From fa2aff717fa2ba3841b2a29d1d516cda1002abb1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 20 Aug 2025 18:39:42 -0400 Subject: [PATCH 001/408] feat(deployment): Migrate package orchestration to Docker Compose (resolves #1177). --- .../clp_package_utils/general.py | 14 +- .../clp_package_utils/scripts/start_clp.py | 1062 +++-------------- tools/deployment/package/docker-compose.yml | 374 ++++++ 3 files changed, 570 insertions(+), 880 deletions(-) create mode 100644 tools/deployment/package/docker-compose.yml diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 1ae8dbb66e..5191caee23 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -150,6 +150,12 @@ def check_dependencies(): ) except subprocess.CalledProcessError: raise EnvironmentError("docker cannot run without superuser privileges (sudo).") + try: + subprocess.run( + ["docker", "compose", "version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True + ) + except subprocess.CalledProcessError: + raise EnvironmentError("docker-compose is not installed") def is_container_running(container_name): @@ -432,10 +438,10 @@ def load_config_file( validate_path_for_container_mount(clp_config.data_directory) validate_path_for_container_mount(clp_config.logs_directory) - # Make data and logs directories node-specific - hostname = socket.gethostname() - clp_config.data_directory /= hostname - clp_config.logs_directory /= hostname + # # Make data and logs directories node-specific + # hostname = socket.gethostname() + # clp_config.data_directory /= hostname + # clp_config.logs_directory /= hostname return clp_config diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 24a45e7dc0..1938211ca0 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -75,6 +75,10 @@ ) logger = logging.getLogger(__file__) +env_dict = {} + +def get_ip_from_hostname(hostname: str) -> str: + return socket.gethostbyname(hostname) def container_exists(container_name): @@ -156,13 +160,9 @@ def wait_for_container_cmd(container_name: str, cmd_to_run: [str], timeout: int) return False -def start_db(instance_id: str, clp_config: CLPConfig, conf_dir: pathlib.Path): +def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = DB_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") db_data_dir = clp_config.data_directory / component_name db_logs_dir = clp_config.logs_directory / component_name @@ -174,321 +174,69 @@ def start_db(instance_id: str, clp_config: CLPConfig, conf_dir: pathlib.Path): db_logs_dir.mkdir(exist_ok=True, parents=True) # Start container - mounts = [ - DockerMount( - DockerMountType.BIND, - conf_dir / "mysql" / "conf.d", - pathlib.Path("/") / "etc" / "mysql" / "conf.d", - True, - ), - DockerMount(DockerMountType.BIND, db_data_dir, pathlib.Path("/") / "var" / "lib" / "mysql"), - DockerMount(DockerMountType.BIND, db_logs_dir, pathlib.Path("/") / "var" / "log" / "mysql"), - ] - env_vars = [ - f"MYSQL_ROOT_PASSWORD={clp_config.database.password}", - f"MYSQL_USER={clp_config.database.username}", - f"MYSQL_PASSWORD={clp_config.database.password}", - f"MYSQL_DATABASE={clp_config.database.name}", - ] - # fmt: off - cmd = [ - "docker", "run", - "-d", - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - append_docker_options(cmd, mounts, env_vars) - append_docker_port_settings_for_host_ips( - clp_config.database.host, clp_config.database.port, 3306, cmd - ) + env_dict["CLP_HOST_DB_CONF_DIR"] = str(conf_dir / "mysql" / "conf.d") + env_dict["CLP_HOST_DB_DATA_DIR"] = str(db_data_dir) + env_dict["CLP_HOST_DB_LOGS_DIR"] = str(db_logs_dir) + + # TODO: Consider a Docker Compose overwrite / extend approach for pick the right image. if "mysql" == clp_config.database.type: - cmd.append("mysql:8.0.23") + env_dict["CLP_DB_IMAGE"] = "mysql:8.0.23" elif "mariadb" == clp_config.database.type: - cmd.append("mariadb:10-jammy") - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) + env_dict["CLP_DB_IMAGE"] = "mariadb:10-jammy" - # fmt: off - mysqladmin_cmd = [ - "mysqladmin", "ping", - "--silent", - "-h", "127.0.0.1", - "-u", str(clp_config.database.username), - f"--password={clp_config.database.password}", - ] - # fmt: on + env_dict["CLP_DB_HOST"] = get_ip_from_hostname(clp_config.database.host) + env_dict["CLP_DB_PORT"] = str(clp_config.database.port) + env_dict["CLP_DB_NAME"] = clp_config.database.name + env_dict["CLP_DB_USER"] = clp_config.database.username + env_dict["CLP_DB_PASS"] = clp_config.database.password - if not wait_for_container_cmd(container_name, mysqladmin_cmd, 180): - raise EnvironmentError(f"{component_name} did not initialize in time") - logger.info(f"Started {component_name}.") - - -def create_db_tables( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, -): - component_name = DB_COMPONENT_NAME - logger.info(f"Creating {component_name} tables...") - - container_name = f"clp-{component_name}-table-creator-{instance_id}" - - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-i", - "--network", "host", - "--rm", - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - env_vars = [ - *get_common_env_vars_list(), - *get_credential_env_vars_list(container_clp_config, include_db_credentials=True), - ] - necessary_mounts = [ - mounts.clp_home, - mounts.data_dir, - mounts.logs_dir, - mounts.generated_config_file, - ] - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - clp_py_utils_dir = clp_site_packages_dir / "clp_py_utils" - # fmt: off - create_tables_cmd = [ - "python3", - str(clp_py_utils_dir / "create-db-tables.py"), - "--config", str(container_clp_config.get_shared_config_file_path()), - "--storage-engine", str(container_clp_config.package.storage_engine), - ] - # fmt: on - - cmd = container_start_cmd + create_tables_cmd - logger.debug(" ".join(cmd)) - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Created {component_name} tables.") - - -def create_results_cache_indices( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, -): - component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Creating {component_name} indices...") - - container_name = f"clp-{component_name}-indices-creator-{instance_id}" - - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-i", - "--network", "host", - "--rm", - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - env_vars = [f"PYTHONPATH={clp_site_packages_dir}"] - necessary_mounts = [mounts.clp_home, mounts.data_dir, mounts.logs_dir] - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - clp_py_utils_dir = clp_site_packages_dir / "clp_py_utils" - # fmt: off - init_cmd = [ - "python3", - str(clp_py_utils_dir / "initialize-results-cache.py"), - "--uri", container_clp_config.results_cache.get_uri(), - "--stream-collection", container_clp_config.results_cache.stream_collection_name, - ] - # fmt: on - - cmd = container_start_cmd + init_cmd - logger.debug(" ".join(cmd)) - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Created {component_name} indices.") - - -def start_queue(instance_id: str, clp_config: CLPConfig): +def start_queue(clp_config: CLPConfig): component_name = QUEUE_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") queue_logs_dir = clp_config.logs_directory / component_name validate_queue_config(clp_config, queue_logs_dir) - log_filename = "rabbitmq.log" - - # Generate config file - config_filename = f"{container_name}.conf" - host_config_file_path = clp_config.logs_directory / config_filename - with open(host_config_file_path, "w") as f: - f.write(f"default_user = {clp_config.queue.username}\n") - f.write(f"default_pass = {clp_config.queue.password}\n") - f.write(f"log.file = {log_filename}\n") + env_dict["CLP_QUEUE_HOST"] = get_ip_from_hostname(clp_config.queue.host) + env_dict["CLP_QUEUE_PORT"] = str(clp_config.queue.port) + env_dict["CLP_QUEUE_USER"] = clp_config.queue.username + env_dict["CLP_QUEUE_PASS"] = clp_config.queue.password # Create directories queue_logs_dir.mkdir(exist_ok=True, parents=True) - # Start container - rabbitmq_logs_dir = pathlib.Path("/") / "var" / "log" / "rabbitmq" - mounts = [ - DockerMount( - DockerMountType.BIND, - host_config_file_path, - pathlib.Path("/") / "etc" / "rabbitmq" / "rabbitmq.conf", - True, - ), - DockerMount(DockerMountType.BIND, queue_logs_dir, rabbitmq_logs_dir), - ] - rabbitmq_pid_file_path = pathlib.Path("/") / "tmp" / "rabbitmq.pid" - - host_user_id = os.getuid() - if 0 != host_user_id: - container_user = f"{host_user_id}:{os.getgid()}" - else: - # The host user is `root` so use the container's default user and make this component's - # directories writable by that user. - # NOTE: This doesn't affect the host user's access to the directories since they're `root`. - container_user = "rabbitmq" - default_container_user_id = 999 - default_container_group_id = 999 - chown_recursively(queue_logs_dir, default_container_user_id, default_container_group_id) - - # fmt: off - cmd = [ - "docker", "run", - "-d", - "--name", container_name, - "--log-driver", "local", - # Override RABBITMQ_LOGS since the image sets it to *only* log to stdout - "-u", container_user - ] - env_vars = [ - f"RABBITMQ_LOGS={rabbitmq_logs_dir / log_filename}", - f"RABBITMQ_PID_FILE={rabbitmq_pid_file_path}", - ] - # fmt: on - append_docker_options(cmd, mounts, env_vars) - append_docker_port_settings_for_host_ips( - clp_config.queue.host, clp_config.queue.port, 5672, cmd - ) - cmd.append("rabbitmq:3.9.8") - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - # Wait for queue to start up - rabbitmq_cmd = ["rabbitmq-diagnostics", "check_running"] - if not wait_for_container_cmd(container_name, rabbitmq_cmd, 60): - raise EnvironmentError(f"{component_name} did not initialize in time") - - logger.info(f"Started {component_name}.") + env_dict["CLP_HOST_QUEUE_LOGS_DIR"] = str(queue_logs_dir) -def start_redis(instance_id: str, clp_config: CLPConfig, conf_dir: pathlib.Path): +def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = REDIS_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") redis_logs_dir = clp_config.logs_directory / component_name redis_data_dir = clp_config.data_directory / component_name - base_config_file_path = conf_dir / "redis" / "redis.conf" - validate_redis_config(clp_config, redis_data_dir, redis_logs_dir, base_config_file_path) + config_file_path = conf_dir / "redis" / "redis.conf" + validate_redis_config(clp_config, redis_data_dir, redis_logs_dir, config_file_path) - config_filename = f"{container_name}.conf" - host_config_file_path = clp_config.logs_directory / config_filename - with open(base_config_file_path, "r") as base, open(host_config_file_path, "w") as full: - for line in base.readlines(): - full.write(line) - full.write(f"requirepass {clp_config.redis.password}\n") + env_dict["CLP_HOST_REDIS_CONF_PATH"] = str(config_file_path) + env_dict["CLP_HOST_REDIS_DATA_DIR"] = str(redis_data_dir) + env_dict["CLP_HOST_REDIS_LOGS_DIR"] = str(redis_logs_dir) redis_data_dir.mkdir(exist_ok=True, parents=True) redis_logs_dir.mkdir(exist_ok=True, parents=True) - # Start container - config_file_path = pathlib.Path("/") / "usr" / "local" / "etc" / "redis" / "redis.conf" - mounts = [ - DockerMount(DockerMountType.BIND, host_config_file_path, config_file_path, True), - DockerMount( - DockerMountType.BIND, redis_logs_dir, pathlib.Path("/") / "var" / "log" / "redis" - ), - DockerMount(DockerMountType.BIND, redis_data_dir, pathlib.Path("/") / "data"), - ] - - host_user_id = os.getuid() - if 0 != host_user_id: - container_user = f"{host_user_id}:{os.getgid()}" - else: - # The host user is `root` so use the container's default user and make this component's - # directories writable by that user. - # NOTE: This doesn't affect the host user's access to the directories since they're `root`. - container_user = "redis" - default_container_user_id = 999 - default_container_group_id = 999 - chown_recursively(redis_data_dir, default_container_user_id, default_container_group_id) - chown_recursively(redis_logs_dir, default_container_user_id, default_container_group_id) - - # fmt: off - cmd = [ - "docker", "run", - "-d", - "--name", container_name, - "--log-driver", "local", - "-u", container_user, - ] - # fmt: on - append_docker_options(cmd, mounts) - append_docker_port_settings_for_host_ips( - clp_config.redis.host, clp_config.redis.port, 6379, cmd - ) - cmd.append("redis:7.2.4") - cmd.append("redis-server") - cmd.append(str(config_file_path)) - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - # fmt: off - redis_ping_cmd = [ - "redis-cli", - "-h", "127.0.0.1", - "-p", "6379", - "-a", clp_config.redis.password, - "PING" - ] - # fmt: on - - if not wait_for_container_cmd(container_name, redis_ping_cmd, 30): - raise EnvironmentError(f"{component_name} did not initialize in time") - - logger.info(f"Started {component_name}.") + env_dict["CLP_REDIS_HOST"] = get_ip_from_hostname(clp_config.redis.host) + env_dict["CLP_REDIS_PORT"] = str(clp_config.redis.port) + env_dict["CLP_REDIS_PASS"] = clp_config.redis.password + env_dict["CLP_REDIS_QUERY_BACKEND_DB"] = str(clp_config.redis.query_backend_database) + env_dict["CLP_REDIS_COMPRESSION_BACKEND_DB"] = str( + clp_config.redis.compression_backend_database) -def start_results_cache(instance_id: str, clp_config: CLPConfig, conf_dir: pathlib.Path): +def start_results_cache(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") data_dir = clp_config.data_directory / component_name logs_dir = clp_config.logs_directory / component_name @@ -498,298 +246,110 @@ def start_results_cache(instance_id: str, clp_config: CLPConfig, conf_dir: pathl data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) - mounts = [ - DockerMount( - DockerMountType.BIND, conf_dir / "mongo", pathlib.Path("/") / "etc" / "mongo", True - ), - DockerMount(DockerMountType.BIND, data_dir, pathlib.Path("/") / "data" / "db"), - DockerMount(DockerMountType.BIND, logs_dir, pathlib.Path("/") / "var" / "log" / "mongodb"), - ] - - host_user_id = os.getuid() - if 0 != host_user_id: - container_user = f"{host_user_id}:{os.getgid()}" - else: - # The host user is `root` so use the container's default user and make this component's - # directories writable by that user. - # NOTE: This doesn't affect the host user's access to the directories since they're `root`. - container_user = "mongodb" - default_container_user_id = 999 - default_container_group_id = 999 - chown_recursively(data_dir, default_container_user_id, default_container_group_id) - chown_recursively(logs_dir, default_container_user_id, default_container_group_id) - - # fmt: off - cmd = [ - "docker", "run", - "-d", - "--network", "host", - "--name", container_name, - "--log-driver", "local", - "-u", container_user, - ] - # fmt: on - append_docker_options(cmd, mounts) - cmd.append("mongo:7.0.1") - cmd.append("--config") - cmd.append(str(pathlib.Path("/") / "etc" / "mongo" / "mongod.conf")) - cmd.append("--bind_ip") - cmd.append(clp_config.results_cache.host) - cmd.append("--port") - cmd.append(str(clp_config.results_cache.port)) - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + env_dict["CLP_HOST_RESULTS_CACHE_CONF_DIR"] = str(conf_dir / "mongo") + env_dict["CLP_HOST_RESULTS_CACHE_DATA_DIR"] = str(data_dir) + env_dict["CLP_HOST_RESULTS_CACHE_LOGS_DIR"] = str(logs_dir) + + env_dict["CLP_RESULTS_CACHE_HOST"] = get_ip_from_hostname(clp_config.results_cache.host) + env_dict["CLP_RESULTS_CACHE_PORT"] = str(clp_config.results_cache.port) + env_dict["CLP_RESULTS_CACHE_DB_NAME"] = clp_config.results_cache.db_name + env_dict[ + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME"] = clp_config.results_cache.stream_collection_name def start_compression_scheduler( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, + clp_config: CLPConfig, ): - module_name = "job_orchestration.scheduler.compress.compression_scheduler" - generic_start_scheduler( - COMPRESSION_SCHEDULER_COMPONENT_NAME, - module_name, - instance_id, - clp_config, - container_clp_config, - mounts, - ) + component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + logs_dir = clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) -def start_query_scheduler( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, -): - module_name = "job_orchestration.scheduler.query.query_scheduler" - generic_start_scheduler( - QUERY_SCHEDULER_COMPONENT_NAME, - module_name, - instance_id, - clp_config, - container_clp_config, - mounts, - ) + env_dict["CLP_HOST_COMPRESSION_SCHEDULER_LOGS_DIR"] = str(logs_dir) + env_dict[ + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL"] = clp_config.compression_scheduler.logging_level + # FIXME: handle S3 -def generic_start_scheduler( - component_name: str, - module_name: str, - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, -): - logger.info(f"Starting {component_name}...") - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return +def start_query_scheduler( + clp_config: CLPConfig, +): + component_name = QUERY_SCHEDULER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - container_logs_dir = container_clp_config.logs_directory / component_name - - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-di", - "--network", "host", - "-w", str(CONTAINER_CLP_HOME), - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - - env_vars = [ - *get_common_env_vars_list(), - *get_credential_env_vars_list(container_clp_config, include_db_credentials=True), - *get_celery_connection_env_vars_list(container_clp_config), - f"CLP_LOGS_DIR={container_logs_dir}", - f"CLP_LOGGING_LEVEL={clp_config.query_scheduler.logging_level}", - ] - necessary_mounts = [mounts.clp_home, mounts.logs_dir, mounts.generated_config_file] - aws_mount, aws_env_vars = generate_container_auth_options(clp_config, component_name) - if aws_mount: - necessary_mounts.append(mounts.aws_config_dir) - if aws_env_vars: - env_vars.extend(aws_env_vars) - if ( - COMPRESSION_SCHEDULER_COMPONENT_NAME == component_name - and StorageType.FS == clp_config.logs_input.type - ): - necessary_mounts.append(mounts.input_logs_dir) - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - # fmt: off - scheduler_cmd = [ - "python3", "-u", - "-m", module_name, - "--config", str(container_clp_config.get_shared_config_file_path()), - ] - # fmt: on - cmd = container_start_cmd + scheduler_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + + env_dict["CLP_HOST_QUERY_SCHEDULER_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_QUERY_SCHEDULER_LOGGING_LEVEL"] = clp_config.query_scheduler.logging_level + + # FIXME: handle S3 def start_compression_worker( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - num_cpus: int, - mounts: CLPDockerMounts, + clp_config: CLPConfig, + num_cpus: int, ): - celery_method = "job_orchestration.executor.compress" - celery_route = f"{QueueName.COMPRESSION}" - compression_worker_mounts = [mounts.archives_output_dir] - generic_start_worker( - COMPRESSION_WORKER_COMPONENT_NAME, - instance_id, - clp_config, - clp_config.compression_worker, - container_clp_config, - celery_method, - celery_route, - clp_config.redis.compression_backend_database, - num_cpus, - mounts, - compression_worker_mounts, - ) + component_name = COMPRESSION_WORKER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + logs_dir = clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) -def start_query_worker( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - num_cpus: int, - mounts: CLPDockerMounts, -): - celery_method = "job_orchestration.executor.query" - celery_route = f"{QueueName.QUERY}" - - query_worker_mounts = [mounts.stream_output_dir] - if StorageType.FS == clp_config.archive_output.storage.type: - query_worker_mounts.append(mounts.archives_output_dir) - - generic_start_worker( - QUERY_WORKER_COMPONENT_NAME, - instance_id, - clp_config, - clp_config.query_worker, - container_clp_config, - celery_method, - celery_route, - clp_config.redis.query_backend_database, - num_cpus, - mounts, - query_worker_mounts, - ) + env_dict["CLP_COMPRESSION_WORKER_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_COMPRESSION_WORKER_LOGGING_LEVEL"] = clp_config.compression_worker.logging_level + env_dict["CLP_COMPRESSION_WORKER_CONCURRENCY"] = str(num_cpus) + # Create necessary directories + clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) + clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + + # FIXME: handle S3 -def generic_start_worker( - component_name: str, - instance_id: str, - clp_config: CLPConfig, - worker_config: BaseModel, - container_clp_config: CLPConfig, - celery_method: str, - celery_route: str, - redis_database: int, - num_cpus: int, - mounts: CLPDockerMounts, - worker_specific_mounts: Optional[List[Optional[DockerMount]]], -): - logger.info(f"Starting {component_name}...") - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return +def start_query_worker( + clp_config: CLPConfig, + num_cpus: int, +): + component_name = QUERY_WORKER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - container_logs_dir = container_clp_config.logs_directory / component_name + + env_dict["CLP_QUERY_WORKER_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_QUERY_WORKER_LOGGING_LEVEL"] = clp_config.query_worker.logging_level + env_dict["CLP_QUERY_WORKER_CONCURRENCY"] = str(num_cpus) # Create necessary directories clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - container_worker_log_path = container_logs_dir / "worker.log" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-di", - "--network", "host", - "-w", str(CONTAINER_CLP_HOME), - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - - env_vars = [ - *get_common_env_vars_list(include_clp_home_env_var=True), - *get_celery_connection_env_vars_list(container_clp_config), - f"CLP_CONFIG_PATH={container_clp_config.get_shared_config_file_path()}", - f"CLP_LOGS_DIR={container_logs_dir}", - f"CLP_LOGGING_LEVEL={worker_config.logging_level}", - f"CLP_WORKER_LOG_PATH={container_worker_log_path}", - ] - necessary_mounts = [ - mounts.clp_home, - mounts.data_dir, - mounts.logs_dir, - ] - if StorageType.FS == clp_config.logs_input.type: - necessary_mounts.append(mounts.input_logs_dir) - if worker_specific_mounts: - necessary_mounts.extend(worker_specific_mounts) - - aws_mount, aws_env_vars = generate_container_auth_options(clp_config, component_name) - if aws_mount: - necessary_mounts.append(mounts.aws_config_dir) - if aws_env_vars: - env_vars.extend(aws_env_vars) - - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - worker_cmd = [ - "python3", - str(clp_site_packages_dir / "bin" / "celery"), - "-A", - celery_method, - "worker", - "--concurrency", - str(num_cpus), - "--loglevel", - "WARNING", - "-f", - str(container_worker_log_path), - "-Q", - celery_route, - "-n", - component_name, - ] - cmd = container_start_cmd + worker_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + # FIXME: handle S3 + + +def start_reducer( + clp_config: CLPConfig, + num_workers: int, +): + component_name = REDUCER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + env_dict["CLP_REDUCER_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_REDUCER_LOGGING_LEVEL"] = clp_config.reducer.logging_level + env_dict["CLP_REDUCER_CONCURRENCY"] = str(num_workers) + env_dict["CLP_REDUCER_UPSERT_INTERVAL"] = str(clp_config.reducer.upsert_interval) def update_settings_object( - parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], + parent_key_prefix: str, + settings: Dict[str, Any], + updates: Dict[str, Any], ): """ Recursively updates the given settings object with the values from `updates`. @@ -824,25 +384,18 @@ def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dic def start_webui( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, + clp_config: CLPConfig, + container_clp_config: CLPConfig, ): component_name = WEBUI_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" - node_path = str(container_webui_dir / "server" / "node_modules") client_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" ) validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path) @@ -874,22 +427,20 @@ def start_webui( client_settings_json_file.write(json.dumps(client_settings_json)) server_settings_json_updates = { - "SqlDbHost": clp_config.database.host, + "SqlDbHost": container_clp_config.database.host, "SqlDbPort": clp_config.database.port, "SqlDbName": clp_config.database.name, "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME, - "MongoDbHost": clp_config.results_cache.host, + "MongoDbHost": container_clp_config.results_cache.host, "MongoDbPort": clp_config.results_cache.port, "MongoDbName": clp_config.results_cache.db_name, "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name, "MongoDbStreamFilesCollectionName": clp_config.results_cache.stream_collection_name, "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), - "StreamTargetUncompressedSize": container_clp_config.stream_output.target_uncompressed_size, + "StreamTargetUncompressedSize": clp_config.stream_output.target_uncompressed_size, } - container_cmd_extra_opts = [] - stream_storage = clp_config.stream_output.storage if StorageType.S3 == stream_storage.type: s3_config = stream_storage.s3_config @@ -917,193 +468,22 @@ def start_webui( with open(server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) - # fmt: off - container_cmd = [ - "docker", "run", - "-d", - "--network", "host", - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - container_cmd.extend(container_cmd_extra_opts) - - env_vars = [ - *get_common_env_vars_list(), - *get_credential_env_vars_list(container_clp_config, include_db_credentials=True), - f"NODE_PATH={node_path}", - f"HOST={clp_config.webui.host}", - f"PORT={clp_config.webui.port}", - f"NODE_ENV=production", - ] - necessary_mounts = [ - mounts.clp_home, - ] - if StorageType.S3 == stream_storage.type: - auth = stream_storage.s3_config.aws_authentication - if AwsAuthType.credentials == auth.type: - credentials = auth.credentials - env_vars.append(f"AWS_ACCESS_KEY_ID={credentials.access_key_id}") - env_vars.append(f"AWS_SECRET_ACCESS_KEY={credentials.secret_access_key}") - else: - aws_mount, aws_env_vars = generate_container_auth_options( - clp_config, WEBUI_COMPONENT_NAME - ) - if aws_mount: - necessary_mounts.append(mounts.aws_config_dir) - if aws_env_vars: - env_vars.extend(aws_env_vars) - elif StorageType.FS == stream_storage.type: - necessary_mounts.append(mounts.stream_output_dir) - - append_docker_options(container_cmd, necessary_mounts, env_vars) - container_cmd.append(clp_config.execution_container) - - node_cmd = [ - str(CONTAINER_CLP_HOME / "bin" / "node-22"), - str(container_webui_dir / "server" / "dist" / "server" / "src" / "main.js"), - ] - cmd = container_cmd + node_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") - - -def start_reducer( - instance_id: str, - clp_config: CLPConfig, - container_clp_config: CLPConfig, - num_workers: int, - mounts: CLPDockerMounts, -): - component_name = REDUCER_COMPONENT_NAME - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return - - logs_dir = clp_config.logs_directory / component_name - validate_reducer_config(clp_config, logs_dir, num_workers) - - logs_dir.mkdir(parents=True, exist_ok=True) - container_logs_dir = container_clp_config.logs_directory / component_name - - clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages" - # fmt: off - container_start_cmd = [ - "docker", "run", - "-di", - "--network", "host", - "-w", str(CONTAINER_CLP_HOME), - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - env_vars = [ - *get_common_env_vars_list(include_clp_home_env_var=True), - f"CLP_LOGS_DIR={container_logs_dir}", - f"CLP_LOGGING_LEVEL={clp_config.reducer.logging_level}", - ] - necessary_mounts = [ - mounts.clp_home, - mounts.logs_dir, - mounts.generated_config_file, - ] - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - # fmt: off - reducer_cmd = [ - "python3", "-u", - "-m", "job_orchestration.reducer.reducer", - "--config", str(container_clp_config.get_shared_config_file_path()), - "--concurrency", str(num_workers), - "--upsert-interval", str(clp_config.reducer.upsert_interval), - ] - # fmt: on - cmd = container_start_cmd + reducer_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + env_dict["CLP_WEBUI_HOST"] = get_ip_from_hostname(clp_config.webui.host) + env_dict["CLP_WEBUI_PORT"] = clp_config.webui.port + # FIXME: Handle S3 def start_garbage_collector( - instance_id: str, clp_config: CLPConfig, - container_clp_config: CLPConfig, - mounts: CLPDockerMounts, ): component_name = GARBAGE_COLLECTOR_COMPONENT_NAME - - if not is_retention_period_configured(clp_config): - logger.info(f"Retention period is not configured, skipping {component_name} creation...") - return - - logger.info(f"Starting {component_name}...") - - container_name = f"clp-{component_name}-{instance_id}" - if container_exists(container_name): - return + logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name - validate_log_directory(logs_dir, component_name) - # Create logs directory if necessary logs_dir.mkdir(parents=True, exist_ok=True) - container_logs_dir = container_clp_config.logs_directory / component_name - - # fmt: off - container_start_cmd = [ - "docker", "run", - "-di", - "--network", "host", - "-w", str(CONTAINER_CLP_HOME), - "--name", container_name, - "--log-driver", "local", - "-u", f"{os.getuid()}:{os.getgid()}", - ] - # fmt: on - - necessary_mounts = [ - mounts.clp_home, - mounts.logs_dir, - mounts.generated_config_file, - ] - env_vars = [ - *get_common_env_vars_list(include_clp_home_env_var=True), - *get_credential_env_vars_list(container_clp_config, include_db_credentials=True), - f"CLP_LOGS_DIR={container_logs_dir}", - f"CLP_LOGGING_LEVEL={clp_config.garbage_collector.logging_level}", - ] - - # Add necessary mounts for archives and streams. - if StorageType.FS == clp_config.archive_output.storage.type: - necessary_mounts.append(mounts.archives_output_dir) - if StorageType.FS == clp_config.stream_output.storage.type: - necessary_mounts.append(mounts.stream_output_dir) - - aws_mount, aws_env_vars = generate_container_auth_options(clp_config, component_name) - if aws_mount: - necessary_mounts.append(mounts.aws_config_dir) - if aws_env_vars: - env_vars.extend(aws_env_vars) - - append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) - - # fmt: off - garbage_collector_cmd = [ - "python3", "-u", - "-m", "job_orchestration.garbage_collector.garbage_collector", - "--config", str(container_clp_config.get_shared_config_file_path()), - ] - # fmt: on - cmd = container_start_cmd + garbage_collector_cmd - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Started {component_name}.") + + env_dict["CLP_GC_LOGS_DIR"] = str(logs_dir) + env_dict["CLP_GC_LOGGING_LEVEL"] = clp_config.garbage_collector.logging_level def add_num_workers_argument(parser): @@ -1113,6 +493,7 @@ def add_num_workers_argument(parser): default=multiprocessing.cpu_count(), help="Number of workers to start", ) + # FIXME: handle S3 def main(argv): @@ -1146,11 +527,6 @@ def main(argv): parsed_args = args_parser.parse_args(argv[1:]) - if parsed_args.target: - target = parsed_args.target - else: - target = ALL_TARGET_NAME - try: check_dependencies() except: @@ -1162,64 +538,17 @@ def main(argv): config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) - runnable_components = clp_config.get_runnable_components() - components_to_start = get_components_for_target(target) - components_to_start = components_to_start.intersection(runnable_components) - - # Exit early if no components to start - if len(components_to_start) == 0: - logger.error(f"{target} not available with current configuration") - return -1 - - # Validate and load necessary credentials - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - WEBUI_COMPONENT_NAME, - ): - validate_and_load_db_credentials_file(clp_config, clp_home, True) - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - QUEUE_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - ): - validate_and_load_queue_credentials_file(clp_config, clp_home, True) - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - REDIS_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - ): - validate_and_load_redis_credentials_file(clp_config, clp_home, True) - if target in ( - ALL_TARGET_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - ): - validate_logs_input_config(clp_config) - if target in ( - ALL_TARGET_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - ): - validate_output_storage_config(clp_config) - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - ): - validate_retention_config(clp_config) + validate_and_load_db_credentials_file(clp_config, clp_home, True) + validate_and_load_queue_credentials_file(clp_config, clp_home, True) + validate_and_load_redis_credentials_file(clp_config, clp_home, True) + validate_logs_input_config(clp_config) + validate_output_storage_config(clp_config) + validate_retention_config(clp_config) + + # validate_and_load_db_credentials_file(clp_config, clp_home, True) + # validate_and_load_queue_credentials_file(clp_config, clp_home, True) + # validate_and_load_redis_credentials_file(clp_config, clp_home, True) + # validate_worker_config(clp_config) clp_config.validate_data_dir() clp_config.validate_logs_dir() @@ -1228,14 +557,8 @@ def main(argv): logger.exception("Failed to load config.") return -1 - if target in ( - COMPRESSION_WORKER_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - ): - num_workers = parsed_args.num_workers - else: - num_workers = multiprocessing.cpu_count() // 2 + # FIXME: handle this + num_workers = multiprocessing.cpu_count() // 2 container_clp_config, mounts = generate_container_config(clp_config, clp_home) @@ -1243,75 +566,62 @@ def main(argv): clp_config.data_directory.mkdir(parents=True, exist_ok=True) clp_config.logs_directory.mkdir(parents=True, exist_ok=True) + # Set container services' hosts in container config + container_clp_config.database.host = DB_COMPONENT_NAME + container_clp_config.queue.host = QUEUE_COMPONENT_NAME + container_clp_config.redis.host = REDIS_COMPONENT_NAME + container_clp_config.results_cache.host = RESULTS_CACHE_COMPONENT_NAME + container_clp_config.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME + container_clp_config.reducer.host = REDUCER_COMPONENT_NAME + dump_shared_container_config(container_clp_config, clp_config) - try: - # Create instance-id file - instance_id_file_path = clp_config.logs_directory / "instance-id" - if instance_id_file_path.exists(): - with open(instance_id_file_path, "r") as f: - instance_id = f.readline() - else: - instance_id = str(uuid.uuid4())[-4:] - with open(instance_id_file_path, "w") as f: - f.write(instance_id) - f.flush() + env_dict["CLP_USER_ID"] = os.getuid() + env_dict["CLP_GROUP_ID"] = os.getgid() + env_dict["CLP_STORAGE_ENGINE"] = clp_config.package.storage_engine + + env_dict["CLP_HOST_DATA_DIR"] = str(clp_config.data_directory) + env_dict["CLP_HOST_LOGS_DIR"] = str(clp_config.logs_directory) + env_dict["CLP_HOST_ARCHIVE_OUTPUT_DIR"] = str(clp_config.archive_output.get_directory()) + env_dict["CLP_HOST_STREAM_OUTPUT_DIR"] = str(clp_config.stream_output.get_directory()) + try: conf_dir = clp_home / "etc" # Start components - if DB_COMPONENT_NAME in components_to_start: - start_db(instance_id, clp_config, conf_dir) - - if ( - target == CONTROLLER_TARGET_NAME and DB_COMPONENT_NAME in runnable_components - ) or DB_COMPONENT_NAME in components_to_start: - create_db_tables(instance_id, clp_config, container_clp_config, mounts) - - if QUEUE_COMPONENT_NAME in components_to_start: - start_queue(instance_id, clp_config) - - if REDIS_COMPONENT_NAME in components_to_start: - start_redis(instance_id, clp_config, conf_dir) - - if RESULTS_CACHE_COMPONENT_NAME in components_to_start: - start_results_cache(instance_id, clp_config, conf_dir) - - if ( - target == CONTROLLER_TARGET_NAME and RESULTS_CACHE_COMPONENT_NAME in runnable_components - ) or RESULTS_CACHE_COMPONENT_NAME in components_to_start: - create_results_cache_indices(instance_id, clp_config, container_clp_config, mounts) - - if COMPRESSION_SCHEDULER_COMPONENT_NAME in components_to_start: - start_compression_scheduler(instance_id, clp_config, container_clp_config, mounts) - - if QUERY_SCHEDULER_COMPONENT_NAME in components_to_start: - start_query_scheduler(instance_id, clp_config, container_clp_config, mounts) - - if COMPRESSION_WORKER_COMPONENT_NAME in components_to_start: - start_compression_worker( - instance_id, clp_config, container_clp_config, num_workers, mounts - ) - - if QUERY_WORKER_COMPONENT_NAME in components_to_start: - start_query_worker(instance_id, clp_config, container_clp_config, num_workers, mounts) - - if REDUCER_COMPONENT_NAME in components_to_start: - start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts) - - if WEBUI_COMPONENT_NAME in components_to_start: - start_webui(instance_id, clp_config, container_clp_config, mounts) - - if GARBAGE_COLLECTOR_COMPONENT_NAME in components_to_start: - start_garbage_collector(instance_id, clp_config, container_clp_config, mounts) - + start_db( clp_config, conf_dir) + start_queue( clp_config) + start_redis( clp_config, conf_dir) + start_results_cache( clp_config, conf_dir) + start_compression_scheduler(clp_config) + start_query_scheduler(clp_config) + start_compression_worker(clp_config, num_workers) + start_query_worker( clp_config, num_workers) + start_reducer( clp_config, num_workers) + start_webui( clp_config, container_clp_config) + start_garbage_collector(clp_config) + + with open(f"{clp_home}/.env", "w") as env_file: + for key, value in env_dict.items(): + env_file.write(f"{key}={value}\n") except Exception as ex: if type(ex) == ValueError: - logger.error(f"Failed to start CLP: {ex}") + logger.error(f"Failed to initialize CLP: {ex}") else: - logger.exception("Failed to start CLP.") + logger.exception("Failed to initialize CLP.") return -1 + # try: + # subprocess.run( + # "docker compose up", + # shell=True, + # stderr=subprocess.STDOUT, + # check=True, + # ) + # except subprocess.CalledProcessError: + # logger.exception("Failed to start CLP.") + # return -1 + return 0 diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml new file mode 100644 index 0000000000..1bfd80d95e --- /dev/null +++ b/tools/deployment/package/docker-compose.yml @@ -0,0 +1,374 @@ +secrets: + CLP_DB_PASS_FILE: + environment: "CLP_DB_PASS" +services: + db: + image: ${CLP_DB_IMAGE:-mysql:8.0.23} + container_name: "database" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + secrets: + - CLP_DB_PASS_FILE + environment: + MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_USER: "${CLP_DB_USER}" + MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_DATABASE: "${CLP_DB_NAME}" + volumes: + - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" + - "./var/data/database:/var/lib/mysql" + - "./var/log/database:/var/log/mysql" + networks: + - "clp-network" + ports: + - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + healthcheck: + test: [ + "CMD", + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1", + "-u", "${CLP_DB_USER}", + "--password=${CLP_DB_PASS}" + ] + start_period: "10s" + start_interval: "2s" + interval: "30s" + timeout: "10s" + retries: 3 + + db-table-creator: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "db_table_creator" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", "${CLP_STORAGE_ENGINE}" + ] + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml" + networks: + - "clp-network" + depends_on: + db: + condition: "service_healthy" + + queue: + image: "rabbitmq:3.9.8" + container_name: "queue" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" + volumes: + - "./var/log/queue:/var/log/rabbitmq" + networks: + - "clp-network" + ports: + - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" + healthcheck: + test: [ + "CMD", + "rabbitmq-diagnostics", "check_running" + ] + start_period: "10s" + start_interval: "2s" + interval: "30s" + timeout: "10s" + retries: 3 + + redis: + image: "redis:7.2.4" + container_name: "redis" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS}" + ] + volumes: + - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" + - "./var/log/redis:/var/log/redis" + - "./var/data/redis:/data" + networks: + - "clp-network" + ports: + - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" + healthcheck: + test: [ + "CMD", + "redis-cli", + "-h", "127.0.0.1", + "-p", "6379", + "-a", "${CLP_REDIS_PASS}", + "PING" + ] + start_period: "10s" + start_interval: "2s" + interval: 30s + timeout: 10s + retries: 3 + + results-cache: + image: "mongo:7.0.1" + container_name: "results_cache" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] + volumes: + - "./etc/mongo:/etc/mongo:ro" + - "./var/data/results_cache:/data/db" + - "./var/log/results_cache:/var/log/mongodb" + networks: + - "clp-network" + ports: + - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet + start_period: "10s" + start_interval: "2s" + interval: 30s + timeout: 10s + retries: 3 + + results-cache-indices-creator: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "results_cache_indices_creator" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", + "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", + ] + networks: + - "clp-network" + depends_on: + results-cache: + condition: "service_healthy" + + compression-scheduler: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "compression_scheduler" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1}" + CLP_LOGS_DIR: "/var/log" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log:/var/log" + # FIXME: why not call this /mnt/input? + - "/:/mnt/logs:ro" + networks: + - "clp-network" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "--config", "/etc/clp-config.yml" + ] + healthcheck: + # FIXME + test: [ + "CMD", + "true" + ] + + query-scheduler: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "query_scheduler" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0}" + CLP_LOGS_DIR: "/var/log" + CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log:/var/log" + networks: + - "clp-network" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "job_orchestration.scheduler.query.query_scheduler", + "--config", "/etc/clp-config.yml" + ] + healthcheck: + # FIXME + test: [ + "CMD", + "true" + ] + + compression-worker: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "compression_worker" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1}" + CLP_HOME: "/opt/clp" + CLP_LOGS_DIR: "/var/log/compression_worker" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" + CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" + CLP_CONFIG_PATH: "/etc/clp-config.yml" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log/compression_worker:/var/log/compression_worker" + # FIXME: why dump "var/data/compression-job-4-task-4-db-config.yml"? + - "./var/data:/var/data" + - "./var/data/archives:/var/data/archives" + # FIXME: why not call this /mnt/input? + - "/:/mnt/logs:ro" + networks: + - "clp-network" + depends_on: + compression-scheduler: + condition: "service_healthy" + # FIXME: add healthcheck + command: [ + "python3", + "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.compress", + "worker", + "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", + "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", + "-f", "/var/log/compression_worker/worker.log", + "-Q", "compression", + "-n", "compression-worker" + ] + + query-worker: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "query_worker" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0}" + CLP_HOME: "/opt/clp" + CLP_LOGS_DIR: "/var/log/query_worker" + CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" + CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" + CLP_CONFIG_PATH: "/etc/clp-config.yml" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log/query_worker:/var/log/query_worker" + - "./var/data/archives:/var/data/archives" + - "./var/data/streams:/var/data/streams" + networks: + - "clp-network" + depends_on: + query-scheduler: + condition: "service_healthy" + command: [ + "python3", + "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.query", + "worker", + "--concurrency", "${CLP_QUERY_WORKER_CONCURRENCY:-1}", + "--loglevel", "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}", + "-f", "/var/log/query_worker/worker.log", + "-Q", "query", + "-n", "query-worker" + ] + + reducer: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "reducer" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + CLP_HOME: "/opt/clp" + CLP_LOGS_DIR: "/var/log/reducer" + CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "./var/log/reducer:/var/log/reducer" + networks: + - "clp-network" + depends_on: + query-scheduler: + condition: "service_healthy" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "python3", "-u", + "-m", "job_orchestration.reducer.reducer", + "--config", "/etc/clp-config.yml", + "--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}", + "--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}" + ] + + webui: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "webui" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" + HOST: "0.0.0.0" + PORT: "4000" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + NODE_ENV: "production" + volumes: + - "./var/data/streams:/var/data/streams" + - type: "bind" + source: "./var/www/webui/server/dist/server/settings.json" + target: "/opt/clp/var/www/webui/server/dist/server/settings.json" + read_only: true + networks: + - "clp-network" + ports: + - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/server/src/main.js" + ] + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + +networks: + clp-network: + driver: bridge \ No newline at end of file From 32ea452622a05e6e4f4c90c3436cf68c0bfc7c43 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 22 Aug 2025 04:27:49 -0400 Subject: [PATCH 002/408] Add garbage collector service to Docker Compose configuration --- tools/deployment/package/docker-compose.yml | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 1bfd80d95e..03410db3f2 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -369,6 +369,35 @@ services: results-cache-indices-creator: condition: "service_completed_successfully" + garbage-collector: + image: "clp-package-x86-ubuntu-jammy:dev" + container_name: "garbage_collector" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + CLP_HOME: "/opt/clp" + CLP_LOGS_DIR: "/var/log/garbage_collector" + CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS}" + volumes: + - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" + networks: + - "clp-network" + depends_on: + query-scheduler: + condition: "service_healthy" + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yml", + ] + networks: clp-network: driver: bridge \ No newline at end of file From 217c63f4756f0be44cb513d8691f1dc680569294 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 08:36:52 -0400 Subject: [PATCH 003/408] fix s3 support and various issues --- .../clp_package_utils/general.py | 56 ++++++++- .../clp_package_utils/scripts/start_clp.py | 83 +++++++------ .../clp-py-utils/clp_py_utils/clp_config.py | 8 +- tools/deployment/package/docker-compose.yml | 116 +++++++++--------- 4 files changed, 167 insertions(+), 96 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 5191caee23..b27c62606c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -14,9 +14,11 @@ import yaml from clp_py_utils.clp_config import ( CLP_DEFAULT_CREDENTIALS_FILE_PATH, + CLP_DEFAULT_DATA_DIRECTORY_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, DB_COMPONENT_NAME, + QUERY_SCHEDULER_COMPONENT_NAME, QueryEngine, QUEUE_COMPONENT_NAME, REDIS_COMPONENT_NAME, @@ -152,7 +154,10 @@ def check_dependencies(): raise EnvironmentError("docker cannot run without superuser privileges (sudo).") try: subprocess.run( - ["docker", "compose", "version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True + ["docker", "compose", "version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, ) except subprocess.CalledProcessError: raise EnvironmentError("docker-compose is not installed") @@ -316,6 +321,55 @@ def generate_container_config( return container_clp_config, docker_mounts +def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig: + """ + Copies the given config and corrects mount paths and hosts for Docker Compose. + + :param clp_config: + :return: The container config and the mounts. + """ + container_clp_config = clp_config.copy(deep=True) + + # FIXME: consider removing credentials_file_path + + # Set container paths + container_clp_config.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH + container_clp_config.logs_directory = pathlib.Path("/") / "var" / "log" + if StorageType.FS == clp_config.logs_input.type: + container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR + + if StorageType.FS == clp_config.archive_output.storage.type: + container_clp_config.archive_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" + ) + elif StorageType.S3 == clp_config.archive_output.storage.type: + container_clp_config.archive_output.storage.staging_directory = ( + CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" + ) + + if StorageType.FS == clp_config.stream_output.storage.type: + container_clp_config.stream_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" + ) + elif StorageType.S3 == clp_config.stream_output.storage.type: + container_clp_config.stream_output.storage.staging_directory = ( + CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" + ) + + if clp_config.aws_config_directory is not None: + container_clp_config.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY + + # Set container services' hosts + container_clp_config.database.host = DB_COMPONENT_NAME + container_clp_config.queue.host = QUEUE_COMPONENT_NAME + container_clp_config.redis.host = REDIS_COMPONENT_NAME + container_clp_config.results_cache.host = RESULTS_CACHE_COMPONENT_NAME + container_clp_config.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME + container_clp_config.reducer.host = REDUCER_COMPONENT_NAME + + return container_clp_config + + def generate_worker_config(clp_config: CLPConfig) -> WorkerConfig: worker_config = WorkerConfig() worker_config.package = clp_config.package.copy(deep=True) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 1938211ca0..c163df9e0a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -51,6 +51,7 @@ DockerMountType, dump_shared_container_config, generate_container_config, + generate_docker_compose_container_config, get_celery_connection_env_vars_list, get_clp_home, get_common_env_vars_list, @@ -77,6 +78,7 @@ logger = logging.getLogger(__file__) env_dict = {} + def get_ip_from_hostname(hostname: str) -> str: return socket.gethostbyname(hostname) @@ -231,7 +233,8 @@ def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): env_dict["CLP_REDIS_PASS"] = clp_config.redis.password env_dict["CLP_REDIS_QUERY_BACKEND_DB"] = str(clp_config.redis.query_backend_database) env_dict["CLP_REDIS_COMPRESSION_BACKEND_DB"] = str( - clp_config.redis.compression_backend_database) + clp_config.redis.compression_backend_database + ) def start_results_cache(clp_config: CLPConfig, conf_dir: pathlib.Path): @@ -253,12 +256,13 @@ def start_results_cache(clp_config: CLPConfig, conf_dir: pathlib.Path): env_dict["CLP_RESULTS_CACHE_HOST"] = get_ip_from_hostname(clp_config.results_cache.host) env_dict["CLP_RESULTS_CACHE_PORT"] = str(clp_config.results_cache.port) env_dict["CLP_RESULTS_CACHE_DB_NAME"] = clp_config.results_cache.db_name - env_dict[ - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME"] = clp_config.results_cache.stream_collection_name + env_dict["CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME"] = ( + clp_config.results_cache.stream_collection_name + ) def start_compression_scheduler( - clp_config: CLPConfig, + clp_config: CLPConfig, ): component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -267,14 +271,15 @@ def start_compression_scheduler( logs_dir.mkdir(parents=True, exist_ok=True) env_dict["CLP_HOST_COMPRESSION_SCHEDULER_LOGS_DIR"] = str(logs_dir) - env_dict[ - "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL"] = clp_config.compression_scheduler.logging_level + env_dict["CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL"] = ( + clp_config.compression_scheduler.logging_level + ) # FIXME: handle S3 def start_query_scheduler( - clp_config: CLPConfig, + clp_config: CLPConfig, ): component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -289,8 +294,8 @@ def start_query_scheduler( def start_compression_worker( - clp_config: CLPConfig, - num_cpus: int, + clp_config: CLPConfig, + num_cpus: int, ): component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -310,8 +315,8 @@ def start_compression_worker( def start_query_worker( - clp_config: CLPConfig, - num_cpus: int, + clp_config: CLPConfig, + num_cpus: int, ): component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -331,8 +336,8 @@ def start_query_worker( def start_reducer( - clp_config: CLPConfig, - num_workers: int, + clp_config: CLPConfig, + num_workers: int, ): component_name = REDUCER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -347,9 +352,9 @@ def start_reducer( def update_settings_object( - parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], + parent_key_prefix: str, + settings: Dict[str, Any], + updates: Dict[str, Any], ): """ Recursively updates the given settings object with the values from `updates`. @@ -384,18 +389,18 @@ def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dic def start_webui( - clp_config: CLPConfig, - container_clp_config: CLPConfig, + clp_config: CLPConfig, + container_clp_config: CLPConfig, ): component_name = WEBUI_COMPONENT_NAME logger.info(f"Initializing {component_name}...") container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" ) validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path) @@ -473,6 +478,7 @@ def start_webui( # FIXME: Handle S3 + def start_garbage_collector( clp_config: CLPConfig, ): @@ -560,20 +566,12 @@ def main(argv): # FIXME: handle this num_workers = multiprocessing.cpu_count() // 2 - container_clp_config, mounts = generate_container_config(clp_config, clp_home) + container_clp_config = generate_docker_compose_container_config(clp_config) # Create necessary directories clp_config.data_directory.mkdir(parents=True, exist_ok=True) clp_config.logs_directory.mkdir(parents=True, exist_ok=True) - # Set container services' hosts in container config - container_clp_config.database.host = DB_COMPONENT_NAME - container_clp_config.queue.host = QUEUE_COMPONENT_NAME - container_clp_config.redis.host = REDIS_COMPONENT_NAME - container_clp_config.results_cache.host = RESULTS_CACHE_COMPONENT_NAME - container_clp_config.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME - container_clp_config.reducer.host = REDUCER_COMPONENT_NAME - dump_shared_container_config(container_clp_config, clp_config) env_dict["CLP_USER_ID"] = os.getuid() @@ -585,20 +583,33 @@ def main(argv): env_dict["CLP_HOST_ARCHIVE_OUTPUT_DIR"] = str(clp_config.archive_output.get_directory()) env_dict["CLP_HOST_STREAM_OUTPUT_DIR"] = str(clp_config.stream_output.get_directory()) + if clp_config.aws_config_directory is not None: + env_dict["CLP_HOST_AWS_CONFIG_DIR"] = str(clp_config.aws_config_directory) + if StorageType.S3 == clp_config.archive_output.storage.type: + clp_config.archive_output.storage.staging_directory.mkdir(parents=True, exist_ok=True) + env_dict["CLP_HOST_ARCHIVE_STAGING_DIR"] = str( + clp_config.archive_output.storage.staging_directory + ) + if StorageType.S3 == clp_config.stream_output.storage.type: + clp_config.stream_output.storage.staging_directory.mkdir(parents=True, exist_ok=True) + env_dict["CLP_HOST_STREAM_STAGING_DIR"] = str( + clp_config.stream_output.storage.staging_directory + ) + try: conf_dir = clp_home / "etc" # Start components - start_db( clp_config, conf_dir) - start_queue( clp_config) - start_redis( clp_config, conf_dir) - start_results_cache( clp_config, conf_dir) + start_db(clp_config, conf_dir) + start_queue(clp_config) + start_redis(clp_config, conf_dir) + start_results_cache(clp_config, conf_dir) start_compression_scheduler(clp_config) start_query_scheduler(clp_config) start_compression_worker(clp_config, num_workers) - start_query_worker( clp_config, num_workers) - start_reducer( clp_config, num_workers) - start_webui( clp_config, container_clp_config) + start_query_worker(clp_config, num_workers) + start_reducer(clp_config, num_workers) + start_webui(clp_config, container_clp_config) start_garbage_collector(clp_config) with open(f"{clp_home}/.env", "w") as env_file: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 175ab38f7d..3d9932a2c5 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -793,7 +793,7 @@ class CLPConfig(BaseModel): database: Database = Database() queue: Queue = Queue() redis: Redis = Redis() - reducer: Reducer() = Reducer() + reducer: Reducer = Reducer() results_cache: ResultsCache = ResultsCache() compression_scheduler: CompressionScheduler = CompressionScheduler() query_scheduler: QueryScheduler = QueryScheduler() @@ -932,15 +932,15 @@ def get_runnable_components(self) -> Set[str]: return ALL_COMPONENTS def dump_to_primitive_dict(self): - custom_serialized_fields = ( + custom_serialized_fields = { "database", "queue", "redis", "logs_input", "archive_output", "stream_output", - ) - d = self.dict(exclude=set(custom_serialized_fields)) + } + d = self.dict(exclude=custom_serialized_fields) for key in custom_serialized_fields: d[key] = getattr(self, key).dump_to_primitive_dict() diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 03410db3f2..0cd1c84aa1 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -1,13 +1,14 @@ +# FIXME: avoid hardcoding data / log paths secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" services: db: - image: ${CLP_DB_IMAGE:-mysql:8.0.23} + image: "${CLP_DB_IMAGE:-mysql:8.0.23}" container_name: "database" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" secrets: - - CLP_DB_PASS_FILE + - "CLP_DB_PASS_FILE" environment: MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" MYSQL_USER: "${CLP_DB_USER}" @@ -15,8 +16,8 @@ services: MYSQL_DATABASE: "${CLP_DB_NAME}" volumes: - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" - - "./var/data/database:/var/lib/mysql" - - "./var/log/database:/var/log/mysql" + - "${CLP_HOST_DATA_DIR:-./var/data}/database:/var/lib/mysql" + - "${CLP_HOST_LOGS_DIR:-./var/log}/database:/var/log/mysql" networks: - "clp-network" ports: @@ -31,7 +32,7 @@ services: "--password=${CLP_DB_PASS}" ] start_period: "10s" - start_interval: "2s" + start_interval: "1s" interval: "30s" timeout: "10s" retries: 3 @@ -52,7 +53,7 @@ services: "--storage-engine", "${CLP_STORAGE_ENGINE}" ] volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" networks: - "clp-network" depends_on: @@ -68,7 +69,7 @@ services: RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" volumes: - - "./var/log/queue:/var/log/rabbitmq" + - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" networks: - "clp-network" ports: @@ -79,7 +80,7 @@ services: "rabbitmq-diagnostics", "check_running" ] start_period: "10s" - start_interval: "2s" + start_interval: "1s" interval: "30s" timeout: "10s" retries: 3 @@ -95,8 +96,8 @@ services: ] volumes: - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" - - "./var/log/redis:/var/log/redis" - - "./var/data/redis:/data" + - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" + - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" networks: - "clp-network" ports: @@ -111,9 +112,9 @@ services: "PING" ] start_period: "10s" - start_interval: "2s" - interval: 30s - timeout: 10s + start_interval: "1s" + interval: "30s" + timeout: "10s" retries: 3 results-cache: @@ -126,18 +127,18 @@ services: ] volumes: - "./etc/mongo:/etc/mongo:ro" - - "./var/data/results_cache:/data/db" - - "./var/log/results_cache:/var/log/mongodb" + - "${CLP_HOST_DATA_DIR:-./var/data}/results_cache:/data/db" + - "${CLP_HOST_LOGS_DIR:-./var/log}/results_cache:/var/log/mongodb" networks: - "clp-network" ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet + test: "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet" start_period: "10s" - start_interval: "2s" - interval: 30s - timeout: 10s + start_interval: "1s" + interval: "30s" + timeout: "10s" retries: 3 results-cache-indices-creator: @@ -172,10 +173,11 @@ services: CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log:/var/log" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" # FIXME: why not call this /mnt/input? - "/:/mnt/logs:ro" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" networks: - "clp-network" depends_on: @@ -191,12 +193,6 @@ services: "-m", "job_orchestration.scheduler.compress.compression_scheduler", "--config", "/etc/clp-config.yml" ] - healthcheck: - # FIXME - test: [ - "CMD", - "true" - ] query-scheduler: image: "clp-package-x86-ubuntu-jammy:dev" @@ -211,8 +207,8 @@ services: CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log:/var/log" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" networks: - "clp-network" depends_on: @@ -229,11 +225,18 @@ services: "--config", "/etc/clp-config.yml" ] healthcheck: - # FIXME + # FIXME: need to suppressing warnings in the schduler for reading 0 out of 8 expected bytes test: [ "CMD", - "true" + "bash", + "-c", + "< /dev/tcp/query_scheduler/7000" ] + start_period: "10s" + start_interval: "1s" + interval: "30s" + timeout: "10s" + retries: 3 compression-worker: image: "clp-package-x86-ubuntu-jammy:dev" @@ -249,19 +252,17 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" CLP_CONFIG_PATH: "/etc/clp-config.yml" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log/compression_worker:/var/log/compression_worker" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" # FIXME: why dump "var/data/compression-job-4-task-4-db-config.yml"? - - "./var/data:/var/data" - - "./var/data/archives:/var/data/archives" + - "${CLP_HOST_DATA_DIR:-./var/data}:/var/data" + - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" # FIXME: why not call this /mnt/input? - "/:/mnt/logs:ro" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" + - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" networks: - "clp-network" - depends_on: - compression-scheduler: - condition: "service_healthy" - # FIXME: add healthcheck command: [ "python3", "-u", @@ -289,15 +290,14 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" CLP_CONFIG_PATH: "/etc/clp-config.yml" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log/query_worker:/var/log/query_worker" - - "./var/data/archives:/var/data/archives" - - "./var/data/streams:/var/data/streams" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/query_worker:/var/log/query_worker" + - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" + - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" + - "${CLP_HOST_STREAM_STAGING_DIR:-./var/data/staged-streams}:/var/data/staged-streams" networks: - "clp-network" - depends_on: - query-scheduler: - condition: "service_healthy" command: [ "python3", "-u", @@ -321,8 +321,8 @@ services: CLP_LOGS_DIR: "/var/log/reducer" CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" - - "./var/log/reducer:/var/log/reducer" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/reducer:/var/log/reducer" networks: - "clp-network" depends_on: @@ -350,11 +350,10 @@ services: CLP_DB_PASS: "${CLP_DB_PASS}" NODE_ENV: "production" volumes: - - "./var/data/streams:/var/data/streams" - - type: "bind" - source: "./var/www/webui/server/dist/server/settings.json" - target: "/opt/clp/var/www/webui/server/dist/server/settings.json" - read_only: true + - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" + - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" + - "./var/www/webui/server/dist/server/settings.json:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" networks: - "clp-network" ports: @@ -368,6 +367,13 @@ services: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" + healthcheck: + test: [ + "CMD", + "bash", + "-c", + "< /dev/tcp/webui/4000" + ] garbage-collector: image: "clp-package-x86-ubuntu-jammy:dev" @@ -381,7 +387,7 @@ services: CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "./var/log/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" networks: - "clp-network" @@ -400,4 +406,4 @@ services: networks: clp-network: - driver: bridge \ No newline at end of file + driver: "bridge" From c229fe035fc762ce29246d145c18ed7ef674dbd1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 08:56:22 -0400 Subject: [PATCH 004/408] lint --- tools/deployment/package/docker-compose.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 0cd1c84aa1..5f445f0b45 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -134,7 +134,9 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" healthcheck: - test: "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet" + test: >- + echo 'db.runCommand("ping").ok' | + mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet start_period: "10s" start_interval: "1s" interval: "30s" @@ -167,7 +169,8 @@ services: environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1}" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} CLP_LOGS_DIR: "/var/log" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_DB_USER: "${CLP_DB_USER}" @@ -201,7 +204,8 @@ services: environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0}" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} CLP_LOGS_DIR: "/var/log" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_DB_USER: "${CLP_DB_USER}" @@ -245,7 +249,8 @@ services: environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1}" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} CLP_HOME: "/opt/clp" CLP_LOGS_DIR: "/var/log/compression_worker" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" @@ -283,7 +288,8 @@ services: environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0}" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} CLP_HOME: "/opt/clp" CLP_LOGS_DIR: "/var/log/query_worker" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" @@ -352,7 +358,8 @@ services: volumes: - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/server/settings.json:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" + - "./var/www/webui/server/dist/server/settings.json\ +:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" networks: - "clp-network" From a850f6948a01235c5f1c1610ce2eab183f9f241d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:02:14 -0400 Subject: [PATCH 005/408] add Docker Compose launch --- .../clp_package_utils/scripts/start_clp.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index c163df9e0a..b3954c34f7 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -622,16 +622,17 @@ def main(argv): logger.exception("Failed to initialize CLP.") return -1 - # try: - # subprocess.run( - # "docker compose up", - # shell=True, - # stderr=subprocess.STDOUT, - # check=True, - # ) - # except subprocess.CalledProcessError: - # logger.exception("Failed to start CLP.") - # return -1 + try: + logger.info(f"Starting CLP with Docker Compose...") + subprocess.run( + "docker compose up -d", + shell=True, + stderr=subprocess.STDOUT, + check=True, + ) + except subprocess.CalledProcessError: + logger.exception("Failed to start CLP.") + return -1 return 0 From b28323139c292286ac75330c98173556a9391edd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:10:22 -0400 Subject: [PATCH 006/408] remove unused import --- .../clp_package_utils/scripts/start_clp.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index b3954c34f7..4519cb1310 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -8,11 +8,9 @@ import subprocess import sys import time -import uuid from typing import Any, Dict, List, Optional from clp_py_utils.clp_config import ( - ALL_TARGET_NAME, AwsAuthType, CLPConfig, COMPRESSION_JOBS_TABLE_NAME, @@ -21,7 +19,6 @@ CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, GARBAGE_COLLECTOR_COMPONENT_NAME, - get_components_for_target, QUERY_JOBS_TABLE_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, @@ -38,38 +35,26 @@ get_datasets_table_name, get_files_table_name, ) -from clp_py_utils.s3_utils import generate_container_auth_options -from job_orchestration.scheduler.constants import QueueName -from pydantic import BaseModel from clp_package_utils.general import ( check_dependencies, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, - CLPDockerMounts, CONTAINER_CLP_HOME, DockerMount, - DockerMountType, dump_shared_container_config, - generate_container_config, generate_docker_compose_container_config, - get_celery_connection_env_vars_list, get_clp_home, - get_common_env_vars_list, - get_credential_env_vars_list, is_container_exited, is_container_running, - is_retention_period_configured, load_config_file, validate_and_load_db_credentials_file, validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, validate_db_config, - validate_log_directory, validate_logs_input_config, validate_output_storage_config, validate_queue_config, validate_redis_config, - validate_reducer_config, validate_results_cache_config, validate_retention_config, validate_webui_config, From 743268cfc59d7afc4e7ed7e83bf7c04522aa76bc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:23:18 -0400 Subject: [PATCH 007/408] add proper stop support; reduce stop_grace_period from 10s to 3s --- .../clp_package_utils/scripts/start_clp.py | 5 +- .../clp_package_utils/scripts/stop_clp.py | 94 ++----------------- tools/deployment/package/docker-compose.yml | 17 +++- 3 files changed, 27 insertions(+), 89 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 4519cb1310..2f44d064c3 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -607,11 +607,10 @@ def main(argv): logger.exception("Failed to initialize CLP.") return -1 + logger.info(f"Starting CLP using Docker Compose...") try: - logger.info(f"Starting CLP with Docker Compose...") subprocess.run( - "docker compose up -d", - shell=True, + ["docker", "compose", "up", "-d"], stderr=subprocess.STDOUT, check=True, ) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 899b7a96a3..632bc8f7f1 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -120,92 +120,16 @@ def main(argv): logger.exception("Failed to load config.") return -1 + logger.info("Stopping all CLP containers using Docker Compose...") try: - # Read instance ID from file - logs_dir = clp_config.logs_directory - instance_id_file_path = logs_dir / "instance-id" - if not (logs_dir.exists() and logs_dir.is_dir() and instance_id_file_path.exists()): - # No instance ID file, so nothing to do - return 0 - with open(instance_id_file_path, "r") as f: - instance_id = f.readline() - - already_exited_containers = [] - force = parsed_args.force - if target in (ALL_TARGET_NAME, GARBAGE_COLLECTOR_COMPONENT_NAME): - container_name = f"clp-{GARBAGE_COLLECTOR_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME): - container_name = f"clp-{WEBUI_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, REDUCER_COMPONENT_NAME): - container_name = f"clp-{REDUCER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - container_config_file_path = logs_dir / f"{container_name}.yml" - if container_config_file_path.exists(): - container_config_file_path.unlink() - if target in (ALL_TARGET_NAME, QUERY_WORKER_COMPONENT_NAME): - container_name = f"clp-{QUERY_WORKER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, COMPRESSION_WORKER_COMPONENT_NAME): - container_name = f"clp-{COMPRESSION_WORKER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, QUERY_SCHEDULER_COMPONENT_NAME): - container_name = f"clp-{QUERY_SCHEDULER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - container_config_file_path = logs_dir / f"{container_name}.yml" - if container_config_file_path.exists(): - container_config_file_path.unlink() - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - ): - container_name = f"clp-{COMPRESSION_SCHEDULER_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - container_config_file_path = logs_dir / f"{container_name}.yml" - if container_config_file_path.exists(): - container_config_file_path.unlink() - if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, REDIS_COMPONENT_NAME): - container_name = f"clp-{REDIS_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - redis_config_file_path = logs_dir / f"{container_name}.conf" - if redis_config_file_path.exists(): - redis_config_file_path.unlink() - if target in (ALL_TARGET_NAME, RESULTS_CACHE_COMPONENT_NAME): - container_name = f"clp-{RESULTS_CACHE_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, QUEUE_COMPONENT_NAME): - container_name = f"clp-{QUEUE_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - queue_config_file_path = logs_dir / f"{container_name}.conf" - if queue_config_file_path.exists(): - queue_config_file_path.unlink() - if target in (ALL_TARGET_NAME, DB_COMPONENT_NAME): - container_name = f"clp-{DB_COMPONENT_NAME}-{instance_id}" - stop_running_container(container_name, already_exited_containers, force) - - if already_exited_containers: - container_list = " ".join(already_exited_containers) - logger.warning( - f"The following containers have already exited and were not removed:" - f" {container_list}" - ) - logger.warning(f"Run with --force to remove them") - elif target in ALL_TARGET_NAME: - # NOTE: We can only remove the instance ID file if all containers have been stopped. - # Currently, we only remove the instance file when all containers are stopped at once. - # If a single container is stopped, it's expensive to check if the others are running, - # so instead we don't remove the instance file. In the worst case, a user will have to - # remove it manually. - instance_id_file_path.unlink() - except: - logger.exception("Failed to stop CLP.") + subprocess.run( + ["docker", "compose", "down"], + stderr=subprocess.STDOUT, + check=True, + ) + logger.info("All CLP containers stopped.") + except subprocess.CalledProcessError: + logger.exception("Failed to stop CLP containers using Docker Compose.") return -1 return 0 diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 5f445f0b45..22f4de3d28 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -1,9 +1,12 @@ -# FIXME: avoid hardcoding data / log paths +x-service-defaults: &service_defaults + stop_grace_period: "3s" + secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" services: db: + <<: *service_defaults image: "${CLP_DB_IMAGE:-mysql:8.0.23}" container_name: "database" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -38,6 +41,7 @@ services: retries: 3 db-table-creator: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "db_table_creator" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -61,6 +65,7 @@ services: condition: "service_healthy" queue: + <<: *service_defaults image: "rabbitmq:3.9.8" container_name: "queue" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -86,6 +91,7 @@ services: retries: 3 redis: + <<: *service_defaults image: "redis:7.2.4" container_name: "redis" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -118,6 +124,7 @@ services: retries: 3 results-cache: + <<: *service_defaults image: "mongo:7.0.1" container_name: "results_cache" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -144,6 +151,7 @@ services: retries: 3 results-cache-indices-creator: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "results_cache_indices_creator" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -163,6 +171,7 @@ services: condition: "service_healthy" compression-scheduler: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "compression_scheduler" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -198,6 +207,7 @@ services: ] query-scheduler: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "query_scheduler" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -243,6 +253,7 @@ services: retries: 3 compression-worker: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "compression_worker" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -282,6 +293,7 @@ services: ] query-worker: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "query_worker" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -318,6 +330,7 @@ services: ] reducer: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "reducer" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -345,6 +358,7 @@ services: ] webui: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "webui" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" @@ -383,6 +397,7 @@ services: ] garbage-collector: + <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "garbage_collector" user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" From ee1b9f28f23ced8fb3fc65e72e371530ca0b4df5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:33:32 -0400 Subject: [PATCH 008/408] de-duplicate docker-compose.yml --- tools/deployment/package/docker-compose.yml | 88 +++++---------------- 1 file changed, 20 insertions(+), 68 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 22f4de3d28..9f79b41aef 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -1,15 +1,28 @@ x-service-defaults: &service_defaults + networks: ["clp-network"] stop_grace_period: "3s" + user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + +x-healthcheck-defaults: &healthcheck_defaults + start_period: "10s" + start_interval: "1s" + interval: "30s" + timeout: "10s" + retries: 3 + +networks: + clp-network: + driver: "bridge" secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" + services: db: <<: *service_defaults image: "${CLP_DB_IMAGE:-mysql:8.0.23}" container_name: "database" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" secrets: - "CLP_DB_PASS_FILE" environment: @@ -21,11 +34,10 @@ services: - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/database:/var/lib/mysql" - "${CLP_HOST_LOGS_DIR:-./var/log}/database:/var/log/mysql" - networks: - - "clp-network" ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" healthcheck: + <<: *healthcheck_defaults test: [ "CMD", "mysqladmin", "ping", @@ -34,17 +46,11 @@ services: "-u", "${CLP_DB_USER}", "--password=${CLP_DB_PASS}" ] - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 db-table-creator: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "db_table_creator" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" CLP_DB_USER: "${CLP_DB_USER}" @@ -58,8 +64,6 @@ services: ] volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" - networks: - - "clp-network" depends_on: db: condition: "service_healthy" @@ -68,33 +72,25 @@ services: <<: *service_defaults image: "rabbitmq:3.9.8" container_name: "queue" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" - networks: - - "clp-network" ports: - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" healthcheck: + <<: *healthcheck_defaults test: [ "CMD", "rabbitmq-diagnostics", "check_running" ] - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 redis: <<: *service_defaults image: "redis:7.2.4" container_name: "redis" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" command: [ "redis-server", "/usr/local/etc/redis/redis.conf", @@ -104,11 +100,10 @@ services: - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" - networks: - - "clp-network" ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" healthcheck: + <<: *healthcheck_defaults test: [ "CMD", "redis-cli", @@ -117,17 +112,11 @@ services: "-a", "${CLP_REDIS_PASS}", "PING" ] - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 results-cache: <<: *service_defaults image: "mongo:7.0.1" container_name: "results_cache" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" command: [ "--config", "/etc/mongo/mongod.conf", "--bind_ip", "0.0.0.0", @@ -136,25 +125,18 @@ services: - "./etc/mongo:/etc/mongo:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/results_cache:/data/db" - "${CLP_HOST_LOGS_DIR:-./var/log}/results_cache:/var/log/mongodb" - networks: - - "clp-network" ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" healthcheck: + <<: *healthcheck_defaults test: >- echo 'db.runCommand("ping").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 results-cache-indices-creator: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "results_cache_indices_creator" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" command: [ @@ -164,8 +146,6 @@ services: "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", ] - networks: - - "clp-network" depends_on: results-cache: condition: "service_healthy" @@ -174,7 +154,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "compression_scheduler" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -190,8 +169,6 @@ services: # FIXME: why not call this /mnt/input? - "/:/mnt/logs:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - networks: - - "clp-network" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -210,7 +187,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "query_scheduler" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -223,8 +199,6 @@ services: volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" - networks: - - "clp-network" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -239,6 +213,7 @@ services: "--config", "/etc/clp-config.yml" ] healthcheck: + <<: *healthcheck_defaults # FIXME: need to suppressing warnings in the schduler for reading 0 out of 8 expected bytes test: [ "CMD", @@ -246,17 +221,11 @@ services: "-c", "< /dev/tcp/query_scheduler/7000" ] - start_period: "10s" - start_interval: "1s" - interval: "30s" - timeout: "10s" - retries: 3 compression-worker: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "compression_worker" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -277,8 +246,6 @@ services: - "/:/mnt/logs:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" - networks: - - "clp-network" command: [ "python3", "-u", @@ -296,7 +263,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "query_worker" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -314,8 +280,6 @@ services: - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_STREAM_STAGING_DIR:-./var/data/staged-streams}:/var/data/staged-streams" - networks: - - "clp-network" command: [ "python3", "-u", @@ -333,7 +297,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "reducer" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" CLP_HOME: "/opt/clp" @@ -342,8 +305,6 @@ services: volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}/reducer:/var/log/reducer" - networks: - - "clp-network" depends_on: query-scheduler: condition: "service_healthy" @@ -361,7 +322,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "webui" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" HOST: "0.0.0.0" @@ -375,8 +335,6 @@ services: - "./var/www/webui/server/dist/server/settings.json\ :/opt/clp/var/www/webui/server/dist/server/settings.json:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - networks: - - "clp-network" ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" command: [ @@ -389,6 +347,7 @@ services: results-cache-indices-creator: condition: "service_completed_successfully" healthcheck: + <<: *healthcheck_defaults test: [ "CMD", "bash", @@ -400,7 +359,6 @@ services: <<: *service_defaults image: "clp-package-x86-ubuntu-jammy:dev" container_name: "garbage_collector" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" CLP_HOME: "/opt/clp" @@ -411,8 +369,6 @@ services: volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" - networks: - - "clp-network" depends_on: query-scheduler: condition: "service_healthy" @@ -425,7 +381,3 @@ services: "-m", "job_orchestration.garbage_collector.garbage_collector", "--config", "/etc/clp-config.yml", ] - -networks: - clp-network: - driver: "bridge" From ec0cfa587c212030bc0cb85e034e89f8c1624f6e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:53:58 -0400 Subject: [PATCH 009/408] replace hardcoded image name with environment variable for container configuration --- .../clp_package_utils/scripts/start_clp.py | 2 ++ tools/deployment/package/docker-compose.yml | 18 +++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 2f44d064c3..c8eb0334ce 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -559,6 +559,8 @@ def main(argv): dump_shared_container_config(container_clp_config, clp_config) + env_dict["CLP_PACKAGE_CONTAINER"] = "clp-package-x86-ubuntu-jammy:dev" + env_dict["CLP_USER_ID"] = os.getuid() env_dict["CLP_GROUP_ID"] = os.getgid() env_dict["CLP_STORAGE_ENGINE"] = clp_config.package.storage_engine diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 9f79b41aef..31efd363a9 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -49,7 +49,7 @@ services: db-table-creator: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "db_table_creator" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -135,7 +135,7 @@ services: results-cache-indices-creator: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "results_cache_indices_creator" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -152,7 +152,7 @@ services: compression-scheduler: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "compression_scheduler" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -185,7 +185,7 @@ services: query-scheduler: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "query_scheduler" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -224,7 +224,7 @@ services: compression-worker: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "compression_worker" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -261,7 +261,7 @@ services: query-worker: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "query_worker" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -295,7 +295,7 @@ services: reducer: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "reducer" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -320,7 +320,7 @@ services: webui: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "webui" environment: NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" @@ -357,7 +357,7 @@ services: garbage-collector: <<: *service_defaults - image: "clp-package-x86-ubuntu-jammy:dev" + image: "${CLP_PACKAGE_CONTAINER}" container_name: "garbage_collector" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" From ab68c259eb63eb353c69750baa657f7c0cebdd22 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 09:55:48 -0400 Subject: [PATCH 010/408] remove FIXME --- tools/deployment/package/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 31efd363a9..fc39782f0c 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -239,10 +239,8 @@ services: volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" - # FIXME: why dump "var/data/compression-job-4-task-4-db-config.yml"? - "${CLP_HOST_DATA_DIR:-./var/data}:/var/data" - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" - # FIXME: why not call this /mnt/input? - "/:/mnt/logs:ro" - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" From 934a83c394a1a8bc478b6163583e2e92aa28994e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:10:40 -0400 Subject: [PATCH 011/408] remove FIXME --- .../clp_package_utils/scripts/start_clp.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index c8eb0334ce..307574511e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -260,7 +260,6 @@ def start_compression_scheduler( clp_config.compression_scheduler.logging_level ) - # FIXME: handle S3 def start_query_scheduler( @@ -275,8 +274,6 @@ def start_query_scheduler( env_dict["CLP_HOST_QUERY_SCHEDULER_LOGS_DIR"] = str(logs_dir) env_dict["CLP_QUERY_SCHEDULER_LOGGING_LEVEL"] = clp_config.query_scheduler.logging_level - # FIXME: handle S3 - def start_compression_worker( clp_config: CLPConfig, @@ -296,8 +293,6 @@ def start_compression_worker( clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - # FIXME: handle S3 - def start_query_worker( clp_config: CLPConfig, @@ -317,8 +312,6 @@ def start_query_worker( clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - # FIXME: handle S3 - def start_reducer( clp_config: CLPConfig, @@ -461,8 +454,6 @@ def start_webui( env_dict["CLP_WEBUI_HOST"] = get_ip_from_hostname(clp_config.webui.host) env_dict["CLP_WEBUI_PORT"] = clp_config.webui.port - # FIXME: Handle S3 - def start_garbage_collector( clp_config: CLPConfig, @@ -484,7 +475,6 @@ def add_num_workers_argument(parser): default=multiprocessing.cpu_count(), help="Number of workers to start", ) - # FIXME: handle S3 def main(argv): @@ -548,7 +538,7 @@ def main(argv): logger.exception("Failed to load config.") return -1 - # FIXME: handle this + # TODO: Rely on Docker Compose to spawn multiple workers num_workers = multiprocessing.cpu_count() // 2 container_clp_config = generate_docker_compose_container_config(clp_config) From 83cc9d12ac05b876df3623c723694f2e2aff0a87 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:39:50 -0400 Subject: [PATCH 012/408] reformat --- tools/deployment/package/docker-compose.yml | 210 ++++++++++---------- 1 file changed, 104 insertions(+), 106 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index fc39782f0c..8501be6dae 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -4,11 +4,11 @@ x-service-defaults: &service_defaults user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" x-healthcheck-defaults: &healthcheck_defaults - start_period: "10s" - start_interval: "1s" interval: "30s" - timeout: "10s" retries: 3 + start_interval: "1s" + start_period: "10s" + timeout: "10s" networks: clp-network: @@ -21,21 +21,21 @@ secrets: services: db: <<: *service_defaults - image: "${CLP_DB_IMAGE:-mysql:8.0.23}" container_name: "database" - secrets: - - "CLP_DB_PASS_FILE" + image: "${CLP_DB_IMAGE:-mysql:8.0.23}" environment: + MYSQL_DATABASE: "${CLP_DB_NAME}" + MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" MYSQL_USER: "${CLP_DB_USER}" - MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_DATABASE: "${CLP_DB_NAME}" + secrets: + - "CLP_DB_PASS_FILE" + ports: + - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/database:/var/lib/mysql" - "${CLP_HOST_LOGS_DIR:-./var/log}/database:/var/log/mysql" - ports: - - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" healthcheck: <<: *healthcheck_defaults test: [ @@ -49,12 +49,17 @@ services: db-table-creator: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "db_table_creator" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + depends_on: + db: + condition: "service_healthy" command: [ "python3", "-u", @@ -62,24 +67,19 @@ services: "--config", "/etc/clp-config.yml", "--storage-engine", "${CLP_STORAGE_ENGINE}" ] - volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" - depends_on: - db: - condition: "service_healthy" queue: <<: *service_defaults - image: "rabbitmq:3.9.8" container_name: "queue" + image: "rabbitmq:3.9.8" environment: - RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" - volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" + volumes: + - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" healthcheck: <<: *healthcheck_defaults test: [ @@ -89,19 +89,14 @@ services: redis: <<: *service_defaults - image: "redis:7.2.4" container_name: "redis" - command: [ - "redis-server", - "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS}" - ] - volumes: - - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" - - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" + image: "redis:7.2.4" ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" + volumes: + - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" + - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" + - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" healthcheck: <<: *healthcheck_defaults test: [ @@ -112,33 +107,41 @@ services: "-a", "${CLP_REDIS_PASS}", "PING" ] + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS}" + ] results-cache: <<: *service_defaults - image: "mongo:7.0.1" container_name: "results_cache" - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] + image: "mongo:7.0.1" + ports: + - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "./etc/mongo:/etc/mongo:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/results_cache:/data/db" - "${CLP_HOST_LOGS_DIR:-./var/log}/results_cache:/var/log/mongodb" - ports: - - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + - "./etc/mongo:/etc/mongo:ro" healthcheck: <<: *healthcheck_defaults test: >- echo 'db.runCommand("ping").ok' | mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] results-cache-indices-creator: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "results_cache_indices_creator" + image: "${CLP_PACKAGE_CONTAINER}" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" + depends_on: + results-cache: + condition: "service_healthy" command: [ "python3", "-u", @@ -146,29 +149,25 @@ services: "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", ] - depends_on: - results-cache: - condition: "service_healthy" compression-scheduler: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "compression_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} - CLP_LOGS_DIR: "/var/log" - CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" - # FIXME: why not call this /mnt/input? + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -185,20 +184,20 @@ services: query-scheduler: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "query_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} - CLP_LOGS_DIR: "/var/log" - CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_DB_PASS: "${CLP_DB_PASS}" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -224,26 +223,26 @@ services: compression-worker: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "compression_worker" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" - CLP_LOGS_DIR: "/var/log/compression_worker" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/compression_worker" CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" - CLP_CONFIG_PATH: "/etc/clp-config.yml" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" + - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}:/var/data" - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" - "/:/mnt/logs:ro" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" command: [ "python3", "-u", @@ -259,24 +258,24 @@ services: query-worker: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "query_worker" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} + CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" - CLP_LOGS_DIR: "/var/log/query_worker" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/query_worker" CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" - CLP_CONFIG_PATH: "/etc/clp-config.yml" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/query_worker:/var/log/query_worker" + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/query_worker:/var/log/query_worker" - "${CLP_HOST_STREAM_STAGING_DIR:-./var/data/staged-streams}:/var/data/staged-streams" command: [ "python3", @@ -293,13 +292,13 @@ services: reducer: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "reducer" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" CLP_HOME: "/opt/clp" - CLP_LOGS_DIR: "/var/log/reducer" CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/reducer" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_HOST_LOGS_DIR:-./var/log}/reducer:/var/log/reducer" @@ -318,32 +317,31 @@ services: webui: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "webui" + image: "${CLP_PACKAGE_CONTAINER}" environment: - NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" - HOST: "0.0.0.0" - PORT: "4000" - CLP_DB_USER: "${CLP_DB_USER}" CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + HOST: "0.0.0.0" NODE_ENV: "production" + NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" + PORT: "4000" + ports: + - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: + - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/server/settings.json\ -:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - ports: - - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" - command: [ - "/opt/clp/bin/node-22", - "/opt/clp/var/www/webui/server/dist/server/src/main.js" - ] + - "./var/www/webui/server/dist/server/settings.json:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" depends_on: db-table-creator: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/server/src/main.js" + ] healthcheck: <<: *healthcheck_defaults test: [ @@ -355,23 +353,23 @@ services: garbage-collector: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" container_name: "garbage_collector" + image: "${CLP_PACKAGE_CONTAINER}" environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" CLP_HOME: "/opt/clp" - CLP_LOGS_DIR: "/var/log/garbage_collector" CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_LOGS_DIR: "/var/log/garbage_collector" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" + - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" depends_on: - query-scheduler: - condition: "service_healthy" db-table-creator: condition: "service_completed_successfully" + query-scheduler: + condition: "service_healthy" results-cache-indices-creator: condition: "service_completed_successfully" command: [ From 82abc077193ac67267ff419e4fb406a56eab68a0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:41:07 -0400 Subject: [PATCH 013/408] reformat --- tools/deployment/package/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 8501be6dae..f5057ebe3f 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -332,7 +332,8 @@ services: - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/server/settings.json:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" + - "./var/www/webui/server/dist/server/settings.json\ +:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" depends_on: db-table-creator: condition: "service_completed_successfully" From 0fb229416d1fea146d66c2d6a50fc1728812ac7c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:43:32 -0400 Subject: [PATCH 014/408] Update garbage collector logs directory mapping --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 1 - tools/deployment/package/docker-compose.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 307574511e..7eb790ac47 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -464,7 +464,6 @@ def start_garbage_collector( logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_GC_LOGS_DIR"] = str(logs_dir) env_dict["CLP_GC_LOGGING_LEVEL"] = clp_config.garbage_collector.logging_level diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index f5057ebe3f..8648d6492a 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -364,8 +364,8 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_GC_LOGS_DIR}:/var/log/garbage_collector" - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_HOST_LOGS_DIR:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: db-table-creator: condition: "service_completed_successfully" From c8ffb949b4e854bde8203cbfa64e0c8e6961ec0e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:46:08 -0400 Subject: [PATCH 015/408] Remove unused component argument parsers --- .../clp_package_utils/scripts/start_clp.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 7eb790ac47..e1e52dbfef 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -488,23 +488,6 @@ def main(argv): help="CLP package configuration file.", ) - component_args_parser = args_parser.add_subparsers(dest="target") - component_args_parser.add_parser(CONTROLLER_TARGET_NAME) - component_args_parser.add_parser(DB_COMPONENT_NAME) - component_args_parser.add_parser(QUEUE_COMPONENT_NAME) - component_args_parser.add_parser(REDIS_COMPONENT_NAME) - component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME) - component_args_parser.add_parser(COMPRESSION_SCHEDULER_COMPONENT_NAME) - component_args_parser.add_parser(QUERY_SCHEDULER_COMPONENT_NAME) - compression_worker_parser = component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME) - add_num_workers_argument(compression_worker_parser) - query_worker_parser = component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME) - add_num_workers_argument(query_worker_parser) - reducer_server_parser = component_args_parser.add_parser(REDUCER_COMPONENT_NAME) - add_num_workers_argument(reducer_server_parser) - component_args_parser.add_parser(WEBUI_COMPONENT_NAME) - component_args_parser.add_parser(GARBAGE_COLLECTOR_COMPONENT_NAME) - parsed_args = args_parser.parse_args(argv[1:]) try: From 83e902a39ec34ffcb6f57b10c0120478e7903c0a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:48:11 -0400 Subject: [PATCH 016/408] Remove unused component argument parsers --- .../clp_package_utils/scripts/start_clp.py | 1 - .../clp_package_utils/scripts/stop_clp.py | 118 +----------------- 2 files changed, 2 insertions(+), 117 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index e1e52dbfef..2d0850eea8 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -261,7 +261,6 @@ def start_compression_scheduler( ) - def start_query_scheduler( clp_config: CLPConfig, ): diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 632bc8f7f1..03981bd474 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -1,125 +1,11 @@ -import argparse import logging -import pathlib import subprocess import sys -from typing import List - -from clp_py_utils.clp_config import ( - ALL_TARGET_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - CONTROLLER_TARGET_NAME, - DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - QUEUE_COMPONENT_NAME, - REDIS_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, - RESULTS_CACHE_COMPONENT_NAME, - WEBUI_COMPONENT_NAME, -) - -from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, - get_clp_home, - is_container_exited, - is_container_running, - load_config_file, - validate_and_load_db_credentials_file, - validate_and_load_queue_credentials_file, -) logger = logging.getLogger(__file__) -def stop_running_container(container_name: str, already_exited_containers: List[str], force: bool): - if is_container_running(container_name): - logger.info(f"Stopping {container_name}...") - cmd = ["docker", "stop", container_name] - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Removing {container_name}...") - cmd = ["docker", "rm", container_name] - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - - logger.info(f"Stopped and removed {container_name}.") - elif is_container_exited(container_name): - if force: - logger.info(f"Forcibly removing exited {container_name}...") - cmd = ["docker", "rm", container_name] - subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) - logger.info(f"Removed {container_name}...") - else: - already_exited_containers.append(container_name) - - -def main(argv): - clp_home = get_clp_home() - default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH - - args_parser = argparse.ArgumentParser(description="Stops CLP") - args_parser.add_argument( - "--config", - "-c", - default=str(default_config_file_path), - help="CLP package configuration file.", - ) - args_parser.add_argument( - "--force", - "-f", - action="store_true", - help="Forcibly remove exited containers", - ) - - component_args_parser = args_parser.add_subparsers(dest="target") - component_args_parser.add_parser(CONTROLLER_TARGET_NAME) - component_args_parser.add_parser(DB_COMPONENT_NAME) - component_args_parser.add_parser(QUEUE_COMPONENT_NAME) - component_args_parser.add_parser(REDIS_COMPONENT_NAME) - component_args_parser.add_parser(REDUCER_COMPONENT_NAME) - component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME) - component_args_parser.add_parser(COMPRESSION_SCHEDULER_COMPONENT_NAME) - component_args_parser.add_parser(QUERY_SCHEDULER_COMPONENT_NAME) - component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME) - component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME) - component_args_parser.add_parser(WEBUI_COMPONENT_NAME) - component_args_parser.add_parser(GARBAGE_COLLECTOR_COMPONENT_NAME) - - parsed_args = args_parser.parse_args(argv[1:]) - - if parsed_args.target: - target = parsed_args.target - else: - target = ALL_TARGET_NAME - - # Validate and load config file - try: - config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) - - # Validate and load necessary credentials - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - DB_COMPONENT_NAME, - ): - validate_and_load_db_credentials_file(clp_config, clp_home, False) - if target in ( - ALL_TARGET_NAME, - CONTROLLER_TARGET_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - QUEUE_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - ): - validate_and_load_queue_credentials_file(clp_config, clp_home, False) - except: - logger.exception("Failed to load config.") - return -1 - +def main(): logger.info("Stopping all CLP containers using Docker Compose...") try: subprocess.run( @@ -136,4 +22,4 @@ def main(argv): if "__main__" == __name__: - sys.exit(main(sys.argv)) + sys.exit(main()) From 5f2e5cdbc68599eabdb2416c6f5c405b675ce03c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:22:24 -0400 Subject: [PATCH 017/408] Refactor dependency checks to include docker-compose status validation --- .../clp_package_utils/general.py | 65 +++++-------------- .../clp_package_utils/scripts/start_clp.py | 19 +----- .../clp_package_utils/scripts/stop_clp.py | 8 +++ 3 files changed, 25 insertions(+), 67 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index b27c62606c..963b058157 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -135,65 +135,32 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def check_dependencies(): +def is_compose_running(): + cmd = ["docker", "compose", "ps", "--quiet"] + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + return bool(output.strip()) + except subprocess.CalledProcessError: + raise EnvironmentError("docker-compose is not installed or not functioning properly.") + + +def check_dependencies(should_compose_run: bool = False): try: subprocess.run( "command -v docker", shell=True, - stdout=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, check=True, ) except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - try: - subprocess.run( - ["docker", "ps"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True - ) - except subprocess.CalledProcessError: - raise EnvironmentError("docker cannot run without superuser privileges (sudo).") - try: - subprocess.run( - ["docker", "compose", "version"], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=True, - ) - except subprocess.CalledProcessError: - raise EnvironmentError("docker-compose is not installed") - -def is_container_running(container_name): - # fmt: off - cmd = [ - "docker", "ps", - # Only return container IDs - "--quiet", - "--filter", f"name={container_name}" - ] - # fmt: on - proc = subprocess.run(cmd, stdout=subprocess.PIPE) - if proc.stdout.decode("utf-8"): - return True - - return False - - -def is_container_exited(container_name): - # fmt: off - cmd = [ - "docker", "ps", - # Only return container IDs - "--quiet", - "--filter", f"name={container_name}", - "--filter", "status=exited" - ] - # fmt: on - proc = subprocess.run(cmd, stdout=subprocess.PIPE) - if proc.stdout.decode("utf-8"): - return True - - return False + is_running = is_compose_running() + if should_compose_run and not is_running: + raise EnvironmentError("docker-compose is not running.") + if not should_compose_run and is_running: + raise EnvironmentError("docker-compose is already running.") def validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 2d0850eea8..ae1ac14d48 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -44,8 +44,6 @@ dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, - is_container_exited, - is_container_running, load_config_file, validate_and_load_db_credentials_file, validate_and_load_queue_credentials_file, @@ -68,16 +66,6 @@ def get_ip_from_hostname(hostname: str) -> str: return socket.gethostbyname(hostname) -def container_exists(container_name): - if is_container_running(container_name): - logger.info(f"{container_name} already running.") - return True - elif is_container_exited(container_name): - logger.info(f"{container_name} exited but not removed.") - return True - return False - - def append_docker_options( cmd: List[str], mounts: Optional[List[Optional[DockerMount]]] = None, @@ -490,7 +478,7 @@ def main(argv): parsed_args = args_parser.parse_args(argv[1:]) try: - check_dependencies() + check_dependencies(should_compose_run=False) except: logger.exception("Dependency checking failed.") return -1 @@ -507,11 +495,6 @@ def main(argv): validate_output_storage_config(clp_config) validate_retention_config(clp_config) - # validate_and_load_db_credentials_file(clp_config, clp_home, True) - # validate_and_load_queue_credentials_file(clp_config, clp_home, True) - # validate_and_load_redis_credentials_file(clp_config, clp_home, True) - # validate_worker_config(clp_config) - clp_config.validate_data_dir() clp_config.validate_logs_dir() clp_config.validate_aws_config_dir() diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 03981bd474..39912ce87c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -2,10 +2,18 @@ import subprocess import sys +from clp_package_utils.general import check_dependencies + logger = logging.getLogger(__file__) def main(): + try: + check_dependencies(should_compose_run=True) + except: + logger.exception("Dependency checking failed.") + return -1 + logger.info("Stopping all CLP containers using Docker Compose...") try: subprocess.run( From edfa9c91a7e891aaa351e1e70152121bde5d514c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:25:27 -0400 Subject: [PATCH 018/408] Refactor log directory handling to use constant path definitions --- components/clp-package-utils/clp_package_utils/general.py | 5 ++--- components/clp-py-utils/clp_py_utils/clp_config.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 963b058157..b2a3b67c70 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -15,6 +15,7 @@ from clp_py_utils.clp_config import ( CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLP_DEFAULT_DATA_DIRECTORY_PATH, + CLP_DEFAULT_LOG_DIRECTORY_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, DB_COMPONENT_NAME, @@ -297,11 +298,9 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig """ container_clp_config = clp_config.copy(deep=True) - # FIXME: consider removing credentials_file_path - # Set container paths container_clp_config.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH - container_clp_config.logs_directory = pathlib.Path("/") / "var" / "log" + container_clp_config.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH if StorageType.FS == clp_config.logs_input.type: container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 3d9932a2c5..90fb524f28 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -80,11 +80,11 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" +CLP_DEFAULT_LOG_DIRECTORY_PATH = pathlib.Path("var") / "log" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" CLP_SHARED_CONFIG_FILENAME = ".clp-config.yml" - # Environment variable names CLP_DB_USER_ENV_VAR_NAME = "CLP_DB_USER" CLP_DB_PASS_ENV_VAR_NAME = "CLP_DB_PASS" @@ -805,8 +805,8 @@ class CLPConfig(BaseModel): archive_output: ArchiveOutput = ArchiveOutput() stream_output: StreamOutput = StreamOutput() - data_directory: pathlib.Path = pathlib.Path("var") / "data" - logs_directory: pathlib.Path = pathlib.Path("var") / "log" + data_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH + logs_directory: pathlib.Path = CLP_DEFAULT_LOG_DIRECTORY_PATH aws_config_directory: Optional[pathlib.Path] = None _os_release_file_path: pathlib.Path = PrivateAttr(default=OS_RELEASE_FILE_PATH) From 7b3965e3333f79bf00906a9a30806a74295f18da Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:29:05 -0400 Subject: [PATCH 019/408] Add constants for archive and stream directory paths --- .../clp_package_utils/general.py | 16 ++++++++-------- .../clp-py-utils/clp_py_utils/clp_config.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index b2a3b67c70..505379ce26 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -13,9 +13,13 @@ import yaml from clp_py_utils.clp_config import ( + CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH, + CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH, CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLP_DEFAULT_DATA_DIRECTORY_PATH, CLP_DEFAULT_LOG_DIRECTORY_PATH, + CLP_DEFAULT_STREAM_DIRECTORY_PATH, + CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, DB_COMPONENT_NAME, @@ -305,21 +309,17 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR if StorageType.FS == clp_config.archive_output.storage.type: - container_clp_config.archive_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" - ) + container_clp_config.archive_output.storage.directory = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH elif StorageType.S3 == clp_config.archive_output.storage.type: container_clp_config.archive_output.storage.staging_directory = ( - CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" + CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH ) if StorageType.FS == clp_config.stream_output.storage.type: - container_clp_config.stream_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" - ) + container_clp_config.stream_output.storage.directory = CLP_DEFAULT_STREAM_DIRECTORY_PATH elif StorageType.S3 == clp_config.stream_output.storage.type: container_clp_config.stream_output.storage.staging_directory = ( - CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" + CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH ) if clp_config.aws_config_directory is not None: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 90fb524f28..49aa08d4b8 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -80,6 +80,10 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" +CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" +CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" +CLP_DEFAULT_STREAM_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" +CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" CLP_DEFAULT_LOG_DIRECTORY_PATH = pathlib.Path("var") / "log" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" @@ -609,19 +613,19 @@ class FsIngestionConfig(FsStorage): class ArchiveFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" + directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH class StreamFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" + directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" + staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" + staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( From cd84be806dccc4bb5efdc0363b2eeb848fd7a32b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:33:16 -0400 Subject: [PATCH 020/408] remove unused component groups and functions --- .../clp-py-utils/clp_py_utils/clp_config.py | 56 +------------------ 1 file changed, 1 insertion(+), 55 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 49aa08d4b8..2a0b53cbed 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Literal, Optional, Set, Union +from typing import Literal, Optional, Union from dotenv import dotenv_values from pydantic import BaseModel, PrivateAttr, root_validator, validator @@ -29,45 +29,6 @@ WEBUI_COMPONENT_NAME = "webui" GARBAGE_COLLECTOR_COMPONENT_NAME = "garbage_collector" -# Component groups -GENERAL_SCHEDULING_COMPONENTS = { - QUEUE_COMPONENT_NAME, - REDIS_COMPONENT_NAME, -} -COMPRESSION_COMPONENTS = GENERAL_SCHEDULING_COMPONENTS | { - DB_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, -} -QUERY_COMPONENTS = GENERAL_SCHEDULING_COMPONENTS | { - DB_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, -} -UI_COMPONENTS = { - RESULTS_CACHE_COMPONENT_NAME, - WEBUI_COMPONENT_NAME, -} -STORAGE_MANAGEMENT_COMPONENTS = {GARBAGE_COLLECTOR_COMPONENT_NAME} -ALL_COMPONENTS = ( - COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS | STORAGE_MANAGEMENT_COMPONENTS -) - -# Target names -ALL_TARGET_NAME = "" -CONTROLLER_TARGET_NAME = "controller" - -TARGET_TO_COMPONENTS = { - ALL_TARGET_NAME: ALL_COMPONENTS, - CONTROLLER_TARGET_NAME: GENERAL_SCHEDULING_COMPONENTS - | { - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - } - | STORAGE_MANAGEMENT_COMPONENTS, -} - # Action names ARCHIVE_MANAGER_ACTION_NAME = "archive_manager" @@ -929,12 +890,6 @@ def load_execution_container_name(self): def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME - def get_runnable_components(self) -> Set[str]: - if QueryEngine.PRESTO == self.package.query_engine: - return COMPRESSION_COMPONENTS | UI_COMPONENTS - else: - return ALL_COMPONENTS - def dump_to_primitive_dict(self): custom_serialized_fields = { "database", @@ -978,12 +933,3 @@ def dump_to_primitive_dict(self): d["stream_output"] = self.stream_output.dump_to_primitive_dict() return d - - -def get_components_for_target(target: str) -> Set[str]: - if target in TARGET_TO_COMPONENTS: - return TARGET_TO_COMPONENTS[target] - elif target in ALL_COMPONENTS: - return {target} - else: - return set() From aa12bdbc7e929f2a5f61a9ef34f851172ab146be Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 11:36:20 -0400 Subject: [PATCH 021/408] Remove unused CONTROLLER_TARGET_NAME constant from start_clp.py --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index ae1ac14d48..52d1c4318a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -16,7 +16,6 @@ COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, - CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, GARBAGE_COLLECTOR_COMPONENT_NAME, QUERY_JOBS_TABLE_NAME, From 5365722a30b95485ec386039e754e7f0020c71e8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 12:19:09 -0400 Subject: [PATCH 022/408] fix staging dirs --- components/clp-package-utils/clp_package_utils/general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 505379ce26..aecc7aa89d 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -312,14 +312,14 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig container_clp_config.archive_output.storage.directory = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH elif StorageType.S3 == clp_config.archive_output.storage.type: container_clp_config.archive_output.storage.staging_directory = ( - CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH ) if StorageType.FS == clp_config.stream_output.storage.type: container_clp_config.stream_output.storage.directory = CLP_DEFAULT_STREAM_DIRECTORY_PATH elif StorageType.S3 == clp_config.stream_output.storage.type: container_clp_config.stream_output.storage.staging_directory = ( - CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH ) if clp_config.aws_config_directory is not None: From 21ef70355861132061feccfa172983796118509c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 12:39:36 -0400 Subject: [PATCH 023/408] fix: update command to check if Docker Compose is running --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index aecc7aa89d..2f43dbea9b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -141,7 +141,7 @@ def generate_container_name(job_type: str) -> str: def is_compose_running(): - cmd = ["docker", "compose", "ps", "--quiet"] + cmd = ["docker", "compose", "ls", "--quiet"] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) return bool(output.strip()) From d2cdfbc02a30c6d104866618035ea42e13d6269b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 12:39:55 -0400 Subject: [PATCH 024/408] add AWS env credentials support --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 4 ++++ tools/deployment/package/docker-compose.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 52d1c4318a..19c235af87 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -523,6 +523,10 @@ def main(argv): env_dict["CLP_HOST_ARCHIVE_OUTPUT_DIR"] = str(clp_config.archive_output.get_directory()) env_dict["CLP_HOST_STREAM_OUTPUT_DIR"] = str(clp_config.stream_output.get_directory()) + # TODO: validate if those are required + env_dict["CLP_AWS_ACCESS_KEY_ID"] = os.getenv("AWS_ACCESS_KEY_ID", "") + env_dict["CLP_AWS_SECRET_ACCESS_KEY"] = os.getenv("AWS_SECRET_ACCESS_KEY", "") + if clp_config.aws_config_directory is not None: env_dict["CLP_HOST_AWS_CONFIG_DIR"] = str(clp_config.aws_config_directory) if StorageType.S3 == clp_config.archive_output.storage.type: diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 8648d6492a..536109daec 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -226,6 +226,8 @@ services: container_name: "compression_worker" image: "${CLP_PACKAGE_CONTAINER}" environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" @@ -261,6 +263,8 @@ services: container_name: "query_worker" image: "${CLP_PACKAGE_CONTAINER}" environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" From f0db07fe00c5fbe623f39b5d4f4a53f81489f8dc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 26 Aug 2025 15:21:16 -0400 Subject: [PATCH 025/408] Update container image name in start_clp.py --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 19c235af87..624b14f9ab 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -512,7 +512,7 @@ def main(argv): dump_shared_container_config(container_clp_config, clp_config) - env_dict["CLP_PACKAGE_CONTAINER"] = "clp-package-x86-ubuntu-jammy:dev" + env_dict["CLP_PACKAGE_CONTAINER"] = "clp-package:dev" env_dict["CLP_USER_ID"] = os.getuid() env_dict["CLP_GROUP_ID"] = os.getgid() From 5f24ce70b3738b5efabb0e5c2fe11a55aafec2d4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 18:34:20 -0400 Subject: [PATCH 026/408] add support for configurable CLP WebUI rate limiting --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 1 + tools/deployment/package/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 624b14f9ab..7b2e14ccae 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -439,6 +439,7 @@ def start_webui( env_dict["CLP_WEBUI_HOST"] = get_ip_from_hostname(clp_config.webui.host) env_dict["CLP_WEBUI_PORT"] = clp_config.webui.port + env_dict["CLP_WEBUI_RATE_LIMIT"] = clp_config.webui.rate_limit def start_garbage_collector( diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 536109daec..eee49564a1 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -330,6 +330,7 @@ services: NODE_ENV: "production" NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" PORT: "4000" + RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: From 3df20fce0d50f53f754938d37b471a8b5f82e38a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 18:36:08 -0400 Subject: [PATCH 027/408] update WebUI server path in start_clp.py and docker-compose configuration --- .../clp_package_utils/scripts/start_clp.py | 2 +- tools/deployment/package/docker-compose.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 7b2e14ccae..05991bfd76 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -364,7 +364,7 @@ def start_webui( get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json" + get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index eee49564a1..d3ea05493f 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -337,8 +337,8 @@ services: - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/server/settings.json\ -:/opt/clp/var/www/webui/server/dist/server/settings.json:ro" + - "./var/www/webui/server/dist/settings.json\ +:/opt/clp/var/www/webui/server/dist/settings.json:ro" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -346,7 +346,7 @@ services: condition: "service_completed_successfully" command: [ "/opt/clp/bin/node-22", - "/opt/clp/var/www/webui/server/dist/server/src/main.js" + "/opt/clp/var/www/webui/server/dist/src/main.js" ] healthcheck: <<: *healthcheck_defaults From c6f81ade93518f4dae1a42f8a75d90c4ae472f7d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 19:21:28 -0400 Subject: [PATCH 028/408] copy docker-compose.yml in package task --- taskfile.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 874af69cf0..90a0e886d8 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -113,6 +113,7 @@ tasks: - "components/clp-py-utils/dist/*.whl" - "components/job-orchestration/dist/*.whl" - "components/package-template/src/**/*" + - "tools/deployment/package/docker-compose.yml" generates: ["{{.CHECKSUM_FILE}}"] deps: - "core" @@ -160,6 +161,10 @@ tasks: - |- cd "{{.OUTPUT_DIR}}/var/www/webui" PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm ci --omit=dev + - >- + rsync -a + "tools/deployment/package/docker-compose.yml" + "{{.OUTPUT_DIR}}" # This command must be last - task: "utils:checksum:compute" vars: From db9c20fef293fcac1b776785864c5b4864d4ad8a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 19:48:16 -0400 Subject: [PATCH 029/408] use absolute paths in archive and stream storage configurations --- components/clp-package-utils/clp_package_utils/general.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 2f43dbea9b..a31e00d11c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -309,14 +309,18 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR if StorageType.FS == clp_config.archive_output.storage.type: - container_clp_config.archive_output.storage.directory = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + container_clp_config.archive_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + ) elif StorageType.S3 == clp_config.archive_output.storage.type: container_clp_config.archive_output.storage.staging_directory = ( pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH ) if StorageType.FS == clp_config.stream_output.storage.type: - container_clp_config.stream_output.storage.directory = CLP_DEFAULT_STREAM_DIRECTORY_PATH + container_clp_config.stream_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + ) elif StorageType.S3 == clp_config.stream_output.storage.type: container_clp_config.stream_output.storage.staging_directory = ( pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH From ea03e17956cac6fbb178efb79ba6680ffc67bda8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 00:21:41 -0400 Subject: [PATCH 030/408] refactor: centralize environment variable management and enhance validation --- .../clp_package_utils/general.py | 21 +- .../clp_package_utils/scripts/start_clp.py | 276 +++++++----------- .../clp-py-utils/clp_py_utils/clp_config.py | 122 +++++++- tools/deployment/package/docker-compose.yml | 70 ++--- 4 files changed, 271 insertions(+), 218 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a31e00d11c..302f145e92 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -519,7 +519,13 @@ def validate_and_load_redis_credentials_file( clp_config.redis.load_credentials_from_file(clp_config.credentials_file_path) -def validate_db_config(clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path): +def validate_db_config( + clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path +): + if not base_config.exists(): + raise ValueError( + f"{DB_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + ) _validate_data_directory(data_dir, DB_COMPONENT_NAME) validate_log_directory(logs_dir, DB_COMPONENT_NAME) @@ -533,15 +539,14 @@ def validate_queue_config(clp_config: CLPConfig, logs_dir: pathlib.Path): def validate_redis_config( - clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path, base_config: pathlib.Path + clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path ): - _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) - validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) - if not base_config.exists(): raise ValueError( f"{REDIS_COMPONENT_NAME} base configuration at {str(base_config)} is missing." ) + _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) + validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) validate_port(f"{REDIS_COMPONENT_NAME}.port", clp_config.redis.host, clp_config.redis.port) @@ -558,8 +563,12 @@ def validate_reducer_config(clp_config: CLPConfig, logs_dir: pathlib.Path, num_w def validate_results_cache_config( - clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path + clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path ): + if not base_config.exists(): + raise ValueError( + f"{RESULTS_CACHE_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + ) _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME) validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 05991bfd76..8cf3b63bc7 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -2,9 +2,10 @@ import json import logging import multiprocessing -import os import pathlib +import shlex import socket +import stat import subprocess import sys import time @@ -57,12 +58,9 @@ validate_webui_config, ) -logger = logging.getLogger(__file__) -env_dict = {} - +LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH -def get_ip_from_hostname(hostname: str) -> str: - return socket.gethostbyname(hostname) +logger = logging.getLogger(__file__) def append_docker_options( @@ -114,7 +112,7 @@ def chown_recursively( subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) -def wait_for_container_cmd(container_name: str, cmd_to_run: [str], timeout: int): +def wait_for_container_cmd(container_name: str, cmd_to_run: List[str], timeout: int): container_exec_cmd = ["docker", "exec", container_name] cmd = container_exec_cmd + cmd_to_run @@ -129,7 +127,7 @@ def wait_for_container_cmd(container_name: str, cmd_to_run: [str], timeout: int) break time.sleep(1) - cmd_str = " ".join(cmd_to_run) + cmd_str = shlex.join(cmd_to_run) logger.error(f"Timeout while waiting for command {cmd_str} to run after {timeout} seconds") return False @@ -138,181 +136,140 @@ def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = DB_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - db_data_dir = clp_config.data_directory / component_name - db_logs_dir = clp_config.logs_directory / component_name - - validate_db_config(clp_config, db_data_dir, db_logs_dir) - - # Create directories - db_data_dir.mkdir(exist_ok=True, parents=True) - db_logs_dir.mkdir(exist_ok=True, parents=True) - - # Start container - env_dict["CLP_HOST_DB_CONF_DIR"] = str(conf_dir / "mysql" / "conf.d") - env_dict["CLP_HOST_DB_DATA_DIR"] = str(db_data_dir) - env_dict["CLP_HOST_DB_LOGS_DIR"] = str(db_logs_dir) - - # TODO: Consider a Docker Compose overwrite / extend approach for pick the right image. - if "mysql" == clp_config.database.type: - env_dict["CLP_DB_IMAGE"] = "mysql:8.0.23" - elif "mariadb" == clp_config.database.type: - env_dict["CLP_DB_IMAGE"] = "mariadb:10-jammy" + conf_file = conf_dir / "mysql" / "conf.d" / "logging.cnf" + data_dir = clp_config.data_directory / component_name + logs_dir = clp_config.logs_directory / component_name + validate_db_config(clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) - env_dict["CLP_DB_HOST"] = get_ip_from_hostname(clp_config.database.host) - env_dict["CLP_DB_PORT"] = str(clp_config.database.port) - env_dict["CLP_DB_NAME"] = clp_config.database.name - env_dict["CLP_DB_USER"] = clp_config.database.username - env_dict["CLP_DB_PASS"] = clp_config.database.password + return { + **clp_config.database.dump_to_env_vars_dict(), + "CLP_DB_CONF_FILE_HOST": str(conf_file), + "CLP_DB_DATA_DIR_HOST": str(data_dir), + "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + } def start_queue(clp_config: CLPConfig): component_name = QUEUE_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - queue_logs_dir = clp_config.logs_directory / component_name - validate_queue_config(clp_config, queue_logs_dir) - - env_dict["CLP_QUEUE_HOST"] = get_ip_from_hostname(clp_config.queue.host) - env_dict["CLP_QUEUE_PORT"] = str(clp_config.queue.port) - env_dict["CLP_QUEUE_USER"] = clp_config.queue.username - env_dict["CLP_QUEUE_PASS"] = clp_config.queue.password - - # Create directories - queue_logs_dir.mkdir(exist_ok=True, parents=True) + logs_dir = clp_config.logs_directory / component_name + validate_queue_config(clp_config, logs_dir) + logs_dir.mkdir(exist_ok=True, parents=True) - env_dict["CLP_HOST_QUEUE_LOGS_DIR"] = str(queue_logs_dir) + return { + **clp_config.queue.dump_to_env_vars_dict(), + "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + } def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): component_name = REDIS_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - redis_logs_dir = clp_config.logs_directory / component_name - redis_data_dir = clp_config.data_directory / component_name - - config_file_path = conf_dir / "redis" / "redis.conf" - validate_redis_config(clp_config, redis_data_dir, redis_logs_dir, config_file_path) - - env_dict["CLP_HOST_REDIS_CONF_PATH"] = str(config_file_path) - env_dict["CLP_HOST_REDIS_DATA_DIR"] = str(redis_data_dir) - env_dict["CLP_HOST_REDIS_LOGS_DIR"] = str(redis_logs_dir) - - redis_data_dir.mkdir(exist_ok=True, parents=True) - redis_logs_dir.mkdir(exist_ok=True, parents=True) + conf_file = conf_dir / "redis" / "redis.conf" + logs_dir = clp_config.logs_directory / component_name + data_dir = clp_config.data_directory / component_name + validate_redis_config(clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) - env_dict["CLP_REDIS_HOST"] = get_ip_from_hostname(clp_config.redis.host) - env_dict["CLP_REDIS_PORT"] = str(clp_config.redis.port) - env_dict["CLP_REDIS_PASS"] = clp_config.redis.password - env_dict["CLP_REDIS_QUERY_BACKEND_DB"] = str(clp_config.redis.query_backend_database) - env_dict["CLP_REDIS_COMPRESSION_BACKEND_DB"] = str( - clp_config.redis.compression_backend_database - ) + return { + **clp_config.redis.dump_to_env_vars_dict(), + "CLP_REDIS_CONF_FILE_HOST": str(conf_file), + "CLP_REDIS_DATA_DIR_HOST": str(data_dir), + "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), + } -def start_results_cache(clp_config: CLPConfig, conf_dir: pathlib.Path): +def start_results_cache(clp_config: CLPConfig, conf_file: pathlib.Path): component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Initializing {component_name}...") + conf_file = conf_file / "mongo" / "mongod.conf" data_dir = clp_config.data_directory / component_name logs_dir = clp_config.logs_directory / component_name - - validate_results_cache_config(clp_config, data_dir, logs_dir) - + validate_results_cache_config(clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) - env_dict["CLP_HOST_RESULTS_CACHE_CONF_DIR"] = str(conf_dir / "mongo") - env_dict["CLP_HOST_RESULTS_CACHE_DATA_DIR"] = str(data_dir) - env_dict["CLP_HOST_RESULTS_CACHE_LOGS_DIR"] = str(logs_dir) - - env_dict["CLP_RESULTS_CACHE_HOST"] = get_ip_from_hostname(clp_config.results_cache.host) - env_dict["CLP_RESULTS_CACHE_PORT"] = str(clp_config.results_cache.port) - env_dict["CLP_RESULTS_CACHE_DB_NAME"] = clp_config.results_cache.db_name - env_dict["CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME"] = ( - clp_config.results_cache.stream_collection_name - ) + return { + **clp_config.results_cache.dump_to_env_vars_dict(), + "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), + "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), + "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), + } -def start_compression_scheduler( - clp_config: CLPConfig, -): +def start_compression_scheduler(clp_config: CLPConfig): component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + logs_file = clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) - env_dict["CLP_HOST_COMPRESSION_SCHEDULER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL"] = ( - clp_config.compression_scheduler.logging_level - ) + return { + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": clp_config.compression_scheduler.logging_level, + "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + } -def start_query_scheduler( - clp_config: CLPConfig, -): +def start_query_scheduler(clp_config: CLPConfig): component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + logs_file = clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) - env_dict["CLP_HOST_QUERY_SCHEDULER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_QUERY_SCHEDULER_LOGGING_LEVEL"] = clp_config.query_scheduler.logging_level + return { + "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": clp_config.query_scheduler.logging_level, + "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + } -def start_compression_worker( - clp_config: CLPConfig, - num_cpus: int, -): +def start_compression_worker(clp_config: CLPConfig, num_cpus: int): component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_COMPRESSION_WORKER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_COMPRESSION_WORKER_LOGGING_LEVEL"] = clp_config.compression_worker.logging_level - env_dict["CLP_COMPRESSION_WORKER_CONCURRENCY"] = str(num_cpus) - - # Create necessary directories - clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + return { + "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_cpus), + "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": clp_config.compression_worker.logging_level, + "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), + } -def start_query_worker( - clp_config: CLPConfig, - num_cpus: int, -): +def start_query_worker(clp_config: CLPConfig, num_cpus: int): component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_QUERY_WORKER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_QUERY_WORKER_LOGGING_LEVEL"] = clp_config.query_worker.logging_level - env_dict["CLP_QUERY_WORKER_CONCURRENCY"] = str(num_cpus) - - # Create necessary directories - clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + return { + "CLP_QUERY_WORKER_LOGGING_LEVEL": clp_config.query_worker.logging_level, + "CLP_QUERY_WORKER_LOGS_DIR": str(logs_dir), + "CLP_QUERY_WORKER_CONCURRENCY": str(num_cpus), + } -def start_reducer( - clp_config: CLPConfig, - num_workers: int, -): +def start_reducer(clp_config: CLPConfig, num_workers: int): component_name = REDUCER_COMPONENT_NAME logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_REDUCER_LOGS_DIR"] = str(logs_dir) - env_dict["CLP_REDUCER_LOGGING_LEVEL"] = clp_config.reducer.logging_level - env_dict["CLP_REDUCER_CONCURRENCY"] = str(num_workers) - env_dict["CLP_REDUCER_UPSERT_INTERVAL"] = str(clp_config.reducer.upsert_interval) + return { + "CLP_REDUCER_LOGGING_LEVEL": clp_config.reducer.logging_level, + "CLP_REDUCER_LOGS_DIR": str(logs_dir), + "CLP_REDUCER_CONCURRENCY": str(num_workers), + "CLP_REDUCER_UPSERT_INTERVAL": str(clp_config.reducer.upsert_interval), + } def update_settings_object( @@ -352,10 +309,7 @@ def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dic return settings_object -def start_webui( - clp_config: CLPConfig, - container_clp_config: CLPConfig, -): +def start_webui(clp_config: CLPConfig, container_clp_config: CLPConfig): component_name = WEBUI_COMPONENT_NAME logger.info(f"Initializing {component_name}...") @@ -437,21 +391,17 @@ def start_webui( with open(server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) - env_dict["CLP_WEBUI_HOST"] = get_ip_from_hostname(clp_config.webui.host) - env_dict["CLP_WEBUI_PORT"] = clp_config.webui.port - env_dict["CLP_WEBUI_RATE_LIMIT"] = clp_config.webui.rate_limit + return clp_config.webui.dump_to_env_vars_dict() -def start_garbage_collector( - clp_config: CLPConfig, -): +def start_garbage_collector(clp_config: CLPConfig): component_name = GARBAGE_COLLECTOR_COMPONENT_NAME logger.info(f"Initializing {component_name}...") logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - env_dict["CLP_GC_LOGGING_LEVEL"] = clp_config.garbage_collector.logging_level + return clp_config.garbage_collector.dump_to_env_vars_dict() def add_num_workers_argument(parser): @@ -510,53 +460,31 @@ def main(argv): # Create necessary directories clp_config.data_directory.mkdir(parents=True, exist_ok=True) clp_config.logs_directory.mkdir(parents=True, exist_ok=True) + clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) + clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) dump_shared_container_config(container_clp_config, clp_config) - env_dict["CLP_PACKAGE_CONTAINER"] = "clp-package:dev" - - env_dict["CLP_USER_ID"] = os.getuid() - env_dict["CLP_GROUP_ID"] = os.getgid() - env_dict["CLP_STORAGE_ENGINE"] = clp_config.package.storage_engine - - env_dict["CLP_HOST_DATA_DIR"] = str(clp_config.data_directory) - env_dict["CLP_HOST_LOGS_DIR"] = str(clp_config.logs_directory) - env_dict["CLP_HOST_ARCHIVE_OUTPUT_DIR"] = str(clp_config.archive_output.get_directory()) - env_dict["CLP_HOST_STREAM_OUTPUT_DIR"] = str(clp_config.stream_output.get_directory()) - - # TODO: validate if those are required - env_dict["CLP_AWS_ACCESS_KEY_ID"] = os.getenv("AWS_ACCESS_KEY_ID", "") - env_dict["CLP_AWS_SECRET_ACCESS_KEY"] = os.getenv("AWS_SECRET_ACCESS_KEY", "") - - if clp_config.aws_config_directory is not None: - env_dict["CLP_HOST_AWS_CONFIG_DIR"] = str(clp_config.aws_config_directory) - if StorageType.S3 == clp_config.archive_output.storage.type: - clp_config.archive_output.storage.staging_directory.mkdir(parents=True, exist_ok=True) - env_dict["CLP_HOST_ARCHIVE_STAGING_DIR"] = str( - clp_config.archive_output.storage.staging_directory - ) - if StorageType.S3 == clp_config.stream_output.storage.type: - clp_config.stream_output.storage.staging_directory.mkdir(parents=True, exist_ok=True) - env_dict["CLP_HOST_STREAM_STAGING_DIR"] = str( - clp_config.stream_output.storage.staging_directory - ) - try: conf_dir = clp_home / "etc" - # Start components - start_db(clp_config, conf_dir) - start_queue(clp_config) - start_redis(clp_config, conf_dir) - start_results_cache(clp_config, conf_dir) - start_compression_scheduler(clp_config) - start_query_scheduler(clp_config) - start_compression_worker(clp_config, num_workers) - start_query_worker(clp_config, num_workers) - start_reducer(clp_config, num_workers) - start_webui(clp_config, container_clp_config) - start_garbage_collector(clp_config) - + env_dict = { + **clp_config.dump_to_env_vars_dict(), + **start_db(clp_config, conf_dir), + **start_queue(clp_config), + **start_redis(clp_config, conf_dir), + **start_results_cache(clp_config, conf_dir), + **start_compression_scheduler(clp_config), + **start_query_scheduler(clp_config), + **start_compression_worker(clp_config, num_workers), + **start_query_worker(clp_config, num_workers), + **start_reducer(clp_config, num_workers), + **start_webui(clp_config, container_clp_config), + **start_garbage_collector(clp_config), + **start_reducer(clp_config, num_workers), + **start_webui(clp_config, container_clp_config), + **start_garbage_collector(clp_config), + } with open(f"{clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index fc51ae3149..7c53b8f940 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,8 @@ import os import pathlib +import socket from enum import auto -from typing import Literal, Optional, Union +from typing import Dict, Literal, Optional, Union from dotenv import dotenv_values from pydantic import BaseModel, PrivateAttr, root_validator, validator @@ -129,6 +130,11 @@ def validate_query_engine_package_compatibility(cls, values): return values + def dump_to_env_vars(self): + return { + "CLP_PACKAGE_STORAGE_ENGINE": self.storage_engine, + } + class Database(BaseModel): type: str = "mariadb" @@ -214,6 +220,19 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_DB_HOST": get_ip_from_hostname(self.host), + "CLP_DB_PORT": str(self.port), + "CLP_DB_NAME": self.name, + "CLP_DB_USER": self.username, + "CLP_DB_PASS": self.password, + "CLP_DB_IMAGE": "mysql:8.0.23" if "mysql" == self.type else "mariadb:10-jammy", + } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -267,8 +286,8 @@ def validate_logging_level(cls, field): class QueryScheduler(BaseModel): - host = "localhost" - port = 7000 + host: str = "localhost" + port: int = 7000 jobs_poll_delay: float = 0.1 # seconds num_archives_to_search_per_sub_job: int = 16 logging_level: str = "INFO" @@ -326,6 +345,18 @@ def validate_host(cls, field): def dump_to_primitive_dict(self): return self.dict(exclude={"password"}) + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_REDIS_HOST": get_ip_from_hostname(self.host), + "CLP_REDIS_PORT": str(self.port), + "CLP_REDIS_PASS": self.password, + "CLP_REDIS_QUERY_BACKEND_DB": str(self.query_backend_database), + "CLP_REDIS_COMPRESSION_BACKEND_DB": str(self.compression_backend_database), + } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -407,6 +438,17 @@ def validate_retention_period(cls, field): raise ValueError("retention_period must be greater than 0") return field + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(self.host), + "CLP_RESULTS_CACHE_PORT": str(self.port), + "CLP_RESULTS_CACHE_DB_NAME": self.db_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.stream_collection_name, + } + def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" @@ -421,6 +463,17 @@ class Queue(BaseModel): def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_QUEUE_HOST": get_ip_from_hostname(self.host), + "CLP_QUEUE_PORT": str(self.port), + "CLP_QUEUE_USER": self.username, + "CLP_QUEUE_PASS": self.password, + } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -722,6 +775,16 @@ def validate_rate_limit(cls, field): raise ValueError(f"rate_limit must be greater than 0") return field + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return { + "CLP_WEBUI_HOST": get_ip_from_hostname(self.host), + "CLP_WEBUI_PORT": str(self.port), + "CLP_WEBUI_RATE_LIMIT": str(self.rate_limit), + } + class SweepInterval(BaseModel): archive: int = 60 @@ -748,6 +811,12 @@ def validate_logging_level(cls, field): _validate_logging_level(cls, field) return field + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + return {"CLP_GC_LOGGING_LEVEL": self.logging_level} + def _get_env_var(name: str) -> str: value = os.getenv(name) @@ -756,6 +825,16 @@ def _get_env_var(name: str) -> str: return value +def get_ip_from_hostname(hostname: str) -> str: + """ + Resolves a hostname to an IP address. + + :param hostname: The hostname to resolve. + :return: The resolved IP address. + """ + return socket.gethostbyname(hostname) + + class CLPConfig(BaseModel): execution_container: Optional[str] = None @@ -921,6 +1000,43 @@ def dump_to_primitive_dict(self): return d + def dump_to_env_vars_dict(self) -> Dict[str, str]: + """ + :return: Dictionary of environment variables. + """ + env_vars = { + **self.package.dump_to_env_vars(), + # User and group IDs + "CLP_USER_ID": str(os.getuid()), + "CLP_GROUP_ID": str(os.getgid()), + # Package container + "CLP_PACKAGE_CONTAINER": "clp-package:dev", + # Global paths + "CLP_DATA_DIR_HOST": str(self.data_directory), + "CLP_LOGS_DIR_HOST": str(self.logs_directory), + "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.archive_output.get_directory()), + "CLP_STREAM_OUTPUT_DIR_HOST": str(self.stream_output.get_directory()), + # AWS credentials + "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), + "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + } + + # AWS config directory + if self.aws_config_directory is not None: + env_vars["CLP_AWS_CONFIG_DIR_HOST"] = str(self.aws_config_directory) + + # Staging directories for S3 storage + if StorageType.S3 == self.archive_output.storage.type: + env_vars["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( + self.archive_output.storage.staging_directory + ) + if StorageType.S3 == self.stream_output.storage.type: + env_vars["CLP_STREAM_STAGING_DIR_HOST"] = str( + self.stream_output.storage.staging_directory + ) + + return env_vars + class WorkerConfig(BaseModel): package: Package = Package() diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index d3ea05493f..b76224a969 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -33,9 +33,9 @@ services: ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "${CLP_HOST_DATA_DIR:-./var/data}/database:/var/lib/mysql" - - "${CLP_HOST_LOGS_DIR:-./var/log}/database:/var/log/mysql" - - "./etc/mysql/conf.d/logging.cnf:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" + - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" healthcheck: <<: *healthcheck_defaults test: [ @@ -56,7 +56,7 @@ services: CLP_DB_USER: "${CLP_DB_USER}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" depends_on: db: condition: "service_healthy" @@ -65,7 +65,7 @@ services: "-u", "-m", "clp_py_utils.create-db-tables", "--config", "/etc/clp-config.yml", - "--storage-engine", "${CLP_STORAGE_ENGINE}" + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" ] queue: @@ -79,7 +79,7 @@ services: ports: - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/queue:/var/log/rabbitmq" + - "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}:/var/log/rabbitmq" healthcheck: <<: *healthcheck_defaults test: [ @@ -94,9 +94,9 @@ services: ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" volumes: - - "${CLP_HOST_DATA_DIR:-./var/data}/redis:/data" - - "${CLP_HOST_LOGS_DIR:-./var/log}/redis:/var/log/redis" - - "./etc/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro" + - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf:ro" + - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" + - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" healthcheck: <<: *healthcheck_defaults test: [ @@ -120,9 +120,9 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "${CLP_HOST_DATA_DIR:-./var/data}/results_cache:/data/db" - - "${CLP_HOST_LOGS_DIR:-./var/log}/results_cache:/var/log/mongodb" - - "./etc/mongo:/etc/mongo:ro" + - "${CLP_RESULTS_CACHE_CONF_DIR_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" + - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" healthcheck: <<: *healthcheck_defaults test: >- @@ -164,9 +164,9 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}:/var/log/compression_scheduler.log" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" depends_on: db-table-creator: @@ -196,8 +196,8 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}:/var/log" - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_QUERY_SCHEDULER_LOGS_FILE_HOST:-./var/log/query_scheduler.log}:/var/log/query_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -238,12 +238,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_HOST_ARCHIVE_STAGING_DIR:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_DATA_DIR:-./var/data}:/var/data" - - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/compression_worker:/var/log/compression_worker" + - "${CLP_ARCHIVE_STAGING_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:/var/log/compression_worker" + - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" + - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" command: [ "python3", @@ -275,12 +275,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_DATA_DIR:-./var/data}/archives:/var/data/archives" - - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/query_worker:/var/log/query_worker" - - "${CLP_HOST_STREAM_STAGING_DIR:-./var/data/staged-streams}:/var/data/staged-streams" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" + - "${CLP_DATA_DIR_HOST:-./var/data}/streams:/var/data/streams" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_QUERY_WORKER_LOGS_DIR:-./var/log/query_worker}:/var/log/query_worker" + - "${CLP_STREAM_STAGING_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" command: [ "python3", "-u", @@ -304,8 +304,8 @@ services: CLP_LOGS_DIR: "/var/log/reducer" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/reducer:/var/log/reducer" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_REDUCER_LOGS_DIR:-./var/log/reducer}:/var/log/reducer" depends_on: query-scheduler: condition: "service_healthy" @@ -334,8 +334,8 @@ services: ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: - - "${CLP_HOST_AWS_CONFIG_DIR:-~/.aws}:/.aws:ro" - - "${CLP_HOST_DATA_DIR:-./var/data}/streams:/var/data/streams" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_DATA_DIR_HOST:-./var/data}/streams:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - "./var/www/webui/server/dist/settings.json\ :/opt/clp/var/www/webui/server/dist/settings.json:ro" @@ -369,8 +369,8 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_HOST_LOGS_DIR:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_HOST_LOGS_DIR:-./var/log}/garbage_collector:/var/log/garbage_collector" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: db-table-creator: condition: "service_completed_successfully" From 60994ee02d23600c17733bc0776203017017b2c1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 00:23:42 -0400 Subject: [PATCH 031/408] fix: use List[str] type hint for command parameter in start_clp.py --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 8cf3b63bc7..53e367dbe6 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -89,7 +89,7 @@ def append_docker_options( def append_docker_port_settings_for_host_ips( - hostname: str, host_port: int, container_port: int, cmd: [str] + hostname: str, host_port: int, container_port: int, cmd: List[str] ): # Note: We use a set because gethostbyname_ex can return the same IP twice for one hostname for ip in set(socket.gethostbyname_ex(hostname)[2]): From 7e25d7556226cb5dba9a466908445d591077c1ec Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 00:48:08 -0400 Subject: [PATCH 032/408] refactor: remove `dump_to_env_vars_dict` methods and centralize environment variable handling --- .../clp_package_utils/scripts/start_clp.py | 81 ++++++++++++-- .../clp-py-utils/clp_py_utils/clp_config.py | 104 +----------------- 2 files changed, 73 insertions(+), 112 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 53e367dbe6..0f8171a559 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -2,6 +2,7 @@ import json import logging import multiprocessing +import os import pathlib import shlex import socket @@ -63,6 +64,16 @@ logger = logging.getLogger(__file__) +def get_ip_from_hostname(hostname: str) -> str: + """ + Resolves a hostname to an IP address. + + :param hostname: The hostname to resolve. + :return: The resolved IP address. + """ + return socket.gethostbyname(hostname) + + def append_docker_options( cmd: List[str], mounts: Optional[List[Optional[DockerMount]]] = None, @@ -144,10 +155,16 @@ def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): logs_dir.mkdir(exist_ok=True, parents=True) return { - **clp_config.database.dump_to_env_vars_dict(), "CLP_DB_CONF_FILE_HOST": str(conf_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + + "CLP_DB_HOST": get_ip_from_hostname(clp_config.database.host), + "CLP_DB_PORT": str(clp_config.database.port), + "CLP_DB_NAME": clp_config.database.name, + "CLP_DB_USER": clp_config.database.username, + "CLP_DB_PASS": clp_config.database.password, + "CLP_DB_IMAGE": "mysql:8.0.23" if "mysql" == clp_config.database.type else "mariadb:10-jammy", } @@ -160,8 +177,12 @@ def start_queue(clp_config: CLPConfig): logs_dir.mkdir(exist_ok=True, parents=True) return { - **clp_config.queue.dump_to_env_vars_dict(), "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + + "CLP_QUEUE_HOST": get_ip_from_hostname(clp_config.queue.host), + "CLP_QUEUE_PORT": str(clp_config.queue.port), + "CLP_QUEUE_USER": clp_config.queue.username, + "CLP_QUEUE_PASS": clp_config.queue.password, } @@ -177,10 +198,15 @@ def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): logs_dir.mkdir(exist_ok=True, parents=True) return { - **clp_config.redis.dump_to_env_vars_dict(), "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), + + "CLP_REDIS_HOST": get_ip_from_hostname(clp_config.redis.host), + "CLP_REDIS_PORT": str(clp_config.redis.port), + "CLP_REDIS_PASS": clp_config.redis.password, + "CLP_REDIS_QUERY_BACKEND_DB": str(clp_config.redis.query_backend_database), + "CLP_REDIS_COMPRESSION_BACKEND_DB": str(clp_config.redis.compression_backend_database), } @@ -196,10 +222,14 @@ def start_results_cache(clp_config: CLPConfig, conf_file: pathlib.Path): logs_dir.mkdir(exist_ok=True, parents=True) return { - **clp_config.results_cache.dump_to_env_vars_dict(), "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), + + "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(clp_config.results_cache.host), + "CLP_RESULTS_CACHE_PORT": str(clp_config.results_cache.port), + "CLP_RESULTS_CACHE_DB_NAME": clp_config.results_cache.db_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": clp_config.results_cache.stream_collection_name, } @@ -391,7 +421,11 @@ def start_webui(clp_config: CLPConfig, container_clp_config: CLPConfig): with open(server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) - return clp_config.webui.dump_to_env_vars_dict() + return { + "CLP_WEBUI_HOST": get_ip_from_hostname(clp_config.webui.host), + "CLP_WEBUI_PORT": str(clp_config.webui.port), + "CLP_WEBUI_RATE_LIMIT": str(clp_config.webui.rate_limit), + } def start_garbage_collector(clp_config: CLPConfig): @@ -401,8 +435,9 @@ def start_garbage_collector(clp_config: CLPConfig): logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return clp_config.garbage_collector.dump_to_env_vars_dict() - + return { + "CLP_GC_LOGGING_LEVEL": clp_config.garbage_collector.logging_level + } def add_num_workers_argument(parser): parser.add_argument( @@ -469,7 +504,21 @@ def main(argv): conf_dir = clp_home / "etc" env_dict = { - **clp_config.dump_to_env_vars_dict(), + "CLP_PACKAGE_STORAGE_ENGINE": clp_config.package.storage_engine, + # User and group IDs + "CLP_USER_ID": str(os.getuid()), + "CLP_GROUP_ID": str(os.getgid()), + # Package container + "CLP_PACKAGE_CONTAINER": "clp-package:dev", + # Global paths + "CLP_DATA_DIR_HOST": str(clp_config.data_directory), + "CLP_LOGS_DIR_HOST": str(clp_config.logs_directory), + "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(clp_config.archive_output.get_directory()), + "CLP_STREAM_OUTPUT_DIR_HOST": str(clp_config.stream_output.get_directory()), + # AWS credentials + "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), + "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + **start_db(clp_config, conf_dir), **start_queue(clp_config), **start_redis(clp_config, conf_dir), @@ -481,10 +530,20 @@ def main(argv): **start_reducer(clp_config, num_workers), **start_webui(clp_config, container_clp_config), **start_garbage_collector(clp_config), - **start_reducer(clp_config, num_workers), - **start_webui(clp_config, container_clp_config), - **start_garbage_collector(clp_config), } + # AWS config directory + if clp_config.aws_config_directory is not None: + env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(clp_config.aws_config_directory) + + # Staging directories for S3 storage + if StorageType.S3 == clp_config.archive_output.storage.type: + env_dict["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( + clp_config.archive_output.storage.staging_directory + ) + if StorageType.S3 == clp_config.stream_output.storage.type: + env_dict["CLP_STREAM_STAGING_DIR_HOST"] = str( + clp_config.stream_output.storage.staging_directory + ) with open(f"{clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 7c53b8f940..7bec8926ad 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -130,11 +130,6 @@ def validate_query_engine_package_compatibility(cls, values): return values - def dump_to_env_vars(self): - return { - "CLP_PACKAGE_STORAGE_ENGINE": self.storage_engine, - } - class Database(BaseModel): type: str = "mariadb" @@ -220,18 +215,7 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_DB_HOST": get_ip_from_hostname(self.host), - "CLP_DB_PORT": str(self.port), - "CLP_DB_NAME": self.name, - "CLP_DB_USER": self.username, - "CLP_DB_PASS": self.password, - "CLP_DB_IMAGE": "mysql:8.0.23" if "mysql" == self.type else "mariadb:10-jammy", - } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) @@ -345,17 +329,7 @@ def validate_host(cls, field): def dump_to_primitive_dict(self): return self.dict(exclude={"password"}) - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_REDIS_HOST": get_ip_from_hostname(self.host), - "CLP_REDIS_PORT": str(self.port), - "CLP_REDIS_PASS": self.password, - "CLP_REDIS_QUERY_BACKEND_DB": str(self.query_backend_database), - "CLP_REDIS_COMPRESSION_BACKEND_DB": str(self.compression_backend_database), - } + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) @@ -438,16 +412,7 @@ def validate_retention_period(cls, field): raise ValueError("retention_period must be greater than 0") return field - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(self.host), - "CLP_RESULTS_CACHE_PORT": str(self.port), - "CLP_RESULTS_CACHE_DB_NAME": self.db_name, - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.stream_collection_name, - } + def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" @@ -463,16 +428,6 @@ class Queue(BaseModel): def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_QUEUE_HOST": get_ip_from_hostname(self.host), - "CLP_QUEUE_PORT": str(self.port), - "CLP_QUEUE_USER": self.username, - "CLP_QUEUE_PASS": self.password, - } def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) @@ -775,16 +730,6 @@ def validate_rate_limit(cls, field): raise ValueError(f"rate_limit must be greater than 0") return field - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return { - "CLP_WEBUI_HOST": get_ip_from_hostname(self.host), - "CLP_WEBUI_PORT": str(self.port), - "CLP_WEBUI_RATE_LIMIT": str(self.rate_limit), - } - class SweepInterval(BaseModel): archive: int = 60 @@ -811,12 +756,6 @@ def validate_logging_level(cls, field): _validate_logging_level(cls, field) return field - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - return {"CLP_GC_LOGGING_LEVEL": self.logging_level} - def _get_env_var(name: str) -> str: value = os.getenv(name) @@ -1000,43 +939,6 @@ def dump_to_primitive_dict(self): return d - def dump_to_env_vars_dict(self) -> Dict[str, str]: - """ - :return: Dictionary of environment variables. - """ - env_vars = { - **self.package.dump_to_env_vars(), - # User and group IDs - "CLP_USER_ID": str(os.getuid()), - "CLP_GROUP_ID": str(os.getgid()), - # Package container - "CLP_PACKAGE_CONTAINER": "clp-package:dev", - # Global paths - "CLP_DATA_DIR_HOST": str(self.data_directory), - "CLP_LOGS_DIR_HOST": str(self.logs_directory), - "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.archive_output.get_directory()), - "CLP_STREAM_OUTPUT_DIR_HOST": str(self.stream_output.get_directory()), - # AWS credentials - "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), - "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - } - - # AWS config directory - if self.aws_config_directory is not None: - env_vars["CLP_AWS_CONFIG_DIR_HOST"] = str(self.aws_config_directory) - - # Staging directories for S3 storage - if StorageType.S3 == self.archive_output.storage.type: - env_vars["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( - self.archive_output.storage.staging_directory - ) - if StorageType.S3 == self.stream_output.storage.type: - env_vars["CLP_STREAM_STAGING_DIR_HOST"] = str( - self.stream_output.storage.staging_directory - ) - - return env_vars - class WorkerConfig(BaseModel): package: Package = Package() From 3e24e4e2b63ed0d9b312385013bb4eeeb7103226 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 00:48:32 -0400 Subject: [PATCH 033/408] lint --- .../clp_package_utils/scripts/start_clp.py | 14 +++++--------- components/clp-py-utils/clp_py_utils/clp_config.py | 7 ------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 0f8171a559..9abfdf9a14 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -158,13 +158,14 @@ def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): "CLP_DB_CONF_FILE_HOST": str(conf_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), - "CLP_DB_HOST": get_ip_from_hostname(clp_config.database.host), "CLP_DB_PORT": str(clp_config.database.port), "CLP_DB_NAME": clp_config.database.name, "CLP_DB_USER": clp_config.database.username, "CLP_DB_PASS": clp_config.database.password, - "CLP_DB_IMAGE": "mysql:8.0.23" if "mysql" == clp_config.database.type else "mariadb:10-jammy", + "CLP_DB_IMAGE": ( + "mysql:8.0.23" if "mysql" == clp_config.database.type else "mariadb:10-jammy" + ), } @@ -178,7 +179,6 @@ def start_queue(clp_config: CLPConfig): return { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), - "CLP_QUEUE_HOST": get_ip_from_hostname(clp_config.queue.host), "CLP_QUEUE_PORT": str(clp_config.queue.port), "CLP_QUEUE_USER": clp_config.queue.username, @@ -201,7 +201,6 @@ def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": get_ip_from_hostname(clp_config.redis.host), "CLP_REDIS_PORT": str(clp_config.redis.port), "CLP_REDIS_PASS": clp_config.redis.password, @@ -225,7 +224,6 @@ def start_results_cache(clp_config: CLPConfig, conf_file: pathlib.Path): "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(clp_config.results_cache.port), "CLP_RESULTS_CACHE_DB_NAME": clp_config.results_cache.db_name, @@ -435,9 +433,8 @@ def start_garbage_collector(clp_config: CLPConfig): logs_dir = clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return { - "CLP_GC_LOGGING_LEVEL": clp_config.garbage_collector.logging_level - } + return {"CLP_GC_LOGGING_LEVEL": clp_config.garbage_collector.logging_level} + def add_num_workers_argument(parser): parser.add_argument( @@ -518,7 +515,6 @@ def main(argv): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **start_db(clp_config, conf_dir), **start_queue(clp_config), **start_redis(clp_config, conf_dir), diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 7bec8926ad..abef21babc 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -215,8 +215,6 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) - - def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -329,8 +327,6 @@ def validate_host(cls, field): def dump_to_primitive_dict(self): return self.dict(exclude={"password"}) - - def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: @@ -412,8 +408,6 @@ def validate_retention_period(cls, field): raise ValueError("retention_period must be greater than 0") return field - - def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" @@ -428,7 +422,6 @@ class Queue(BaseModel): def dump_to_primitive_dict(self): return self.dict(exclude={"username", "password"}) - def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: From cbb9ce1f537be96525a47087ec228acbbb5f3b7b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 01:47:30 -0400 Subject: [PATCH 034/408] refactor: modularize and simplify start_clp.py by introducing DockerComposeController --- .../clp_package_utils/controllers.py | 455 ++++++++++++++++ .../clp_package_utils/general.py | 6 +- .../clp_package_utils/scripts/start_clp.py | 500 +----------------- .../clp_package_utils/scripts/stop_clp.py | 4 +- 4 files changed, 463 insertions(+), 502 deletions(-) create mode 100644 components/clp-package-utils/clp_package_utils/controllers.py diff --git a/components/clp-package-utils/clp_package_utils/controllers.py b/components/clp-package-utils/clp_package_utils/controllers.py new file mode 100644 index 0000000000..880f1f7266 --- /dev/null +++ b/components/clp-package-utils/clp_package_utils/controllers.py @@ -0,0 +1,455 @@ +import json +import logging +import multiprocessing +import os +import pathlib +import socket +import stat +import subprocess +from abc import ABC, abstractmethod +from typing import Any, Dict + +from clp_py_utils.clp_config import ( + AwsAuthType, + CLPConfig, + COMPRESSION_SCHEDULER_COMPONENT_NAME, + COMPRESSION_WORKER_COMPONENT_NAME, + DB_COMPONENT_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, + QUERY_SCHEDULER_COMPONENT_NAME, + QUERY_WORKER_COMPONENT_NAME, + QUEUE_COMPONENT_NAME, + REDIS_COMPONENT_NAME, + REDUCER_COMPONENT_NAME, + RESULTS_CACHE_COMPONENT_NAME, + StorageEngine, + StorageType, + WEBUI_COMPONENT_NAME, +) +from clp_py_utils.clp_metadata_db_utils import ( + get_archives_table_name, + get_datasets_table_name, + get_files_table_name, +) + +from clp_package_utils.general import ( + check_docker_dependencies, + CONTAINER_CLP_HOME, + generate_docker_compose_container_config, + get_clp_home, + validate_db_config, + validate_queue_config, + validate_redis_config, + validate_results_cache_config, + validate_webui_config, +) + +LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + +logger = logging.getLogger(__name__) + + +def get_ip_from_hostname(hostname: str) -> str: + """ + Resolves a hostname to an IP address. + + :param hostname: The hostname to resolve. + :return: The resolved IP address. + """ + return socket.gethostbyname(hostname) + + +class BaseController(ABC): + def __init__(self, clp_config: CLPConfig): + self.clp_config = clp_config + self.clp_home = get_clp_home() + + def provision_database(self, conf_dir: pathlib.Path): + component_name = DB_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + conf_file = conf_dir / "mysql" / "conf.d" / "logging.cnf" + data_dir = self.clp_config.data_directory / component_name + logs_dir = self.clp_config.logs_directory / component_name + validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) + + return { + "CLP_DB_CONF_FILE_HOST": str(conf_file), + "CLP_DB_DATA_DIR_HOST": str(data_dir), + "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + "CLP_DB_HOST": get_ip_from_hostname(self.clp_config.database.host), + "CLP_DB_PORT": str(self.clp_config.database.port), + "CLP_DB_NAME": self.clp_config.database.name, + "CLP_DB_USER": self.clp_config.database.username, + "CLP_DB_PASS": self.clp_config.database.password, + "CLP_DB_IMAGE": ( + "mysql:8.0.23" if "mysql" == self.clp_config.database.type else "mariadb:10-jammy" + ), + } + + def provision_queue(self): + component_name = QUEUE_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + validate_queue_config(self.clp_config, logs_dir) + logs_dir.mkdir(exist_ok=True, parents=True) + + return { + "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + "CLP_QUEUE_HOST": get_ip_from_hostname(self.clp_config.queue.host), + "CLP_QUEUE_PORT": str(self.clp_config.queue.port), + "CLP_QUEUE_USER": self.clp_config.queue.username, + "CLP_QUEUE_PASS": self.clp_config.queue.password, + } + + def provision_redis(self, conf_dir: pathlib.Path): + component_name = REDIS_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + conf_file = conf_dir / "redis" / "redis.conf" + logs_dir = self.clp_config.logs_directory / component_name + data_dir = self.clp_config.data_directory / component_name + validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) + + return { + "CLP_REDIS_CONF_FILE_HOST": str(conf_file), + "CLP_REDIS_DATA_DIR_HOST": str(data_dir), + "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), + "CLP_REDIS_HOST": get_ip_from_hostname(self.clp_config.redis.host), + "CLP_REDIS_PORT": str(self.clp_config.redis.port), + "CLP_REDIS_PASS": self.clp_config.redis.password, + "CLP_REDIS_QUERY_BACKEND_DB": str(self.clp_config.redis.query_backend_database), + "CLP_REDIS_COMPRESSION_BACKEND_DB": str( + self.clp_config.redis.compression_backend_database + ), + } + + def provision_results_cache(self, conf_dir: pathlib.Path): + component_name = RESULTS_CACHE_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + conf_file = conf_dir / "mongo" / "mongod.conf" + data_dir = self.clp_config.data_directory / component_name + logs_dir = self.clp_config.logs_directory / component_name + validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) + logs_dir.mkdir(exist_ok=True, parents=True) + + return { + "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), + "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), + "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), + "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(self.clp_config.results_cache.host), + "CLP_RESULTS_CACHE_PORT": str(self.clp_config.results_cache.port), + "CLP_RESULTS_CACHE_DB_NAME": self.clp_config.results_cache.db_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, + } + + def provision_compression_scheduler(self): + component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_file = self.clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + + return { + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, + "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + } + + def provision_query_scheduler(self): + component_name = QUERY_SCHEDULER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_file = self.clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + + return { + "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, + "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + } + + def provision_compression_worker(self, num_workers: int): + component_name = COMPRESSION_WORKER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + return { + "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), + "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": self.clp_config.compression_worker.logging_level, + "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), + } + + def provision_query_worker(self, num_workers: int): + component_name = QUERY_WORKER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + return { + "CLP_QUERY_WORKER_LOGGING_LEVEL": self.clp_config.query_worker.logging_level, + "CLP_QUERY_WORKER_LOGS_DIR": str(logs_dir), + "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), + } + + def provision_reducer(self, num_workers: int): + component_name = REDUCER_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + return { + "CLP_REDUCER_LOGGING_LEVEL": self.clp_config.reducer.logging_level, + "CLP_REDUCER_LOGS_DIR": str(logs_dir), + "CLP_REDUCER_CONCURRENCY": str(num_workers), + "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), + } + + def _update_settings_object( + self, + parent_key_prefix: str, + settings: Dict[str, Any], + updates: Dict[str, Any], + ): + """ + Recursively updates the given settings object with the values from `updates`. + + :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. + :param settings: The settings to update. + :param updates: The updates. + :raises ValueError: If a key in `updates` doesn't exist in `settings`. + """ + for key, value in updates.items(): + if key not in settings: + error_msg = ( + f"{parent_key_prefix}{key} is not a valid configuration key for the webui." + ) + raise ValueError(error_msg) + if isinstance(value, dict): + self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) + else: + settings[key] = updates[key] + + def _read_and_update_settings_json( + self, settings_file_path: pathlib.Path, updates: Dict[str, Any] + ): + """ + Reads and updates a settings JSON file. + + :param settings_file_path: + :param updates: + """ + with open(settings_file_path, "r") as settings_json_file: + settings_object = json.loads(settings_json_file.read()) + self._update_settings_object("", settings_object, updates) + + return settings_object + + def provision_webui(self, container_clp_config: CLPConfig): + component_name = WEBUI_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" + client_settings_json_path = ( + self.clp_home / "var" / "www" / "webui" / "client" / "settings.json" + ) + server_settings_json_path = ( + self.clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" + ) + + validate_webui_config(self.clp_config, client_settings_json_path, server_settings_json_path) + + # Read, update, and write back client's and server's settings.json + clp_db_connection_params = self.clp_config.database.get_clp_connection_params_and_type(True) + table_prefix = clp_db_connection_params["table_prefix"] + if StorageEngine.CLP_S == self.clp_config.package.storage_engine: + archives_table_name = "" + files_table_name = "" + else: + archives_table_name = get_archives_table_name(table_prefix, None) + files_table_name = get_files_table_name(table_prefix, None) + + client_settings_json_updates = { + "ClpStorageEngine": self.clp_config.package.storage_engine, + "ClpQueryEngine": self.clp_config.package.query_engine, + "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, + "SqlDbClpArchivesTableName": archives_table_name, + "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), + "SqlDbClpFilesTableName": files_table_name, + "SqlDbClpTablePrefix": table_prefix, + "SqlDbCompressionJobsTableName": "compression_jobs", + } + client_settings_json = self._read_and_update_settings_json( + client_settings_json_path, client_settings_json_updates + ) + with open(client_settings_json_path, "w") as client_settings_json_file: + client_settings_json_file.write(json.dumps(client_settings_json)) + + server_settings_json_updates = { + "SqlDbHost": container_clp_config.database.host, + "SqlDbPort": self.clp_config.database.port, + "SqlDbName": self.clp_config.database.name, + "SqlDbQueryJobsTableName": "query_jobs", + "MongoDbHost": container_clp_config.results_cache.host, + "MongoDbPort": self.clp_config.results_cache.port, + "MongoDbName": self.clp_config.results_cache.db_name, + "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, + "MongoDbStreamFilesCollectionName": self.clp_config.results_cache.stream_collection_name, + "ClientDir": str(container_webui_dir / "client"), + "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), + "StreamTargetUncompressedSize": self.clp_config.stream_output.target_uncompressed_size, + } + + stream_storage = self.clp_config.stream_output.storage + if StorageType.S3 == stream_storage.type: + s3_config = stream_storage.s3_config + server_settings_json_updates["StreamFilesDir"] = None + server_settings_json_updates["StreamFilesS3Region"] = s3_config.region_code + server_settings_json_updates["StreamFilesS3PathPrefix"] = ( + f"{s3_config.bucket}/{s3_config.key_prefix}" + ) + auth = s3_config.aws_authentication + if AwsAuthType.profile == auth.type: + server_settings_json_updates["StreamFilesS3Profile"] = auth.profile + else: + server_settings_json_updates["StreamFilesS3Profile"] = None + elif StorageType.FS == stream_storage.type: + server_settings_json_updates["StreamFilesDir"] = str( + container_clp_config.stream_output.get_directory() + ) + server_settings_json_updates["StreamFilesS3Region"] = None + server_settings_json_updates["StreamFilesS3PathPrefix"] = None + server_settings_json_updates["StreamFilesS3Profile"] = None + + server_settings_json = self._read_and_update_settings_json( + server_settings_json_path, server_settings_json_updates + ) + with open(server_settings_json_path, "w") as settings_json_file: + settings_json_file.write(json.dumps(server_settings_json)) + + return { + "CLP_WEBUI_HOST": get_ip_from_hostname(self.clp_config.webui.host), + "CLP_WEBUI_PORT": str(self.clp_config.webui.port), + "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), + } + + def provision_garbage_collector(self): + component_name = GARBAGE_COLLECTOR_COMPONENT_NAME + logger.info(f"Initializing {component_name}...") + + logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) + + return {"CLP_GC_LOGGING_LEVEL": self.clp_config.garbage_collector.logging_level} + + @abstractmethod + def deploy(self): + """ + Deploys the provisioned components with orchestrator-specific logic. + """ + pass + + @abstractmethod + def _provision(self) -> Dict[str, str]: + """ + Provisions all components with orchestrator-specific logic. + + :return: Dictionary of environment variables for the orchestrator + """ + pass + + +class DockerComposeController(BaseController): + @staticmethod + def _get_num_workers(): + """ + Gets the parallelism number for worker components. + TODO: Revisit after moving from single-container to multi-container workers. + """ + return multiprocessing.cpu_count() // 2 + + def __init__(self, clp_config: CLPConfig): + super().__init__(clp_config) + check_docker_dependencies(should_compose_run=False) + + def deploy(self): + self._provision() + + # Start Docker Compose + logger.info(f"Starting CLP using Docker Compose...") + try: + subprocess.run( + ["docker", "compose", "up", "-d"], + cwd=self.clp_home, + stderr=subprocess.STDOUT, + check=True, + ) + except subprocess.CalledProcessError: + logger.exception("Failed to start CLP.") + raise + + def _provision(self): + # Create necessary directories + self.clp_config.data_directory.mkdir(parents=True, exist_ok=True) + self.clp_config.logs_directory.mkdir(parents=True, exist_ok=True) + self.clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) + self.clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + + container_clp_config = generate_docker_compose_container_config(self.clp_config) + conf_dir = self.clp_home / "etc" + num_workers = self._get_num_workers() + + env_dict = { + "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, + # User and group IDs + "CLP_USER_ID": str(os.getuid()), + "CLP_GROUP_ID": str(os.getgid()), + # Package container + "CLP_PACKAGE_CONTAINER": "clp-package:dev", + # Global paths + "CLP_DATA_DIR_HOST": str(self.clp_config.data_directory), + "CLP_LOGS_DIR_HOST": str(self.clp_config.logs_directory), + "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.clp_config.archive_output.get_directory()), + "CLP_STREAM_OUTPUT_DIR_HOST": str(self.clp_config.stream_output.get_directory()), + # AWS credentials + "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), + "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + **self.provision_database(conf_dir), + **self.provision_queue(), + **self.provision_redis(conf_dir), + **self.provision_results_cache(conf_dir), + **self.provision_compression_scheduler(), + **self.provision_query_scheduler(), + **self.provision_compression_worker(num_workers), + **self.provision_query_worker(num_workers), + **self.provision_reducer(num_workers), + **self.provision_webui(container_clp_config), + **self.provision_garbage_collector(), + } + + # AWS config directory + if self.clp_config.aws_config_directory is not None: + env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) + + # Staging directories for S3 storage + if StorageType.S3 == self.clp_config.archive_output.storage.type: + env_dict["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( + self.clp_config.archive_output.storage.staging_directory + ) + if StorageType.S3 == self.clp_config.stream_output.storage.type: + env_dict["CLP_STREAM_STAGING_DIR_HOST"] = str( + self.clp_config.stream_output.storage.staging_directory + ) + + with open(f"{self.clp_home}/.env", "w") as env_file: + for key, value in env_dict.items(): + env_file.write(f"{key}={value}\n") diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 302f145e92..20c31b265f 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -140,7 +140,7 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def is_compose_running(): +def is_docker_compose_running(): cmd = ["docker", "compose", "ls", "--quiet"] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) @@ -149,7 +149,7 @@ def is_compose_running(): raise EnvironmentError("docker-compose is not installed or not functioning properly.") -def check_dependencies(should_compose_run: bool = False): +def check_docker_dependencies(should_compose_run: bool = False): try: subprocess.run( "command -v docker", @@ -161,7 +161,7 @@ def check_dependencies(should_compose_run: bool = False): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = is_compose_running() + is_running = is_docker_compose_running() if should_compose_run and not is_running: raise EnvironmentError("docker-compose is not running.") if not should_compose_run and is_running: diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 9abfdf9a14..0b7021768a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -1,47 +1,11 @@ import argparse -import json import logging -import multiprocessing -import os import pathlib -import shlex -import socket -import stat -import subprocess import sys -import time -from typing import Any, Dict, List, Optional - -from clp_py_utils.clp_config import ( - AwsAuthType, - CLPConfig, - COMPRESSION_JOBS_TABLE_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, - DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_COMPONENT_NAME, - QUERY_JOBS_TABLE_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - QUEUE_COMPONENT_NAME, - REDIS_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, - RESULTS_CACHE_COMPONENT_NAME, - StorageEngine, - StorageType, - WEBUI_COMPONENT_NAME, -) -from clp_py_utils.clp_metadata_db_utils import ( - get_archives_table_name, - get_datasets_table_name, - get_files_table_name, -) +from clp_package_utils.controllers import DockerComposeController from clp_package_utils.general import ( - check_dependencies, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, - CONTAINER_CLP_HOME, - DockerMount, dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, @@ -49,402 +13,14 @@ validate_and_load_db_credentials_file, validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, - validate_db_config, validate_logs_input_config, validate_output_storage_config, - validate_queue_config, - validate_redis_config, - validate_results_cache_config, validate_retention_config, - validate_webui_config, ) -LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH - logger = logging.getLogger(__file__) -def get_ip_from_hostname(hostname: str) -> str: - """ - Resolves a hostname to an IP address. - - :param hostname: The hostname to resolve. - :return: The resolved IP address. - """ - return socket.gethostbyname(hostname) - - -def append_docker_options( - cmd: List[str], - mounts: Optional[List[Optional[DockerMount]]] = None, - env_vars: Optional[List[str]] = None, -): - """ - Appends Docker mount and environment variable options to a command list. - - :param cmd: The command list to append options to. - :param mounts: Optional list of DockerMount objects to add as --mount options. - :param env_vars: Optional list of environment variables to add as -e options. - """ - if mounts: - for mount in mounts: - if mount: - cmd.append("--mount") - cmd.append(str(mount)) - - if env_vars: - for env_var in env_vars: - if "" != env_var: - cmd.append("-e") - cmd.append(env_var) - - -def append_docker_port_settings_for_host_ips( - hostname: str, host_port: int, container_port: int, cmd: List[str] -): - # Note: We use a set because gethostbyname_ex can return the same IP twice for one hostname - for ip in set(socket.gethostbyname_ex(hostname)[2]): - cmd.append("-p") - cmd.append(f"{ip}:{host_port}:{container_port}") - - -def chown_recursively( - path: pathlib.Path, - user_id: int, - group_id: int, -): - """ - Recursively changes the owner of the given path to the given user ID and group ID. - :param path: - :param user_id: - :param group_id: - """ - chown_cmd = ["chown", "--recursive", f"{user_id}:{group_id}", str(path)] - subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) - - -def wait_for_container_cmd(container_name: str, cmd_to_run: List[str], timeout: int): - container_exec_cmd = ["docker", "exec", container_name] - cmd = container_exec_cmd + cmd_to_run - - begin_time = time.time() - - while True: - try: - subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) - return True - except subprocess.CalledProcessError: - if time.time() - begin_time > timeout: - break - time.sleep(1) - - cmd_str = shlex.join(cmd_to_run) - logger.error(f"Timeout while waiting for command {cmd_str} to run after {timeout} seconds") - return False - - -def start_db(clp_config: CLPConfig, conf_dir: pathlib.Path): - component_name = DB_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - conf_file = conf_dir / "mysql" / "conf.d" / "logging.cnf" - data_dir = clp_config.data_directory / component_name - logs_dir = clp_config.logs_directory / component_name - validate_db_config(clp_config, conf_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - - return { - "CLP_DB_CONF_FILE_HOST": str(conf_file), - "CLP_DB_DATA_DIR_HOST": str(data_dir), - "CLP_DB_LOGS_DIR_HOST": str(logs_dir), - "CLP_DB_HOST": get_ip_from_hostname(clp_config.database.host), - "CLP_DB_PORT": str(clp_config.database.port), - "CLP_DB_NAME": clp_config.database.name, - "CLP_DB_USER": clp_config.database.username, - "CLP_DB_PASS": clp_config.database.password, - "CLP_DB_IMAGE": ( - "mysql:8.0.23" if "mysql" == clp_config.database.type else "mariadb:10-jammy" - ), - } - - -def start_queue(clp_config: CLPConfig): - component_name = QUEUE_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - validate_queue_config(clp_config, logs_dir) - logs_dir.mkdir(exist_ok=True, parents=True) - - return { - "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), - "CLP_QUEUE_HOST": get_ip_from_hostname(clp_config.queue.host), - "CLP_QUEUE_PORT": str(clp_config.queue.port), - "CLP_QUEUE_USER": clp_config.queue.username, - "CLP_QUEUE_PASS": clp_config.queue.password, - } - - -def start_redis(clp_config: CLPConfig, conf_dir: pathlib.Path): - component_name = REDIS_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - conf_file = conf_dir / "redis" / "redis.conf" - logs_dir = clp_config.logs_directory / component_name - data_dir = clp_config.data_directory / component_name - validate_redis_config(clp_config, conf_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - - return { - "CLP_REDIS_CONF_FILE_HOST": str(conf_file), - "CLP_REDIS_DATA_DIR_HOST": str(data_dir), - "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": get_ip_from_hostname(clp_config.redis.host), - "CLP_REDIS_PORT": str(clp_config.redis.port), - "CLP_REDIS_PASS": clp_config.redis.password, - "CLP_REDIS_QUERY_BACKEND_DB": str(clp_config.redis.query_backend_database), - "CLP_REDIS_COMPRESSION_BACKEND_DB": str(clp_config.redis.compression_backend_database), - } - - -def start_results_cache(clp_config: CLPConfig, conf_file: pathlib.Path): - component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - conf_file = conf_file / "mongo" / "mongod.conf" - data_dir = clp_config.data_directory / component_name - logs_dir = clp_config.logs_directory / component_name - validate_results_cache_config(clp_config, conf_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - - return { - "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), - "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), - "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(clp_config.results_cache.host), - "CLP_RESULTS_CACHE_PORT": str(clp_config.results_cache.port), - "CLP_RESULTS_CACHE_DB_NAME": clp_config.results_cache.db_name, - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": clp_config.results_cache.stream_collection_name, - } - - -def start_compression_scheduler(clp_config: CLPConfig): - component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_file = clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) - - return { - "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": clp_config.compression_scheduler.logging_level, - "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), - } - - -def start_query_scheduler(clp_config: CLPConfig): - component_name = QUERY_SCHEDULER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_file = clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) - - return { - "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": clp_config.query_scheduler.logging_level, - "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), - } - - -def start_compression_worker(clp_config: CLPConfig, num_cpus: int): - component_name = COMPRESSION_WORKER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) - - return { - "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_cpus), - "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": clp_config.compression_worker.logging_level, - "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), - } - - -def start_query_worker(clp_config: CLPConfig, num_cpus: int): - component_name = QUERY_WORKER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) - - return { - "CLP_QUERY_WORKER_LOGGING_LEVEL": clp_config.query_worker.logging_level, - "CLP_QUERY_WORKER_LOGS_DIR": str(logs_dir), - "CLP_QUERY_WORKER_CONCURRENCY": str(num_cpus), - } - - -def start_reducer(clp_config: CLPConfig, num_workers: int): - component_name = REDUCER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) - - return { - "CLP_REDUCER_LOGGING_LEVEL": clp_config.reducer.logging_level, - "CLP_REDUCER_LOGS_DIR": str(logs_dir), - "CLP_REDUCER_CONCURRENCY": str(num_workers), - "CLP_REDUCER_UPSERT_INTERVAL": str(clp_config.reducer.upsert_interval), - } - - -def update_settings_object( - parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], -): - """ - Recursively updates the given settings object with the values from `updates`. - - :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. - :param settings: The settings to update. - :param updates: The updates. - :raises ValueError: If a key in `updates` doesn't exist in `settings`. - """ - for key, value in updates.items(): - if key not in settings: - error_msg = f"{parent_key_prefix}{key} is not a valid configuration key for the webui." - raise ValueError(error_msg) - if isinstance(value, dict): - update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) - else: - settings[key] = updates[key] - - -def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dict[str, Any]): - """ - Reads and updates a settings JSON file. - - :param settings_file_path: - :param updates: - """ - with open(settings_file_path, "r") as settings_json_file: - settings_object = json.loads(settings_json_file.read()) - update_settings_object("", settings_object, updates) - - return settings_object - - -def start_webui(clp_config: CLPConfig, container_clp_config: CLPConfig): - component_name = WEBUI_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" - client_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json" - ) - server_settings_json_path = ( - get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "settings.json" - ) - - validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path) - - # Read, update, and write back client's and server's settings.json - clp_db_connection_params = clp_config.database.get_clp_connection_params_and_type(True) - table_prefix = clp_db_connection_params["table_prefix"] - if StorageEngine.CLP_S == clp_config.package.storage_engine: - archives_table_name = "" - files_table_name = "" - else: - archives_table_name = get_archives_table_name(table_prefix, None) - files_table_name = get_files_table_name(table_prefix, None) - - client_settings_json_updates = { - "ClpStorageEngine": clp_config.package.storage_engine, - "ClpQueryEngine": clp_config.package.query_engine, - "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name, - "SqlDbClpArchivesTableName": archives_table_name, - "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), - "SqlDbClpFilesTableName": files_table_name, - "SqlDbClpTablePrefix": table_prefix, - "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME, - } - client_settings_json = read_and_update_settings_json( - client_settings_json_path, client_settings_json_updates - ) - with open(client_settings_json_path, "w") as client_settings_json_file: - client_settings_json_file.write(json.dumps(client_settings_json)) - - server_settings_json_updates = { - "SqlDbHost": container_clp_config.database.host, - "SqlDbPort": clp_config.database.port, - "SqlDbName": clp_config.database.name, - "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME, - "MongoDbHost": container_clp_config.results_cache.host, - "MongoDbPort": clp_config.results_cache.port, - "MongoDbName": clp_config.results_cache.db_name, - "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name, - "MongoDbStreamFilesCollectionName": clp_config.results_cache.stream_collection_name, - "ClientDir": str(container_webui_dir / "client"), - "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), - "StreamTargetUncompressedSize": clp_config.stream_output.target_uncompressed_size, - } - - stream_storage = clp_config.stream_output.storage - if StorageType.S3 == stream_storage.type: - s3_config = stream_storage.s3_config - server_settings_json_updates["StreamFilesDir"] = None - server_settings_json_updates["StreamFilesS3Region"] = s3_config.region_code - server_settings_json_updates["StreamFilesS3PathPrefix"] = ( - f"{s3_config.bucket}/{s3_config.key_prefix}" - ) - auth = s3_config.aws_authentication - if AwsAuthType.profile == auth.type: - server_settings_json_updates["StreamFilesS3Profile"] = auth.profile - else: - server_settings_json_updates["StreamFilesS3Profile"] = None - elif StorageType.FS == stream_storage.type: - server_settings_json_updates["StreamFilesDir"] = str( - container_clp_config.stream_output.get_directory() - ) - server_settings_json_updates["StreamFilesS3Region"] = None - server_settings_json_updates["StreamFilesS3PathPrefix"] = None - server_settings_json_updates["StreamFilesS3Profile"] = None - - server_settings_json = read_and_update_settings_json( - server_settings_json_path, server_settings_json_updates - ) - with open(server_settings_json_path, "w") as settings_json_file: - settings_json_file.write(json.dumps(server_settings_json)) - - return { - "CLP_WEBUI_HOST": get_ip_from_hostname(clp_config.webui.host), - "CLP_WEBUI_PORT": str(clp_config.webui.port), - "CLP_WEBUI_RATE_LIMIT": str(clp_config.webui.rate_limit), - } - - -def start_garbage_collector(clp_config: CLPConfig): - component_name = GARBAGE_COLLECTOR_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") - - logs_dir = clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) - - return {"CLP_GC_LOGGING_LEVEL": clp_config.garbage_collector.logging_level} - - -def add_num_workers_argument(parser): - parser.add_argument( - "--num-workers", - type=int, - default=multiprocessing.cpu_count(), - help="Number of workers to start", - ) - - def main(argv): clp_home = get_clp_home() default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH @@ -459,12 +35,6 @@ def main(argv): parsed_args = args_parser.parse_args(argv[1:]) - try: - check_dependencies(should_compose_run=False) - except: - logger.exception("Dependency checking failed.") - return -1 - # Validate and load config file try: config_file_path = pathlib.Path(parsed_args.config) @@ -484,65 +54,12 @@ def main(argv): logger.exception("Failed to load config.") return -1 - # TODO: Rely on Docker Compose to spawn multiple workers - num_workers = multiprocessing.cpu_count() // 2 - container_clp_config = generate_docker_compose_container_config(clp_config) - - # Create necessary directories - clp_config.data_directory.mkdir(parents=True, exist_ok=True) - clp_config.logs_directory.mkdir(parents=True, exist_ok=True) - clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - dump_shared_container_config(container_clp_config, clp_config) try: - conf_dir = clp_home / "etc" - - env_dict = { - "CLP_PACKAGE_STORAGE_ENGINE": clp_config.package.storage_engine, - # User and group IDs - "CLP_USER_ID": str(os.getuid()), - "CLP_GROUP_ID": str(os.getgid()), - # Package container - "CLP_PACKAGE_CONTAINER": "clp-package:dev", - # Global paths - "CLP_DATA_DIR_HOST": str(clp_config.data_directory), - "CLP_LOGS_DIR_HOST": str(clp_config.logs_directory), - "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(clp_config.archive_output.get_directory()), - "CLP_STREAM_OUTPUT_DIR_HOST": str(clp_config.stream_output.get_directory()), - # AWS credentials - "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), - "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **start_db(clp_config, conf_dir), - **start_queue(clp_config), - **start_redis(clp_config, conf_dir), - **start_results_cache(clp_config, conf_dir), - **start_compression_scheduler(clp_config), - **start_query_scheduler(clp_config), - **start_compression_worker(clp_config, num_workers), - **start_query_worker(clp_config, num_workers), - **start_reducer(clp_config, num_workers), - **start_webui(clp_config, container_clp_config), - **start_garbage_collector(clp_config), - } - # AWS config directory - if clp_config.aws_config_directory is not None: - env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(clp_config.aws_config_directory) - - # Staging directories for S3 storage - if StorageType.S3 == clp_config.archive_output.storage.type: - env_dict["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( - clp_config.archive_output.storage.staging_directory - ) - if StorageType.S3 == clp_config.stream_output.storage.type: - env_dict["CLP_STREAM_STAGING_DIR_HOST"] = str( - clp_config.stream_output.storage.staging_directory - ) - with open(f"{clp_home}/.env", "w") as env_file: - for key, value in env_dict.items(): - env_file.write(f"{key}={value}\n") + controller = DockerComposeController(clp_config) + controller.deploy() except Exception as ex: if type(ex) == ValueError: logger.error(f"Failed to initialize CLP: {ex}") @@ -550,17 +67,6 @@ def main(argv): logger.exception("Failed to initialize CLP.") return -1 - logger.info(f"Starting CLP using Docker Compose...") - try: - subprocess.run( - ["docker", "compose", "up", "-d"], - stderr=subprocess.STDOUT, - check=True, - ) - except subprocess.CalledProcessError: - logger.exception("Failed to start CLP.") - return -1 - return 0 diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 39912ce87c..7cabe58ecc 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -2,14 +2,14 @@ import subprocess import sys -from clp_package_utils.general import check_dependencies +from clp_package_utils.general import check_docker_dependencies logger = logging.getLogger(__file__) def main(): try: - check_dependencies(should_compose_run=True) + check_docker_dependencies(should_compose_run=True) except: logger.exception("Dependency checking failed.") return -1 From 3eb8dfe591ad0399ebe09e84db5dff6f88e6450d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 01:48:50 -0400 Subject: [PATCH 035/408] refactor: remove obsolete node-specific directory configuration comments --- components/clp-package-utils/clp_package_utils/general.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 20c31b265f..a0292fa848 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -462,11 +462,6 @@ def load_config_file( validate_path_for_container_mount(clp_config.data_directory) validate_path_for_container_mount(clp_config.logs_directory) - # # Make data and logs directories node-specific - # hostname = socket.gethostname() - # clp_config.data_directory /= hostname - # clp_config.logs_directory /= hostname - return clp_config From 669fa9c9e6a3fd006129e301cbe0d56f255b2847 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 01:58:24 -0400 Subject: [PATCH 036/408] refactor: remove redundant `conf_dir` parameter and use centralized config directory in controller methods --- .../clp_package_utils/controllers.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controllers.py b/components/clp-package-utils/clp_package_utils/controllers.py index 880f1f7266..9ea293c2d7 100644 --- a/components/clp-package-utils/clp_package_utils/controllers.py +++ b/components/clp-package-utils/clp_package_utils/controllers.py @@ -63,12 +63,13 @@ class BaseController(ABC): def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config self.clp_home = get_clp_home() + self.conf_dir = self.clp_home / "etc" - def provision_database(self, conf_dir: pathlib.Path): + def provision_database(self): component_name = DB_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = conf_dir / "mysql" / "conf.d" / "logging.cnf" + conf_file = self.conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -105,11 +106,11 @@ def provision_queue(self): "CLP_QUEUE_PASS": self.clp_config.queue.password, } - def provision_redis(self, conf_dir: pathlib.Path): + def provision_redis(self): component_name = REDIS_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = conf_dir / "redis" / "redis.conf" + conf_file = self.conf_dir / "redis" / "redis.conf" logs_dir = self.clp_config.logs_directory / component_name data_dir = self.clp_config.data_directory / component_name validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -129,11 +130,11 @@ def provision_redis(self, conf_dir: pathlib.Path): ), } - def provision_results_cache(self, conf_dir: pathlib.Path): + def provision_results_cache(self): component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = conf_dir / "mongo" / "mongod.conf" + conf_file = self.conf_dir / "mongo" / "mongod.conf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -405,7 +406,6 @@ def _provision(self): self.clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) container_clp_config = generate_docker_compose_container_config(self.clp_config) - conf_dir = self.clp_home / "etc" num_workers = self._get_num_workers() env_dict = { @@ -423,10 +423,10 @@ def _provision(self): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **self.provision_database(conf_dir), + **self.provision_database(), **self.provision_queue(), - **self.provision_redis(conf_dir), - **self.provision_results_cache(conf_dir), + **self.provision_redis(), + **self.provision_results_cache(), **self.provision_compression_scheduler(), **self.provision_query_scheduler(), **self.provision_compression_worker(num_workers), From acee071ed12bee4cea5b10def41df3438663eb18 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 01:58:41 -0400 Subject: [PATCH 037/408] refactor: rename `controllers` module to `controller` and update import paths in start_clp.py --- .../clp_package_utils/{controllers.py => controller.py} | 0 .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename components/clp-package-utils/clp_package_utils/{controllers.py => controller.py} (100%) diff --git a/components/clp-package-utils/clp_package_utils/controllers.py b/components/clp-package-utils/clp_package_utils/controller.py similarity index 100% rename from components/clp-package-utils/clp_package_utils/controllers.py rename to components/clp-package-utils/clp_package_utils/controller.py diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 0b7021768a..3ccc91eee6 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -3,7 +3,7 @@ import pathlib import sys -from clp_package_utils.controllers import DockerComposeController +from clp_package_utils.controller import DockerComposeController from clp_package_utils.general import ( CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, dump_shared_container_config, From 3c45cfa8572b83e165a2c42f86603e7bd0454ccd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:01:28 -0400 Subject: [PATCH 038/408] refactor: make `validate_log_directory` private and update references accordingly --- .../clp-package-utils/clp_package_utils/general.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a0292fa848..8aa271c4ad 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -168,7 +168,7 @@ def check_docker_dependencies(should_compose_run: bool = False): raise EnvironmentError("docker-compose is already running.") -def validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: +def _validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: try: validate_path_could_be_dir(logs_dir) except ValueError as ex: @@ -522,13 +522,13 @@ def validate_db_config( f"{DB_COMPONENT_NAME} base configuration at {str(base_config)} is missing." ) _validate_data_directory(data_dir, DB_COMPONENT_NAME) - validate_log_directory(logs_dir, DB_COMPONENT_NAME) + _validate_log_directory(logs_dir, DB_COMPONENT_NAME) validate_port(f"{DB_COMPONENT_NAME}.port", clp_config.database.host, clp_config.database.port) def validate_queue_config(clp_config: CLPConfig, logs_dir: pathlib.Path): - validate_log_directory(logs_dir, QUEUE_COMPONENT_NAME) + _validate_log_directory(logs_dir, QUEUE_COMPONENT_NAME) validate_port(f"{QUEUE_COMPONENT_NAME}.port", clp_config.queue.host, clp_config.queue.port) @@ -541,13 +541,13 @@ def validate_redis_config( f"{REDIS_COMPONENT_NAME} base configuration at {str(base_config)} is missing." ) _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) - validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) + _validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) validate_port(f"{REDIS_COMPONENT_NAME}.port", clp_config.redis.host, clp_config.redis.port) def validate_reducer_config(clp_config: CLPConfig, logs_dir: pathlib.Path, num_workers: int): - validate_log_directory(logs_dir, REDUCER_COMPONENT_NAME) + _validate_log_directory(logs_dir, REDUCER_COMPONENT_NAME) for i in range(0, num_workers): validate_port( @@ -565,7 +565,7 @@ def validate_results_cache_config( f"{RESULTS_CACHE_COMPONENT_NAME} base configuration at {str(base_config)} is missing." ) _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME) - validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) + _validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) validate_port( f"{RESULTS_CACHE_COMPONENT_NAME}.port", From ed20110f6bf2effc769d03546d7489a1592377ee Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:07:19 -0400 Subject: [PATCH 039/408] refactor: rename `conf_dir` to `_conf_dir` and update references to reflect private usage --- .../clp-package-utils/clp_package_utils/controller.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 9ea293c2d7..ae63a15a34 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -63,13 +63,13 @@ class BaseController(ABC): def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config self.clp_home = get_clp_home() - self.conf_dir = self.clp_home / "etc" + self._conf_dir = self.clp_home / "etc" def provision_database(self): component_name = DB_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = self.conf_dir / "mysql" / "conf.d" / "logging.cnf" + conf_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -110,7 +110,7 @@ def provision_redis(self): component_name = REDIS_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = self.conf_dir / "redis" / "redis.conf" + conf_file = self._conf_dir / "redis" / "redis.conf" logs_dir = self.clp_config.logs_directory / component_name data_dir = self.clp_config.data_directory / component_name validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) @@ -134,7 +134,7 @@ def provision_results_cache(self): component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Initializing {component_name}...") - conf_file = self.conf_dir / "mongo" / "mongod.conf" + conf_file = self._conf_dir / "mongo" / "mongod.conf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) From 4d1f5aae21b40a331d037bbce7cdd6ecc4fbecea Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:21:16 -0400 Subject: [PATCH 040/408] refactor: make `get_ip_from_hostname` private and update all references --- .../clp_package_utils/controller.py | 12 ++++++------ components/clp-py-utils/clp_py_utils/clp_config.py | 10 ---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ae63a15a34..a7e4aa6826 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -49,7 +49,7 @@ logger = logging.getLogger(__name__) -def get_ip_from_hostname(hostname: str) -> str: +def _get_ip_from_hostname(hostname: str) -> str: """ Resolves a hostname to an IP address. @@ -80,7 +80,7 @@ def provision_database(self): "CLP_DB_CONF_FILE_HOST": str(conf_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), - "CLP_DB_HOST": get_ip_from_hostname(self.clp_config.database.host), + "CLP_DB_HOST": _get_ip_from_hostname(self.clp_config.database.host), "CLP_DB_PORT": str(self.clp_config.database.port), "CLP_DB_NAME": self.clp_config.database.name, "CLP_DB_USER": self.clp_config.database.username, @@ -100,7 +100,7 @@ def provision_queue(self): return { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), - "CLP_QUEUE_HOST": get_ip_from_hostname(self.clp_config.queue.host), + "CLP_QUEUE_HOST": _get_ip_from_hostname(self.clp_config.queue.host), "CLP_QUEUE_PORT": str(self.clp_config.queue.port), "CLP_QUEUE_USER": self.clp_config.queue.username, "CLP_QUEUE_PASS": self.clp_config.queue.password, @@ -121,7 +121,7 @@ def provision_redis(self): "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": get_ip_from_hostname(self.clp_config.redis.host), + "CLP_REDIS_HOST": _get_ip_from_hostname(self.clp_config.redis.host), "CLP_REDIS_PORT": str(self.clp_config.redis.port), "CLP_REDIS_PASS": self.clp_config.redis.password, "CLP_REDIS_QUERY_BACKEND_DB": str(self.clp_config.redis.query_backend_database), @@ -145,7 +145,7 @@ def provision_results_cache(self): "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": get_ip_from_hostname(self.clp_config.results_cache.host), + "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self.clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(self.clp_config.results_cache.port), "CLP_RESULTS_CACHE_DB_NAME": self.clp_config.results_cache.db_name, "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, @@ -338,7 +338,7 @@ def provision_webui(self, container_clp_config: CLPConfig): settings_json_file.write(json.dumps(server_settings_json)) return { - "CLP_WEBUI_HOST": get_ip_from_hostname(self.clp_config.webui.host), + "CLP_WEBUI_HOST": _get_ip_from_hostname(self.clp_config.webui.host), "CLP_WEBUI_PORT": str(self.clp_config.webui.port), "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), } diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index abef21babc..2135bab404 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -757,16 +757,6 @@ def _get_env_var(name: str) -> str: return value -def get_ip_from_hostname(hostname: str) -> str: - """ - Resolves a hostname to an IP address. - - :param hostname: The hostname to resolve. - :return: The resolved IP address. - """ - return socket.gethostbyname(hostname) - - class CLPConfig(BaseModel): execution_container: Optional[str] = None From 3a698bb91d25b481efeff97951cbebeb67555760 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:24:15 -0400 Subject: [PATCH 041/408] refactor: extract `transform_for_container_config` method to simplify container config handling --- .../clp_package_utils/general.py | 38 +------------------ .../clp-py-utils/clp_py_utils/clp_config.py | 36 ++++++++++++++++++ 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 8aa271c4ad..95266da90b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -295,47 +295,13 @@ def generate_container_config( def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig: """ - Copies the given config and corrects mount paths and hosts for Docker Compose. + Copies the given config and transforms mount paths and hosts for Docker Compose. :param clp_config: :return: The container config and the mounts. """ container_clp_config = clp_config.copy(deep=True) - - # Set container paths - container_clp_config.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH - container_clp_config.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH - if StorageType.FS == clp_config.logs_input.type: - container_clp_config.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR - - if StorageType.FS == clp_config.archive_output.storage.type: - container_clp_config.archive_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH - ) - elif StorageType.S3 == clp_config.archive_output.storage.type: - container_clp_config.archive_output.storage.staging_directory = ( - pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH - ) - - if StorageType.FS == clp_config.stream_output.storage.type: - container_clp_config.stream_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH - ) - elif StorageType.S3 == clp_config.stream_output.storage.type: - container_clp_config.stream_output.storage.staging_directory = ( - pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH - ) - - if clp_config.aws_config_directory is not None: - container_clp_config.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY - - # Set container services' hosts - container_clp_config.database.host = DB_COMPONENT_NAME - container_clp_config.queue.host = QUEUE_COMPONENT_NAME - container_clp_config.redis.host = REDIS_COMPONENT_NAME - container_clp_config.results_cache.host = RESULTS_CACHE_COMPONENT_NAME - container_clp_config.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME - container_clp_config.reducer.host = REDUCER_COMPONENT_NAME + container_clp_config.transform_for_container_config() return container_clp_config diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 2135bab404..4bd13aae5f 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, PrivateAttr, root_validator, validator from strenum import KebabCaseStrEnum, LowercaseStrEnum +from clp_package_utils.general import CONTAINER_AWS_CONFIG_DIRECTORY, CONTAINER_INPUT_LOGS_ROOT_DIR from .clp_logging import get_valid_logging_level, is_valid_logging_level from .core import ( get_config_value, @@ -922,6 +923,41 @@ def dump_to_primitive_dict(self): return d + def transform_for_container_config(self): + self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH + self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH + if StorageType.FS == self.logs_input.type: + self.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR + + if StorageType.FS == self.archive_output.storage.type: + self.archive_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + ) + elif StorageType.S3 == self.archive_output.storage.type: + self.archive_output.storage.staging_directory = ( + pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + ) + + if StorageType.FS == self.stream_output.storage.type: + self.stream_output.storage.directory = ( + pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + ) + elif StorageType.S3 == self.stream_output.storage.type: + self.stream_output.storage.staging_directory = ( + pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + ) + + if self.aws_config_directory is not None: + self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY + + # Set container services' hosts + self.database.host = DB_COMPONENT_NAME + self.queue.host = QUEUE_COMPONENT_NAME + self.redis.host = REDIS_COMPONENT_NAME + self.results_cache.host = RESULTS_CACHE_COMPONENT_NAME + self.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME + self.reducer.host = REDUCER_COMPONENT_NAME + class WorkerConfig(BaseModel): package: Package = Package() From cca84f491529bac205d99b3e0c16fcd8d732fe8a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:39:54 -0400 Subject: [PATCH 042/408] refactor: centralize and simplify path definitions and container config handling --- .../clp_package_utils/general.py | 15 +------ .../clp_package_utils/scripts/start_clp.py | 2 +- .../clp-py-utils/clp_py_utils/clp_config.py | 45 +++++++++---------- 3 files changed, 23 insertions(+), 39 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 95266da90b..575c321f14 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -13,17 +13,10 @@ import yaml from clp_py_utils.clp_config import ( - CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH, - CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH, CLP_DEFAULT_CREDENTIALS_FILE_PATH, - CLP_DEFAULT_DATA_DIRECTORY_PATH, - CLP_DEFAULT_LOG_DIRECTORY_PATH, - CLP_DEFAULT_STREAM_DIRECTORY_PATH, - CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, DB_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, QueryEngine, QUEUE_COMPONENT_NAME, REDIS_COMPONENT_NAME, @@ -31,7 +24,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageType, WEBUI_COMPONENT_NAME, - WorkerConfig, + WorkerConfig, CONTAINER_CLP_HOME, CONTAINER_INPUT_LOGS_ROOT_DIR, CONTAINER_AWS_CONFIG_DIRECTORY, ) from clp_py_utils.clp_metadata_db_utils import ( MYSQL_TABLE_NAME_MAX_LEN, @@ -50,12 +43,6 @@ EXTRACT_IR_CMD = "i" EXTRACT_JSON_CMD = "j" -# Paths -CONTAINER_AWS_CONFIG_DIRECTORY = pathlib.Path("/") / ".aws" -CONTAINER_CLP_HOME = pathlib.Path("/") / "opt" / "clp" -CONTAINER_INPUT_LOGS_ROOT_DIR = pathlib.Path("/") / "mnt" / "logs" -CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH = pathlib.Path("etc") / "clp-config.yml" - DOCKER_MOUNT_TYPE_STRINGS = ["bind"] diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 3ccc91eee6..501d593dbb 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -5,7 +5,6 @@ from clp_package_utils.controller import DockerComposeController from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, @@ -17,6 +16,7 @@ validate_output_storage_config, validate_retention_config, ) +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH logger = logging.getLogger(__file__) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4bd13aae5f..db932a8c67 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -8,7 +8,6 @@ from pydantic import BaseModel, PrivateAttr, root_validator, validator from strenum import KebabCaseStrEnum, LowercaseStrEnum -from clp_package_utils.general import CONTAINER_AWS_CONFIG_DIRECTORY, CONTAINER_INPUT_LOGS_ROOT_DIR from .clp_logging import get_valid_logging_level, is_valid_logging_level from .core import ( get_config_value, @@ -41,6 +40,11 @@ OS_RELEASE_FILE_PATH = pathlib.Path("etc") / "os-release" +# Paths +CONTAINER_AWS_CONFIG_DIRECTORY = pathlib.Path("/") / ".aws" +CONTAINER_CLP_HOME = pathlib.Path("/") / "opt" / "clp" +CONTAINER_INPUT_LOGS_ROOT_DIR = pathlib.Path("/") / "mnt" / "logs" +CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH = pathlib.Path("etc") / "clp-config.yml" CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" @@ -519,6 +523,9 @@ class S3IngestionConfig(BaseModel): def dump_to_primitive_dict(self): return self.dict() + def transform_for_container_config(self): + pass + class FsStorage(BaseModel): type: Literal[StorageType.FS.value] = StorageType.FS.value @@ -574,21 +581,29 @@ def dump_to_primitive_dict(self): class FsIngestionConfig(FsStorage): directory: pathlib.Path = pathlib.Path("/") + def transform_for_container_config(self): + self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR class ArchiveFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + def transform_for_container_config(self): + self.directory = pathlib.Path("/") /CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH class StreamFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH + def transform_for_container_config(self): + self.directory = pathlib.Path("/") /CLP_DEFAULT_STREAM_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + def transform_for_container_config(self): + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( @@ -926,27 +941,9 @@ def dump_to_primitive_dict(self): def transform_for_container_config(self): self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH - if StorageType.FS == self.logs_input.type: - self.logs_input.directory = CONTAINER_INPUT_LOGS_ROOT_DIR - - if StorageType.FS == self.archive_output.storage.type: - self.archive_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH - ) - elif StorageType.S3 == self.archive_output.storage.type: - self.archive_output.storage.staging_directory = ( - pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH - ) - - if StorageType.FS == self.stream_output.storage.type: - self.stream_output.storage.directory = ( - pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH - ) - elif StorageType.S3 == self.stream_output.storage.type: - self.stream_output.storage.staging_directory = ( - pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH - ) - + self.logs_input.transform_for_container_config() + self.archive_output.storage.transform_for_container_config() + self.stream_output.storage.transform_for_container_config() if self.aws_config_directory is not None: self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY From 537398a36b4847b105c1bf6e01435b7ca69afbf9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:45:19 -0400 Subject: [PATCH 043/408] refactor: streamline container service configuration with `transform_for_container_config` method --- .../clp_package_utils/general.py | 5 +- .../clp_package_utils/scripts/start_clp.py | 3 +- .../clp-py-utils/clp_py_utils/clp_config.py | 49 +++++++++++++------ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 575c321f14..47d0472531 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -16,6 +16,9 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLP_SHARED_CONFIG_FILENAME, CLPConfig, + CONTAINER_AWS_CONFIG_DIRECTORY, + CONTAINER_CLP_HOME, + CONTAINER_INPUT_LOGS_ROOT_DIR, DB_COMPONENT_NAME, QueryEngine, QUEUE_COMPONENT_NAME, @@ -24,7 +27,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageType, WEBUI_COMPONENT_NAME, - WorkerConfig, CONTAINER_CLP_HOME, CONTAINER_INPUT_LOGS_ROOT_DIR, CONTAINER_AWS_CONFIG_DIRECTORY, + WorkerConfig, ) from clp_py_utils.clp_metadata_db_utils import ( MYSQL_TABLE_NAME_MAX_LEN, diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 501d593dbb..62a8aef324 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -3,6 +3,8 @@ import pathlib import sys +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + from clp_package_utils.controller import DockerComposeController from clp_package_utils.general import ( dump_shared_container_config, @@ -16,7 +18,6 @@ validate_output_storage_config, validate_retention_config, ) -from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH logger = logging.getLogger(__file__) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index db932a8c67..21706cd5f8 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -239,6 +239,9 @@ def load_credentials_from_env(self): self.username = _get_env_var(CLP_DB_USER_ENV_VAR_NAME) self.password = _get_env_var(CLP_DB_PASS_ENV_VAR_NAME) + def transform_for_container_config(self): + self.host = DB_COMPONENT_NAME + def _validate_logging_level(cls, field): if not is_valid_logging_level(field): @@ -296,6 +299,9 @@ def validate_port(cls, field): raise ValueError(f"{field} is not greater than zero") return field + def transform_for_container_config(self): + self.host = QUERY_SCHEDULER_COMPONENT_NAME + class CompressionWorker(BaseModel): logging_level: str = "INFO" @@ -349,6 +355,9 @@ def load_credentials_from_env(self): """ self.password = _get_env_var(CLP_REDIS_PASS_ENV_VAR_NAME) + def transform_for_container_config(self): + self.host = REDIS_COMPONENT_NAME + class Reducer(BaseModel): host: str = "localhost" @@ -379,6 +388,9 @@ def validate_upsert_interval(cls, field): raise ValueError(f"{field} is not greater than zero") return field + def transform_for_container_config(self): + self.host = REDUCER_COMPONENT_NAME + class ResultsCache(BaseModel): host: str = "localhost" @@ -416,6 +428,9 @@ def validate_retention_period(cls, field): def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" + def transform_for_container_config(self): + self.host = RESULTS_CACHE_COMPONENT_NAME + class Queue(BaseModel): host: str = "localhost" @@ -446,6 +461,9 @@ def load_credentials_from_env(self): self.username = _get_env_var(CLP_QUEUE_USER_ENV_VAR_NAME) self.password = _get_env_var(CLP_QUEUE_PASS_ENV_VAR_NAME) + def transform_for_container_config(self): + self.host = QUEUE_COMPONENT_NAME + class S3Credentials(BaseModel): access_key_id: str @@ -584,26 +602,30 @@ class FsIngestionConfig(FsStorage): def transform_for_container_config(self): self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR + class ArchiveFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH def transform_for_container_config(self): - self.directory = pathlib.Path("/") /CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + class StreamFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH def transform_for_container_config(self): - self.directory = pathlib.Path("/") /CLP_DEFAULT_STREAM_DIRECTORY_PATH + self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH def transform_for_container_config(self): - self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( @@ -941,19 +963,18 @@ def dump_to_primitive_dict(self): def transform_for_container_config(self): self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH + if self.aws_config_directory is not None: + self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY self.logs_input.transform_for_container_config() self.archive_output.storage.transform_for_container_config() self.stream_output.storage.transform_for_container_config() - if self.aws_config_directory is not None: - self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY - # Set container services' hosts - self.database.host = DB_COMPONENT_NAME - self.queue.host = QUEUE_COMPONENT_NAME - self.redis.host = REDIS_COMPONENT_NAME - self.results_cache.host = RESULTS_CACHE_COMPONENT_NAME - self.query_scheduler.host = QUERY_SCHEDULER_COMPONENT_NAME - self.reducer.host = REDUCER_COMPONENT_NAME + self.database.transform_for_container_config() + self.queue.transform_for_container_config() + self.redis.transform_for_container_config() + self.results_cache.transform_for_container_config() + self.query_scheduler.transform_for_container_config() + self.reducer.transform_for_container_config() class WorkerConfig(BaseModel): From 42442b838a5ef1ce9a9d46c00b53bb2d82c52691 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:49:41 -0400 Subject: [PATCH 044/408] fix imports --- .../clp_package_utils/scripts/archive_manager.py | 3 +-- .../clp-package-utils/clp_package_utils/scripts/compress.py | 3 +-- .../clp_package_utils/scripts/dataset_manager.py | 2 +- .../clp-package-utils/clp_package_utils/scripts/decompress.py | 3 +-- .../clp_package_utils/scripts/native/archive_manager.py | 3 +-- .../clp_package_utils/scripts/native/compress.py | 3 +-- .../clp_package_utils/scripts/native/dataset_manager.py | 4 ++-- .../clp_package_utils/scripts/native/decompress.py | 3 +-- .../clp_package_utils/scripts/native/search.py | 3 +-- .../clp-package-utils/clp_package_utils/scripts/search.py | 3 +-- 10 files changed, 11 insertions(+), 19 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index a86543c157..8706fe86c4 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -11,11 +11,10 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_DATASET_NAME, StorageEngine, - StorageType, + StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, DockerMount, dump_container_config, diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 8cd2e6637c..9b33bb1995 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -11,12 +11,11 @@ CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_DATASET_NAME, - StorageEngine, + StorageEngine, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from job_orchestration.scheduler.job_config import InputType from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CONTAINER_INPUT_LOGS_ROOT_DIR, dump_container_config, generate_container_config, diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 55c280e489..60fc994878 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -12,11 +12,11 @@ CLP_DB_USER_ENV_VAR_NAME, StorageEngine, StorageType, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.s3_utils import generate_container_auth_options from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, dump_container_config, generate_container_config, generate_container_name, diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index b7e9893487..e7ece78b31 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -12,11 +12,10 @@ CLP_DEFAULT_DATASET_NAME, CLPConfig, StorageEngine, - StorageType, + StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, DockerMount, DockerMountType, dump_container_config, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py index 79825dcac7..968660aecf 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, List, Optional -from clp_py_utils.clp_config import Database +from clp_py_utils.clp_config import Database, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH from clp_py_utils.clp_metadata_db_utils import ( delete_archives_from_metadata_db, get_archives_table_name, @@ -15,7 +15,6 @@ from clp_py_utils.sql_adapter import SQL_Adapter from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, get_clp_home, load_config_file, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py index 6f8411dfaa..0078467b88 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py @@ -11,7 +11,7 @@ import msgpack from clp_py_utils.clp_config import ( CLPConfig, - COMPRESSION_JOBS_TABLE_NAME, + COMPRESSION_JOBS_TABLE_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.pretty_size import pretty_size from clp_py_utils.s3_utils import parse_s3_url @@ -29,7 +29,6 @@ ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CONTAINER_INPUT_LOGS_ROOT_DIR, get_clp_home, load_config_file, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py index 44ab9d2a10..c6896e00e8 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py @@ -6,7 +6,8 @@ from pathlib import Path from typing import Dict, List -from clp_py_utils.clp_config import ArchiveOutput, Database, S3Config, StorageType +from clp_py_utils.clp_config import ArchiveOutput, Database, S3Config, StorageType, \ + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH from clp_py_utils.clp_metadata_db_utils import ( delete_dataset_from_metadata_db, get_datasets_table_name, @@ -15,7 +16,6 @@ from clp_py_utils.sql_adapter import SQL_Adapter from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, get_clp_home, load_config_file, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py index 1418271c37..9c1212ff58 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py @@ -13,7 +13,7 @@ CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, CLPConfig, - Database, + Database, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.clp_metadata_db_utils import get_files_table_name from clp_py_utils.sql_adapter import SQL_Adapter @@ -25,7 +25,6 @@ ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, EXTRACT_FILE_CMD, EXTRACT_IR_CMD, EXTRACT_JSON_CMD, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index 7c3a02e7d5..326a067d88 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -12,14 +12,13 @@ import pymongo from clp_py_utils.clp_config import ( Database, - ResultsCache, + ResultsCache, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.sql_adapter import SQL_Adapter from job_orchestration.scheduler.constants import QueryJobStatus, QueryJobType from job_orchestration.scheduler.job_config import AggregationConfig, SearchJobConfig from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, get_clp_home, load_config_file, ) diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index eb5a2118b0..fccbdae114 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -10,11 +10,10 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_DATASET_NAME, StorageEngine, - StorageType, + StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_package_utils.general import ( - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, dump_container_config, generate_container_config, generate_container_name, From a3288ae22ffba0d24614a8337d230fc0da7ff7fa Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:51:37 -0400 Subject: [PATCH 045/408] reorganize imports --- .../clp_package_utils/scripts/archive_manager.py | 3 ++- .../clp_package_utils/scripts/compress.py | 3 ++- .../clp_package_utils/scripts/dataset_manager.py | 2 +- .../clp_package_utils/scripts/decompress.py | 3 ++- .../clp_package_utils/scripts/native/archive_manager.py | 2 +- .../clp_package_utils/scripts/native/compress.py | 5 +++-- .../clp_package_utils/scripts/native/dataset_manager.py | 9 +++++++-- .../clp_package_utils/scripts/native/decompress.py | 3 ++- .../clp_package_utils/scripts/native/search.py | 3 ++- .../clp_package_utils/scripts/search.py | 3 ++- components/clp-py-utils/clp_py_utils/clp_config.py | 3 +-- 11 files changed, 25 insertions(+), 14 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index 8706fe86c4..d137d90883 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -9,9 +9,10 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, StorageEngine, - StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + StorageType, ) from clp_package_utils.general import ( diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 9b33bb1995..010d4b7612 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -10,8 +10,9 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, - StorageEngine, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + StorageEngine, ) from job_orchestration.scheduler.job_config import InputType diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 60fc994878..7cd81766f1 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -10,9 +10,9 @@ ARCHIVE_MANAGER_ACTION_NAME, CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, StorageEngine, StorageType, - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ) from clp_py_utils.s3_utils import generate_container_auth_options diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index e7ece78b31..379316007b 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -9,10 +9,11 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, CLPConfig, StorageEngine, - StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + StorageType, ) from clp_package_utils.general import ( diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py index 968660aecf..e0349e5f2c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, List, Optional -from clp_py_utils.clp_config import Database, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, Database from clp_py_utils.clp_metadata_db_utils import ( delete_archives_from_metadata_db, get_archives_table_name, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py index 0078467b88..de4f8330a3 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py @@ -5,13 +5,14 @@ import sys import time from contextlib import closing -from typing import List, Optional, Union +from typing import List, Union import brotli import msgpack from clp_py_utils.clp_config import ( + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, - COMPRESSION_JOBS_TABLE_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + COMPRESSION_JOBS_TABLE_NAME, ) from clp_py_utils.pretty_size import pretty_size from clp_py_utils.s3_utils import parse_s3_url diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py index c6896e00e8..4ff30cd9fd 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py @@ -6,8 +6,13 @@ from pathlib import Path from typing import Dict, List -from clp_py_utils.clp_config import ArchiveOutput, Database, S3Config, StorageType, \ - CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH +from clp_py_utils.clp_config import ( + ArchiveOutput, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + Database, + S3Config, + StorageType, +) from clp_py_utils.clp_metadata_db_utils import ( delete_dataset_from_metadata_db, get_datasets_table_name, diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py index 9c1212ff58..300609439e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py @@ -12,8 +12,9 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLPConfig, - Database, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + Database, ) from clp_py_utils.clp_metadata_db_utils import get_files_table_name from clp_py_utils.sql_adapter import SQL_Adapter diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index 326a067d88..d08d484463 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -11,8 +11,9 @@ import msgpack import pymongo from clp_py_utils.clp_config import ( + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, Database, - ResultsCache, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + ResultsCache, ) from clp_py_utils.sql_adapter import SQL_Adapter from job_orchestration.scheduler.constants import QueryJobStatus, QueryJobType diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index fccbdae114..66d63ab2d2 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -8,9 +8,10 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, StorageEngine, - StorageType, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + StorageType, ) from clp_package_utils.general import ( diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 21706cd5f8..b16ffe5051 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,8 +1,7 @@ import os import pathlib -import socket from enum import auto -from typing import Dict, Literal, Optional, Union +from typing import Literal, Optional, Union from dotenv import dotenv_values from pydantic import BaseModel, PrivateAttr, root_validator, validator From 5b3677947ce97c891fb249b9b2565a133ce927c0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:53:52 -0400 Subject: [PATCH 046/408] fix: adjust volume path formatting in docker-compose --- tools/deployment/package/docker-compose.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index b76224a969..1bf99117f2 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -165,7 +165,8 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}:/var/log/compression_scheduler.log" + - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ +:/var/log/compression_scheduler.log" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" depends_on: @@ -197,7 +198,8 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_QUERY_SCHEDULER_LOGS_FILE_HOST:-./var/log/query_scheduler.log}:/var/log/query_scheduler.log" + - "${CLP_QUERY_SCHEDULER_LOGS_FILE_HOST:-./var/log/query_scheduler.log}:\ +/var/log/query_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -240,7 +242,8 @@ services: volumes: - "${CLP_ARCHIVE_STAGING_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:/var/log/compression_worker" + - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ +/var/log/compression_worker" - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" From 93882affdde9a19b882f6d5a500bbe13c03fd224 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 02:55:45 -0400 Subject: [PATCH 047/408] fix: correct typo in comment for scheduler healthcheck in docker-compose --- tools/deployment/package/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 1bf99117f2..4364401614 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -215,7 +215,7 @@ services: ] healthcheck: <<: *healthcheck_defaults - # FIXME: need to suppressing warnings in the schduler for reading 0 out of 8 expected bytes + # FIXME: need to suppressing warnings in the scheduler for reading 0 out of 8 expected bytes test: [ "CMD", "bash", From a4d8e1f5701c45980795cea70971618594584cfd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:04:58 -0400 Subject: [PATCH 048/408] refactor: unify volume path definitions and remove redundant staging directory handling in container configuration --- .../clp_package_utils/controller.py | 10 ---------- tools/deployment/package/docker-compose.yml | 12 ++++++------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index a7e4aa6826..ba94d134b8 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -440,16 +440,6 @@ def _provision(self): if self.clp_config.aws_config_directory is not None: env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) - # Staging directories for S3 storage - if StorageType.S3 == self.clp_config.archive_output.storage.type: - env_dict["CLP_ARCHIVE_STAGING_DIR_HOST"] = str( - self.clp_config.archive_output.storage.staging_directory - ) - if StorageType.S3 == self.clp_config.stream_output.storage.type: - env_dict["CLP_STREAM_STAGING_DIR_HOST"] = str( - self.clp_config.stream_output.storage.staging_directory - ) - with open(f"{self.clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 4364401614..7b69a00585 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -240,12 +240,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_ARCHIVE_STAGING_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ /var/log/compression_worker" - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" command: [ @@ -279,11 +279,11 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_DATA_DIR_HOST:-./var/data}/archives:/var/data/archives" - - "${CLP_DATA_DIR_HOST:-./var/data}/streams:/var/data/streams" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_QUERY_WORKER_LOGS_DIR:-./var/log/query_worker}:/var/log/query_worker" - - "${CLP_STREAM_STAGING_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" command: [ "python3", "-u", @@ -338,7 +338,7 @@ services: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_DATA_DIR_HOST:-./var/data}/streams:/var/data/streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - "./var/www/webui/server/dist/settings.json\ :/opt/clp/var/www/webui/server/dist/settings.json:ro" From 63e6d72da77112b23e51bcca88c276df1fb6e220 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:05:37 -0400 Subject: [PATCH 049/408] remove comment --- components/clp-package-utils/clp_package_utils/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ba94d134b8..12716c8afa 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -436,7 +436,6 @@ def _provision(self): **self.provision_garbage_collector(), } - # AWS config directory if self.clp_config.aws_config_directory is not None: env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) From 0531a7bd2c533e103680b29fad68e125ab2a0e13 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:10:50 -0400 Subject: [PATCH 050/408] feat: implement `stop` method in `DockerComposeController` and update stop script to use it --- .../clp_package_utils/controller.py | 26 ++++++++++++++-- .../clp_package_utils/scripts/stop_clp.py | 30 +++++++++++-------- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 12716c8afa..55bcb32b0a 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -359,6 +359,13 @@ def deploy(self): """ pass + @abstractmethod + def stop(self): + """ + Stops the deployed components with orchestrator-specific logic. + """ + pass + @abstractmethod def _provision(self) -> Dict[str, str]: """ @@ -380,12 +387,11 @@ def _get_num_workers(): def __init__(self, clp_config: CLPConfig): super().__init__(clp_config) - check_docker_dependencies(should_compose_run=False) def deploy(self): + check_docker_dependencies(should_compose_run=False) self._provision() - # Start Docker Compose logger.info(f"Starting CLP using Docker Compose...") try: subprocess.run( @@ -398,6 +404,22 @@ def deploy(self): logger.exception("Failed to start CLP.") raise + def stop(self): + check_docker_dependencies(should_compose_run=True) + + logger.info("Stopping all CLP containers using Docker Compose...") + try: + subprocess.run( + ["docker", "compose", "down"], + cwd=self.clp_home, + stderr=subprocess.STDOUT, + check=True, + ) + logger.info("All CLP containers stopped.") + except subprocess.CalledProcessError: + logger.exception("Failed to stop CLP containers using Docker Compose.") + raise + def _provision(self): # Create necessary directories self.clp_config.data_directory.mkdir(parents=True, exist_ok=True) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 7cabe58ecc..80bdbeee10 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -1,29 +1,33 @@ import logging -import subprocess import sys -from clp_package_utils.general import check_docker_dependencies +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + +from clp_package_utils.controller import DockerComposeController +from clp_package_utils.general import ( + get_clp_home, + load_config_file, +) logger = logging.getLogger(__file__) def main(): + clp_home = get_clp_home() + default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + + # Load config file try: - check_docker_dependencies(should_compose_run=True) + clp_config = load_config_file(default_config_file_path, default_config_file_path, clp_home) except: - logger.exception("Dependency checking failed.") + logger.exception("Failed to load config.") return -1 - logger.info("Stopping all CLP containers using Docker Compose...") try: - subprocess.run( - ["docker", "compose", "down"], - stderr=subprocess.STDOUT, - check=True, - ) - logger.info("All CLP containers stopped.") - except subprocess.CalledProcessError: - logger.exception("Failed to stop CLP containers using Docker Compose.") + controller = DockerComposeController(clp_config) + controller.stop() + except: + logger.exception("Failed to stop CLP.") return -1 return 0 From f93aec76c43636bb23537b6d3c266775d12f9052 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:13:10 -0400 Subject: [PATCH 051/408] refactor: reorder volume definitions in docker-compose for consistency and clarity --- tools/deployment/package/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yml index 7b69a00585..fe42dd968f 100644 --- a/tools/deployment/package/docker-compose.yml +++ b/tools/deployment/package/docker-compose.yml @@ -240,12 +240,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ /var/log/compression_worker" - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" command: [ @@ -278,12 +278,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_QUERY_WORKER_LOGS_DIR:-./var/log/query_worker}:/var/log/query_worker" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" command: [ "python3", "-u", From 6a206285c1127d9b4b277ea041c225cd4e9a49d5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:14:38 -0400 Subject: [PATCH 052/408] revert error message in start_clp.py --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 62a8aef324..36ae82de93 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -63,9 +63,9 @@ def main(argv): controller.deploy() except Exception as ex: if type(ex) == ValueError: - logger.error(f"Failed to initialize CLP: {ex}") + logger.error(f"Failed to start CLP: {ex}") else: - logger.exception("Failed to initialize CLP.") + logger.exception("Failed to start CLP.") return -1 return 0 From 09356586078aae503da290f6a5f6a5dfb395c4fe Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:15:35 -0400 Subject: [PATCH 053/408] refactor: standardize logger messages from "Initializing" to "Provisioning" in `controller.py` --- .../clp_package_utils/controller.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 55bcb32b0a..c7c6a52ba6 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -67,7 +67,7 @@ def __init__(self, clp_config: CLPConfig): def provision_database(self): component_name = DB_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") conf_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name @@ -92,7 +92,7 @@ def provision_database(self): def provision_queue(self): component_name = QUEUE_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name validate_queue_config(self.clp_config, logs_dir) @@ -108,7 +108,7 @@ def provision_queue(self): def provision_redis(self): component_name = REDIS_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" logs_dir = self.clp_config.logs_directory / component_name @@ -132,7 +132,7 @@ def provision_redis(self): def provision_results_cache(self): component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") conf_file = self._conf_dir / "mongo" / "mongod.conf" data_dir = self.clp_config.data_directory / component_name @@ -153,7 +153,7 @@ def provision_results_cache(self): def provision_compression_scheduler(self): component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) @@ -165,7 +165,7 @@ def provision_compression_scheduler(self): def provision_query_scheduler(self): component_name = QUERY_SCHEDULER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) @@ -177,7 +177,7 @@ def provision_query_scheduler(self): def provision_compression_worker(self, num_workers: int): component_name = COMPRESSION_WORKER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -190,7 +190,7 @@ def provision_compression_worker(self, num_workers: int): def provision_query_worker(self, num_workers: int): component_name = QUERY_WORKER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -203,7 +203,7 @@ def provision_query_worker(self, num_workers: int): def provision_reducer(self, num_workers: int): component_name = REDUCER_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -257,7 +257,7 @@ def _read_and_update_settings_json( def provision_webui(self, container_clp_config: CLPConfig): component_name = WEBUI_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( @@ -345,7 +345,7 @@ def provision_webui(self, container_clp_config: CLPConfig): def provision_garbage_collector(self): component_name = GARBAGE_COLLECTOR_COMPONENT_NAME - logger.info(f"Initializing {component_name}...") + logger.info(f"Provisioning {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) From ad6192a4acab284132151a6be35729fe33fb7163 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 03:25:26 -0400 Subject: [PATCH 054/408] remove comment --- .../clp-package-utils/clp_package_utils/scripts/stop_clp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 80bdbeee10..f0d54e6de5 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -16,7 +16,6 @@ def main(): clp_home = get_clp_home() default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH - # Load config file try: clp_config = load_config_file(default_config_file_path, default_config_file_path, clp_home) except: From 9469db4ec4804c3f19df1f104e7d00a4f5c62f75 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 04:24:32 -0400 Subject: [PATCH 055/408] docs: update multi-node deployment guide and add Docker Compose design instructions --- docs/src/dev-docs/building-package.md | 5 + docs/src/dev-docs/design-docker-compose.md | 101 +++++++++++++++ docs/src/dev-docs/index.md | 1 + docs/src/user-docs/guides-multi-node.md | 141 +-------------------- docs/src/user-docs/guides-overview.md | 1 - 5 files changed, 111 insertions(+), 138 deletions(-) create mode 100644 docs/src/dev-docs/design-docker-compose.md diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index b212b5d3c1..0c84f1c758 100644 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -78,5 +78,10 @@ task docker-images:package This will create a Docker image named `clp-package:dev`. +The package includes a `docker-compose.yml` file that can be used to deploy CLP using Docker Compose. +If you want to manually deploy with Docker Compose instead of using the package scripts, see the +[Docker Compose design][docker-compose-design] for more information. + [clp-issue-872]: https://github.com/y-scope/clp/issues/872 +[docker-compose-design]: ../dev-docs/design-docker-compose.md [Task]: https://taskfile.dev/ diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md new file mode 100644 index 0000000000..097559c74d --- /dev/null +++ b/docs/src/dev-docs/design-docker-compose.md @@ -0,0 +1,101 @@ +# Docker Compose design + +This document explains the technical details of CLP's Docker Compose implementation. + +## Overview + +The Docker Compose implementation depends on a new controller architecture with a `BaseController` +abstract class and a `DockerComposeController` implementation. + +## Architecture + +### Controller + +The implementation uses a controller pattern: + +* `BaseController` (abstract): Defines the interface for provisioning and managing CLP components. +* `DockerComposeController`: Implements the Docker Compose-specific logic. + +### Key Components + +1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the controller: + * `provision_database()` + * `provision_queue()` + * `provision_redis()` + * `provision_results_cache()` + * `provision_compression_scheduler()` + * `provision_query_scheduler()` + * `provision_compression_worker()` + * `provision_query_worker()` + * `provision_reducer()` + * `provision_webui()` + * `provision_garbage_collector()` + +2. **Environment Generation**: The controller generates a `.env` file with all necessary environment + variables for Docker Compose. + +3. **Configuration Transformation**: The `transform_for_container_config()` method in `CLPConfig` + and related classes adapts the configuration for containerized environments. + +## Docker Compose File + +The `docker-compose.yml` file defines all services with: + +* Proper service dependencies using `depends_on` +* Health checks for critical services +* Volume mounts for persistent data +* Network configuration +* User permissions +* Resource limits + +## Deployment Process + +1. **Configuration Loading**: The start script loads and validates the CLP configuration. +2. **Provisioning**: The controller provisions all components and generates environment variables. +3. **Environment File Generation**: A `.env` file is created with all necessary variables. +4. **Docker Compose Execution**: `docker compose up -d` is executed to start all services. + +## Service Architecture + +The Docker Compose setup includes the following services: + +* **database**: MySQL/MariaDB for metadata storage +* **queue**: RabbitMQ for job queuing +* **redis**: Redis for task result storage +* **results-cache**: MongoDB for search results caching +* **compression-scheduler**: Schedules compression jobs +* **query-scheduler**: Schedules search jobs +* **compression-worker**: Executes compression tasks +* **query-worker**: Executes search tasks +* **reducer**: Handles aggregation operations +* **webui**: Web interface for CLP +* **garbage-collector**: Manages retention policies +* **db-table-creator**: Initializes database tables +* **results-cache-indices-creator**: Sets up MongoDB indices + +## Service Dependencies + +Docker Compose manages service startup order through: + +* `depends_on` directives. +* Health checks with `condition: service_healthy`. +* Init containers for one-time setup tasks (e.g., `db-table-creator`). + +## Troubleshooting + +If you encounter issues with the Docker Compose deployment: + +1. Check service status: + ```bash + docker compose ps + ``` + +2. View service logs: + ```bash + docker compose logs + ``` + +3. Validate configuration: + ```bash + docker compose config + ``` diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md index 5372121755..e46c339e21 100644 --- a/docs/src/dev-docs/index.md +++ b/docs/src/dev-docs/index.md @@ -79,6 +79,7 @@ tooling-gh-workflows :hidden: design-project-structure +design-docker-compose design-kv-ir-streams/index design-metadata-db diff --git a/docs/src/user-docs/guides-multi-node.md b/docs/src/user-docs/guides-multi-node.md index 44078605ad..9a56f4c631 100644 --- a/docs/src/user-docs/guides-multi-node.md +++ b/docs/src/user-docs/guides-multi-node.md @@ -2,145 +2,12 @@ A multi-node deployment allows you to run CLP across a distributed set of hosts. -## Requirements - -* [Docker] - * If you're not running as root, ensure docker can be run - [without superuser privileges][docker-non-root]. -* Python 3.9 or higher -* One or more hosts networked together -* A distributed filesystem (e.g. [SeaweedFS]) accessible by all worker hosts through a filesystem - mount - * See [below](#setting-up-seaweedfs) for how to set up a simple SeaweedFS cluster. - -## Cluster overview - -The CLP package is composed of several components--controller components and worker components. In a -cluster, there should be a single instance of each controller component and one or more instances of -worker components. The tables below list the components and their functions. - -:::{table} Controller components -:align: left - -| Component | Description | -|-----------------------|-----------------------------------------------------------------| -| database | Database for archive metadata, compression jobs, and query jobs | -| queue | Task queue for schedulers | -| redis | Task result storage for workers | -| compression_scheduler | Scheduler for compression jobs | -| query_scheduler | Scheduler for search/aggregation jobs | -| results_cache | Storage for the workers to return search results to the UI | -| webui | Web server for the UI | -| garbage_collector | Background process for retention control | -::: - -:::{table} Worker components -:align: left - -| Component | Description | -|--------------------|--------------------------------------------------------------| -| compression_worker | Worker processes for compression jobs | -| query_worker | Worker processes for search/aggregation jobs | -| reducer | Reducers for performing the final stages of aggregation jobs | -::: - -:::{note} -Running additional workers increases the parallelism of compression and search/aggregation jobs. +:::{warning} +CLP now uses Docker Compose for orchestration and support for multi-node deployments is +temporarily removed. Please contact us if you need immediate support for multi-node deployments, or +stay tuned for future updates on Kubernetes Helm support. ::: -## Configuring CLP - -1. Copy `etc/credentials.template.yml` to `etc/credentials.yml`. -2. Edit `etc/credentials.yml`: - - {style=lower-alpha} - 1. Uncomment the file. - 2. Choose an appropriate username and password. - * Note that these are *new* credentials that will be used by the components. - -3. Choose which hosts you would like to use for the controller components. - * You can use a single host for all controller components. -4. Edit `etc/clp-config.yml`: - - {style=lower-alpha} - 1. Uncomment the file. - 2. Set the `host` config of each controller component to the host that you'd like to run them - on. - * If desired, you can run different controller components on different hosts. - 3. Change any of the controller components' ports that will conflict with services you already - have running. - 4. Set `archive_output.directory` to a directory on the distributed filesystem. - * Ideally, the directory should be empty or should not yet exist (CLP will create it) since - CLP will write several files and directories directly to the given directory. - -5. Download and extract the package on all nodes. -6. Copy the `credentials.yml` and `clp-config.yml` files that you created above and paste them - into `etc` on all the hosts where you extracted the package. - -## Starting CLP - -Before starting each CLP component, note that some components must be started before others. We -organize the components into groups below, where components in a group can be started in any order, -but all components in a group must be started before starting a component in the next group. - -**Group 1 components:** - -* `database` -* `queue` -* `redis` -* `results_cache` - -**Group 2 components:** - -* `compression_scheduler` -* `query_scheduler` -* `garbage_collector` - -**Group 3 components:** - -* `compression_worker` -* `query_worker` -* `reducer` - -For each component, on the host where you want to run the component, run: - -```bash -sbin/start-clp.sh -``` - -Where `` is the name of the component in the groups above. - -## Using CLP - -To learn how to compress and search your logs, check out the quick-start guide that corresponds to -the flavor of CLP you're running: - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: quick-start/clp-json -Using clp-json -^^^ -How to compress and search JSON logs. -::: - -:::{grid-item-card} -:link: quick-start/clp-text -Using clp-text -^^^ -How to compress and search unstructured text logs. -::: -:::: - -## Stopping CLP - -If you need to stop the cluster, run: - -```bash -sbin/stop-clp.sh -``` - ## Setting up SeaweedFS The instructions below are for running a simple SeaweedFS cluster on a set of hosts. For other use diff --git a/docs/src/user-docs/guides-overview.md b/docs/src/user-docs/guides-overview.md index 60d7bbacd1..62f01a9835 100644 --- a/docs/src/user-docs/guides-overview.md +++ b/docs/src/user-docs/guides-overview.md @@ -32,4 +32,3 @@ Using Presto with CLP ^^^ How to use Presto to query compressed logs in CLP. ::: -:::: From 110e9fa2f05756de7839412e9b39f6fc986517af Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 04:27:47 -0400 Subject: [PATCH 056/408] docs: refine Docker Compose design doc for clarity and consistency --- docs/src/dev-docs/design-docker-compose.md | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 097559c74d..1a3261c20d 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -18,22 +18,10 @@ The implementation uses a controller pattern: ### Key Components -1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the controller: - * `provision_database()` - * `provision_queue()` - * `provision_redis()` - * `provision_results_cache()` - * `provision_compression_scheduler()` - * `provision_query_scheduler()` - * `provision_compression_worker()` - * `provision_query_worker()` - * `provision_reducer()` - * `provision_webui()` - * `provision_garbage_collector()` - +1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the + controller: `provision_()`. 2. **Environment Generation**: The controller generates a `.env` file with all necessary environment variables for Docker Compose. - 3. **Configuration Transformation**: The `transform_for_container_config()` method in `CLPConfig` and related classes adapts the configuration for containerized environments. @@ -50,6 +38,8 @@ The `docker-compose.yml` file defines all services with: ## Deployment Process +The `start-clp.py` script performs the following steps: + 1. **Configuration Loading**: The start script loads and validates the CLP configuration. 2. **Provisioning**: The controller provisions all components and generates environment variables. 3. **Environment File Generation**: A `.env` file is created with all necessary variables. @@ -59,6 +49,8 @@ The `docker-compose.yml` file defines all services with: The Docker Compose setup includes the following services: +### Services + * **database**: MySQL/MariaDB for metadata storage * **queue**: RabbitMQ for job queuing * **redis**: Redis for task result storage @@ -70,6 +62,8 @@ The Docker Compose setup includes the following services: * **reducer**: Handles aggregation operations * **webui**: Web interface for CLP * **garbage-collector**: Manages retention policies + +### One-time initialization jobs * **db-table-creator**: Initializes database tables * **results-cache-indices-creator**: Sets up MongoDB indices From 045dde623204f3bb88646d03bfe9207431d80c40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 04:40:55 -0400 Subject: [PATCH 057/408] refactor: update CLP container configuration to use execution container from config --- .../clp_package_utils/controller.py | 2 +- components/clp-py-utils/clp_py_utils/clp_config.py | 14 ++------------ components/package-template/src/etc/clp-config.yml | 3 +++ 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index c7c6a52ba6..9b4750c471 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -436,7 +436,7 @@ def _provision(self): "CLP_USER_ID": str(os.getuid()), "CLP_GROUP_ID": str(os.getgid()), # Package container - "CLP_PACKAGE_CONTAINER": "clp-package:dev", + "CLP_PACKAGE_CONTAINER": self.clp_config.execution_container, # Global paths "CLP_DATA_DIR_HOST": str(self.clp_config.data_directory), "CLP_LOGS_DIR_HOST": str(self.clp_config.logs_directory), diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index b16ffe5051..d6d3439fbd 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -917,20 +917,10 @@ def validate_aws_config_dir(self): def load_execution_container_name(self): if self.execution_container is not None: - # Accept configured value for debug purposes + # Accept configured value for releases return - os_release = dotenv_values(self._os_release_file_path) - if "ubuntu" == os_release["ID"]: - self.execution_container = ( - f"clp-execution-x86-{os_release['ID']}-{os_release['VERSION_CODENAME']}:main" - ) - else: - raise NotImplementedError( - f"Unsupported OS {os_release['ID']} in {OS_RELEASE_FILE_PATH}" - ) - - self.execution_container = "ghcr.io/y-scope/clp/" + self.execution_container + self.execution_container = "clp-package:dev" def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index 8e96b179b8..d67d4f3a56 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -1,3 +1,6 @@ +## Docker image to use for CLP package execution. +#execution_container: "clp-package:dev" +# ## Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all ## workers. #logs_input: From fa87d9281d9766e0782e6770b7ea0693590aac87 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 12:36:18 -0400 Subject: [PATCH 058/408] refactor(controller): move directory creation logic from `_provision` to `start_clp` script. --- .../clp-package-utils/clp_package_utils/controller.py | 6 ------ .../clp_package_utils/scripts/start_clp.py | 9 ++++++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 9b4750c471..19d4e2ad69 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -421,12 +421,6 @@ def stop(self): raise def _provision(self): - # Create necessary directories - self.clp_config.data_directory.mkdir(parents=True, exist_ok=True) - self.clp_config.logs_directory.mkdir(parents=True, exist_ok=True) - self.clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - self.clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) - container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 36ae82de93..b4e4a3449c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -51,13 +51,16 @@ def main(argv): clp_config.validate_data_dir() clp_config.validate_logs_dir() clp_config.validate_aws_config_dir() + + # Create necessary directories + clp_config.data_directory.mkdir(parents=True, exist_ok=True) + clp_config.logs_directory.mkdir(parents=True, exist_ok=True) + clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) + clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) except: logger.exception("Failed to load config.") return -1 - container_clp_config = generate_docker_compose_container_config(clp_config) - dump_shared_container_config(container_clp_config, clp_config) - try: controller = DockerComposeController(clp_config) controller.deploy() From b6ac2c6561b91b1324970dd77f4bbce250223574 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 12:40:41 -0400 Subject: [PATCH 059/408] feat(controller): add function to dump shared container configuration in `_provision`. --- components/clp-package-utils/clp_package_utils/controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 19d4e2ad69..956a930928 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -41,7 +41,7 @@ validate_queue_config, validate_redis_config, validate_results_cache_config, - validate_webui_config, + validate_webui_config, dump_shared_container_config, ) LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH @@ -423,6 +423,7 @@ def stop(self): def _provision(self): container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() + dump_shared_container_config(container_clp_config, self.clp_config) env_dict = { "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, From bfd8c7ba6683f1d5b7c2742740159efcffd1ebed Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 13:13:14 -0400 Subject: [PATCH 060/408] lint --- components/clp-package-utils/clp_package_utils/controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 956a930928..d0f78f7bd9 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -35,13 +35,14 @@ from clp_package_utils.general import ( check_docker_dependencies, CONTAINER_CLP_HOME, + dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, validate_db_config, validate_queue_config, validate_redis_config, validate_results_cache_config, - validate_webui_config, dump_shared_container_config, + validate_webui_config, ) LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH From 2b7959bf45551d076a36d37a43516411a2795bab Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 13:13:27 -0400 Subject: [PATCH 061/408] refactor(clp-py-utils): update staging directory handling in S3 storage classes. --- components/clp-py-utils/clp_py_utils/clp_config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index d6d3439fbd..846b8a9b72 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -617,14 +617,17 @@ def transform_for_container_config(self): class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH def transform_for_container_config(self): self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + + def transform_for_container_config(self): + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( From f9eb88c8a94faf66cba954f2bfd9a5376c7fd566 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 16:05:11 -0400 Subject: [PATCH 062/408] docs(design): Enhance Docker Compose design doc with diagrams and detailed service tables. --- docs/src/dev-docs/design-docker-compose.md | 118 +++++++++++++++++---- 1 file changed, 96 insertions(+), 22 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 1a3261c20d..2c9e25b516 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -11,12 +11,12 @@ abstract class and a `DockerComposeController` implementation. ### Controller -The implementation uses a controller pattern: +The orchestration implementation uses a controller pattern: * `BaseController` (abstract): Defines the interface for provisioning and managing CLP components. * `DockerComposeController`: Implements the Docker Compose-specific logic. -### Key Components +### Initialization 1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the controller: `provision_()`. @@ -49,31 +49,105 @@ The `start-clp.py` script performs the following steps: The Docker Compose setup includes the following services: -### Services - -* **database**: MySQL/MariaDB for metadata storage -* **queue**: RabbitMQ for job queuing -* **redis**: Redis for task result storage -* **results-cache**: MongoDB for search results caching -* **compression-scheduler**: Schedules compression jobs -* **query-scheduler**: Schedules search jobs -* **compression-worker**: Executes compression tasks -* **query-worker**: Executes search tasks -* **reducer**: Handles aggregation operations -* **webui**: Web interface for CLP -* **garbage-collector**: Manages retention policies +:::{mermaid} +graph LR + %% Services + db["db (MySQL)"] + queue["queue (RabbitMQ)"] + redis["redis (Redis)"] + results_cache["results-cache (MongoDB)"] + compression_scheduler["compression-scheduler"] + query_scheduler["query-scheduler"] + compression_worker["compression-worker"] + query_worker["query-worker"] + reducer["reducer"] + webui["webui"] + garbage_collector["garbage-collector"] + + %% One-time jobs + db_table_creator["db-table-creator"] + results_cache_indices_creator["results-cache-indices-creator"] + + %% Dependencies + db -->|healthy| db_table_creator + results_cache -->|healthy| results_cache_indices_creator + db_table_creator -->|completed_successfully| compression_scheduler + queue -->|healthy| compression_scheduler + redis -->|healthy| compression_scheduler + db_table_creator -->|completed_successfully| query_scheduler + queue -->|healthy| query_scheduler + redis -->|healthy| query_scheduler + query_scheduler -->|healthy| reducer + results_cache_indices_creator -->|completed_successfully| reducer + db_table_creator -->|completed_successfully| webui + results_cache_indices_creator -->|completed_successfully| webui + db_table_creator -->|completed_successfully| garbage_collector + query_scheduler -->|healthy| garbage_collector + results_cache_indices_creator -->|completed_successfully| garbage_collector + + subgraph Databases + db + queue + redis + results_cache + end + + subgraph DB Migration Jobs + db_table_creator + results_cache_indices_creator + end + + subgraph Schedulers + compression_scheduler + query_scheduler + end + + subgraph Workers + compression_worker + query_worker + reducer + end + + subgraph UI & Management + webui + garbage_collector + end +::: + +### Services overview + +The CLP package is composed of several service components. The tables below list the services and their functions. + +:::{table} Services +:align: left + +| Service | Description | +|-----------------------|-----------------------------------------------------------------| +| database | Database for archive metadata, compression jobs, and query jobs | +| queue | Task queue for schedulers | +| redis | Task result storage for workers | +| compression_scheduler | Scheduler for compression jobs | +| query_scheduler | Scheduler for search/aggregation jobs | +| results_cache | Storage for the workers to return search results to the UI | +| compression_worker | Worker processes for compression jobs | +| query_worker | Worker processes for search/aggregation jobs | +| reducer | Reducers for performing the final stages of aggregation jobs | +| webui | Web server for the UI | +| garbage_collector | Background process for retention control | +::: ### One-time initialization jobs -* **db-table-creator**: Initializes database tables -* **results-cache-indices-creator**: Sets up MongoDB indices -## Service Dependencies +We also set up short-lived run-once "services" to initialize some services listed above. -Docker Compose manages service startup order through: +:::{table} Initialization jobs +:align: left -* `depends_on` directives. -* Health checks with `condition: service_healthy`. -* Init containers for one-time setup tasks (e.g., `db-table-creator`). +| Job | Description | +|-------------------------------|---------------------------------------------------------| +| db-table-creator | Initializes database tables | +| results-cache-indices-creator | Initializes single-node replica set and sets up indices | +::: ## Troubleshooting From 0f5bd45bc8f0c3023f74ffdba83ec85e5470cb39 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 16:56:58 -0400 Subject: [PATCH 063/408] fix title case --- docs/src/dev-docs/design-docker-compose.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 2c9e25b516..10c17edd28 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -45,7 +45,7 @@ The `start-clp.py` script performs the following steps: 3. **Environment File Generation**: A `.env` file is created with all necessary variables. 4. **Docker Compose Execution**: `docker compose up -d` is executed to start all services. -## Service Architecture +## Service architecture The Docker Compose setup includes the following services: From 065692880f2583cae41bd28d9a85a1bfca97e57a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 4 Sep 2025 17:05:40 -0400 Subject: [PATCH 064/408] Update references to docker-compose.yml to docker-compose.yaml. --- docs/src/dev-docs/building-package.md | 2 +- docs/src/dev-docs/design-docker-compose.md | 2 +- taskfile.yaml | 4 ++-- .../package/{docker-compose.yml => docker-compose.yaml} | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename tools/deployment/package/{docker-compose.yml => docker-compose.yaml} (100%) diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index 0c84f1c758..c1422c00c9 100644 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -78,7 +78,7 @@ task docker-images:package This will create a Docker image named `clp-package:dev`. -The package includes a `docker-compose.yml` file that can be used to deploy CLP using Docker Compose. +The package includes a `docker-compose.yaml` file that can be used to deploy CLP using Docker Compose. If you want to manually deploy with Docker Compose instead of using the package scripts, see the [Docker Compose design][docker-compose-design] for more information. diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 10c17edd28..d845a242ae 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -27,7 +27,7 @@ The orchestration implementation uses a controller pattern: ## Docker Compose File -The `docker-compose.yml` file defines all services with: +The `docker-compose.yaml` file defines all services with: * Proper service dependencies using `depends_on` * Health checks for critical services diff --git a/taskfile.yaml b/taskfile.yaml index 90a0e886d8..933b8012dc 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -113,7 +113,7 @@ tasks: - "components/clp-py-utils/dist/*.whl" - "components/job-orchestration/dist/*.whl" - "components/package-template/src/**/*" - - "tools/deployment/package/docker-compose.yml" + - "tools/deployment/package/docker-compose.yaml" generates: ["{{.CHECKSUM_FILE}}"] deps: - "core" @@ -163,7 +163,7 @@ tasks: PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm ci --omit=dev - >- rsync -a - "tools/deployment/package/docker-compose.yml" + "tools/deployment/package/docker-compose.yaml" "{{.OUTPUT_DIR}}" # This command must be last - task: "utils:checksum:compute" diff --git a/tools/deployment/package/docker-compose.yml b/tools/deployment/package/docker-compose.yaml similarity index 100% rename from tools/deployment/package/docker-compose.yml rename to tools/deployment/package/docker-compose.yaml From 66dcbb21c4c5feff6841211611d6f96cad457c67 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Sep 2025 13:28:42 -0400 Subject: [PATCH 065/408] feat(docker): add project name to docker-compose and enhance running check --- .../clp-package-utils/clp_package_utils/general.py | 14 ++++++++++---- tools/deployment/package/docker-compose.yaml | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 47d0472531..7c83983e5e 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -1,5 +1,6 @@ import enum import errno +import json import os import pathlib import re @@ -130,11 +131,16 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def is_docker_compose_running(): - cmd = ["docker", "compose", "ls", "--quiet"] +def is_docker_compose_running(project_name: str) -> bool: + cmd = [ + "docker", "compose", "ls", + "--format", "json", + "--filter", f"name={project_name}" + ] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - return bool(output.strip()) + running_instances = json.loads(output) + return len(running_instances) >= 1 except subprocess.CalledProcessError: raise EnvironmentError("docker-compose is not installed or not functioning properly.") @@ -151,7 +157,7 @@ def check_docker_dependencies(should_compose_run: bool = False): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = is_docker_compose_running() + is_running = is_docker_compose_running("clp-package") if should_compose_run and not is_running: raise EnvironmentError("docker-compose is not running.") if not should_compose_run and is_running: diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index fe42dd968f..ca6e6d7bf1 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -14,6 +14,8 @@ networks: clp-network: driver: "bridge" +name: clp-package + secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" From 09ef298ed730fc1f8b4a0f3624eaab360eff9617 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Sep 2025 13:30:28 -0400 Subject: [PATCH 066/408] fix lint --- components/clp-package-utils/clp_package_utils/general.py | 6 +----- tools/deployment/package/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 7c83983e5e..f8679b6919 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -132,11 +132,7 @@ def generate_container_name(job_type: str) -> str: def is_docker_compose_running(project_name: str) -> bool: - cmd = [ - "docker", "compose", "ls", - "--format", "json", - "--filter", f"name={project_name}" - ] + cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) running_instances = json.loads(output) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index ca6e6d7bf1..ecd14a1799 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -14,7 +14,7 @@ networks: clp-network: driver: "bridge" -name: clp-package +name: "clp-package" secrets: CLP_DB_PASS_FILE: From d3b6a679f75667c79a889c92e8c68e768983842e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Sep 2025 14:35:12 -0400 Subject: [PATCH 067/408] fix(taskfile): update default task to `docker-images:package` --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index 933b8012dc..984b6de348 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -48,7 +48,7 @@ vars: tasks: default: - deps: ["package"] + deps: ["docker-images:package"] clean: cmds: From 6148a65fdc001605bb77c5aa62442ba0f0807e2d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 05:19:52 -0400 Subject: [PATCH 068/408] feat: reset default ports for container configs. --- .../clp-py-utils/clp_py_utils/clp_config.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 846b8a9b72..54548c9873 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Literal, Optional, Union +from typing import ClassVar, Literal, Optional, Union from dotenv import dotenv_values from pydantic import BaseModel, PrivateAttr, root_validator, validator @@ -136,9 +136,11 @@ def validate_query_engine_package_compatibility(cls, values): class Database(BaseModel): + DEFAULT_PORT: ClassVar[int] = 3306 + type: str = "mariadb" host: str = "localhost" - port: int = 3306 + port: int = DEFAULT_PORT name: str = "clp-db" ssl_cert: Optional[str] = None auto_commit: bool = False @@ -240,6 +242,7 @@ def load_credentials_from_env(self): def transform_for_container_config(self): self.host = DB_COMPONENT_NAME + self.port = self.DEFAULT_PORT def _validate_logging_level(cls, field): @@ -275,6 +278,8 @@ def validate_logging_level(cls, field): class QueryScheduler(BaseModel): + DEFAULT_PORT: ClassVar[int] = 7000 + host: str = "localhost" port: int = 7000 jobs_poll_delay: float = 0.1 # seconds @@ -300,6 +305,7 @@ def validate_port(cls, field): def transform_for_container_config(self): self.host = QUERY_SCHEDULER_COMPONENT_NAME + self.port = self.DEFAULT_PORT class CompressionWorker(BaseModel): @@ -321,8 +327,10 @@ def validate_logging_level(cls, field): class Redis(BaseModel): + DEFAULT_PORT: ClassVar[int] = 6379 + host: str = "localhost" - port: int = 6379 + port: int = DEFAULT_PORT query_backend_database: int = 0 compression_backend_database: int = 1 # redis can perform authentication without a username @@ -356,11 +364,14 @@ def load_credentials_from_env(self): def transform_for_container_config(self): self.host = REDIS_COMPONENT_NAME + self.port = self.DEFAULT_PORT class Reducer(BaseModel): + DEFAULT_PORT: ClassVar[int] = 14009 + host: str = "localhost" - base_port: int = 14009 + base_port: int = DEFAULT_PORT logging_level: str = "INFO" upsert_interval: int = 100 # milliseconds @@ -389,11 +400,14 @@ def validate_upsert_interval(cls, field): def transform_for_container_config(self): self.host = REDUCER_COMPONENT_NAME + self.base_port = self.DEFAULT_PORT class ResultsCache(BaseModel): + DEFAULT_PORT: ClassVar[int] = 27017 + host: str = "localhost" - port: int = 27017 + port: int = DEFAULT_PORT db_name: str = "clp-query-results" stream_collection_name: str = "stream-files" retention_period: Optional[int] = 60 @@ -429,11 +443,14 @@ def get_uri(self): def transform_for_container_config(self): self.host = RESULTS_CACHE_COMPONENT_NAME + self.port = self.DEFAULT_PORT class Queue(BaseModel): + DEFAULT_PORT: ClassVar[int] = 5672 + host: str = "localhost" - port: int = 5672 + port: int = DEFAULT_PORT username: Optional[str] = None password: Optional[str] = None @@ -462,6 +479,7 @@ def load_credentials_from_env(self): def transform_for_container_config(self): self.host = QUEUE_COMPONENT_NAME + self.port = self.DEFAULT_PORT class S3Credentials(BaseModel): From 0ab99f0d0a525220f44c4229339d64c58d16b04d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 05:26:41 -0400 Subject: [PATCH 069/408] fix(docker): update MongoDB connection string to use internal port --- tools/deployment/package/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index ecd14a1799..adba39dd42 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -129,7 +129,7 @@ services: <<: *healthcheck_defaults test: >- echo 'db.runCommand("ping").ok' | - mongosh 127.0.0.1:${CLP_RESULTS_CACHE_PORT:-27017}/test --quiet + mongosh 127.0.0.1:27017/test --quiet command: [ "--config", "/etc/mongo/mongod.conf", "--bind_ip", "0.0.0.0", From 263be6f600589377d5bf5360ec745bf243be2d48 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 05:26:57 -0400 Subject: [PATCH 070/408] fix(controller): update webui configuration to use container-specific ports --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index d0f78f7bd9..49fbe1e6f9 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -298,11 +298,11 @@ def provision_webui(self, container_clp_config: CLPConfig): server_settings_json_updates = { "SqlDbHost": container_clp_config.database.host, - "SqlDbPort": self.clp_config.database.port, + "SqlDbPort": container_clp_config.database.port, "SqlDbName": self.clp_config.database.name, "SqlDbQueryJobsTableName": "query_jobs", "MongoDbHost": container_clp_config.results_cache.host, - "MongoDbPort": self.clp_config.results_cache.port, + "MongoDbPort": container_clp_config.results_cache.port, "MongoDbName": self.clp_config.results_cache.db_name, "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, "MongoDbStreamFilesCollectionName": self.clp_config.results_cache.stream_collection_name, From 51d1b55f5bca5fdd6704e0f62d443ea851662bc2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 18:09:17 -0400 Subject: [PATCH 071/408] add ownership management for data and logs directories when running as root --- .../clp_package_utils/controller.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 49fbe1e6f9..eab3b46883 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -46,10 +46,40 @@ ) LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH +DEFAULT_CONTAINER_USER_ID = 999 +DEFAULT_CONTAINER_GROUP_ID = 999 logger = logging.getLogger(__name__) +def _chown_recursively( + path: pathlib.Path, + user_id: int, + group_id: int, +): + """ + Recursively changes the owner of the given path to the given user ID and group ID. + :param path: + :param user_id: + :param group_id: + """ + chown_cmd = ["chown", "--recursive", f"{user_id}:{group_id}", str(path)] + subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) + + +def _chown_paths_if_root(*paths: pathlib.Path): + """ + Changes ownership of the given paths to the default container user/group IDs + if the current process is running as root. + + :param paths: + """ + if os.getuid() != 0: + return + for path in paths: + _chown_recursively(path, DEFAULT_CONTAINER_USER_ID, DEFAULT_CONTAINER_GROUP_ID) + + def _get_ip_from_hostname(hostname: str) -> str: """ Resolves a hostname to an IP address. @@ -76,6 +106,7 @@ def provision_database(self): validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(data_dir, logs_dir) return { "CLP_DB_CONF_FILE_HOST": str(conf_file), @@ -98,6 +129,7 @@ def provision_queue(self): logs_dir = self.clp_config.logs_directory / component_name validate_queue_config(self.clp_config, logs_dir) logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(logs_dir) return { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), @@ -117,6 +149,7 @@ def provision_redis(self): validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(data_dir, logs_dir) return { "CLP_REDIS_CONF_FILE_HOST": str(conf_file), @@ -141,6 +174,7 @@ def provision_results_cache(self): validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(data_dir, logs_dir) return { "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), @@ -459,4 +493,4 @@ def _provision(self): with open(f"{self.clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): - env_file.write(f"{key}={value}\n") + env_file.write(f"{key}={value}\n") \ No newline at end of file From 0066386245481ba59afeadf27a77baea15b62469 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 17 Sep 2025 19:09:10 -0400 Subject: [PATCH 072/408] fix(controller, docker-compose): update user and group ID handling for containers --- .../clp_package_utils/controller.py | 12 +++++++----- tools/deployment/package/docker-compose.yaml | 6 +++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index eab3b46883..904a290d76 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -46,8 +46,10 @@ ) LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH -DEFAULT_CONTAINER_USER_ID = 999 -DEFAULT_CONTAINER_GROUP_ID = 999 +DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" +SERVICE_CONTAINER_USER_ID = 999 +SERVICE_CONTAINER_GROUP_ID = 999 +SERVICE_CONTAINER_UID_GID = f"{SERVICE_CONTAINER_USER_ID}:{SERVICE_CONTAINER_GROUP_ID}" logger = logging.getLogger(__name__) @@ -77,7 +79,7 @@ def _chown_paths_if_root(*paths: pathlib.Path): if os.getuid() != 0: return for path in paths: - _chown_recursively(path, DEFAULT_CONTAINER_USER_ID, DEFAULT_CONTAINER_GROUP_ID) + _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) def _get_ip_from_hostname(hostname: str) -> str: @@ -463,8 +465,8 @@ def _provision(self): env_dict = { "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, # User and group IDs - "CLP_USER_ID": str(os.getuid()), - "CLP_GROUP_ID": str(os.getgid()), + "CLP_UID_GID": DEFAULT_UID_GID, + "CLP_SERVICE_CONTAINER_UID_GID": SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID, # Package container "CLP_PACKAGE_CONTAINER": self.clp_config.execution_container, # Global paths diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index adba39dd42..2e85855633 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -1,7 +1,7 @@ x-service-defaults: &service_defaults networks: ["clp-network"] stop_grace_period: "3s" - user: "${CLP_USER_ID:-1000}:${CLP_GROUP_ID:-1000}" + user: "${CLP_UID_GID:-1000:1000}" x-healthcheck-defaults: &healthcheck_defaults interval: "30s" @@ -25,6 +25,7 @@ services: <<: *service_defaults container_name: "database" image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" @@ -74,6 +75,7 @@ services: <<: *service_defaults container_name: "queue" image: "rabbitmq:3.9.8" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" @@ -93,6 +95,7 @@ services: <<: *service_defaults container_name: "redis" image: "redis:7.2.4" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" volumes: @@ -119,6 +122,7 @@ services: <<: *service_defaults container_name: "results_cache" image: "mongo:7.0.1" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: From bc9ab99fce30375409a5d88e77b2a2a8afcf2c9b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Sep 2025 21:29:59 -0400 Subject: [PATCH 073/408] fix(search): enable direct connection for MongoDB client in search.py --- .../clp_package_utils/scripts/native/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index d08d484463..60fd16d657 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -76,7 +76,7 @@ def create_and_monitor_job_in_db( if do_count_aggregation is None and count_by_time_bucket_size is None: return - with pymongo.MongoClient(results_cache.get_uri()) as client: + with pymongo.MongoClient(results_cache.get_uri(), directConnection=True) as client: search_results_collection = client[results_cache.db_name][str(job_id)] if do_count_aggregation is not None: for document in search_results_collection.find(): From 80081385bd51d3f9576478edaf52f2ca8331fab0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 20 Sep 2025 03:53:42 -0400 Subject: [PATCH 074/408] revert direct connection option for MongoDB client in search.py --- .../clp_package_utils/scripts/native/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py index 60fd16d657..d08d484463 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py @@ -76,7 +76,7 @@ def create_and_monitor_job_in_db( if do_count_aggregation is None and count_by_time_bucket_size is None: return - with pymongo.MongoClient(results_cache.get_uri(), directConnection=True) as client: + with pymongo.MongoClient(results_cache.get_uri()) as client: search_results_collection = client[results_cache.db_name][str(job_id)] if do_count_aggregation is not None: for document in search_results_collection.find(): From 34b60bd96ee340371c92eef46aae8edbd51e4350 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 01:38:13 -0400 Subject: [PATCH 075/408] refactor(controller): rename provision methods to set_up_env_for_* to better reflect their purpose. --- .../clp_package_utils/controller.py | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 904a290d76..2ad6fca226 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -98,9 +98,9 @@ def __init__(self, clp_config: CLPConfig): self.clp_home = get_clp_home() self._conf_dir = self.clp_home / "etc" - def provision_database(self): + def set_up_env_for_database(self): component_name = DB_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name @@ -124,9 +124,9 @@ def provision_database(self): ), } - def provision_queue(self): + def set_up_env_for_queue(self): component_name = QUEUE_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name validate_queue_config(self.clp_config, logs_dir) @@ -141,9 +141,9 @@ def provision_queue(self): "CLP_QUEUE_PASS": self.clp_config.queue.password, } - def provision_redis(self): + def set_up_env_for_redis(self): component_name = REDIS_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" logs_dir = self.clp_config.logs_directory / component_name @@ -166,9 +166,9 @@ def provision_redis(self): ), } - def provision_results_cache(self): + def set_up_env_for_results_cache(self): component_name = RESULTS_CACHE_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "mongo" / "mongod.conf" data_dir = self.clp_config.data_directory / component_name @@ -188,9 +188,9 @@ def provision_results_cache(self): "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, } - def provision_compression_scheduler(self): + def set_up_env_for_compression_scheduler(self): component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) @@ -200,9 +200,9 @@ def provision_compression_scheduler(self): "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def provision_query_scheduler(self): + def set_up_env_for_query_scheduler(self): component_name = QUERY_SCHEDULER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) @@ -212,9 +212,9 @@ def provision_query_scheduler(self): "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def provision_compression_worker(self, num_workers: int): + def set_up_env_for_compression_worker(self, num_workers: int): component_name = COMPRESSION_WORKER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -225,9 +225,9 @@ def provision_compression_worker(self, num_workers: int): "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } - def provision_query_worker(self, num_workers: int): + def set_up_env_for_query_worker(self, num_workers: int): component_name = QUERY_WORKER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -238,9 +238,9 @@ def provision_query_worker(self, num_workers: int): "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } - def provision_reducer(self, num_workers: int): + def set_up_env_for_reducer(self, num_workers: int): component_name = REDUCER_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -292,9 +292,9 @@ def _read_and_update_settings_json( return settings_object - def provision_webui(self, container_clp_config: CLPConfig): + def set_up_env_for_webui(self, container_clp_config: CLPConfig): component_name = WEBUI_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( @@ -380,9 +380,9 @@ def provision_webui(self, container_clp_config: CLPConfig): "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), } - def provision_garbage_collector(self): + def set_up_env_for_garbage_collector(self): component_name = GARBAGE_COLLECTOR_COMPONENT_NAME - logger.info(f"Provisioning {component_name}...") + logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) @@ -477,17 +477,17 @@ def _provision(self): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **self.provision_database(), - **self.provision_queue(), - **self.provision_redis(), - **self.provision_results_cache(), - **self.provision_compression_scheduler(), - **self.provision_query_scheduler(), - **self.provision_compression_worker(num_workers), - **self.provision_query_worker(num_workers), - **self.provision_reducer(num_workers), - **self.provision_webui(container_clp_config), - **self.provision_garbage_collector(), + **self.set_up_env_for_database(), + **self.set_up_env_for_queue(), + **self.set_up_env_for_redis(), + **self.set_up_env_for_results_cache(), + **self.set_up_env_for_compression_scheduler(), + **self.set_up_env_for_query_scheduler(), + **self.set_up_env_for_compression_worker(num_workers), + **self.set_up_env_for_query_worker(num_workers), + **self.set_up_env_for_reducer(num_workers), + **self.set_up_env_for_webui(container_clp_config), + **self.set_up_env_for_garbage_collector(), } if self.clp_config.aws_config_directory is not None: From 9f6348120f334fc265a588c1576adbcbf29cd3eb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 01:44:05 -0400 Subject: [PATCH 076/408] reorder private functions --- .../clp_package_utils/controller.py | 174 +++++++++--------- 1 file changed, 87 insertions(+), 87 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 2ad6fca226..ca9fa0f30e 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -54,44 +54,6 @@ logger = logging.getLogger(__name__) -def _chown_recursively( - path: pathlib.Path, - user_id: int, - group_id: int, -): - """ - Recursively changes the owner of the given path to the given user ID and group ID. - :param path: - :param user_id: - :param group_id: - """ - chown_cmd = ["chown", "--recursive", f"{user_id}:{group_id}", str(path)] - subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) - - -def _chown_paths_if_root(*paths: pathlib.Path): - """ - Changes ownership of the given paths to the default container user/group IDs - if the current process is running as root. - - :param paths: - """ - if os.getuid() != 0: - return - for path in paths: - _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) - - -def _get_ip_from_hostname(hostname: str) -> str: - """ - Resolves a hostname to an IP address. - - :param hostname: The hostname to resolve. - :return: The resolved IP address. - """ - return socket.gethostbyname(hostname) - - class BaseController(ABC): def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config @@ -252,46 +214,6 @@ def set_up_env_for_reducer(self, num_workers: int): "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), } - def _update_settings_object( - self, - parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], - ): - """ - Recursively updates the given settings object with the values from `updates`. - - :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. - :param settings: The settings to update. - :param updates: The updates. - :raises ValueError: If a key in `updates` doesn't exist in `settings`. - """ - for key, value in updates.items(): - if key not in settings: - error_msg = ( - f"{parent_key_prefix}{key} is not a valid configuration key for the webui." - ) - raise ValueError(error_msg) - if isinstance(value, dict): - self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) - else: - settings[key] = updates[key] - - def _read_and_update_settings_json( - self, settings_file_path: pathlib.Path, updates: Dict[str, Any] - ): - """ - Reads and updates a settings JSON file. - - :param settings_file_path: - :param updates: - """ - with open(settings_file_path, "r") as settings_json_file: - settings_object = json.loads(settings_json_file.read()) - self._update_settings_object("", settings_object, updates) - - return settings_object - def set_up_env_for_webui(self, container_clp_config: CLPConfig): component_name = WEBUI_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -403,6 +325,46 @@ def stop(self): """ pass + def _update_settings_object( + self, + parent_key_prefix: str, + settings: Dict[str, Any], + updates: Dict[str, Any], + ): + """ + Recursively updates the given settings object with the values from `updates`. + + :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. + :param settings: The settings to update. + :param updates: The updates. + :raises ValueError: If a key in `updates` doesn't exist in `settings`. + """ + for key, value in updates.items(): + if key not in settings: + error_msg = ( + f"{parent_key_prefix}{key} is not a valid configuration key for the webui." + ) + raise ValueError(error_msg) + if isinstance(value, dict): + self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) + else: + settings[key] = updates[key] + + def _read_and_update_settings_json( + self, settings_file_path: pathlib.Path, updates: Dict[str, Any] + ): + """ + Reads and updates a settings JSON file. + + :param settings_file_path: + :param updates: + """ + with open(settings_file_path, "r") as settings_json_file: + settings_object = json.loads(settings_json_file.read()) + self._update_settings_object("", settings_object, updates) + + return settings_object + @abstractmethod def _provision(self) -> Dict[str, str]: """ @@ -414,14 +376,6 @@ def _provision(self) -> Dict[str, str]: class DockerComposeController(BaseController): - @staticmethod - def _get_num_workers(): - """ - Gets the parallelism number for worker components. - TODO: Revisit after moving from single-container to multi-container workers. - """ - return multiprocessing.cpu_count() // 2 - def __init__(self, clp_config: CLPConfig): super().__init__(clp_config) @@ -457,6 +411,14 @@ def stop(self): logger.exception("Failed to stop CLP containers using Docker Compose.") raise + @staticmethod + def _get_num_workers(): + """ + Gets the parallelism number for worker components. + TODO: Revisit after moving from single-container to multi-container workers. + """ + return multiprocessing.cpu_count() // 2 + def _provision(self): container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() @@ -495,4 +457,42 @@ def _provision(self): with open(f"{self.clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): - env_file.write(f"{key}={value}\n") \ No newline at end of file + env_file.write(f"{key}={value}\n") + + +def _chown_recursively( + path: pathlib.Path, + user_id: int, + group_id: int, +): + """ + Recursively changes the owner of the given path to the given user ID and group ID. + :param path: + :param user_id: + :param group_id: + """ + chown_cmd = ["chown", "--recursive", f"{user_id}:{group_id}", str(path)] + subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) + + +def _chown_paths_if_root(*paths: pathlib.Path): + """ + Changes ownership of the given paths to the default container user/group IDs + if the current process is running as root. + + :param paths: + """ + if os.getuid() != 0: + return + for path in paths: + _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) + + +def _get_ip_from_hostname(hostname: str) -> str: + """ + Resolves a hostname to an IP address. + + :param hostname: The hostname to resolve. + :return: The resolved IP address. + """ + return socket.gethostbyname(hostname) \ No newline at end of file From 2344db9007a7bdd215a54d58530557a58a451e1e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 02:44:47 -0400 Subject: [PATCH 077/408] add documentation --- .../clp_package_utils/controller.py | 96 ++++++++++++++++++- 1 file changed, 92 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ca9fa0f30e..22a7be9f05 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -55,12 +55,23 @@ class BaseController(ABC): + """ + Abstract base controller for preparing and deploying CLP components. + Provides common logic for preparing environment variables, directories, + and configuration files for each service. + """ + def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config self.clp_home = get_clp_home() self._conf_dir = self.clp_home / "etc" def set_up_env_for_database(self): + """ + Prepares environment variables and directories for the database component. + + :return: Dictionary of component-related environment variables. + """ component_name = DB_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -87,6 +98,11 @@ def set_up_env_for_database(self): } def set_up_env_for_queue(self): + """ + Prepares environment variables and directories for the message queue component. + + :return: Dictionary of component-related environment variables. + """ component_name = QUEUE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -104,6 +120,11 @@ def set_up_env_for_queue(self): } def set_up_env_for_redis(self): + """ + Prepares environment variables and directories for the Redis component. + + :return: Dictionary of component-related environment variables. + """ component_name = REDIS_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -129,6 +150,11 @@ def set_up_env_for_redis(self): } def set_up_env_for_results_cache(self): + """ + Prepares environment variables and directories for the results cache (MongoDB) component. + + :return: Dictionary of component-related environment variables. + """ component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -151,6 +177,11 @@ def set_up_env_for_results_cache(self): } def set_up_env_for_compression_scheduler(self): + """ + Prepares environment variables and files for the compression scheduler component. + + :return: Dictionary of component-related environment variables. + """ component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -163,6 +194,11 @@ def set_up_env_for_compression_scheduler(self): } def set_up_env_for_query_scheduler(self): + """ + Prepares environment variables and files for the query scheduler component. + + :return: Dictionary of component-related environment variables. + """ component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -175,6 +211,12 @@ def set_up_env_for_query_scheduler(self): } def set_up_env_for_compression_worker(self, num_workers: int): + """ + Prepares environment variables for the compression worker component. + + :param num_workers: Number of worker processes to run. + :return: Dictionary of compression worker-related environment variables. + """ component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -188,6 +230,12 @@ def set_up_env_for_compression_worker(self, num_workers: int): } def set_up_env_for_query_worker(self, num_workers: int): + """ + Prepares environment variables for the query worker component. + + :param num_workers: Number of worker processes to run. + :return: Dictionary of component-related environment variables. + """ component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -201,6 +249,12 @@ def set_up_env_for_query_worker(self, num_workers: int): } def set_up_env_for_reducer(self, num_workers: int): + """ + Prepares environment variables for the reducer component. + + :param num_workers: Number of worker processes to run. + :return: Dictionary of component-related environment variables. + """ component_name = REDUCER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -215,6 +269,12 @@ def set_up_env_for_reducer(self, num_workers: int): } def set_up_env_for_webui(self, container_clp_config: CLPConfig): + """ + Prepares environment variables and settings for the Web UI component. + + :param container_clp_config: CLP configuration inside the containers. + :return: Dictionary of component-related environment variables. + """ component_name = WEBUI_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -303,6 +363,11 @@ def set_up_env_for_webui(self, container_clp_config: CLPConfig): } def set_up_env_for_garbage_collector(self): + """ + Prepares environment variables for the garbage collector component. + + :return: Dictionary of component-related environment variables. + """ component_name = GARBAGE_COLLECTOR_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -368,18 +433,28 @@ def _read_and_update_settings_json( @abstractmethod def _provision(self) -> Dict[str, str]: """ - Provisions all components with orchestrator-specific logic. + Prepares all components with orchestrator-specific logic. - :return: Dictionary of environment variables for the orchestrator + :return: Dictionary of environment variables to be used by the orchestrator. """ pass class DockerComposeController(BaseController): + """ + Controller for deploying CLP components using Docker Compose. + """ + def __init__(self, clp_config: CLPConfig): super().__init__(clp_config) def deploy(self): + """ + Deploys CLP components using Docker Compose by: + 1. Checking Docker dependencies. + 2. Provisioning environment variables and configuration. + 3. Running `docker compose up -d`. + """ check_docker_dependencies(should_compose_run=False) self._provision() @@ -396,6 +471,9 @@ def deploy(self): raise def stop(self): + """ + Stops CLP components deployed via Docker Compose. + """ check_docker_dependencies(should_compose_run=True) logger.info("Stopping all CLP containers using Docker Compose...") @@ -416,10 +494,19 @@ def _get_num_workers(): """ Gets the parallelism number for worker components. TODO: Revisit after moving from single-container to multi-container workers. + :return: Number of worker processes. """ return multiprocessing.cpu_count() // 2 def _provision(self): + """ + Provisions all CLP components for Docker Compose by: + - Generating container-specific config. + - Preparing environment variables for all components. + - Writing environment variables to `.env`. + + :return: Dictionary of all environment variables. + """ container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() dump_shared_container_config(container_clp_config, self.clp_config) @@ -467,6 +554,7 @@ def _chown_recursively( ): """ Recursively changes the owner of the given path to the given user ID and group ID. + :param path: :param user_id: :param group_id: @@ -477,8 +565,8 @@ def _chown_recursively( def _chown_paths_if_root(*paths: pathlib.Path): """ - Changes ownership of the given paths to the default container user/group IDs - if the current process is running as root. + Changes ownership of the given paths to the default service container user/group IDs if the + current process is running as root. :param paths: """ From 2ba93d7d212b8c001b64e4cd6338d63a5f806e40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 02:53:05 -0400 Subject: [PATCH 078/408] change visibility of `set_up_env_for` function from public to private by prefixing `_`; reorder methods. --- .../clp_package_utils/controller.py | 144 +++++++++--------- 1 file changed, 73 insertions(+), 71 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 22a7be9f05..340bed1763 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -66,7 +66,30 @@ def __init__(self, clp_config: CLPConfig): self.clp_home = get_clp_home() self._conf_dir = self.clp_home / "etc" - def set_up_env_for_database(self): + @abstractmethod + def deploy(self): + """ + Deploys the provisioned components with orchestrator-specific logic. + """ + pass + + @abstractmethod + def stop(self): + """ + Stops the deployed components with orchestrator-specific logic. + """ + pass + + @abstractmethod + def _provision(self) -> Dict[str, str]: + """ + Prepares all components with orchestrator-specific logic. + + :return: Dictionary of environment variables to be used by the orchestrator. + """ + pass + + def _set_up_env_for_database(self): """ Prepares environment variables and directories for the database component. @@ -97,7 +120,7 @@ def set_up_env_for_database(self): ), } - def set_up_env_for_queue(self): + def _set_up_env_for_queue(self): """ Prepares environment variables and directories for the message queue component. @@ -119,7 +142,7 @@ def set_up_env_for_queue(self): "CLP_QUEUE_PASS": self.clp_config.queue.password, } - def set_up_env_for_redis(self): + def _set_up_env_for_redis(self): """ Prepares environment variables and directories for the Redis component. @@ -149,7 +172,7 @@ def set_up_env_for_redis(self): ), } - def set_up_env_for_results_cache(self): + def _set_up_env_for_results_cache(self): """ Prepares environment variables and directories for the results cache (MongoDB) component. @@ -176,7 +199,7 @@ def set_up_env_for_results_cache(self): "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, } - def set_up_env_for_compression_scheduler(self): + def _set_up_env_for_compression_scheduler(self): """ Prepares environment variables and files for the compression scheduler component. @@ -193,7 +216,7 @@ def set_up_env_for_compression_scheduler(self): "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def set_up_env_for_query_scheduler(self): + def _set_up_env_for_query_scheduler(self): """ Prepares environment variables and files for the query scheduler component. @@ -210,7 +233,7 @@ def set_up_env_for_query_scheduler(self): "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def set_up_env_for_compression_worker(self, num_workers: int): + def _set_up_env_for_compression_worker(self, num_workers: int): """ Prepares environment variables for the compression worker component. @@ -229,7 +252,7 @@ def set_up_env_for_compression_worker(self, num_workers: int): "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } - def set_up_env_for_query_worker(self, num_workers: int): + def _set_up_env_for_query_worker(self, num_workers: int): """ Prepares environment variables for the query worker component. @@ -248,7 +271,7 @@ def set_up_env_for_query_worker(self, num_workers: int): "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } - def set_up_env_for_reducer(self, num_workers: int): + def _set_up_env_for_reducer(self, num_workers: int): """ Prepares environment variables for the reducer component. @@ -268,7 +291,7 @@ def set_up_env_for_reducer(self, num_workers: int): "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), } - def set_up_env_for_webui(self, container_clp_config: CLPConfig): + def _set_up_env_for_webui(self, container_clp_config: CLPConfig): """ Prepares environment variables and settings for the Web UI component. @@ -362,7 +385,7 @@ def set_up_env_for_webui(self, container_clp_config: CLPConfig): "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), } - def set_up_env_for_garbage_collector(self): + def _set_up_env_for_garbage_collector(self): """ Prepares environment variables for the garbage collector component. @@ -376,19 +399,20 @@ def set_up_env_for_garbage_collector(self): return {"CLP_GC_LOGGING_LEVEL": self.clp_config.garbage_collector.logging_level} - @abstractmethod - def deploy(self): - """ - Deploys the provisioned components with orchestrator-specific logic. + def _read_and_update_settings_json( + self, settings_file_path: pathlib.Path, updates: Dict[str, Any] + ): """ - pass + Reads and updates a settings JSON file. - @abstractmethod - def stop(self): - """ - Stops the deployed components with orchestrator-specific logic. + :param settings_file_path: + :param updates: """ - pass + with open(settings_file_path, "r") as settings_json_file: + settings_object = json.loads(settings_json_file.read()) + self._update_settings_object("", settings_object, updates) + + return settings_object def _update_settings_object( self, @@ -415,30 +439,6 @@ def _update_settings_object( else: settings[key] = updates[key] - def _read_and_update_settings_json( - self, settings_file_path: pathlib.Path, updates: Dict[str, Any] - ): - """ - Reads and updates a settings JSON file. - - :param settings_file_path: - :param updates: - """ - with open(settings_file_path, "r") as settings_json_file: - settings_object = json.loads(settings_json_file.read()) - self._update_settings_object("", settings_object, updates) - - return settings_object - - @abstractmethod - def _provision(self) -> Dict[str, str]: - """ - Prepares all components with orchestrator-specific logic. - - :return: Dictionary of environment variables to be used by the orchestrator. - """ - pass - class DockerComposeController(BaseController): """ @@ -515,7 +515,9 @@ def _provision(self): "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, # User and group IDs "CLP_UID_GID": DEFAULT_UID_GID, - "CLP_SERVICE_CONTAINER_UID_GID": SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID, + "CLP_SERVICE_CONTAINER_UID_GID": ( + SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID + ), # Package container "CLP_PACKAGE_CONTAINER": self.clp_config.execution_container, # Global paths @@ -526,17 +528,17 @@ def _provision(self): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - **self.set_up_env_for_database(), - **self.set_up_env_for_queue(), - **self.set_up_env_for_redis(), - **self.set_up_env_for_results_cache(), - **self.set_up_env_for_compression_scheduler(), - **self.set_up_env_for_query_scheduler(), - **self.set_up_env_for_compression_worker(num_workers), - **self.set_up_env_for_query_worker(num_workers), - **self.set_up_env_for_reducer(num_workers), - **self.set_up_env_for_webui(container_clp_config), - **self.set_up_env_for_garbage_collector(), + **self._set_up_env_for_database(), + **self._set_up_env_for_queue(), + **self._set_up_env_for_redis(), + **self._set_up_env_for_results_cache(), + **self._set_up_env_for_compression_scheduler(), + **self._set_up_env_for_query_scheduler(), + **self._set_up_env_for_compression_worker(num_workers), + **self._set_up_env_for_query_worker(num_workers), + **self._set_up_env_for_reducer(num_workers), + **self._set_up_env_for_webui(container_clp_config), + **self._set_up_env_for_garbage_collector(), } if self.clp_config.aws_config_directory is not None: @@ -547,6 +549,19 @@ def _provision(self): env_file.write(f"{key}={value}\n") +def _chown_paths_if_root(*paths: pathlib.Path): + """ + Changes ownership of the given paths to the default service container user/group IDs if the + current process is running as root. + + :param paths: + """ + if os.getuid() != 0: + return + for path in paths: + _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) + + def _chown_recursively( path: pathlib.Path, user_id: int, @@ -563,19 +578,6 @@ def _chown_recursively( subprocess.run(chown_cmd, stdout=subprocess.DEVNULL, check=True) -def _chown_paths_if_root(*paths: pathlib.Path): - """ - Changes ownership of the given paths to the default service container user/group IDs if the - current process is running as root. - - :param paths: - """ - if os.getuid() != 0: - return - for path in paths: - _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) - - def _get_ip_from_hostname(hostname: str) -> str: """ Resolves a hostname to an IP address. @@ -583,4 +585,4 @@ def _get_ip_from_hostname(hostname: str) -> str: :param hostname: The hostname to resolve. :return: The resolved IP address. """ - return socket.gethostbyname(hostname) \ No newline at end of file + return socket.gethostbyname(hostname) From ce94225d8a356451b196c74fc4b370cd2ccfa751 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:05:08 -0400 Subject: [PATCH 079/408] Improve type hints. --- .../clp_package_utils/controller.py | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 340bed1763..2251d5d770 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -9,6 +9,9 @@ from abc import ABC, abstractmethod from typing import Any, Dict +# Type alias for environment variables dictionary. +EnvVarsDict = Dict[str, str] + from clp_py_utils.clp_config import ( AwsAuthType, CLPConfig, @@ -81,7 +84,7 @@ def stop(self): pass @abstractmethod - def _provision(self) -> Dict[str, str]: + def _provision(self) -> EnvVarsDict: """ Prepares all components with orchestrator-specific logic. @@ -89,7 +92,7 @@ def _provision(self) -> Dict[str, str]: """ pass - def _set_up_env_for_database(self): + def _set_up_env_for_database(self) -> EnvVarsDict: """ Prepares environment variables and directories for the database component. @@ -120,7 +123,7 @@ def _set_up_env_for_database(self): ), } - def _set_up_env_for_queue(self): + def _set_up_env_for_queue(self) -> EnvVarsDict: """ Prepares environment variables and directories for the message queue component. @@ -142,7 +145,7 @@ def _set_up_env_for_queue(self): "CLP_QUEUE_PASS": self.clp_config.queue.password, } - def _set_up_env_for_redis(self): + def _set_up_env_for_redis(self) -> EnvVarsDict: """ Prepares environment variables and directories for the Redis component. @@ -172,7 +175,7 @@ def _set_up_env_for_redis(self): ), } - def _set_up_env_for_results_cache(self): + def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ Prepares environment variables and directories for the results cache (MongoDB) component. @@ -199,7 +202,7 @@ def _set_up_env_for_results_cache(self): "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, } - def _set_up_env_for_compression_scheduler(self): + def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: """ Prepares environment variables and files for the compression scheduler component. @@ -216,7 +219,7 @@ def _set_up_env_for_compression_scheduler(self): "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def _set_up_env_for_query_scheduler(self): + def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: """ Prepares environment variables and files for the query scheduler component. @@ -233,7 +236,7 @@ def _set_up_env_for_query_scheduler(self): "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), } - def _set_up_env_for_compression_worker(self, num_workers: int): + def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: """ Prepares environment variables for the compression worker component. @@ -252,7 +255,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int): "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } - def _set_up_env_for_query_worker(self, num_workers: int): + def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: """ Prepares environment variables for the query worker component. @@ -271,7 +274,7 @@ def _set_up_env_for_query_worker(self, num_workers: int): "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } - def _set_up_env_for_reducer(self, num_workers: int): + def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: """ Prepares environment variables for the reducer component. @@ -291,7 +294,7 @@ def _set_up_env_for_reducer(self, num_workers: int): "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), } - def _set_up_env_for_webui(self, container_clp_config: CLPConfig): + def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: """ Prepares environment variables and settings for the Web UI component. @@ -385,7 +388,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig): "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), } - def _set_up_env_for_garbage_collector(self): + def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: """ Prepares environment variables for the garbage collector component. @@ -401,7 +404,7 @@ def _set_up_env_for_garbage_collector(self): def _read_and_update_settings_json( self, settings_file_path: pathlib.Path, updates: Dict[str, Any] - ): + ) -> Dict[str, Any]: """ Reads and updates a settings JSON file. @@ -490,7 +493,7 @@ def stop(self): raise @staticmethod - def _get_num_workers(): + def _get_num_workers() -> int: """ Gets the parallelism number for worker components. TODO: Revisit after moving from single-container to multi-container workers. From 52258705962e42ef406d3be97deca8baace0d1c1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:09:05 -0400 Subject: [PATCH 080/408] docs(clp-package-utils): add docstring for is_docker_compose_running function. --- components/clp-package-utils/clp_package_utils/general.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index f8679b6919..bdd3b38bb8 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -132,6 +132,13 @@ def generate_container_name(job_type: str) -> str: def is_docker_compose_running(project_name: str) -> bool: + """ + Checks if a Docker Compose project is running. + + :param project_name: + :return: True if at least one instance is running, else False. + :raises EnvironmentError: If Docker Compose is not installed or fails. + """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) From cecf5b28c010c8f4c6ca291ae4601cda972666dc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:11:31 -0400 Subject: [PATCH 081/408] docs(clp-package-utils): add docstring for check_docker_dependencies function. --- components/clp-package-utils/clp_package_utils/general.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index bdd3b38bb8..cfb03e9ffd 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -149,6 +149,13 @@ def is_docker_compose_running(project_name: str) -> bool: def check_docker_dependencies(should_compose_run: bool = False): + """ + Checks if Docker and Docker Compose are installed, and whether Docker Compose is running or not. + + :param should_compose_run: + :raises EnvironmentError: If any Docker dependency is not installed or Docker Compose state + does not match expectation. + """ try: subprocess.run( "command -v docker", From 0c0c562be5b232c26327bf0a80d6b4c9d5a2c985 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:12:57 -0400 Subject: [PATCH 082/408] docs(clp-package-utils): add docstring for _validate_log_directory function. --- components/clp-package-utils/clp_package_utils/general.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index cfb03e9ffd..b2bba35be7 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -175,6 +175,13 @@ def check_docker_dependencies(should_compose_run: bool = False): def _validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: + """ + Validate that a log directory path of a component is valid. + + :param logs_dir: + :param component_name: + :raises ValueError: If the path is invalid or not a directory. + """ try: validate_path_could_be_dir(logs_dir) except ValueError as ex: From 762884b5bf3b6e403b3ecbfe6f1ee03a81359fb3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:13:13 -0400 Subject: [PATCH 083/408] remove `None` return type annotation from _validate_log_directory function. --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index b2bba35be7..fcc3a3dd77 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -174,7 +174,7 @@ def check_docker_dependencies(should_compose_run: bool = False): raise EnvironmentError("docker-compose is already running.") -def _validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None: +def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): """ Validate that a log directory path of a component is valid. From fc515dbcd580e4bf8d9e59daec09773029bff104 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:19:02 -0400 Subject: [PATCH 084/408] Rename transform_for_container_config to transform_for_container; add docstring for CLPConfig.transform_for_container --- .../clp_package_utils/general.py | 2 +- .../clp-py-utils/clp_py_utils/clp_config.py | 52 +++++++++++-------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index fcc3a3dd77..fd357aad36 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -314,7 +314,7 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig :return: The container config and the mounts. """ container_clp_config = clp_config.copy(deep=True) - container_clp_config.transform_for_container_config() + container_clp_config.transform_for_container() return container_clp_config diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 54548c9873..30962ca83a 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -240,7 +240,7 @@ def load_credentials_from_env(self): self.username = _get_env_var(CLP_DB_USER_ENV_VAR_NAME) self.password = _get_env_var(CLP_DB_PASS_ENV_VAR_NAME) - def transform_for_container_config(self): + def transform_for_container(self): self.host = DB_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -303,7 +303,7 @@ def validate_port(cls, field): raise ValueError(f"{field} is not greater than zero") return field - def transform_for_container_config(self): + def transform_for_container(self): self.host = QUERY_SCHEDULER_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -362,7 +362,7 @@ def load_credentials_from_env(self): """ self.password = _get_env_var(CLP_REDIS_PASS_ENV_VAR_NAME) - def transform_for_container_config(self): + def transform_for_container(self): self.host = REDIS_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -398,7 +398,7 @@ def validate_upsert_interval(cls, field): raise ValueError(f"{field} is not greater than zero") return field - def transform_for_container_config(self): + def transform_for_container(self): self.host = REDUCER_COMPONENT_NAME self.base_port = self.DEFAULT_PORT @@ -441,7 +441,7 @@ def validate_retention_period(cls, field): def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" - def transform_for_container_config(self): + def transform_for_container(self): self.host = RESULTS_CACHE_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -477,7 +477,7 @@ def load_credentials_from_env(self): self.username = _get_env_var(CLP_QUEUE_USER_ENV_VAR_NAME) self.password = _get_env_var(CLP_QUEUE_PASS_ENV_VAR_NAME) - def transform_for_container_config(self): + def transform_for_container(self): self.host = QUEUE_COMPONENT_NAME self.port = self.DEFAULT_PORT @@ -558,7 +558,7 @@ class S3IngestionConfig(BaseModel): def dump_to_primitive_dict(self): return self.dict() - def transform_for_container_config(self): + def transform_for_container(self): pass @@ -616,35 +616,35 @@ def dump_to_primitive_dict(self): class FsIngestionConfig(FsStorage): directory: pathlib.Path = pathlib.Path("/") - def transform_for_container_config(self): + def transform_for_container(self): self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR class ArchiveFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH - def transform_for_container_config(self): + def transform_for_container(self): self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH class StreamFsStorage(FsStorage): directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH - def transform_for_container_config(self): + def transform_for_container(self): self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH - def transform_for_container_config(self): + def transform_for_container(self): self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH - def transform_for_container_config(self): + def transform_for_container(self): self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH @@ -970,21 +970,27 @@ def dump_to_primitive_dict(self): return d - def transform_for_container_config(self): + def transform_for_container(self): + """ + Adjusts paths and service hosts for containerized execution. + + Converts all relevant directories to absolute paths inside the container + and updates service hostnames/ports to their container service names. + """ self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH if self.aws_config_directory is not None: self.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY - self.logs_input.transform_for_container_config() - self.archive_output.storage.transform_for_container_config() - self.stream_output.storage.transform_for_container_config() - - self.database.transform_for_container_config() - self.queue.transform_for_container_config() - self.redis.transform_for_container_config() - self.results_cache.transform_for_container_config() - self.query_scheduler.transform_for_container_config() - self.reducer.transform_for_container_config() + self.logs_input.transform_for_container() + self.archive_output.storage.transform_for_container() + self.stream_output.storage.transform_for_container() + + self.database.transform_for_container() + self.queue.transform_for_container() + self.redis.transform_for_container() + self.results_cache.transform_for_container() + self.query_scheduler.transform_for_container() + self.reducer.transform_for_container() class WorkerConfig(BaseModel): From 268c1444cac04408d618ae5335a68f416f521887 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 22 Sep 2025 03:22:20 -0400 Subject: [PATCH 085/408] add docs for x-service-defaults & x-healthcheck-defaults in docker-compose.yaml --- tools/deployment/package/docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 2e85855633..0f9a7243d5 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -1,8 +1,10 @@ +# Common service defaults. x-service-defaults: &service_defaults networks: ["clp-network"] stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" +# Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 From fc7aae361433f3cf59fe19cfdeb03d3bf0cdad04 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 15:59:03 -0400 Subject: [PATCH 086/408] remove garbage-collector's dependency condition on query-scheduler. --- docs/src/dev-docs/design-docker-compose.md | 1 - tools/deployment/package/docker-compose.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index d845a242ae..d8fa7800ad 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -82,7 +82,6 @@ graph LR db_table_creator -->|completed_successfully| webui results_cache_indices_creator -->|completed_successfully| webui db_table_creator -->|completed_successfully| garbage_collector - query_scheduler -->|healthy| garbage_collector results_cache_indices_creator -->|completed_successfully| garbage_collector subgraph Databases diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 0f9a7243d5..64aa7232eb 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -385,8 +385,6 @@ services: depends_on: db-table-creator: condition: "service_completed_successfully" - query-scheduler: - condition: "service_healthy" results-cache-indices-creator: condition: "service_completed_successfully" command: [ From cda03486b4284efb2e64c2007312b559282fa63f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 19:00:51 -0400 Subject: [PATCH 087/408] Split base services into a separate docker-compose.base.yaml; Launch only base services when Presto query engine is configured. --- .../clp_package_utils/controller.py | 11 +- .../clp-py-utils/clp_py_utils/clp_config.py | 50 +-- taskfile.yaml | 4 +- .../package/docker-compose.base.yaml | 293 ++++++++++++++++++ tools/deployment/package/docker-compose.yaml | 284 +---------------- 5 files changed, 319 insertions(+), 323 deletions(-) create mode 100644 tools/deployment/package/docker-compose.base.yaml diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index a4b763915d..ef9a1870f8 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -28,7 +28,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageEngine, StorageType, - WEBUI_COMPONENT_NAME, + WEBUI_COMPONENT_NAME, DeploymentType, ) from clp_py_utils.clp_metadata_db_utils import ( get_archives_table_name, @@ -470,10 +470,15 @@ def deploy(self): check_docker_dependencies(should_compose_run=False) self._provision() - logger.info(f"Starting CLP using Docker Compose...") + deployment_type = self.clp_config.get_deployment_type() + logger.info(f"Starting CLP using Docker Compose ({deployment_type})...") + cmd = ["docker", "compose"] + if deployment_type == DeploymentType.BASE: + cmd += ["--file", "docker-compose.base.yaml"] + cmd += ["up", "--detach"] try: subprocess.run( - ["docker", "compose", "up", "-d"], + cmd, cwd=self.clp_home, stderr=subprocess.STDOUT, check=True, diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index e431df85bc..691623321c 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -28,45 +28,6 @@ WEBUI_COMPONENT_NAME = "webui" GARBAGE_COLLECTOR_COMPONENT_NAME = "garbage_collector" -# Component groups -GENERAL_SCHEDULING_COMPONENTS = { - QUEUE_COMPONENT_NAME, - REDIS_COMPONENT_NAME, -} -COMPRESSION_COMPONENTS = GENERAL_SCHEDULING_COMPONENTS | { - DB_COMPONENT_NAME, - COMPRESSION_SCHEDULER_COMPONENT_NAME, - COMPRESSION_WORKER_COMPONENT_NAME, -} -QUERY_COMPONENTS = GENERAL_SCHEDULING_COMPONENTS | { - DB_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - QUERY_WORKER_COMPONENT_NAME, - REDUCER_COMPONENT_NAME, -} -UI_COMPONENTS = { - RESULTS_CACHE_COMPONENT_NAME, - WEBUI_COMPONENT_NAME, -} -STORAGE_MANAGEMENT_COMPONENTS = {GARBAGE_COLLECTOR_COMPONENT_NAME} -ALL_COMPONENTS = ( - COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS | STORAGE_MANAGEMENT_COMPONENTS -) - -# Target names -ALL_TARGET_NAME = "" -CONTROLLER_TARGET_NAME = "controller" - -TARGET_TO_COMPONENTS = { - ALL_TARGET_NAME: ALL_COMPONENTS, - CONTROLLER_TARGET_NAME: GENERAL_SCHEDULING_COMPONENTS - | { - COMPRESSION_SCHEDULER_COMPONENT_NAME, - QUERY_SCHEDULER_COMPONENT_NAME, - } - | STORAGE_MANAGEMENT_COMPONENTS, -} - # Action names ARCHIVE_MANAGER_ACTION_NAME = "archive_manager" @@ -101,6 +62,11 @@ CLP_REDIS_PASS_ENV_VAR_NAME = "CLP_REDIS_PASS" +class DeploymentType(KebabCaseStrEnum): + BASE = auto() + FULL = auto() + + class StorageEngine(KebabCaseStrEnum): CLP = auto() CLP_S = auto() @@ -1001,6 +967,12 @@ def load_execution_container_name(self): def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME + def get_deployment_type(self) -> DeploymentType: + if QueryEngine.PRESTO == self.package.query_engine: + return DeploymentType.BASE + else: + return DeploymentType.FULL + def dump_to_primitive_dict(self): custom_serialized_fields = { "database", diff --git a/taskfile.yaml b/taskfile.yaml index 231c83849b..651e001c6e 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -116,7 +116,7 @@ tasks: - "components/clp-py-utils/dist/*.whl" - "components/job-orchestration/dist/*.whl" - "components/package-template/src/**/*" - - "tools/deployment/package/docker-compose.yaml" + - "tools/deployment/package/**/*" generates: ["{{.CHECKSUM_FILE}}"] deps: - "core" @@ -166,7 +166,7 @@ tasks: PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm ci --omit=dev - >- rsync -a - "tools/deployment/package/docker-compose.yaml" + "tools/deployment/package/" "{{.OUTPUT_DIR}}" # This command must be last - task: "utils:checksum:compute" diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml new file mode 100644 index 0000000000..a49e4d3cd5 --- /dev/null +++ b/tools/deployment/package/docker-compose.base.yaml @@ -0,0 +1,293 @@ +name: "clp-package-base" + +# Common service defaults. +x-service-defaults: &service_defaults + networks: ["clp-network"] + stop_grace_period: "3s" + user: "${CLP_UID_GID:-1000:1000}" + +# Common healthcheck defaults. +x-healthcheck-defaults: &healthcheck_defaults + interval: "30s" + retries: 3 + start_interval: "1s" + start_period: "10s" + timeout: "10s" + +networks: + clp-network: + driver: "bridge" + +secrets: + CLP_DB_PASS_FILE: + environment: "CLP_DB_PASS" + +services: + db: + <<: *service_defaults + container_name: "database" + image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + environment: + MYSQL_DATABASE: "${CLP_DB_NAME}" + MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_USER: "${CLP_DB_USER}" + secrets: + - "CLP_DB_PASS_FILE" + ports: + - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + volumes: + - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" + - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1", + "-u", "${CLP_DB_USER}", + "--password=${CLP_DB_PASS}" + ] + + db-table-creator: + <<: *service_defaults + container_name: "db_table_creator" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + depends_on: + db: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" + ] + + queue: + <<: *service_defaults + container_name: "queue" + image: "rabbitmq:3.9.8" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + environment: + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" + ports: + - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" + volumes: + - "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}:/var/log/rabbitmq" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "rabbitmq-diagnostics", "check_running" + ] + + redis: + <<: *service_defaults + container_name: "redis" + image: "redis:7.2.4" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + ports: + - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" + volumes: + - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf:ro" + - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" + - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "redis-cli", + "-h", "127.0.0.1", + "-p", "6379", + "-a", "${CLP_REDIS_PASS}", + "PING" + ] + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS}" + ] + + results-cache: + <<: *service_defaults + container_name: "results_cache" + image: "mongo:7.0.1" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + ports: + - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + volumes: + - "${CLP_RESULTS_CACHE_CONF_DIR_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" + - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" + healthcheck: + <<: *healthcheck_defaults + test: >- + echo 'db.runCommand("ping").ok' | + mongosh 127.0.0.1:27017/test --quiet + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] + + results-cache-indices-creator: + <<: *service_defaults + container_name: "results_cache_indices_creator" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + depends_on: + results-cache: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", + "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", + ] + + compression-scheduler: + <<: *service_defaults + container_name: "compression_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + volumes: + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ +:/var/log/compression_scheduler.log" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "/:/mnt/logs:ro" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "--config", "/etc/clp-config.yml" + ] + + compression-worker: + <<: *service_defaults + container_name: "compression_worker" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_CONFIG_PATH: "/etc/clp-config.yml" + CLP_HOME: "/opt/clp" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/compression_worker" + CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + volumes: + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ +/var/log/compression_worker" + - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "/:/mnt/logs:ro" + command: [ + "python3", + "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.compress", + "worker", + "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", + "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", + "-f", "/var/log/compression_worker/worker.log", + "-Q", "compression", + "-n", "compression-worker" + ] + + webui: + <<: *service_defaults + container_name: "webui" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + HOST: "0.0.0.0" + NODE_ENV: "production" + NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" + PORT: "4000" + RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" + ports: + - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + volumes: + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" + - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" + - "./var/www/webui/server/dist/settings.json\ +:/opt/clp/var/www/webui/server/dist/settings.json:ro" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/src/main.js" + ] + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "bash", + "-c", + "< /dev/tcp/webui/4000" + ] + + garbage-collector: + <<: *service_defaults + container_name: "garbage_collector" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_HOME: "/opt/clp" + CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/garbage_collector" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yml", + ] diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 64aa7232eb..5e60c09797 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -1,3 +1,8 @@ +name: "clp-package" + +include: + - "docker-compose.base.yaml" + # Common service defaults. x-service-defaults: &service_defaults networks: ["clp-network"] @@ -12,185 +17,7 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "10s" timeout: "10s" -networks: - clp-network: - driver: "bridge" - -name: "clp-package" - -secrets: - CLP_DB_PASS_FILE: - environment: "CLP_DB_PASS" - services: - db: - <<: *service_defaults - container_name: "database" - image: "${CLP_DB_IMAGE:-mysql:8.0.23}" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - environment: - MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_USER: "${CLP_DB_USER}" - secrets: - - "CLP_DB_PASS_FILE" - ports: - - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" - volumes: - - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" - - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "mysqladmin", "ping", - "--silent", - "-h", "127.0.0.1", - "-u", "${CLP_DB_USER}", - "--password=${CLP_DB_PASS}" - ] - - db-table-creator: - <<: *service_defaults - container_name: "db_table_creator" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" - depends_on: - db: - condition: "service_healthy" - command: [ - "python3", - "-u", - "-m", "clp_py_utils.create-db-tables", - "--config", "/etc/clp-config.yml", - "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" - ] - - queue: - <<: *service_defaults - container_name: "queue" - image: "rabbitmq:3.9.8" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - environment: - RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" - RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" - ports: - - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" - volumes: - - "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}:/var/log/rabbitmq" - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "rabbitmq-diagnostics", "check_running" - ] - - redis: - <<: *service_defaults - container_name: "redis" - image: "redis:7.2.4" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - ports: - - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" - volumes: - - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf:ro" - - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" - - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "redis-cli", - "-h", "127.0.0.1", - "-p", "6379", - "-a", "${CLP_REDIS_PASS}", - "PING" - ] - command: [ - "redis-server", - "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS}" - ] - - results-cache: - <<: *service_defaults - container_name: "results_cache" - image: "mongo:7.0.1" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - ports: - - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" - volumes: - - "${CLP_RESULTS_CACHE_CONF_DIR_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" - - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" - healthcheck: - <<: *healthcheck_defaults - test: >- - echo 'db.runCommand("ping").ok' | - mongosh 127.0.0.1:27017/test --quiet - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] - - results-cache-indices-creator: - <<: *service_defaults - container_name: "results_cache_indices_creator" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - depends_on: - results-cache: - condition: "service_healthy" - command: [ - "python3", - "-u", - "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", - "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", - ] - - compression-scheduler: - <<: *service_defaults - container_name: "compression_scheduler" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} - volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ -:/var/log/compression_scheduler.log" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" - depends_on: - db-table-creator: - condition: "service_completed_successfully" - queue: - condition: "service_healthy" - redis: - condition: "service_healthy" - command: [ - "python3", - "-u", - "-m", "job_orchestration.scheduler.compress.compression_scheduler", - "--config", "/etc/clp-config.yml" - ] - query-scheduler: <<: *service_defaults container_name: "query_scheduler" @@ -231,44 +58,6 @@ services: "< /dev/tcp/query_scheduler/7000" ] - compression-worker: - <<: *service_defaults - container_name: "compression_worker" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_CONFIG_PATH: "/etc/clp-config.yml" - CLP_HOME: "/opt/clp" - CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log/compression_worker" - CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} - volumes: - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ -/var/log/compression_worker" - - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" - command: [ - "python3", - "-u", - "/opt/clp/lib/python3/site-packages/bin/celery", - "-A", "job_orchestration.executor.compress", - "worker", - "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", - "-f", "/var/log/compression_worker/worker.log", - "-Q", "compression", - "-n", "compression-worker" - ] - query-worker: <<: *service_defaults container_name: "query_worker" @@ -329,66 +118,3 @@ services: "--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}", "--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}" ] - - webui: - <<: *service_defaults - container_name: "webui" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - HOST: "0.0.0.0" - NODE_ENV: "production" - NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" - PORT: "4000" - RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" - ports: - - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" - volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/settings.json\ -:/opt/clp/var/www/webui/server/dist/settings.json:ro" - depends_on: - db-table-creator: - condition: "service_completed_successfully" - results-cache-indices-creator: - condition: "service_completed_successfully" - command: [ - "/opt/clp/bin/node-22", - "/opt/clp/var/www/webui/server/dist/src/main.js" - ] - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "bash", - "-c", - "< /dev/tcp/webui/4000" - ] - - garbage-collector: - <<: *service_defaults - container_name: "garbage_collector" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_HOME: "/opt/clp" - CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log/garbage_collector" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" - depends_on: - db-table-creator: - condition: "service_completed_successfully" - results-cache-indices-creator: - condition: "service_completed_successfully" - command: [ - "python3", "-u", - "-m", "job_orchestration.garbage_collector.garbage_collector", - "--config", "/etc/clp-config.yml", - ] From e506f6db72a84cd22dbc7b00a94a024b56875252 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 19:31:03 -0400 Subject: [PATCH 088/408] feat(controller): Add instance ID support for Docker Compose project names. --- .../clp_package_utils/controller.py | 36 +++++++++++++++---- .../clp_package_utils/general.py | 5 +-- .../clp_package_utils/scripts/start_clp.py | 7 ++-- .../clp_package_utils/scripts/stop_clp.py | 5 +-- .../package/docker-compose.base.yaml | 1 + 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ef9a1870f8..407937766b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -6,6 +6,7 @@ import socket import stat import subprocess +import uuid from abc import ABC, abstractmethod from typing import Any, Dict @@ -18,6 +19,7 @@ COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, DB_COMPONENT_NAME, + DeploymentType, GARBAGE_COLLECTOR_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, @@ -28,7 +30,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageEngine, StorageType, - WEBUI_COMPONENT_NAME, DeploymentType, + WEBUI_COMPONENT_NAME, ) from clp_py_utils.clp_metadata_db_utils import ( get_archives_table_name, @@ -457,7 +459,8 @@ class DockerComposeController(BaseController): Controller for deploying CLP components using Docker Compose. """ - def __init__(self, clp_config: CLPConfig): + def __init__(self, clp_config: CLPConfig, instance_id: str): + self._project_name = f"clp-package-{instance_id}" super().__init__(clp_config) def deploy(self): @@ -467,12 +470,13 @@ def deploy(self): 2. Provisioning environment variables and configuration. 3. Running `docker compose up -d`. """ - check_docker_dependencies(should_compose_run=False) + check_docker_dependencies(should_compose_run=False, project_name=self._project_name) self._provision() deployment_type = self.clp_config.get_deployment_type() logger.info(f"Starting CLP using Docker Compose ({deployment_type})...") - cmd = ["docker", "compose"] + + cmd = ["docker", "compose", "--project-name", self._project_name] if deployment_type == DeploymentType.BASE: cmd += ["--file", "docker-compose.base.yaml"] cmd += ["up", "--detach"] @@ -491,12 +495,12 @@ def stop(self): """ Stops CLP components deployed via Docker Compose. """ - check_docker_dependencies(should_compose_run=True) + check_docker_dependencies(should_compose_run=True, project_name=self._project_name) logger.info("Stopping all CLP containers using Docker Compose...") try: subprocess.run( - ["docker", "compose", "down"], + ["docker", "compose", "--project-name", self._project_name, "down"], cwd=self.clp_home, stderr=subprocess.STDOUT, check=True, @@ -566,6 +570,26 @@ def _provision(self): env_file.write(f"{key}={value}\n") +def get_or_create_instance_id(clp_config: CLPConfig): + """ + Gets or create a unique instance ID for this CLP instance. + :param clp_config: + :return: The instance ID. + """ + instance_id_file_path = clp_config.logs_directory / "instance-id" + + if instance_id_file_path.exists(): + with open(instance_id_file_path, "r") as f: + instance_id = f.readline() + else: + instance_id = str(uuid.uuid4())[-4:] + with open(instance_id_file_path, "w") as f: + f.write(instance_id) + f.flush() + + return instance_id + + def _chown_paths_if_root(*paths: pathlib.Path): """ Changes ownership of the given paths to the default service container user/group IDs if the diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index fd357aad36..b7dcf14936 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -148,11 +148,12 @@ def is_docker_compose_running(project_name: str) -> bool: raise EnvironmentError("docker-compose is not installed or not functioning properly.") -def check_docker_dependencies(should_compose_run: bool = False): +def check_docker_dependencies(should_compose_run: bool, project_name: str): """ Checks if Docker and Docker Compose are installed, and whether Docker Compose is running or not. :param should_compose_run: + :param project_name: The Docker Compose project name to check. :raises EnvironmentError: If any Docker dependency is not installed or Docker Compose state does not match expectation. """ @@ -167,7 +168,7 @@ def check_docker_dependencies(should_compose_run: bool = False): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = is_docker_compose_running("clp-package") + is_running = is_docker_compose_running(project_name) if should_compose_run and not is_running: raise EnvironmentError("docker-compose is not running.") if not should_compose_run and is_running: diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index b4e4a3449c..78fc6739a8 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -5,10 +5,8 @@ from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH -from clp_package_utils.controller import DockerComposeController +from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( - dump_shared_container_config, - generate_docker_compose_container_config, get_clp_home, load_config_file, validate_and_load_db_credentials_file, @@ -61,8 +59,9 @@ def main(argv): logger.exception("Failed to load config.") return -1 + instance_id = get_or_create_instance_id(clp_config) try: - controller = DockerComposeController(clp_config) + controller = DockerComposeController(clp_config, instance_id) controller.deploy() except Exception as ex: if type(ex) == ValueError: diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index f0d54e6de5..5506e81f08 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -3,7 +3,7 @@ from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH -from clp_package_utils.controller import DockerComposeController +from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( get_clp_home, load_config_file, @@ -22,8 +22,9 @@ def main(): logger.exception("Failed to load config.") return -1 + instance_id = get_or_create_instance_id(clp_config) try: - controller = DockerComposeController(clp_config) + controller = DockerComposeController(clp_config, instance_id) controller.stop() except: logger.exception("Failed to stop CLP.") diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a49e4d3cd5..e259afb985 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -17,6 +17,7 @@ x-healthcheck-defaults: &healthcheck_defaults networks: clp-network: driver: "bridge" + name: "clp-network" secrets: CLP_DB_PASS_FILE: From 93034fe2a0a43335e0eaef9d21cd62be296f8d84 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 19:34:28 -0400 Subject: [PATCH 089/408] remove network to use default network with bridge driver --- tools/deployment/package/docker-compose.base.yaml | 6 ------ tools/deployment/package/docker-compose.yaml | 1 - 2 files changed, 7 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e259afb985..01afde0fb9 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -2,7 +2,6 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults - networks: ["clp-network"] stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" @@ -14,11 +13,6 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "10s" timeout: "10s" -networks: - clp-network: - driver: "bridge" - name: "clp-network" - secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 5e60c09797..9e89a28a0d 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -5,7 +5,6 @@ include: # Common service defaults. x-service-defaults: &service_defaults - networks: ["clp-network"] stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" From ca620094ce7622be4711895f9827a13d32d2b886 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 24 Sep 2025 23:43:02 -0400 Subject: [PATCH 090/408] fix(controller): update clp_home to be private variable. --- .../clp_package_utils/controller.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 407937766b..1ee3c17aff 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -69,8 +69,8 @@ class BaseController(ABC): def __init__(self, clp_config: CLPConfig): self.clp_config = clp_config - self.clp_home = get_clp_home() - self._conf_dir = self.clp_home / "etc" + self._clp_home = get_clp_home() + self._conf_dir = self._clp_home / "etc" @abstractmethod def deploy(self): @@ -309,10 +309,10 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( - self.clp_home / "var" / "www" / "webui" / "client" / "settings.json" + self._clp_home / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - self.clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" + self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) validate_webui_config(self.clp_config, client_settings_json_path, server_settings_json_path) @@ -483,7 +483,7 @@ def deploy(self): try: subprocess.run( cmd, - cwd=self.clp_home, + cwd=self._clp_home, stderr=subprocess.STDOUT, check=True, ) @@ -501,7 +501,7 @@ def stop(self): try: subprocess.run( ["docker", "compose", "--project-name", self._project_name, "down"], - cwd=self.clp_home, + cwd=self._clp_home, stderr=subprocess.STDOUT, check=True, ) @@ -565,7 +565,7 @@ def _provision(self): if self.clp_config.aws_config_directory is not None: env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) - with open(f"{self.clp_home}/.env", "w") as env_file: + with open(f"{self._clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") From 82cc48c3527306734e620b37cbdb0a0ab84d437f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 00:41:52 -0400 Subject: [PATCH 091/408] Use unique ids to launch dev clp package. --- .../clp-py-utils/clp_py_utils/clp_config.py | 22 ++++++++++++++----- taskfile.yaml | 1 + taskfiles/docker-images.yaml | 1 + tools/docker-images/clp-package/build.sh | 17 +++++++++++++- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 691623321c..33658baad5 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,6 +1,7 @@ import os import pathlib from enum import auto +from importlib.metadata import version from typing import ClassVar, Literal, Optional, Set, Union from pydantic import BaseModel, PrivateAttr, root_validator, validator @@ -36,8 +37,6 @@ COMPRESSION_JOBS_TABLE_NAME = "compression_jobs" COMPRESSION_TASKS_TABLE_NAME = "compression_tasks" -OS_RELEASE_FILE_PATH = pathlib.Path("etc") / "os-release" - # Paths CONTAINER_AWS_CONFIG_DIRECTORY = pathlib.Path("/") / ".aws" CONTAINER_CLP_HOME = pathlib.Path("/") / "opt" / "clp" @@ -52,7 +51,9 @@ CLP_DEFAULT_LOG_DIRECTORY_PATH = pathlib.Path("var") / "log" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" +CLP_PACKAGE_IMAGE_ID_PATH = pathlib.Path("image.id") CLP_SHARED_CONFIG_FILENAME = ".clp-config.yml" +CLP_VERSION_FILE_PATH = pathlib.Path("VERSION") # Environment variable names CLP_DB_USER_ENV_VAR_NAME = "CLP_DB_USER" @@ -861,7 +862,8 @@ class CLPConfig(BaseModel): logs_directory: pathlib.Path = CLP_DEFAULT_LOG_DIRECTORY_PATH aws_config_directory: Optional[pathlib.Path] = None - _os_release_file_path: pathlib.Path = PrivateAttr(default=OS_RELEASE_FILE_PATH) + _image_id_path: pathlib.Path = PrivateAttr(default=CLP_PACKAGE_IMAGE_ID_PATH) + _version_file_path: pathlib.Path = PrivateAttr(default=CLP_VERSION_FILE_PATH) def make_config_paths_absolute(self, clp_home: pathlib.Path): if StorageType.FS == self.logs_input.type: @@ -871,7 +873,8 @@ def make_config_paths_absolute(self, clp_home: pathlib.Path): self.stream_output.storage.make_config_paths_absolute(clp_home) self.data_directory = make_config_path_absolute(clp_home, self.data_directory) self.logs_directory = make_config_path_absolute(clp_home, self.logs_directory) - self._os_release_file_path = make_config_path_absolute(clp_home, self._os_release_file_path) + self._image_id_path = make_config_path_absolute(clp_home, self._image_id_path) + self._version_file_path = make_config_path_absolute(clp_home, self._version_file_path) def validate_logs_input_config(self): logs_input_type = self.logs_input.type @@ -959,10 +962,17 @@ def validate_aws_config_dir(self): def load_execution_container_name(self): if self.execution_container is not None: - # Accept configured value for releases + # Accept configured value for debug purposes return - self.execution_container = "clp-package:dev" + if self._image_id_path.exists(): + with open(self._image_id_path) as image_id_file: + self.execution_container = image_id_file.read().strip() + + if not bool(self.execution_container): + with open(self._version_file_path) as version_file: + package_version = version_file.read().strip() + self.execution_container = f"ghcr.io/y-scope/clp/clp-package:{package_version}" def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME diff --git a/taskfile.yaml b/taskfile.yaml index 347d89866b..f9933a5121 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -172,6 +172,7 @@ tasks: rsync -a "tools/deployment/package/" "{{.OUTPUT_DIR}}" + - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" # This command must be last - task: "utils:checksum:compute" vars: diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 9a546f713c..c3e6ca57b5 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -12,3 +12,4 @@ tasks: - ":package" cmds: - "./build.sh" + - "rsync -a 'image.id' '{{.G_PACKAGE_BUILD_DIR}}'" diff --git a/tools/docker-images/clp-package/build.sh b/tools/docker-images/clp-package/build.sh index 39df1a1105..f44e40bc1e 100755 --- a/tools/docker-images/clp-package/build.sh +++ b/tools/docker-images/clp-package/build.sh @@ -4,11 +4,26 @@ set -eu set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +iid_file="${script_dir}/image.id" repo_root=${script_dir}/../../../ +# Remove the previous image after the build to allow layer reuse. +prev_image_id="" +if [[ -f "$iid_file" ]]; then + prev_image_id=$(cat "$iid_file") +fi +cleanup() { + if [[ -n "$prev_image_id" ]] && docker image inspect "$prev_image_id" >/dev/null 2>&1; then + echo "Removing previous image $prev_image_id" + docker image remove "$prev_image_id" + fi +} +trap cleanup EXIT + build_cmd=( docker build - --tag "clp-package:dev" + --iidfile "$iid_file" + --tag "clp-package:dev-${USER}-$(date +%s)" "$repo_root" --file "${script_dir}/Dockerfile" ) From 8f6ba1bdf03b2d75144c4f82bd8c06f6dc20e11d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 00:47:56 -0400 Subject: [PATCH 092/408] remove trailing space --- docs/src/dev-docs/design-docker-compose.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index d8fa7800ad..0f58aca712 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -4,7 +4,7 @@ This document explains the technical details of CLP's Docker Compose implementat ## Overview -The Docker Compose implementation depends on a new controller architecture with a `BaseController` +The Docker Compose implementation depends on a new controller architecture with a `BaseController` abstract class and a `DockerComposeController` implementation. ## Architecture From 2aff301de7a8f588f11894db6fce16f0496ab72a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 00:53:59 -0400 Subject: [PATCH 093/408] remove `execution_container` field from clp-config.yml template --- components/package-template/src/etc/clp-config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index 3433a3e3a0..736f9a2ad5 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -1,6 +1,3 @@ -## Docker image to use for CLP package execution. -#execution_container: "clp-package:dev" -# ## Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all ## workers. #logs_input: From 5c91bf391075808eea1f2ee1684c4622f222fd09 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 00:55:07 -0400 Subject: [PATCH 094/408] fix(docker-compose): update `include` array syntax to use [] format. --- tools/deployment/package/docker-compose.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 9e89a28a0d..5e63b59884 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -1,7 +1,6 @@ name: "clp-package" -include: - - "docker-compose.base.yaml" +include: ["docker-compose.base.yaml"] # Common service defaults. x-service-defaults: &service_defaults From 049e269d1fe62b50b039ffdce8063112db1aa163 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 01:07:37 -0400 Subject: [PATCH 095/408] update service name from `db` to `database`. --- docs/src/dev-docs/design-docker-compose.md | 6 +++--- tools/deployment/package/docker-compose.base.yaml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index 0f58aca712..add169870b 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -52,7 +52,7 @@ The Docker Compose setup includes the following services: :::{mermaid} graph LR %% Services - db["db (MySQL)"] + database["database (MySQL)"] queue["queue (RabbitMQ)"] redis["redis (Redis)"] results_cache["results-cache (MongoDB)"] @@ -69,7 +69,7 @@ graph LR results_cache_indices_creator["results-cache-indices-creator"] %% Dependencies - db -->|healthy| db_table_creator + database -->|healthy| db_table_creator results_cache -->|healthy| results_cache_indices_creator db_table_creator -->|completed_successfully| compression_scheduler queue -->|healthy| compression_scheduler @@ -85,7 +85,7 @@ graph LR results_cache_indices_creator -->|completed_successfully| garbage_collector subgraph Databases - db + database queue redis results_cache diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 01afde0fb9..2b5bcfdb6b 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -18,7 +18,7 @@ secrets: environment: "CLP_DB_PASS" services: - db: + database: <<: *service_defaults container_name: "database" image: "${CLP_DB_IMAGE:-mysql:8.0.23}" @@ -58,7 +58,7 @@ services: volumes: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" depends_on: - db: + database: condition: "service_healthy" command: [ "python3", @@ -173,7 +173,7 @@ services: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" depends_on: - db-table-creator: + database: condition: "service_completed_successfully" queue: condition: "service_healthy" @@ -245,7 +245,7 @@ services: - "./var/www/webui/server/dist/settings.json\ :/opt/clp/var/www/webui/server/dist/settings.json:ro" depends_on: - db-table-creator: + database: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" @@ -277,7 +277,7 @@ services: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: - db-table-creator: + database: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" From fc70c054a1865d38db8e226d30d62741a3dbea40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 01:09:44 -0400 Subject: [PATCH 096/408] fix(docker-compose, controller): correct configuration file variable name. --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- tools/deployment/package/docker-compose.base.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 1ee3c17aff..1b4132ad74 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -196,7 +196,7 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: _chown_paths_if_root(data_dir, logs_dir) return { - "CLP_RESULTS_CACHE_CONF_DIR_HOST": str(conf_file), + "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self.clp_config.results_cache.host), diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 2b5bcfdb6b..628af466ae 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -123,7 +123,7 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "${CLP_RESULTS_CACHE_CONF_DIR_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" healthcheck: From 2cb1807234020cf6b734b9eaf14636de39dedf23 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 01:14:21 -0400 Subject: [PATCH 097/408] add _HOST postfix to path env var names for consistency --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- tools/deployment/package/docker-compose.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 1b4132ad74..7e3d7067fa 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -273,7 +273,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: return { "CLP_QUERY_WORKER_LOGGING_LEVEL": self.clp_config.query_worker.logging_level, - "CLP_QUERY_WORKER_LOGS_DIR": str(logs_dir), + "CLP_QUERY_WORKER_LOGS_DIR_HOST": str(logs_dir), "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } @@ -292,7 +292,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: return { "CLP_REDUCER_LOGGING_LEVEL": self.clp_config.reducer.logging_level, - "CLP_REDUCER_LOGS_DIR": str(logs_dir), + "CLP_REDUCER_LOGS_DIR_HOST": str(logs_dir), "CLP_REDUCER_CONCURRENCY": str(num_workers), "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), } diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 5e63b59884..319e11c694 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -76,7 +76,7 @@ services: - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_QUERY_WORKER_LOGS_DIR:-./var/log/query_worker}:/var/log/query_worker" + - "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}:/var/log/query_worker" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" command: [ @@ -103,7 +103,7 @@ services: PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_REDUCER_LOGS_DIR:-./var/log/reducer}:/var/log/reducer" + - "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}:/var/log/reducer" depends_on: query-scheduler: condition: "service_healthy" From d31e9705fb20a1d91f8f65e3b765f97838e327b8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 25 Sep 2025 11:29:24 -0400 Subject: [PATCH 098/408] Move celery compression task to separate file --- .../executor/compress/celery_compress.py | 114 ++++++++++++++++++ .../executor/compress/compression_task.py | 102 +--------------- .../compress/compression_scheduler.py | 2 +- 3 files changed, 117 insertions(+), 101 deletions(-) create mode 100644 components/job-orchestration/job_orchestration/executor/compress/celery_compress.py diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py new file mode 100644 index 0000000000..e1eabd56ea --- /dev/null +++ b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py @@ -0,0 +1,114 @@ +import datetime +import os +import pathlib +from contextlib import closing + +from celery import signals +from celery.app.task import Task +from celery.utils.log import get_task_logger +from clp_py_utils.clp_config import Database, WorkerConfig +from clp_py_utils.clp_logging import set_logging_level +from clp_py_utils.core import read_yaml_config_file +from clp_py_utils.sql_adapter import SQL_Adapter +from job_orchestration.executor.compress.celery import app +from job_orchestration.executor.compress.compression_task import ( + increment_compression_job_metadata, + run_clp, + update_compression_task_metadata, +) +from job_orchestration.scheduler.constants import CompressionTaskStatus +from job_orchestration.scheduler.job_config import ClpIoConfig, PathsToCompress +from job_orchestration.scheduler.scheduler_data import CompressionTaskResult + +# Setup logging +logger = get_task_logger(__name__) + + +@signals.worker_shutdown.connect +def worker_shutdown_handler(signal=None, sender=None, **kwargs): + logger.info("Shutdown signal received.") + + +@app.task(bind=True) +def compress( + self: Task, + job_id: int, + task_id: int, + tag_ids, + clp_io_config_json: str, + paths_to_compress_json: str, + clp_metadata_db_connection_config, +): + clp_home = pathlib.Path(os.getenv("CLP_HOME")) + + # Set logging level + logs_dir = pathlib.Path(os.getenv("CLP_LOGS_DIR")) + clp_logging_level = str(os.getenv("CLP_LOGGING_LEVEL")) + set_logging_level(logger, clp_logging_level) + + # Load configuration + try: + worker_config = WorkerConfig.parse_obj( + read_yaml_config_file(pathlib.Path(os.getenv("CLP_CONFIG_PATH"))) + ) + except Exception as ex: + error_msg = "Failed to load worker config" + logger.exception(error_msg) + return CompressionTaskResult( + task_id=task_id, + status=CompressionTaskStatus.FAILED, + duration=0, + error_message=error_msg, + ) + + clp_io_config = ClpIoConfig.parse_raw(clp_io_config_json) + paths_to_compress = PathsToCompress.parse_raw(paths_to_compress_json) + + sql_adapter = SQL_Adapter(Database.parse_obj(clp_metadata_db_connection_config)) + + start_time = datetime.datetime.now() + logger.info(f"[job_id={job_id} task_id={task_id}] COMPRESSION STARTED.") + compression_task_status, worker_output = run_clp( + worker_config, + clp_io_config, + clp_home, + logs_dir, + job_id, + task_id, + tag_ids, + paths_to_compress, + sql_adapter, + clp_metadata_db_connection_config, + logger, + ) + duration = (datetime.datetime.now() - start_time).total_seconds() + logger.info(f"[job_id={job_id} task_id={task_id}] COMPRESSION COMPLETED.") + + with closing(sql_adapter.create_connection(True)) as db_conn, closing( + db_conn.cursor(dictionary=True) + ) as db_cursor: + update_compression_task_metadata( + db_cursor, + task_id, + dict( + start_time=start_time, + status=compression_task_status, + partition_uncompressed_size=worker_output["total_uncompressed_size"], + partition_compressed_size=worker_output["total_compressed_size"], + duration=duration, + ), + ) + if CompressionTaskStatus.SUCCEEDED == compression_task_status: + increment_compression_job_metadata(db_cursor, job_id, dict(num_tasks_completed=1)) + db_conn.commit() + + compression_task_result = CompressionTaskResult( + task_id=task_id, + status=compression_task_status, + duration=duration, + ) + + if CompressionTaskStatus.FAILED == compression_task_status: + compression_task_result.error_message = worker_output["error_message"] + + return compression_task_result.dict() diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index a80f762c79..115b9ef3a5 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -1,4 +1,3 @@ -import datetime import json import os import pathlib @@ -6,9 +5,6 @@ from contextlib import closing from typing import Any, Dict, List, Optional, Tuple -from celery import signals -from celery.app.task import Task -from celery.utils.log import get_task_logger from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, @@ -20,19 +16,16 @@ StorageType, WorkerConfig, ) -from clp_py_utils.clp_logging import set_logging_level from clp_py_utils.clp_metadata_db_utils import ( get_archive_tags_table_name, get_archives_table_name, ) -from clp_py_utils.core import read_yaml_config_file from clp_py_utils.s3_utils import ( generate_s3_virtual_hosted_style_url, get_credential_env_vars, s3_put, ) from clp_py_utils.sql_adapter import SQL_Adapter -from job_orchestration.executor.compress.celery import app from job_orchestration.scheduler.constants import CompressionTaskStatus from job_orchestration.scheduler.job_config import ( ClpIoConfig, @@ -40,10 +33,6 @@ PathsToCompress, S3InputConfig, ) -from job_orchestration.scheduler.scheduler_data import CompressionTaskResult - -# Setup logging -logger = get_task_logger(__name__) def update_compression_task_metadata(db_cursor, task_id, kv): @@ -322,6 +311,7 @@ def run_clp( paths_to_compress: PathsToCompress, sql_adapter: SQL_Adapter, clp_metadata_db_connection_config, + logger, ): """ Compresses logs into archives. @@ -336,6 +326,7 @@ def run_clp( :param paths_to_compress: PathToCompress :param sql_adapter: SQL_Adapter :param clp_metadata_db_connection_config + :param logger :return: tuple -- (whether compression was successful, output messages) """ instance_id_str = f"compression-job-{job_id}-task-{task_id}" @@ -521,92 +512,3 @@ def run_clp( error_msgs.append(s3_error) worker_output["error_message"] = "\n".join(error_msgs) return CompressionTaskStatus.FAILED, worker_output - - -@signals.worker_shutdown.connect -def worker_shutdown_handler(signal=None, sender=None, **kwargs): - logger.info("Shutdown signal received.") - - -@app.task(bind=True) -def compress( - self: Task, - job_id: int, - task_id: int, - tag_ids, - clp_io_config_json: str, - paths_to_compress_json: str, - clp_metadata_db_connection_config, -): - clp_home = pathlib.Path(os.getenv("CLP_HOME")) - - # Set logging level - logs_dir = pathlib.Path(os.getenv("CLP_LOGS_DIR")) - clp_logging_level = str(os.getenv("CLP_LOGGING_LEVEL")) - set_logging_level(logger, clp_logging_level) - - # Load configuration - try: - worker_config = WorkerConfig.parse_obj( - read_yaml_config_file(pathlib.Path(os.getenv("CLP_CONFIG_PATH"))) - ) - except Exception as ex: - error_msg = "Failed to load worker config" - logger.exception(error_msg) - return CompressionTaskResult( - task_id=task_id, - status=CompressionTaskStatus.FAILED, - duration=0, - error_message=error_msg, - ) - - clp_io_config = ClpIoConfig.parse_raw(clp_io_config_json) - paths_to_compress = PathsToCompress.parse_raw(paths_to_compress_json) - - sql_adapter = SQL_Adapter(Database.parse_obj(clp_metadata_db_connection_config)) - - start_time = datetime.datetime.now() - logger.info(f"[job_id={job_id} task_id={task_id}] COMPRESSION STARTED.") - compression_task_status, worker_output = run_clp( - worker_config, - clp_io_config, - clp_home, - logs_dir, - job_id, - task_id, - tag_ids, - paths_to_compress, - sql_adapter, - clp_metadata_db_connection_config, - ) - duration = (datetime.datetime.now() - start_time).total_seconds() - logger.info(f"[job_id={job_id} task_id={task_id}] COMPRESSION COMPLETED.") - - with closing(sql_adapter.create_connection(True)) as db_conn, closing( - db_conn.cursor(dictionary=True) - ) as db_cursor: - update_compression_task_metadata( - db_cursor, - task_id, - dict( - start_time=start_time, - status=compression_task_status, - partition_uncompressed_size=worker_output["total_uncompressed_size"], - partition_compressed_size=worker_output["total_compressed_size"], - duration=duration, - ), - ) - if CompressionTaskStatus.SUCCEEDED == compression_task_status: - increment_compression_job_metadata(db_cursor, job_id, dict(num_tasks_completed=1)) - db_conn.commit() - - compression_task_result = CompressionTaskResult( - task_id=task_id, - status=compression_task_status, - duration=duration, - ) - - if CompressionTaskStatus.FAILED == compression_task_status: - compression_task_result.error_message = worker_output["error_message"] - - return compression_task_result.dict() diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index e844394f8e..46fcb2dcc4 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -31,7 +31,7 @@ from clp_py_utils.core import read_yaml_config_file from clp_py_utils.s3_utils import s3_get_object_metadata from clp_py_utils.sql_adapter import SQL_Adapter -from job_orchestration.executor.compress.compression_task import compress +from job_orchestration.executor.compress.celery_compress import compress from job_orchestration.scheduler.compress.partition import PathsToCompressBuffer from job_orchestration.scheduler.constants import ( CompressionJobStatus, From 562ddce9710657ba6bd119271ea06f43d07e5d90 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 25 Sep 2025 12:09:18 -0400 Subject: [PATCH 099/408] Add spider task and refactor common compression task --- .../executor/compress/celery_compress.py | 90 ++----------------- .../executor/compress/compression_task.py | 88 ++++++++++++++++++ .../executor/compress/spider_comrpess.py | 26 ++++++ 3 files changed, 119 insertions(+), 85 deletions(-) create mode 100644 components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py index e1eabd56ea..818b958e85 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py @@ -1,24 +1,8 @@ -import datetime -import os -import pathlib -from contextlib import closing - from celery import signals from celery.app.task import Task from celery.utils.log import get_task_logger -from clp_py_utils.clp_config import Database, WorkerConfig -from clp_py_utils.clp_logging import set_logging_level -from clp_py_utils.core import read_yaml_config_file -from clp_py_utils.sql_adapter import SQL_Adapter from job_orchestration.executor.compress.celery import app -from job_orchestration.executor.compress.compression_task import ( - increment_compression_job_metadata, - run_clp, - update_compression_task_metadata, -) -from job_orchestration.scheduler.constants import CompressionTaskStatus -from job_orchestration.scheduler.job_config import ClpIoConfig, PathsToCompress -from job_orchestration.scheduler.scheduler_data import CompressionTaskResult +from job_orchestration.executor.compress.compression_task import general_compress # Setup logging logger = get_task_logger(__name__) @@ -31,7 +15,7 @@ def worker_shutdown_handler(signal=None, sender=None, **kwargs): @app.task(bind=True) def compress( - self: Task, + _: Task, job_id: int, task_id: int, tag_ids, @@ -39,76 +23,12 @@ def compress( paths_to_compress_json: str, clp_metadata_db_connection_config, ): - clp_home = pathlib.Path(os.getenv("CLP_HOME")) - - # Set logging level - logs_dir = pathlib.Path(os.getenv("CLP_LOGS_DIR")) - clp_logging_level = str(os.getenv("CLP_LOGGING_LEVEL")) - set_logging_level(logger, clp_logging_level) - - # Load configuration - try: - worker_config = WorkerConfig.parse_obj( - read_yaml_config_file(pathlib.Path(os.getenv("CLP_CONFIG_PATH"))) - ) - except Exception as ex: - error_msg = "Failed to load worker config" - logger.exception(error_msg) - return CompressionTaskResult( - task_id=task_id, - status=CompressionTaskStatus.FAILED, - duration=0, - error_message=error_msg, - ) - - clp_io_config = ClpIoConfig.parse_raw(clp_io_config_json) - paths_to_compress = PathsToCompress.parse_raw(paths_to_compress_json) - - sql_adapter = SQL_Adapter(Database.parse_obj(clp_metadata_db_connection_config)) - - start_time = datetime.datetime.now() - logger.info(f"[job_id={job_id} task_id={task_id}] COMPRESSION STARTED.") - compression_task_status, worker_output = run_clp( - worker_config, - clp_io_config, - clp_home, - logs_dir, + general_compress( job_id, task_id, tag_ids, - paths_to_compress, - sql_adapter, + clp_io_config_json, + paths_to_compress_json, clp_metadata_db_connection_config, logger, ) - duration = (datetime.datetime.now() - start_time).total_seconds() - logger.info(f"[job_id={job_id} task_id={task_id}] COMPRESSION COMPLETED.") - - with closing(sql_adapter.create_connection(True)) as db_conn, closing( - db_conn.cursor(dictionary=True) - ) as db_cursor: - update_compression_task_metadata( - db_cursor, - task_id, - dict( - start_time=start_time, - status=compression_task_status, - partition_uncompressed_size=worker_output["total_uncompressed_size"], - partition_compressed_size=worker_output["total_compressed_size"], - duration=duration, - ), - ) - if CompressionTaskStatus.SUCCEEDED == compression_task_status: - increment_compression_job_metadata(db_cursor, job_id, dict(num_tasks_completed=1)) - db_conn.commit() - - compression_task_result = CompressionTaskResult( - task_id=task_id, - status=compression_task_status, - duration=duration, - ) - - if CompressionTaskStatus.FAILED == compression_task_status: - compression_task_result.error_message = worker_output["error_message"] - - return compression_task_result.dict() diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index 115b9ef3a5..78ceada7b2 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -1,3 +1,4 @@ +import datetime import json import os import pathlib @@ -16,10 +17,12 @@ StorageType, WorkerConfig, ) +from clp_py_utils.clp_logging import set_logging_level from clp_py_utils.clp_metadata_db_utils import ( get_archive_tags_table_name, get_archives_table_name, ) +from clp_py_utils.core import read_yaml_config_file from clp_py_utils.s3_utils import ( generate_s3_virtual_hosted_style_url, get_credential_env_vars, @@ -33,6 +36,7 @@ PathsToCompress, S3InputConfig, ) +from job_orchestration.scheduler.scheduler_data import CompressionTaskResult def update_compression_task_metadata(db_cursor, task_id, kv): @@ -512,3 +516,87 @@ def run_clp( error_msgs.append(s3_error) worker_output["error_message"] = "\n".join(error_msgs) return CompressionTaskStatus.FAILED, worker_output + + +def general_compress( + job_id: int, + task_id: int, + tag_ids, + clp_io_config_json: str, + paths_to_compress_json: str, + clp_metadata_db_connection_config, + logger, +): + clp_home = pathlib.Path(os.getenv("CLP_HOME")) + + # Set logging level + logs_dir = pathlib.Path(os.getenv("CLP_LOGS_DIR")) + clp_logging_level = str(os.getenv("CLP_LOGGING_LEVEL")) + set_logging_level(logger, clp_logging_level) + + # Load configuration + try: + worker_config = WorkerConfig.parse_obj( + read_yaml_config_file(pathlib.Path(os.getenv("CLP_CONFIG_PATH"))) + ) + except Exception as ex: + error_msg = "Failed to load worker config" + logger.exception(error_msg) + return CompressionTaskResult( + task_id=task_id, + status=CompressionTaskStatus.FAILED, + duration=0, + error_message=error_msg, + ) + + clp_io_config = ClpIoConfig.parse_raw(clp_io_config_json) + paths_to_compress = PathsToCompress.parse_raw(paths_to_compress_json) + + sql_adapter = SQL_Adapter(Database.parse_obj(clp_metadata_db_connection_config)) + + start_time = datetime.datetime.now() + logger.info(f"[job_id={job_id} task_id={task_id}] COMPRESSION STARTED.") + compression_task_status, worker_output = run_clp( + worker_config, + clp_io_config, + clp_home, + logs_dir, + job_id, + task_id, + tag_ids, + paths_to_compress, + sql_adapter, + clp_metadata_db_connection_config, + logger, + ) + duration = (datetime.datetime.now() - start_time).total_seconds() + logger.info(f"[job_id={job_id} task_id={task_id}] COMPRESSION COMPLETED.") + + with closing(sql_adapter.create_connection(True)) as db_conn, closing( + db_conn.cursor(dictionary=True) + ) as db_cursor: + update_compression_task_metadata( + db_cursor, + task_id, + dict( + start_time=start_time, + status=compression_task_status, + partition_uncompressed_size=worker_output["total_uncompressed_size"], + partition_compressed_size=worker_output["total_compressed_size"], + duration=duration, + ), + ) + if CompressionTaskStatus.SUCCEEDED == compression_task_status: + increment_compression_job_metadata(db_cursor, job_id, dict(num_tasks_completed=1)) + db_conn.commit() + + compression_task_result = CompressionTaskResult( + task_id=task_id, + status=compression_task_status, + duration=duration, + ) + + if CompressionTaskStatus.FAILED == compression_task_status: + compression_task_result.error_message = worker_output["error_message"] + + return compression_task_result.dict() diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py new file mode 100644 index 0000000000..442124eac3 --- /dev/null +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py @@ -0,0 +1,26 @@ +from clp_py_utils.clp_logging import get_logger +from job_orchestration.executor.compress.compression_task import general_compress +from spider_py import TaskContext + +# Setup logging +logger = get_logger("spider_compression_scheduler") + + +def compress( + _: TaskContext, + job_id: int, + task_id: int, + tag_ids, + clp_io_config_json: str, + paths_to_compress_json: str, + clp_metadata_db_connection_config, +): + general_compress( + job_id, + task_id, + tag_ids, + clp_io_config_json, + paths_to_compress_json, + clp_metadata_db_connection_config, + logger, + ) From de92f5a26b4fabbab0b8691db3260f2ae5ccb6aa Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 25 Sep 2025 22:32:53 -0400 Subject: [PATCH 100/408] Fix celery setup --- .../job_orchestration/executor/compress/celeryconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/celeryconfig.py b/components/job-orchestration/job_orchestration/executor/compress/celeryconfig.py index 034e0870dd..9cd1e4d749 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celeryconfig.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celeryconfig.py @@ -6,13 +6,13 @@ # Force workers to consume only one task at a time worker_prefetch_multiplier = 1 imports = [ - "job_orchestration.executor.compress.compression_task", + "job_orchestration.executor.compress.celery_compress", ] # Queue settings task_queue_max_priority = TASK_QUEUE_HIGHEST_PRIORITY task_routes = { - "job_orchestration.executor.compress.compression_task.compress": SchedulerType.COMPRESSION, + "job_orchestration.executor.compress.celery_compress.compress": SchedulerType.COMPRESSION, } task_create_missing_queues = True From 336ca41dad1a237c3ddb7b105f5a9f2687b1dcbb Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 25 Sep 2025 22:50:09 -0400 Subject: [PATCH 101/408] Fix return --- .../executor/compress/celery_compress.py | 2 +- .../executor/compress/spider_comrpess.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py index 818b958e85..86a1a8afaf 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py @@ -23,7 +23,7 @@ def compress( paths_to_compress_json: str, clp_metadata_db_connection_config, ): - general_compress( + return general_compress( job_id, task_id, tag_ids, diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py index 442124eac3..96f20408d4 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py @@ -10,12 +10,12 @@ def compress( _: TaskContext, job_id: int, task_id: int, - tag_ids, + tag_ids: list[int], clp_io_config_json: str, paths_to_compress_json: str, - clp_metadata_db_connection_config, -): - general_compress( + clp_metadata_db_connection_config: str, +) -> dict[str, int | float | str | None]: + return general_compress( job_id, task_id, tag_ids, From 0ba30e6f8b2767fadaf6664c853b18574c662354 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 25 Sep 2025 23:30:17 -0400 Subject: [PATCH 102/408] Add docstring --- .../executor/compress/spider_comrpess.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py index 96f20408d4..77bdd111e5 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py @@ -13,8 +13,19 @@ def compress( tag_ids: list[int], clp_io_config_json: str, paths_to_compress_json: str, - clp_metadata_db_connection_config: str, + clp_metadata_db_connection_config: dict[str, bool | int | str | None], ) -> dict[str, int | float | str | None]: + """ + Compress files using the general compression function. + :param _: + :param job_id: + :param task_id: + :param tag_ids: + :param clp_io_config_json: A Json string representing `ClpIoConfig`. + :param paths_to_compress_json: A Json string representing `PathsToCompress`. + :param clp_metadata_db_connection_config: A dict representing `Database`. + :return: A dict representing `CompressionTaskResult`. + """ return general_compress( job_id, task_id, From e657feb111a86ac730a3b585536cacfc4078f030 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 25 Sep 2025 23:31:53 -0400 Subject: [PATCH 103/408] fix: the webui service should depend on db-table-creator rather than database --- tools/deployment/package/docker-compose.base.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 628af466ae..a0d5e301ae 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -173,7 +173,7 @@ services: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" depends_on: - database: + db-table-creator: condition: "service_completed_successfully" queue: condition: "service_healthy" @@ -245,7 +245,7 @@ services: - "./var/www/webui/server/dist/settings.json\ :/opt/clp/var/www/webui/server/dist/settings.json:ro" depends_on: - database: + db-table-creator: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" @@ -277,7 +277,7 @@ services: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: - database: + db-table-creator: condition: "service_completed_successfully" results-cache-indices-creator: condition: "service_completed_successfully" From 4330fc053a419d1083a76de87562e634761fd5da Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 25 Sep 2025 23:37:14 -0400 Subject: [PATCH 104/408] Fix docstring --- .../job_orchestration/executor/compress/spider_comrpess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py index 77bdd111e5..e04d10dc64 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py @@ -10,7 +10,7 @@ def compress( _: TaskContext, job_id: int, task_id: int, - tag_ids: list[int], + tag_ids: list[int] | None, clp_io_config_json: str, paths_to_compress_json: str, clp_metadata_db_connection_config: dict[str, bool | int | str | None], From 81bb21e95393a09a08517de1f165e5e4bc47812d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 26 Sep 2025 21:35:36 -0400 Subject: [PATCH 105/408] Fix file name --- .../executor/compress/{spider_comrpess.py => spider_compress.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename components/job-orchestration/job_orchestration/executor/compress/{spider_comrpess.py => spider_compress.py} (100%) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py similarity index 100% rename from components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py rename to components/job-orchestration/job_orchestration/executor/compress/spider_compress.py From 5c112d7e389cdba5cbbddc272bddd65748a98589 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 26 Sep 2025 22:21:28 -0400 Subject: [PATCH 106/408] Improve docstring Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- .../executor/compress/spider_comrpess.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py index e04d10dc64..b33b5f328d 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_comrpess.py @@ -16,15 +16,18 @@ def compress( clp_metadata_db_connection_config: dict[str, bool | int | str | None], ) -> dict[str, int | float | str | None]: """ - Compress files using the general compression function. - :param _: + Compresses files using the general compression entry point. + :param _: Spider's task context. Not used in the function. :param job_id: :param task_id: :param tag_ids: - :param clp_io_config_json: A Json string representing `ClpIoConfig`. - :param paths_to_compress_json: A Json string representing `PathsToCompress`. - :param clp_metadata_db_connection_config: A dict representing `Database`. - :return: A dict representing `CompressionTaskResult`. + :param clp_io_config_json: A JSON string representation of + `job_orchestration.scheduler.constants.ClpIoConfig`. + :param paths_to_compress_json: A JSON string representation of + `job_orchestration.scheduler.constants.PathToCompress`. + :param clp_metadata_db_connection_config: A dict representation of + `clp_py_utils.clp_config.Database`. + :return: A dict representation of `job_orchestration.scheduler.constants.CompressionTaskResult`. """ return general_compress( job_id, From 03093e7b4c2cc07fc112e5318b9ff32f4ab86c63 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 26 Sep 2025 22:23:16 -0400 Subject: [PATCH 107/408] Rename function --- .../job_orchestration/executor/compress/celery_compress.py | 4 ++-- .../job_orchestration/executor/compress/compression_task.py | 2 +- .../job_orchestration/executor/compress/spider_compress.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py index 86a1a8afaf..9c2d3c0583 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py @@ -2,7 +2,7 @@ from celery.app.task import Task from celery.utils.log import get_task_logger from job_orchestration.executor.compress.celery import app -from job_orchestration.executor.compress.compression_task import general_compress +from job_orchestration.executor.compress.compression_task import compression_entry_point # Setup logging logger = get_task_logger(__name__) @@ -23,7 +23,7 @@ def compress( paths_to_compress_json: str, clp_metadata_db_connection_config, ): - return general_compress( + return compression_entry_point( job_id, task_id, tag_ids, diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index 78ceada7b2..acc8c3f839 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -518,7 +518,7 @@ def run_clp( return CompressionTaskStatus.FAILED, worker_output -def general_compress( +def compression_entry_point( job_id: int, task_id: int, tag_ids, diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index b33b5f328d..c1297e0d47 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -1,5 +1,5 @@ from clp_py_utils.clp_logging import get_logger -from job_orchestration.executor.compress.compression_task import general_compress +from job_orchestration.executor.compress.compression_task import compression_entry_point from spider_py import TaskContext # Setup logging @@ -29,7 +29,7 @@ def compress( `clp_py_utils.clp_config.Database`. :return: A dict representation of `job_orchestration.scheduler.constants.CompressionTaskResult`. """ - return general_compress( + return compression_entry_point( job_id, task_id, tag_ids, From 28652b19ecdcb2420081c35371331b9fa526bd1f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 28 Sep 2025 11:02:06 -0400 Subject: [PATCH 108/408] Use self for celery task --- .../job_orchestration/executor/compress/celery_compress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py index 9c2d3c0583..8f9b82d8e2 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/celery_compress.py @@ -15,7 +15,7 @@ def worker_shutdown_handler(signal=None, sender=None, **kwargs): @app.task(bind=True) def compress( - _: Task, + self: Task, job_id: int, task_id: int, tag_ids, From 2889a290eb6afc402103c392e68a89d7876484ec Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 28 Sep 2025 11:05:24 -0400 Subject: [PATCH 109/408] Use json string for dict --- .../executor/compress/spider_compress.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index c1297e0d47..8a42f62f87 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -1,3 +1,5 @@ +import json + from clp_py_utils.clp_logging import get_logger from job_orchestration.executor.compress.compression_task import compression_entry_point from spider_py import TaskContext @@ -10,11 +12,11 @@ def compress( _: TaskContext, job_id: int, task_id: int, - tag_ids: list[int] | None, + tag_ids: list[int], clp_io_config_json: str, paths_to_compress_json: str, - clp_metadata_db_connection_config: dict[str, bool | int | str | None], -) -> dict[str, int | float | str | None]: + clp_metadata_db_connection_config_json: str, +) -> str: """ Compresses files using the general compression entry point. :param _: Spider's task context. Not used in the function. @@ -25,16 +27,17 @@ def compress( `job_orchestration.scheduler.constants.ClpIoConfig`. :param paths_to_compress_json: A JSON string representation of `job_orchestration.scheduler.constants.PathToCompress`. - :param clp_metadata_db_connection_config: A dict representation of + :param clp_metadata_db_connection_config_json: A JSON string representation of `clp_py_utils.clp_config.Database`. - :return: A dict representation of `job_orchestration.scheduler.constants.CompressionTaskResult`. + :return: A JSON string representation of + `job_orchestration.scheduler.constants.CompressionTaskResult`. """ - return compression_entry_point( + return json.load(compression_entry_point( job_id, task_id, tag_ids, clp_io_config_json, paths_to_compress_json, - clp_metadata_db_connection_config, + json.dumps(clp_metadata_db_connection_config_json), logger, - ) + )) From 8e07c387a3593618a1abfe5c0017615dd3e86ccf Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 28 Sep 2025 20:21:00 -0400 Subject: [PATCH 110/408] Use standard TDL list[Int8] for tasks --- .../executor/compress/spider_compress.py | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index 8a42f62f87..bc481c4e3c 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -1,22 +1,30 @@ -import json - +from clp_py_utils.clp_config import Database from clp_py_utils.clp_logging import get_logger from job_orchestration.executor.compress.compression_task import compression_entry_point -from spider_py import TaskContext +from spider_py import Int8, TaskContext # Setup logging logger = get_logger("spider_compression_scheduler") +def convert_to_str(byte_list: list[Int8]) -> str: + """ + Converts a list of Int8 to a UTF-8 string. + :param byte_list: A list of Int8 representing bytes. + :return: A UTF-8 string. + """ + return bytes(int(byte) for byte in byte_list).decode("utf-8") + + def compress( _: TaskContext, job_id: int, task_id: int, tag_ids: list[int], - clp_io_config_json: str, - paths_to_compress_json: str, - clp_metadata_db_connection_config_json: str, -) -> str: + clp_io_config_json: list[Int8], + paths_to_compress_json: list[Int8], + clp_metadata_db_connection_config_json: list[Int8], +) -> list[Int8]: """ Compresses files using the general compression entry point. :param _: Spider's task context. Not used in the function. @@ -32,12 +40,18 @@ def compress( :return: A JSON string representation of `job_orchestration.scheduler.constants.CompressionTaskResult`. """ - return json.load(compression_entry_point( - job_id, - task_id, - tag_ids, - clp_io_config_json, - paths_to_compress_json, - json.dumps(clp_metadata_db_connection_config_json), - logger, - )) + result_json_bytes = ( + compression_entry_point( + job_id, + task_id, + tag_ids, + convert_to_str(clp_io_config_json), + convert_to_str(paths_to_compress_json), + Database.model_validate_json(convert_to_str(clp_metadata_db_connection_config_json)), + logger, + ) + .model_dump_json() + .encode("utf-8") + ) + + return [Int8(byte) for byte in result_json_bytes] From 14a8efc11ffdd54918c3c2fcb9ae3d7de1c3908b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 28 Sep 2025 20:41:40 -0400 Subject: [PATCH 111/408] Fix json dump --- .../executor/compress/spider_compress.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index bc481c4e3c..c3f7023979 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -1,3 +1,5 @@ +import json + from clp_py_utils.clp_config import Database from clp_py_utils.clp_logging import get_logger from job_orchestration.executor.compress.compression_task import compression_entry_point @@ -40,7 +42,7 @@ def compress( :return: A JSON string representation of `job_orchestration.scheduler.constants.CompressionTaskResult`. """ - result_json_bytes = ( + result_json_bytes = json.dumps( compression_entry_point( job_id, task_id, @@ -50,8 +52,6 @@ def compress( Database.model_validate_json(convert_to_str(clp_metadata_db_connection_config_json)), logger, ) - .model_dump_json() - .encode("utf-8") - ) + ).encode("utf-8") return [Int8(byte) for byte in result_json_bytes] From e8f6e48cffb119fb26b374f04ead54f3f019fae9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 29 Sep 2025 20:33:28 -0400 Subject: [PATCH 112/408] Add scheduler interface --- .../scheduler/scheduler/__init__.py | 0 .../scheduler/scheduler/scheduler.py | 32 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 components/job-orchestration/job_orchestration/scheduler/scheduler/__init__.py create mode 100644 components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/__init__.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py new file mode 100644 index 0000000000..6baf5ee0ce --- /dev/null +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + +from job_orchestration.scheduler.scheduler_data import CompressionTaskResult + + +class Scheduler(ABC): + """Abstract base class for a scheduler framework.""" + + @abstractmethod + def compress(self, task_params: list[dict[str, Any]]) -> Any: + """ + Starts a batch of compression tasks as a job. + :param task_params: A list of dictionaries containing parameters for each compression task. + :return: A handle through which to get the result of the job. + """ + pass + + @abstractmethod + def get_compress_result( + self, result_handle: Any, timeout: float = 0.1 + ) -> list[CompressionTaskResult] | None: + """ + Gets the result of a compression job. + :param result_handle: The handle returned by the compress method. + :param timeout: The maximum time to wait for the result. Notice that some schedulers ignore + this parameter. + :return: A list of task results. + """ + pass From 48b9955bbbc73e49d32cca319107d8b3cf3c1bde Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 29 Sep 2025 21:12:22 -0400 Subject: [PATCH 113/408] Add spider scheduler --- .../compress/compression_scheduler.py | 30 +++++------- .../scheduler/scheduler/celery_scheduler.py | 28 +++++++++++ .../scheduler/scheduler/spider_scheduler.py | 46 +++++++++++++++++++ 3 files changed, 86 insertions(+), 18 deletions(-) create mode 100644 components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py create mode 100644 components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 46fcb2dcc4..66dfb7783f 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -10,7 +10,6 @@ from typing import Any, Dict, List, Optional, Set import brotli -import celery import msgpack from clp_package_utils.general import CONTAINER_INPUT_LOGS_ROOT_DIR from clp_py_utils.clp_config import ( @@ -31,7 +30,6 @@ from clp_py_utils.core import read_yaml_config_file from clp_py_utils.s3_utils import s3_get_object_metadata from clp_py_utils.sql_adapter import SQL_Adapter -from job_orchestration.executor.compress.celery_compress import compress from job_orchestration.scheduler.compress.partition import PathsToCompressBuffer from job_orchestration.scheduler.constants import ( CompressionJobStatus, @@ -44,9 +42,10 @@ InputType, S3InputConfig, ) +from job_orchestration.scheduler.scheduler.celery_scheduler import CeleryScheduler +from job_orchestration.scheduler.scheduler.scheduler import Scheduler from job_orchestration.scheduler.scheduler_data import ( CompressionJob, - CompressionTaskResult, ) from job_orchestration.scheduler.utils import kill_hanging_jobs from pydantic import ValidationError @@ -218,6 +217,7 @@ def search_and_schedule_new_tasks( db_conn, db_cursor, clp_metadata_db_connection_config: Dict[str, Any], + scheduler: Scheduler ): """ For all jobs with PENDING status, splits the job into tasks and schedules them. @@ -225,6 +225,7 @@ def search_and_schedule_new_tasks( :param db_conn: :param db_cursor: :param clp_metadata_db_connection_config: + :param scheduler: """ global scheduled_jobs @@ -357,7 +358,6 @@ def search_and_schedule_new_tasks( tag_ids = [tags["tag_id"] for tags in db_cursor.fetchall()] db_conn.commit() - task_instances = [] for task_idx, task in enumerate(tasks): db_cursor.execute( f""" @@ -370,11 +370,10 @@ def search_and_schedule_new_tasks( db_conn.commit() task["task_id"] = db_cursor.lastrowid task["tag_ids"] = tag_ids - task_instances.append(compress.s(**task)) - tasks_group = celery.group(task_instances) + result_handle = scheduler.compress(tasks) job = CompressionJob( - id=job_id, start_time=start_time, async_task_result=tasks_group.apply_async() + id=job_id, start_time=start_time, async_task_result=result_handle ) db_cursor.execute( f""" @@ -387,14 +386,7 @@ def search_and_schedule_new_tasks( scheduled_jobs[job_id] = job -def get_results_or_timeout(result): - try: - return result.get(timeout=0.1) - except celery.exceptions.TimeoutError: - return None - - -def poll_running_jobs(db_conn, db_cursor): +def poll_running_jobs(db_conn, db_cursor, scheduler: Scheduler): """ Poll for running jobs and update their status. """ @@ -408,14 +400,13 @@ def poll_running_jobs(db_conn, db_cursor): error_message = "" try: - returned_results = get_results_or_timeout(job.async_task_result) + returned_results = scheduler.get_compress_result(job.result_handle) if returned_results is None: continue duration = (datetime.datetime.now() - job.start_time).total_seconds() # Check for finished jobs for task_result in returned_results: - task_result = CompressionTaskResult.parse_obj(task_result) if task_result.status == CompressionTaskStatus.SUCCEEDED: logger.info( f"Compression task job-{job_id}-task-{task_result.task_id} completed in" @@ -498,6 +489,8 @@ def main(argv): logger.info(f"Starting {COMPRESSION_SCHEDULER_COMPONENT_NAME}") sql_adapter = SQL_Adapter(clp_config.database) + scheduler = CeleryScheduler() + try: killed_jobs = kill_hanging_jobs(sql_adapter, SchedulerType.COMPRESSION) if killed_jobs is not None: @@ -522,8 +515,9 @@ def main(argv): db_conn, db_cursor, clp_metadata_db_connection_config, + scheduler ) - poll_running_jobs(db_conn, db_cursor) + poll_running_jobs(db_conn, db_cursor, scheduler) time.sleep(clp_config.compression_scheduler.jobs_poll_delay) except KeyboardInterrupt: logger.info("Forcefully shutting down") diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py new file mode 100644 index 0000000000..b5a04a5b7e --- /dev/null +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Any + +import celery + +from job_orchestration.executor.compress.celery_compress import compress +from job_orchestration.scheduler.scheduler.scheduler import Scheduler +from job_orchestration.scheduler.scheduler_data import CompressionTaskResult + + +class CeleryScheduler(Scheduler): + + def compress(self, task_params: list[dict[str, Any]]) -> Any: + task_instances = [ + compress.s(**params) for params in task_params] + task_group = celery.group(task_instances) + return task_group.apply_async() + + + def get_compress_result( + self, result_handle: Any, timeout: float = 0.1 + ) -> list[CompressionTaskResult] | None: + try: + results = result_handle.get(timeout=timeout) + return [CompressionTaskResult.model_validate(**res) for res in results] + except celery.exceptions.TimeoutError: + return None diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py new file mode 100644 index 0000000000..81833722d1 --- /dev/null +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import json +from typing import Any + +import spider_py + +from job_orchestration.executor.compress.spider_compress import compress +from job_orchestration.scheduler.scheduler.scheduler import Scheduler +from job_orchestration.scheduler.scheduler_data import CompressionTaskResult + + +def convert_from_str(string: str) -> list[spider_py.Int8]: + """ + Convert a string to a list of `Int8` as utf-8 bytes. + :param string: The string to convert. + :return: The list of `Int8` representing the string. + """ + return [spider_py.Int8(byte) for byte in string.encode("utf-8")] + + +class SpiderScheduler(Scheduler): + + def __init__(self, storage_url: str) -> None: + self.driver = spider_py.Driver(storage_url) + + def compress(self, task_params: list[dict[str, Any]]) -> Any: + job = spider_py.group( + [compress for _ in range(len(task_params))], + ) + return self.driver.submit_jobs(job, [ + spider_py.Int64(task_params["job_id"]), + spider_py.Int64(task_params["task_id"]), + [spider_py.Int64(tag_id) for tag_id in task_params["tag_ids"]], + convert_from_str(task_params["clp_io_config_json"]), + convert_from_str(task_params["paths_to_compress_json"]), + convert_from_str(json.loads(task_params["clp_io_config_json"])), + ]) + + def get_compress_result( + self, result_handle: Any, timeout: float = 0.1 + ) -> list[CompressionTaskResult] | None: + results = result_handle.get_results() + if results is None: + return None + return [CompressionTaskResult.validate_model_json(result) for result in results] \ No newline at end of file From 8adc2d5493a004ca0bc5128368f7ef4a078b961a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 29 Sep 2025 21:15:14 -0400 Subject: [PATCH 114/408] Bug fix --- .../executor/compress/spider_compress.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index c3f7023979..99d2e848ba 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -3,7 +3,7 @@ from clp_py_utils.clp_config import Database from clp_py_utils.clp_logging import get_logger from job_orchestration.executor.compress.compression_task import compression_entry_point -from spider_py import Int8, TaskContext +from spider_py import Int8, Int64, TaskContext # Setup logging logger = get_logger("spider_compression_scheduler") @@ -20,9 +20,9 @@ def convert_to_str(byte_list: list[Int8]) -> str: def compress( _: TaskContext, - job_id: int, - task_id: int, - tag_ids: list[int], + job_id: Int64, + task_id: Int64, + tag_ids: list[Int64], clp_io_config_json: list[Int8], paths_to_compress_json: list[Int8], clp_metadata_db_connection_config_json: list[Int8], @@ -44,9 +44,9 @@ def compress( """ result_json_bytes = json.dumps( compression_entry_point( - job_id, - task_id, - tag_ids, + int(job_id), + int(task_id), + [int(tag_id) for tag_id in tag_ids], convert_to_str(clp_io_config_json), convert_to_str(paths_to_compress_json), Database.model_validate_json(convert_to_str(clp_metadata_db_connection_config_json)), From ebc746fd4b8c967ae1037958d4907927a50a1a8c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 29 Sep 2025 21:17:35 -0400 Subject: [PATCH 115/408] Bug fix --- .../job_orchestration/scheduler/scheduler/spider_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py index 81833722d1..43db40d7de 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py @@ -5,7 +5,7 @@ import spider_py -from job_orchestration.executor.compress.spider_compress import compress +from job_orchestration.executor.compress.spider_compress import compress, convert_to_str from job_orchestration.scheduler.scheduler.scheduler import Scheduler from job_orchestration.scheduler.scheduler_data import CompressionTaskResult @@ -43,4 +43,4 @@ def get_compress_result( results = result_handle.get_results() if results is None: return None - return [CompressionTaskResult.validate_model_json(result) for result in results] \ No newline at end of file + return [CompressionTaskResult.validate_model_json(convert_to_str(result)) for result in results] \ No newline at end of file From 3f6355dd76aec42cf2c530aabfe6d80fb6abbbdd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 29 Sep 2025 21:22:00 -0400 Subject: [PATCH 116/408] Fix lint --- .../compress/compression_scheduler.py | 12 +++------- .../scheduler/scheduler/celery_scheduler.py | 4 +--- .../scheduler/scheduler/spider_scheduler.py | 23 +++++++++++-------- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 66dfb7783f..54519d071e 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -217,7 +217,7 @@ def search_and_schedule_new_tasks( db_conn, db_cursor, clp_metadata_db_connection_config: Dict[str, Any], - scheduler: Scheduler + scheduler: Scheduler, ): """ For all jobs with PENDING status, splits the job into tasks and schedules them. @@ -372,9 +372,7 @@ def search_and_schedule_new_tasks( task["tag_ids"] = tag_ids result_handle = scheduler.compress(tasks) - job = CompressionJob( - id=job_id, start_time=start_time, async_task_result=result_handle - ) + job = CompressionJob(id=job_id, start_time=start_time, async_task_result=result_handle) db_cursor.execute( f""" UPDATE {COMPRESSION_TASKS_TABLE_NAME} @@ -511,11 +509,7 @@ def main(argv): try: if not received_sigterm: search_and_schedule_new_tasks( - clp_config, - db_conn, - db_cursor, - clp_metadata_db_connection_config, - scheduler + clp_config, db_conn, db_cursor, clp_metadata_db_connection_config, scheduler ) poll_running_jobs(db_conn, db_cursor, scheduler) time.sleep(clp_config.compression_scheduler.jobs_poll_delay) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py index b5a04a5b7e..74ca62cf7b 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py @@ -12,12 +12,10 @@ class CeleryScheduler(Scheduler): def compress(self, task_params: list[dict[str, Any]]) -> Any: - task_instances = [ - compress.s(**params) for params in task_params] + task_instances = [compress.s(**params) for params in task_params] task_group = celery.group(task_instances) return task_group.apply_async() - def get_compress_result( self, result_handle: Any, timeout: float = 0.1 ) -> list[CompressionTaskResult] | None: diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py index 43db40d7de..382b74499f 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py @@ -28,14 +28,17 @@ def compress(self, task_params: list[dict[str, Any]]) -> Any: job = spider_py.group( [compress for _ in range(len(task_params))], ) - return self.driver.submit_jobs(job, [ - spider_py.Int64(task_params["job_id"]), - spider_py.Int64(task_params["task_id"]), - [spider_py.Int64(tag_id) for tag_id in task_params["tag_ids"]], - convert_from_str(task_params["clp_io_config_json"]), - convert_from_str(task_params["paths_to_compress_json"]), - convert_from_str(json.loads(task_params["clp_io_config_json"])), - ]) + return self.driver.submit_jobs( + job, + [ + spider_py.Int64(task_params["job_id"]), + spider_py.Int64(task_params["task_id"]), + [spider_py.Int64(tag_id) for tag_id in task_params["tag_ids"]], + convert_from_str(task_params["clp_io_config_json"]), + convert_from_str(task_params["paths_to_compress_json"]), + convert_from_str(json.loads(task_params["clp_io_config_json"])), + ], + ) def get_compress_result( self, result_handle: Any, timeout: float = 0.1 @@ -43,4 +46,6 @@ def get_compress_result( results = result_handle.get_results() if results is None: return None - return [CompressionTaskResult.validate_model_json(convert_to_str(result)) for result in results] \ No newline at end of file + return [ + CompressionTaskResult.validate_model_json(convert_to_str(result)) for result in results + ] From a5759cd26a9a558267b725d11e8c342e66cf4c85 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 29 Sep 2025 21:22:27 -0400 Subject: [PATCH 117/408] Fix import --- .../job_orchestration/scheduler/scheduler/celery_scheduler.py | 1 - .../job_orchestration/scheduler/scheduler/spider_scheduler.py | 1 - 2 files changed, 2 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py index 74ca62cf7b..6da3e91319 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py @@ -3,7 +3,6 @@ from typing import Any import celery - from job_orchestration.executor.compress.celery_compress import compress from job_orchestration.scheduler.scheduler.scheduler import Scheduler from job_orchestration.scheduler.scheduler_data import CompressionTaskResult diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py index 382b74499f..7b28b698ff 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py @@ -4,7 +4,6 @@ from typing import Any import spider_py - from job_orchestration.executor.compress.spider_compress import compress, convert_to_str from job_orchestration.scheduler.scheduler.scheduler import Scheduler from job_orchestration.scheduler.scheduler_data import CompressionTaskResult From 17ff4c9c0a97e8bc14fb600fefec1b99e8f9ab87 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 22:46:27 -0400 Subject: [PATCH 118/408] refactor(config): Use enum types for package storage and query engines and update serialization. --- .../clp-py-utils/clp_py_utils/clp_config.py | 34 +++++-------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 1ac4d7083f..58bff02a08 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -123,33 +123,9 @@ class AwsAuthType(LowercaseStrEnum): ec2 = auto() -VALID_STORAGE_ENGINES = [storage_engine.value for storage_engine in StorageEngine] -VALID_QUERY_ENGINES = [query_engine.value for query_engine in QueryEngine] - - class Package(BaseModel): - storage_engine: str = "clp" - query_engine: str = "clp" - - @field_validator("storage_engine") - @classmethod - def validate_storage_engine(cls, value): - if value not in VALID_STORAGE_ENGINES: - raise ValueError( - f"package.storage_engine must be one of the following" - f" {'|'.join(VALID_STORAGE_ENGINES)}" - ) - return value - - @field_validator("query_engine") - @classmethod - def validate_query_engine(cls, value): - if value not in VALID_QUERY_ENGINES: - raise ValueError( - f"package.query_engine must be one of the following" - f" {'|'.join(VALID_QUERY_ENGINES)}" - ) - return value + storage_engine: StorageEngine = StorageEngine.CLP + query_engine: QueryEngine = QueryEngine.CLP @model_validator(mode="after") def validate_query_engine_package_compatibility(self): @@ -173,6 +149,11 @@ def validate_query_engine_package_compatibility(self): return self + def dump_to_primitive_dict(self): + d = self.model_dump() + d["storage_engine"] = d["storage_engine"].value + d["query_engine"] = d["query_engine"].value + return d class Database(BaseModel): type: str = "mariadb" @@ -1035,6 +1016,7 @@ def get_runnable_components(self) -> Set[str]: def dump_to_primitive_dict(self): custom_serialized_fields = ( + "package", "database", "queue", "redis", From 4c014852a6588f614c6440682ec8a2cd02714448 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 22:48:31 -0400 Subject: [PATCH 119/408] refactor(config): Change custom_serialized_fields to a set and pass it directly to model_dump. --- components/clp-py-utils/clp_py_utils/clp_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 58bff02a08..8b1dfc784b 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1015,7 +1015,7 @@ def get_runnable_components(self) -> Set[str]: return ALL_COMPONENTS def dump_to_primitive_dict(self): - custom_serialized_fields = ( + custom_serialized_fields = { "package", "database", "queue", @@ -1023,8 +1023,8 @@ def dump_to_primitive_dict(self): "logs_input", "archive_output", "stream_output", - ) - d = self.model_dump(exclude=set(custom_serialized_fields)) + } + d = self.model_dump(exclude=custom_serialized_fields) for key in custom_serialized_fields: d[key] = getattr(self, key).dump_to_primitive_dict() From 3e484e20d4d3ade00a48c8288b91b9c342c3b85f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:04:35 -0400 Subject: [PATCH 120/408] refactor(config): Use shared annotated Port type for port fields and drop custom port validators. --- .../clp-py-utils/clp_py_utils/clp_config.py | 79 +++---------------- 1 file changed, 13 insertions(+), 66 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 8b1dfc784b..131d6fe293 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Any, Literal, Optional, Set, Union +from typing import Annotated, Any, Literal, Optional, Set, Union from dotenv import dotenv_values from pydantic import ( @@ -99,6 +99,9 @@ CLP_QUEUE_PASS_ENV_VAR_NAME = "CLP_QUEUE_PASS" CLP_REDIS_PASS_ENV_VAR_NAME = "CLP_REDIS_PASS" +# Types +Port = Annotated[int, Field(gt=0, lt=2**16)] + class StorageEngine(KebabCaseStrEnum): CLP = auto() @@ -155,10 +158,11 @@ def dump_to_primitive_dict(self): d["query_engine"] = d["query_engine"].value return d + class Database(BaseModel): type: str = "mariadb" host: str = "localhost" - port: int = 3306 + port: Port = 3306 name: str = "clp-db" ssl_cert: Optional[str] = None auto_commit: bool = False @@ -191,12 +195,6 @@ def validate_host(cls, value): raise ValueError("database.host cannot be empty.") return value - @field_validator("port") - @classmethod - def validate_port(cls, value): - _validate_port(cls, value) - return value - def ensure_credentials_loaded(self): if self.username is None or self.password is None: raise ValueError("Credentials not loaded.") @@ -281,15 +279,6 @@ def _validate_host(cls, value): raise ValueError(f"{cls.__name__}.host cannot be empty.") -def _validate_port(cls, value): - min_valid_port = 0 - max_valid_port = 2**16 - 1 - if min_valid_port > value or max_valid_port < value: - raise ValueError( - f"{cls.__name__}.port is not within valid range " f"{min_valid_port}-{max_valid_port}." - ) - - class CompressionScheduler(BaseModel): jobs_poll_delay: float = 0.1 # seconds logging_level: str = "INFO" @@ -303,7 +292,7 @@ def validate_logging_level(cls, value): class QueryScheduler(BaseModel): host: str = "localhost" - port: int = 7000 + port: Port = 7000 jobs_poll_delay: float = 0.1 # seconds num_archives_to_search_per_sub_job: int = 16 logging_level: str = "INFO" @@ -321,12 +310,6 @@ def validate_host(cls, value): raise ValueError(f"Cannot be empty.") return value - @field_validator("port") - @classmethod - def validate_port(cls, value): - _validate_port(cls, value) - return value - class CompressionWorker(BaseModel): logging_level: str = "INFO" @@ -350,7 +333,7 @@ def validate_logging_level(cls, value): class Redis(BaseModel): host: str = "localhost" - port: int = 6379 + port: Port = 6379 query_backend_database: int = 0 compression_backend_database: int = 1 # redis can perform authentication without a username @@ -363,12 +346,6 @@ def validate_host(cls, value): raise ValueError(f"{REDIS_COMPONENT_NAME}.host cannot be empty.") return value - @field_validator("port") - @classmethod - def validate_port(cls, value): - _validate_port(cls, value) - return value - def dump_to_primitive_dict(self): return self.model_dump(exclude={"password"}) @@ -392,7 +369,7 @@ def load_credentials_from_env(self): class Reducer(BaseModel): host: str = "localhost" - base_port: int = 14009 + base_port: Port = 14009 logging_level: str = "INFO" upsert_interval: int = 100 # milliseconds @@ -409,12 +386,6 @@ def validate_logging_level(cls, value): _validate_logging_level(cls, value) return value - @field_validator("base_port") - @classmethod - def validate_base_port(cls, value): - _validate_port(cls, value) - return value - @field_validator("upsert_interval") @classmethod def validate_upsert_interval(cls, value): @@ -425,7 +396,7 @@ def validate_upsert_interval(cls, value): class ResultsCache(BaseModel): host: str = "localhost" - port: int = 27017 + port: Port = 27017 db_name: str = "clp-query-results" stream_collection_name: str = "stream-files" retention_period: Optional[int] = 60 @@ -437,12 +408,6 @@ def validate_host(cls, value): raise ValueError(f"{RESULTS_CACHE_COMPONENT_NAME}.host cannot be empty.") return value - @field_validator("port") - @classmethod - def validate_port(cls, value): - _validate_port(cls, value) - return value - @field_validator("db_name") @classmethod def validate_db_name(cls, value): @@ -472,7 +437,7 @@ def get_uri(self): class Queue(BaseModel): host: str = "localhost" - port: int = 5672 + port: Port = 5672 username: Optional[str] = None password: Optional[str] = None @@ -484,12 +449,6 @@ def validate_host(cls, value): raise ValueError(f"{QUEUE_COMPONENT_NAME}.host cannot be empty.") return value - @field_validator("port") - @classmethod - def validate_port(cls, value): - _validate_port(cls, value) - return value - def dump_to_primitive_dict(self): return self.model_dump(exclude={"username", "password"}) @@ -781,7 +740,7 @@ def dump_to_primitive_dict(self): class WebUi(BaseModel): host: str = "localhost" - port: int = 4000 + port: Port = 4000 results_metadata_collection_name: str = "results-metadata" rate_limit: int = 1000 @@ -791,12 +750,6 @@ def validate_host(cls, value): _validate_host(cls, value) return value - @field_validator("port") - @classmethod - def validate_port(cls, value): - _validate_port(cls, value) - return value - @field_validator("results_metadata_collection_name") @classmethod def validate_results_metadata_collection_name(cls, value): @@ -834,7 +787,7 @@ def validate_logging_level(cls, value): class Presto(BaseModel): host: str - port: int + port: Port @field_validator("host") @classmethod @@ -842,12 +795,6 @@ def validate_host(cls, value): _validate_host(cls, value) return value - @field_validator("port") - @classmethod - def validate_port(cls, value): - _validate_port(cls, value) - return value - def _get_env_var(name: str) -> str: value = os.getenv(name) From d4419938e8976c2d6edd79949eacaba3a6d6c89a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:08:37 -0400 Subject: [PATCH 121/408] refactor(config): Consolidate host validation by introducing shared Host type and removing duplicated validators. --- .../clp-py-utils/clp_py_utils/clp_config.py | 76 +++---------------- 1 file changed, 9 insertions(+), 67 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 131d6fe293..b04fc553ad 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -100,6 +100,7 @@ CLP_REDIS_PASS_ENV_VAR_NAME = "CLP_REDIS_PASS" # Types +Host = Annotated[str, Field(min_length=1)] Port = Annotated[int, Field(gt=0, lt=2**16)] @@ -161,7 +162,7 @@ def dump_to_primitive_dict(self): class Database(BaseModel): type: str = "mariadb" - host: str = "localhost" + host: Host = "localhost" port: Port = 3306 name: str = "clp-db" ssl_cert: Optional[str] = None @@ -188,13 +189,6 @@ def validate_name(cls, value): raise ValueError("database.name cannot be empty.") return value - @field_validator("host") - @classmethod - def validate_host(cls, value): - if "" == value: - raise ValueError("database.host cannot be empty.") - return value - def ensure_credentials_loaded(self): if self.username is None or self.password is None: raise ValueError("Credentials not loaded.") @@ -274,11 +268,6 @@ def _validate_logging_level(cls, value): ) -def _validate_host(cls, value): - if "" == value: - raise ValueError(f"{cls.__name__}.host cannot be empty.") - - class CompressionScheduler(BaseModel): jobs_poll_delay: float = 0.1 # seconds logging_level: str = "INFO" @@ -291,7 +280,7 @@ def validate_logging_level(cls, value): class QueryScheduler(BaseModel): - host: str = "localhost" + host: Host = "localhost" port: Port = 7000 jobs_poll_delay: float = 0.1 # seconds num_archives_to_search_per_sub_job: int = 16 @@ -303,13 +292,6 @@ def validate_logging_level(cls, value): _validate_logging_level(cls, value) return value - @field_validator("host") - @classmethod - def validate_host(cls, value): - if "" == value: - raise ValueError(f"Cannot be empty.") - return value - class CompressionWorker(BaseModel): logging_level: str = "INFO" @@ -332,20 +314,13 @@ def validate_logging_level(cls, value): class Redis(BaseModel): - host: str = "localhost" + host: Host = "localhost" port: Port = 6379 query_backend_database: int = 0 compression_backend_database: int = 1 # redis can perform authentication without a username password: Optional[str] = None - @field_validator("host") - @classmethod - def validate_host(cls, value): - if "" == value: - raise ValueError(f"{REDIS_COMPONENT_NAME}.host cannot be empty.") - return value - def dump_to_primitive_dict(self): return self.model_dump(exclude={"password"}) @@ -368,18 +343,11 @@ def load_credentials_from_env(self): class Reducer(BaseModel): - host: str = "localhost" + host: Host = "localhost" base_port: Port = 14009 logging_level: str = "INFO" upsert_interval: int = 100 # milliseconds - @field_validator("host") - @classmethod - def validate_host(cls, value): - if "" == value: - raise ValueError(f"{value} cannot be empty") - return value - @field_validator("logging_level") @classmethod def validate_logging_level(cls, value): @@ -395,19 +363,12 @@ def validate_upsert_interval(cls, value): class ResultsCache(BaseModel): - host: str = "localhost" + host: Host = "localhost" port: Port = 27017 db_name: str = "clp-query-results" stream_collection_name: str = "stream-files" retention_period: Optional[int] = 60 - @field_validator("host") - @classmethod - def validate_host(cls, value): - if "" == value: - raise ValueError(f"{RESULTS_CACHE_COMPONENT_NAME}.host cannot be empty.") - return value - @field_validator("db_name") @classmethod def validate_db_name(cls, value): @@ -436,19 +397,12 @@ def get_uri(self): class Queue(BaseModel): - host: str = "localhost" + host: Host = "localhost" port: Port = 5672 username: Optional[str] = None password: Optional[str] = None - @field_validator("host") - @classmethod - def validate_host(cls, value): - if "" == value: - raise ValueError(f"{QUEUE_COMPONENT_NAME}.host cannot be empty.") - return value - def dump_to_primitive_dict(self): return self.model_dump(exclude={"username", "password"}) @@ -739,17 +693,11 @@ def dump_to_primitive_dict(self): class WebUi(BaseModel): - host: str = "localhost" + host: Host = "localhost" port: Port = 4000 results_metadata_collection_name: str = "results-metadata" rate_limit: int = 1000 - @field_validator("host") - @classmethod - def validate_host(cls, value): - _validate_host(cls, value) - return value - @field_validator("results_metadata_collection_name") @classmethod def validate_results_metadata_collection_name(cls, value): @@ -786,15 +734,9 @@ def validate_logging_level(cls, value): class Presto(BaseModel): - host: str + host: Host port: Port - @field_validator("host") - @classmethod - def validate_host(cls, value): - _validate_host(cls, value) - return value - def _get_env_var(name: str) -> str: value = os.getenv(name) From 68a3a06093a9598fb4f1dca9d7588a3279f45272 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:17:16 -0400 Subject: [PATCH 122/408] refactor(config): Introduce DatabaseEngine enum, use it for database.type and adjust serialization accordingly. --- .../clp-py-utils/clp_py_utils/clp_config.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index b04fc553ad..f5a7ca2e59 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -109,6 +109,11 @@ class StorageEngine(KebabCaseStrEnum): CLP_S = auto() +class DatabaseEngine(KebabCaseStrEnum): + MARIADB = auto() + MYSQL = auto() + + class QueryEngine(KebabCaseStrEnum): CLP = auto() CLP_S = auto() @@ -161,7 +166,7 @@ def dump_to_primitive_dict(self): class Database(BaseModel): - type: str = "mariadb" + type: DatabaseEngine = DatabaseEngine.MARIADB host: Host = "localhost" port: Port = 3306 name: str = "clp-db" @@ -172,16 +177,6 @@ class Database(BaseModel): username: Optional[str] = None password: Optional[str] = None - @field_validator("type") - @classmethod - def validate_type(cls, value): - supported_database_types = ["mysql", "mariadb"] - if value not in supported_database_types: - raise ValueError( - f"database.type must be one of the following {'|'.join(supported_database_types)}" - ) - return value - @field_validator("name") @classmethod def validate_name(cls, value): @@ -223,7 +218,7 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection connection_params_and_type = { # NOTE: clp-core does not distinguish between mysql and mariadb - "type": "mysql", + "type": DatabaseEngine.MYSQL.value, "host": host, "port": self.port, "username": self.username, @@ -238,7 +233,9 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection return connection_params_and_type def dump_to_primitive_dict(self): - return self.model_dump(exclude={"username", "password"}) + d = self.model_dump(exclude={"username", "password"}) + d["type"] = d["type"].value + return d def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) From fbe91b0d19a4cbdd06b0a96304c6e94d21619e45 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:21:51 -0400 Subject: [PATCH 123/408] =?UTF-8?q?refactor(config):=20Replace=20manual=20?= =?UTF-8?q?non=E2=80=91empty=20string=20validators=20with=20shared=20NonEm?= =?UTF-8?q?ptyStr=20type.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clp-py-utils/clp_py_utils/clp_config.py | 79 +++---------------- 1 file changed, 10 insertions(+), 69 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index f5a7ca2e59..97758f631d 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -100,7 +100,8 @@ CLP_REDIS_PASS_ENV_VAR_NAME = "CLP_REDIS_PASS" # Types -Host = Annotated[str, Field(min_length=1)] +NonEmptyStr = Annotated[str, Field(min_length=1)] +Host = NonEmptyStr Port = Annotated[int, Field(gt=0, lt=2**16)] @@ -169,7 +170,7 @@ class Database(BaseModel): type: DatabaseEngine = DatabaseEngine.MARIADB host: Host = "localhost" port: Port = 3306 - name: str = "clp-db" + name: NonEmptyStr = "clp-db" ssl_cert: Optional[str] = None auto_commit: bool = False compress: bool = True @@ -177,13 +178,6 @@ class Database(BaseModel): username: Optional[str] = None password: Optional[str] = None - @field_validator("name") - @classmethod - def validate_name(cls, value): - if "" == value: - raise ValueError("database.name cannot be empty.") - return value - def ensure_credentials_loaded(self): if self.username is None or self.password is None: raise ValueError("Credentials not loaded.") @@ -362,26 +356,10 @@ def validate_upsert_interval(cls, value): class ResultsCache(BaseModel): host: Host = "localhost" port: Port = 27017 - db_name: str = "clp-query-results" - stream_collection_name: str = "stream-files" + db_name: NonEmptyStr = "clp-query-results" + stream_collection_name: NonEmptyStr = "stream-files" retention_period: Optional[int] = 60 - @field_validator("db_name") - @classmethod - def validate_db_name(cls, value): - if "" == value: - raise ValueError(f"{RESULTS_CACHE_COMPONENT_NAME}.db_name cannot be empty.") - return value - - @field_validator("stream_collection_name") - @classmethod - def validate_stream_collection_name(cls, value): - if "" == value: - raise ValueError( - f"{RESULTS_CACHE_COMPONENT_NAME}.stream_collection_name cannot be empty." - ) - return value - @field_validator("retention_period") @classmethod def validate_retention_period(cls, value): @@ -424,24 +402,10 @@ def load_credentials_from_env(self): class S3Credentials(BaseModel): - access_key_id: str - secret_access_key: str + access_key_id: NonEmptyStr + secret_access_key: NonEmptyStr session_token: Optional[str] = None - @field_validator("access_key_id") - @classmethod - def validate_access_key_id(cls, value): - if "" == value: - raise ValueError("access_key_id cannot be empty") - return value - - @field_validator("secret_access_key") - @classmethod - def validate_secret_access_key(cls, value): - if "" == value: - raise ValueError("secret_access_key cannot be empty") - return value - class AwsAuthentication(BaseModel): type: Literal[ @@ -482,25 +446,11 @@ def validate_authentication(cls, data): class S3Config(BaseModel): - region_code: str - bucket: str + region_code: NonEmptyStr + bucket: NonEmptyStr key_prefix: str aws_authentication: AwsAuthentication - @field_validator("region_code") - @classmethod - def validate_region_code(cls, value): - if "" == value: - raise ValueError("region_code cannot be empty") - return value - - @field_validator("bucket") - @classmethod - def validate_bucket(cls, value): - if "" == value: - raise ValueError("bucket cannot be empty") - return value - class S3IngestionConfig(BaseModel): type: Literal[StorageType.S3.value] = StorageType.S3.value @@ -692,18 +642,9 @@ def dump_to_primitive_dict(self): class WebUi(BaseModel): host: Host = "localhost" port: Port = 4000 - results_metadata_collection_name: str = "results-metadata" + results_metadata_collection_name: NonEmptyStr = "results-metadata" rate_limit: int = 1000 - @field_validator("results_metadata_collection_name") - @classmethod - def validate_results_metadata_collection_name(cls, value): - if "" == value: - raise ValueError( - f"{WEBUI_COMPONENT_NAME}.results_metadata_collection_name cannot be empty." - ) - return value - @field_validator("rate_limit") @classmethod def validate_rate_limit(cls, value): From e829a8615702d39225db266da5c366e42e5cf964 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:30:21 -0400 Subject: [PATCH 124/408] refactor(config): Use PositiveFloat type for jobs_poll_delay fields in scheduler configs. --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 97758f631d..fbc4bc9e0f 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -100,6 +100,7 @@ CLP_REDIS_PASS_ENV_VAR_NAME = "CLP_REDIS_PASS" # Types +PositiveFloat = Annotated[float, Field(gt=0)] NonEmptyStr = Annotated[str, Field(min_length=1)] Host = NonEmptyStr Port = Annotated[int, Field(gt=0, lt=2**16)] @@ -260,7 +261,7 @@ def _validate_logging_level(cls, value): class CompressionScheduler(BaseModel): - jobs_poll_delay: float = 0.1 # seconds + jobs_poll_delay: PositiveFloat = 0.1 # seconds logging_level: str = "INFO" @field_validator("logging_level") @@ -273,7 +274,7 @@ def validate_logging_level(cls, value): class QueryScheduler(BaseModel): host: Host = "localhost" port: Port = 7000 - jobs_poll_delay: float = 0.1 # seconds + jobs_poll_delay: PositiveFloat = 0.1 # seconds num_archives_to_search_per_sub_job: int = 16 logging_level: str = "INFO" From 79b80433e9ab6672f790ae09a98422453f4072c8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:31:26 -0400 Subject: [PATCH 125/408] Refactor(config): Rename jobs_poll_delay to jobs_poll_delay_sec in scheduler configs and update references. --- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ++-- .../scheduler/compress/compression_scheduler.py | 2 +- .../job_orchestration/scheduler/query/query_scheduler.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index fbc4bc9e0f..4b45ba191b 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -261,7 +261,7 @@ def _validate_logging_level(cls, value): class CompressionScheduler(BaseModel): - jobs_poll_delay: PositiveFloat = 0.1 # seconds + jobs_poll_delay_sec: PositiveFloat = 0.1 logging_level: str = "INFO" @field_validator("logging_level") @@ -274,7 +274,7 @@ def validate_logging_level(cls, value): class QueryScheduler(BaseModel): host: Host = "localhost" port: Port = 7000 - jobs_poll_delay: PositiveFloat = 0.1 # seconds + jobs_poll_delay_sec: PositiveFloat = 0.1 num_archives_to_search_per_sub_job: int = 16 logging_level: str = "INFO" diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index a12a0cf6a6..4613b72276 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -523,7 +523,7 @@ def main(argv): clp_metadata_db_connection_config, ) poll_running_jobs(db_conn, db_cursor) - time.sleep(clp_config.compression_scheduler.jobs_poll_delay) + time.sleep(clp_config.compression_scheduler.jobs_poll_delay_sec) except KeyboardInterrupt: logger.info("Forcefully shutting down") return -1 diff --git a/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py index 7e73572725..417bbecdf1 100644 --- a/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py @@ -1180,7 +1180,7 @@ async def main(argv: List[str]) -> int: logger.exception("Failed to kill hanging query jobs.") return -1 - logger.debug(f"Job polling interval {clp_config.query_scheduler.jobs_poll_delay} seconds.") + logger.debug(f"Job polling interval {clp_config.query_scheduler.jobs_poll_delay_sec} seconds.") try: reducer_handler = await asyncio.start_server( lambda reader, writer: handle_reducer_connection( @@ -1214,7 +1214,7 @@ async def main(argv: List[str]) -> int: ), results_cache_uri=clp_config.results_cache.get_uri(), stream_collection_name=clp_config.results_cache.stream_collection_name, - jobs_poll_delay=clp_config.query_scheduler.jobs_poll_delay, + jobs_poll_delay=clp_config.query_scheduler.jobs_poll_delay_sec, num_archives_to_search_per_sub_job=batch_size, archive_retention_period=clp_config.archive_output.retention_period, ) From 354b019fb3f1ec592ed43d74cc0332d100791248 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:37:03 -0400 Subject: [PATCH 126/408] refactor(config): Replace int fields with PositiveInt and remove redundant validators. --- .../clp-py-utils/clp_py_utils/clp_config.py | 93 ++++--------------- 1 file changed, 16 insertions(+), 77 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4b45ba191b..e3c8b7730d 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -99,9 +99,11 @@ CLP_QUEUE_PASS_ENV_VAR_NAME = "CLP_QUEUE_PASS" CLP_REDIS_PASS_ENV_VAR_NAME = "CLP_REDIS_PASS" -# Types -PositiveFloat = Annotated[float, Field(gt=0)] +# Generic types NonEmptyStr = Annotated[str, Field(min_length=1)] +PositiveFloat = Annotated[float, Field(gt=0)] +PositiveInt = Annotated[int, Field(gt=0)] +# Type aliases Host = NonEmptyStr Port = Annotated[int, Field(gt=0, lt=2**16)] @@ -275,7 +277,7 @@ class QueryScheduler(BaseModel): host: Host = "localhost" port: Port = 7000 jobs_poll_delay_sec: PositiveFloat = 0.1 - num_archives_to_search_per_sub_job: int = 16 + num_archives_to_search_per_sub_job: PositiveInt = 16 logging_level: str = "INFO" @field_validator("logging_level") @@ -338,7 +340,7 @@ class Reducer(BaseModel): host: Host = "localhost" base_port: Port = 14009 logging_level: str = "INFO" - upsert_interval: int = 100 # milliseconds + upsert_interval: PositiveInt = 100 # milliseconds @field_validator("logging_level") @classmethod @@ -346,27 +348,13 @@ def validate_logging_level(cls, value): _validate_logging_level(cls, value) return value - @field_validator("upsert_interval") - @classmethod - def validate_upsert_interval(cls, value): - if not value > 0: - raise ValueError(f"{value} is not greater than zero") - return value - class ResultsCache(BaseModel): host: Host = "localhost" port: Port = 27017 db_name: NonEmptyStr = "clp-query-results" stream_collection_name: NonEmptyStr = "stream-files" - retention_period: Optional[int] = 60 - - @field_validator("retention_period") - @classmethod - def validate_retention_period(cls, value): - if value is not None and value <= 0: - raise ValueError("retention_period must be greater than 0") - return value + retention_period: Optional[PositiveInt] = 60 def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" @@ -556,40 +544,12 @@ def _set_directory_for_storage_config( class ArchiveOutput(BaseModel): storage: Union[ArchiveFsStorage, ArchiveS3Storage] = ArchiveFsStorage() - target_archive_size: int = 256 * 1024 * 1024 # 256 MB - target_dictionaries_size: int = 32 * 1024 * 1024 # 32 MB - target_encoded_file_size: int = 256 * 1024 * 1024 # 256 MB - target_segment_size: int = 256 * 1024 * 1024 # 256 MB + target_archive_size: PositiveInt = 256 * 1024 * 1024 # 256 MB + target_dictionaries_size: PositiveInt = 32 * 1024 * 1024 # 32 MB + target_encoded_file_size: PositiveInt = 256 * 1024 * 1024 # 256 MB + target_segment_size: PositiveInt = 256 * 1024 * 1024 # 256 MB compression_level: int = 3 - retention_period: Optional[int] = None - - @field_validator("target_archive_size") - @classmethod - def validate_target_archive_size(cls, value): - if value <= 0: - raise ValueError("target_archive_size must be greater than 0") - return value - - @field_validator("target_dictionaries_size") - @classmethod - def validate_target_dictionaries_size(cls, value): - if value <= 0: - raise ValueError("target_dictionaries_size must be greater than 0") - return value - - @field_validator("target_encoded_file_size") - @classmethod - def validate_target_encoded_file_size(cls, value): - if value <= 0: - raise ValueError("target_encoded_file_size must be greater than 0") - return value - - @field_validator("target_segment_size") - @classmethod - def validate_target_segment_size(cls, value): - if value <= 0: - raise ValueError("target_segment_size must be greater than 0") - return value + retention_period: Optional[PositiveInt] = None @field_validator("compression_level") @classmethod @@ -598,13 +558,6 @@ def validate_compression_level(cls, value): raise ValueError("compression_level must be a value from 1 to 19") return value - @field_validator("retention_period") - @classmethod - def validate_retention_period(cls, value): - if value is not None and value <= 0: - raise ValueError("retention_period must be greater than 0") - return value - def set_directory(self, directory: pathlib.Path): _set_directory_for_storage_config(self.storage, directory) @@ -619,14 +572,7 @@ def dump_to_primitive_dict(self): class StreamOutput(BaseModel): storage: Union[StreamFsStorage, StreamS3Storage] = StreamFsStorage() - target_uncompressed_size: int = 128 * 1024 * 1024 - - @field_validator("target_uncompressed_size") - @classmethod - def validate_target_uncompressed_size(cls, value): - if value <= 0: - raise ValueError("target_uncompressed_size must be greater than 0") - return value + target_uncompressed_size: PositiveInt = 128 * 1024 * 1024 def set_directory(self, directory: pathlib.Path): _set_directory_for_storage_config(self.storage, directory) @@ -644,21 +590,14 @@ class WebUi(BaseModel): host: Host = "localhost" port: Port = 4000 results_metadata_collection_name: NonEmptyStr = "results-metadata" - rate_limit: int = 1000 - - @field_validator("rate_limit") - @classmethod - def validate_rate_limit(cls, value): - if value <= 0: - raise ValueError(f"rate_limit must be greater than 0") - return value + rate_limit: PositiveInt = 1000 class SweepInterval(BaseModel): model_config = ConfigDict(extra="forbid") - archive: int = Field(default=60, gt=0) - search_result: int = Field(default=30, gt=0) + archive: PositiveInt = 60 + search_result: PositiveInt = 30 class GarbageCollector(BaseModel): From e60b39a6bfbab3133577b67fb8da3b4e41d69105 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:41:53 -0400 Subject: [PATCH 127/408] refactor(config): Update optional string fields to use NonEmptyStr type. --- components/clp-py-utils/clp_py_utils/clp_config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index e3c8b7730d..b7de55246b 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -174,7 +174,7 @@ class Database(BaseModel): host: Host = "localhost" port: Port = 3306 name: NonEmptyStr = "clp-db" - ssl_cert: Optional[str] = None + ssl_cert: Optional[NonEmptyStr] = None auto_commit: bool = False compress: bool = True @@ -364,7 +364,7 @@ class Queue(BaseModel): host: Host = "localhost" port: Port = 5672 - username: Optional[str] = None + username: Optional[NonEmptyStr] = None password: Optional[str] = None def dump_to_primitive_dict(self): @@ -393,7 +393,7 @@ def load_credentials_from_env(self): class S3Credentials(BaseModel): access_key_id: NonEmptyStr secret_access_key: NonEmptyStr - session_token: Optional[str] = None + session_token: Optional[NonEmptyStr] = None class AwsAuthentication(BaseModel): @@ -403,7 +403,7 @@ class AwsAuthentication(BaseModel): AwsAuthType.env_vars.value, AwsAuthType.ec2.value, ] - profile: Optional[str] = None + profile: Optional[NonEmptyStr] = None credentials: Optional[S3Credentials] = None @model_validator(mode="before") @@ -624,7 +624,7 @@ def _get_env_var(name: str) -> str: class CLPConfig(BaseModel): - execution_container: Optional[str] = None + execution_container: Optional[NonEmptyStr] = None logs_input: Union[FsIngestionConfig, S3IngestionConfig] = FsIngestionConfig() From 3372a4be33edcf420eab72dd63d38f119a44fc03 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:46:39 -0400 Subject: [PATCH 128/408] docs(config): Update comment to specify specific types. --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index b7de55246b..bd0afe8cab 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -103,7 +103,7 @@ NonEmptyStr = Annotated[str, Field(min_length=1)] PositiveFloat = Annotated[float, Field(gt=0)] PositiveInt = Annotated[int, Field(gt=0)] -# Type aliases +# Specific types Host = NonEmptyStr Port = Annotated[int, Field(gt=0, lt=2**16)] From 68740f5558d76e10742cd4f93874584ea0ebb873 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 29 Sep 2025 23:47:27 -0400 Subject: [PATCH 129/408] refactor(config): Use ZstdCompressionLevel for compression_level and remove its validator. --- components/clp-py-utils/clp_py_utils/clp_config.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index bd0afe8cab..d36157ea05 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -106,6 +106,7 @@ # Specific types Host = NonEmptyStr Port = Annotated[int, Field(gt=0, lt=2**16)] +ZstdCompressionLevel = Annotated[int, Field(ge=1, le=19)] class StorageEngine(KebabCaseStrEnum): @@ -548,16 +549,9 @@ class ArchiveOutput(BaseModel): target_dictionaries_size: PositiveInt = 32 * 1024 * 1024 # 32 MB target_encoded_file_size: PositiveInt = 256 * 1024 * 1024 # 256 MB target_segment_size: PositiveInt = 256 * 1024 * 1024 # 256 MB - compression_level: int = 3 + compression_level: ZstdCompressionLevel = 3 retention_period: Optional[PositiveInt] = None - @field_validator("compression_level") - @classmethod - def validate_compression_level(cls, value): - if value < 1 or value > 19: - raise ValueError("compression_level must be a value from 1 to 19") - return value - def set_directory(self, directory: pathlib.Path): _set_directory_for_storage_config(self.storage, directory) From 420f30f58f16a9d6eb2257e3738b26b4f8dd4788 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 30 Sep 2025 02:21:30 -0400 Subject: [PATCH 130/408] refactor(config): Replace string logging_level fields with LoggingLevel type and remove validation helpers. --- .../clp-py-utils/clp_py_utils/clp_config.py | 58 +++---------------- .../clp-py-utils/clp_py_utils/clp_logging.py | 30 ++++------ 2 files changed, 19 insertions(+), 69 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index d36157ea05..546096a173 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -14,7 +14,7 @@ ) from strenum import KebabCaseStrEnum, LowercaseStrEnum -from .clp_logging import get_valid_logging_level, is_valid_logging_level +from .clp_logging import LoggingLevel from .core import ( get_config_value, make_config_path_absolute, @@ -255,23 +255,9 @@ def load_credentials_from_env(self): self.password = _get_env_var(CLP_DB_PASS_ENV_VAR_NAME) -def _validate_logging_level(cls, value): - if not is_valid_logging_level(value): - raise ValueError( - f"{cls.__name__}: '{value}' is not a valid logging level. Use one of" - f" {get_valid_logging_level()}" - ) - - class CompressionScheduler(BaseModel): jobs_poll_delay_sec: PositiveFloat = 0.1 - logging_level: str = "INFO" - - @field_validator("logging_level") - @classmethod - def validate_logging_level(cls, value): - _validate_logging_level(cls, value) - return value + logging_level: LoggingLevel = "INFO" class QueryScheduler(BaseModel): @@ -279,33 +265,15 @@ class QueryScheduler(BaseModel): port: Port = 7000 jobs_poll_delay_sec: PositiveFloat = 0.1 num_archives_to_search_per_sub_job: PositiveInt = 16 - logging_level: str = "INFO" - - @field_validator("logging_level") - @classmethod - def validate_logging_level(cls, value): - _validate_logging_level(cls, value) - return value + logging_level: LoggingLevel = "INFO" class CompressionWorker(BaseModel): - logging_level: str = "INFO" - - @field_validator("logging_level") - @classmethod - def validate_logging_level(cls, value): - _validate_logging_level(cls, value) - return value + logging_level: LoggingLevel = "INFO" class QueryWorker(BaseModel): - logging_level: str = "INFO" - - @field_validator("logging_level") - @classmethod - def validate_logging_level(cls, value): - _validate_logging_level(cls, value) - return value + logging_level: LoggingLevel = "INFO" class Redis(BaseModel): @@ -340,15 +308,9 @@ def load_credentials_from_env(self): class Reducer(BaseModel): host: Host = "localhost" base_port: Port = 14009 - logging_level: str = "INFO" + logging_level: LoggingLevel = "INFO" upsert_interval: PositiveInt = 100 # milliseconds - @field_validator("logging_level") - @classmethod - def validate_logging_level(cls, value): - _validate_logging_level(cls, value) - return value - class ResultsCache(BaseModel): host: Host = "localhost" @@ -595,15 +557,9 @@ class SweepInterval(BaseModel): class GarbageCollector(BaseModel): - logging_level: str = "INFO" + logging_level: LoggingLevel = "INFO" sweep_interval: SweepInterval = SweepInterval() - @field_validator("logging_level") - @classmethod - def validate_logging_level(cls, value): - _validate_logging_level(cls, value) - return value - class Presto(BaseModel): host: Host diff --git a/components/clp-py-utils/clp_py_utils/clp_logging.py b/components/clp-py-utils/clp_py_utils/clp_logging.py index dfe2ae4d8e..23d58602a2 100644 --- a/components/clp-py-utils/clp_py_utils/clp_logging.py +++ b/components/clp-py-utils/clp_py_utils/clp_logging.py @@ -1,13 +1,14 @@ import logging +from typing import get_args, Literal -LOGGING_LEVEL_MAPPING = { - "INFO": logging.INFO, - "DEBUG": logging.DEBUG, - "WARN": logging.WARNING, - "WARNING": logging.WARNING, - "ERROR": logging.ERROR, - "CRITICAL": logging.CRITICAL, -} +LoggingLevel = Literal[ + "INFO", + "DEBUG", + "WARN", + "WARNING", + "ERROR", + "CRITICAL", +] def get_logging_formatter(): @@ -25,17 +26,10 @@ def get_logger(name: str): return logger -def get_valid_logging_level(): - return [i for i in LOGGING_LEVEL_MAPPING.keys()] - - -def is_valid_logging_level(level: str): - return level in LOGGING_LEVEL_MAPPING - - def set_logging_level(logger: logging.Logger, level: str): - if not is_valid_logging_level(level): + if level not in get_args(LoggingLevel): logger.warning(f"Invalid logging level: {level}, using INFO as default") logger.setLevel(logging.INFO) return - logger.setLevel(LOGGING_LEVEL_MAPPING[level]) + + logger.setLevel(level) From 6a584e016e984f98e36c8fc8bc28cf2cb6846f44 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 30 Sep 2025 02:41:22 -0400 Subject: [PATCH 131/408] revert jobs_poll_delay rename --- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ++-- .../scheduler/compress/compression_scheduler.py | 2 +- .../job_orchestration/scheduler/query/query_scheduler.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 546096a173..b6743d7f94 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -256,14 +256,14 @@ def load_credentials_from_env(self): class CompressionScheduler(BaseModel): - jobs_poll_delay_sec: PositiveFloat = 0.1 + jobs_poll_delay: PositiveFloat = 0.1 # seconds logging_level: LoggingLevel = "INFO" class QueryScheduler(BaseModel): host: Host = "localhost" port: Port = 7000 - jobs_poll_delay_sec: PositiveFloat = 0.1 + jobs_poll_delay: PositiveFloat = 0.1 # seconds num_archives_to_search_per_sub_job: PositiveInt = 16 logging_level: LoggingLevel = "INFO" diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 4613b72276..a12a0cf6a6 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -523,7 +523,7 @@ def main(argv): clp_metadata_db_connection_config, ) poll_running_jobs(db_conn, db_cursor) - time.sleep(clp_config.compression_scheduler.jobs_poll_delay_sec) + time.sleep(clp_config.compression_scheduler.jobs_poll_delay) except KeyboardInterrupt: logger.info("Forcefully shutting down") return -1 diff --git a/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py index 417bbecdf1..7e73572725 100644 --- a/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py @@ -1180,7 +1180,7 @@ async def main(argv: List[str]) -> int: logger.exception("Failed to kill hanging query jobs.") return -1 - logger.debug(f"Job polling interval {clp_config.query_scheduler.jobs_poll_delay_sec} seconds.") + logger.debug(f"Job polling interval {clp_config.query_scheduler.jobs_poll_delay} seconds.") try: reducer_handler = await asyncio.start_server( lambda reader, writer: handle_reducer_connection( @@ -1214,7 +1214,7 @@ async def main(argv: List[str]) -> int: ), results_cache_uri=clp_config.results_cache.get_uri(), stream_collection_name=clp_config.results_cache.stream_collection_name, - jobs_poll_delay=clp_config.query_scheduler.jobs_poll_delay_sec, + jobs_poll_delay=clp_config.query_scheduler.jobs_poll_delay, num_archives_to_search_per_sub_job=batch_size, archive_retention_period=clp_config.archive_output.retention_period, ) From 7f234a095ad9dfe593347a3f79bb23018c2f51ee Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 30 Sep 2025 11:16:40 -0400 Subject: [PATCH 132/408] Make ResultHandle an abstract class of the scheduler. --- .../compress/compression_scheduler.py | 5 +- .../scheduler/scheduler/celery_scheduler.py | 24 ++++---- .../scheduler/scheduler/scheduler.py | 26 ++++----- .../scheduler/scheduler/spider_scheduler.py | 58 +++++++++++-------- .../scheduler/scheduler_data.py | 3 +- 5 files changed, 64 insertions(+), 52 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 54519d071e..83643acbe1 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -372,7 +372,8 @@ def search_and_schedule_new_tasks( task["tag_ids"] = tag_ids result_handle = scheduler.compress(tasks) - job = CompressionJob(id=job_id, start_time=start_time, async_task_result=result_handle) + # TODO: Figure out whether we need `async_task_result` + job = CompressionJob(id=job_id, start_time=start_time, result_handle=result_handle) db_cursor.execute( f""" UPDATE {COMPRESSION_TASKS_TABLE_NAME} @@ -398,7 +399,7 @@ def poll_running_jobs(db_conn, db_cursor, scheduler: Scheduler): error_message = "" try: - returned_results = scheduler.get_compress_result(job.result_handle) + returned_results = job.result_handle.get_result() if returned_results is None: continue diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py index 6da3e91319..dbe2f10171 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py @@ -10,16 +10,18 @@ class CeleryScheduler(Scheduler): - def compress(self, task_params: list[dict[str, Any]]) -> Any: + class ResultHandle(Scheduler.ResultHandle): + def __init__(self, celery_result: celery.result.GroupResult) -> None: + self._celery_result: celery.result.GroupResult = celery_result + + def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: + try: + results = self._celery_result.get(timeout=timeout) + return [CompressionTaskResult.model_validate(**res) for res in results] + except celery.exceptions.TimeoutError: + return None + + def compress(self, task_params: list[dict[str, Any]]) -> Scheduler.ResultHandle: task_instances = [compress.s(**params) for params in task_params] task_group = celery.group(task_instances) - return task_group.apply_async() - - def get_compress_result( - self, result_handle: Any, timeout: float = 0.1 - ) -> list[CompressionTaskResult] | None: - try: - results = result_handle.get(timeout=timeout) - return [CompressionTaskResult.model_validate(**res) for res in results] - except celery.exceptions.TimeoutError: - return None + return CeleryScheduler.ResultHandle(task_group) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py index 6baf5ee0ce..4cac6045cd 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py @@ -9,24 +9,22 @@ class Scheduler(ABC): """Abstract base class for a scheduler framework.""" + class ResultHandle(ABC): + @abstractmethod + def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: + """ + Gets the result of a compression job. + :param timeout: The maximum time to wait for the result. Notice that some schedulers + ignore this parameter. + :return: A list of task results. + """ + pass + @abstractmethod - def compress(self, task_params: list[dict[str, Any]]) -> Any: + def compress(self, task_params: list[dict[str, Any]]) -> ResultHandle: """ Starts a batch of compression tasks as a job. :param task_params: A list of dictionaries containing parameters for each compression task. :return: A handle through which to get the result of the job. """ pass - - @abstractmethod - def get_compress_result( - self, result_handle: Any, timeout: float = 0.1 - ) -> list[CompressionTaskResult] | None: - """ - Gets the result of a compression job. - :param result_handle: The handle returned by the compress method. - :param timeout: The maximum time to wait for the result. Notice that some schedulers ignore - this parameter. - :return: A list of task results. - """ - pass diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py index 7b28b698ff..df276e6923 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py @@ -1,50 +1,60 @@ from __future__ import annotations import json -from typing import Any +from typing import Any, Sequence import spider_py from job_orchestration.executor.compress.spider_compress import compress, convert_to_str from job_orchestration.scheduler.scheduler.scheduler import Scheduler from job_orchestration.scheduler.scheduler_data import CompressionTaskResult - - -def convert_from_str(string: str) -> list[spider_py.Int8]: - """ - Convert a string to a list of `Int8` as utf-8 bytes. - :param string: The string to convert. - :return: The list of `Int8` representing the string. - """ - return [spider_py.Int8(byte) for byte in string.encode("utf-8")] +from spider_py.client.job import Job class SpiderScheduler(Scheduler): + class ResultHandle(Scheduler.ResultHandle): + def __init__(self, spider_jobs: Sequence[Job]) -> None: + self._spider_jobs: Sequence[Job] = spider_jobs + + def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: + results: list[CompressionTaskResult] = [] + for job in self._spider_jobs: + job_results = job.get_results() + if job_results is None: + return None + results.extend( + [ + CompressionTaskResult.validate_model_json(convert_to_str(result)) + for result in job_results + ] + ) + return results + def __init__(self, storage_url: str) -> None: - self.driver = spider_py.Driver(storage_url) + self._driver = spider_py.Driver(storage_url) def compress(self, task_params: list[dict[str, Any]]) -> Any: job = spider_py.group( [compress for _ in range(len(task_params))], ) - return self.driver.submit_jobs( + submitted_jobs = self._driver.submit_jobs( job, [ spider_py.Int64(task_params["job_id"]), spider_py.Int64(task_params["task_id"]), [spider_py.Int64(tag_id) for tag_id in task_params["tag_ids"]], - convert_from_str(task_params["clp_io_config_json"]), - convert_from_str(task_params["paths_to_compress_json"]), - convert_from_str(json.loads(task_params["clp_io_config_json"])), + _from_str(task_params["clp_io_config_json"]), + _from_str(task_params["paths_to_compress_json"]), + _from_str(json.loads(task_params["clp_io_config_json"])), ], ) + return SpiderScheduler.ResultHandle(submitted_jobs) + - def get_compress_result( - self, result_handle: Any, timeout: float = 0.1 - ) -> list[CompressionTaskResult] | None: - results = result_handle.get_results() - if results is None: - return None - return [ - CompressionTaskResult.validate_model_json(convert_to_str(result)) for result in results - ] +def _from_str(string: str) -> list[spider_py.Int8]: + """ + Converts a string to a list of `spider_py.Int8` as utf-8 bytes. + :param string: The string to convert. + :return: A list of `spider_py.Int8` representing the string. + """ + return [spider_py.Int8(byte) for byte in string.encode("utf-8")] diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py index 4f49a7c1a4..81f53e6c10 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py @@ -16,13 +16,14 @@ SearchJobConfig, ) from job_orchestration.scheduler.query.reducer_handler import ReducerHandlerMessageQueues +from job_orchestration.scheduler.scheduler.scheduler import Scheduler from pydantic import BaseModel, validator class CompressionJob(BaseModel): id: int start_time: datetime.datetime - async_task_result: Any + result_handle: Scheduler.ResultHandle class CompressionTaskResult(BaseModel): From 0f1b97c251869bf7de4c0f6aa7d137417154df1b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 11:36:57 -0400 Subject: [PATCH 133/408] Rename class; Fix spider job --- .../scheduler/scheduler/celery_scheduler.py | 6 +-- .../scheduler/scheduler/scheduler.py | 4 +- .../scheduler/scheduler/spider_scheduler.py | 51 +++++++++---------- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py index dbe2f10171..a79efdf970 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py @@ -10,7 +10,7 @@ class CeleryScheduler(Scheduler): - class ResultHandle(Scheduler.ResultHandle): + class CompressResultHandle(Scheduler.CompressResultHandle): def __init__(self, celery_result: celery.result.GroupResult) -> None: self._celery_result: celery.result.GroupResult = celery_result @@ -21,7 +21,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None except celery.exceptions.TimeoutError: return None - def compress(self, task_params: list[dict[str, Any]]) -> Scheduler.ResultHandle: + def compress(self, task_params: list[dict[str, Any]]) -> Scheduler.CompressResultHandle: task_instances = [compress.s(**params) for params in task_params] task_group = celery.group(task_instances) - return CeleryScheduler.ResultHandle(task_group) + return CeleryScheduler.CompressResultHandle(task_group) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py index 4cac6045cd..e02f8dc48c 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py @@ -9,7 +9,7 @@ class Scheduler(ABC): """Abstract base class for a scheduler framework.""" - class ResultHandle(ABC): + class CompressResultHandle(ABC): @abstractmethod def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: """ @@ -21,7 +21,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None pass @abstractmethod - def compress(self, task_params: list[dict[str, Any]]) -> ResultHandle: + def compress(self, task_params: list[dict[str, Any]]) -> CompressionTaskResult: """ Starts a batch of compression tasks as a job. :param task_params: A list of dictionaries containing parameters for each compression task. diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py index df276e6923..927766695d 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import itertools import json from typing import Any, Sequence @@ -12,43 +13,39 @@ class SpiderScheduler(Scheduler): - class ResultHandle(Scheduler.ResultHandle): - def __init__(self, spider_jobs: Sequence[Job]) -> None: - self._spider_jobs: Sequence[Job] = spider_jobs + class CompressResultHandle(Scheduler.CompressResultHandle): + def __init__(self, spider_job: Job) -> None: + self._spider_job: Job = spider_job def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: - results: list[CompressionTaskResult] = [] - for job in self._spider_jobs: - job_results = job.get_results() - if job_results is None: - return None - results.extend( - [ - CompressionTaskResult.validate_model_json(convert_to_str(result)) - for result in job_results - ] - ) - return results + job_result = self._spider_job.get_results() + if job_result is None: + return None + return [ + CompressionTaskResult.validate_model_json(convert_to_str(task_result)) + for task_result in job_result + ] def __init__(self, storage_url: str) -> None: self._driver = spider_py.Driver(storage_url) - def compress(self, task_params: list[dict[str, Any]]) -> Any: + def compress(self, task_params: list[dict[str, Any]]) -> Scheduler.CompressResultHandle: job = spider_py.group( [compress for _ in range(len(task_params))], ) - submitted_jobs = self._driver.submit_jobs( - job, + task_params_list = [ [ - spider_py.Int64(task_params["job_id"]), - spider_py.Int64(task_params["task_id"]), - [spider_py.Int64(tag_id) for tag_id in task_params["tag_ids"]], - _from_str(task_params["clp_io_config_json"]), - _from_str(task_params["paths_to_compress_json"]), - _from_str(json.loads(task_params["clp_io_config_json"])), - ], - ) - return SpiderScheduler.ResultHandle(submitted_jobs) + spider_py.Int64(task_param["job_id"]), + spider_py.Int64(task_param["task_id"]), + [spider_py.Int64(tag_id) for tag_id in task_param["tag_ids"]], + _from_str(task_param["clp_io_config_json"]), + _from_str(task_param["paths_to_compress_json"]), + _from_str(json.loads(task_param["clp_io_config_json"])), + ] + for task_param in task_params + ] + submitted_job = self._driver.submit_jobs([job], [list(itertools.chain(*task_params_list))]) + return SpiderScheduler.CompressResultHandle(submitted_job) def _from_str(string: str) -> list[spider_py.Int8]: From bbf9b38754596445cb7d72fa891f0338c6f6b600 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 11:39:47 -0400 Subject: [PATCH 134/408] Remove TODO --- .../scheduler/compress/compression_scheduler.py | 1 - .../job_orchestration/scheduler/scheduler_data.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 83643acbe1..7105ea29af 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -372,7 +372,6 @@ def search_and_schedule_new_tasks( task["tag_ids"] = tag_ids result_handle = scheduler.compress(tasks) - # TODO: Figure out whether we need `async_task_result` job = CompressionJob(id=job_id, start_time=start_time, result_handle=result_handle) db_cursor.execute( f""" diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py index 81f53e6c10..6ea683b022 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py @@ -23,7 +23,7 @@ class CompressionJob(BaseModel): id: int start_time: datetime.datetime - result_handle: Scheduler.ResultHandle + result_handle: Scheduler.CompressResultHandle class CompressionTaskResult(BaseModel): From 200567cfe47e7ef5fc355b8aacd0c7035ddced19 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 12:56:03 -0400 Subject: [PATCH 135/408] Move and rename files and classes --- .../compress/compression_scheduler.py | 23 +++++++++++-------- .../task_manager}/__init__.py | 0 .../task_manager/celery_task_manager.py} | 11 +++++---- .../task_manager/spider_task_manager.py} | 15 ++++++------ .../task_manager/task_manager.py} | 2 +- .../scheduler/scheduler_data.py | 4 ++-- 6 files changed, 30 insertions(+), 25 deletions(-) rename components/job-orchestration/job_orchestration/scheduler/{scheduler => compress/task_manager}/__init__.py (100%) rename components/job-orchestration/job_orchestration/scheduler/{scheduler/celery_scheduler.py => compress/task_manager/celery_task_manager.py} (70%) rename components/job-orchestration/job_orchestration/scheduler/{scheduler/spider_scheduler.py => compress/task_manager/spider_task_manager.py} (83%) rename components/job-orchestration/job_orchestration/scheduler/{scheduler/scheduler.py => compress/task_manager/task_manager.py} (97%) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 7105ea29af..a734f78957 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -13,7 +13,6 @@ import msgpack from clp_package_utils.general import CONTAINER_INPUT_LOGS_ROOT_DIR from clp_py_utils.clp_config import ( - ArchiveOutput, CLPConfig, COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, @@ -31,6 +30,8 @@ from clp_py_utils.s3_utils import s3_get_object_metadata from clp_py_utils.sql_adapter import SQL_Adapter from job_orchestration.scheduler.compress.partition import PathsToCompressBuffer +from job_orchestration.scheduler.compress.task_manager.celery_task_manager import CeleryTaskManager +from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.constants import ( CompressionJobStatus, CompressionTaskStatus, @@ -42,8 +43,6 @@ InputType, S3InputConfig, ) -from job_orchestration.scheduler.scheduler.celery_scheduler import CeleryScheduler -from job_orchestration.scheduler.scheduler.scheduler import Scheduler from job_orchestration.scheduler.scheduler_data import ( CompressionJob, ) @@ -217,7 +216,7 @@ def search_and_schedule_new_tasks( db_conn, db_cursor, clp_metadata_db_connection_config: Dict[str, Any], - scheduler: Scheduler, + task_manager: TaskManager, ): """ For all jobs with PENDING status, splits the job into tasks and schedules them. @@ -225,7 +224,7 @@ def search_and_schedule_new_tasks( :param db_conn: :param db_cursor: :param clp_metadata_db_connection_config: - :param scheduler: + :param task_manager: """ global scheduled_jobs @@ -370,7 +369,7 @@ def search_and_schedule_new_tasks( db_conn.commit() task["task_id"] = db_cursor.lastrowid task["tag_ids"] = tag_ids - result_handle = scheduler.compress(tasks) + result_handle = task_manager.compress(tasks) job = CompressionJob(id=job_id, start_time=start_time, result_handle=result_handle) db_cursor.execute( @@ -384,7 +383,7 @@ def search_and_schedule_new_tasks( scheduled_jobs[job_id] = job -def poll_running_jobs(db_conn, db_cursor, scheduler: Scheduler): +def poll_running_jobs(db_conn, db_cursor): """ Poll for running jobs and update their status. """ @@ -487,7 +486,7 @@ def main(argv): logger.info(f"Starting {COMPRESSION_SCHEDULER_COMPONENT_NAME}") sql_adapter = SQL_Adapter(clp_config.database) - scheduler = CeleryScheduler() + task_manager = CeleryTaskManager() try: killed_jobs = kill_hanging_jobs(sql_adapter, SchedulerType.COMPRESSION) @@ -509,9 +508,13 @@ def main(argv): try: if not received_sigterm: search_and_schedule_new_tasks( - clp_config, db_conn, db_cursor, clp_metadata_db_connection_config, scheduler + clp_config, + db_conn, + db_cursor, + clp_metadata_db_connection_config, + task_manager, ) - poll_running_jobs(db_conn, db_cursor, scheduler) + poll_running_jobs(db_conn, db_cursor) time.sleep(clp_config.compression_scheduler.jobs_poll_delay) except KeyboardInterrupt: logger.info("Forcefully shutting down") diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/__init__.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/__init__.py similarity index 100% rename from components/job-orchestration/job_orchestration/scheduler/scheduler/__init__.py rename to components/job-orchestration/job_orchestration/scheduler/compress/task_manager/__init__.py diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py similarity index 70% rename from components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py rename to components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py index a79efdf970..1140768c0f 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/celery_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py @@ -3,14 +3,15 @@ from typing import Any import celery + from job_orchestration.executor.compress.celery_compress import compress -from job_orchestration.scheduler.scheduler.scheduler import Scheduler +from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.scheduler_data import CompressionTaskResult -class CeleryScheduler(Scheduler): +class CeleryTaskManager(TaskManager): - class CompressResultHandle(Scheduler.CompressResultHandle): + class CompressResultHandle(TaskManager.CompressResultHandle): def __init__(self, celery_result: celery.result.GroupResult) -> None: self._celery_result: celery.result.GroupResult = celery_result @@ -21,7 +22,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None except celery.exceptions.TimeoutError: return None - def compress(self, task_params: list[dict[str, Any]]) -> Scheduler.CompressResultHandle: + def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.CompressResultHandle: task_instances = [compress.s(**params) for params in task_params] task_group = celery.group(task_instances) - return CeleryScheduler.CompressResultHandle(task_group) + return CeleryTaskManager.CompressResultHandle(task_group) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py similarity index 83% rename from components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py rename to components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index 927766695d..ad7d9c9cb7 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/spider_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -2,18 +2,19 @@ import itertools import json -from typing import Any, Sequence +from typing import Any import spider_py +from spider_py.client.job import Job + from job_orchestration.executor.compress.spider_compress import compress, convert_to_str -from job_orchestration.scheduler.scheduler.scheduler import Scheduler +from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.scheduler_data import CompressionTaskResult -from spider_py.client.job import Job -class SpiderScheduler(Scheduler): +class SpiderTaskManager(TaskManager): - class CompressResultHandle(Scheduler.CompressResultHandle): + class CompressResultHandle(TaskManager.CompressResultHandle): def __init__(self, spider_job: Job) -> None: self._spider_job: Job = spider_job @@ -29,7 +30,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None def __init__(self, storage_url: str) -> None: self._driver = spider_py.Driver(storage_url) - def compress(self, task_params: list[dict[str, Any]]) -> Scheduler.CompressResultHandle: + def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.CompressResultHandle: job = spider_py.group( [compress for _ in range(len(task_params))], ) @@ -45,7 +46,7 @@ def compress(self, task_params: list[dict[str, Any]]) -> Scheduler.CompressResul for task_param in task_params ] submitted_job = self._driver.submit_jobs([job], [list(itertools.chain(*task_params_list))]) - return SpiderScheduler.CompressResultHandle(submitted_job) + return SpiderTaskManager.CompressResultHandle(submitted_job) def _from_str(string: str) -> list[spider_py.Int8]: diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py similarity index 97% rename from components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py rename to components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py index e02f8dc48c..7f754c2373 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler/scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py @@ -6,7 +6,7 @@ from job_orchestration.scheduler.scheduler_data import CompressionTaskResult -class Scheduler(ABC): +class TaskManager(ABC): """Abstract base class for a scheduler framework.""" class CompressResultHandle(ABC): diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py index 6ea683b022..d923750017 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py @@ -4,6 +4,7 @@ from enum import auto, Enum from typing import Any, Dict, List, Optional +from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.constants import ( CompressionTaskStatus, QueryJobType, @@ -16,14 +17,13 @@ SearchJobConfig, ) from job_orchestration.scheduler.query.reducer_handler import ReducerHandlerMessageQueues -from job_orchestration.scheduler.scheduler.scheduler import Scheduler from pydantic import BaseModel, validator class CompressionJob(BaseModel): id: int start_time: datetime.datetime - result_handle: Scheduler.CompressResultHandle + result_handle: TaskManager.CompressResultHandle class CompressionTaskResult(BaseModel): From 7f17280dac9889d4af450b1c02abc3d2624e95df Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 12:56:42 -0400 Subject: [PATCH 136/408] Fix lint --- .../scheduler/compress/task_manager/celery_task_manager.py | 1 - .../scheduler/compress/task_manager/spider_task_manager.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py index 1140768c0f..3ee7db7c6f 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py @@ -3,7 +3,6 @@ from typing import Any import celery - from job_orchestration.executor.compress.celery_compress import compress from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.scheduler_data import CompressionTaskResult diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index ad7d9c9cb7..d2a792ee98 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -5,11 +5,10 @@ from typing import Any import spider_py -from spider_py.client.job import Job - from job_orchestration.executor.compress.spider_compress import compress, convert_to_str from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.scheduler_data import CompressionTaskResult +from spider_py.client.job import Job class SpiderTaskManager(TaskManager): From ed04ede41f6e03499e1915c477621ee49a51d3ca Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 14:35:02 -0400 Subject: [PATCH 137/408] Split the file --- .../executor/compress/compression_task.py | 2 +- .../task_manager/celery_task_manager.py | 2 +- .../task_manager/spider_task_manager.py | 2 +- .../compress/task_manager/task_manager.py | 2 +- .../scheduler/scheduler_data.py | 15 --------------- .../scheduler/scheduler_result.py | 18 ++++++++++++++++++ 6 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 components/job-orchestration/job_orchestration/scheduler/scheduler_result.py diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index acc8c3f839..e9683800d6 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -36,7 +36,7 @@ PathsToCompress, S3InputConfig, ) -from job_orchestration.scheduler.scheduler_data import CompressionTaskResult +from job_orchestration.scheduler.scheduler_result import CompressionTaskResult def update_compression_task_metadata(db_cursor, task_id, kv): diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py index 3ee7db7c6f..c9bd42d2ac 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py @@ -5,7 +5,7 @@ import celery from job_orchestration.executor.compress.celery_compress import compress from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager -from job_orchestration.scheduler.scheduler_data import CompressionTaskResult +from job_orchestration.scheduler.scheduler_result import CompressionTaskResult class CeleryTaskManager(TaskManager): diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index d2a792ee98..3de7f9e379 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -7,7 +7,7 @@ import spider_py from job_orchestration.executor.compress.spider_compress import compress, convert_to_str from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager -from job_orchestration.scheduler.scheduler_data import CompressionTaskResult +from job_orchestration.scheduler.scheduler_result import CompressionTaskResult from spider_py.client.job import Job diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py index 7f754c2373..b4bd5cbd8e 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from typing import Any -from job_orchestration.scheduler.scheduler_data import CompressionTaskResult +from job_orchestration.scheduler.scheduler_result import CompressionTaskResult class TaskManager(ABC): diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py index d923750017..2542dc7a7f 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py @@ -6,7 +6,6 @@ from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.constants import ( - CompressionTaskStatus, QueryJobType, QueryTaskStatus, ) @@ -26,20 +25,6 @@ class CompressionJob(BaseModel): result_handle: TaskManager.CompressResultHandle -class CompressionTaskResult(BaseModel): - task_id: int - status: int - duration: float - error_message: Optional[str] - - @validator("status") - def valid_status(cls, field): - supported_status = [CompressionTaskStatus.SUCCEEDED, CompressionTaskStatus.FAILED] - if field not in supported_status: - raise ValueError(f'must be one of the following {"|".join(supported_status)}') - return field - - class InternalJobState(Enum): WAITING_FOR_REDUCER = auto() WAITING_FOR_DISPATCH = auto() diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py new file mode 100644 index 0000000000..efeb503c26 --- /dev/null +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py @@ -0,0 +1,18 @@ +from typing import Optional + +from job_orchestration.scheduler.constants import CompressionTaskStatus +from pydantic import BaseModel, validator + + +class CompressionTaskResult(BaseModel): + task_id: int + status: int + duration: float + error_message: Optional[str] + + @validator("status") + def valid_status(cls, field): + supported_status = [CompressionTaskStatus.SUCCEEDED, CompressionTaskStatus.FAILED] + if field not in supported_status: + raise ValueError(f'must be one of the following {"|".join(supported_status)}') + return field From 3c7ce34d46db4d0935ee4b6474fd86b194b56458 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 15:17:06 -0400 Subject: [PATCH 138/408] Fix bug --- .../scheduler/compress/task_manager/celery_task_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py index c9bd42d2ac..7c52bfed55 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py @@ -11,8 +11,8 @@ class CeleryTaskManager(TaskManager): class CompressResultHandle(TaskManager.CompressResultHandle): - def __init__(self, celery_result: celery.result.GroupResult) -> None: - self._celery_result: celery.result.GroupResult = celery_result + def __init__(self, celery_result: celery.result.AsyncResult) -> None: + self._celery_result: celery.result.AsyncResult = celery_result def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: try: @@ -24,4 +24,4 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.CompressResultHandle: task_instances = [compress.s(**params) for params in task_params] task_group = celery.group(task_instances) - return CeleryTaskManager.CompressResultHandle(task_group) + return CeleryTaskManager.CompressResultHandle(task_group.apply_async()) From 53847917ed3be8619fbe861e14d74107bf9e8910 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 18:43:53 -0400 Subject: [PATCH 139/408] Bug fix --- .../scheduler/compress/task_manager/celery_task_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py index 7c52bfed55..a6fe14e3f7 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py @@ -17,7 +17,7 @@ def __init__(self, celery_result: celery.result.AsyncResult) -> None: def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: try: results = self._celery_result.get(timeout=timeout) - return [CompressionTaskResult.model_validate(**res) for res in results] + return [CompressionTaskResult.parse_obj(res) for res in results] except celery.exceptions.TimeoutError: return None From 885dcbebbf90f5e02e0faa3276671c3b7f50521c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 19:39:10 -0400 Subject: [PATCH 140/408] Bug fix --- .../job_orchestration/executor/compress/spider_compress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index 99d2e848ba..9e613237af 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -49,7 +49,7 @@ def compress( [int(tag_id) for tag_id in tag_ids], convert_to_str(clp_io_config_json), convert_to_str(paths_to_compress_json), - Database.model_validate_json(convert_to_str(clp_metadata_db_connection_config_json)), + Database.parse_raw(convert_to_str(clp_metadata_db_connection_config_json)), logger, ) ).encode("utf-8") From bc315cf2a8e2056a407368d84ea0c13178473a88 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 20:12:08 -0400 Subject: [PATCH 141/408] Allow arbitrary type --- .../job_orchestration/scheduler/scheduler_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py index 2542dc7a7f..d65c2293c8 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py @@ -24,6 +24,10 @@ class CompressionJob(BaseModel): start_time: datetime.datetime result_handle: TaskManager.CompressResultHandle + class Config: + # Allow the use of TaskManager.CompressResultHandle + arbitrary_types_allowed = True + class InternalJobState(Enum): WAITING_FOR_REDUCER = auto() From 64e5924f69c42a3880c3790074ac51559bd71f9b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 30 Sep 2025 20:24:02 -0400 Subject: [PATCH 142/408] Rename result handle --- .../scheduler/compress/task_manager/celery_task_manager.py | 6 +++--- .../scheduler/compress/task_manager/spider_task_manager.py | 6 +++--- .../scheduler/compress/task_manager/task_manager.py | 2 +- .../job_orchestration/scheduler/scheduler_data.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py index a6fe14e3f7..67259c92d4 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py @@ -10,7 +10,7 @@ class CeleryTaskManager(TaskManager): - class CompressResultHandle(TaskManager.CompressResultHandle): + class ResultHandle(TaskManager.ResultHandle): def __init__(self, celery_result: celery.result.AsyncResult) -> None: self._celery_result: celery.result.AsyncResult = celery_result @@ -21,7 +21,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None except celery.exceptions.TimeoutError: return None - def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.CompressResultHandle: + def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.ResultHandle: task_instances = [compress.s(**params) for params in task_params] task_group = celery.group(task_instances) - return CeleryTaskManager.CompressResultHandle(task_group.apply_async()) + return CeleryTaskManager.ResultHandle(task_group.apply_async()) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index 3de7f9e379..a515fe5693 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -13,7 +13,7 @@ class SpiderTaskManager(TaskManager): - class CompressResultHandle(TaskManager.CompressResultHandle): + class ResultHandle(TaskManager.ResultHandle): def __init__(self, spider_job: Job) -> None: self._spider_job: Job = spider_job @@ -29,7 +29,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None def __init__(self, storage_url: str) -> None: self._driver = spider_py.Driver(storage_url) - def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.CompressResultHandle: + def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.ResultHandle: job = spider_py.group( [compress for _ in range(len(task_params))], ) @@ -45,7 +45,7 @@ def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.CompressRes for task_param in task_params ] submitted_job = self._driver.submit_jobs([job], [list(itertools.chain(*task_params_list))]) - return SpiderTaskManager.CompressResultHandle(submitted_job) + return SpiderTaskManager.ResultHandle(submitted_job) def _from_str(string: str) -> list[spider_py.Int8]: diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py index b4bd5cbd8e..72acf20625 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py @@ -9,7 +9,7 @@ class TaskManager(ABC): """Abstract base class for a scheduler framework.""" - class CompressResultHandle(ABC): + class ResultHandle(ABC): @abstractmethod def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: """ diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py index d65c2293c8..90efdf2e9f 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py @@ -22,7 +22,7 @@ class CompressionJob(BaseModel): id: int start_time: datetime.datetime - result_handle: TaskManager.CompressResultHandle + result_handle: TaskManager.ResultHandle class Config: # Allow the use of TaskManager.CompressResultHandle From 2115872f6280fa93a97bda3f5849d57f68f6a398 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 1 Oct 2025 14:59:54 -0400 Subject: [PATCH 143/408] Use pydantic v2 --- .../job_orchestration/scheduler/scheduler_data.py | 7 +++---- .../job_orchestration/scheduler/scheduler_result.py | 10 +++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py index 7b5e292d30..15a2cb0dee 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_data.py @@ -20,14 +20,13 @@ class CompressionJob(BaseModel): + # Allow the use of TaskManager.ResultHandle + model_config = ConfigDict(arbitrary_types_allowed=True) + id: int start_time: datetime.datetime result_handle: TaskManager.ResultHandle - class Config: - # Allow the use of TaskManager.CompressResultHandle - arbitrary_types_allowed = True - class InternalJobState(Enum): WAITING_FOR_REDUCER = auto() diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py index efeb503c26..3c774843a0 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py @@ -1,7 +1,7 @@ from typing import Optional from job_orchestration.scheduler.constants import CompressionTaskStatus -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator class CompressionTaskResult(BaseModel): @@ -10,9 +10,9 @@ class CompressionTaskResult(BaseModel): duration: float error_message: Optional[str] - @validator("status") - def valid_status(cls, field): + @field_validator("status") + def valid_status(cls, value): supported_status = [CompressionTaskStatus.SUCCEEDED, CompressionTaskStatus.FAILED] - if field not in supported_status: + if value not in supported_status: raise ValueError(f'must be one of the following {"|".join(supported_status)}') - return field + return value From 5ff438f39f79e9bda696457f355b4927c2fae2fe Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 1 Oct 2025 15:23:09 -0400 Subject: [PATCH 144/408] Fix missing value --- .../job_orchestration/scheduler/scheduler_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py b/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py index 3c774843a0..355eb1327d 100644 --- a/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py +++ b/components/job-orchestration/job_orchestration/scheduler/scheduler_result.py @@ -8,7 +8,7 @@ class CompressionTaskResult(BaseModel): task_id: int status: int duration: float - error_message: Optional[str] + error_message: Optional[str] = None @field_validator("status") def valid_status(cls, value): From af96fc8606e8817319c1b8a9975a2eedd748120f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 1 Oct 2025 18:32:03 -0400 Subject: [PATCH 145/408] lint --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 7e3d7067fa..73b3986614 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -309,10 +309,10 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" client_settings_json_path = ( - self._clp_home / "var" / "www" / "webui" / "client" / "settings.json" + self._clp_home / "var" / "www" / "webui" / "client" / "settings.json" ) server_settings_json_path = ( - self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" + self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) validate_webui_config(self.clp_config, client_settings_json_path, server_settings_json_path) From d5edb1087605c6038d87c173617427db744baf26 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 1 Oct 2025 21:03:47 -0400 Subject: [PATCH 146/408] Use model_validate instead of parse_obj --- .../scheduler/compress/task_manager/celery_task_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py index 67259c92d4..c34c32a31a 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py @@ -17,7 +17,7 @@ def __init__(self, celery_result: celery.result.AsyncResult) -> None: def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: try: results = self._celery_result.get(timeout=timeout) - return [CompressionTaskResult.parse_obj(res) for res in results] + return [CompressionTaskResult.model_validate(res) for res in results] except celery.exceptions.TimeoutError: return None From a66440ac4a818ad0ada1ce6ceb2a73712e738391 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 1 Oct 2025 21:28:26 -0400 Subject: [PATCH 147/408] Fix docstring --- .../scheduler/compress/task_manager/task_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py index 72acf20625..8066c3dc3a 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py @@ -14,8 +14,8 @@ class ResultHandle(ABC): def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: """ Gets the result of a compression job. - :param timeout: The maximum time to wait for the result. Notice that some schedulers - ignore this parameter. + :param timeout: The maximum time to wait for the result. Notice that some task managers + ignore this parameter. :return: A list of task results. """ pass From 3211d8a02ed4e60a81331d492678e30c95c1455c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 1 Oct 2025 21:31:28 -0400 Subject: [PATCH 148/408] Use spider util function --- .../compress/task_manager/spider_task_manager.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index a515fe5693..341c543e2f 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -8,6 +8,7 @@ from job_orchestration.executor.compress.spider_compress import compress, convert_to_str from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.scheduler_result import CompressionTaskResult +from job_orchestration.utils.spider_utils import utf8_str_to_int8_list from spider_py.client.job import Job @@ -38,20 +39,11 @@ def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.ResultHandl spider_py.Int64(task_param["job_id"]), spider_py.Int64(task_param["task_id"]), [spider_py.Int64(tag_id) for tag_id in task_param["tag_ids"]], - _from_str(task_param["clp_io_config_json"]), - _from_str(task_param["paths_to_compress_json"]), - _from_str(json.loads(task_param["clp_io_config_json"])), + utf8_str_to_int8_list(task_param["clp_io_config_json"]), + utf8_str_to_int8_list(task_param["paths_to_compress_json"]), + utf8_str_to_int8_list(json.loads(task_param["clp_io_config_json"])), ] for task_param in task_params ] submitted_job = self._driver.submit_jobs([job], [list(itertools.chain(*task_params_list))]) return SpiderTaskManager.ResultHandle(submitted_job) - - -def _from_str(string: str) -> list[spider_py.Int8]: - """ - Converts a string to a list of `spider_py.Int8` as utf-8 bytes. - :param string: The string to convert. - :return: A list of `spider_py.Int8` representing the string. - """ - return [spider_py.Int8(byte) for byte in string.encode("utf-8")] From 3133b071db4d3e4b72270d9f8a50bd8f8cbd936f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 1 Oct 2025 21:39:18 -0400 Subject: [PATCH 149/408] Bug fix --- .../scheduler/compress/task_manager/spider_task_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index 341c543e2f..be9987bb07 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -41,7 +41,7 @@ def compress(self, task_params: list[dict[str, Any]]) -> TaskManager.ResultHandl [spider_py.Int64(tag_id) for tag_id in task_param["tag_ids"]], utf8_str_to_int8_list(task_param["clp_io_config_json"]), utf8_str_to_int8_list(task_param["paths_to_compress_json"]), - utf8_str_to_int8_list(json.loads(task_param["clp_io_config_json"])), + utf8_str_to_int8_list(json.dumps(task_param["clp_metadata_db_connection_config"])), ] for task_param in task_params ] From 7184d869163b4cc8ccb2bd6718582d4f9cab031f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 1 Oct 2025 21:40:29 -0400 Subject: [PATCH 150/408] Fix return type --- .../scheduler/compress/task_manager/task_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py index 8066c3dc3a..17de419743 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/task_manager.py @@ -21,7 +21,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None pass @abstractmethod - def compress(self, task_params: list[dict[str, Any]]) -> CompressionTaskResult: + def compress(self, task_params: list[dict[str, Any]]) -> ResultHandle: """ Starts a batch of compression tasks as a job. :param task_params: A list of dictionaries containing parameters for each compression task. From 630a780576c8b1d5560fef3b3e19b7c51e876ff1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 1 Oct 2025 21:48:39 -0400 Subject: [PATCH 151/408] Bug fix --- .../scheduler/compress/task_manager/spider_task_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index be9987bb07..32781c0fa9 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -23,7 +23,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None if job_result is None: return None return [ - CompressionTaskResult.validate_model_json(convert_to_str(task_result)) + CompressionTaskResult.model_validate_json(convert_to_str(task_result)) for task_result in job_result ] From dd56d04324ea7bad3135da6443e7ab7112939244 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 18:01:52 -0400 Subject: [PATCH 152/408] fix: Replace container_name with hostname for services to avoid name conflicts with multiple of the projects starting on the same host. --- .../package/docker-compose.base.yaml | 20 +++++++++---------- tools/deployment/package/docker-compose.yaml | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a0d5e301ae..62996dec0a 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -20,8 +20,8 @@ secrets: services: database: <<: *service_defaults - container_name: "database" image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + hostname: "database" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" @@ -49,8 +49,8 @@ services: db-table-creator: <<: *service_defaults - container_name: "db_table_creator" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "db_table_creator" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" @@ -70,8 +70,8 @@ services: queue: <<: *service_defaults - container_name: "queue" image: "rabbitmq:3.9.8" + hostname: "queue" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" @@ -90,8 +90,8 @@ services: redis: <<: *service_defaults - container_name: "redis" image: "redis:7.2.4" + hostname: "redis" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" @@ -117,8 +117,8 @@ services: results-cache: <<: *service_defaults - container_name: "results_cache" image: "mongo:7.0.1" + hostname: "results_cache" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" @@ -138,8 +138,8 @@ services: results-cache-indices-creator: <<: *service_defaults - container_name: "results_cache_indices_creator" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "results_cache_indices_creator" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" depends_on: @@ -155,8 +155,8 @@ services: compression-scheduler: <<: *service_defaults - container_name: "compression_scheduler" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "compression_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS}" @@ -188,8 +188,8 @@ services: compression-worker: <<: *service_defaults - container_name: "compression_worker" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "compression_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" @@ -226,8 +226,8 @@ services: webui: <<: *service_defaults - container_name: "webui" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "webui" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" @@ -264,8 +264,8 @@ services: garbage-collector: <<: *service_defaults - container_name: "garbage_collector" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "garbage_collector" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 319e11c694..7d938a238b 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -18,8 +18,8 @@ x-healthcheck-defaults: &healthcheck_defaults services: query-scheduler: <<: *service_defaults - container_name: "query_scheduler" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "query_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS}" @@ -58,8 +58,8 @@ services: query-worker: <<: *service_defaults - container_name: "query_worker" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "query_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" @@ -94,8 +94,8 @@ services: reducer: <<: *service_defaults - container_name: "reducer" image: "${CLP_PACKAGE_CONTAINER}" + hostname: "reducer" environment: CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" From 5298540ba24630873fff6dd7ed36b0e9635352c2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 18:09:54 -0400 Subject: [PATCH 153/408] refactor(docker-compose): move image definition to service defaults. --- tools/deployment/package/docker-compose.base.yaml | 7 +------ tools/deployment/package/docker-compose.yaml | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 62996dec0a..a78168376e 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -2,6 +2,7 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults + image: "${CLP_PACKAGE_CONTAINER}" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" @@ -49,7 +50,6 @@ services: db-table-creator: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "db_table_creator" environment: CLP_DB_PASS: "${CLP_DB_PASS}" @@ -138,7 +138,6 @@ services: results-cache-indices-creator: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "results_cache_indices_creator" environment: PYTHONPATH: "/opt/clp/lib/python3/site-packages" @@ -155,7 +154,6 @@ services: compression-scheduler: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "compression_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -188,7 +186,6 @@ services: compression-worker: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "compression_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" @@ -226,7 +223,6 @@ services: webui: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "webui" environment: CLP_DB_PASS: "${CLP_DB_PASS}" @@ -264,7 +260,6 @@ services: garbage-collector: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "garbage_collector" environment: CLP_DB_PASS: "${CLP_DB_PASS}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 7d938a238b..71c5492f3d 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -4,6 +4,7 @@ include: ["docker-compose.base.yaml"] # Common service defaults. x-service-defaults: &service_defaults + image: "${CLP_PACKAGE_CONTAINER}" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" @@ -18,7 +19,6 @@ x-healthcheck-defaults: &healthcheck_defaults services: query-scheduler: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "query_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" @@ -58,7 +58,6 @@ services: query-worker: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "query_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" From 140187f550d4d61653239ed9109e902b281f9d6e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 18:14:36 -0400 Subject: [PATCH 154/408] refactor(docker-compose): Remove image entry from reducer service. --- tools/deployment/package/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 71c5492f3d..94aef25e42 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -93,7 +93,6 @@ services: reducer: <<: *service_defaults - image: "${CLP_PACKAGE_CONTAINER}" hostname: "reducer" environment: CLP_HOME: "/opt/clp" From 68586f014b8cc721f0ec8f513d26ed57a9e83450 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 18:16:49 -0400 Subject: [PATCH 155/408] refactor(docker-compose): add default fallback for image and storage engine variables. --- tools/deployment/package/docker-compose.base.yaml | 4 ++-- tools/deployment/package/docker-compose.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a78168376e..07b760e649 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -2,7 +2,7 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER}" + image: "${CLP_PACKAGE_CONTAINER:-clp-package}" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" @@ -65,7 +65,7 @@ services: "-u", "-m", "clp_py_utils.create-db-tables", "--config", "/etc/clp-config.yml", - "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE:-clp}", ] queue: diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 94aef25e42..24ad6b242f 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -4,7 +4,7 @@ include: ["docker-compose.base.yaml"] # Common service defaults. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER}" + image: "${CLP_PACKAGE_CONTAINER:-clp-package}" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" From cba044a12f17420ac5f89afef1234fa22e656f67 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 21:55:42 -0400 Subject: [PATCH 156/408] improve docs --- docs/src/dev-docs/design-docker-compose.md | 77 +++++++++++++++------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index add169870b..e365decd63 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -4,46 +4,71 @@ This document explains the technical details of CLP's Docker Compose implementat ## Overview -The Docker Compose implementation depends on a new controller architecture with a `BaseController` -abstract class and a `DockerComposeController` implementation. +The Docker Compose implementation follows a controller architecture with a `BaseController` abstract class and a +`DockerComposeController` implementation. ## Architecture -### Controller +### Controller Pattern -The orchestration implementation uses a controller pattern: +The orchestration uses a controller pattern: * `BaseController` (abstract): Defines the interface for provisioning and managing CLP components. -* `DockerComposeController`: Implements the Docker Compose-specific logic. +* `DockerComposeController`: Implements Docker Compose-specific logic. -### Initialization +## Initialization -1. **Provisioning Methods**: Each CLP component has a dedicated provisioning method in the - controller: `provision_()`. -2. **Environment Generation**: The controller generates a `.env` file with all necessary environment - variables for Docker Compose. -3. **Configuration Transformation**: The `transform_for_container_config()` method in `CLPConfig` - and related classes adapts the configuration for containerized environments. +The controller performs these initialization steps: -## Docker Compose File +1. **Provisioning**: Provisions all components and generates component specific configuration variables. +2. **Configuration Transformation**: The `transform_for_container()` method in `CLPConfig` adapts configurations for + containerized environments +3. **Environment Generation**: Creates a `.env` file with necessary Docker Compose variables -The `docker-compose.yaml` file defines all services with: +### Configuration Transformation -* Proper service dependencies using `depends_on` -* Health checks for critical services -* Volume mounts for persistent data -* Network configuration -* User permissions -* Resource limits +The `transform_for_container()` method in the `CLPConfig` class and related component classes adapts the configuration for containerized environments by: + +1. Converting host paths to container paths +2. Updating service hostnames to match Docker Compose service names +3. Setting appropriate ports for container communication + +### Environment Variables + +The controller generates a comprehensive set of environment variables that are written to a `.env` file, including: + +* Component-specific settings (ports, logging levels, concurrency) +* Credentials for database, queue, and Redis services +* Paths for data, logs, archives, and streams +* AWS credentials when needed ## Deployment Process -The `start-clp.py` script performs the following steps: +The `start-clp.sh` script executes the `start_clp.py` Python script to orchestrate the deployment. + +### Deployment Types + +CLP supports two deployment types determined by the `package.query_engine` configuration setting: -1. **Configuration Loading**: The start script loads and validates the CLP configuration. -2. **Provisioning**: The controller provisions all components and generates environment variables. -3. **Environment File Generation**: A `.env` file is created with all necessary variables. -4. **Docker Compose Execution**: `docker compose up -d` is executed to start all services. +1. **BASE**: For deployments using [Presto][presto-integration] as the query engine. Uses only + `docker-compose.base.yaml`. +2. **FULL**: For deployments using CLP's native query engine. Uses both compose files. + +## Docker Compose Files + +The Docker Compose setup uses two files: + +* `docker-compose.base.yaml`: Defines base services for all deployment types, excluding Celery scheduler and worker + components to allow separate Presto [integration][presto-integration]. +* `docker-compose.yaml`: Extends the base file with additional services for complete deployments + +Each file defines services with: + +* Service dependencies via `depends_on` +* Health checks for critical services +* Volume binding mounts for persistent data +* Network configuration +* User permissions ## Service architecture @@ -166,3 +191,5 @@ If you encounter issues with the Docker Compose deployment: ```bash docker compose config ``` + +[presto-integration]: ../user-docs/guides-using-presto.md \ No newline at end of file From f41e22abec53c3a384b377cd8b740ad787ef2b45 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 2 Oct 2025 22:09:02 -0400 Subject: [PATCH 157/408] refactor(docker-compose): use list syntax for healthcheck test command. --- tools/deployment/package/docker-compose.base.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 07b760e649..35ff1a6635 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -128,9 +128,10 @@ services: - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" healthcheck: <<: *healthcheck_defaults - test: >- - echo 'db.runCommand("ping").ok' | - mongosh 127.0.0.1:27017/test --quiet + test: [ + "CMD-SHELL", + "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:27017/test --quiet" + ] command: [ "--config", "/etc/mongo/mongod.conf", "--bind_ip", "0.0.0.0", From 02fcbe7b2f853c57d1e87fbe65a39d9202cf88ed Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 14:47:09 -0400 Subject: [PATCH 158/408] fix(docker-compose): revert workers log level to hardcoded "warning". --- tools/deployment/package/docker-compose.base.yaml | 2 +- tools/deployment/package/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 35ff1a6635..68ec1e5c9d 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -216,7 +216,7 @@ services: "-A", "job_orchestration.executor.compress", "worker", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", + "--loglevel", "WARNING", "-f", "/var/log/compression_worker/worker.log", "-Q", "compression", "-n", "compression-worker" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 24ad6b242f..e793d08dfe 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -85,7 +85,7 @@ services: "-A", "job_orchestration.executor.query", "worker", "--concurrency", "${CLP_QUERY_WORKER_CONCURRENCY:-1}", - "--loglevel", "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}", + "--loglevel", "WARNING", "-f", "/var/log/query_worker/worker.log", "-Q", "query", "-n", "query-worker" From 6b0e1f25cc5735c907992634be4fc4d2d6b79c64 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:20:07 -0400 Subject: [PATCH 159/408] fix(docker-compose): Fix environment variable name for results cache uri. --- tools/deployment/package/docker-compose.base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 68ec1e5c9d..11d00dbd9d 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -149,7 +149,7 @@ services: "python3", "-u", "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", + "--uri", "mongodb://results_cache:27017/${CLP_RESULTS_CACHE_DB_NAME:-clp-query-results}", "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", ] From ddcf760d73cd25116929ab011bf86184be7c9596 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:35:22 -0400 Subject: [PATCH 160/408] refactor(clp-package-utils): Move EnvVarsDict type alias below imports. --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index b0348eac85..6923f095d2 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -10,9 +10,6 @@ from abc import ABC, abstractmethod from typing import Any, Dict -# Type alias for environment variables dictionary. -EnvVarsDict = Dict[str, str] - from clp_py_utils.clp_config import ( AwsAuthType, CLPConfig, @@ -51,6 +48,9 @@ validate_webui_config, ) +# Type alias for environment variables dictionary. +EnvVarsDict = Dict[str, str] + LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" SERVICE_CONTAINER_USER_ID = 999 From b1866fee335ce6a7ca5cd96abdc4843372e5cbf3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 6 Oct 2025 15:36:11 -0400 Subject: [PATCH 161/408] Add config --- .../clp-py-utils/clp_py_utils/clp_config.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index b2eb995ea2..4225910aad 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -267,6 +267,63 @@ def transform_for_container(self): self.port = self.DEFAULT_PORT +class SpiderDb(BaseModel): + DEFAULT_PORT: ClassVar[int] = 3306 + + type: str = "mariadb" + host: str = "localhost" + port: int = DEFAULT_PORT + name: str = "spider-db" + ssl_cert: Optional[str] = None + auto_commit: bool = False + compress: bool = True + + username: Optional[str] = None + password: Optional[str] = None + + @field_validator("type") + @classmethod + def validate_type(cls, value): + supported_database_types = ["mysql", "mariadb"] + if value not in supported_database_types: + raise ValueError( + f"database.type must be one of the following {'|'.join(supported_database_types)}" + ) + return value + + @field_validator("name") + @classmethod + def validate_name(cls, value): + if "" == value: + raise ValueError("database.name cannot be empty.") + return value + + @field_validator("host") + @classmethod + def validate_host(cls, value): + if "" == value: + raise ValueError("database.host cannot be empty.") + return value + + @field_validator("port") + @classmethod + def validate_port(cls, value): + _validate_port(cls, value) + return value + + +class SpiderScheduler(BaseModel): + DEFAULT_PORT: ClassVar[int] = 6000 + + port: int = DEFAULT_PORT + + @field_validator("port") + @classmethod + def validate_port(cls, value): + _validate_port(cls, value) + return value + + def _validate_logging_level(cls, value): if not is_valid_logging_level(value): raise ValueError( @@ -289,9 +346,15 @@ def _validate_port(cls, value): ) +class OrchestrationType(KebabCaseStrEnum): + celery = auto() + spider = auto() + + class CompressionScheduler(BaseModel): jobs_poll_delay: float = 0.1 # seconds logging_level: str = "INFO" + type: OrchestrationType = OrchestrationType.celery @field_validator("logging_level") @classmethod From 4134b1a43a4fbc0da762242a418adbd28e983f62 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:38:01 -0400 Subject: [PATCH 162/408] refactor(clp-package-utils): Rename LOGS_FILE_MODE to LOG_FILE_ACCESS_MODE for clarity. --- .../clp-package-utils/clp_package_utils/controller.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 6923f095d2..289a07710f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -51,7 +51,8 @@ # Type alias for environment variables dictionary. EnvVarsDict = Dict[str, str] -LOGS_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH +LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" SERVICE_CONTAINER_USER_ID = 999 SERVICE_CONTAINER_GROUP_ID = 999 @@ -215,7 +216,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + logs_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, @@ -232,7 +233,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_file = self.clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + logs_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, From f483abda2ee5f59e9a47f083b128b5260efc4da7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:39:39 -0400 Subject: [PATCH 163/408] reflow comments to 100 char per line --- components/clp-package-utils/clp_package_utils/controller.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 289a07710f..e0efcf404d 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -63,9 +63,8 @@ class BaseController(ABC): """ - Abstract base controller for preparing and deploying CLP components. - Provides common logic for preparing environment variables, directories, - and configuration files for each service. + Abstract base controller for preparing and deploying CLP components. Provides common logic for + preparing environment variables, directories, and configuration files for each service. """ def __init__(self, clp_config: CLPConfig): From 7d4f47d813460bd1ac5d2fea53ebc090d1152815 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:40:17 -0400 Subject: [PATCH 164/408] refactor(clp-package-utils): Rename deploy method to start and adjust script usage. --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++--- .../clp_package_utils/scripts/start_clp.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e0efcf404d..a447bf814e 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -73,9 +73,9 @@ def __init__(self, clp_config: CLPConfig): self._conf_dir = self._clp_home / "etc" @abstractmethod - def deploy(self): + def start(self): """ - Deploys the provisioned components with orchestrator-specific logic. + Starts the provisioned components with orchestrator-specific logic. """ pass @@ -463,7 +463,7 @@ def __init__(self, clp_config: CLPConfig, instance_id: str): self._project_name = f"clp-package-{instance_id}" super().__init__(clp_config) - def deploy(self): + def start(self): """ Deploys CLP components using Docker Compose by: 1. Checking Docker dependencies. diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 78fc6739a8..368612530f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -62,7 +62,7 @@ def main(argv): instance_id = get_or_create_instance_id(clp_config) try: controller = DockerComposeController(clp_config, instance_id) - controller.deploy() + controller.start() except Exception as ex: if type(ex) == ValueError: logger.error(f"Failed to start CLP: {ex}") From 064f36512da63585d68ba06d08402e69bd7fd98f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:41:24 -0400 Subject: [PATCH 165/408] refactor(clp-package-utils): Rename _provision method to _set_up_env and update calls and documentation. --- .../clp-package-utils/clp_package_utils/controller.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index a447bf814e..3c833d562e 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -87,9 +87,10 @@ def stop(self): pass @abstractmethod - def _provision(self) -> EnvVarsDict: + def _set_up_env(self) -> EnvVarsDict: """ - Prepares all components with orchestrator-specific logic. + Sets up all components for the orchestrator by preparing environment variables, directories, + and configuration files. :return: Dictionary of environment variables to be used by the orchestrator. """ @@ -471,7 +472,7 @@ def start(self): 3. Running `docker compose up -d`. """ check_docker_dependencies(should_compose_run=False, project_name=self._project_name) - self._provision() + self._set_up_env() deployment_type = self.clp_config.get_deployment_type() logger.info(f"Starting CLP using Docker Compose ({deployment_type})...") @@ -519,7 +520,7 @@ def _get_num_workers() -> int: """ return multiprocessing.cpu_count() // 2 - def _provision(self): + def _set_up_env(self): """ Provisions all CLP components for Docker Compose by: - Generating container-specific config. From 7beb19998dc502b376e7b94a2e014bb98c32135a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 15:46:32 -0400 Subject: [PATCH 166/408] =?UTF-8?q?docs(clp-package-utils):=20Update=20doc?= =?UTF-8?q?strings=20to=20use=20=E2=80=9Csets=20up=E2=80=9D=20phrasing.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clp_package_utils/controller.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 3c833d562e..e5749b50f5 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -75,7 +75,7 @@ def __init__(self, clp_config: CLPConfig): @abstractmethod def start(self): """ - Starts the provisioned components with orchestrator-specific logic. + Starts the set-up components with orchestrator-specific logic. """ pass @@ -98,7 +98,7 @@ def _set_up_env(self) -> EnvVarsDict: def _set_up_env_for_database(self) -> EnvVarsDict: """ - Prepares environment variables and directories for the database component. + Sets up environment variables and directories for the database component. :return: Dictionary of component-related environment variables. """ @@ -129,7 +129,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: def _set_up_env_for_queue(self) -> EnvVarsDict: """ - Prepares environment variables and directories for the message queue component. + Sets up environment variables and directories for the message queue component. :return: Dictionary of component-related environment variables. """ @@ -151,7 +151,7 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: def _set_up_env_for_redis(self) -> EnvVarsDict: """ - Prepares environment variables and directories for the Redis component. + Sets up environment variables and directories for the Redis component. :return: Dictionary of component-related environment variables. """ @@ -181,7 +181,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ - Prepares environment variables and directories for the results cache (MongoDB) component. + Sets up environment variables and directories for the results cache (MongoDB) component. :return: Dictionary of component-related environment variables. """ @@ -208,7 +208,7 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: """ - Prepares environment variables and files for the compression scheduler component. + Sets up environment variables and files for the compression scheduler component. :return: Dictionary of component-related environment variables. """ @@ -225,7 +225,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: """ - Prepares environment variables and files for the query scheduler component. + Sets up environment variables and files for the query scheduler component. :return: Dictionary of component-related environment variables. """ @@ -242,7 +242,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: """ - Prepares environment variables for the compression worker component. + Sets up environment variables for the compression worker component. :param num_workers: Number of worker processes to run. :return: Dictionary of compression worker-related environment variables. @@ -261,7 +261,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: """ - Prepares environment variables for the query worker component. + Sets up environment variables for the query worker component. :param num_workers: Number of worker processes to run. :return: Dictionary of component-related environment variables. @@ -280,7 +280,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: """ - Prepares environment variables for the reducer component. + Sets up environment variables for the reducer component. :param num_workers: Number of worker processes to run. :return: Dictionary of component-related environment variables. @@ -300,7 +300,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: """ - Prepares environment variables and settings for the Web UI component. + Sets up environment variables and settings for the Web UI component. :param container_clp_config: CLP configuration inside the containers. :return: Dictionary of component-related environment variables. @@ -402,7 +402,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: """ - Prepares environment variables for the garbage collector component. + Sets up environment variables for the garbage collector component. :return: Dictionary of component-related environment variables. """ @@ -468,7 +468,7 @@ def start(self): """ Deploys CLP components using Docker Compose by: 1. Checking Docker dependencies. - 2. Provisioning environment variables and configuration. + 2. Setting up environment variables and configuration. 3. Running `docker compose up -d`. """ check_docker_dependencies(should_compose_run=False, project_name=self._project_name) @@ -522,7 +522,7 @@ def _get_num_workers() -> int: def _set_up_env(self): """ - Provisions all CLP components for Docker Compose by: + Sets up all CLP components for Docker Compose by: - Generating container-specific config. - Preparing environment variables for all components. - Writing environment variables to `.env`. From 3507a1bcc9740b9727ef8d58c5df47b11280a643 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:08:44 -0400 Subject: [PATCH 167/408] docs(clp-package-utils): Standardize return docstrings; Remove return type hint from _set_up_env(). --- .../clp_package_utils/controller.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e5749b50f5..98a1bed560 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -87,12 +87,10 @@ def stop(self): pass @abstractmethod - def _set_up_env(self) -> EnvVarsDict: + def _set_up_env(self): """ Sets up all components for the orchestrator by preparing environment variables, directories, and configuration files. - - :return: Dictionary of environment variables to be used by the orchestrator. """ pass @@ -100,7 +98,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: """ Sets up environment variables and directories for the database component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = DB_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -131,7 +129,7 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: """ Sets up environment variables and directories for the message queue component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = QUEUE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -153,7 +151,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: """ Sets up environment variables and directories for the Redis component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = REDIS_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -183,7 +181,7 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ Sets up environment variables and directories for the results cache (MongoDB) component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = RESULTS_CACHE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -210,7 +208,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: """ Sets up environment variables and files for the compression scheduler component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -227,7 +225,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: """ Sets up environment variables and files for the query scheduler component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -245,7 +243,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: Sets up environment variables for the compression worker component. :param num_workers: Number of worker processes to run. - :return: Dictionary of compression worker-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -264,7 +262,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: Sets up environment variables for the query worker component. :param num_workers: Number of worker processes to run. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -283,7 +281,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: Sets up environment variables for the reducer component. :param num_workers: Number of worker processes to run. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = REDUCER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -303,7 +301,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: Sets up environment variables and settings for the Web UI component. :param container_clp_config: CLP configuration inside the containers. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = WEBUI_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -404,7 +402,7 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: """ Sets up environment variables for the garbage collector component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of environment variables necessary to launch the component. """ component_name = GARBAGE_COLLECTOR_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") @@ -526,8 +524,6 @@ def _set_up_env(self): - Generating container-specific config. - Preparing environment variables for all components. - Writing environment variables to `.env`. - - :return: Dictionary of all environment variables. """ container_clp_config = generate_docker_compose_container_config(self.clp_config) num_workers = self._get_num_workers() From eb2754a84c3d99571846f06bd6e1271073ea9fdc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:10:00 -0400 Subject: [PATCH 168/408] refactor(clp-package-utils): Add spacing before log and data directory setup for readability. --- .../clp-package-utils/clp_package_utils/controller.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 98a1bed560..47d4d2aec1 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -107,6 +107,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) @@ -136,6 +137,7 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: logs_dir = self.clp_config.logs_directory / component_name validate_queue_config(self.clp_config, logs_dir) + logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(logs_dir) @@ -160,6 +162,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logs_dir = self.clp_config.logs_directory / component_name data_dir = self.clp_config.data_directory / component_name validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) @@ -190,6 +193,7 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) @@ -249,6 +253,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -268,6 +273,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -287,6 +293,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -408,6 +415,7 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name + logs_dir.mkdir(parents=True, exist_ok=True) return {"CLP_GC_LOGGING_LEVEL": self.clp_config.garbage_collector.logging_level} From cae66df46e60c54bf38571f7a0978ef51a047569 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:22:47 -0400 Subject: [PATCH 169/408] refactor(clp-package-utils): Move logs directory assignment after data directory assignment --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 47d4d2aec1..cbaad687e9 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -159,8 +159,8 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" - logs_dir = self.clp_config.logs_directory / component_name data_dir = self.clp_config.data_directory / component_name + logs_dir = self.clp_config.logs_directory / component_name validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) From dffc2f2a648064b7407b9a4da0169d42f370c522 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:23:44 -0400 Subject: [PATCH 170/408] refactor(clp-package-utils): Rename logs_file -> log_file --- .../clp_package_utils/controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index cbaad687e9..2aea50f55c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -217,12 +217,12 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_file = self.clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) + log_file = self.clp_config.logs_directory / f"{component_name}.log" + log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, - "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(log_file), } def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: @@ -234,12 +234,12 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_file = self.clp_config.logs_directory / f"{component_name}.log" - logs_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) + log_file = self.clp_config.logs_directory / f"{component_name}.log" + log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, - "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(log_file), } def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: From c0ef4ff24b5ecf12f863729b625a013c5d1a1e29 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:27:59 -0400 Subject: [PATCH 171/408] refactor(clp-package-utils): Rename LOGS_FILE variables to LOG_FILE in Docker compose files and controller. --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- tools/deployment/package/docker-compose.base.yaml | 2 +- tools/deployment/package/docker-compose.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 2aea50f55c..d87e29b3a1 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -222,7 +222,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: return { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, - "CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST": str(log_file), + "CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST": str(log_file), } def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: @@ -239,7 +239,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: return { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, - "CLP_QUERY_SCHEDULER_LOGS_FILE_HOST": str(log_file), + "CLP_QUERY_SCHEDULER_LOG_FILE_HOST": str(log_file), } def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 11d00dbd9d..b70b06e1c4 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -167,7 +167,7 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ + - "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}\ :/var/log/compression_scheduler.log" - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - "/:/mnt/logs:ro" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index e793d08dfe..a58310cb3f 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -31,7 +31,7 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_QUERY_SCHEDULER_LOGS_FILE_HOST:-./var/log/query_scheduler.log}:\ + - "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}:\ /var/log/query_scheduler.log" depends_on: db-table-creator: From 95404d0f72dbcf0e025989d201401f3b6d15cb99 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:29:55 -0400 Subject: [PATCH 172/408] refactor(clp-package-utils): Remove redundant flush after writing instance id. --- components/clp-package-utils/clp_package_utils/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index d87e29b3a1..0f31a423c6 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -590,7 +590,6 @@ def get_or_create_instance_id(clp_config: CLPConfig): instance_id = str(uuid.uuid4())[-4:] with open(instance_id_file_path, "w") as f: f.write(instance_id) - f.flush() return instance_id From e1db192b53477422b5c1db429458d6b6a8dc8f1a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:32:14 -0400 Subject: [PATCH 173/408] Clarify resolving to IPv4 in the docstring - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0f31a423c6..51b9af2b6d 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -625,7 +625,7 @@ def _chown_recursively( def _get_ip_from_hostname(hostname: str) -> str: """ - Resolves a hostname to an IP address. + Resolves a hostname to an IPv4 IP address. :param hostname: The hostname to resolve. :return: The resolved IP address. From 630329b71fd5bca9f185c48d0c8dd0e0918e751c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:33:33 -0400 Subject: [PATCH 174/408] Remove redundant param description in `_get_ip_from_hostname()` - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 51b9af2b6d..3b30875844 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -627,7 +627,7 @@ def _get_ip_from_hostname(hostname: str) -> str: """ Resolves a hostname to an IPv4 IP address. - :param hostname: The hostname to resolve. + :param hostname: :return: The resolved IP address. """ return socket.gethostbyname(hostname) From d9da76384a041f3b5b7644d40dbd27dfb0edaee0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:48:40 -0400 Subject: [PATCH 175/408] refactor(clp-package-utils): Rename db logging config variable and env placeholder for clarity. --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++--- tools/deployment/package/docker-compose.base.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 3b30875844..136ae09f04 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -103,17 +103,17 @@ def _set_up_env_for_database(self) -> EnvVarsDict: component_name = DB_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - conf_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" + conf_logging_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" data_dir = self.clp_config.data_directory / component_name logs_dir = self.clp_config.logs_directory / component_name - validate_db_config(self.clp_config, conf_file, data_dir, logs_dir) + validate_db_config(self.clp_config, conf_logging_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) return { - "CLP_DB_CONF_FILE_HOST": str(conf_file), + "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), "CLP_DB_HOST": _get_ip_from_hostname(self.clp_config.database.host), diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index b70b06e1c4..fa545b68d3 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -34,7 +34,7 @@ services: ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" healthcheck: From c2005028e22d7d9c2f54250ff891f6e44d82a7b1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 18:55:56 -0400 Subject: [PATCH 176/408] use long form `:readonly` instead of short form `:ro` in binding mount specifications --- .../package/docker-compose.base.yaml | 29 ++++++++++--------- tools/deployment/package/docker-compose.yaml | 8 ++--- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index fa545b68d3..956aab4649 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -34,7 +34,8 @@ services: ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}\ +:/etc/mysql/conf.d/logging.cnf:readonly" - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" healthcheck: @@ -96,7 +97,8 @@ services: ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" volumes: - - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf:ro" + - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf\ +:readonly" - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" healthcheck: @@ -123,7 +125,8 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf\ +:readonly" - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" healthcheck: @@ -166,11 +169,11 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}\ :/var/log/compression_scheduler.log" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" + - "/:/mnt/logs:readonly" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -203,12 +206,12 @@ services: volumes: - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ /var/log/compression_worker" - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" + - "/:/mnt/logs:readonly" command: [ "python3", "-u", @@ -236,11 +239,11 @@ services: ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" + - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:readonly" - "./var/www/webui/server/dist/settings.json\ -:/opt/clp/var/www/webui/server/dist/settings.json:ro" +:/opt/clp/var/www/webui/server/dist/settings.json:readonly" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -270,7 +273,7 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" depends_on: db-table-creator: diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index a58310cb3f..45bfc222bd 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -30,7 +30,7 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}:\ /var/log/query_scheduler.log" depends_on: @@ -73,8 +73,8 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}:/var/log/query_worker" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" @@ -100,7 +100,7 @@ services: CLP_LOGS_DIR: "/var/log/reducer" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}:/var/log/reducer" depends_on: query-scheduler: From 5751ddf58505ffbf690ea9888b6995549f4cdc6a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 19:24:03 -0400 Subject: [PATCH 177/408] revert unintentional change --- docs/src/user-docs/guides-overview.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/user-docs/guides-overview.md b/docs/src/user-docs/guides-overview.md index 62f01a9835..60d7bbacd1 100644 --- a/docs/src/user-docs/guides-overview.md +++ b/docs/src/user-docs/guides-overview.md @@ -32,3 +32,4 @@ Using Presto with CLP ^^^ How to use Presto to query compressed logs in CLP. ::: +:::: From 3e9734ed8e079c8e1f3ab637579bbd659702db1e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 19:29:56 -0400 Subject: [PATCH 178/408] chore(deployment): Add local logging driver to service defaults. --- tools/deployment/package/docker-compose.base.yaml | 2 ++ tools/deployment/package/docker-compose.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 956aab4649..a120906d1c 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -3,6 +3,8 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + logging: + driver: "local" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 45bfc222bd..70b062458f 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -5,6 +5,8 @@ include: ["docker-compose.base.yaml"] # Common service defaults. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + logging: + driver: "local" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" From ab64f4aca35473fcc54287e19b0321098fd6a526 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 20:12:33 -0400 Subject: [PATCH 179/408] Improve docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/user-docs/guides-multi-node.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-docs/guides-multi-node.md b/docs/src/user-docs/guides-multi-node.md index 9a56f4c631..52f206f7af 100644 --- a/docs/src/user-docs/guides-multi-node.md +++ b/docs/src/user-docs/guides-multi-node.md @@ -3,8 +3,8 @@ A multi-node deployment allows you to run CLP across a distributed set of hosts. :::{warning} -CLP now uses Docker Compose for orchestration and support for multi-node deployments is -temporarily removed. Please contact us if you need immediate support for multi-node deployments, or +CLP now uses Docker Compose for orchestration and support for multi-node deployments has been +removed temporarily. Please contact us if you need immediate support for multi-node deployments, or stay tuned for future updates on Kubernetes Helm support. ::: From 2d9906f8723c7dace88f87ff184c72312d97475b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 20:48:20 -0400 Subject: [PATCH 180/408] chore(deployment): Adjust healthcheck defaults (start interval, period, timeout). --- tools/deployment/package/docker-compose.base.yaml | 6 +++--- tools/deployment/package/docker-compose.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a120906d1c..a89b0bf4b9 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -12,9 +12,9 @@ x-service-defaults: &service_defaults x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 - start_interval: "1s" - start_period: "10s" - timeout: "10s" + start_interval: "2s" + start_period: "60s" + timeout: "2s" secrets: CLP_DB_PASS_FILE: diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 70b062458f..4aae85e367 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -14,9 +14,9 @@ x-service-defaults: &service_defaults x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 - start_interval: "1s" - start_period: "10s" - timeout: "10s" + start_interval: "2s" + start_period: "60s" + timeout: "2s" services: query-scheduler: From 3635883b314789b310b51c9fdde0519cb3564d45 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 22:31:19 -0400 Subject: [PATCH 181/408] refactor(deployment): Use long form volume definitions and use bind objects with anchors. --- .../package/docker-compose.base.yaml | 123 +++++++++++++----- tools/deployment/package/docker-compose.yaml | 46 +++++-- 2 files changed, 124 insertions(+), 45 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a89b0bf4b9..1ff53e5b5f 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -16,6 +16,24 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "60s" timeout: "2s" +# Common volume definitions. +x-volume-definitions: + aws-config-readonly: &volume_aws_config_readonly + type: "bind" + source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" + target: "/.aws" + read_only: true + clp-config-readonly: &volume_clp_config_readonly + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" + target: "/etc/clp-config.yml" + read_only: true + logs-input-readonly: &volume_root_logs_readonly + type: "bind" + source: "/" + target: "/mnt/logs" + read_only: true + secrets: CLP_DB_PASS_FILE: environment: "CLP_DB_PASS" @@ -36,10 +54,16 @@ services: ports: - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" volumes: - - "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}\ -:/etc/mysql/conf.d/logging.cnf:readonly" - - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" + - type: "bind" + source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}" + target: "/etc/mysql/conf.d/logging.cnf" + read_only: true + - type: "bind" + source: "${CLP_DB_DATA_DIR_HOST:-./var/data/database}" + target: "/var/lib/mysql" + - type: "bind" + source: "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}" + target: "/var/log/mysql" healthcheck: <<: *healthcheck_defaults test: [ @@ -59,7 +83,7 @@ services: CLP_DB_USER: "${CLP_DB_USER}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + - *volume_clp_config_readonly depends_on: database: condition: "service_healthy" @@ -83,7 +107,9 @@ services: ports: - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" volumes: - - "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}:/var/log/rabbitmq" + - type: "bind" + source: "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}" + target: "/var/log/rabbitmq" healthcheck: <<: *healthcheck_defaults test: [ @@ -99,10 +125,16 @@ services: ports: - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" volumes: - - "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}:/usr/local/etc/redis/redis.conf\ -:readonly" - - "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}:/data" - - "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}:/var/log/redis" + - type: "bind" + source: "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}" + target: "/usr/local/etc/redis/redis.conf" + read_only: true + - type: "bind" + source: "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}" + target: "/data" + - type: "bind" + source: "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}" + target: "/var/log/redis" healthcheck: <<: *healthcheck_defaults test: [ @@ -127,10 +159,16 @@ services: ports: - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" volumes: - - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf\ -:readonly" - - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" + - type: "bind" + source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}" + target: "/etc/mongo/mongod.conf" + read_only: true + - type: "bind" + source: "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}" + target: "/data/db" + - type: "bind" + source: "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}" + target: "/var/log/mongodb" healthcheck: <<: *healthcheck_defaults test: [ @@ -171,11 +209,12 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - - "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}\ -:/var/log/compression_scheduler.log" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "/:/mnt/logs:readonly" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - *volume_root_logs_readonly + - type: "bind" + source: "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}" + target: "/var/log/compression_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -206,14 +245,21 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ -/var/log/compression_worker" - - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "/:/mnt/logs:readonly" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - *volume_root_logs_readonly + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" + target: "/var/data/archives" + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" + target: "/var/data/staged-archives" + - type: "bind" + source: "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}" + target: "/var/log/compression_worker" + - type: "bind" + source: "${CLP_DATA_DIR_HOST:-./var/data}" + target: "/var/data" command: [ "python3", "-u", @@ -241,11 +287,18 @@ services: ports: - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:readonly" - - "./var/www/webui/server/dist/settings.json\ -:/opt/clp/var/www/webui/server/dist/settings.json:readonly" + - *volume_aws_config_readonly + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" + target: "/var/data/streams" + - type: "bind" + source: "./var/www/webui/client/settings.json" + target: "/opt/clp/var/www/webui/client/settings.json" + read_only: true + - type: "bind" + source: "./var/www/webui/server/dist/settings.json" + target: "/opt/clp/var/www/webui/server/dist/settings.json" + read_only: true depends_on: db-table-creator: condition: "service_completed_successfully" @@ -275,8 +328,10 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector" + target: "/var/log/garbage_collector" depends_on: db-table-creator: condition: "service_completed_successfully" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 4aae85e367..acd54ed415 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -18,6 +18,19 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "60s" timeout: "2s" +# Common volume definitions. +x-volume-definitions: + aws-config-readonly: &volume_aws_config_readonly + type: "bind" + source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" + target: "/.aws" + read_only: true + clp-config-readonly: &volume_clp_config_readonly + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" + target: "/etc/clp-config.yml" + read_only: true + services: query-scheduler: <<: *service_defaults @@ -32,9 +45,10 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}:\ -/var/log/query_scheduler.log" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}" + target: "/var/log/query_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -74,12 +88,20 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:readonly" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}:/var/log/query_worker" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}:/var/data/staged-streams" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" + target: "/var/data/archives" + - type: "bind" + source: "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}" + target: "/var/log/query_worker" + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}" + target: "/var/data/staged-streams" + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" + target: "/var/data/streams" command: [ "python3", "-u", @@ -102,8 +124,10 @@ services: CLP_LOGS_DIR: "/var/log/reducer" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:readonly" - - "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}:/var/log/reducer" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}" + target: "/var/log/reducer" depends_on: query-scheduler: condition: "service_healthy" From 5e0d0d2311813fa5c01970812a6a9bd6ebc6ce89 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 23:13:59 -0400 Subject: [PATCH 182/408] refactor(deployment): Use long form port definitions; fix CLP_RESULTS_CACHE_PORT fallback 6379 -> 27017. --- .../package/docker-compose.base.yaml | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 1ff53e5b5f..c6e0730a9b 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -52,7 +52,9 @@ services: secrets: - "CLP_DB_PASS_FILE" ports: - - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + - host_ip: "${CLP_DB_HOST:-127.0.0.1}" + target: 3306 + published: "${CLP_DB_PORT:-3306}" volumes: - type: "bind" source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}" @@ -105,7 +107,9 @@ services: RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - - "${CLP_QUEUE_HOST:-127.0.0.1}:${CLP_QUEUE_PORT:-5672}:5672" + - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" + target: 5672 + published: "${CLP_QUEUE_PORT:-5672}" volumes: - type: "bind" source: "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}" @@ -123,7 +127,9 @@ services: hostname: "redis" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - - "${CLP_REDIS_HOST:-127.0.0.1}:${CLP_REDIS_PORT:-6379}:6379" + - host_ip: "${CLP_REDIS_HOST:-127.0.0.1}" + target: 6379 + published: "${CLP_REDIS_PORT:-6379}" volumes: - type: "bind" source: "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}" @@ -157,7 +163,9 @@ services: hostname: "results_cache" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + - host_ip: "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}" + target: 27017 + published: "${CLP_RESULTS_CACHE_PORT:-27017}" volumes: - type: "bind" source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}" @@ -285,7 +293,9 @@ services: PORT: "4000" RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" ports: - - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + - host_ip: "${CLP_WEBUI_HOST:-127.0.0.1}" + target: 4000 + published: "${CLP_WEBUI_PORT:-4000}" volumes: - *volume_aws_config_readonly - type: "bind" From e771800aed92323320a5194cae23516f12bec62b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 23:20:50 -0400 Subject: [PATCH 183/408] refactor(deployment): Reorder port definitions to long form order. --- tools/deployment/package/docker-compose.base.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index c6e0730a9b..7461ce3e6c 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -53,8 +53,8 @@ services: - "CLP_DB_PASS_FILE" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" - target: 3306 published: "${CLP_DB_PORT:-3306}" + target: 3306 volumes: - type: "bind" source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}" @@ -108,8 +108,8 @@ services: RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" - target: 5672 published: "${CLP_QUEUE_PORT:-5672}" + target: 5672 volumes: - type: "bind" source: "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}" @@ -128,8 +128,8 @@ services: user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_REDIS_HOST:-127.0.0.1}" - target: 6379 published: "${CLP_REDIS_PORT:-6379}" + target: 6379 volumes: - type: "bind" source: "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}" @@ -164,8 +164,8 @@ services: user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}" - target: 27017 published: "${CLP_RESULTS_CACHE_PORT:-27017}" + target: 27017 volumes: - type: "bind" source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}" @@ -294,8 +294,8 @@ services: RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" ports: - host_ip: "${CLP_WEBUI_HOST:-127.0.0.1}" - target: 4000 published: "${CLP_WEBUI_PORT:-4000}" + target: 4000 volumes: - *volume_aws_config_readonly - type: "bind" From 6e878cf3a6aa58baf5286b145c8365eb3c256112 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 6 Oct 2025 23:47:15 -0400 Subject: [PATCH 184/408] refactor(clp-package-utils): Replace deprecated Pydantic (V2) copy() with model_copy() --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 9fa7f946b1..c428e0b351 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -313,7 +313,7 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig :param clp_config: :return: The container config and the mounts. """ - container_clp_config = clp_config.copy(deep=True) + container_clp_config = clp_config.model_copy(deep=True) container_clp_config.transform_for_container() return container_clp_config From f2b096765f82c4c77e5e05b7eb2fdc8c2a568ecf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 00:15:22 -0400 Subject: [PATCH 185/408] feat(deployment): Add support for configurable logs input directory mapping via env vars. --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++++- components/clp-py-utils/clp_py_utils/clp_config.py | 4 +++- tools/deployment/package/docker-compose.base.yaml | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 136ae09f04..0b3807a3dd 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -546,9 +546,13 @@ def _set_up_env(self): ), # Package container "CLP_PACKAGE_CONTAINER": self.clp_config.container_image_ref, - # Global paths + # Runtime data directories "CLP_DATA_DIR_HOST": str(self.clp_config.data_directory), "CLP_LOGS_DIR_HOST": str(self.clp_config.logs_directory), + # Input directories + "CLP_LOGS_INPUT_DIR_HOST": str(self.clp_config.logs_input.directory), + "CLP_LOGS_INPUT_DIR_CONTAINER": str(container_clp_config.logs_input.directory), + # Output directories "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.clp_config.archive_output.get_directory()), "CLP_STREAM_OUTPUT_DIR_HOST": str(self.clp_config.stream_output.get_directory()), # AWS credentials diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index b2eb995ea2..29fc9c6310 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -685,7 +685,9 @@ class FsIngestionConfig(FsStorage): directory: pathlib.Path = pathlib.Path("/") def transform_for_container(self): - self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR + input_logs_dir = self.directory.resolve() + self.directory = (CONTAINER_INPUT_LOGS_ROOT_DIR / + input_logs_dir.relative_to(input_logs_dir.anchor)) class ArchiveFsStorage(FsStorage): diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 7461ce3e6c..52a4d8eeec 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -30,8 +30,8 @@ x-volume-definitions: read_only: true logs-input-readonly: &volume_root_logs_readonly type: "bind" - source: "/" - target: "/mnt/logs" + source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" + target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" read_only: true secrets: From 01558ba3dc6412342007b512391d109ea812f26a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 03:48:23 -0400 Subject: [PATCH 186/408] lint --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 29fc9c6310..43d1695e60 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -686,8 +686,9 @@ class FsIngestionConfig(FsStorage): def transform_for_container(self): input_logs_dir = self.directory.resolve() - self.directory = (CONTAINER_INPUT_LOGS_ROOT_DIR / - input_logs_dir.relative_to(input_logs_dir.anchor)) + self.directory = CONTAINER_INPUT_LOGS_ROOT_DIR / input_logs_dir.relative_to( + input_logs_dir.anchor + ) class ArchiveFsStorage(FsStorage): From 8a3985ba496e7c008e263297d554dd71aa5f28d0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 04:05:20 -0400 Subject: [PATCH 187/408] refactor(deployment): Remove secrets usage; replace with environment variables. --- tools/deployment/package/docker-compose.base.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 52a4d8eeec..6c27235240 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -34,10 +34,6 @@ x-volume-definitions: target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" read_only: true -secrets: - CLP_DB_PASS_FILE: - environment: "CLP_DB_PASS" - services: database: <<: *service_defaults @@ -46,11 +42,9 @@ services: user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_PASSWORD: "${CLP_DB_PASS}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS}" MYSQL_USER: "${CLP_DB_USER}" - secrets: - - "CLP_DB_PASS_FILE" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" published: "${CLP_DB_PORT:-3306}" From c691735e4ab28c7068ad65bf5738472e694242db Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 04:18:42 -0400 Subject: [PATCH 188/408] docs(design): Add health check defaults. --- docs/src/dev-docs/design-docker-compose.md | 42 ++++++++++++++++------ 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index e365decd63..eb14b23e8a 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -4,8 +4,8 @@ This document explains the technical details of CLP's Docker Compose implementat ## Overview -The Docker Compose implementation follows a controller architecture with a `BaseController` abstract class and a -`DockerComposeController` implementation. +The Docker Compose implementation follows a controller architecture with a `BaseController` abstract +class and a `DockerComposeController` implementation. ## Architecture @@ -20,14 +20,16 @@ The orchestration uses a controller pattern: The controller performs these initialization steps: -1. **Provisioning**: Provisions all components and generates component specific configuration variables. -2. **Configuration Transformation**: The `transform_for_container()` method in `CLPConfig` adapts configurations for - containerized environments +1. **Provisioning**: Provisions all components and generates component specific configuration + variables. +2. **Configuration Transformation**: The `transform_for_container()` method in `CLPConfig` adapts + configurations for containerized environments 3. **Environment Generation**: Creates a `.env` file with necessary Docker Compose variables ### Configuration Transformation -The `transform_for_container()` method in the `CLPConfig` class and related component classes adapts the configuration for containerized environments by: +The `transform_for_container()` method in the `CLPConfig` class and related component classes +adapts the configuration for containerized environments by: 1. Converting host paths to container paths 2. Updating service hostnames to match Docker Compose service names @@ -35,7 +37,8 @@ The `transform_for_container()` method in the `CLPConfig` class and related comp ### Environment Variables -The controller generates a comprehensive set of environment variables that are written to a `.env` file, including: +The controller generates a comprehensive set of environment variables that are written to a `.env` +file, including: * Component-specific settings (ports, logging levels, concurrency) * Credentials for database, queue, and Redis services @@ -58,8 +61,8 @@ CLP supports two deployment types determined by the `package.query_engine` confi The Docker Compose setup uses two files: -* `docker-compose.base.yaml`: Defines base services for all deployment types, excluding Celery scheduler and worker - components to allow separate Presto [integration][presto-integration]. +* `docker-compose.base.yaml`: Defines base services for all deployment types, excluding Celery + scheduler and worker components to allow separate Presto [integration][presto-integration]. * `docker-compose.yaml`: Extends the base file with additional services for complete deployments Each file defines services with: @@ -70,6 +73,22 @@ Each file defines services with: * Network configuration * User permissions +### Health check defaults + +Below are the default health check settings and the rationale for each: + +* `interval: 30s` — default probe interval in steady state. Avoid setting this too low to avoid + excessive resource usage. +* `timeout: 2s` — no remote communication is expected, so a short timeout is sufficient. +* `retries: 3` + - a service is deemed unhealthy if it does not respond in ~(30s * 3) = 90s since it is in the + steady state. + - a service is deemed unhealthy if it does not respond within ~(60s + 90s) since it is started. +* `start_interval: 2s` — A short interval allows the service to become healthy quickly once it + is ready, allowing other services which depend on it to start. +* `start_period: 60s` — the first minute of startup ignores failures, effectively granting around 30 + fast attempts to become healthy before retries start counting. + ## Service architecture The Docker Compose setup includes the following services: @@ -140,7 +159,8 @@ graph LR ### Services overview -The CLP package is composed of several service components. The tables below list the services and their functions. +The CLP package is composed of several service components. The tables below list the services and +their functions. :::{table} Services :align: left @@ -192,4 +212,4 @@ If you encounter issues with the Docker Compose deployment: docker compose config ``` -[presto-integration]: ../user-docs/guides-using-presto.md \ No newline at end of file +[presto-integration]: ../user-docs/guides-using-presto.md From 787c7d77b0bf9d21e99d874335b807d85c8538e2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 7 Oct 2025 04:27:37 -0400 Subject: [PATCH 189/408] refactor(clp-package-utils): Improve error handling for config loading and directory creation. --- .../clp_package_utils/scripts/start_clp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 368612530f..a9ff5b596e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -34,8 +34,8 @@ def main(argv): parsed_args = args_parser.parse_args(argv[1:]) - # Validate and load config file try: + # Validate and load config file. config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) @@ -49,14 +49,18 @@ def main(argv): clp_config.validate_data_dir() clp_config.validate_logs_dir() clp_config.validate_aws_config_dir() + except: + logger.exception("Failed to load config.") + return -1 - # Create necessary directories + try: + # Create necessary directories. clp_config.data_directory.mkdir(parents=True, exist_ok=True) clp_config.logs_directory.mkdir(parents=True, exist_ok=True) clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) except: - logger.exception("Failed to load config.") + logger.exception("Failed to create necessary directories.") return -1 instance_id = get_or_create_instance_id(clp_config) From ba2940606832dbf296f0d94fa123959f88b856a4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 8 Oct 2025 10:00:08 -0400 Subject: [PATCH 190/408] Use model inheretence for spider db --- .../clp-py-utils/clp_py_utils/clp_config.py | 40 ++----------------- .../clp-py-utils/clp_py_utils/sql_adapter.py | 8 +++- 2 files changed, 9 insertions(+), 39 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4225910aad..68e4e2ec2a 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -267,49 +267,15 @@ def transform_for_container(self): self.port = self.DEFAULT_PORT -class SpiderDb(BaseModel): - DEFAULT_PORT: ClassVar[int] = 3306 +class SpiderDb(Database): - type: str = "mariadb" - host: str = "localhost" - port: int = DEFAULT_PORT name: str = "spider-db" - ssl_cert: Optional[str] = None - auto_commit: bool = False - compress: bool = True - - username: Optional[str] = None - password: Optional[str] = None @field_validator("type") @classmethod def validate_type(cls, value): - supported_database_types = ["mysql", "mariadb"] - if value not in supported_database_types: - raise ValueError( - f"database.type must be one of the following {'|'.join(supported_database_types)}" - ) - return value - - @field_validator("name") - @classmethod - def validate_name(cls, value): - if "" == value: - raise ValueError("database.name cannot be empty.") - return value - - @field_validator("host") - @classmethod - def validate_host(cls, value): - if "" == value: - raise ValueError("database.host cannot be empty.") - return value - - @field_validator("port") - @classmethod - def validate_port(cls, value): - _validate_port(cls, value) - return value + if value != "mariadb": + raise ValueError(f"Spider only support MariaDB storage.") class SpiderScheduler(BaseModel): diff --git a/components/clp-py-utils/clp_py_utils/sql_adapter.py b/components/clp-py-utils/clp_py_utils/sql_adapter.py index 3dfd4d8e52..6f767c8622 100644 --- a/components/clp-py-utils/clp_py_utils/sql_adapter.py +++ b/components/clp-py-utils/clp_py_utils/sql_adapter.py @@ -1,6 +1,7 @@ import contextlib import logging import time +from typing import Optional import mariadb import mysql.connector @@ -8,7 +9,7 @@ from mysql.connector import errorcode from sqlalchemy.dialects.mysql import mariadbconnector, mysqlconnector -from clp_py_utils.clp_config import Database +from clp_py_utils.clp_config import Database, SpiderDb class DummyCloseableObject: @@ -58,8 +59,11 @@ def alive(self): class SQL_Adapter: - def __init__(self, database_config: Database): + def __init__( + self, database_config: Database, spider_database_config: Optional[SpiderDb] = None + ): self.database_config = database_config + self._spider_database_config = spider_database_config def create_mysql_connection( self, disable_localhost_socket_connection: bool = False From ffd8d0e0c6232415d5ca4468f9daf42d7615b43b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 8 Oct 2025 10:02:51 -0400 Subject: [PATCH 191/408] Add spider db and scheduler in clp config --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 68e4e2ec2a..117f016e8c 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -944,6 +944,8 @@ class CLPConfig(BaseModel): reducer: Reducer = Reducer() results_cache: ResultsCache = ResultsCache() compression_scheduler: CompressionScheduler = CompressionScheduler() + spider_db : Optional[SpiderDb] = None + spider_scheduler: Optional[SpiderScheduler] = None query_scheduler: QueryScheduler = QueryScheduler() compression_worker: CompressionWorker = CompressionWorker() query_worker: QueryWorker = QueryWorker() From f9d72905e6889cb786c7ed3dd859f4b18a362a0f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 8 Oct 2025 10:44:30 -0400 Subject: [PATCH 192/408] Make setting optional and validate relevant scheduler backend exists --- .../clp-py-utils/clp_py_utils/clp_config.py | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 117f016e8c..24cf208d08 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -939,12 +939,12 @@ class CLPConfig(BaseModel): package: Package = Package() database: Database = Database() - queue: Queue = Queue() - redis: Redis = Redis() + queue: Optional[Queue] = None + redis: Optional[Redis] = None reducer: Reducer = Reducer() results_cache: ResultsCache = ResultsCache() compression_scheduler: CompressionScheduler = CompressionScheduler() - spider_db : Optional[SpiderDb] = None + spider_db: Optional[SpiderDb] = None spider_scheduler: Optional[SpiderScheduler] = None query_scheduler: QueryScheduler = QueryScheduler() compression_worker: CompressionWorker = CompressionWorker() @@ -1128,6 +1128,30 @@ def validate_presto_config(self): ) return self + @model_validator(mode="after") + def validate_spider_config(self): + orchestration_type = self.compression_scheduler.type + if orchestration_type != OrchestrationType.spider: + return self + if self.spider_db is None: + raise ValueError("SpiderDb config must be set when using spider orchestration.") + if self.spider_scheduler is None: + raise ValueError("SpiderScheduler config must be set when using spider orchestration.") + if self.database.type != "mariadb": + raise ValueError("Database type must be 'mariadb' when using spider orchestration.") + return self + + @model_validator(mode="after") + def validate_celery_config(self): + orchestration_type = self.compression_scheduler.type + if orchestration_type != OrchestrationType.celery: + return self + if self.queue is None: + raise ValueError("Queue config must be set when using celery orchestration.") + if self.redis is None: + raise ValueError("Redis config must be set when using celery orchestration.") + return self + def transform_for_container(self): """ Adjusts paths and service hosts for containerized execution. From 172772bf9cce226e66f242c079aaf4b1eed127e7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 8 Oct 2025 10:50:35 -0400 Subject: [PATCH 193/408] Add root connection in sql adapter --- components/clp-py-utils/clp_py_utils/sql_adapter.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/sql_adapter.py b/components/clp-py-utils/clp_py_utils/sql_adapter.py index 6f767c8622..341737b1e1 100644 --- a/components/clp-py-utils/clp_py_utils/sql_adapter.py +++ b/components/clp-py-utils/clp_py_utils/sql_adapter.py @@ -108,6 +108,16 @@ def create_connection(self, disable_localhost_socket_connection: bool = False): else: raise NotImplementedError + def create_root_mariadb_connection(self, disable_localhost_socket_connection: bool = False): + if "mariadb" != self.database_config.type: + raise NotImplementedError + params = self.database_config.get_mysql_connection_params( + disable_localhost_socket_connection + ) + params["user"] = "root" + params.pop("database", None) + return mariadb.connect(**params) + def create_connection_pool( self, logger: logging.Logger, From 6546baff4074d42d9687c51153936dcc812821c0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 8 Oct 2025 11:26:38 -0400 Subject: [PATCH 194/408] Add spider database setup --- .../clp_py_utils/initialize-spider-db.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 components/clp-py-utils/clp_py_utils/initialize-spider-db.py diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py new file mode 100644 index 0000000000..9577b96da7 --- /dev/null +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +import argparse +import logging +import pathlib +import re +import sys +from contextlib import closing + +from pydantic import ValidationError +from sql_adapter import SQL_Adapter + +from clp_py_utils.clp_config import CLPConfig +from clp_py_utils.core import read_yaml_config_file + +# Setup logging +# Create logger +logger = logging.getLogger(__file__) +logger.setLevel(logging.INFO) +# Setup console logging +logging_console_handler = logging.StreamHandler() +logging_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") +logging_console_handler.setFormatter(logging_formatter) +logger.addHandler(logging_console_handler) + + +def main(argv): + args_parser = argparse.ArgumentParser(description="Sets up Spider database.") + args_parser.add_argument("--config", "-c", required=True, help="CLP configuration file.") + parsed_args = args_parser.parse_args(argv[1:]) + + config_path = pathlib.Path(parsed_args.config) + try: + clp_config = CLPConfig.model_validate(read_yaml_config_file(config_path)) + clp_config.database.load_credentials_from_env() + clp_config.spider_db.load_credentials_from_env() + except (ValidationError, ValueError) as err: + logger.error(err) + return -1 + except: + logger.exception("Failed to load CLP configuration.") + return -1 + + spider_db_config = clp_config.spider_db + if not spider_db_config: + logger.error("Spider database configuration not found in CLP configuration.") + return -1 + + try: + sql_adapter = SQL_Adapter(clp_config.database) + with closing(sql_adapter.create_root_mariadb_connection()) as db_conn, closing( + db_conn.cursor() + ) as db_cursor: + db_name = spider_db_config.name + db_user = spider_db_config.username + db_password = spider_db_config.password + clp_user = clp_config.database.username + if not _validate_name(db_name): + logger.error(f"Invalid database name: {db_name}") + return -1 + if not _validate_name(db_user): + logger.error(f"Invalid database user name: {db_user}") + return -1 + if not _validate_name(clp_user): + logger.error(f"Invalid CLP database user name: {clp_user}") + return -1 + if not _validate_name(db_password): + logger.error(f"Invalid database user password") + return -1 + + db_cursor.execute(f"""CREATE DATABASE IF NOT EXISTS `{db_name}`""") + if db_password is not None: + db_cursor.execute( + f"""CREATE USER IF NOT EXISTS '{db_user}'@'%' IDENTIFIED BY '{db_password}'""" + ) + else: + db_cursor.execute(f"""CREATE USER IF NOT EXISTS '{db_user}'@'%' IDENTIFIED BY ''""") + db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_user}'@'%'""") + db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{clp_user}'@'%'""") + + except: + logger.exception("Failed to setup Spider database.") + return -1 + + return 0 + + +_name_pattern = re.compile(r"^[A-Za-z0-9_-]+$") + + +def _validate_name(name: str) -> bool: + """ + Validates that the input string contains only alphanumeric characters, underscores, or hyphens. + :param name: The input string to validate. + :return: If the input string is valid. + """ + return _name_pattern.match(name) is not None + + +if "__main__" == __name__: + sys.exit(main(sys.argv)) From 256f00b3045c840e35848095a6880b081ee4ad92 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 8 Oct 2025 11:28:10 -0400 Subject: [PATCH 195/408] Add spider init script call to create db tables script --- components/clp-py-utils/clp_py_utils/create-db-tables.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/create-db-tables.py b/components/clp-py-utils/clp_py_utils/create-db-tables.py index c47ec53d2a..03c268b441 100644 --- a/components/clp-py-utils/clp_py_utils/create-db-tables.py +++ b/components/clp-py-utils/clp_py_utils/create-db-tables.py @@ -51,6 +51,14 @@ def main(argv): # fmt: on subprocess.run(cmd, check=True) + # fmt: off + cmd = [ + "python3", str(script_dir / "initialize-spider-db.py"), + "--config", str(config_file_path), + ] + # fmt: on + subprocess.run(cmd, check=True) + return 0 From 0da12eb0aa5e43954a6edf8338c4a72dc1cd80ee Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 8 Oct 2025 22:23:27 -0400 Subject: [PATCH 196/408] Add spider scheduler service; Fix config; Add controller setup for spider scheduler --- .../clp_package_utils/controller.py | 52 +++- .../clp-py-utils/clp_py_utils/clp_config.py | 6 + .../package/docker-compose.spider.yaml | 255 ++++++++++++++++++ 3 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 tools/deployment/package/docker-compose.spider.yaml diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index b0348eac85..e5cf9521f0 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -28,6 +28,7 @@ REDIS_COMPONENT_NAME, REDUCER_COMPONENT_NAME, RESULTS_CACHE_COMPONENT_NAME, + SPIDER_SCHEDULER_COMPONENT_NAME, StorageEngine, StorageType, WEBUI_COMPONENT_NAME, @@ -130,9 +131,13 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: """ Prepares environment variables and directories for the message queue component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of component-related environment variables. Empty if queue is not + configured. """ component_name = QUEUE_COMPONENT_NAME + if self.clp_config.queue is None: + logger.info(f"{component_name} is not configured, skipping setup.") + return {} logger.info(f"Setting up environment for {component_name}...") logs_dir = self.clp_config.logs_directory / component_name @@ -152,9 +157,13 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: """ Prepares environment variables and directories for the Redis component. - :return: Dictionary of component-related environment variables. + :return: Dictionary of component-related environment variables. Empty if Redis is not + configured. """ component_name = REDIS_COMPONENT_NAME + if self.clp_config.redis is None: + logger.info(f"{component_name} is not configured, skipping setup.") + return {} logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" @@ -178,6 +187,43 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: ), } + def _set_up_env_for_spider_db(self) -> EnvVarsDict: + """ + Prepares environment variables and directories for the spider database component. + + :return: Dictionary of component-related environment variables. Empty if spider DB is not + configured. + """ + component_name = "spider_db" + if self.clp_config.spider_db is None: + logger.info(f"{component_name} is not configured, skipping setup.") + return {} + logger.info(f"Setting up environment for {component_name}...") + + return {"SPIDER_DB_URL": self.clp_config.spider_db.get_url()} + + def _set_up_env_for_spider_scheduler(self) -> EnvVarsDict: + """ + Prepares environment variables and files for the spider scheduler component. + + :return: Dictionary of component-related environment variables. Empty if spider scheduler + is not configured. + """ + component_name = SPIDER_SCHEDULER_COMPONENT_NAME + if self.clp_config.spider_scheduler is None: + logger.info(f"{component_name} is not configured, skipping setup.") + return {} + logger.info(f"Setting up environment for {component_name}...") + + logs_file = self.clp_config.logs_directory / f"{component_name}.log" + logs_file.touch(mode=LOGS_FILE_MODE, exist_ok=True) + + return { + "SPIDER_SCHEDULER_LOGS_FILE_HOST": str(logs_file), + "SPIDER_SCHEDULER_HOST": _get_ip_from_hostname(self.clp_config.spider_scheduler.host), + "SPIDER_SCHEDULER_PORT": str(self.clp_config.scheduler.port), + } + def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ Prepares environment variables and directories for the results cache (MongoDB) component. @@ -552,6 +598,8 @@ def _provision(self): **self._set_up_env_for_database(), **self._set_up_env_for_queue(), **self._set_up_env_for_redis(), + **self._set_up_env_for_spider_db(), + **self._set_up_env_for_spider_scheduler(), **self._set_up_env_for_results_cache(), **self._set_up_env_for_compression_scheduler(), **self._set_up_env_for_query_scheduler(), diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 24cf208d08..ded3411eb6 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -26,6 +26,7 @@ DB_COMPONENT_NAME = "database" QUEUE_COMPONENT_NAME = "queue" REDIS_COMPONENT_NAME = "redis" +SPIDER_SCHEDULER_COMPONENT_NAME = "spider_scheduler" REDUCER_COMPONENT_NAME = "reducer" RESULTS_CACHE_COMPONENT_NAME = "results_cache" COMPRESSION_SCHEDULER_COMPONENT_NAME = "compression_scheduler" @@ -277,10 +278,15 @@ def validate_type(cls, value): if value != "mariadb": raise ValueError(f"Spider only support MariaDB storage.") + def get_url(self): + self.ensure_credentials_loaded() + return f"mariadb://{self.host}:{self.port}/{self.name}?user={self.username}&password={self.password}" + class SpiderScheduler(BaseModel): DEFAULT_PORT: ClassVar[int] = 6000 + host: str = "localhost" port: int = DEFAULT_PORT @field_validator("port") diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml new file mode 100644 index 0000000000..f0fdc24d83 --- /dev/null +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -0,0 +1,255 @@ +name: "clp-package-base" + +# Common service defaults. +x-service-defaults: &service_defaults + stop_grace_period: "3s" + user: "${CLP_UID_GID:-1000:1000}" + +# Common healthcheck defaults. +x-healthcheck-defaults: &healthcheck_defaults + interval: "30s" + retries: 3 + start_interval: "1s" + start_period: "10s" + timeout: "10s" + +secrets: + CLP_DB_PASS_FILE: + environment: "CLP_DB_PASS" + +services: + database: + <<: *service_defaults + container_name: "database" + image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + environment: + MYSQL_DATABASE: "${CLP_DB_NAME}" + MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_USER: "${CLP_DB_USER}" + secrets: + - "CLP_DB_PASS_FILE" + ports: + - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + volumes: + - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" + - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1", + "-u", "${CLP_DB_USER}", + "--password=${CLP_DB_PASS}" + ] + + db-table-creator: + <<: *service_defaults + container_name: "db_table_creator" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + depends_on: + database: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" + ] + + spider-scheduler: + <<: *service_defaults + container_name: "spider_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" + depends_on: + database: + condition: "service_healthy" + command: [ + "/opt/clp/bin/spider_scheduler", + "--host", "${SPIDER_SCHEDULER_HOST}", + "--port", "${SPIDER_SCHEDULER_PORT}", + "--storage_url", "${SPIDER_DB_URL}" + ] + + results-cache: + <<: *service_defaults + container_name: "results_cache" + image: "mongo:7.0.1" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + ports: + - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + volumes: + - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" + - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" + healthcheck: + <<: *healthcheck_defaults + test: >- + echo 'db.runCommand("ping").ok' | + mongosh 127.0.0.1:27017/test --quiet + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] + + results-cache-indices-creator: + <<: *service_defaults + container_name: "results_cache_indices_creator" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + depends_on: + results-cache: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", + "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", + ] + + compression-scheduler: + <<: *service_defaults + container_name: "compression_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + volumes: + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ +:/var/log/compression_scheduler.log" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "/:/mnt/logs:ro" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "--config", "/etc/clp-config.yml" + ] + + compression-worker: + <<: *service_defaults + container_name: "compression_worker" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_CONFIG_PATH: "/etc/clp-config.yml" + CLP_HOME: "/opt/clp" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/compression_worker" + CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + volumes: + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ +/var/log/compression_worker" + - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "/:/mnt/logs:ro" + command: [ + "python3", + "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.compress", + "worker", + "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", + "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", + "-f", "/var/log/compression_worker/worker.log", + "-Q", "compression", + "-n", "compression-worker" + ] + + webui: + <<: *service_defaults + container_name: "webui" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + HOST: "0.0.0.0" + NODE_ENV: "production" + NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" + PORT: "4000" + RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" + ports: + - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + volumes: + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" + - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" + - "./var/www/webui/server/dist/settings.json\ +:/opt/clp/var/www/webui/server/dist/settings.json:ro" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/src/main.js" + ] + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "bash", + "-c", + "< /dev/tcp/webui/4000" + ] + + garbage-collector: + <<: *service_defaults + container_name: "garbage_collector" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_HOME: "/opt/clp" + CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/garbage_collector" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yml", + ] From 8b97bc2f857402bf2ea12ae0b39ac91e2fc2e6a3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 9 Oct 2025 01:53:31 -0400 Subject: [PATCH 197/408] refactor(deployment): Remove outdated comment in reducer healthcheck configuration. --- tools/deployment/package/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index acd54ed415..979f9c314c 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -64,7 +64,6 @@ services: ] healthcheck: <<: *healthcheck_defaults - # FIXME: need to suppressing warnings in the scheduler for reading 0 out of 8 expected bytes test: [ "CMD", "bash", From cb821b55f4c5e7033c8a74daf0603ea533da2fbc Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 08:18:46 -0400 Subject: [PATCH 198/408] Make compression scheduler depends on spider scheduler --- .../clp-package-utils/clp_package_utils/controller.py | 6 +++++- tools/deployment/package/docker-compose.spider.yaml | 6 ++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e5cf9521f0..f26fe6bb3d 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -200,7 +200,11 @@ def _set_up_env_for_spider_db(self) -> EnvVarsDict: return {} logger.info(f"Setting up environment for {component_name}...") - return {"SPIDER_DB_URL": self.clp_config.spider_db.get_url()} + return { + "SPIDER_DB_USER": self.clp_config.spider_db.username, + "SPIDER_DB_PASS": self.clp_config.spider_db.password, + "SPIDER_DB_URL": self.clp_config.spider_db.get_url(), + } def _set_up_env_for_spider_scheduler(self) -> EnvVarsDict: """ diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index f0fdc24d83..c0bfeb6f2e 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -142,10 +142,8 @@ services: depends_on: db-table-creator: condition: "service_completed_successfully" - queue: - condition: "service_healthy" - redis: - condition: "service_healthy" + spider-scheduler: + condition: "service_started" command: [ "python3", "-u", From 41b90d74936de848b6bfc0f531f7bd82166586a6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 10:51:47 -0400 Subject: [PATCH 199/408] Add script to start multiple spider worker; Add docker service to start spider worker --- .../clp_py_utils/start-spider-worker.py | 41 +++++++++++++++++++ .../package/docker-compose.spider.yaml | 14 ++----- 2 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 components/clp-py-utils/clp_py_utils/start-spider-worker.py diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py new file mode 100644 index 0000000000..3eb892dba8 --- /dev/null +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -0,0 +1,41 @@ +import argparse +import os +import pathlib +import subprocess + + +def parse_args() -> argparse.Namespace: + """ + Parses command line arguments. + :return: The parsed arguments. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--concurrency", type=int, default=1, help="Number of concurrent spider workers." + ) + parser.add_argument("--storage-url", type=str, required=True, help="Spider storage URL.") + parser.add_argument("--host", type=str, required=True, help="Worker host address.") + return parser.parse_args() + + +def main() -> None: + """Main function to start multiple spider workers.""" + # Parse arguments + args = parse_args() + concurrency = args.concurrency + storage_url = args.storage_url + host = args.host + + clp_home = os.getenv("CLP_HOME", "/opt/clp") + spider_worker_path = pathlib.Path(clp_home) / "bin" / "spider_worker" + + # Start multiple spider workers + processes = [] + for _ in range(concurrency): + process = subprocess.Popen( + [spider_worker_path, "--storage_url", storage_url, "--host", host] + ) + processes.append(process) + + for process in processes: + process.wait() diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index c0bfeb6f2e..daf5942ec7 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -158,15 +158,12 @@ services: environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/compression_worker" CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} volumes: - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" @@ -179,14 +176,11 @@ services: command: [ "python3", "-u", - "/opt/clp/lib/python3/site-packages/bin/celery", - "-A", "job_orchestration.executor.compress", - "worker", + "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--loglevel", "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}", - "-f", "/var/log/compression_worker/worker.log", - "-Q", "compression", - "-n", "compression-worker" + "--storage-url", "${SPIDER_DB_URL}", + # NOTE: Leave host to spider scheduler's host. This only affects task placement. + "--host", "${SPIDER_SCHEDULER_HOST}", ] webui: From 4e1378574a42e12adb3df09eb00932f22dfa2f7d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 10:58:41 -0400 Subject: [PATCH 200/408] Fix credential lading in start-clp --- .../clp-package-utils/clp_package_utils/general.py | 7 +++++++ .../clp_package_utils/scripts/start_clp.py | 11 ++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 9fa7f946b1..3a808578de 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -493,6 +493,13 @@ def validate_and_load_redis_credentials_file( clp_config.redis.load_credentials_from_file(clp_config.credentials_file_path) +def validate_and_load_spider_db_credentials_file( + clp_config: CLPConfig, clp_home: pathlib.Path, generate_default_file: bool +): + validate_credentials_file_path(clp_config, clp_home, generate_default_file) + clp_config.spider_db.load_credentials_from_file(clp_config.credentials_file_path) + + def validate_db_config( clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path ): diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 78fc6739a8..2db5cde5ba 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -3,7 +3,7 @@ import pathlib import sys -from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH +from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, OrchestrationType from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( @@ -12,6 +12,7 @@ validate_and_load_db_credentials_file, validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, + validate_and_load_spider_db_credentials_file, validate_logs_input_config, validate_output_storage_config, validate_retention_config, @@ -40,8 +41,12 @@ def main(argv): clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) validate_and_load_db_credentials_file(clp_config, clp_home, True) - validate_and_load_queue_credentials_file(clp_config, clp_home, True) - validate_and_load_redis_credentials_file(clp_config, clp_home, True) + if clp_config.queue is not None: + validate_and_load_queue_credentials_file(clp_config, clp_home, True) + if clp_config.redis is not None: + validate_and_load_redis_credentials_file(clp_config, clp_home, True) + if clp_config.spider_db is not None: + validate_and_load_spider_db_credentials_file(clp_config, clp_home, True) validate_logs_input_config(clp_config) validate_output_storage_config(clp_config) validate_retention_config(clp_config) From f9dcf77eef9bcba132da2ffbc4da3b78851b01f6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 11:05:00 -0400 Subject: [PATCH 201/408] Add compression scheduler check for which task manager to use --- .../scheduler/compress/compression_scheduler.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 655f90ecac..180143c19e 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -17,7 +17,7 @@ COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_TASKS_TABLE_NAME, - StorageEngine, + StorageEngine, OrchestrationType, ) from clp_py_utils.clp_logging import get_logger, get_logging_formatter, set_logging_level from clp_py_utils.clp_metadata_db_utils import ( @@ -31,6 +31,7 @@ from clp_py_utils.sql_adapter import SQL_Adapter from job_orchestration.scheduler.compress.partition import PathsToCompressBuffer from job_orchestration.scheduler.compress.task_manager.celery_task_manager import CeleryTaskManager +from job_orchestration.scheduler.compress.task_manager.spider_task_manager import SpiderTaskManager from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.constants import ( CompressionJobStatus, @@ -486,7 +487,16 @@ def main(argv): logger.info(f"Starting {COMPRESSION_SCHEDULER_COMPONENT_NAME}") sql_adapter = SQL_Adapter(clp_config.database) - task_manager = CeleryTaskManager() + if clp_config.compression_scheduler.type == OrchestrationType.celery: + task_manager = CeleryTaskManager() + elif clp_config.compression_scheduler.type == OrchestrationType.spider: + task_manager = SpiderTaskManager() + else: + logger.error( + f"Unsupported compression scheduler type:" + f" {clp_config.compression_scheduler.type}" + ) + return -1 try: killed_jobs = kill_hanging_jobs(sql_adapter, SchedulerType.COMPRESSION) From 4ff8e05b8126764cbdf769643078ed243a88239a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 11:44:25 -0400 Subject: [PATCH 202/408] Fix lint --- .../scheduler/compress/compression_scheduler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 180143c19e..33a375eb5c 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -17,7 +17,8 @@ COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_TASKS_TABLE_NAME, - StorageEngine, OrchestrationType, + OrchestrationType, + StorageEngine, ) from clp_py_utils.clp_logging import get_logger, get_logging_formatter, set_logging_level from clp_py_utils.clp_metadata_db_utils import ( @@ -493,8 +494,7 @@ def main(argv): task_manager = SpiderTaskManager() else: logger.error( - f"Unsupported compression scheduler type:" - f" {clp_config.compression_scheduler.type}" + f"Unsupported compression scheduler type:" f" {clp_config.compression_scheduler.type}" ) return -1 From 28abb0f466531e7d878a9e437b51050a289a2593 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 14:17:06 -0400 Subject: [PATCH 203/408] Skip spider table creation if spider_db not configured --- components/clp-py-utils/clp_py_utils/initialize-spider-db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 9577b96da7..4ec5e383b5 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -32,6 +32,8 @@ def main(argv): try: clp_config = CLPConfig.model_validate(read_yaml_config_file(config_path)) clp_config.database.load_credentials_from_env() + if clp_config.spider_db is None: + return 0 clp_config.spider_db.load_credentials_from_env() except (ValidationError, ValueError) as err: logger.error(err) From ccb8f66a993f2f1f65f5b38dfea5ed96e09d1f85 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 9 Oct 2025 14:48:12 -0400 Subject: [PATCH 204/408] refactor(deployment): Remove duplicated docstring from docker-compose.yaml; comment that the other file should be referred --- tools/deployment/package/docker-compose.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 979f9c314c..44ecae5e89 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -2,23 +2,20 @@ name: "clp-package" include: ["docker-compose.base.yaml"] -# Common service defaults. +# Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for +# documentation. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER:-clp-package}" logging: driver: "local" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" - -# Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 start_interval: "2s" start_period: "60s" timeout: "2s" - -# Common volume definitions. x-volume-definitions: aws-config-readonly: &volume_aws_config_readonly type: "bind" From 40b88a07eefe5236d0d7db5ceed892d8106d3c75 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 9 Oct 2025 15:02:52 -0400 Subject: [PATCH 205/408] docs(design): Remove health check defaults section; add comments to healthcheck defaults in docker-compose.base.yaml --- docs/src/dev-docs/design-docker-compose.md | 16 ---------------- .../deployment/package/docker-compose.base.yaml | 7 +++++++ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md index eb14b23e8a..9bec8bb49b 100644 --- a/docs/src/dev-docs/design-docker-compose.md +++ b/docs/src/dev-docs/design-docker-compose.md @@ -73,22 +73,6 @@ Each file defines services with: * Network configuration * User permissions -### Health check defaults - -Below are the default health check settings and the rationale for each: - -* `interval: 30s` — default probe interval in steady state. Avoid setting this too low to avoid - excessive resource usage. -* `timeout: 2s` — no remote communication is expected, so a short timeout is sufficient. -* `retries: 3` - - a service is deemed unhealthy if it does not respond in ~(30s * 3) = 90s since it is in the - steady state. - - a service is deemed unhealthy if it does not respond within ~(60s + 90s) since it is started. -* `start_interval: 2s` — A short interval allows the service to become healthy quickly once it - is ready, allowing other services which depend on it to start. -* `start_period: 60s` — the first minute of startup ignores failures, effectively granting around 30 - fast attempts to become healthy before retries start counting. - ## Service architecture The Docker Compose setup includes the following services: diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 6c27235240..2c00c58f15 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -10,10 +10,17 @@ x-service-defaults: &service_defaults # Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults + # Avoid lowering to prevent excessive resource usage. interval: "30s" + # Mark unhealthy after 3 failed probes. + # - In steady state, ( + ) × 3 = ~90s before the service is marked unhealthy. + # - From startup, (60s) + ~90s = ~150s before the service is marked unhealthy. retries: 3 + # Frequent checks during startup allow fast transition to healthy. start_interval: "2s" + # Ignore failures for ~15 frequent checks before counting retries. start_period: "60s" + # Short timeout since no remote communication is expected. timeout: "2s" # Common volume definitions. From 71b90e5144cdda7777cf1318d9cdb3d8ce99a177 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 15:30:18 -0400 Subject: [PATCH 206/408] Use redis&queue by default in config --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 55dcbf85ca..4b67c50877 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -948,8 +948,9 @@ class CLPConfig(BaseModel): package: Package = Package() database: Database = Database() - queue: Optional[Queue] = None - redis: Optional[Redis] = None + # Default to use celery backend + queue: Optional[Queue] = Queue() + redis: Optional[Redis] = Redis() reducer: Reducer = Reducer() results_cache: ResultsCache = ResultsCache() compression_scheduler: CompressionScheduler = CompressionScheduler() From 7357a7e95c64357da5d6401357742ac1dbacc09d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 22:29:26 -0400 Subject: [PATCH 207/408] Fix enum --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4b67c50877..5f1b90888d 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -324,6 +324,8 @@ class OrchestrationType(KebabCaseStrEnum): class CompressionScheduler(BaseModel): + model_config = ConfigDict(use_enum_values=True) + jobs_poll_delay: float = 0.1 # seconds logging_level: str = "INFO" type: OrchestrationType = OrchestrationType.celery From e58512a7fc42f8a225dce7da945870ea3ab085c8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 9 Oct 2025 22:38:57 -0400 Subject: [PATCH 208/408] docs(quick-start): Add required containerd, Docker CE, CLI, and compose plugin versions to requirements list. --- docs/src/user-docs/quick-start/index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/user-docs/quick-start/index.md b/docs/src/user-docs/quick-start/index.md index cfe7fea552..e46086ed26 100644 --- a/docs/src/user-docs/quick-start/index.md +++ b/docs/src/user-docs/quick-start/index.md @@ -13,6 +13,10 @@ This guide describes the following: To run a CLP release, you'll need: * [Docker](#docker) + * `containerd.io` >= 1.7.18 + * `docker-ce` >= 27.0.3 + * `docker-ce-cli` >= 27.0.3 + * `docker-compose-plugin` >= 2.28.1 * [Python](#python) ### Docker From d3b34cfeacc159f3574c0074034727503906dc9e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 9 Oct 2025 22:41:34 -0400 Subject: [PATCH 209/408] Revert "Fix enum" This reverts commit 7357a7e95c64357da5d6401357742ac1dbacc09d. --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 5f1b90888d..4b67c50877 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -324,8 +324,6 @@ class OrchestrationType(KebabCaseStrEnum): class CompressionScheduler(BaseModel): - model_config = ConfigDict(use_enum_values=True) - jobs_poll_delay: float = 0.1 # seconds logging_level: str = "INFO" type: OrchestrationType = OrchestrationType.celery From cc9a1f4384ca898891deec1f8c739ed05f44b608 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 10 Oct 2025 03:18:04 -0400 Subject: [PATCH 210/408] refactor(config): Rename type `Host` -> `DomainStr`; Add TODO docstring about plan for replacement. --- .../clp-py-utils/clp_py_utils/clp_config.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index b6743d7f94..7c8cd9feb4 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -104,7 +104,8 @@ PositiveFloat = Annotated[float, Field(gt=0)] PositiveInt = Annotated[int, Field(gt=0)] # Specific types -Host = NonEmptyStr +# TODO: Replace this with pydantic_extra_types.domain.DomainStr. +DomainStr = NonEmptyStr Port = Annotated[int, Field(gt=0, lt=2**16)] ZstdCompressionLevel = Annotated[int, Field(ge=1, le=19)] @@ -172,7 +173,7 @@ def dump_to_primitive_dict(self): class Database(BaseModel): type: DatabaseEngine = DatabaseEngine.MARIADB - host: Host = "localhost" + host: DomainStr = "localhost" port: Port = 3306 name: NonEmptyStr = "clp-db" ssl_cert: Optional[NonEmptyStr] = None @@ -261,7 +262,7 @@ class CompressionScheduler(BaseModel): class QueryScheduler(BaseModel): - host: Host = "localhost" + host: DomainStr = "localhost" port: Port = 7000 jobs_poll_delay: PositiveFloat = 0.1 # seconds num_archives_to_search_per_sub_job: PositiveInt = 16 @@ -277,7 +278,7 @@ class QueryWorker(BaseModel): class Redis(BaseModel): - host: Host = "localhost" + host: DomainStr = "localhost" port: Port = 6379 query_backend_database: int = 0 compression_backend_database: int = 1 @@ -306,14 +307,14 @@ def load_credentials_from_env(self): class Reducer(BaseModel): - host: Host = "localhost" + host: DomainStr = "localhost" base_port: Port = 14009 logging_level: LoggingLevel = "INFO" upsert_interval: PositiveInt = 100 # milliseconds class ResultsCache(BaseModel): - host: Host = "localhost" + host: DomainStr = "localhost" port: Port = 27017 db_name: NonEmptyStr = "clp-query-results" stream_collection_name: NonEmptyStr = "stream-files" @@ -324,7 +325,7 @@ def get_uri(self): class Queue(BaseModel): - host: Host = "localhost" + host: DomainStr = "localhost" port: Port = 5672 username: Optional[NonEmptyStr] = None @@ -543,7 +544,7 @@ def dump_to_primitive_dict(self): class WebUi(BaseModel): - host: Host = "localhost" + host: DomainStr = "localhost" port: Port = 4000 results_metadata_collection_name: NonEmptyStr = "results-metadata" rate_limit: PositiveInt = 1000 @@ -562,7 +563,7 @@ class GarbageCollector(BaseModel): class Presto(BaseModel): - host: Host + host: DomainStr port: Port From 04929ea358620e0143a3747e9ac6a0d474096250 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 11:31:26 -0400 Subject: [PATCH 211/408] Add custom annotation for serialization --- .../clp-py-utils/clp_py_utils/clp_config.py | 95 ++++++------------- .../pydantic_serialization_utils.py | 8 ++ 2 files changed, 37 insertions(+), 66 deletions(-) create mode 100644 components/clp-py-utils/clp_py_utils/pydantic_serialization_utils.py diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4d408dcc75..020c28ca30 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -21,6 +21,7 @@ read_yaml_config_file, validate_path_could_be_dir, ) +from .pydantic_serialization_utils import PathStr, StrEnumSerializer # Constants # Component names @@ -114,17 +115,26 @@ class StorageEngine(KebabCaseStrEnum): CLP_S = auto() +StorageEngineStr = Annotated[StorageEngine, StrEnumSerializer] + + class DatabaseEngine(KebabCaseStrEnum): MARIADB = auto() MYSQL = auto() +DatabaseEngineStr = Annotated[DatabaseEngine, StrEnumSerializer] + + class QueryEngine(KebabCaseStrEnum): CLP = auto() CLP_S = auto() PRESTO = auto() +QueryEngineStr = Annotated[QueryEngine, StrEnumSerializer] + + class StorageType(LowercaseStrEnum): FS = auto() S3 = auto() @@ -137,9 +147,12 @@ class AwsAuthType(LowercaseStrEnum): ec2 = auto() +AwsAuthTypeStr = Annotated[AwsAuthType, StrEnumSerializer] + + class Package(BaseModel): - storage_engine: StorageEngine = StorageEngine.CLP - query_engine: QueryEngine = QueryEngine.CLP + storage_engine: StorageEngineStr = StorageEngine.CLP + query_engine: QueryEngineStr = QueryEngine.CLP @model_validator(mode="after") def validate_query_engine_package_compatibility(self): @@ -163,15 +176,9 @@ def validate_query_engine_package_compatibility(self): return self - def dump_to_primitive_dict(self): - d = self.model_dump() - d["storage_engine"] = d["storage_engine"].value - d["query_engine"] = d["query_engine"].value - return d - class Database(BaseModel): - type: DatabaseEngine = DatabaseEngine.MARIADB + type: DatabaseEngineStr = DatabaseEngine.MARIADB host: DomainStr = "localhost" port: Port = 3306 name: NonEmptyStr = "clp-db" @@ -232,7 +239,6 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection def dump_to_primitive_dict(self): d = self.model_dump(exclude={"username", "password"}) - d["type"] = d["type"].value return d def load_credentials_from_file(self, credentials_file_path: pathlib.Path): @@ -360,12 +366,7 @@ class S3Credentials(BaseModel): class AwsAuthentication(BaseModel): - type: Literal[ - AwsAuthType.credentials.value, - AwsAuthType.profile.value, - AwsAuthType.env_vars.value, - AwsAuthType.ec2.value, - ] + type: AwsAuthTypeStr profile: Optional[NonEmptyStr] = None credentials: Optional[S3Credentials] = None @@ -408,13 +409,10 @@ class S3IngestionConfig(BaseModel): type: Literal[StorageType.S3.value] = StorageType.S3.value aws_authentication: AwsAuthentication - def dump_to_primitive_dict(self): - return self.model_dump() - class FsStorage(BaseModel): type: Literal[StorageType.FS.value] = StorageType.FS.value - directory: pathlib.Path + directory: PathStr @field_validator("directory", mode="before") @classmethod @@ -425,16 +423,11 @@ def validate_directory(cls, value): def make_config_paths_absolute(self, clp_home: pathlib.Path): self.directory = make_config_path_absolute(clp_home, self.directory) - def dump_to_primitive_dict(self): - d = self.model_dump() - d["directory"] = str(d["directory"]) - return d - class S3Storage(BaseModel): type: Literal[StorageType.S3.value] = StorageType.S3.value s3_config: S3Config - staging_directory: pathlib.Path + staging_directory: PathStr @field_validator("staging_directory", mode="before") @classmethod @@ -455,30 +448,25 @@ def validate_key_prefix(cls, value): def make_config_paths_absolute(self, clp_home: pathlib.Path): self.staging_directory = make_config_path_absolute(clp_home, self.staging_directory) - def dump_to_primitive_dict(self): - d = self.model_dump() - d["staging_directory"] = str(d["staging_directory"]) - return d - class FsIngestionConfig(FsStorage): - directory: pathlib.Path = pathlib.Path("/") + directory: PathStr = pathlib.Path("/") class ArchiveFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" + directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" class StreamFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" + directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" + staging_directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" + staging_directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" def _get_directory_from_storage_config( @@ -520,11 +508,6 @@ def set_directory(self, directory: pathlib.Path): def get_directory(self) -> pathlib.Path: return _get_directory_from_storage_config(self.storage) - def dump_to_primitive_dict(self): - d = self.model_dump() - d["storage"] = self.storage.dump_to_primitive_dict() - return d - class StreamOutput(BaseModel): storage: Union[StreamFsStorage, StreamS3Storage] = StreamFsStorage() @@ -536,11 +519,6 @@ def set_directory(self, directory: pathlib.Path): def get_directory(self) -> pathlib.Path: return _get_directory_from_storage_config(self.storage) - def dump_to_primitive_dict(self): - d = self.model_dump() - d["storage"] = self.storage.dump_to_primitive_dict() - return d - class WebUi(BaseModel): host: DomainStr = "localhost" @@ -590,20 +568,18 @@ class CLPConfig(BaseModel): query_worker: QueryWorker = QueryWorker() webui: WebUi = WebUi() garbage_collector: GarbageCollector = GarbageCollector() - credentials_file_path: pathlib.Path = CLP_DEFAULT_CREDENTIALS_FILE_PATH + credentials_file_path: PathStr = CLP_DEFAULT_CREDENTIALS_FILE_PATH presto: Optional[Presto] = None archive_output: ArchiveOutput = ArchiveOutput() stream_output: StreamOutput = StreamOutput() - data_directory: pathlib.Path = pathlib.Path("var") / "data" - logs_directory: pathlib.Path = pathlib.Path("var") / "log" + data_directory: PathStr = pathlib.Path("var") / "data" + logs_directory: PathStr = pathlib.Path("var") / "log" aws_config_directory: Optional[pathlib.Path] = None - _container_image_id_path: pathlib.Path = PrivateAttr( - default=CLP_PACKAGE_CONTAINER_IMAGE_ID_PATH - ) - _version_file_path: pathlib.Path = PrivateAttr(default=CLP_VERSION_FILE_PATH) + _container_image_id_path: PathStr = PrivateAttr(default=CLP_PACKAGE_CONTAINER_IMAGE_ID_PATH) + _version_file_path: PathStr = PrivateAttr(default=CLP_VERSION_FILE_PATH) @field_validator("aws_config_directory") @classmethod @@ -748,9 +724,6 @@ def dump_to_primitive_dict(self): d[key] = getattr(self, key).dump_to_primitive_dict() # Turn paths into primitive strings - d["credentials_file_path"] = str(self.credentials_file_path) - d["data_directory"] = str(self.data_directory) - d["logs_directory"] = str(self.logs_directory) if self.aws_config_directory is not None: d["aws_config_directory"] = str(self.aws_config_directory) else: @@ -778,16 +751,6 @@ class WorkerConfig(BaseModel): stream_output: StreamOutput = StreamOutput() stream_collection_name: str = ResultsCache().stream_collection_name - def dump_to_primitive_dict(self): - d = self.model_dump() - d["archive_output"] = self.archive_output.dump_to_primitive_dict() - - # Turn paths into primitive strings - d["data_directory"] = str(self.data_directory) - d["stream_output"] = self.stream_output.dump_to_primitive_dict() - - return d - def get_components_for_target(target: str) -> Set[str]: if target in TARGET_TO_COMPONENTS: diff --git a/components/clp-py-utils/clp_py_utils/pydantic_serialization_utils.py b/components/clp-py-utils/clp_py_utils/pydantic_serialization_utils.py new file mode 100644 index 0000000000..83141724d5 --- /dev/null +++ b/components/clp-py-utils/clp_py_utils/pydantic_serialization_utils.py @@ -0,0 +1,8 @@ +import pathlib +from typing import Annotated + +from pydantic import PlainSerializer + +StrEnumSerializer = PlainSerializer(lambda enum_value: enum_value.value) + +PathStr = Annotated[pathlib.Path, PlainSerializer(lambda path_value: str(path_value))] From 57b2789691d4ec3c3dc74bc5513843819381da40 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 11:44:14 -0400 Subject: [PATCH 212/408] Remove field without custom serialization --- components/clp-py-utils/clp_py_utils/clp_config.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 020c28ca30..dc19ccd140 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -517,7 +517,7 @@ def set_directory(self, directory: pathlib.Path): _set_directory_for_storage_config(self.storage, directory) def get_directory(self) -> pathlib.Path: - return _get_directory_from_storage_config(self.storage) + return _get_directory_from_storage_config(self.pathlib.Path) class WebUi(BaseModel): @@ -711,13 +711,9 @@ def get_runnable_components(self) -> Set[str]: def dump_to_primitive_dict(self): custom_serialized_fields = { - "package", "database", "queue", "redis", - "logs_input", - "archive_output", - "stream_output", } d = self.model_dump(exclude=custom_serialized_fields) for key in custom_serialized_fields: From 0b6f43c226098455ce88f261a757ab374bfb48fd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 11:59:30 -0400 Subject: [PATCH 213/408] Bug fix --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index dc19ccd140..e80940148d 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -517,7 +517,7 @@ def set_directory(self, directory: pathlib.Path): _set_directory_for_storage_config(self.storage, directory) def get_directory(self) -> pathlib.Path: - return _get_directory_from_storage_config(self.pathlib.Path) + return _get_directory_from_storage_config(self.storage) class WebUi(BaseModel): From 06d9c391125d8a854401e8d1add82a8247b24183 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 11:31:26 -0400 Subject: [PATCH 214/408] Add custom annotation for serialization --- .../clp-py-utils/clp_py_utils/clp_config.py | 130 +++++------------- .../pydantic_serialization_utils.py | 8 ++ 2 files changed, 41 insertions(+), 97 deletions(-) create mode 100644 components/clp-py-utils/clp_py_utils/pydantic_serialization_utils.py diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4b67c50877..96445c9698 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Any, ClassVar, Literal, Optional, Union +from typing import Any, ClassVar, Literal, Optional, Union, Annotated from pydantic import ( BaseModel, @@ -20,6 +20,7 @@ read_yaml_config_file, validate_path_could_be_dir, ) +from .pydantic_serialization_utils import PathStr, StrEnumSerializer # Constants # Component names @@ -80,12 +81,26 @@ class StorageEngine(KebabCaseStrEnum): CLP_S = auto() +StorageEngineStr = Annotated[StorageEngine, StrEnumSerializer] + + +class DatabaseEngine(KebabCaseStrEnum): + MARIADB = auto() + MYSQL = auto() + + +DatabaseEngineStr = Annotated[DatabaseEngine, StrEnumSerializer] + + class QueryEngine(KebabCaseStrEnum): CLP = auto() CLP_S = auto() PRESTO = auto() +QueryEngineStr = Annotated[QueryEngine, StrEnumSerializer] + + class StorageType(LowercaseStrEnum): FS = auto() S3 = auto() @@ -98,33 +113,12 @@ class AwsAuthType(LowercaseStrEnum): ec2 = auto() -VALID_STORAGE_ENGINES = [storage_engine.value for storage_engine in StorageEngine] -VALID_QUERY_ENGINES = [query_engine.value for query_engine in QueryEngine] +AwsAuthTypeStr = Annotated[AwsAuthType, StrEnumSerializer] class Package(BaseModel): - storage_engine: str = "clp" - query_engine: str = "clp" - - @field_validator("storage_engine") - @classmethod - def validate_storage_engine(cls, value): - if value not in VALID_STORAGE_ENGINES: - raise ValueError( - f"package.storage_engine must be one of the following" - f" {'|'.join(VALID_STORAGE_ENGINES)}" - ) - return value - - @field_validator("query_engine") - @classmethod - def validate_query_engine(cls, value): - if value not in VALID_QUERY_ENGINES: - raise ValueError( - f"package.query_engine must be one of the following" - f" {'|'.join(VALID_QUERY_ENGINES)}" - ) - return value + storage_engine: StorageEngineStr = StorageEngine.CLP + query_engine: QueryEngineStr = QueryEngine.CLP @model_validator(mode="after") def validate_query_engine_package_compatibility(self): @@ -152,7 +146,7 @@ def validate_query_engine_package_compatibility(self): class Database(BaseModel): DEFAULT_PORT: ClassVar[int] = 3306 - type: str = "mariadb" + type: DatabaseEngineStr = DatabaseEngine.MARIADB host: str = "localhost" port: int = DEFAULT_PORT name: str = "clp-db" @@ -598,12 +592,7 @@ def validate_secret_access_key(cls, value): class AwsAuthentication(BaseModel): - type: Literal[ - AwsAuthType.credentials.value, - AwsAuthType.profile.value, - AwsAuthType.env_vars.value, - AwsAuthType.ec2.value, - ] + type: AwsAuthTypeStr profile: Optional[str] = None credentials: Optional[S3Credentials] = None @@ -660,16 +649,10 @@ class S3IngestionConfig(BaseModel): type: Literal[StorageType.S3.value] = StorageType.S3.value aws_authentication: AwsAuthentication - def dump_to_primitive_dict(self): - return self.model_dump() - - def transform_for_container(self): - pass - class FsStorage(BaseModel): type: Literal[StorageType.FS.value] = StorageType.FS.value - directory: pathlib.Path + directory: PathStr @field_validator("directory", mode="before") @classmethod @@ -680,16 +663,11 @@ def validate_directory(cls, value): def make_config_paths_absolute(self, clp_home: pathlib.Path): self.directory = make_config_path_absolute(clp_home, self.directory) - def dump_to_primitive_dict(self): - d = self.model_dump() - d["directory"] = str(d["directory"]) - return d - class S3Storage(BaseModel): type: Literal[StorageType.S3.value] = StorageType.S3.value s3_config: S3Config - staging_directory: pathlib.Path + staging_directory: PathStr @field_validator("staging_directory", mode="before") @classmethod @@ -710,14 +688,9 @@ def validate_key_prefix(cls, value): def make_config_paths_absolute(self, clp_home: pathlib.Path): self.staging_directory = make_config_path_absolute(clp_home, self.staging_directory) - def dump_to_primitive_dict(self): - d = self.model_dump() - d["staging_directory"] = str(d["staging_directory"]) - return d - class FsIngestionConfig(FsStorage): - directory: pathlib.Path = pathlib.Path("/") + directory: PathStr = pathlib.Path("/") def transform_for_container(self): input_logs_dir = self.directory.resolve() @@ -727,31 +700,19 @@ def transform_for_container(self): class ArchiveFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH - - def transform_for_container(self): - self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" class StreamFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH - - def transform_for_container(self): - self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH - - def transform_for_container(self): - self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH - - def transform_for_container(self): - self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" def _get_directory_from_storage_config( @@ -835,11 +796,6 @@ def set_directory(self, directory: pathlib.Path): def get_directory(self) -> pathlib.Path: return _get_directory_from_storage_config(self.storage) - def dump_to_primitive_dict(self): - d = self.model_dump() - d["storage"] = self.storage.dump_to_primitive_dict() - return d - class StreamOutput(BaseModel): storage: Union[StreamFsStorage, StreamS3Storage] = StreamFsStorage() @@ -858,11 +814,6 @@ def set_directory(self, directory: pathlib.Path): def get_directory(self) -> pathlib.Path: return _get_directory_from_storage_config(self.storage) - def dump_to_primitive_dict(self): - d = self.model_dump() - d["storage"] = self.storage.dump_to_primitive_dict() - return d - class WebUi(BaseModel): host: str = "localhost" @@ -961,20 +912,18 @@ class CLPConfig(BaseModel): query_worker: QueryWorker = QueryWorker() webui: WebUi = WebUi() garbage_collector: GarbageCollector = GarbageCollector() - credentials_file_path: pathlib.Path = CLP_DEFAULT_CREDENTIALS_FILE_PATH + credentials_file_path: PathStr = CLP_DEFAULT_CREDENTIALS_FILE_PATH presto: Optional[Presto] = None archive_output: ArchiveOutput = ArchiveOutput() stream_output: StreamOutput = StreamOutput() - data_directory: pathlib.Path = CLP_DEFAULT_DATA_DIRECTORY_PATH - logs_directory: pathlib.Path = CLP_DEFAULT_LOG_DIRECTORY_PATH + data_directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH + logs_directory: PathStr = CLP_DEFAULT_LOG_DIRECTORY_PATH aws_config_directory: Optional[pathlib.Path] = None - _container_image_id_path: pathlib.Path = PrivateAttr( - default=CLP_PACKAGE_CONTAINER_IMAGE_ID_PATH - ) - _version_file_path: pathlib.Path = PrivateAttr(default=CLP_VERSION_FILE_PATH) + _container_image_id_path: PathStr = PrivateAttr(default=CLP_PACKAGE_CONTAINER_IMAGE_ID_PATH) + _version_file_path: PathStr = PrivateAttr(default=CLP_VERSION_FILE_PATH) @field_validator("aws_config_directory") @classmethod @@ -1118,9 +1067,6 @@ def dump_to_primitive_dict(self): d[key] = getattr(self, key).dump_to_primitive_dict() # Turn paths into primitive strings - d["credentials_file_path"] = str(self.credentials_file_path) - d["data_directory"] = str(self.data_directory) - d["logs_directory"] = str(self.logs_directory) if self.aws_config_directory is not None: d["aws_config_directory"] = str(self.aws_config_directory) else: @@ -1194,16 +1140,6 @@ class WorkerConfig(BaseModel): stream_output: StreamOutput = StreamOutput() stream_collection_name: str = ResultsCache().stream_collection_name - def dump_to_primitive_dict(self): - d = self.model_dump() - d["archive_output"] = self.archive_output.dump_to_primitive_dict() - - # Turn paths into primitive strings - d["data_directory"] = str(self.data_directory) - d["stream_output"] = self.stream_output.dump_to_primitive_dict() - - return d - def _validate_directory(value: Any): """ diff --git a/components/clp-py-utils/clp_py_utils/pydantic_serialization_utils.py b/components/clp-py-utils/clp_py_utils/pydantic_serialization_utils.py new file mode 100644 index 0000000000..83141724d5 --- /dev/null +++ b/components/clp-py-utils/clp_py_utils/pydantic_serialization_utils.py @@ -0,0 +1,8 @@ +import pathlib +from typing import Annotated + +from pydantic import PlainSerializer + +StrEnumSerializer = PlainSerializer(lambda enum_value: enum_value.value) + +PathStr = Annotated[pathlib.Path, PlainSerializer(lambda path_value: str(path_value))] From 43c229560701b445f0e4bb13abe238566d725194 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 11:44:14 -0400 Subject: [PATCH 215/408] Remove field without custom serialization --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 96445c9698..c7d8e7ff79 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -812,7 +812,7 @@ def set_directory(self, directory: pathlib.Path): _set_directory_for_storage_config(self.storage, directory) def get_directory(self) -> pathlib.Path: - return _get_directory_from_storage_config(self.storage) + return _get_directory_from_storage_config(self.pathlib.Path) class WebUi(BaseModel): @@ -1058,9 +1058,6 @@ def dump_to_primitive_dict(self): "database", "queue", "redis", - "logs_input", - "archive_output", - "stream_output", } d = self.model_dump(exclude=custom_serialized_fields) for key in custom_serialized_fields: From e09d497bd6e5e697648aa091059dd3bb444c4503 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 11:59:30 -0400 Subject: [PATCH 216/408] Bug fix --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c7d8e7ff79..74a06697b4 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -812,7 +812,7 @@ def set_directory(self, directory: pathlib.Path): _set_directory_for_storage_config(self.storage, directory) def get_directory(self) -> pathlib.Path: - return _get_directory_from_storage_config(self.pathlib.Path) + return _get_directory_from_storage_config(self.storage) class WebUi(BaseModel): From 8e0f163e3458ebf68471edb9ba5c75d06e424fb3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 14:16:08 -0400 Subject: [PATCH 217/408] Fix lint --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 74a06697b4..be33af2b26 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Any, ClassVar, Literal, Optional, Union, Annotated +from typing import Annotated, Any, ClassVar, Literal, Optional, Union from pydantic import ( BaseModel, From 37a0a8a6d5e45d9afd7b6fa5be76b9f94e8fd708 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 10 Oct 2025 14:25:28 -0400 Subject: [PATCH 218/408] restore verbose logging option to start_clp script. --- .../clp_package_utils/scripts/start_clp.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index a9ff5b596e..f84ba7c7a3 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -31,9 +31,20 @@ def main(argv): default=str(default_config_file_path), help="CLP package configuration file.", ) + args_parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Enable debug logging.", + ) parsed_args = args_parser.parse_args(argv[1:]) + if parsed_args.verbose: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + try: # Validate and load config file. config_file_path = pathlib.Path(parsed_args.config) From 4878bd2dbb429917e3c22ca4db39e460a0bcdf41 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 14:36:52 -0400 Subject: [PATCH 219/408] Fix merge --- .../clp-py-utils/clp_py_utils/clp_config.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index be33af2b26..43dfa68793 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -649,6 +649,12 @@ class S3IngestionConfig(BaseModel): type: Literal[StorageType.S3.value] = StorageType.S3.value aws_authentication: AwsAuthentication + def dump_to_primitive_dict(self): + return self.model_dump() + + def transform_for_container(self): + pass + class FsStorage(BaseModel): type: Literal[StorageType.FS.value] = StorageType.FS.value @@ -700,19 +706,31 @@ def transform_for_container(self): class ArchiveFsStorage(FsStorage): - directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" + directory: PathStr = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + + def transform_for_container(self): + self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH class StreamFsStorage(FsStorage): - directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" + directory: PathStr = CLP_DEFAULT_STREAM_DIRECTORY_PATH + + def transform_for_container(self): + self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): - staging_directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" + staging_directory: PathStr = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + + def transform_for_container(self): + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: PathStr = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" + staging_directory: PathStr = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + + def transform_for_container(self): + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( From cc71ad19f54193e63d382c4e442c05e7b7c5990e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 15:01:35 -0400 Subject: [PATCH 220/408] Fix orchestration type --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 43dfa68793..a3d249e56f 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -317,10 +317,13 @@ class OrchestrationType(KebabCaseStrEnum): spider = auto() +OrchestrationTypeStr = Annotated[OrchestrationType, StrEnumSerializer] + + class CompressionScheduler(BaseModel): jobs_poll_delay: float = 0.1 # seconds logging_level: str = "INFO" - type: OrchestrationType = OrchestrationType.celery + type: OrchestrationTypeStr = OrchestrationType.celery @field_validator("logging_level") @classmethod From c608e7b865bd6d6d4188be1bf25be500b1615e00 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 18:06:26 -0400 Subject: [PATCH 221/408] Bug fix --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 5f58e27ef5..196964900f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -223,7 +223,7 @@ def _set_up_env_for_spider_scheduler(self) -> EnvVarsDict: return { "SPIDER_SCHEDULER_HOST": _get_ip_from_hostname(self.clp_config.spider_scheduler.host), - "SPIDER_SCHEDULER_PORT": str(self.clp_config.scheduler.port), + "SPIDER_SCHEDULER_PORT": str(self.clp_config.spider_scheduler.port), } def _set_up_env_for_results_cache(self) -> EnvVarsDict: From 358fe4b9d66e0e68c80b86264693a97dae9b551a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 18:21:03 -0400 Subject: [PATCH 222/408] Fix compression scheduler --- .../scheduler/compress/compression_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 33a375eb5c..d1e20ecbb2 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -491,7 +491,7 @@ def main(argv): if clp_config.compression_scheduler.type == OrchestrationType.celery: task_manager = CeleryTaskManager() elif clp_config.compression_scheduler.type == OrchestrationType.spider: - task_manager = SpiderTaskManager() + task_manager = SpiderTaskManager(clp_config.spider_db.get_url()) else: logger.error( f"Unsupported compression scheduler type:" f" {clp_config.compression_scheduler.type}" From 535b6964892e8532ff8a37c7e1c01980890f0228 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 18:38:21 -0400 Subject: [PATCH 223/408] Add spider table creation --- .../clp_py_utils/initialize-spider-db.py | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 4ec5e383b5..8193c60923 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -23,6 +23,203 @@ logger.addHandler(logging_console_handler) +table_creators = [ + """ +CREATE TABLE IF NOT EXISTS `drivers` +( + `id` BINARY(16) NOT NULL, + `heartbeat` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS `schedulers` +( + `id` BINARY(16) NOT NULL, + `address` VARCHAR(40) NOT NULL, + `port` INT UNSIGNED NOT NULL, + CONSTRAINT `scheduler_driver_id` FOREIGN KEY (`id`) REFERENCES `drivers` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + PRIMARY KEY (`id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS jobs +( + `id` BINARY(16) NOT NULL, + `client_id` BINARY(16) NOT NULL, + `creation_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `state` ENUM ('running', 'success', 'fail', 'cancel') NOT NULL DEFAULT 'running', + KEY (`client_id`) USING BTREE, + INDEX idx_jobs_creation_time (`creation_time`), + INDEX idx_jobs_state (`state`), + PRIMARY KEY (`id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS tasks +( + `id` BINARY(16) NOT NULL, + `job_id` BINARY(16) NOT NULL, + `func_name` VARCHAR(64) NOT NULL, + `language` ENUM('cpp', 'python') NOT NULL, + `state` ENUM ('pending', 'ready', 'running', 'success', 'cancel', 'fail') NOT NULL, + `timeout` FLOAT, + `max_retry` INT UNSIGNED DEFAULT 0, + `retry` INT UNSIGNED DEFAULT 0, + `instance_id` BINARY(16), + CONSTRAINT `task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + PRIMARY KEY (`id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS input_tasks +( + `job_id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + `position` INT UNSIGNED NOT NULL, + CONSTRAINT `input_task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `input_task_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + INDEX (`job_id`, `position`), + PRIMARY KEY (`task_id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS output_tasks +( + `job_id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + `position` INT UNSIGNED NOT NULL, + CONSTRAINT `output_task_job_id` FOREIGN KEY (`job_id`) REFERENCES `jobs` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `output_task_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + INDEX (`job_id`, `position`), + PRIMARY KEY (`task_id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS `data` +( + `id` BINARY(16) NOT NULL, + `value` VARBINARY(999) NOT NULL, + `hard_locality` BOOL DEFAULT FALSE, + `persisted` BOOL DEFAULT FALSE, + PRIMARY KEY (`id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS `task_outputs` +( + `task_id` BINARY(16) NOT NULL, + `position` INT UNSIGNED NOT NULL, + `type` VARCHAR(999) NOT NULL, + `value` VARBINARY(999), + `data_id` BINARY(16), + CONSTRAINT `output_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `output_data_id` FOREIGN KEY (`data_id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE NO ACTION, + PRIMARY KEY (`task_id`, `position`) +); +""", + """ +CREATE TABLE IF NOT EXISTS `task_inputs` +( + `task_id` BINARY(16) NOT NULL, + `position` INT UNSIGNED NOT NULL, + `type` VARCHAR(999) NOT NULL, + `output_task_id` BINARY(16), + `output_task_position` INT UNSIGNED, + `value` VARBINARY(999), -- Use VARBINARY for all types of values + `data_id` BINARY(16), + CONSTRAINT `input_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `input_task_output_match` FOREIGN KEY (`output_task_id`, `output_task_position`) REFERENCES task_outputs (`task_id`, `position`) ON UPDATE NO ACTION ON DELETE SET NULL, + CONSTRAINT `input_data_id` FOREIGN KEY (`data_id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE NO ACTION, + PRIMARY KEY (`task_id`, `position`) +); +""", + """ +CREATE TABLE IF NOT EXISTS `task_dependencies` +( + `parent` BINARY(16) NOT NULL, + `child` BINARY(16) NOT NULL, + KEY (`parent`) USING BTREE, + KEY (`child`) USING BTREE, + CONSTRAINT `task_dep_parent` FOREIGN KEY (`parent`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `task_dep_child` FOREIGN KEY (`child`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE +); +""", + """ +CREATE TABLE IF NOT EXISTS `task_instances` +( + `id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + `start_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `instance_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + PRIMARY KEY (`id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS `scheduler_leases` +( + `scheduler_id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + `lease_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `lease_scheduler_id` FOREIGN KEY (`scheduler_id`) REFERENCES `schedulers` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `lease_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + INDEX (`scheduler_id`), + PRIMARY KEY (`scheduler_id`, `task_id`) +); +""", + """ +CREATE TABLE IF NOT EXISTS `data_locality` +( + `id` BINARY(16) NOT NULL, + `address` VARCHAR(40) NOT NULL, + KEY (`id`) USING BTREE, + CONSTRAINT `locality_data_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE +); +""", + """ +CREATE TABLE IF NOT EXISTS `data_ref_driver` +( + `id` BINARY(16) NOT NULL, + `driver_id` BINARY(16) NOT NULL, + KEY (`id`) USING BTREE, + KEY (`driver_id`) USING BTREE, + CONSTRAINT `data_driver_ref_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `data_ref_driver_id` FOREIGN KEY (`driver_id`) REFERENCES `drivers` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE +); +""", + """ +CREATE TABLE IF NOT EXISTS `data_ref_task` +( + `id` BINARY(16) NOT NULL, + `task_id` BINARY(16) NOT NULL, + KEY (`id`) USING BTREE, + KEY (`task_id`) USING BTREE, + CONSTRAINT `data_task_ref_id` FOREIGN KEY (`id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, + CONSTRAINT `data_ref_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE +); +""", + """ +CREATE TABLE IF NOT EXISTS `client_kv_data` +( + `kv_key` VARCHAR(64) NOT NULL, + `value` VARBINARY(999) NOT NULL, + `client_id` BINARY(16) NOT NULL, + PRIMARY KEY (`client_id`, `kv_key`) +); +""", + """ +CREATE TABLE IF NOT EXISTS `task_kv_data` +( + `kv_key` VARCHAR(64) NOT NULL, + `value` VARBINARY(999) NOT NULL, + `task_id` BINARY(16) NOT NULL, + PRIMARY KEY (`task_id`, `kv_key`), + CONSTRAINT `kv_data_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE +); +""", +] + + def main(argv): args_parser = argparse.ArgumentParser(description="Sets up Spider database.") args_parser.add_argument("--config", "-c", required=True, help="CLP configuration file.") @@ -79,6 +276,9 @@ def main(argv): db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_user}'@'%'""") db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{clp_user}'@'%'""") + db_cursor.execute(f"""USE `{db_name}`""") + for table_creator in table_creators: + db_cursor.execute(table_creator) except: logger.exception("Failed to setup Spider database.") return -1 From a405aea890e8eca19774239d05e8cb9b28ee81bc Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 10 Oct 2025 19:17:47 -0400 Subject: [PATCH 224/408] Fix none tag id --- .../job_orchestration/executor/compress/spider_compress.py | 2 +- .../scheduler/compress/task_manager/spider_task_manager.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index 3cd74b6ae5..64c1a55268 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -38,7 +38,7 @@ def compress( compression_entry_point( int(job_id), int(task_id), - [int(tag_id) for tag_id in tag_ids], + [int(tag_id) for tag_id in tag_ids] if len(tag_ids) > 0 else None, int8_list_to_utf8_str(clp_io_config_json), int8_list_to_utf8_str(paths_to_compress_json), json.loads(int8_list_to_utf8_str(clp_metadata_db_connection_config_json)), diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index 6148720b3d..0e8bdb4586 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -37,7 +37,10 @@ def submit(self, task_params: list[dict[str, Any]]) -> TaskManager.ResultHandle: for task_param in task_params: job_args.append(spider_py.Int64(task_param["job_id"])) job_args.append(spider_py.Int64(task_param["task_id"])) - job_args.append([spider_py.Int64(tag_id) for tag_id in task_param["tag_ids"]]) + if "tag_ids" in task_param and task_param["tag_ids"] is not None: + job_args.append([spider_py.Int64(tag_id) for tag_id in task_param["tag_ids"]]) + else: + job_args.append([]) job_args.append(utf8_str_to_int8_list(task_param["clp_io_config_json"])) job_args.append(utf8_str_to_int8_list(task_param["paths_to_compress_json"])) job_args.append( From 7d1812974a3933906340ab497441613906ac3c39 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 12 Oct 2025 22:38:04 -0400 Subject: [PATCH 225/408] fix(docker): Move USER and ENV directives after image flattening in clp-package Dockerfile (fixes #1379); Reduce layers using multi-line ENV (fixes #1378). --- tools/docker-images/clp-package/Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 9d1bc5c71e..4dbafc16e4 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -10,15 +10,15 @@ RUN ./setup-scripts/install-prebuilt-packages.sh \ RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -ENV CLP_HOME="/opt/clp" -ENV PATH="${CLP_HOME}/bin:${PATH}" -ENV PATH="${CLP_HOME}/sbin:${PATH}" -ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" - -USER 1000:1000 - COPY ./build/clp-package /opt/clp # Flatten the image FROM scratch COPY --from=base / / + +ENV CLP_HOME="/opt/clp" +ENV PATH="${CLP_HOME}/bin:${PATH}" \ + PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" +ENV PATH="${CLP_HOME}/sbin:${PATH}" + +USER 1000:1000 From ef046e5d2661792d04f9619ca976aef2a2ca6a1c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 12 Oct 2025 22:34:34 -0400 Subject: [PATCH 226/408] fix(docker): Include `libmariadbcpp.so` (required by `spider_scheduler` and `spider_worker`) in the `clp-package` image; Set `mariadb-connector-cpp` build type to `Release` (fixes #1410). --- taskfiles/deps/main.yaml | 1 + tools/docker-images/clp-package/Dockerfile | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 9a6b0ab478..ed37d48251 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -405,6 +405,7 @@ tasks: - task: "utils:install-remote-cmake-lib" vars: CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=DEB" LIB_NAME: "mariadb-connector-cpp" diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 9d1bc5c71e..21d9d5795b 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -10,15 +10,18 @@ RUN ./setup-scripts/install-prebuilt-packages.sh \ RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -ENV CLP_HOME="/opt/clp" -ENV PATH="${CLP_HOME}/bin:${PATH}" -ENV PATH="${CLP_HOME}/sbin:${PATH}" -ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" - -USER 1000:1000 - COPY ./build/clp-package /opt/clp +COPY --link ./build/deps/cpp/mariadb-connector-cpp-install/lib/x86_64-linux-gnu/libmariadbcpp.so \ + /opt/clp/lib/libmariadbcpp.so # Flatten the image FROM scratch COPY --from=base / / + +ENV CLP_HOME="/opt/clp" +ENV LD_LIBRARY_PATH="${CLP_HOME}/lib" \ + PATH="${CLP_HOME}/bin:${PATH}" \ + PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" +ENV PATH="${CLP_HOME}/sbin:${PATH}" + +USER 1000:1000 From cfa71d899cc0314882daa738c3950fe6caa35469 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 12 Oct 2025 23:29:25 -0400 Subject: [PATCH 227/408] Make COPY arch-agnostic and include SONAME symlinks --- tools/docker-images/clp-package/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 21d9d5795b..b4ad1cdf22 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -11,8 +11,7 @@ RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* COPY ./build/clp-package /opt/clp -COPY --link ./build/deps/cpp/mariadb-connector-cpp-install/lib/x86_64-linux-gnu/libmariadbcpp.so \ - /opt/clp/lib/libmariadbcpp.so +COPY --link ./build/deps/cpp/mariadb-connector-cpp-install/lib/*/libmariadbcpp.so* /opt/clp/lib/ # Flatten the image FROM scratch From 3195ef03f5105054d7f1488c6b18277ebeffb66b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 14 Oct 2025 01:51:09 -0400 Subject: [PATCH 228/408] feat(docker): Update clp-package Dockerfile; add non-root user and set working directory. --- tools/docker-images/clp-package/Dockerfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 4dbafc16e4..3bd45031dd 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -1,7 +1,5 @@ FROM ubuntu:jammy AS base -WORKDIR /root - COPY ./tools/docker-images/clp-package/setup-scripts ./setup-scripts RUN ./setup-scripts/install-prebuilt-packages.sh \ && rm -rf ./setup-scripts/ @@ -10,15 +8,19 @@ RUN ./setup-scripts/install-prebuilt-packages.sh \ RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -COPY ./build/clp-package /opt/clp - # Flatten the image FROM scratch COPY --from=base / / ENV CLP_HOME="/opt/clp" +COPY ./build/clp-package ${CLP_HOME} + ENV PATH="${CLP_HOME}/bin:${PATH}" \ PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" ENV PATH="${CLP_HOME}/sbin:${PATH}" -USER 1000:1000 +# Create a non-root user. +ENV USER="clp-user" +RUN useradd --uid 1000 --shell /bin/bash --home-dir ${CLP_HOME} ${USER} +USER ${USER} +WORKDIR ${CLP_HOME} From 18a1b18fa0c836b02da45b9c3967cd0f659bf0db Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 14 Oct 2025 02:27:58 -0400 Subject: [PATCH 229/408] fix(docker): Reorder and restructure ENV directives in clp-package Dockerfile. --- tools/docker-images/clp-package/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 3bd45031dd..ccd02402a3 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -15,12 +15,11 @@ COPY --from=base / / ENV CLP_HOME="/opt/clp" COPY ./build/clp-package ${CLP_HOME} -ENV PATH="${CLP_HOME}/bin:${PATH}" \ +ENV PATH="${CLP_HOME}/bin:${PATH}" +ENV PATH="${CLP_HOME}/sbin:${PATH}" \ PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" -ENV PATH="${CLP_HOME}/sbin:${PATH}" + USER="clp-user" -# Create a non-root user. -ENV USER="clp-user" RUN useradd --uid 1000 --shell /bin/bash --home-dir ${CLP_HOME} ${USER} USER ${USER} WORKDIR ${CLP_HOME} From 3ef8714b53ce78a57c1c9867c0a94ec7cd198823 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 14 Oct 2025 02:28:22 -0400 Subject: [PATCH 230/408] add \ --- tools/docker-images/clp-package/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index ccd02402a3..625ba8e044 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -17,7 +17,7 @@ COPY ./build/clp-package ${CLP_HOME} ENV PATH="${CLP_HOME}/bin:${PATH}" ENV PATH="${CLP_HOME}/sbin:${PATH}" \ - PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" + PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" \ USER="clp-user" RUN useradd --uid 1000 --shell /bin/bash --home-dir ${CLP_HOME} ${USER} From e487f82c7c1e9cc5caa840d8dc645c4cb9eadf3a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 14 Oct 2025 11:46:26 -0400 Subject: [PATCH 231/408] Fix enum --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index e80940148d..21f0f63233 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -669,7 +669,7 @@ def validate_aws_config_dir(self): auth_configs.append(self.stream_output.storage.s3_config.aws_authentication) for auth in auth_configs: - if AwsAuthType.profile.value == auth.type: + if AwsAuthType.profile == auth.type: profile_auth_used = True break From 95d900a07a831a28ae3b23c96bc19040c63410ac Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 14 Oct 2025 19:31:06 -0400 Subject: [PATCH 232/408] refactor(clp-py-utils): Use DEFAULT_PORT for WebUi port configuration. --- components/clp-py-utils/clp_py_utils/clp_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index f20026d154..41a9ee431d 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -575,8 +575,10 @@ def dump_to_primitive_dict(self): class WebUi(BaseModel): + DEFAULT_PORT: ClassVar[int] = 4000 + host: DomainStr = "localhost" - port: Port = 4000 + port: Port = DEFAULT_PORT results_metadata_collection_name: NonEmptyStr = "results-metadata" rate_limit: PositiveInt = 1000 From f6cbc7f97b36767ee069c64d8ec6551431ffe829 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:15:21 -0400 Subject: [PATCH 233/408] apply docs suggestions --- .../clp_package_utils/controller.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0b3807a3dd..6162022266 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -48,7 +48,6 @@ validate_webui_config, ) -# Type alias for environment variables dictionary. EnvVarsDict = Dict[str, str] LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH @@ -63,8 +62,9 @@ class BaseController(ABC): """ - Abstract base controller for preparing and deploying CLP components. Provides common logic for - preparing environment variables, directories, and configuration files for each service. + Base controller for orchestrating CLP components. Derived classes should implement any + orchestrator-specific logic. This class provides common logic for preparing environment + variables, directories, and configuration files for each component. """ def __init__(self, clp_config: CLPConfig): @@ -75,22 +75,22 @@ def __init__(self, clp_config: CLPConfig): @abstractmethod def start(self): """ - Starts the set-up components with orchestrator-specific logic. + Starts the components. """ pass @abstractmethod def stop(self): """ - Stops the deployed components with orchestrator-specific logic. + Stops the components. """ pass @abstractmethod def _set_up_env(self): """ - Sets up all components for the orchestrator by preparing environment variables, directories, - and configuration files. + Sets up all components to run by preparing environment variables, directories, and + configuration files. """ pass @@ -463,7 +463,7 @@ def _update_settings_object( class DockerComposeController(BaseController): """ - Controller for deploying CLP components using Docker Compose. + Controller for orchestrating CLP components using Docker Compose. """ def __init__(self, clp_config: CLPConfig, instance_id: str): @@ -472,16 +472,13 @@ def __init__(self, clp_config: CLPConfig, instance_id: str): def start(self): """ - Deploys CLP components using Docker Compose by: - 1. Checking Docker dependencies. - 2. Setting up environment variables and configuration. - 3. Running `docker compose up -d`. + Starts CLP's components using Docker Compose. """ check_docker_dependencies(should_compose_run=False, project_name=self._project_name) self._set_up_env() deployment_type = self.clp_config.get_deployment_type() - logger.info(f"Starting CLP using Docker Compose ({deployment_type})...") + logger.info(f"Starting CLP using Docker Compose ({deployment_type} deployment)...") cmd = ["docker", "compose", "--project-name", self._project_name] if deployment_type == DeploymentType.BASE: @@ -520,9 +517,8 @@ def stop(self): @staticmethod def _get_num_workers() -> int: """ - Gets the parallelism number for worker components. TODO: Revisit after moving from single-container to multi-container workers. - :return: Number of worker processes. + :return: Number of worker processes to run. """ return multiprocessing.cpu_count() // 2 @@ -558,6 +554,7 @@ def _set_up_env(self): # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + # Component-specific environment variables **self._set_up_env_for_database(), **self._set_up_env_for_queue(), **self._set_up_env_for_redis(), @@ -581,7 +578,8 @@ def _set_up_env(self): def get_or_create_instance_id(clp_config: CLPConfig): """ - Gets or create a unique instance ID for this CLP instance. + Gets or creates a unique instance ID for this CLP instance. + :param clp_config: :return: The instance ID. """ From 5bf08873fdb05c2c726c10c606694a92fcbbf32b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:17:02 -0400 Subject: [PATCH 234/408] refactor(controller): Rename clp_config -> _clp_config for consistent private attribute naming. --- .../clp_package_utils/controller.py | 148 +++++++++--------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 6162022266..84ecfb3726 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -68,7 +68,7 @@ class BaseController(ABC): """ def __init__(self, clp_config: CLPConfig): - self.clp_config = clp_config + self._clp_config = clp_config self._clp_home = get_clp_home() self._conf_dir = self._clp_home / "etc" @@ -104,9 +104,9 @@ def _set_up_env_for_database(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") conf_logging_file = self._conf_dir / "mysql" / "conf.d" / "logging.cnf" - data_dir = self.clp_config.data_directory / component_name - logs_dir = self.clp_config.logs_directory / component_name - validate_db_config(self.clp_config, conf_logging_file, data_dir, logs_dir) + data_dir = self._clp_config.data_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name + validate_db_config(self._clp_config, conf_logging_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) @@ -116,13 +116,13 @@ def _set_up_env_for_database(self) -> EnvVarsDict: "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), - "CLP_DB_HOST": _get_ip_from_hostname(self.clp_config.database.host), - "CLP_DB_PORT": str(self.clp_config.database.port), - "CLP_DB_NAME": self.clp_config.database.name, - "CLP_DB_USER": self.clp_config.database.username, - "CLP_DB_PASS": self.clp_config.database.password, + "CLP_DB_HOST": _get_ip_from_hostname(self._clp_config.database.host), + "CLP_DB_PORT": str(self._clp_config.database.port), + "CLP_DB_NAME": self._clp_config.database.name, + "CLP_DB_USER": self._clp_config.database.username, + "CLP_DB_PASS": self._clp_config.database.password, "CLP_DB_IMAGE": ( - "mysql:8.0.23" if "mysql" == self.clp_config.database.type else "mariadb:10-jammy" + "mysql:8.0.23" if "mysql" == self._clp_config.database.type else "mariadb:10-jammy" ), } @@ -135,18 +135,18 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: component_name = QUEUE_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name - validate_queue_config(self.clp_config, logs_dir) + logs_dir = self._clp_config.logs_directory / component_name + validate_queue_config(self._clp_config, logs_dir) logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(logs_dir) return { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), - "CLP_QUEUE_HOST": _get_ip_from_hostname(self.clp_config.queue.host), - "CLP_QUEUE_PORT": str(self.clp_config.queue.port), - "CLP_QUEUE_USER": self.clp_config.queue.username, - "CLP_QUEUE_PASS": self.clp_config.queue.password, + "CLP_QUEUE_HOST": _get_ip_from_hostname(self._clp_config.queue.host), + "CLP_QUEUE_PORT": str(self._clp_config.queue.port), + "CLP_QUEUE_USER": self._clp_config.queue.username, + "CLP_QUEUE_PASS": self._clp_config.queue.password, } def _set_up_env_for_redis(self) -> EnvVarsDict: @@ -159,9 +159,9 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "redis" / "redis.conf" - data_dir = self.clp_config.data_directory / component_name - logs_dir = self.clp_config.logs_directory / component_name - validate_redis_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir = self._clp_config.data_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name + validate_redis_config(self._clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) @@ -171,12 +171,12 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": _get_ip_from_hostname(self.clp_config.redis.host), - "CLP_REDIS_PORT": str(self.clp_config.redis.port), - "CLP_REDIS_PASS": self.clp_config.redis.password, - "CLP_REDIS_QUERY_BACKEND_DB": str(self.clp_config.redis.query_backend_database), + "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), + "CLP_REDIS_PORT": str(self._clp_config.redis.port), + "CLP_REDIS_PASS": self._clp_config.redis.password, + "CLP_REDIS_QUERY_BACKEND_DB": str(self._clp_config.redis.query_backend_database), "CLP_REDIS_COMPRESSION_BACKEND_DB": str( - self.clp_config.redis.compression_backend_database + self._clp_config.redis.compression_backend_database ), } @@ -190,9 +190,9 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") conf_file = self._conf_dir / "mongo" / "mongod.conf" - data_dir = self.clp_config.data_directory / component_name - logs_dir = self.clp_config.logs_directory / component_name - validate_results_cache_config(self.clp_config, conf_file, data_dir, logs_dir) + data_dir = self._clp_config.data_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name + validate_results_cache_config(self._clp_config, conf_file, data_dir, logs_dir) data_dir.mkdir(exist_ok=True, parents=True) logs_dir.mkdir(exist_ok=True, parents=True) @@ -202,10 +202,10 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self.clp_config.results_cache.host), - "CLP_RESULTS_CACHE_PORT": str(self.clp_config.results_cache.port), - "CLP_RESULTS_CACHE_DB_NAME": self.clp_config.results_cache.db_name, - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self.clp_config.results_cache.stream_collection_name, + "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), + "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), + "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self._clp_config.results_cache.stream_collection_name, } def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: @@ -217,11 +217,11 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - log_file = self.clp_config.logs_directory / f"{component_name}.log" + log_file = self._clp_config.logs_directory / f"{component_name}.log" log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { - "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self.clp_config.compression_scheduler.logging_level, + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self._clp_config.compression_scheduler.logging_level, "CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST": str(log_file), } @@ -234,11 +234,11 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - log_file = self.clp_config.logs_directory / f"{component_name}.log" + log_file = self._clp_config.logs_directory / f"{component_name}.log" log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) return { - "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self.clp_config.query_scheduler.logging_level, + "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, "CLP_QUERY_SCHEDULER_LOG_FILE_HOST": str(log_file), } @@ -252,13 +252,13 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: component_name = COMPRESSION_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) return { "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), - "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": self.clp_config.compression_worker.logging_level, + "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": self._clp_config.compression_worker.logging_level, "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } @@ -272,12 +272,12 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: component_name = QUERY_WORKER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) return { - "CLP_QUERY_WORKER_LOGGING_LEVEL": self.clp_config.query_worker.logging_level, + "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, "CLP_QUERY_WORKER_LOGS_DIR_HOST": str(logs_dir), "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } @@ -292,15 +292,15 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: component_name = REDUCER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) return { - "CLP_REDUCER_LOGGING_LEVEL": self.clp_config.reducer.logging_level, + "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, "CLP_REDUCER_LOGS_DIR_HOST": str(logs_dir), "CLP_REDUCER_CONCURRENCY": str(num_workers), - "CLP_REDUCER_UPSERT_INTERVAL": str(self.clp_config.reducer.upsert_interval), + "CLP_REDUCER_UPSERT_INTERVAL": str(self._clp_config.reducer.upsert_interval), } def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: @@ -321,12 +321,12 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) - validate_webui_config(self.clp_config, client_settings_json_path, server_settings_json_path) + validate_webui_config(self._clp_config, client_settings_json_path, server_settings_json_path) # Read, update, and write back client's and server's settings.json - clp_db_connection_params = self.clp_config.database.get_clp_connection_params_and_type(True) + clp_db_connection_params = self._clp_config.database.get_clp_connection_params_and_type(True) table_prefix = clp_db_connection_params["table_prefix"] - if StorageEngine.CLP_S == self.clp_config.package.storage_engine: + if StorageEngine.CLP_S == self._clp_config.package.storage_engine: archives_table_name = "" files_table_name = "" else: @@ -334,9 +334,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: files_table_name = get_files_table_name(table_prefix, None) client_settings_json_updates = { - "ClpStorageEngine": self.clp_config.package.storage_engine, - "ClpQueryEngine": self.clp_config.package.query_engine, - "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, + "ClpStorageEngine": self._clp_config.package.storage_engine, + "ClpQueryEngine": self._clp_config.package.query_engine, + "MongoDbSearchResultsMetadataCollectionName": self._clp_config.webui.results_metadata_collection_name, "SqlDbClpArchivesTableName": archives_table_name, "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), "SqlDbClpFilesTableName": files_table_name, @@ -352,19 +352,19 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_updates = { "SqlDbHost": container_clp_config.database.host, "SqlDbPort": container_clp_config.database.port, - "SqlDbName": self.clp_config.database.name, + "SqlDbName": self._clp_config.database.name, "SqlDbQueryJobsTableName": "query_jobs", "MongoDbHost": container_clp_config.results_cache.host, "MongoDbPort": container_clp_config.results_cache.port, - "MongoDbName": self.clp_config.results_cache.db_name, - "MongoDbSearchResultsMetadataCollectionName": self.clp_config.webui.results_metadata_collection_name, - "MongoDbStreamFilesCollectionName": self.clp_config.results_cache.stream_collection_name, + "MongoDbName": self._clp_config.results_cache.db_name, + "MongoDbSearchResultsMetadataCollectionName": self._clp_config.webui.results_metadata_collection_name, + "MongoDbStreamFilesCollectionName": self._clp_config.results_cache.stream_collection_name, "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), - "StreamTargetUncompressedSize": self.clp_config.stream_output.target_uncompressed_size, + "StreamTargetUncompressedSize": self._clp_config.stream_output.target_uncompressed_size, } - stream_storage = self.clp_config.stream_output.storage + stream_storage = self._clp_config.stream_output.storage if StorageType.S3 == stream_storage.type: s3_config = stream_storage.s3_config server_settings_json_updates["StreamFilesDir"] = None @@ -385,10 +385,10 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_updates["StreamFilesS3PathPrefix"] = None server_settings_json_updates["StreamFilesS3Profile"] = None - query_engine = self.clp_config.package.query_engine + query_engine = self._clp_config.package.query_engine if QueryEngine.PRESTO == query_engine: - server_settings_json_updates["PrestoHost"] = self.clp_config.presto.host - server_settings_json_updates["PrestoPort"] = self.clp_config.presto.port + server_settings_json_updates["PrestoHost"] = self._clp_config.presto.host + server_settings_json_updates["PrestoPort"] = self._clp_config.presto.port else: server_settings_json_updates["PrestoHost"] = None server_settings_json_updates["PrestoPort"] = None @@ -400,9 +400,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: settings_json_file.write(json.dumps(server_settings_json)) return { - "CLP_WEBUI_HOST": _get_ip_from_hostname(self.clp_config.webui.host), - "CLP_WEBUI_PORT": str(self.clp_config.webui.port), - "CLP_WEBUI_RATE_LIMIT": str(self.clp_config.webui.rate_limit), + "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), + "CLP_WEBUI_PORT": str(self._clp_config.webui.port), + "CLP_WEBUI_RATE_LIMIT": str(self._clp_config.webui.rate_limit), } def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: @@ -414,11 +414,11 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: component_name = GARBAGE_COLLECTOR_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - logs_dir = self.clp_config.logs_directory / component_name + logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return {"CLP_GC_LOGGING_LEVEL": self.clp_config.garbage_collector.logging_level} + return {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} def _read_and_update_settings_json( self, settings_file_path: pathlib.Path, updates: Dict[str, Any] @@ -477,7 +477,7 @@ def start(self): check_docker_dependencies(should_compose_run=False, project_name=self._project_name) self._set_up_env() - deployment_type = self.clp_config.get_deployment_type() + deployment_type = self._clp_config.get_deployment_type() logger.info(f"Starting CLP using Docker Compose ({deployment_type} deployment)...") cmd = ["docker", "compose", "--project-name", self._project_name] @@ -529,28 +529,28 @@ def _set_up_env(self): - Preparing environment variables for all components. - Writing environment variables to `.env`. """ - container_clp_config = generate_docker_compose_container_config(self.clp_config) + container_clp_config = generate_docker_compose_container_config(self._clp_config) num_workers = self._get_num_workers() - dump_shared_container_config(container_clp_config, self.clp_config) + dump_shared_container_config(container_clp_config, self._clp_config) env_dict = { - "CLP_PACKAGE_STORAGE_ENGINE": self.clp_config.package.storage_engine, + "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, # User and group IDs "CLP_UID_GID": DEFAULT_UID_GID, "CLP_SERVICE_CONTAINER_UID_GID": ( SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID ), # Package container - "CLP_PACKAGE_CONTAINER": self.clp_config.container_image_ref, + "CLP_PACKAGE_CONTAINER": self._clp_config.container_image_ref, # Runtime data directories - "CLP_DATA_DIR_HOST": str(self.clp_config.data_directory), - "CLP_LOGS_DIR_HOST": str(self.clp_config.logs_directory), + "CLP_DATA_DIR_HOST": str(self._clp_config.data_directory), + "CLP_LOGS_DIR_HOST": str(self._clp_config.logs_directory), # Input directories - "CLP_LOGS_INPUT_DIR_HOST": str(self.clp_config.logs_input.directory), + "CLP_LOGS_INPUT_DIR_HOST": str(self._clp_config.logs_input.directory), "CLP_LOGS_INPUT_DIR_CONTAINER": str(container_clp_config.logs_input.directory), # Output directories - "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self.clp_config.archive_output.get_directory()), - "CLP_STREAM_OUTPUT_DIR_HOST": str(self.clp_config.stream_output.get_directory()), + "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self._clp_config.archive_output.get_directory()), + "CLP_STREAM_OUTPUT_DIR_HOST": str(self._clp_config.stream_output.get_directory()), # AWS credentials "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), @@ -568,8 +568,8 @@ def _set_up_env(self): **self._set_up_env_for_garbage_collector(), } - if self.clp_config.aws_config_directory is not None: - env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self.clp_config.aws_config_directory) + if self._clp_config.aws_config_directory is not None: + env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self._clp_config.aws_config_directory) with open(f"{self._clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): From 26dfacacf2856d997745db31b81719414b8b684a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:37:25 -0400 Subject: [PATCH 235/408] refactor(deployment): Rename Redis environment variables for improved clarity. --- components/clp-package-utils/clp_package_utils/controller.py | 4 ++-- tools/deployment/package/docker-compose.base.yaml | 4 ++-- tools/deployment/package/docker-compose.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 84ecfb3726..7d5c84f96c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -174,8 +174,8 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), "CLP_REDIS_PORT": str(self._clp_config.redis.port), "CLP_REDIS_PASS": self._clp_config.redis.password, - "CLP_REDIS_QUERY_BACKEND_DB": str(self._clp_config.redis.query_backend_database), - "CLP_REDIS_COMPRESSION_BACKEND_DB": str( + "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), + "CLP_REDIS_BACKEND_DB_COMPRESSION": str( self._clp_config.redis.compression_backend_database ), } diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 2c00c58f15..03e1dcf867 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -216,7 +216,7 @@ services: CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -252,7 +252,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 44ecae5e89..b66757980a 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -40,7 +40,7 @@ services: CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - type: "bind" @@ -82,7 +82,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly From 99e983f29e5f3a79a0efe0c491125c62c1ac2c1a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:51:02 -0400 Subject: [PATCH 236/408] Apply suggestions - Reformat multiline statements and remove unnecessary blank lines. --- .../clp_package_utils/controller.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 7d5c84f96c..96f78dbd89 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -205,7 +205,9 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, - "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": self._clp_config.results_cache.stream_collection_name, + "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": ( + self._clp_config.results_cache.stream_collection_name + ), } def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: @@ -253,7 +255,6 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -273,7 +274,6 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -293,7 +293,6 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) return { @@ -320,11 +319,14 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_path = ( self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) - - validate_webui_config(self._clp_config, client_settings_json_path, server_settings_json_path) + validate_webui_config( + self._clp_config, client_settings_json_path, server_settings_json_path + ) # Read, update, and write back client's and server's settings.json - clp_db_connection_params = self._clp_config.database.get_clp_connection_params_and_type(True) + clp_db_connection_params = self._clp_config.database.get_clp_connection_params_and_type( + True + ) table_prefix = clp_db_connection_params["table_prefix"] if StorageEngine.CLP_S == self._clp_config.package.storage_engine: archives_table_name = "" @@ -415,7 +417,6 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) return {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} From 0b028dcef5f19d58d019a04d53d3904b466c34d6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 03:58:19 -0400 Subject: [PATCH 237/408] Apply suggestions - Rename UID/GID environment variables for improved clarity. --- .../clp_package_utils/controller.py | 14 +++++++------- tools/deployment/package/docker-compose.base.yaml | 10 +++++----- tools/deployment/package/docker-compose.yaml | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 96f78dbd89..0d0b19ca1f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -53,9 +53,9 @@ LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" -SERVICE_CONTAINER_USER_ID = 999 -SERVICE_CONTAINER_GROUP_ID = 999 -SERVICE_CONTAINER_UID_GID = f"{SERVICE_CONTAINER_USER_ID}:{SERVICE_CONTAINER_GROUP_ID}" +THIRD_PARTY_SERVICE_UID = 999 +THIRD_PARTY_SERVICE_GID = 999 +THIRD_PARTY_SERVICE_UID_GID = f"{THIRD_PARTY_SERVICE_UID}:{THIRD_PARTY_SERVICE_GID}" logger = logging.getLogger(__name__) @@ -537,9 +537,9 @@ def _set_up_env(self): env_dict = { "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, # User and group IDs - "CLP_UID_GID": DEFAULT_UID_GID, - "CLP_SERVICE_CONTAINER_UID_GID": ( - SERVICE_CONTAINER_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID + "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID, + "CLP_THIRD_PARTY_SERVICE_UID_GID": ( + THIRD_PARTY_SERVICE_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID ), # Package container "CLP_PACKAGE_CONTAINER": self._clp_config.container_image_ref, @@ -607,7 +607,7 @@ def _chown_paths_if_root(*paths: pathlib.Path): if os.getuid() != 0: return for path in paths: - _chown_recursively(path, SERVICE_CONTAINER_USER_ID, SERVICE_CONTAINER_GROUP_ID) + _chown_recursively(path, THIRD_PARTY_SERVICE_UID, THIRD_PARTY_SERVICE_GID) def _chown_recursively( diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 03e1dcf867..205aa9c387 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -6,7 +6,7 @@ x-service-defaults: &service_defaults logging: driver: "local" stop_grace_period: "3s" - user: "${CLP_UID_GID:-1000:1000}" + user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" # Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults @@ -46,7 +46,7 @@ services: <<: *service_defaults image: "${CLP_DB_IMAGE:-mysql:8.0.23}" hostname: "database" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" MYSQL_PASSWORD: "${CLP_DB_PASS}" @@ -102,7 +102,7 @@ services: <<: *service_defaults image: "rabbitmq:3.9.8" hostname: "queue" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" @@ -126,7 +126,7 @@ services: <<: *service_defaults image: "redis:7.2.4" hostname: "redis" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_REDIS_HOST:-127.0.0.1}" published: "${CLP_REDIS_PORT:-6379}" @@ -162,7 +162,7 @@ services: <<: *service_defaults image: "mongo:7.0.1" hostname: "results_cache" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}" published: "${CLP_RESULTS_CACHE_PORT:-27017}" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index b66757980a..e803fd3cf7 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -9,7 +9,7 @@ x-service-defaults: &service_defaults logging: driver: "local" stop_grace_period: "3s" - user: "${CLP_UID_GID:-1000:1000}" + user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 From f11963c338eb9ce9ab04c3c99b0bbca17685ced3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 04:03:49 -0400 Subject: [PATCH 238/408] Apply suggestions - Use constants for jobs table names. --- .../clp-package-utils/clp_package_utils/controller.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0d0b19ca1f..cde2c9d071 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -13,11 +13,13 @@ from clp_py_utils.clp_config import ( AwsAuthType, CLPConfig, + COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, DB_COMPONENT_NAME, DeploymentType, GARBAGE_COLLECTOR_COMPONENT_NAME, + QUERY_JOBS_TABLE_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, QueryEngine, @@ -343,7 +345,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), "SqlDbClpFilesTableName": files_table_name, "SqlDbClpTablePrefix": table_prefix, - "SqlDbCompressionJobsTableName": "compression_jobs", + "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME, } client_settings_json = self._read_and_update_settings_json( client_settings_json_path, client_settings_json_updates @@ -355,7 +357,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "SqlDbHost": container_clp_config.database.host, "SqlDbPort": container_clp_config.database.port, "SqlDbName": self._clp_config.database.name, - "SqlDbQueryJobsTableName": "query_jobs", + "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME, "MongoDbHost": container_clp_config.results_cache.host, "MongoDbPort": container_clp_config.results_cache.port, "MongoDbName": self._clp_config.results_cache.db_name, From 50dc07579139b72279d013575a43f03894a2efcb Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 15 Oct 2025 10:33:50 -0400 Subject: [PATCH 239/408] Add spider docker files and the use of them --- .../clp_package_utils/controller.py | 8 +- .../package/docker-compose.spider.base.yaml | 247 ++++++++++++++ .../package/docker-compose.spider.yaml | 301 ++++++++---------- 3 files changed, 380 insertions(+), 176 deletions(-) create mode 100644 tools/deployment/package/docker-compose.spider.base.yaml diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 196964900f..a9fe3e7926 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -18,6 +18,7 @@ DB_COMPONENT_NAME, DeploymentType, GARBAGE_COLLECTOR_COMPONENT_NAME, + OrchestrationType, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, QueryEngine, @@ -531,7 +532,12 @@ def start(self): cmd = ["docker", "compose", "--project-name", self._project_name] if deployment_type == DeploymentType.BASE: - cmd += ["--file", "docker-compose.base.yaml"] + if self.clp_config.compression_scheduler.type == OrchestrationType.spider: + cmd += ["--file", "docker-compose.spider.base.yaml"] + else: + cmd += ["--file", "docker-compose.base.yaml"] + if self.clp_config.compression_scheduler.type == OrchestrationType.spider: + cmd += ["--file", "docker-compose.spider.yaml"] cmd += ["up", "--detach"] try: subprocess.run( diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml new file mode 100644 index 0000000000..daf5942ec7 --- /dev/null +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -0,0 +1,247 @@ +name: "clp-package-base" + +# Common service defaults. +x-service-defaults: &service_defaults + stop_grace_period: "3s" + user: "${CLP_UID_GID:-1000:1000}" + +# Common healthcheck defaults. +x-healthcheck-defaults: &healthcheck_defaults + interval: "30s" + retries: 3 + start_interval: "1s" + start_period: "10s" + timeout: "10s" + +secrets: + CLP_DB_PASS_FILE: + environment: "CLP_DB_PASS" + +services: + database: + <<: *service_defaults + container_name: "database" + image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + environment: + MYSQL_DATABASE: "${CLP_DB_NAME}" + MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_USER: "${CLP_DB_USER}" + secrets: + - "CLP_DB_PASS_FILE" + ports: + - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + volumes: + - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" + - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" + - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1", + "-u", "${CLP_DB_USER}", + "--password=${CLP_DB_PASS}" + ] + + db-table-creator: + <<: *service_defaults + container_name: "db_table_creator" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + depends_on: + database: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" + ] + + spider-scheduler: + <<: *service_defaults + container_name: "spider_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" + depends_on: + database: + condition: "service_healthy" + command: [ + "/opt/clp/bin/spider_scheduler", + "--host", "${SPIDER_SCHEDULER_HOST}", + "--port", "${SPIDER_SCHEDULER_PORT}", + "--storage_url", "${SPIDER_DB_URL}" + ] + + results-cache: + <<: *service_defaults + container_name: "results_cache" + image: "mongo:7.0.1" + user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + ports: + - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + volumes: + - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" + - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" + - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" + healthcheck: + <<: *healthcheck_defaults + test: >- + echo 'db.runCommand("ping").ok' | + mongosh 127.0.0.1:27017/test --quiet + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] + + results-cache-indices-creator: + <<: *service_defaults + container_name: "results_cache_indices_creator" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + depends_on: + results-cache: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", + "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", + ] + + compression-scheduler: + <<: *service_defaults + container_name: "compression_scheduler" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + volumes: + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ +:/var/log/compression_scheduler.log" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "/:/mnt/logs:ro" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + spider-scheduler: + condition: "service_started" + command: [ + "python3", + "-u", + "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "--config", "/etc/clp-config.yml" + ] + + compression-worker: + <<: *service_defaults + container_name: "compression_worker" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" + CLP_CONFIG_PATH: "/etc/clp-config.yml" + CLP_HOME: "/opt/clp" + CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/compression_worker" + CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ +/var/log/compression_worker" + - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "/:/mnt/logs:ro" + command: [ + "python3", + "-u", + "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", + "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", + "--storage-url", "${SPIDER_DB_URL}", + # NOTE: Leave host to spider scheduler's host. This only affects task placement. + "--host", "${SPIDER_SCHEDULER_HOST}", + ] + + webui: + <<: *service_defaults + container_name: "webui" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + HOST: "0.0.0.0" + NODE_ENV: "production" + NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" + PORT: "4000" + RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" + ports: + - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + volumes: + - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" + - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" + - "./var/www/webui/server/dist/settings.json\ +:/opt/clp/var/www/webui/server/dist/settings.json:ro" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/src/main.js" + ] + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "bash", + "-c", + "< /dev/tcp/webui/4000" + ] + + garbage-collector: + <<: *service_defaults + container_name: "garbage_collector" + image: "${CLP_PACKAGE_CONTAINER}" + environment: + CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_USER: "${CLP_DB_USER}" + CLP_HOME: "/opt/clp" + CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/garbage_collector" + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + volumes: + - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" + - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" + depends_on: + db-table-creator: + condition: "service_completed_successfully" + results-cache-indices-creator: + condition: "service_completed_successfully" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yml", + ] diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index daf5942ec7..5f34c39fbc 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -1,247 +1,198 @@ -name: "clp-package-base" +name: "clp-package" -# Common service defaults. +include: ["docker-compose.spider.base.yaml"] + +# Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for +# documentation. x-service-defaults: &service_defaults + image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + logging: + driver: "local" stop_grace_period: "3s" user: "${CLP_UID_GID:-1000:1000}" - -# Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 - start_interval: "1s" - start_period: "10s" - timeout: "10s" - -secrets: - CLP_DB_PASS_FILE: - environment: "CLP_DB_PASS" + start_interval: "2s" + start_period: "60s" + timeout: "2s" +x-volume-definitions: + aws-config-readonly: &volume_aws_config_readonly + type: "bind" + source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" + target: "/.aws" + read_only: true + clp-config-readonly: &volume_clp_config_readonly + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" + target: "/etc/clp-config.yml" + read_only: true services: - database: + queue: <<: *service_defaults - container_name: "database" - image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + image: "rabbitmq:3.9.8" + hostname: "queue" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" environment: - MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_USER: "${CLP_DB_USER}" - secrets: - - "CLP_DB_PASS_FILE" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" + published: "${CLP_QUEUE_PORT:-5672}" + target: 5672 volumes: - - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" - - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" + - type: "bind" + source: "${CLP_QUEUE_LOGS_DIR_HOST:-./var/log/queue}" + target: "/var/log/rabbitmq" healthcheck: <<: *healthcheck_defaults test: [ "CMD", - "mysqladmin", "ping", - "--silent", - "-h", "127.0.0.1", - "-u", "${CLP_DB_USER}", - "--password=${CLP_DB_PASS}" + "rabbitmq-diagnostics", "check_running" ] - db-table-creator: - <<: *service_defaults - container_name: "db_table_creator" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" - depends_on: - database: - condition: "service_healthy" - command: [ - "python3", - "-u", - "-m", "clp_py_utils.create-db-tables", - "--config", "/etc/clp-config.yml", - "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" - ] - - spider-scheduler: - <<: *service_defaults - container_name: "spider_scheduler" - image: "${CLP_PACKAGE_CONTAINER}" - depends_on: - database: - condition: "service_healthy" - command: [ - "/opt/clp/bin/spider_scheduler", - "--host", "${SPIDER_SCHEDULER_HOST}", - "--port", "${SPIDER_SCHEDULER_PORT}", - "--storage_url", "${SPIDER_DB_URL}" - ] - - results-cache: + redis: <<: *service_defaults - container_name: "results_cache" - image: "mongo:7.0.1" + image: "redis:7.2.4" + hostname: "redis" user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" ports: - - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" + - host_ip: "${CLP_REDIS_HOST:-127.0.0.1}" + published: "${CLP_REDIS_PORT:-6379}" + target: 6379 volumes: - - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" - - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" + - type: "bind" + source: "${CLP_REDIS_CONF_FILE_HOST:-./etc/redis/redis.conf}" + target: "/usr/local/etc/redis/redis.conf" + read_only: true + - type: "bind" + source: "${CLP_REDIS_DATA_DIR_HOST:-./var/data/redis}" + target: "/data" + - type: "bind" + source: "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}" + target: "/var/log/redis" healthcheck: <<: *healthcheck_defaults - test: >- - echo 'db.runCommand("ping").ok' | - mongosh 127.0.0.1:27017/test --quiet - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] - - results-cache-indices-creator: - <<: *service_defaults - container_name: "results_cache_indices_creator" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - depends_on: - results-cache: - condition: "service_healthy" + test: [ + "CMD", + "redis-cli", + "-h", "127.0.0.1", + "-p", "6379", + "-a", "${CLP_REDIS_PASS}", + "PING" + ] command: [ - "python3", - "-u", - "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", - "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS}" ] - compression-scheduler: + query-scheduler: <<: *service_defaults - container_name: "compression_scheduler" - image: "${CLP_PACKAGE_CONTAINER}" + hostname: "query_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" - CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" + CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ -:/var/log/compression_scheduler.log" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}" + target: "/var/log/query_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" - spider-scheduler: - condition: "service_started" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" command: [ "python3", "-u", - "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "-m", "job_orchestration.scheduler.query.query_scheduler", "--config", "/etc/clp-config.yml" ] + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD", + "bash", + "-c", + "< /dev/tcp/query_scheduler/7000" + ] - compression-worker: + query-worker: <<: *service_defaults - container_name: "compression_worker" - image: "${CLP_PACKAGE_CONTAINER}" + hostname: "query_worker" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" + BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" - CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log/compression_worker" - CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" + CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/query_worker" + CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" + RESULT_BACKEND: >- + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} volumes: - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ -/var/log/compression_worker" - - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" + target: "/var/data/archives" + - type: "bind" + source: "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}" + target: "/var/log/query_worker" + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}" + target: "/var/data/staged-streams" + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" + target: "/var/data/streams" command: [ "python3", "-u", - "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", - "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--storage-url", "${SPIDER_DB_URL}", - # NOTE: Leave host to spider scheduler's host. This only affects task placement. - "--host", "${SPIDER_SCHEDULER_HOST}", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.query", + "worker", + "--concurrency", "${CLP_QUERY_WORKER_CONCURRENCY:-1}", + "--loglevel", "WARNING", + "-f", "/var/log/query_worker/worker.log", + "-Q", "query", + "-n", "query-worker" ] - webui: + reducer: <<: *service_defaults - container_name: "webui" - image: "${CLP_PACKAGE_CONTAINER}" + hostname: "reducer" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" - HOST: "0.0.0.0" - NODE_ENV: "production" - NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" - PORT: "4000" - RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" - ports: - - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" - volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/settings.json\ -:/opt/clp/var/www/webui/server/dist/settings.json:ro" - depends_on: - db-table-creator: - condition: "service_completed_successfully" - results-cache-indices-creator: - condition: "service_completed_successfully" - command: [ - "/opt/clp/bin/node-22", - "/opt/clp/var/www/webui/server/dist/src/main.js" - ] - healthcheck: - <<: *healthcheck_defaults - test: [ - "CMD", - "bash", - "-c", - "< /dev/tcp/webui/4000" - ] - - garbage-collector: - <<: *service_defaults - container_name: "garbage_collector" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" CLP_HOME: "/opt/clp" - CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log/garbage_collector" + CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}" + CLP_LOGS_DIR: "/var/log/reducer" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}" + target: "/var/log/reducer" depends_on: - db-table-creator: - condition: "service_completed_successfully" + query-scheduler: + condition: "service_healthy" results-cache-indices-creator: condition: "service_completed_successfully" command: [ "python3", "-u", - "-m", "job_orchestration.garbage_collector.garbage_collector", + "-m", "job_orchestration.reducer.reducer", "--config", "/etc/clp-config.yml", + "--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}", + "--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}" ] From 2fc6a72276dd17edb14d69c56b24850cf2420389 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 15 Oct 2025 12:03:21 -0400 Subject: [PATCH 240/408] Bug fix --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index be853963fd..b02816a84e 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -138,7 +138,7 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: queue is not configured. """ component_name = QUEUE_COMPONENT_NAME - if self.clp_config.queue is None: + if self._clp_config.queue is None: logger.info(f"{component_name} is not configured, skipping setup.") return {} logger.info(f"Setting up environment for {component_name}...") From 3a6dcd8fd468274d1f996be76b73f5ee9a59f63c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 15 Oct 2025 12:08:46 -0400 Subject: [PATCH 241/408] Fix private var --- .../clp_package_utils/controller.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index b02816a84e..b3155487a3 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -165,7 +165,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: Redis is not configured. """ component_name = REDIS_COMPONENT_NAME - if self.clp_config.redis is None: + if self._clp_config.redis is None: logger.info(f"{component_name} is not configured, skipping setup.") return {} logger.info(f"Setting up environment for {component_name}...") @@ -200,15 +200,15 @@ def _set_up_env_for_spider_db(self) -> EnvVarsDict: configured. """ component_name = "spider_db" - if self.clp_config.spider_db is None: + if self._clp_config.spider_db is None: logger.info(f"{component_name} is not configured, skipping setup.") return {} logger.info(f"Setting up environment for {component_name}...") return { - "SPIDER_DB_USER": self.clp_config.spider_db.username, - "SPIDER_DB_PASS": self.clp_config.spider_db.password, - "SPIDER_DB_URL": self.clp_config.spider_db.get_url(), + "SPIDER_DB_USER": self._clp_config.spider_db.username, + "SPIDER_DB_PASS": self._clp_config.spider_db.password, + "SPIDER_DB_URL": self._clp_config.spider_db.get_url(), } def _set_up_env_for_spider_scheduler(self) -> EnvVarsDict: @@ -219,14 +219,14 @@ def _set_up_env_for_spider_scheduler(self) -> EnvVarsDict: is not configured. """ component_name = SPIDER_SCHEDULER_COMPONENT_NAME - if self.clp_config.spider_scheduler is None: + if self._clp_config.spider_scheduler is None: logger.info(f"{component_name} is not configured, skipping setup.") return {} logger.info(f"Setting up environment for {component_name}...") return { - "SPIDER_SCHEDULER_HOST": _get_ip_from_hostname(self.clp_config.spider_scheduler.host), - "SPIDER_SCHEDULER_PORT": str(self.clp_config.spider_scheduler.port), + "SPIDER_SCHEDULER_HOST": _get_ip_from_hostname(self._clp_config.spider_scheduler.host), + "SPIDER_SCHEDULER_PORT": str(self._clp_config.spider_scheduler.port), } def _set_up_env_for_results_cache(self) -> EnvVarsDict: @@ -532,11 +532,11 @@ def start(self): cmd = ["docker", "compose", "--project-name", self._project_name] if deployment_type == DeploymentType.BASE: - if self.clp_config.compression_scheduler.type == OrchestrationType.spider: + if self._clp_config.compression_scheduler.type == OrchestrationType.spider: cmd += ["--file", "docker-compose.spider.base.yaml"] else: cmd += ["--file", "docker-compose.base.yaml"] - if self.clp_config.compression_scheduler.type == OrchestrationType.spider: + if self._clp_config.compression_scheduler.type == OrchestrationType.spider: cmd += ["--file", "docker-compose.spider.yaml"] cmd += ["up", "--detach"] try: From 8253a1d2d062a7f5e3462bc899c78ef3695a6aec Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 15 Oct 2025 12:40:42 -0400 Subject: [PATCH 242/408] Update docker files --- .../package/docker-compose.spider.base.yaml | 189 +++++++++--------- .../package/docker-compose.spider.yaml | 60 +++++- 2 files changed, 154 insertions(+), 95 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index daf5942ec7..58f089a929 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -2,40 +2,72 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults + image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + logging: + driver: "local" stop_grace_period: "3s" - user: "${CLP_UID_GID:-1000:1000}" + user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" # Common healthcheck defaults. x-healthcheck-defaults: &healthcheck_defaults + # Avoid lowering to prevent excessive resource usage. interval: "30s" + # Mark unhealthy after 3 failed probes. + # - In steady state, ( + ) × 3 = ~90s before the service is marked unhealthy. + # - From startup, (60s) + ~90s = ~150s before the service is marked unhealthy. retries: 3 - start_interval: "1s" - start_period: "10s" - timeout: "10s" + # Frequent checks during startup allow fast transition to healthy. + start_interval: "2s" + # Ignore failures for ~15 frequent checks before counting retries. + start_period: "60s" + # Short timeout since no remote communication is expected. + timeout: "2s" + +# Common volume definitions. +x-volume-definitions: + aws-config-readonly: &volume_aws_config_readonly + type: "bind" + source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" + target: "/.aws" + read_only: true + clp-config-readonly: &volume_clp_config_readonly + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" + target: "/etc/clp-config.yml" + read_only: true + logs-input-readonly: &volume_root_logs_readonly + type: "bind" + source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" + target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" + read_only: true -secrets: - CLP_DB_PASS_FILE: - environment: "CLP_DB_PASS" services: database: <<: *service_defaults - container_name: "database" image: "${CLP_DB_IMAGE:-mysql:8.0.23}" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + hostname: "database" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" - MYSQL_ROOT_PASSWORD_FILE: "/run/secrets/CLP_DB_PASS_FILE" + MYSQL_PASSWORD: "${CLP_DB_PASS}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS}" MYSQL_USER: "${CLP_DB_USER}" - secrets: - - "CLP_DB_PASS_FILE" ports: - - "${CLP_DB_HOST:-127.0.0.1}:${CLP_DB_PORT:-3306}:3306" + - host_ip: "${CLP_DB_HOST:-127.0.0.1}" + published: "${CLP_DB_PORT:-3306}" + target: 3306 volumes: - - "${CLP_DB_CONF_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}:/etc/mysql/conf.d/logging.cnf:ro" - - "${CLP_DB_DATA_DIR_HOST:-./var/data/database}:/var/lib/mysql" - - "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}:/var/log/mysql" + - type: "bind" + source: "${CLP_DB_CONF_LOGGING_FILE_HOST:-./etc/mysql/conf.d/logging.cnf}" + target: "/etc/mysql/conf.d/logging.cnf" + read_only: true + - type: "bind" + source: "${CLP_DB_DATA_DIR_HOST:-./var/data/database}" + target: "/var/lib/mysql" + - type: "bind" + source: "${CLP_DB_LOGS_DIR_HOST:-./var/log/database}" + target: "/var/log/mysql" healthcheck: <<: *healthcheck_defaults test: [ @@ -49,14 +81,13 @@ services: db-table-creator: <<: *service_defaults - container_name: "db_table_creator" - image: "${CLP_PACKAGE_CONTAINER}" + hostname: "db_table_creator" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml" + - *volume_clp_config_readonly depends_on: database: condition: "service_healthy" @@ -65,9 +96,10 @@ services: "-u", "-m", "clp_py_utils.create-db-tables", "--config", "/etc/clp-config.yml", - "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE}" + "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE:-clp}", ] + spider-scheduler: <<: *service_defaults container_name: "spider_scheduler" @@ -82,48 +114,9 @@ services: "--storage_url", "${SPIDER_DB_URL}" ] - results-cache: - <<: *service_defaults - container_name: "results_cache" - image: "mongo:7.0.1" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" - ports: - - "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}:${CLP_RESULTS_CACHE_PORT:-6379}:27017" - volumes: - - "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}:/etc/mongo/mongod.conf:ro" - - "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}:/data/db" - - "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}:/var/log/mongodb" - healthcheck: - <<: *healthcheck_defaults - test: >- - echo 'db.runCommand("ping").ok' | - mongosh 127.0.0.1:27017/test --quiet - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] - - results-cache-indices-creator: - <<: *service_defaults - container_name: "results_cache_indices_creator" - image: "${CLP_PACKAGE_CONTAINER}" - environment: - PYTHONPATH: "/opt/clp/lib/python3/site-packages" - depends_on: - results-cache: - condition: "service_healthy" - command: [ - "python3", - "-u", - "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://results_cache:27017/${RESULTS_CACHE_CLP_DB_NAME:-clp-query-results}", - "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", - ] - compression-scheduler: <<: *service_defaults - container_name: "compression_scheduler" - image: "${CLP_PACKAGE_CONTAINER}" + hostname: "compression_scheduler" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS}" @@ -132,18 +125,31 @@ services: CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_COMPRESSION_BACKEND_DB:-1} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_SCHEDULER_LOGS_FILE_HOST:-./var/log/compression_scheduler.log}\ -:/var/log/compression_scheduler.log" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - *volume_root_logs_readonly + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" + target: "/var/data/archives" + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" + target: "/var/data/staged-archives" + - type: "bind" + source: "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}" + target: "/var/log/compression_worker" + - type: "bind" + source: "${CLP_DATA_DIR_HOST:-./var/data}" + target: "/var/data" + depends_on: db-table-creator: condition: "service_completed_successfully" - spider-scheduler: - condition: "service_started" + queue: + condition: "service_healthy" + redis: + condition: "service_healthy" command: [ "python3", "-u", @@ -165,14 +171,9 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}:/var/data/archives" - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}:/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}:\ -/var/log/compression_worker" - - "${CLP_DATA_DIR_HOST:-./var/data}:/var/data" - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "/:/mnt/logs:ro" + - *volume_aws_config_readonly + - *volume_clp_config_readonly + - *volume_root_logs_readonly command: [ "python3", "-u", @@ -185,8 +186,7 @@ services: webui: <<: *service_defaults - container_name: "webui" - image: "${CLP_PACKAGE_CONTAINER}" + hostname: "webui" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" @@ -196,13 +196,22 @@ services: PORT: "4000" RATE_LIMIT: "${CLP_WEBUI_RATE_LIMIT:-1000}" ports: - - "${CLP_WEBUI_HOST:-127.0.0.1}:${CLP_WEBUI_PORT:-4000}:4000" + - host_ip: "${CLP_WEBUI_HOST:-127.0.0.1}" + published: "${CLP_WEBUI_PORT:-4000}" + target: 4000 volumes: - - "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}:/.aws:ro" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}:/var/data/streams" - - "./var/www/webui/client/settings.json:/opt/clp/var/www/webui/client/settings.json:ro" - - "./var/www/webui/server/dist/settings.json\ -:/opt/clp/var/www/webui/server/dist/settings.json:ro" + - *volume_aws_config_readonly + - type: "bind" + source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" + target: "/var/data/streams" + - type: "bind" + source: "./var/www/webui/client/settings.json" + target: "/opt/clp/var/www/webui/client/settings.json" + read_only: true + - type: "bind" + source: "./var/www/webui/server/dist/settings.json" + target: "/opt/clp/var/www/webui/server/dist/settings.json" + read_only: true depends_on: db-table-creator: condition: "service_completed_successfully" @@ -223,8 +232,7 @@ services: garbage-collector: <<: *service_defaults - container_name: "garbage_collector" - image: "${CLP_PACKAGE_CONTAINER}" + hostname: "garbage_collector" environment: CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" @@ -233,8 +241,10 @@ services: CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml:/etc/clp-config.yml:ro" - - "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector:/var/log/garbage_collector" + - *volume_clp_config_readonly + - type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector" + target: "/var/log/garbage_collector" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -245,3 +255,4 @@ services: "-m", "job_orchestration.garbage_collector.garbage_collector", "--config", "/etc/clp-config.yml", ] + diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index 5f34c39fbc..66eb7ecd4e 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -1,6 +1,6 @@ name: "clp-package" -include: ["docker-compose.spider.base.yaml"] +include: ["docker-compose.base.yaml"] # Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for # documentation. @@ -9,7 +9,7 @@ x-service-defaults: &service_defaults logging: driver: "local" stop_grace_period: "3s" - user: "${CLP_UID_GID:-1000:1000}" + user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" x-healthcheck-defaults: &healthcheck_defaults interval: "30s" retries: 3 @@ -28,12 +28,13 @@ x-volume-definitions: target: "/etc/clp-config.yml" read_only: true + services: queue: <<: *service_defaults image: "rabbitmq:3.9.8" hostname: "queue" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" @@ -57,7 +58,7 @@ services: <<: *service_defaults image: "redis:7.2.4" hostname: "redis" - user: "${CLP_SERVICE_CONTAINER_UID_GID:-1000:1000}" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" ports: - host_ip: "${CLP_REDIS_HOST:-127.0.0.1}" published: "${CLP_REDIS_PORT:-6379}" @@ -89,6 +90,53 @@ services: "--requirepass", "${CLP_REDIS_PASS}" ] + results-cache: + <<: *service_defaults + image: "mongo:7.0.1" + hostname: "results_cache" + user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" + ports: + - host_ip: "${CLP_RESULTS_CACHE_HOST:-127.0.0.1}" + published: "${CLP_RESULTS_CACHE_PORT:-27017}" + target: 27017 + volumes: + - type: "bind" + source: "${CLP_RESULTS_CACHE_CONF_FILE_HOST:-./etc/mongo/mongod.conf}" + target: "/etc/mongo/mongod.conf" + read_only: true + - type: "bind" + source: "${CLP_RESULTS_CACHE_DATA_DIR_HOST:-./var/data/results_cache}" + target: "/data/db" + - type: "bind" + source: "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}" + target: "/var/log/mongodb" + healthcheck: + <<: *healthcheck_defaults + test: [ + "CMD-SHELL", + "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:27017/test --quiet" + ] + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] + + results-cache-indices-creator: + <<: *service_defaults + hostname: "results_cache_indices_creator" + environment: + PYTHONPATH: "/opt/clp/lib/python3/site-packages" + depends_on: + results-cache: + condition: "service_healthy" + command: [ + "python3", + "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://results_cache:27017/${CLP_RESULTS_CACHE_DB_NAME:-clp-query-results}", + "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", + ] + query-scheduler: <<: *service_defaults hostname: "query_scheduler" @@ -100,7 +148,7 @@ services: CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - type: "bind" @@ -142,7 +190,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_QUERY_BACKEND_DB:-0} + redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly From b7b4b7cb3ff90521851035289dcd6dde7d4d1aed Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 15 Oct 2025 12:47:40 -0400 Subject: [PATCH 243/408] Fix docker --- tools/deployment/package/docker-compose.spider.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index 66eb7ecd4e..82bfc88eb3 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -1,6 +1,6 @@ name: "clp-package" -include: ["docker-compose.base.yaml"] +include: ["docker-compose.spider.base.yaml"] # Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for # documentation. From ad957841fb0aeef312e731ec83f063c3ae8ee35e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 15 Oct 2025 12:59:38 -0400 Subject: [PATCH 244/408] Fix storage url --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 324ddeb289..e420ddb595 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -263,7 +263,7 @@ def validate_type(cls, value): def get_url(self): self.ensure_credentials_loaded() - return f"mariadb://{self.host}:{self.port}/{self.name}?user={self.username}&password={self.password}" + return f"jdbc:mariadb://{self.host}:{self.port}/{self.name}?user={self.username}&password={self.password}" class SpiderScheduler(BaseModel): From d79b58e4da594b183cb56dacded4df6645226e01 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 15 Oct 2025 14:29:33 -0400 Subject: [PATCH 245/408] Fix docker file --- .../package/docker-compose.spider.base.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index 58f089a929..edc33053b6 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -105,8 +105,8 @@ services: container_name: "spider_scheduler" image: "${CLP_PACKAGE_CONTAINER}" depends_on: - database: - condition: "service_healthy" + db-table-creator: + condition: "service_completed_successfully" command: [ "/opt/clp/bin/spider_scheduler", "--host", "${SPIDER_SCHEDULER_HOST}", @@ -142,14 +142,9 @@ services: - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" - depends_on: db-table-creator: condition: "service_completed_successfully" - queue: - condition: "service_healthy" - redis: - condition: "service_healthy" command: [ "python3", "-u", @@ -174,10 +169,13 @@ services: - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_root_logs_readonly + depends_on: + db-table-creator: + condition: "service_completed_successfully" command: [ "python3", "-u", - "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", + "-m", "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", "--storage-url", "${SPIDER_DB_URL}", # NOTE: Leave host to spider scheduler's host. This only affects task placement. From 91cb9fc805df0d33c663e6ed2827501a62da9c5d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 15:44:52 -0400 Subject: [PATCH 246/408] refactor(controller): Remove redundant `stderr` redirection from subprocess commands --- components/clp-package-utils/clp_package_utils/controller.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index cde2c9d071..dd977c11bd 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -491,7 +491,6 @@ def start(self): subprocess.run( cmd, cwd=self._clp_home, - stderr=subprocess.STDOUT, check=True, ) except subprocess.CalledProcessError: @@ -509,7 +508,6 @@ def stop(self): subprocess.run( ["docker", "compose", "--project-name", self._project_name, "down"], cwd=self._clp_home, - stderr=subprocess.STDOUT, check=True, ) logger.info("All CLP containers stopped.") From 1d57a8986b6c30e30cd88cbf5ee8a521b8eb3941 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 15:49:51 -0400 Subject: [PATCH 247/408] docs: move high-level comments to individual blocks --- .../clp-package-utils/clp_package_utils/controller.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index dd977c11bd..f549af7084 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -524,16 +524,12 @@ def _get_num_workers() -> int: return multiprocessing.cpu_count() // 2 def _set_up_env(self): - """ - Sets up all CLP components for Docker Compose by: - - Generating container-specific config. - - Preparing environment variables for all components. - - Writing environment variables to `.env`. - """ + # Generate container-specific config. container_clp_config = generate_docker_compose_container_config(self._clp_config) num_workers = self._get_num_workers() dump_shared_container_config(container_clp_config, self._clp_config) + # Prepare environment variables for all components. env_dict = { "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, # User and group IDs @@ -572,6 +568,7 @@ def _set_up_env(self): if self._clp_config.aws_config_directory is not None: env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self._clp_config.aws_config_directory) + # Write the environment variables to the `.env ` file. with open(f"{self._clp_home}/.env", "w") as env_file: for key, value in env_dict.items(): env_file.write(f"{key}={value}\n") From f4254fa78b5518d789ce17752da2a9aeb3b9631f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 15:52:13 -0400 Subject: [PATCH 248/408] docs(controller): reference issue for revisiting worker count logic --- components/clp-package-utils/clp_package_utils/controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index f549af7084..c4cf248d6f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -518,7 +518,8 @@ def stop(self): @staticmethod def _get_num_workers() -> int: """ - TODO: Revisit after moving from single-container to multi-container workers. + TODO: Revisit after moving from single-container to multi-container workers. See issue + @y-scope/clp#1424 for details. :return: Number of worker processes to run. """ return multiprocessing.cpu_count() // 2 From 932c3343ce3de59b8f547d904f4ffd8f2d091c12 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 16:03:22 -0400 Subject: [PATCH 249/408] refactor(controller): add `--wait` flag to `docker compose up` command to wait for all services become healthy before existing --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index c4cf248d6f..e7bedaef8b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -486,7 +486,7 @@ def start(self): cmd = ["docker", "compose", "--project-name", self._project_name] if deployment_type == DeploymentType.BASE: cmd += ["--file", "docker-compose.base.yaml"] - cmd += ["up", "--detach"] + cmd += ["up", "--detach", "--wait"] try: subprocess.run( cmd, From a8cee64234aef8bd1b4bf650f827d0073f9e03bb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 20:29:01 -0400 Subject: [PATCH 250/408] refactor(controller): replace `Dict` with a custom `EnvVarsDict` for consistent type-safe environment variable management; group the env vars; replace `Dict` usages with `dict`. --- .../clp_package_utils/controller.py | 266 +++++++++++++----- 1 file changed, 197 insertions(+), 69 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e7bedaef8b..0a2b1e3fd2 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -8,7 +8,7 @@ import subprocess import uuid from abc import ABC, abstractmethod -from typing import Any, Dict +from typing import Any, Optional from clp_py_utils.clp_config import ( AwsAuthType, @@ -50,8 +50,6 @@ validate_webui_config, ) -EnvVarsDict = Dict[str, str] - LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH DEFAULT_UID_GID = f"{os.getuid()}:{os.getgid()}" @@ -62,6 +60,15 @@ logger = logging.getLogger(__name__) +class EnvVarsDict(dict[str, Optional[str]]): + def __ior__(self, other: "EnvVarsDict") -> "EnvVarsDict": + """ + Overloads the `|=` operator for static type checking on `other`. + """ + super().__ior__(other) + return self + + class BaseController(ABC): """ Base controller for orchestrating CLP components. Derived classes should implement any @@ -114,20 +121,33 @@ def _set_up_env_for_database(self) -> EnvVarsDict: logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) - return { - "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), - "CLP_DB_DATA_DIR_HOST": str(data_dir), - "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + env_vars = EnvVarsDict() + # Connection + env_vars |= { "CLP_DB_HOST": _get_ip_from_hostname(self._clp_config.database.host), - "CLP_DB_PORT": str(self._clp_config.database.port), "CLP_DB_NAME": self._clp_config.database.name, - "CLP_DB_USER": self._clp_config.database.username, + "CLP_DB_PORT": str(self._clp_config.database.port), + } + # Credential + env_vars |= { "CLP_DB_PASS": self._clp_config.database.password, + "CLP_DB_USER": self._clp_config.database.username, + } + # Path + env_vars |= { + "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), + "CLP_DB_DATA_DIR_HOST": str(data_dir), + "CLP_DB_LOGS_DIR_HOST": str(logs_dir), + } + # Runtime + env_vars |= { "CLP_DB_IMAGE": ( - "mysql:8.0.23" if "mysql" == self._clp_config.database.type else "mariadb:10-jammy" + "mysql:8.0.23" if self._clp_config.database.type == "mysql" else "mariadb:10-jammy" ), } + return env_vars + def _set_up_env_for_queue(self) -> EnvVarsDict: """ Sets up environment variables and directories for the message queue component. @@ -143,13 +163,23 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(logs_dir) - return { - "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + env_vars = EnvVarsDict() + # Connection + env_vars |= { "CLP_QUEUE_HOST": _get_ip_from_hostname(self._clp_config.queue.host), "CLP_QUEUE_PORT": str(self._clp_config.queue.port), + } + # Credential + env_vars |= { "CLP_QUEUE_USER": self._clp_config.queue.username, "CLP_QUEUE_PASS": self._clp_config.queue.password, } + # Path + env_vars |= { + "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), + } + + return env_vars def _set_up_env_for_redis(self) -> EnvVarsDict: """ @@ -169,19 +199,32 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) - return { - "CLP_REDIS_CONF_FILE_HOST": str(conf_file), - "CLP_REDIS_DATA_DIR_HOST": str(data_dir), - "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), - "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), - "CLP_REDIS_PORT": str(self._clp_config.redis.port), - "CLP_REDIS_PASS": self._clp_config.redis.password, - "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), + env_vars = EnvVarsDict() + # Backend + env_vars |= { "CLP_REDIS_BACKEND_DB_COMPRESSION": str( self._clp_config.redis.compression_backend_database ), + "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), + } + # Connection + env_vars |= { + "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), + "CLP_REDIS_PORT": str(self._clp_config.redis.port), + } + # Credential + env_vars |= { + "CLP_REDIS_PASS": self._clp_config.redis.password, + } + # Path + env_vars |= { + "CLP_REDIS_CONF_FILE_HOST": str(conf_file), + "CLP_REDIS_DATA_DIR_HOST": str(data_dir), + "CLP_REDIS_LOGS_DIR_HOST": str(logs_dir), } + return env_vars + def _set_up_env_for_results_cache(self) -> EnvVarsDict: """ Sets up environment variables and directories for the results cache (MongoDB) component. @@ -200,17 +243,27 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(data_dir, logs_dir) - return { - "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), - "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), - "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), - "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), - "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), - "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, + env_vars = EnvVarsDict() + # Collection + env_vars |= { "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": ( self._clp_config.results_cache.stream_collection_name ), } + # Connection + env_vars |= { + "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, + "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), + "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), + } + # Path + env_vars |= { + "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), + "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), + "CLP_RESULTS_CACHE_LOGS_DIR_HOST": str(logs_dir), + } + + return env_vars def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: """ @@ -224,11 +277,20 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: log_file = self._clp_config.logs_directory / f"{component_name}.log" log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) - return { - "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": self._clp_config.compression_scheduler.logging_level, + env_vars = EnvVarsDict() + # Logging + env_vars |= { + "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": ( + self._clp_config.compression_scheduler.logging_level + ), + } + # Path + env_vars |= { "CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST": str(log_file), } + return env_vars + def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: """ Sets up environment variables and files for the query scheduler component. @@ -241,11 +303,18 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: log_file = self._clp_config.logs_directory / f"{component_name}.log" log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) - return { + env_vars = EnvVarsDict() + # Logging + env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, + } + # Path + env_vars |= { "CLP_QUERY_SCHEDULER_LOG_FILE_HOST": str(log_file), } + return env_vars + def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: """ Sets up environment variables for the compression worker component. @@ -259,11 +328,23 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return { - "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), - "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": self._clp_config.compression_worker.logging_level, + env_vars = EnvVarsDict() + # Logging + env_vars |= { + "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": ( + self._clp_config.compression_worker.logging_level + ), + } + # Path + env_vars |= { "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), } + # Resource + env_vars |= { + "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), + } + + return env_vars def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: """ @@ -278,12 +359,22 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return { + env_vars = EnvVarsDict() + # Logging + env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, + } + # Path + env_vars |= { "CLP_QUERY_WORKER_LOGS_DIR_HOST": str(logs_dir), + } + # Resource + env_vars |= { "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } + return env_vars + def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: """ Sets up environment variables for the reducer component. @@ -297,13 +388,23 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return { + env_vars = EnvVarsDict() + # Logging + env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, + } + # Path + env_vars |= { "CLP_REDUCER_LOGS_DIR_HOST": str(logs_dir), + } + # Resource + env_vars |= { "CLP_REDUCER_CONCURRENCY": str(num_workers), "CLP_REDUCER_UPSERT_INTERVAL": str(self._clp_config.reducer.upsert_interval), } + return env_vars + def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: """ Sets up environment variables and settings for the Web UI component. @@ -403,12 +504,19 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: with open(server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) - return { + env_vars = EnvVarsDict() + # Connection + env_vars |= { "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), "CLP_WEBUI_PORT": str(self._clp_config.webui.port), + } + # Security + env_vars |= { "CLP_WEBUI_RATE_LIMIT": str(self._clp_config.webui.rate_limit), } + return env_vars + def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: """ Sets up environment variables for the garbage collector component. @@ -421,11 +529,16 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name logs_dir.mkdir(parents=True, exist_ok=True) - return {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} + env_vars = EnvVarsDict() + + # Logging + env_vars |= {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} + + return env_vars def _read_and_update_settings_json( - self, settings_file_path: pathlib.Path, updates: Dict[str, Any] - ) -> Dict[str, Any]: + self, settings_file_path: pathlib.Path, updates: dict[str, Any] + ) -> dict[str, Any]: """ Reads and updates a settings JSON file. @@ -441,8 +554,8 @@ def _read_and_update_settings_json( def _update_settings_object( self, parent_key_prefix: str, - settings: Dict[str, Any], - updates: Dict[str, Any], + settings: dict[str, Any], + updates: dict[str, Any], ): """ Recursively updates the given settings object with the values from `updates`. @@ -531,47 +644,62 @@ def _set_up_env(self): dump_shared_container_config(container_clp_config, self._clp_config) # Prepare environment variables for all components. - env_dict = { - "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, - # User and group IDs + env_vars = EnvVarsDict() + + # Credential + env_vars |= { + "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), + "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), + } + + # Identity + env_vars |= { "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID, "CLP_THIRD_PARTY_SERVICE_UID_GID": ( THIRD_PARTY_SERVICE_UID_GID if os.geteuid() == 0 else DEFAULT_UID_GID ), - # Package container + } + + # Package + env_vars |= { "CLP_PACKAGE_CONTAINER": self._clp_config.container_image_ref, - # Runtime data directories + "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, + } + + # Path + aws_config_dir = self._clp_config.aws_config_directory + env_vars |= { + # General "CLP_DATA_DIR_HOST": str(self._clp_config.data_directory), "CLP_LOGS_DIR_HOST": str(self._clp_config.logs_directory), - # Input directories - "CLP_LOGS_INPUT_DIR_HOST": str(self._clp_config.logs_input.directory), + # Config + "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), + # Input "CLP_LOGS_INPUT_DIR_CONTAINER": str(container_clp_config.logs_input.directory), - # Output directories + "CLP_LOGS_INPUT_DIR_HOST": str(self._clp_config.logs_input.directory), + # Output "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self._clp_config.archive_output.get_directory()), "CLP_STREAM_OUTPUT_DIR_HOST": str(self._clp_config.stream_output.get_directory()), - # AWS credentials - "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), - "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - # Component-specific environment variables - **self._set_up_env_for_database(), - **self._set_up_env_for_queue(), - **self._set_up_env_for_redis(), - **self._set_up_env_for_results_cache(), - **self._set_up_env_for_compression_scheduler(), - **self._set_up_env_for_query_scheduler(), - **self._set_up_env_for_compression_worker(num_workers), - **self._set_up_env_for_query_worker(num_workers), - **self._set_up_env_for_reducer(num_workers), - **self._set_up_env_for_webui(container_clp_config), - **self._set_up_env_for_garbage_collector(), - } - - if self._clp_config.aws_config_directory is not None: - env_dict["CLP_AWS_CONFIG_DIR_HOST"] = str(self._clp_config.aws_config_directory) + } + + # Component-specific + env_vars |= self._set_up_env_for_database() + env_vars |= self._set_up_env_for_queue() + env_vars |= self._set_up_env_for_redis() + env_vars |= self._set_up_env_for_results_cache() + env_vars |= self._set_up_env_for_compression_scheduler() + env_vars |= self._set_up_env_for_query_scheduler() + env_vars |= self._set_up_env_for_compression_worker(num_workers) + env_vars |= self._set_up_env_for_query_worker(num_workers) + env_vars |= self._set_up_env_for_reducer(num_workers) + env_vars |= self._set_up_env_for_webui(container_clp_config) + env_vars |= self._set_up_env_for_garbage_collector() # Write the environment variables to the `.env ` file. with open(f"{self._clp_home}/.env", "w") as env_file: - for key, value in env_dict.items(): + for key, value in env_vars.items(): + if value is None: + continue env_file.write(f"{key}={value}\n") From f0e174040a7bcdd455bd4c82affa3bf7e9c7d8cc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 20:30:22 -0400 Subject: [PATCH 251/408] lint --- .../clp_package_utils/controller.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0a2b1e3fd2..c31ffc444d 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -441,7 +441,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: client_settings_json_updates = { "ClpStorageEngine": self._clp_config.package.storage_engine, "ClpQueryEngine": self._clp_config.package.query_engine, - "MongoDbSearchResultsMetadataCollectionName": self._clp_config.webui.results_metadata_collection_name, + "MongoDbSearchResultsMetadataCollectionName": ( + self._clp_config.webui.results_metadata_collection_name + ), "SqlDbClpArchivesTableName": archives_table_name, "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), "SqlDbClpFilesTableName": files_table_name, @@ -462,8 +464,12 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "MongoDbHost": container_clp_config.results_cache.host, "MongoDbPort": container_clp_config.results_cache.port, "MongoDbName": self._clp_config.results_cache.db_name, - "MongoDbSearchResultsMetadataCollectionName": self._clp_config.webui.results_metadata_collection_name, - "MongoDbStreamFilesCollectionName": self._clp_config.results_cache.stream_collection_name, + "MongoDbSearchResultsMetadataCollectionName": ( + self._clp_config.webui.results_metadata_collection_name + ), + "MongoDbStreamFilesCollectionName": ( + self._clp_config.results_cache.stream_collection_name + ), "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), "StreamTargetUncompressedSize": self._clp_config.stream_output.target_uncompressed_size, From 6679c2dad4d4bf9299ec70bea0a85c1d420d3f32 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 20:34:06 -0400 Subject: [PATCH 252/408] use local variable in assignment in _update_settings_object() --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index c31ffc444d..3e18b47d46 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -580,7 +580,7 @@ def _update_settings_object( if isinstance(value, dict): self._update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) else: - settings[key] = updates[key] + settings[key] = value class DockerComposeController(BaseController): From 416c31ae61d805085be26a2ef73f6c09fa7a7cd6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 20:40:01 -0400 Subject: [PATCH 253/408] refactor(controller): remove redundant try-except blocks around `subprocess.run` and update docstrings accordingly --- .../clp_package_utils/controller.py | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 3e18b47d46..566ecd54b1 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -595,6 +595,8 @@ def __init__(self, clp_config: CLPConfig, instance_id: str): def start(self): """ Starts CLP's components using Docker Compose. + + :raise: Propagates `subprocess.run`'s exceptions. """ check_docker_dependencies(should_compose_run=False, project_name=self._project_name) self._set_up_env() @@ -606,33 +608,28 @@ def start(self): if deployment_type == DeploymentType.BASE: cmd += ["--file", "docker-compose.base.yaml"] cmd += ["up", "--detach", "--wait"] - try: - subprocess.run( - cmd, - cwd=self._clp_home, - check=True, - ) - except subprocess.CalledProcessError: - logger.exception("Failed to start CLP.") - raise + subprocess.run( + cmd, + cwd=self._clp_home, + check=True, + ) + logger.info("CLP is started.") def stop(self): """ Stops CLP components deployed via Docker Compose. + + :raise: Propagates `subprocess.run`'s exceptions. """ check_docker_dependencies(should_compose_run=True, project_name=self._project_name) logger.info("Stopping all CLP containers using Docker Compose...") - try: - subprocess.run( - ["docker", "compose", "--project-name", self._project_name, "down"], - cwd=self._clp_home, - check=True, - ) - logger.info("All CLP containers stopped.") - except subprocess.CalledProcessError: - logger.exception("Failed to stop CLP containers using Docker Compose.") - raise + subprocess.run( + ["docker", "compose", "--project-name", self._project_name, "down"], + cwd=self._clp_home, + check=True, + ) + logger.info("All CLP containers stopped.") @staticmethod def _get_num_workers() -> int: From 9ab85d2f05917802f96ff3b126409a3acd63ed37 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:00:15 -0400 Subject: [PATCH 254/408] add warning log for failed Docker dependency check before attempting container stop --- .../clp_package_utils/controller.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 566ecd54b1..424b6bc66c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -621,9 +621,17 @@ def stop(self): :raise: Propagates `subprocess.run`'s exceptions. """ - check_docker_dependencies(should_compose_run=True, project_name=self._project_name) + try: + check_docker_dependencies(should_compose_run=True, project_name=self._project_name) + except EnvironmentError as e: + logger.warning( + 'Docker dependencies check failed: "%s". Attempting to stop CLP containers ' + "anyway...", + e, + ) + else: + logger.info("Stopping all CLP containers using Docker Compose...") - logger.info("Stopping all CLP containers using Docker Compose...") subprocess.run( ["docker", "compose", "--project-name", self._project_name, "down"], cwd=self._clp_home, From 1345de36c807d2849774fbdf0fc1f2468a504d1d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:08:44 -0400 Subject: [PATCH 255/408] refactor(controller): add explicit return type annotations for all methods and functions --- .../clp_package_utils/controller.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 424b6bc66c..8e5bedda4a 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -76,27 +76,27 @@ class BaseController(ABC): variables, directories, and configuration files for each component. """ - def __init__(self, clp_config: CLPConfig): + def __init__(self, clp_config: CLPConfig) -> None: self._clp_config = clp_config self._clp_home = get_clp_home() self._conf_dir = self._clp_home / "etc" @abstractmethod - def start(self): + def start(self) -> None: """ Starts the components. """ pass @abstractmethod - def stop(self): + def stop(self) -> None: """ Stops the components. """ pass @abstractmethod - def _set_up_env(self): + def _set_up_env(self) -> None: """ Sets up all components to run by preparing environment variables, directories, and configuration files. @@ -562,7 +562,7 @@ def _update_settings_object( parent_key_prefix: str, settings: dict[str, Any], updates: dict[str, Any], - ): + ) -> None: """ Recursively updates the given settings object with the values from `updates`. @@ -588,11 +588,11 @@ class DockerComposeController(BaseController): Controller for orchestrating CLP components using Docker Compose. """ - def __init__(self, clp_config: CLPConfig, instance_id: str): + def __init__(self, clp_config: CLPConfig, instance_id: str) -> None: self._project_name = f"clp-package-{instance_id}" super().__init__(clp_config) - def start(self): + def start(self) -> None: """ Starts CLP's components using Docker Compose. @@ -615,7 +615,7 @@ def start(self): ) logger.info("CLP is started.") - def stop(self): + def stop(self) -> None: """ Stops CLP components deployed via Docker Compose. @@ -648,7 +648,7 @@ def _get_num_workers() -> int: """ return multiprocessing.cpu_count() // 2 - def _set_up_env(self): + def _set_up_env(self) -> None: # Generate container-specific config. container_clp_config = generate_docker_compose_container_config(self._clp_config) num_workers = self._get_num_workers() @@ -714,7 +714,7 @@ def _set_up_env(self): env_file.write(f"{key}={value}\n") -def get_or_create_instance_id(clp_config: CLPConfig): +def get_or_create_instance_id(clp_config: CLPConfig) -> str: """ Gets or creates a unique instance ID for this CLP instance. @@ -734,7 +734,7 @@ def get_or_create_instance_id(clp_config: CLPConfig): return instance_id -def _chown_paths_if_root(*paths: pathlib.Path): +def _chown_paths_if_root(*paths: pathlib.Path) -> None: """ Changes ownership of the given paths to the default service container user/group IDs if the current process is running as root. @@ -751,7 +751,7 @@ def _chown_recursively( path: pathlib.Path, user_id: int, group_id: int, -): +) -> None: """ Recursively changes the owner of the given path to the given user ID and group ID. From 3c3e23d8365a939406adddae464340e94e019a1d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:12:53 -0400 Subject: [PATCH 256/408] refactor(scripts): move `instance_id` assignment inside `try` block in start/stop scripts --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 +- .../clp-package-utils/clp_package_utils/scripts/stop_clp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index f84ba7c7a3..c64fb6ca6a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -74,8 +74,8 @@ def main(argv): logger.exception("Failed to create necessary directories.") return -1 - instance_id = get_or_create_instance_id(clp_config) try: + instance_id = get_or_create_instance_id(clp_config) controller = DockerComposeController(clp_config, instance_id) controller.start() except Exception as ex: diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 5506e81f08..ae4d6c329c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -22,8 +22,8 @@ def main(): logger.exception("Failed to load config.") return -1 - instance_id = get_or_create_instance_id(clp_config) try: + instance_id = get_or_create_instance_id(clp_config) controller = DockerComposeController(clp_config, instance_id) controller.stop() except: From 7c43aad09d3ea976b9a911ddb40921f7c84fe9a1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:21:46 -0400 Subject: [PATCH 257/408] revert `--config` argument removal in `stop_clp` to allow specifying configuration file path --- .../clp_package_utils/scripts/stop_clp.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index ae4d6c329c..105e929df7 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -1,4 +1,6 @@ +import argparse import logging +import pathlib import sys from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH @@ -12,12 +14,23 @@ logger = logging.getLogger(__file__) -def main(): +def main(argv): clp_home = get_clp_home() default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH + args_parser = argparse.ArgumentParser(description="Stops CLP") + args_parser.add_argument( + "--config", + "-c", + default=str(default_config_file_path), + help="CLP package configuration file.", + ) + + parsed_args = args_parser.parse_args(argv[1:]) + try: - clp_config = load_config_file(default_config_file_path, default_config_file_path, clp_home) + config_file_path = pathlib.Path(parsed_args.config) + clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) except: logger.exception("Failed to load config.") return -1 @@ -34,4 +47,4 @@ def main(): if "__main__" == __name__: - sys.exit(main()) + sys.exit(main(sys.argv)) From 63c700ad20d14d40051bf4aeeeedbddcccf2c95b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:23:45 -0400 Subject: [PATCH 258/408] refactor(config): Rename CLP_DEFAULT_ARCHIVE_* -> CLP_DEFAULT_ARCHIVES_* and CLP_DEFAULT_STREAM_* -> CLP_DEFAULT_STREAMS_*. --- .../clp-py-utils/clp_py_utils/clp_config.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 41a9ee431d..8615a00039 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -50,10 +50,10 @@ CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH = pathlib.Path("etc") / "clp-config.yml" CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" -CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" -CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" -CLP_DEFAULT_STREAM_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" -CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" +CLP_DEFAULT_ARCHIVES_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "archives" +CLP_DEFAULT_ARCHIVES_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-archives" +CLP_DEFAULT_STREAMS_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "streams" +CLP_DEFAULT_STREAMS_STAGING_DIRECTORY_PATH = CLP_DEFAULT_DATA_DIRECTORY_PATH / "staged-streams" CLP_DEFAULT_LOG_DIRECTORY_PATH = pathlib.Path("var") / "log" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" @@ -486,31 +486,31 @@ def transform_for_container(self): class ArchiveFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + directory: pathlib.Path = CLP_DEFAULT_ARCHIVES_DIRECTORY_PATH def transform_for_container(self): - self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_DIRECTORY_PATH + self.directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVES_DIRECTORY_PATH class StreamFsStorage(FsStorage): - directory: pathlib.Path = CLP_DEFAULT_STREAM_DIRECTORY_PATH + directory: pathlib.Path = CLP_DEFAULT_STREAMS_DIRECTORY_PATH def transform_for_container(self): - self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_DIRECTORY_PATH + self.directory = pathlib.Path("/") / CLP_DEFAULT_STREAMS_DIRECTORY_PATH class ArchiveS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = CLP_DEFAULT_ARCHIVES_STAGING_DIRECTORY_PATH def transform_for_container(self): - self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVE_STAGING_DIRECTORY_PATH + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_ARCHIVES_STAGING_DIRECTORY_PATH class StreamS3Storage(S3Storage): - staging_directory: pathlib.Path = CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + staging_directory: pathlib.Path = CLP_DEFAULT_STREAMS_STAGING_DIRECTORY_PATH def transform_for_container(self): - self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAM_STAGING_DIRECTORY_PATH + self.staging_directory = pathlib.Path("/") / CLP_DEFAULT_STREAMS_STAGING_DIRECTORY_PATH def _get_directory_from_storage_config( From 0f7c78c015fcd702df6161d2b96d7043250b05d5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 21:47:39 -0400 Subject: [PATCH 259/408] apply docs suggestions --- components/clp-py-utils/clp_py_utils/clp_config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 8615a00039..c0fc401827 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -804,10 +804,8 @@ def validate_presto_config(self): def transform_for_container(self): """ - Adjusts paths and service hosts for containerized execution. - - Converts all relevant directories to absolute paths inside the container - and updates service hostnames/ports to their container service names. + Converts all relevant directories to absolute paths inside the container and updates + component hostnames and ports to their container service names and default ports. """ self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH self.logs_directory = pathlib.Path("/") / CLP_DEFAULT_LOG_DIRECTORY_PATH From 5e0775679713157e4f0d4d6ab199804a9b0627dd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:13:32 -0400 Subject: [PATCH 260/408] docs(clp-package-utils): Update docstrings for clarity and consistency. --- .../clp_package_utils/general.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index c428e0b351..71a74441ad 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -135,7 +135,7 @@ def is_docker_compose_running(project_name: str) -> bool: Checks if a Docker Compose project is running. :param project_name: - :return: True if at least one instance is running, else False. + :return: Whether at least one instance is running. :raises EnvironmentError: If Docker Compose is not installed or fails. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] @@ -149,12 +149,13 @@ def is_docker_compose_running(project_name: str) -> bool: def check_docker_dependencies(should_compose_run: bool, project_name: str): """ - Checks if Docker and Docker Compose are installed, and whether Docker Compose is running or not. + Checks if Docker and Docker Compose are installed, and whether a Docker Compose project is + running. :param should_compose_run: :param project_name: The Docker Compose project name to check. - :raises EnvironmentError: If any Docker dependency is not installed or Docker Compose state - does not match expectation. + :raises EnvironmentError: If any Docker dependency is not installed or the Docker Compose + project state doesn't match `should_compose_run`. """ try: subprocess.run( @@ -176,11 +177,11 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): """ - Validate that a log directory path of a component is valid. + Validates that the logs directory path for a component is valid. :param logs_dir: :param component_name: - :raises ValueError: If the path is invalid or not a directory. + :raises ValueError: If the path is invalid or can't be a directory. """ try: validate_path_could_be_dir(logs_dir) @@ -311,7 +312,7 @@ def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig Copies the given config and transforms mount paths and hosts for Docker Compose. :param clp_config: - :return: The container config and the mounts. + :return: The container config. """ container_clp_config = clp_config.model_copy(deep=True) container_clp_config.transform_for_container() @@ -498,7 +499,7 @@ def validate_db_config( ): if not base_config.exists(): raise ValueError( - f"{DB_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + f"{DB_COMPONENT_NAME} configuration file missing: '{base_config}'." ) _validate_data_directory(data_dir, DB_COMPONENT_NAME) _validate_log_directory(logs_dir, DB_COMPONENT_NAME) @@ -541,7 +542,7 @@ def validate_results_cache_config( ): if not base_config.exists(): raise ValueError( - f"{RESULTS_CACHE_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{base_config}'." ) _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME) _validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) From ff4f06c7d2463f77b11c8d1bc7b30186e1929eb2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:14:55 -0400 Subject: [PATCH 261/408] remove return type from dump_shared_container_config() --- components/clp-package-utils/clp_package_utils/general.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 71a74441ad..ead1f781e9 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -356,9 +356,7 @@ def dump_container_config( return config_file_path_on_container, config_file_path_on_host -def dump_shared_container_config( - container_clp_config: CLPConfig, clp_config: CLPConfig -) -> Tuple[pathlib.Path, pathlib.Path]: +def dump_shared_container_config(container_clp_config: CLPConfig, clp_config: CLPConfig): """ Dumps the given container config to `CLP_SHARED_CONFIG_FILENAME` in the logs directory, so that it's accessible in the container. @@ -498,9 +496,7 @@ def validate_db_config( clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path ): if not base_config.exists(): - raise ValueError( - f"{DB_COMPONENT_NAME} configuration file missing: '{base_config}'." - ) + raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{base_config}'.") _validate_data_directory(data_dir, DB_COMPONENT_NAME) _validate_log_directory(logs_dir, DB_COMPONENT_NAME) From e381c3d1083882f1fe79fc1ef5ad273810dd397f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:16:11 -0400 Subject: [PATCH 262/408] refactor(clp-package-utils): Rename base_config -> component_config in validation functions. --- .../clp_package_utils/general.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index ead1f781e9..a894085189 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -493,10 +493,13 @@ def validate_and_load_redis_credentials_file( def validate_db_config( - clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path + clp_config: CLPConfig, + component_config: pathlib.Path, + data_dir: pathlib.Path, + logs_dir: pathlib.Path, ): - if not base_config.exists(): - raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{base_config}'.") + if not component_config.exists(): + raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{component_config}'.") _validate_data_directory(data_dir, DB_COMPONENT_NAME) _validate_log_directory(logs_dir, DB_COMPONENT_NAME) @@ -510,11 +513,14 @@ def validate_queue_config(clp_config: CLPConfig, logs_dir: pathlib.Path): def validate_redis_config( - clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path + clp_config: CLPConfig, + component_config: pathlib.Path, + data_dir: pathlib.Path, + logs_dir: pathlib.Path, ): - if not base_config.exists(): + if not component_config.exists(): raise ValueError( - f"{REDIS_COMPONENT_NAME} base configuration at {str(base_config)} is missing." + f"{REDIS_COMPONENT_NAME} base configuration at {str(component_config)} is missing." ) _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) _validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) @@ -534,11 +540,14 @@ def validate_reducer_config(clp_config: CLPConfig, logs_dir: pathlib.Path, num_w def validate_results_cache_config( - clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path + clp_config: CLPConfig, + component_config: pathlib.Path, + data_dir: pathlib.Path, + logs_dir: pathlib.Path, ): - if not base_config.exists(): + if not component_config.exists(): raise ValueError( - f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{base_config}'." + f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{component_config}'." ) _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME) _validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME) From f426c82d6866d7c3c513e0f72f80d977929906c0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:17:35 -0400 Subject: [PATCH 263/408] Simplify raised error message for missing configuration file: redis --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a894085189..71e3cd3559 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -520,7 +520,7 @@ def validate_redis_config( ): if not component_config.exists(): raise ValueError( - f"{REDIS_COMPONENT_NAME} base configuration at {str(component_config)} is missing." + f"{REDIS_COMPONENT_NAME} configuration file missing: '{component_config}'." ) _validate_data_directory(data_dir, REDIS_COMPONENT_NAME) _validate_log_directory(logs_dir, REDIS_COMPONENT_NAME) From 73ec68f3db4c1b2ea2e7e27f969f49de9943a309 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 22:20:44 -0400 Subject: [PATCH 264/408] Apply suggestion - To alphabetize --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 8e5bedda4a..0a5047e00b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -171,8 +171,8 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: } # Credential env_vars |= { - "CLP_QUEUE_USER": self._clp_config.queue.username, "CLP_QUEUE_PASS": self._clp_config.queue.password, + "CLP_QUEUE_USER": self._clp_config.queue.username, } # Path env_vars |= { From 4e707ae1a5be399165357cd652a721cdbe435e23 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 23:17:28 -0400 Subject: [PATCH 265/408] Remove individual log file paths and use shared logging volume --- .../clp_package_utils/controller.py | 26 ------------------- .../package/docker-compose.base.yaml | 16 +++++------- tools/deployment/package/docker-compose.yaml | 16 +++++------- 3 files changed, 14 insertions(+), 44 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0a5047e00b..00ca451e5c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -274,9 +274,6 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - log_file = self._clp_config.logs_directory / f"{component_name}.log" - log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) - env_vars = EnvVarsDict() # Logging env_vars |= { @@ -284,10 +281,6 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: self._clp_config.compression_scheduler.logging_level ), } - # Path - env_vars |= { - "CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST": str(log_file), - } return env_vars @@ -300,18 +293,11 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: component_name = QUERY_SCHEDULER_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") - log_file = self._clp_config.logs_directory / f"{component_name}.log" - log_file.touch(mode=LOG_FILE_ACCESS_MODE, exist_ok=True) - env_vars = EnvVarsDict() # Logging env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, } - # Path - env_vars |= { - "CLP_QUERY_SCHEDULER_LOG_FILE_HOST": str(log_file), - } return env_vars @@ -335,10 +321,6 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: self._clp_config.compression_worker.logging_level ), } - # Path - env_vars |= { - "CLP_COMPRESSION_WORKER_LOGS_DIR_HOST": str(logs_dir), - } # Resource env_vars |= { "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), @@ -364,10 +346,6 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, } - # Path - env_vars |= { - "CLP_QUERY_WORKER_LOGS_DIR_HOST": str(logs_dir), - } # Resource env_vars |= { "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), @@ -393,10 +371,6 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, } - # Path - env_vars |= { - "CLP_REDUCER_LOGS_DIR_HOST": str(logs_dir), - } # Resource env_vars |= { "CLP_REDUCER_CONCURRENCY": str(num_workers), diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 205aa9c387..a13cf77162 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -35,6 +35,10 @@ x-volume-definitions: source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" target: "/etc/clp-config.yml" read_only: true + clp-logs: &volume_clp_logs + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}" + target: "/var/log" logs-input-readonly: &volume_root_logs_readonly type: "bind" source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" @@ -220,10 +224,8 @@ services: volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly + - *volume_clp_logs - *volume_root_logs_readonly - - type: "bind" - source: "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}" - target: "/var/log/compression_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -256,6 +258,7 @@ services: volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly + - *volume_clp_logs - *volume_root_logs_readonly - type: "bind" source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" @@ -263,9 +266,6 @@ services: - type: "bind" source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" target: "/var/data/staged-archives" - - type: "bind" - source: "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}" - target: "/var/log/compression_worker" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" @@ -340,9 +340,7 @@ services: PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector" - target: "/var/log/garbage_collector" + - *volume_clp_logs depends_on: db-table-creator: condition: "service_completed_successfully" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index e803fd3cf7..c8192b30c7 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -27,6 +27,10 @@ x-volume-definitions: source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" target: "/etc/clp-config.yml" read_only: true + clp-logs: &volume_clp_logs + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}" + target: "/var/log" services: query-scheduler: @@ -43,9 +47,7 @@ services: redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}" - target: "/var/log/query_scheduler.log" + - *volume_clp_logs depends_on: db-table-creator: condition: "service_completed_successfully" @@ -86,12 +88,10 @@ services: volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly + - *volume_clp_logs - type: "bind" source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" target: "/var/data/archives" - - type: "bind" - source: "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}" - target: "/var/log/query_worker" - type: "bind" source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}" target: "/var/data/staged-streams" @@ -121,9 +121,7 @@ services: PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}" - target: "/var/log/reducer" + - *volume_clp_logs depends_on: query-scheduler: condition: "service_healthy" From 6f53877fa68bc456d0ff7f57b714dc60d73c50cc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 23:22:26 -0400 Subject: [PATCH 266/408] fix(controller): revert removal of "ClpQueryEngine" in server_settings_json_updates --- components/clp-package-utils/clp_package_utils/controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 00ca451e5c..cdb048f31b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -447,6 +447,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), "StreamTargetUncompressedSize": self._clp_config.stream_output.target_uncompressed_size, + "ClpQueryEngine": self._clp_config.package.query_engine, } stream_storage = self._clp_config.stream_output.storage From e7addd3114359226d6fd1ec0b1a3c54273c3d666 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 15 Oct 2025 23:25:15 -0400 Subject: [PATCH 267/408] refactor(clp-package-utils): Rename CLP_PACKAGE_CONTAINER -> CLP_PACKAGE_CONTAINER_IMAGE_REF. --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- tools/deployment/package/docker-compose.base.yaml | 2 +- tools/deployment/package/docker-compose.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index cdb048f31b..af3ae11e13 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -648,7 +648,7 @@ def _set_up_env(self) -> None: # Package env_vars |= { - "CLP_PACKAGE_CONTAINER": self._clp_config.container_image_ref, + "CLP_PACKAGE_CONTAINER_IMAGE_REF": self._clp_config.container_image_ref, "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, } diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index a13cf77162..e81deadbaf 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -2,7 +2,7 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" stop_grace_period: "3s" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index c8192b30c7..e9797f3b24 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -5,7 +5,7 @@ include: ["docker-compose.base.yaml"] # Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for # documentation. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" stop_grace_period: "3s" From d4e81f327d300d2461a8831d61eb5bd1d3bdbf32 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 00:30:14 -0400 Subject: [PATCH 268/408] fix(docker-compose): use `:?error` syntax to mark required non-empty env vars; add fallback for CLP_DB_NAME; add empty fallbacks for CLP_AWS_ACCESS_KEY_ID and CLP_AWS_SECRET_ACCESS_KEY --- .../package/docker-compose.base.yaml | 48 +++++++++---------- tools/deployment/package/docker-compose.yaml | 16 +++---- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e81deadbaf..e3a3cd4cef 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -52,10 +52,10 @@ services: hostname: "database" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD: "${CLP_DB_PASS}" - MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS}" - MYSQL_USER: "${CLP_DB_USER}" + MYSQL_DATABASE: "${CLP_DB_NAME:-clp-db}" + MYSQL_PASSWORD: "${CLP_DB_PASS:?error}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?error}" + MYSQL_USER: "${CLP_DB_USER:?error}" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" published: "${CLP_DB_PORT:-3306}" @@ -78,16 +78,16 @@ services: "mysqladmin", "ping", "--silent", "-h", "127.0.0.1", - "-u", "${CLP_DB_USER}", - "--password=${CLP_DB_PASS}" + "-u", "${CLP_DB_USER:?error}", + "--password=${CLP_DB_PASS:?error}" ] db-table-creator: <<: *service_defaults hostname: "db_table_creator" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly @@ -108,8 +108,8 @@ services: hostname: "queue" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?error}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?error}" RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" @@ -153,13 +153,13 @@ services: "redis-cli", "-h", "127.0.0.1", "-p", "6379", - "-a", "${CLP_REDIS_PASS}", + "-a", "${CLP_REDIS_PASS:?error}", "PING" ] command: [ "redis-server", "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS}" + "--requirepass", "${CLP_REDIS_PASS:?error}" ] results-cache: @@ -213,14 +213,14 @@ services: <<: *service_defaults hostname: "compression_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -244,9 +244,9 @@ services: <<: *service_defaults hostname: "compression_worker" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" @@ -254,7 +254,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -286,8 +286,8 @@ services: <<: *service_defaults hostname: "webui" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" HOST: "0.0.0.0" NODE_ENV: "production" NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" @@ -332,8 +332,8 @@ services: <<: *service_defaults hostname: "garbage_collector" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/garbage_collector" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index e9797f3b24..d849449c3d 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -37,14 +37,14 @@ services: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -74,9 +74,9 @@ services: <<: *service_defaults hostname: "query_worker" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" @@ -84,7 +84,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly From a5a497dc2a35cd0847af3c9fb6e2368ba725b2a2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 00:32:14 -0400 Subject: [PATCH 269/408] disallow empty `username` and `password` in `Database` because our db init script does not support such; it's a bad practice anyway --- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c0fc401827..8bb05351fa 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -156,8 +156,8 @@ class Database(BaseModel): auto_commit: bool = False compress: bool = True - username: Optional[str] = None - password: Optional[str] = None + username: Optional[NonEmptyStr] = None + password: Optional[NonEmptyStr] = None def ensure_credentials_loaded(self): if self.username is None or self.password is None: From 0dd11ce872c4b5aec7043443222c652a7abd7247 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 00:33:27 -0400 Subject: [PATCH 270/408] fix(docker-compose): Update default database image to mariadb:10-jammy. --- tools/deployment/package/docker-compose.base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e3a3cd4cef..92736468c2 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -48,7 +48,7 @@ x-volume-definitions: services: database: <<: *service_defaults - image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + image: "${CLP_DB_IMAGE:-mariadb:10-jammy}" hostname: "database" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: From d27a0b12450f947e96fe351f7ccf5940648afaa0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 00:58:46 -0400 Subject: [PATCH 271/408] fix(docker-compose): Add stop_grace_period for compression-worker and compression-scheduler. --- tools/deployment/package/docker-compose.base.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 92736468c2..b2e2499fa6 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -212,6 +212,7 @@ services: compression-scheduler: <<: *service_defaults hostname: "compression_scheduler" + stop_grace_period: "300s" environment: BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?error}" @@ -243,6 +244,7 @@ services: compression-worker: <<: *service_defaults hostname: "compression_worker" + stop_grace_period: "60s" environment: AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" From a6478e6bfed0ac49bb3d05cc5e61a45a6d7e7306 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 01:02:38 -0400 Subject: [PATCH 272/408] apply docs suggestions --- tools/deployment/package/docker-compose.base.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index b2e2499fa6..813302760f 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -12,14 +12,19 @@ x-service-defaults: &service_defaults x-healthcheck-defaults: &healthcheck_defaults # Avoid lowering to prevent excessive resource usage. interval: "30s" + # Mark unhealthy after 3 failed probes. # - In steady state, ( + ) × 3 = ~90s before the service is marked unhealthy. # - From startup, (60s) + ~90s = ~150s before the service is marked unhealthy. retries: 3 + # Frequent checks during startup allow fast transition to healthy. start_interval: "2s" - # Ignore failures for ~15 frequent checks before counting retries. + + # Ignore failures for / ( + ) = ~15 frequent checks before + # counting retries. start_period: "60s" + # Short timeout since no remote communication is expected. timeout: "2s" From 7bbd134d42c0660032a7a87c0285d026949a0287 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 01:03:41 -0400 Subject: [PATCH 273/408] refactor(docker-compose): Rename volume_root_logs_readonly -> volume_logs_input_readonly. --- tools/deployment/package/docker-compose.base.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 813302760f..e9209c8e83 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -44,7 +44,7 @@ x-volume-definitions: type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}" target: "/var/log" - logs-input-readonly: &volume_root_logs_readonly + logs-input-readonly: &volume_logs_input_readonly type: "bind" source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" @@ -231,7 +231,7 @@ services: - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - - *volume_root_logs_readonly + - *volume_logs_input_readonly depends_on: db-table-creator: condition: "service_completed_successfully" @@ -266,7 +266,7 @@ services: - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - - *volume_root_logs_readonly + - *volume_logs_input_readonly - type: "bind" source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" target: "/var/data/archives" From 04e56f0eb7ec18bc38a8a27039d4762841b96ecd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 01:18:56 -0400 Subject: [PATCH 274/408] ensure `healthcheck` goes after `command` --- .../package/docker-compose.base.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e9209c8e83..9184cb1759 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -151,6 +151,11 @@ services: - type: "bind" source: "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}" target: "/var/log/redis" + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS:?error}" + ] healthcheck: <<: *healthcheck_defaults test: [ @@ -161,11 +166,6 @@ services: "-a", "${CLP_REDIS_PASS:?error}", "PING" ] - command: [ - "redis-server", - "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS:?error}" - ] results-cache: <<: *service_defaults @@ -187,16 +187,16 @@ services: - type: "bind" source: "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}" target: "/var/log/mongodb" + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] healthcheck: <<: *healthcheck_defaults test: [ "CMD-SHELL", "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:27017/test --quiet" ] - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] results-cache-indices-creator: <<: *service_defaults From 0fc91d74e3b20a149e60cc4011111a430568abe1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 01:28:01 -0400 Subject: [PATCH 275/408] fix(docker-compose): Add AWS credentials environment variables in webui service. --- tools/deployment/package/docker-compose.base.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 9184cb1759..b4a7f0ec74 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -293,6 +293,8 @@ services: <<: *service_defaults hostname: "webui" environment: + AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" CLP_DB_PASS: "${CLP_DB_PASS:?error}" CLP_DB_USER: "${CLP_DB_USER:?error}" HOST: "0.0.0.0" From f307d942484cbde72a958125eb5753877179782a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 08:55:42 -0400 Subject: [PATCH 276/408] fix(docker-compose): Update environment variables for archive and stream output directories based on storage type; conditionally mount output dirs. --- .../clp_package_utils/controller.py | 21 +++++++++++++------ .../package/docker-compose.base.yaml | 20 ++++++++++-------- tools/deployment/package/docker-compose.yaml | 12 +++-------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index af3ae11e13..121241984b 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -660,13 +660,22 @@ def _set_up_env(self) -> None: "CLP_LOGS_DIR_HOST": str(self._clp_config.logs_directory), # Config "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), - # Input - "CLP_LOGS_INPUT_DIR_CONTAINER": str(container_clp_config.logs_input.directory), - "CLP_LOGS_INPUT_DIR_HOST": str(self._clp_config.logs_input.directory), - # Output - "CLP_ARCHIVE_OUTPUT_DIR_HOST": str(self._clp_config.archive_output.get_directory()), - "CLP_STREAM_OUTPUT_DIR_HOST": str(self._clp_config.stream_output.get_directory()), } + # Input + if self._clp_config.logs_input.type == StorageType.FS: + env_vars["CLP_LOGS_INPUT_DIR_CONTAINER"] = str(container_clp_config.logs_input.directory) + env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory) + # Output + archive_output_dir_str = str(self._clp_config.archive_output.get_directory()) + stream_output_dir_str = str(self._clp_config.stream_output.get_directory()) + if self._clp_config.archive_output.storage.type == StorageType.FS: + env_vars["CLP_ARCHIVE_OUTPUT_DIR_HOST"] = archive_output_dir_str + if self._clp_config.archive_output.storage.type == StorageType.S3: + env_vars["CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST"] = archive_output_dir_str + if self._clp_config.stream_output.storage.type == StorageType.FS: + env_vars["CLP_STREAM_OUTPUT_DIR_HOST"] = stream_output_dir_str + if self._clp_config.stream_output.storage.type == StorageType.S3: + env_vars["CLP_STAGED_STREAM_OUTPUT_DIR_HOST"] = stream_output_dir_str # Component-specific env_vars |= self._set_up_env_for_database() diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index b4a7f0ec74..c7f90206f4 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -50,6 +50,14 @@ x-volume-definitions: target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" read_only: true +volumes: + # Dummy volume to use when a bind mount is not desired. + empty: + driver_opts: + device: "tmpfs" + type: "tmpfs" + size: 0 + services: database: <<: *service_defaults @@ -267,12 +275,8 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" - target: "/var/data/archives" - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" - target: "/var/data/staged-archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" @@ -308,9 +312,7 @@ services: target: 4000 volumes: - *volume_aws_config_readonly - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" - target: "/var/data/streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - type: "bind" source: "./var/www/webui/client/settings.json" target: "/opt/clp/var/www/webui/client/settings.json" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index d849449c3d..9bb71a137a 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -89,15 +89,9 @@ services: - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" - target: "/var/data/archives" - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}" - target: "/var/data/staged-streams" - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" - target: "/var/data/streams" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" + - "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams" command: [ "python3", "-u", From e5a90dda029937daf82ea7f36f9dd5c7a13471e2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:08:05 -0400 Subject: [PATCH 277/408] Replace "docker-compose" (usually refer to v1) with "Docker Compose" in error prompts --- components/clp-package-utils/clp_package_utils/general.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 71e3cd3559..7a682156a7 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -144,7 +144,7 @@ def is_docker_compose_running(project_name: str) -> bool: running_instances = json.loads(output) return len(running_instances) >= 1 except subprocess.CalledProcessError: - raise EnvironmentError("docker-compose is not installed or not functioning properly.") + raise EnvironmentError("Docker Compose is not installed or not functioning properly.") def check_docker_dependencies(should_compose_run: bool, project_name: str): @@ -170,9 +170,9 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): is_running = is_docker_compose_running(project_name) if should_compose_run and not is_running: - raise EnvironmentError("docker-compose is not running.") + raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") if not should_compose_run and is_running: - raise EnvironmentError("docker-compose is already running.") + raise EnvironmentError("Docker Compose project '{project_name}' is already running.") def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): From 3015ec4a12bdc8590eecf5b69543c991a5449fd4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:12:21 -0400 Subject: [PATCH 278/408] lint --- components/clp-package-utils/clp_package_utils/controller.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 121241984b..e01db0ad30 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -663,7 +663,9 @@ def _set_up_env(self) -> None: } # Input if self._clp_config.logs_input.type == StorageType.FS: - env_vars["CLP_LOGS_INPUT_DIR_CONTAINER"] = str(container_clp_config.logs_input.directory) + env_vars["CLP_LOGS_INPUT_DIR_CONTAINER"] = str( + container_clp_config.logs_input.directory + ) env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory) # Output archive_output_dir_str = str(self._clp_config.archive_output.get_directory()) From ff43c78562e2710eb9995b647bc0ece742ee2cad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:13:10 -0400 Subject: [PATCH 279/408] refactor(clp-package-utils): Reorder functions and rename is_docker_compose_running -> _is_docker_compose_running. --- .../clp_package_utils/general.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 7a682156a7..e5c0a6d223 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -96,13 +96,6 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path): self.generated_config_file: Optional[DockerMount] = None -def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: - try: - validate_path_could_be_dir(data_dir) - except ValueError as ex: - raise ValueError(f"{component_name} data directory is invalid: {ex}") - - def get_clp_home(): # Determine CLP_HOME from an environment variable or this script's path clp_home = None @@ -130,23 +123,6 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def is_docker_compose_running(project_name: str) -> bool: - """ - Checks if a Docker Compose project is running. - - :param project_name: - :return: Whether at least one instance is running. - :raises EnvironmentError: If Docker Compose is not installed or fails. - """ - cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - running_instances = json.loads(output) - return len(running_instances) >= 1 - except subprocess.CalledProcessError: - raise EnvironmentError("Docker Compose is not installed or not functioning properly.") - - def check_docker_dependencies(should_compose_run: bool, project_name: str): """ Checks if Docker and Docker Compose are installed, and whether a Docker Compose project is @@ -168,13 +144,37 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = is_docker_compose_running(project_name) + is_running = _is_docker_compose_running(project_name) if should_compose_run and not is_running: raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") if not should_compose_run and is_running: raise EnvironmentError("Docker Compose project '{project_name}' is already running.") +def _is_docker_compose_running(project_name: str) -> bool: + """ + Checks if a Docker Compose project is running. + + :param project_name: + :return: Whether at least one instance is running. + :raises EnvironmentError: If Docker Compose is not installed or fails. + """ + cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + running_instances = json.loads(output) + return len(running_instances) >= 1 + except subprocess.CalledProcessError: + raise EnvironmentError("Docker Compose is not installed or not functioning properly.") + + +def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: + try: + validate_path_could_be_dir(data_dir) + except ValueError as ex: + raise ValueError(f"{component_name} data directory is invalid: {ex}") + + def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): """ Validates that the logs directory path for a component is valid. From 699979acc3e2e4586fb0a6f2660fc8643f4c5f4e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:28:06 -0400 Subject: [PATCH 280/408] fix(docker-compose): Add mounts for archive and stream output in garbage-collector --- tools/deployment/package/docker-compose.base.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index c7f90206f4..01edcae002 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -352,6 +352,8 @@ services: volumes: - *volume_clp_config_readonly - *volume_clp_logs + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" depends_on: db-table-creator: condition: "service_completed_successfully" From 092a86c66dbd4143a509bf9dd14479ffa5016dc3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:29:32 -0400 Subject: [PATCH 281/408] refactor(clp-package-utils): Rename _is_docker_compose_running -> _is_docker_compose_project_running. --- components/clp-package-utils/clp_package_utils/general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index e5c0a6d223..863339a42b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -144,14 +144,14 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): except subprocess.CalledProcessError: raise EnvironmentError("docker is not installed or available on the path") - is_running = _is_docker_compose_running(project_name) + is_running = _is_docker_compose_project_running(project_name) if should_compose_run and not is_running: raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") if not should_compose_run and is_running: raise EnvironmentError("Docker Compose project '{project_name}' is already running.") -def _is_docker_compose_running(project_name: str) -> bool: +def _is_docker_compose_project_running(project_name: str) -> bool: """ Checks if a Docker Compose project is running. From 8d7517174eea6975ca35a851b306cf4a9a41cd13 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 09:47:35 -0400 Subject: [PATCH 282/408] fix(docker-compose): Rename CLP_AWS_ACCESS_KEY_ID -> CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID, CLP_AWS_SECRET_ACCESS_KEY -> CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY; Update AWS environment variable handling based on storage type. --- .../clp-package-utils/clp_package_utils/controller.py | 11 +++++++---- tools/deployment/package/docker-compose.base.yaml | 6 ++---- tools/deployment/package/docker-compose.yaml | 2 -- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index e01db0ad30..d6f21c4468 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -633,10 +633,13 @@ def _set_up_env(self) -> None: env_vars = EnvVarsDict() # Credential - env_vars |= { - "CLP_AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", ""), - "CLP_AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", ""), - } + if self._clp_config.stream_output.storage.type == StorageType.S3: + stream_output_aws_auth = self._clp_config.stream_output.storage.s3_config.aws_authentication + if stream_output_aws_auth.type == AwsAuthType.credentials: + env_vars |= { + "CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID": stream_output_aws_auth.credentials.access_key_id, + "CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY": stream_output_aws_auth.credentials.secret_access_key, + } # Identity env_vars |= { diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 01edcae002..82cd4e37d4 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -259,8 +259,6 @@ services: hostname: "compression_worker" stop_grace_period: "60s" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" @@ -297,8 +295,8 @@ services: <<: *service_defaults hostname: "webui" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" + AWS_ACCESS_KEY_ID: "${CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}" CLP_DB_PASS: "${CLP_DB_PASS:?error}" CLP_DB_USER: "${CLP_DB_USER:?error}" HOST: "0.0.0.0" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 9bb71a137a..e227f3ab3b 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -74,8 +74,6 @@ services: <<: *service_defaults hostname: "query_worker" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID:-}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY:-}" BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" From f53339f63f713e06f771accbd32db5c615edf861 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 10:16:19 -0400 Subject: [PATCH 283/408] fix(docker-compose): Break long volume mount lines for improved readability. --- tools/deployment/package/docker-compose.base.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 82cd4e37d4..0663fba97b 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -273,8 +273,10 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" - - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ +${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ +${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" From 18d11091b3b89ba5f9d22ad6dfae9260ce399c49 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 16 Oct 2025 10:19:35 -0400 Subject: [PATCH 284/408] lint --- .../clp_package_utils/controller.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index d6f21c4468..83f2868e58 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -634,11 +634,17 @@ def _set_up_env(self) -> None: # Credential if self._clp_config.stream_output.storage.type == StorageType.S3: - stream_output_aws_auth = self._clp_config.stream_output.storage.s3_config.aws_authentication + stream_output_aws_auth = ( + self._clp_config.stream_output.storage.s3_config.aws_authentication + ) if stream_output_aws_auth.type == AwsAuthType.credentials: env_vars |= { - "CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID": stream_output_aws_auth.credentials.access_key_id, - "CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY": stream_output_aws_auth.credentials.secret_access_key, + "CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID": ( + stream_output_aws_auth.credentials.access_key_id + ), + "CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY": ( + stream_output_aws_auth.credentials.secret_access_key + ), } # Identity From 84c011b2e274c8f91b63d9bc3573f2135d1a0056 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 17 Oct 2025 10:28:54 -0400 Subject: [PATCH 285/408] Add main for script --- .../clp-py-utils/clp_py_utils/start-spider-worker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index 3eb892dba8..893527ca7f 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -38,4 +38,9 @@ def main() -> None: processes.append(process) for process in processes: - process.wait() + exit_code = process.wait() + print(f"Spider worker exited with code {exit_code}") + + +if __name__ == "__main__": + main() From 272706fbff8f28bfcb9f6e4ef2b89f0199914910 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 17 Oct 2025 10:29:08 -0400 Subject: [PATCH 286/408] Bandit solution for column width --- .../clp-py-utils/clp_py_utils/initialize-spider-db.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 8193c60923..2f570aec53 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS `data` ( `id` BINARY(16) NOT NULL, - `value` VARBINARY(999) NOT NULL, + `value` BLOB(60000) NOT NULL, `hard_locality` BOOL DEFAULT FALSE, `persisted` BOOL DEFAULT FALSE, PRIMARY KEY (`id`) @@ -111,7 +111,7 @@ `task_id` BINARY(16) NOT NULL, `position` INT UNSIGNED NOT NULL, `type` VARCHAR(999) NOT NULL, - `value` VARBINARY(999), + `value` BLOB(60000), `data_id` BINARY(16), CONSTRAINT `output_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, CONSTRAINT `output_data_id` FOREIGN KEY (`data_id`) REFERENCES `data` (`id`) ON UPDATE NO ACTION ON DELETE NO ACTION, @@ -126,7 +126,7 @@ `type` VARCHAR(999) NOT NULL, `output_task_id` BINARY(16), `output_task_position` INT UNSIGNED, - `value` VARBINARY(999), -- Use VARBINARY for all types of values + `value` BLOB(60000), `data_id` BINARY(16), CONSTRAINT `input_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, CONSTRAINT `input_task_output_match` FOREIGN KEY (`output_task_id`, `output_task_position`) REFERENCES task_outputs (`task_id`, `position`) ON UPDATE NO ACTION ON DELETE SET NULL, @@ -202,7 +202,7 @@ CREATE TABLE IF NOT EXISTS `client_kv_data` ( `kv_key` VARCHAR(64) NOT NULL, - `value` VARBINARY(999) NOT NULL, + `value` BLOB(60000) NOT NULL, `client_id` BINARY(16) NOT NULL, PRIMARY KEY (`client_id`, `kv_key`) ); @@ -211,7 +211,7 @@ CREATE TABLE IF NOT EXISTS `task_kv_data` ( `kv_key` VARCHAR(64) NOT NULL, - `value` VARBINARY(999) NOT NULL, + `value` BLOB(60000) NOT NULL, `task_id` BINARY(16) NOT NULL, PRIMARY KEY (`task_id`, `kv_key`), CONSTRAINT `kv_data_task_id` FOREIGN KEY (`task_id`) REFERENCES `tasks` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE From 234f0f01f441468ab3e0a4158fe403c977563638 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 17 Oct 2025 10:30:06 -0400 Subject: [PATCH 287/408] Fix docker compose --- .../package/docker-compose.spider.base.yaml | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index edc33053b6..4bfa479ec8 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -107,6 +107,10 @@ services: depends_on: db-table-creator: condition: "service_completed_successfully" + ports: + - host_ip: "${SPIDER_SCHEDULER_HOST}" + published: "${SPIDER_SCHEDULER_PORT}" + target: "${SPIDER_SCHEDULER_PORT}" command: [ "/opt/clp/bin/spider_scheduler", "--host", "${SPIDER_SCHEDULER_HOST}", @@ -131,17 +135,9 @@ services: - *volume_clp_config_readonly - *volume_root_logs_readonly - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" - target: "/var/data/archives" - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" - target: "/var/data/staged-archives" - - type: "bind" - source: "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}" - target: "/var/log/compression_worker" + source: "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}" + target: "/var/log/compression_scheduler.log" - type: "bind" - source: "${CLP_DATA_DIR_HOST:-./var/data}" - target: "/var/data" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -165,17 +161,30 @@ services: CLP_LOGS_DIR: "/var/log/compression_worker" CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" + PATH: "/opt/clp/lib/python3/site-packages/bin:${PATH}" volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_root_logs_readonly + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" + target: "/var/data/archives" + - type: "bind" + source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" + target: "/var/data/staged-archives" + - type: "bind" + source: "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}" + target: "/var/log/compression_worker" + - type: "bind" + source: "${CLP_DATA_DIR_HOST:-./var/data}" + target: "/var/data" depends_on: db-table-creator: condition: "service_completed_successfully" command: [ "python3", "-u", - "-m", "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", + "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", "--storage-url", "${SPIDER_DB_URL}", # NOTE: Leave host to spider scheduler's host. This only affects task placement. From 9d9bd46d824599c527f1fd1928e85739469814e6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 17 Oct 2025 21:07:11 -0400 Subject: [PATCH 288/408] refactor(clp-package-utils): Rename should_compose_run -> should_compose_project_be_running in check_docker_dependencies(). --- .../clp-package-utils/clp_package_utils/controller.py | 8 ++++++-- components/clp-package-utils/clp_package_utils/general.py | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 83f2868e58..f3ed95c172 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -573,7 +573,9 @@ def start(self) -> None: :raise: Propagates `subprocess.run`'s exceptions. """ - check_docker_dependencies(should_compose_run=False, project_name=self._project_name) + check_docker_dependencies( + should_compose_project_be_running=False, project_name=self._project_name + ) self._set_up_env() deployment_type = self._clp_config.get_deployment_type() @@ -597,7 +599,9 @@ def stop(self) -> None: :raise: Propagates `subprocess.run`'s exceptions. """ try: - check_docker_dependencies(should_compose_run=True, project_name=self._project_name) + check_docker_dependencies( + should_compose_project_be_running=True, project_name=self._project_name + ) except EnvironmentError as e: logger.warning( 'Docker dependencies check failed: "%s". Attempting to stop CLP containers ' diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 863339a42b..3ace0c6123 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -123,12 +123,12 @@ def generate_container_name(job_type: str) -> str: return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}" -def check_docker_dependencies(should_compose_run: bool, project_name: str): +def check_docker_dependencies(should_compose_project_be_running: bool, project_name: str): """ Checks if Docker and Docker Compose are installed, and whether a Docker Compose project is running. - :param should_compose_run: + :param should_compose_project_be_running: :param project_name: The Docker Compose project name to check. :raises EnvironmentError: If any Docker dependency is not installed or the Docker Compose project state doesn't match `should_compose_run`. @@ -145,9 +145,9 @@ def check_docker_dependencies(should_compose_run: bool, project_name: str): raise EnvironmentError("docker is not installed or available on the path") is_running = _is_docker_compose_project_running(project_name) - if should_compose_run and not is_running: + if should_compose_project_be_running and not is_running: raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") - if not should_compose_run and is_running: + if not should_compose_project_be_running and is_running: raise EnvironmentError("Docker Compose project '{project_name}' is already running.") From 127aacc68f3670956aa923eb5566557db47d2bfc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 17 Oct 2025 22:49:19 -0400 Subject: [PATCH 289/408] refactor(clp-package-utils): Reorder private functions --- .../clp_package_utils/general.py | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 3ace0c6123..cb8ef8a367 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -151,44 +151,6 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n raise EnvironmentError("Docker Compose project '{project_name}' is already running.") -def _is_docker_compose_project_running(project_name: str) -> bool: - """ - Checks if a Docker Compose project is running. - - :param project_name: - :return: Whether at least one instance is running. - :raises EnvironmentError: If Docker Compose is not installed or fails. - """ - cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - running_instances = json.loads(output) - return len(running_instances) >= 1 - except subprocess.CalledProcessError: - raise EnvironmentError("Docker Compose is not installed or not functioning properly.") - - -def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: - try: - validate_path_could_be_dir(data_dir) - except ValueError as ex: - raise ValueError(f"{component_name} data directory is invalid: {ex}") - - -def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): - """ - Validates that the logs directory path for a component is valid. - - :param logs_dir: - :param component_name: - :raises ValueError: If the path is invalid or can't be a directory. - """ - try: - validate_path_could_be_dir(logs_dir) - except ValueError as ex: - raise ValueError(f"{component_name} logs directory is invalid: {ex}") - - def validate_port(port_name: str, hostname: str, port: int): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -727,3 +689,41 @@ def get_celery_connection_env_vars_list(container_clp_config: CLPConfig) -> List ] return env_vars + + +def _is_docker_compose_project_running(project_name: str) -> bool: + """ + Checks if a Docker Compose project is running. + + :param project_name: + :return: Whether at least one instance is running. + :raises EnvironmentError: If Docker Compose is not installed or fails. + """ + cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + running_instances = json.loads(output) + return len(running_instances) >= 1 + except subprocess.CalledProcessError: + raise EnvironmentError("Docker Compose is not installed or not functioning properly.") + + +def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: + try: + validate_path_could_be_dir(data_dir) + except ValueError as ex: + raise ValueError(f"{component_name} data directory is invalid: {ex}") + + +def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): + """ + Validates that the logs directory path for a component is valid. + + :param logs_dir: + :param component_name: + :raises ValueError: If the path is invalid or can't be a directory. + """ + try: + validate_path_could_be_dir(logs_dir) + except ValueError as ex: + raise ValueError(f"{component_name} logs directory is invalid: {ex}") From f5ea3e2716e74509442ecd99356943399aeb9311 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 17 Oct 2025 22:55:26 -0400 Subject: [PATCH 290/408] Replace EnvironmentError with OSError as per Ruff os-error-alias (UP024) --- .../clp_package_utils/controller.py | 2 +- .../clp-package-utils/clp_package_utils/general.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index f3ed95c172..33d863fb15 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -602,7 +602,7 @@ def stop(self) -> None: check_docker_dependencies( should_compose_project_be_running=True, project_name=self._project_name ) - except EnvironmentError as e: + except OSError as e: logger.warning( 'Docker dependencies check failed: "%s". Attempting to stop CLP containers ' "anyway...", diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index cb8ef8a367..fc0a8996a4 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -130,8 +130,8 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n :param should_compose_project_be_running: :param project_name: The Docker Compose project name to check. - :raises EnvironmentError: If any Docker dependency is not installed or the Docker Compose - project state doesn't match `should_compose_run`. + :raises OSError: If any Docker dependency is not installed or the Docker Compose project state + doesn't match `should_compose_run`. """ try: subprocess.run( @@ -142,13 +142,13 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n check=True, ) except subprocess.CalledProcessError: - raise EnvironmentError("docker is not installed or available on the path") + raise OSError("docker is not installed or available on the path") is_running = _is_docker_compose_project_running(project_name) if should_compose_project_be_running and not is_running: - raise EnvironmentError(f"Docker Compose project '{project_name}' is not running.") + raise OSError(f"Docker Compose project '{project_name}' is not running.") if not should_compose_project_be_running and is_running: - raise EnvironmentError("Docker Compose project '{project_name}' is already running.") + raise OSError("Docker Compose project '{project_name}' is already running.") def validate_port(port_name: str, hostname: str, port: int): @@ -697,7 +697,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: :param project_name: :return: Whether at least one instance is running. - :raises EnvironmentError: If Docker Compose is not installed or fails. + :raises OSError: If Docker Compose is not installed or fails. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: @@ -705,7 +705,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: running_instances = json.loads(output) return len(running_instances) >= 1 except subprocess.CalledProcessError: - raise EnvironmentError("Docker Compose is not installed or not functioning properly.") + raise OSError("Docker Compose is not installed or not functioning properly.") def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: From 22593d890dcdb0977141acba084a5eeca43ee8ca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 18 Oct 2025 09:05:25 -0400 Subject: [PATCH 291/408] refactor(general): Improve error handling for Docker and Docker Compose --- .../clp_package_utils/general.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index fc0a8996a4..363e4d4bf1 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -141,14 +141,17 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n stderr=subprocess.STDOUT, check=True, ) - except subprocess.CalledProcessError: - raise OSError("docker is not installed or available on the path") + except subprocess.CalledProcessError as e: + err_msg = "docker is not installed or available on the path" + raise OSError(err_msg) from e is_running = _is_docker_compose_project_running(project_name) if should_compose_project_be_running and not is_running: - raise OSError(f"Docker Compose project '{project_name}' is not running.") + err_msg = f"Docker Compose project '{project_name}' is not running." + raise OSError(err_msg) if not should_compose_project_be_running and is_running: - raise OSError("Docker Compose project '{project_name}' is already running.") + err_msg = "Docker Compose project '{project_name}' is already running." + raise OSError(err_msg) def validate_port(port_name: str, hostname: str, port: int): @@ -704,8 +707,9 @@ def _is_docker_compose_project_running(project_name: str) -> bool: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) running_instances = json.loads(output) return len(running_instances) >= 1 - except subprocess.CalledProcessError: - raise OSError("Docker Compose is not installed or not functioning properly.") + except subprocess.CalledProcessError as e: + err_msg = "Docker Compose is not installed or not functioning properly." + raise OSError(err_msg) from e def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: From bc69b26d4a001e95e8e182d889592b13282f95cc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 18 Oct 2025 09:06:27 -0400 Subject: [PATCH 292/408] refactor(general): Update dump_container_config call to remove return statement --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 363e4d4bf1..80aa19b8cd 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -329,7 +329,7 @@ def dump_shared_container_config(container_clp_config: CLPConfig, clp_config: CL :param container_clp_config: :param clp_config: """ - return dump_container_config(container_clp_config, clp_config, CLP_SHARED_CONFIG_FILENAME) + dump_container_config(container_clp_config, clp_config, CLP_SHARED_CONFIG_FILENAME) def generate_container_start_cmd( From 1d2ab61b76cbccdd98b1901bdd0cc03f5b1639af Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 18 Oct 2025 09:10:30 -0400 Subject: [PATCH 293/408] refactor(docs): Update exception documentation to use 'raise' format --- .../clp-package-utils/clp_package_utils/controller.py | 2 +- components/clp-package-utils/clp_package_utils/general.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 33d863fb15..676348c00c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -544,7 +544,7 @@ def _update_settings_object( :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. :param settings: The settings to update. :param updates: The updates. - :raises ValueError: If a key in `updates` doesn't exist in `settings`. + :raise ValueError: If a key in `updates` doesn't exist in `settings`. """ for key, value in updates.items(): if key not in settings: diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 80aa19b8cd..dc33f803c0 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -130,7 +130,7 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n :param should_compose_project_be_running: :param project_name: The Docker Compose project name to check. - :raises OSError: If any Docker dependency is not installed or the Docker Compose project state + :raise OSError: If any Docker dependency is not installed or the Docker Compose project state doesn't match `should_compose_run`. """ try: @@ -700,7 +700,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: :param project_name: :return: Whether at least one instance is running. - :raises OSError: If Docker Compose is not installed or fails. + :raise OSError: If Docker Compose is not installed or fails. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: @@ -725,7 +725,7 @@ def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): :param logs_dir: :param component_name: - :raises ValueError: If the path is invalid or can't be a directory. + :raise ValueError: If the path is invalid or can't be a directory. """ try: validate_path_could_be_dir(logs_dir) From d4c921ade727d59fb1ec8baae0f484e12783b22e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 18 Oct 2025 09:22:52 -0400 Subject: [PATCH 294/408] add missing `f` in f-string --- components/clp-package-utils/clp_package_utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index dc33f803c0..2df2e86be5 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -150,7 +150,7 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n err_msg = f"Docker Compose project '{project_name}' is not running." raise OSError(err_msg) if not should_compose_project_be_running and is_running: - err_msg = "Docker Compose project '{project_name}' is already running." + err_msg = f"Docker Compose project '{project_name}' is already running." raise OSError(err_msg) From e6bb002de031e2152480f8d87e6661724f30ab32 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 20 Oct 2025 02:13:30 -0400 Subject: [PATCH 295/408] Add retention periods check for garbage collector and disable the service's launch if none is configured --- .../clp-package-utils/clp_package_utils/controller.py | 11 +++++++++++ tools/deployment/package/docker-compose.base.yaml | 3 +++ 2 files changed, 14 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 676348c00c..1b7d8f5cbd 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -43,6 +43,7 @@ dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, + is_retention_period_configured, validate_db_config, validate_queue_config, validate_redis_config, @@ -505,6 +506,16 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: :return: Dictionary of environment variables necessary to launch the component. """ component_name = GARBAGE_COLLECTOR_COMPONENT_NAME + if not is_retention_period_configured(self._clp_config): + logger.info( + f"Retention period is not configured, skipping {component_name} creation..." + ) + return EnvVarsDict( + { + "CLP_GARBAGE_COLLECTOR_ENABLED": "0", + } + ) + logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 0663fba97b..38d5090510 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -342,6 +342,9 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" garbage-collector: <<: *service_defaults hostname: "garbage_collector" + deploy: + # Value must be either 0 or 1. Set to 0 to disable the garbage collector. + replicas: "${CLP_GARBAGE_COLLECTOR_ENABLED:-1}" environment: CLP_DB_PASS: "${CLP_DB_PASS:?error}" CLP_DB_USER: "${CLP_DB_USER:?error}" From 134e9595a7cb34568d9390e6f3f96d6848feaf01 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 11:11:51 -0400 Subject: [PATCH 296/408] Fix merge --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 0366e6c1b9..1316060538 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -454,6 +454,8 @@ class S3IngestionConfig(BaseModel): type: Literal[StorageType.S3.value] = StorageType.S3.value aws_authentication: AwsAuthentication + def transform_for_container(self): + pass class FsStorage(BaseModel): type: Literal[StorageType.FS.value] = StorageType.FS.value From 45b119e377bf0ef03d6d953d00318bc284049444 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 11:12:56 -0400 Subject: [PATCH 297/408] Fix lint --- components/clp-py-utils/clp_py_utils/clp_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 1316060538..04a61b337a 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -457,6 +457,7 @@ class S3IngestionConfig(BaseModel): def transform_for_container(self): pass + class FsStorage(BaseModel): type: Literal[StorageType.FS.value] = StorageType.FS.value directory: SerializablePath From 8a8400f60b9f7020db4a612e822e94773b0c3adf Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 11:37:02 -0400 Subject: [PATCH 298/408] Fix single return type --- .../scheduler/compress/task_manager/spider_task_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index ccf29c4077..9e8b35ccf2 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -22,6 +22,8 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None job_results = self._spider_job.get_results() if job_results is None: return None + if not isinstance(job_results, tuple): + return CompressionTaskResult.model_validate_json(int8_list_to_utf8_str(job_results)) return [ CompressionTaskResult.model_validate_json(int8_list_to_utf8_str(task_result)) for task_result in job_results From 50037cc46377181f31cad75d830f8ecfc87817a1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 12:25:55 -0400 Subject: [PATCH 299/408] Merge docker file --- .../package/docker-compose.spider.base.yaml | 71 ++++++++++--------- .../package/docker-compose.spider.yaml | 71 ++++++++----------- tools/docker-images/clp-package/Dockerfile | 1 + 3 files changed, 69 insertions(+), 74 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index 4bfa479ec8..40e49dd026 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -2,7 +2,7 @@ name: "clp-package-base" # Common service defaults. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" stop_grace_period: "3s" @@ -12,14 +12,19 @@ x-service-defaults: &service_defaults x-healthcheck-defaults: &healthcheck_defaults # Avoid lowering to prevent excessive resource usage. interval: "30s" + # Mark unhealthy after 3 failed probes. # - In steady state, ( + ) × 3 = ~90s before the service is marked unhealthy. # - From startup, (60s) + ~90s = ~150s before the service is marked unhealthy. retries: 3 + # Frequent checks during startup allow fast transition to healthy. start_interval: "2s" - # Ignore failures for ~15 frequent checks before counting retries. + + # Ignore failures for / ( + ) = ~15 frequent checks before + # counting retries. start_period: "60s" + # Short timeout since no remote communication is expected. timeout: "2s" @@ -35,24 +40,36 @@ x-volume-definitions: source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" target: "/etc/clp-config.yml" read_only: true - logs-input-readonly: &volume_root_logs_readonly + clp-logs: &volume_clp_logs + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}" + target: "/var/log" + logs-input-readonly: &volume_logs_input_readonly type: "bind" source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" read_only: true +volumes: + # Dummy volume to use when a bind mount is not desired. + empty: + driver_opts: + device: "tmpfs" + type: "tmpfs" + size: 0 + services: database: <<: *service_defaults - image: "${CLP_DB_IMAGE:-mysql:8.0.23}" + image: "${CLP_DB_IMAGE:-mariadb:10-jammy}" hostname: "database" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - MYSQL_DATABASE: "${CLP_DB_NAME}" - MYSQL_PASSWORD: "${CLP_DB_PASS}" - MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS}" - MYSQL_USER: "${CLP_DB_USER}" + MYSQL_DATABASE: "${CLP_DB_NAME:-clp-db}" + MYSQL_PASSWORD: "${CLP_DB_PASS:?error}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?error}" + MYSQL_USER: "${CLP_DB_USER:?error}" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" published: "${CLP_DB_PORT:-3306}" @@ -75,16 +92,16 @@ services: "mysqladmin", "ping", "--silent", "-h", "127.0.0.1", - "-u", "${CLP_DB_USER}", - "--password=${CLP_DB_PASS}" + "-u", "${CLP_DB_USER:?error}", + "--password=${CLP_DB_PASS:?error}" ] db-table-creator: <<: *service_defaults hostname: "db_table_creator" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly @@ -103,7 +120,6 @@ services: spider-scheduler: <<: *service_defaults container_name: "spider_scheduler" - image: "${CLP_PACKAGE_CONTAINER}" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -122,22 +138,16 @@ services: <<: *service_defaults hostname: "compression_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS}" CLP_DB_USER: "${CLP_DB_USER}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly - - *volume_root_logs_readonly - - type: "bind" - source: "${CLP_COMPRESSION_SCHEDULER_LOG_FILE_HOST:-./var/log/compression_scheduler.log}" - target: "/var/log/compression_scheduler.log" - - type: "bind" + - *volume_clp_logs + - *volume_logs_input_readonly depends_on: db-table-creator: condition: "service_completed_successfully" @@ -151,10 +161,7 @@ services: compression-worker: <<: *service_defaults container_name: "compression_worker" - image: "${CLP_PACKAGE_CONTAINER}" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" @@ -165,16 +172,12 @@ services: volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly - - *volume_root_logs_readonly - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" - target: "/var/data/archives" - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/staged-archives}" - target: "/var/data/staged-archives" - - type: "bind" - source: "${CLP_COMPRESSION_WORKER_LOGS_DIR_HOST:-./var/log/compression_worker}" - target: "/var/log/compression_worker" + - *volume_clp_logs + - *volume_logs_input_readonly + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ +${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ +${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index 82bfc88eb3..72c1b491d2 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -5,7 +5,7 @@ include: ["docker-compose.spider.base.yaml"] # Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for # documentation. x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER:-clp-package}" + image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" stop_grace_period: "3s" @@ -27,6 +27,10 @@ x-volume-definitions: source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" target: "/etc/clp-config.yml" read_only: true + clp-logs: &volume_clp_logs + type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}" + target: "/var/log" services: @@ -36,8 +40,8 @@ services: hostname: "queue" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS}" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER}" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?error}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?error}" RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" @@ -74,6 +78,11 @@ services: - type: "bind" source: "${CLP_REDIS_LOGS_DIR_HOST:-./var/log/redis}" target: "/var/log/redis" + command: [ + "redis-server", + "/usr/local/etc/redis/redis.conf", + "--requirepass", "${CLP_REDIS_PASS:?error}" + ] healthcheck: <<: *healthcheck_defaults test: [ @@ -81,14 +90,9 @@ services: "redis-cli", "-h", "127.0.0.1", "-p", "6379", - "-a", "${CLP_REDIS_PASS}", + "-a", "${CLP_REDIS_PASS:?error}", "PING" ] - command: [ - "redis-server", - "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS}" - ] results-cache: <<: *service_defaults @@ -110,16 +114,16 @@ services: - type: "bind" source: "${CLP_RESULTS_CACHE_LOGS_DIR_HOST:-./var/log/results_cache}" target: "/var/log/mongodb" + command: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0", + ] healthcheck: <<: *healthcheck_defaults test: [ "CMD-SHELL", "echo 'db.runCommand(\"ping\").ok' | mongosh 127.0.0.1:27017/test --quiet" ] - command: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0", - ] results-cache-indices-creator: <<: *service_defaults @@ -137,23 +141,22 @@ services: "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", ] + query-scheduler: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?error}" + CLP_DB_USER: "${CLP_DB_USER:?error}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_QUERY_SCHEDULER_LOG_FILE_HOST:-./var/log/query_scheduler.log}" - target: "/var/log/query_scheduler.log" + - *volume_clp_logs depends_on: db-table-creator: condition: "service_completed_successfully" @@ -180,9 +183,7 @@ services: <<: *service_defaults hostname: "query_worker" environment: - AWS_ACCESS_KEY_ID: "${CLP_AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${CLP_AWS_SECRET_ACCESS_KEY}" - BROKER_URL: "amqp://${CLP_QUEUE_USER}:${CLP_QUEUE_PASS}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" @@ -190,22 +191,14 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-./var/data/archives}" - target: "/var/data/archives" - - type: "bind" - source: "${CLP_QUERY_WORKER_LOGS_DIR_HOST:-./var/log/query_worker}" - target: "/var/log/query_worker" - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/staged-streams}" - target: "/var/data/staged-streams" - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" - target: "/var/data/streams" + - *volume_clp_logs + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" + - "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams" command: [ "python3", "-u", @@ -229,9 +222,7 @@ services: PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_REDUCER_LOGS_DIR_HOST:-./var/log/reducer}" - target: "/var/log/reducer" + - *volume_clp_logs depends_on: query-scheduler: condition: "service_healthy" @@ -243,4 +234,4 @@ services: "--config", "/etc/clp-config.yml", "--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}", "--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}" - ] + ] \ No newline at end of file diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index a61e665265..95f30c320f 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -16,6 +16,7 @@ COPY ./build/clp-package /opt/clp COPY --link ./build/deps/cpp/mariadb-connector-cpp-install/lib/*/libmariadbcpp.so* /opt/clp/lib/ ENV CLP_HOME="/opt/clp" +ENV LD_LIBRARY_PATH="/opt/clp/lib:${LD_LIBRARY_PATH}" ENV PATH="${CLP_HOME}/bin:${PATH}" ENV PATH="${CLP_HOME}/sbin:${PATH}" ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" From 079ff48f01784c0b2cfbac275c51513874a06c9a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 14:57:02 -0400 Subject: [PATCH 300/408] Fix LD_LIBRARY_PATH --- tools/deployment/package/docker-compose.spider.base.yaml | 4 +++- tools/docker-images/clp-package/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index 40e49dd026..faa4b3061f 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -120,6 +120,8 @@ services: spider-scheduler: <<: *service_defaults container_name: "spider_scheduler" + environment: + LD_LIBRARY_PATH: "/opt/clp/lib" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -167,8 +169,8 @@ services: CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/compression_worker" CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" + LD_LIBRARY_PATH: "/opt/clp/lib" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - PATH: "/opt/clp/lib/python3/site-packages/bin:${PATH}" volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 95f30c320f..f688a0c760 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -16,7 +16,7 @@ COPY ./build/clp-package /opt/clp COPY --link ./build/deps/cpp/mariadb-connector-cpp-install/lib/*/libmariadbcpp.so* /opt/clp/lib/ ENV CLP_HOME="/opt/clp" -ENV LD_LIBRARY_PATH="/opt/clp/lib:${LD_LIBRARY_PATH}" +ENV LD_LIBRARY_PATH="${CLP_HOME}/lib" ENV PATH="${CLP_HOME}/bin:${PATH}" ENV PATH="${CLP_HOME}/sbin:${PATH}" ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" From 2c08c945abc59e8774670f11cba0d12bd15e00fa Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 15:00:58 -0400 Subject: [PATCH 301/408] Bug fix --- .../scheduler/compress/task_manager/spider_task_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index 9e8b35ccf2..9c68ad4f5e 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -23,7 +23,9 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None if job_results is None: return None if not isinstance(job_results, tuple): - return CompressionTaskResult.model_validate_json(int8_list_to_utf8_str(job_results)) + return [ + CompressionTaskResult.model_validate_json(int8_list_to_utf8_str(job_results)) + ] return [ CompressionTaskResult.model_validate_json(int8_list_to_utf8_str(task_result)) for task_result in job_results From 812bb252fe058653a805def91c77eacc6fc47ffe Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 15:13:18 -0400 Subject: [PATCH 302/408] Use empty tag ids instead of None --- .../job_orchestration/executor/compress/compression_task.py | 2 +- .../scheduler/compress/compression_scheduler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index b50e354363..c27f6690dd 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -90,7 +90,7 @@ def update_job_metadata_and_tags( tag_ids: List[int], archive_stats: Dict[str, Any], ) -> None: - if tag_ids is not None: + if not len(tag_ids) == 0: update_tags(db_cursor, table_prefix, dataset, archive_stats["id"], tag_ids) increment_compression_job_metadata( db_cursor, diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index f7a25edc73..9c64ea05c9 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -342,7 +342,7 @@ def search_and_schedule_new_tasks( ) db_conn.commit() - tag_ids = None + tag_ids = [] if clp_io_config.output.tags: tags_table_name = get_tags_table_name(table_prefix, dataset) db_cursor.executemany( From 7ef1c9444cfbc04b820fcc50afc95629fae917d6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 15:15:52 -0400 Subject: [PATCH 303/408] Revert None support for tag_ids --- .../job_orchestration/executor/compress/spider_compress.py | 2 +- .../scheduler/compress/task_manager/spider_task_manager.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index 84bc33e1a1..fa09549134 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -39,7 +39,7 @@ def compress( compression_entry_point( int(job_id), int(task_id), - [int(tag_id) for tag_id in tag_ids] if len(tag_ids) > 0 else None, + [int(tag_id) for tag_id in tag_ids], int8_list_to_utf8_str(clp_io_config_json), int8_list_to_utf8_str(paths_to_compress_json), json.loads(int8_list_to_utf8_str(clp_metadata_db_connection_config_json)), diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index 56b6342681..9e8b35ccf2 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -40,10 +40,7 @@ def submit(self, task_params: list[dict[str, Any]]) -> TaskManager.ResultHandle: for task_param in task_params: job_args.append(spider_py.Int64(task_param["job_id"])) job_args.append(spider_py.Int64(task_param["task_id"])) - if "tag_ids" in task_param and task_param["tag_ids"] is not None: - job_args.append([spider_py.Int64(tag_id) for tag_id in task_param["tag_ids"]]) - else: - job_args.append([]) + job_args.append([spider_py.Int64(tag_id) for tag_id in task_param["tag_ids"]]) job_args.append(utf8_str_to_int8_list(task_param["clp_io_config_json"])) job_args.append(utf8_str_to_int8_list(task_param["paths_to_compress_json"])) job_args.append( From e210cea6b4c9709aadb6aa9f0308203d08089890 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 20 Oct 2025 15:46:59 -0400 Subject: [PATCH 304/408] Fix yaml lint --- tools/deployment/package/docker-compose.spider.base.yaml | 1 - tools/deployment/package/docker-compose.spider.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index faa4b3061f..8319534b56 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -267,4 +267,3 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" "-m", "job_orchestration.garbage_collector.garbage_collector", "--config", "/etc/clp-config.yml", ] - diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index 72c1b491d2..9c11905e24 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -234,4 +234,4 @@ services: "--config", "/etc/clp-config.yml", "--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}", "--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}" - ] \ No newline at end of file + ] From b3e673fa95e94f7a2249585326a6a0e4cb4f8213 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 20 Oct 2025 16:46:12 -0400 Subject: [PATCH 305/408] fix(docker): Use `--link` flag in COPY command for clp-package Dockerfile. --- tools/docker-images/clp-package/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index b3e5a0e166..5ca3e3d6c0 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -17,7 +17,7 @@ FROM scratch COPY --link --from=base / / ENV CLP_HOME="/opt/clp" -COPY ./build/clp-package ${CLP_HOME} +COPY --link ./build/clp-package ${CLP_HOME} ENV PATH="${CLP_HOME}/bin:${PATH}" ENV PATH="${CLP_HOME}/sbin:${PATH}" \ From 65f2d1b98a0565af279de1b794f9a696bfccc912 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 20 Oct 2025 16:48:03 -0400 Subject: [PATCH 306/408] fix(docker): Reorder COPY command in clp-package Dockerfile --- tools/docker-images/clp-package/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 5ca3e3d6c0..38e0c45d18 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -17,13 +17,13 @@ FROM scratch COPY --link --from=base / / ENV CLP_HOME="/opt/clp" -COPY --link ./build/clp-package ${CLP_HOME} - ENV PATH="${CLP_HOME}/bin:${PATH}" ENV PATH="${CLP_HOME}/sbin:${PATH}" \ PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" \ USER="clp-user" +COPY --link ./build/clp-package ${CLP_HOME} + RUN useradd --uid 1000 --shell /bin/bash --home-dir ${CLP_HOME} ${USER} USER ${USER} WORKDIR ${CLP_HOME} From b5222125727b49cc8266eae956e35cf4fdf5a981 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 20 Oct 2025 17:03:41 -0400 Subject: [PATCH 307/408] fix(docker): Add ARG for UID and set ownership in COPY command for clp-package Dockerfile. --- tools/docker-images/clp-package/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 38e0c45d18..4a8a5b2c40 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -16,14 +16,15 @@ RUN apt-get clean \ FROM scratch COPY --link --from=base / / +ARG UID=1000 ENV CLP_HOME="/opt/clp" ENV PATH="${CLP_HOME}/bin:${PATH}" ENV PATH="${CLP_HOME}/sbin:${PATH}" \ PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" \ USER="clp-user" -COPY --link ./build/clp-package ${CLP_HOME} - -RUN useradd --uid 1000 --shell /bin/bash --home-dir ${CLP_HOME} ${USER} +RUN useradd --uid ${UID} --shell /bin/bash --home-dir ${CLP_HOME} ${USER} USER ${USER} WORKDIR ${CLP_HOME} + +COPY --link --chown=${UID} ./build/clp-package ${CLP_HOME} From 42fba6cdeb88f49e7fbef6526d543dff22e94e28 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 20 Oct 2025 17:13:46 -0400 Subject: [PATCH 308/408] merge with the latest from #1413 --- tools/docker-images/clp-package/Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 0057ffd7b7..d93a27b89d 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -16,13 +16,18 @@ RUN apt-get clean \ FROM scratch COPY --link --from=base / / +ARG UID=1000 ENV CLP_HOME="/opt/clp" ENV LD_LIBRARY_PATH="${CLP_HOME}/lib" \ PATH="${CLP_HOME}/bin:${PATH}" \ - PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" + PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" \ + USER="clp-user" ENV PATH="${CLP_HOME}/sbin:${PATH}" -USER 1000:1000 +RUN useradd --uid ${UID} --shell /bin/bash --home-dir ${CLP_HOME} ${USER} +USER ${USER} +WORKDIR ${CLP_HOME} -COPY --link ./build/clp-package ${CLP_HOME} -COPY --link ./build/deps/cpp/mariadb-connector-cpp-install/lib/*/libmariadbcpp.so* ${CLP_HOME}/lib/ +COPY --link --chown=${UID} ./build/clp-package ${CLP_HOME} +COPY --link --chown=${UID} ./build/deps/cpp/mariadb-connector-cpp-install/lib/*/libmariadbcpp.so* \ + ${CLP_HOME}/lib/ From b0d0bccb5918490e14f4ef6769bf34441771e787 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 05:57:51 -0400 Subject: [PATCH 309/408] refactor(clp-package-utils): Remove unused `pass` statements from abstract methods. --- components/clp-package-utils/clp_package_utils/controller.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 1b7d8f5cbd..894153f304 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -87,14 +87,12 @@ def start(self) -> None: """ Starts the components. """ - pass @abstractmethod def stop(self) -> None: """ Stops the components. """ - pass @abstractmethod def _set_up_env(self) -> None: @@ -102,7 +100,6 @@ def _set_up_env(self) -> None: Sets up all components to run by preparing environment variables, directories, and configuration files. """ - pass def _set_up_env_for_database(self) -> EnvVarsDict: """ From 955a0ac0f3e7ae6fc9d2041f6426636661aa25a2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:07:50 -0400 Subject: [PATCH 310/408] refactor(clp-config): Remove unused typing import 'Set'. --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index ce2de4a3e5..c74389a106 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Annotated, Any, ClassVar, Literal, Optional, Set, Union +from typing import Annotated, Any, ClassVar, Literal, Optional, Union from pydantic import ( BaseModel, From 106fdda0aa6ad80c257a64124ad926edb55f5aaa Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:09:10 -0400 Subject: [PATCH 311/408] apply doc suggestions --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c74389a106..572b555845 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -778,7 +778,7 @@ def validate_presto_config(self): def transform_for_container(self): """ - Converts all relevant directories to absolute paths inside the container and updates + Converts all relevant directories to absolute paths inside the container, and updates component hostnames and ports to their container service names and default ports. """ self.data_directory = pathlib.Path("/") / CLP_DEFAULT_DATA_DIRECTORY_PATH From 7a1ea6e957e7145fd94489b95ab6defd079740ba Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:18:54 -0400 Subject: [PATCH 312/408] refactor(docker-compose): Update environment variable error messages for clarity. --- .../package/docker-compose.base.yaml | 42 +++++++++---------- tools/deployment/package/docker-compose.yaml | 12 +++--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 38d5090510..e98eccd70a 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -66,9 +66,9 @@ services: user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME:-clp-db}" - MYSQL_PASSWORD: "${CLP_DB_PASS:?error}" - MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?error}" - MYSQL_USER: "${CLP_DB_USER:?error}" + MYSQL_PASSWORD: "${CLP_DB_PASS:?Please set a value.}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?Please set a value.}" + MYSQL_USER: "${CLP_DB_USER:?Please set a value.}" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" published: "${CLP_DB_PORT:-3306}" @@ -91,16 +91,16 @@ services: "mysqladmin", "ping", "--silent", "-h", "127.0.0.1", - "-u", "${CLP_DB_USER:?error}", - "--password=${CLP_DB_PASS:?error}" + "-u", "${CLP_DB_USER:?Please set a value.}", + "--password=${CLP_DB_PASS:?Please set a value.}" ] db-table-creator: <<: *service_defaults hostname: "db_table_creator" environment: - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly @@ -121,8 +121,8 @@ services: hostname: "queue" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?error}" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?error}" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?Please set a value.}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?Please set a value.}" RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" @@ -162,7 +162,7 @@ services: command: [ "redis-server", "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS:?error}" + "--requirepass", "${CLP_REDIS_PASS:?Please set a value.}" ] healthcheck: <<: *healthcheck_defaults @@ -171,7 +171,7 @@ services: "redis-cli", "-h", "127.0.0.1", "-p", "6379", - "-a", "${CLP_REDIS_PASS:?error}", + "-a", "${CLP_REDIS_PASS:?Please set a value.}", "PING" ] @@ -227,14 +227,14 @@ services: hostname: "compression_scheduler" stop_grace_period: "300s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -259,7 +259,7 @@ services: hostname: "compression_worker" stop_grace_period: "60s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" @@ -267,7 +267,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly @@ -299,8 +299,8 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" environment: AWS_ACCESS_KEY_ID: "${CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID:-}" AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}" - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" HOST: "0.0.0.0" NODE_ENV: "production" NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" @@ -346,8 +346,8 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" # Value must be either 0 or 1. Set to 0 to disable the garbage collector. replicas: "${CLP_GARBAGE_COLLECTOR_ENABLED:-1}" environment: - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/garbage_collector" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index e227f3ab3b..bc731e6fe7 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -37,14 +37,14 @@ services: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -74,7 +74,7 @@ services: <<: *service_defaults hostname: "query_worker" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" @@ -82,7 +82,7 @@ services: CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - *volume_aws_config_readonly - *volume_clp_config_readonly From a732939be9c115e6be9784fc3f01f3c3a822648a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:50:16 -0400 Subject: [PATCH 313/408] Adjust volume bindings to fix directory mappings when CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST/CLP_ARCHIVE_OUTPUT_DIR_HOST is undefined --- tools/deployment/package/docker-compose.base.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index e98eccd70a..927414f930 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -274,9 +274,9 @@ services: - *volume_clp_logs - *volume_logs_input_readonly - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" +${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" +${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" From c687f7839bc82f332e10922059c8a82bc4c9d184 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:51:00 -0400 Subject: [PATCH 314/408] fix string indents --- tools/deployment/package/docker-compose.base.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 927414f930..8ef789711e 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -273,10 +273,10 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ + :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ + :${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" From 19e8a9d64863cd883dcfd3245f04dd3fa92ca922 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 06:52:12 -0400 Subject: [PATCH 315/408] apply doc suggestions --- tools/deployment/package/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index bc731e6fe7..8eb0fe79e6 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -2,7 +2,7 @@ name: "clp-package" include: ["docker-compose.base.yaml"] -# Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for +# The `x-*` definitions below are duplicated from docker-compose.base.yaml. Refer to that file for # documentation. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" From c53c0f1961f0a52e5c4a870fdc6a37709b71da3a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:30:58 -0400 Subject: [PATCH 316/408] apply doc suggestions --- .../clp_package_utils/controller.py | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 894153f304..35987e0cca 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -120,18 +120,19 @@ def _set_up_env_for_database(self) -> EnvVarsDict: _chown_paths_if_root(data_dir, logs_dir) env_vars = EnvVarsDict() - # Connection + + # Connection params env_vars |= { "CLP_DB_HOST": _get_ip_from_hostname(self._clp_config.database.host), "CLP_DB_NAME": self._clp_config.database.name, "CLP_DB_PORT": str(self._clp_config.database.port), } - # Credential + # Credentials env_vars |= { "CLP_DB_PASS": self._clp_config.database.password, "CLP_DB_USER": self._clp_config.database.username, } - # Path + # Paths env_vars |= { "CLP_DB_CONF_LOGGING_FILE_HOST": str(conf_logging_file), "CLP_DB_DATA_DIR_HOST": str(data_dir), @@ -162,17 +163,18 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: _chown_paths_if_root(logs_dir) env_vars = EnvVarsDict() - # Connection + + # Connection params env_vars |= { "CLP_QUEUE_HOST": _get_ip_from_hostname(self._clp_config.queue.host), "CLP_QUEUE_PORT": str(self._clp_config.queue.port), } - # Credential + # Credentials env_vars |= { "CLP_QUEUE_PASS": self._clp_config.queue.password, "CLP_QUEUE_USER": self._clp_config.queue.username, } - # Path + # Paths env_vars |= { "CLP_QUEUE_LOGS_DIR_HOST": str(logs_dir), } @@ -198,23 +200,24 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: _chown_paths_if_root(data_dir, logs_dir) env_vars = EnvVarsDict() - # Backend + + # Backend databases env_vars |= { "CLP_REDIS_BACKEND_DB_COMPRESSION": str( self._clp_config.redis.compression_backend_database ), "CLP_REDIS_BACKEND_DB_QUERY": str(self._clp_config.redis.query_backend_database), } - # Connection + # Connection params env_vars |= { "CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host), "CLP_REDIS_PORT": str(self._clp_config.redis.port), } - # Credential + # Credentials env_vars |= { "CLP_REDIS_PASS": self._clp_config.redis.password, } - # Path + # Paths env_vars |= { "CLP_REDIS_CONF_FILE_HOST": str(conf_file), "CLP_REDIS_DATA_DIR_HOST": str(data_dir), @@ -242,19 +245,20 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: _chown_paths_if_root(data_dir, logs_dir) env_vars = EnvVarsDict() - # Collection + + # Collections env_vars |= { "CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME": ( self._clp_config.results_cache.stream_collection_name ), } - # Connection + # Connection params env_vars |= { "CLP_RESULTS_CACHE_DB_NAME": self._clp_config.results_cache.db_name, "CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(self._clp_config.results_cache.host), "CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port), } - # Path + # Paths env_vars |= { "CLP_RESULTS_CACHE_CONF_FILE_HOST": str(conf_file), "CLP_RESULTS_CACHE_DATA_DIR_HOST": str(data_dir), @@ -273,6 +277,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": ( @@ -292,6 +297,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, @@ -313,13 +319,14 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": ( self._clp_config.compression_worker.logging_level ), } - # Resource + # Resources env_vars |= { "CLP_COMPRESSION_WORKER_CONCURRENCY": str(num_workers), } @@ -340,11 +347,12 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, } - # Resource + # Resources env_vars |= { "CLP_QUERY_WORKER_CONCURRENCY": str(num_workers), } @@ -365,11 +373,12 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() + # Logging env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, } - # Resource + # Resources env_vars |= { "CLP_REDUCER_CONCURRENCY": str(num_workers), "CLP_REDUCER_UPSERT_INTERVAL": str(self._clp_config.reducer.upsert_interval), @@ -484,7 +493,8 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: settings_json_file.write(json.dumps(server_settings_json)) env_vars = EnvVarsDict() - # Connection + + # Connection params env_vars |= { "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), "CLP_WEBUI_PORT": str(self._clp_config.webui.port), @@ -644,7 +654,7 @@ def _set_up_env(self) -> None: # Prepare environment variables for all components. env_vars = EnvVarsDict() - # Credential + # Credentials if self._clp_config.stream_output.storage.type == StorageType.S3: stream_output_aws_auth = ( self._clp_config.stream_output.storage.s3_config.aws_authentication @@ -659,7 +669,7 @@ def _set_up_env(self) -> None: ), } - # Identity + # Identity config env_vars |= { "CLP_FIRST_PARTY_SERVICE_UID_GID": DEFAULT_UID_GID, "CLP_THIRD_PARTY_SERVICE_UID_GID": ( @@ -667,13 +677,13 @@ def _set_up_env(self) -> None: ), } - # Package + # Package config env_vars |= { "CLP_PACKAGE_CONTAINER_IMAGE_REF": self._clp_config.container_image_ref, "CLP_PACKAGE_STORAGE_ENGINE": self._clp_config.package.storage_engine, } - # Path + # Paths aws_config_dir = self._clp_config.aws_config_directory env_vars |= { # General @@ -682,13 +692,15 @@ def _set_up_env(self) -> None: # Config "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), } - # Input + + # Input config if self._clp_config.logs_input.type == StorageType.FS: env_vars["CLP_LOGS_INPUT_DIR_CONTAINER"] = str( container_clp_config.logs_input.directory ) env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory) - # Output + + # Output config archive_output_dir_str = str(self._clp_config.archive_output.get_directory()) stream_output_dir_str = str(self._clp_config.stream_output.get_directory()) if self._clp_config.archive_output.storage.type == StorageType.FS: @@ -700,7 +712,7 @@ def _set_up_env(self) -> None: if self._clp_config.stream_output.storage.type == StorageType.S3: env_vars["CLP_STAGED_STREAM_OUTPUT_DIR_HOST"] = stream_output_dir_str - # Component-specific + # Component-specific config env_vars |= self._set_up_env_for_database() env_vars |= self._set_up_env_for_queue() env_vars |= self._set_up_env_for_redis() From a2fa02c8964d76d0ece0eef0ca45e3a5b651cfa6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:31:10 -0400 Subject: [PATCH 317/408] fix(clp-package-utils): Remove extra space in `.env` file comment. --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 35987e0cca..395157e320 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -725,7 +725,7 @@ def _set_up_env(self) -> None: env_vars |= self._set_up_env_for_webui(container_clp_config) env_vars |= self._set_up_env_for_garbage_collector() - # Write the environment variables to the `.env ` file. + # Write the environment variables to the `.env` file. with open(f"{self._clp_home}/.env", "w") as env_file: for key, value in env_vars.items(): if value is None: From 05810bec3b459cf089580abb59aa2ee3971f3b27 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:32:09 -0400 Subject: [PATCH 318/408] apply doc suggestions --- components/clp-package-utils/clp_package_utils/controller.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 395157e320..ea5be2ae4f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -686,11 +686,9 @@ def _set_up_env(self) -> None: # Paths aws_config_dir = self._clp_config.aws_config_directory env_vars |= { - # General + "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), "CLP_DATA_DIR_HOST": str(self._clp_config.data_directory), "CLP_LOGS_DIR_HOST": str(self._clp_config.logs_directory), - # Config - "CLP_AWS_CONFIG_DIR_HOST": (None if aws_config_dir is None else str(aws_config_dir)), } # Input config From 2a02ecb0d21d1d5b9c799bd5d9700d7907120608 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:32:54 -0400 Subject: [PATCH 319/408] apply doc suggestions --- components/clp-package-utils/clp_package_utils/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ea5be2ae4f..9743fa4803 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -651,7 +651,6 @@ def _set_up_env(self) -> None: num_workers = self._get_num_workers() dump_shared_container_config(container_clp_config, self._clp_config) - # Prepare environment variables for all components. env_vars = EnvVarsDict() # Credentials From 678de80eac42f986fb1d2ca1fd448a9104b9d115 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:34:36 -0400 Subject: [PATCH 320/408] Update comment for `_get_num_workers` method - apply doc suggestion --- components/clp-package-utils/clp_package_utils/controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 9743fa4803..76697bad65 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -639,10 +639,9 @@ def stop(self) -> None: @staticmethod def _get_num_workers() -> int: """ - TODO: Revisit after moving from single-container to multi-container workers. See issue - @y-scope/clp#1424 for details. :return: Number of worker processes to run. """ + # This will change when we move from single to multi-container workers. See y-scope/clp#1424 return multiprocessing.cpu_count() // 2 def _set_up_env(self) -> None: From c30adaec998e6749c86a2d2e5a5dc8c08ee08296 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:35:29 -0400 Subject: [PATCH 321/408] Simplify log message for CLP start action - apply suggestion --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 76697bad65..2f049e56f8 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -608,7 +608,7 @@ def start(self) -> None: cwd=self._clp_home, check=True, ) - logger.info("CLP is started.") + logger.info("CLP started.") def stop(self) -> None: """ From 07d1e3512aefd3b9d9dd1a2723efe52855070ba2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 09:37:55 -0400 Subject: [PATCH 322/408] Update comments for consistency in `controller.py`. --- .../clp_package_utils/controller.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 2f049e56f8..de4390de91 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -138,7 +138,7 @@ def _set_up_env_for_database(self) -> EnvVarsDict: "CLP_DB_DATA_DIR_HOST": str(data_dir), "CLP_DB_LOGS_DIR_HOST": str(logs_dir), } - # Runtime + # Runtime params env_vars |= { "CLP_DB_IMAGE": ( "mysql:8.0.23" if self._clp_config.database.type == "mysql" else "mariadb:10-jammy" @@ -278,7 +278,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL": ( self._clp_config.compression_scheduler.logging_level @@ -298,7 +298,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_QUERY_SCHEDULER_LOGGING_LEVEL": self._clp_config.query_scheduler.logging_level, } @@ -320,7 +320,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_COMPRESSION_WORKER_LOGGING_LEVEL": ( self._clp_config.compression_worker.logging_level @@ -348,7 +348,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_QUERY_WORKER_LOGGING_LEVEL": self._clp_config.query_worker.logging_level, } @@ -374,7 +374,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= { "CLP_REDUCER_LOGGING_LEVEL": self._clp_config.reducer.logging_level, } @@ -499,7 +499,7 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "CLP_WEBUI_HOST": _get_ip_from_hostname(self._clp_config.webui.host), "CLP_WEBUI_PORT": str(self._clp_config.webui.port), } - # Security + # Security params env_vars |= { "CLP_WEBUI_RATE_LIMIT": str(self._clp_config.webui.rate_limit), } @@ -530,7 +530,7 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: env_vars = EnvVarsDict() - # Logging + # Logging params env_vars |= {"CLP_GC_LOGGING_LEVEL": self._clp_config.garbage_collector.logging_level} return env_vars From 0bf6b70fbb3cab4563e3ba9ba8075c74cf3ae3ff Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:18:57 -0400 Subject: [PATCH 323/408] feat(clp-package-utils): Add custom exceptions for Docker dependency errors; do not attempt to stop CLP when DockerComposeProjectNotRunningError is hit --- .../clp_package_utils/controller.py | 11 +++- .../clp_package_utils/general.py | 66 +++++++++++++++---- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index de4390de91..b3c5313f70 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -40,6 +40,7 @@ from clp_package_utils.general import ( check_docker_dependencies, CONTAINER_CLP_HOME, + DockerComposeProjectNotRunningError, dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, @@ -49,6 +50,7 @@ validate_redis_config, validate_results_cache_config, validate_webui_config, + DockerDependencyError, ) LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH @@ -589,6 +591,7 @@ def start(self) -> None: """ Starts CLP's components using Docker Compose. + :raise: Propagates `check_docker_dependencies`'s exceptions. :raise: Propagates `subprocess.run`'s exceptions. """ check_docker_dependencies( @@ -620,7 +623,13 @@ def stop(self) -> None: check_docker_dependencies( should_compose_project_be_running=True, project_name=self._project_name ) - except OSError as e: + except DockerComposeProjectNotRunningError: + logger.info( + "Docker Compose project '%s' is not running. Nothing to stop.", + self._project_name, + ) + return + except DockerDependencyError as e: logger.warning( 'Docker dependencies check failed: "%s". Attempting to stop CLP containers ' "anyway...", diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 2df2e86be5..6d769ee79a 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -49,6 +49,47 @@ DOCKER_MOUNT_TYPE_STRINGS = ["bind"] +class DockerDependencyError(OSError): + """Base class for errors related to Docker dependencies.""" + + +class DockerNotAvailableError(DockerDependencyError): + """Raised when Docker or Docker Compose is unavailable.""" + + def __init__(self, base_message: str, process_error: subprocess.CalledProcessError): + message = base_message + output_chunks: list[str] = [] + for stream in (process_error.output, process_error.stderr): + if stream is None: + continue + if isinstance(stream, bytes): + text = stream.decode(errors="replace") + else: + text = str(stream) + text = text.strip() + if text: + output_chunks.append(text) + if len(output_chunks) > 0: + message = f"{base_message}\n" + "\n".join(output_chunks) + super().__init__(errno.ENOENT, message) + + +class DockerComposeProjectNotRunningError(DockerDependencyError): + """Raised when an expected Docker Compose project is not running.""" + + def __init__(self, project_name: str): + super().__init__(errno.ESRCH, f"Docker Compose project '{project_name}' is not running.") + + +class DockerComposeProjectAlreadyRunningError(DockerDependencyError): + """Raised when a Docker Compose project is already running but should not be.""" + + def __init__(self, project_name: str): + super().__init__( + errno.EEXIST, f"Docker Compose project '{project_name}' is already running." + ) + + class DockerMountType(enum.IntEnum): BIND = 0 @@ -130,8 +171,9 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n :param should_compose_project_be_running: :param project_name: The Docker Compose project name to check. - :raise OSError: If any Docker dependency is not installed or the Docker Compose project state - doesn't match `should_compose_run`. + :raise DockerNotAvailableError: If any Docker dependency is not installed. + :raise DockerComposeProjectNotRunningError: If the project isn't running when it should be. + :raise DockerComposeProjectAlreadyRunningError: If the project is running when it shouldn't be. """ try: subprocess.run( @@ -142,16 +184,16 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n check=True, ) except subprocess.CalledProcessError as e: - err_msg = "docker is not installed or available on the path" - raise OSError(err_msg) from e + raise DockerNotAvailableError( + "docker is not installed or available on the path", e + ) from e is_running = _is_docker_compose_project_running(project_name) + if should_compose_project_be_running and not is_running: - err_msg = f"Docker Compose project '{project_name}' is not running." - raise OSError(err_msg) + raise DockerComposeProjectNotRunningError(project_name) if not should_compose_project_be_running and is_running: - err_msg = f"Docker Compose project '{project_name}' is already running." - raise OSError(err_msg) + raise DockerComposeProjectAlreadyRunningError(project_name) def validate_port(port_name: str, hostname: str, port: int): @@ -700,7 +742,8 @@ def _is_docker_compose_project_running(project_name: str) -> bool: :param project_name: :return: Whether at least one instance is running. - :raise OSError: If Docker Compose is not installed or fails. + :raise DockerNotAvailableError: If Docker Compose is not installed or fails. The error message + includes Docker's command output when available. """ cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"] try: @@ -708,8 +751,9 @@ def _is_docker_compose_project_running(project_name: str) -> bool: running_instances = json.loads(output) return len(running_instances) >= 1 except subprocess.CalledProcessError as e: - err_msg = "Docker Compose is not installed or not functioning properly." - raise OSError(err_msg) from e + raise DockerNotAvailableError( + "Docker Compose is not installed or not functioning properly.", e + ) from e def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: From 2986082589779cd037cbe7fddb36aaf2321ecbe7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:23:42 -0400 Subject: [PATCH 324/408] fix(clp-package-utils): Use `docker --version` to check docker availability; Replace `subprocess.run` with `subprocess.check_output` for Docker check. --- components/clp-package-utils/clp_package_utils/general.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 6d769ee79a..58ff00ea6b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -176,12 +176,9 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n :raise DockerComposeProjectAlreadyRunningError: If the project is running when it shouldn't be. """ try: - subprocess.run( - "command -v docker", - shell=True, - stdout=subprocess.DEVNULL, + subprocess.check_output( + ["docker", "--version"], stderr=subprocess.STDOUT, - check=True, ) except subprocess.CalledProcessError as e: raise DockerNotAvailableError( From e499443df01a084ba34dce3696c5e32a465ee83f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:28:59 -0400 Subject: [PATCH 325/408] Avoid mapping ~/.aws on host when CLP_AWS_CONFIG_DIR_HOST is undefined --- tools/deployment/package/docker-compose.base.yaml | 9 ++------- tools/deployment/package/docker-compose.yaml | 7 +------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 8ef789711e..542bc31adb 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -30,11 +30,6 @@ x-healthcheck-defaults: &healthcheck_defaults # Common volume definitions. x-volume-definitions: - aws-config-readonly: &volume_aws_config_readonly - type: "bind" - source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" - target: "/.aws" - read_only: true clp-config-readonly: &volume_clp_config_readonly type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" @@ -236,10 +231,10 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -269,7 +264,6 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly @@ -277,6 +271,7 @@ services: :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index 8eb0fe79e6..bc283ed7cd 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -17,11 +17,6 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "60s" timeout: "2s" x-volume-definitions: - aws-config-readonly: &volume_aws_config_readonly - type: "bind" - source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" - target: "/.aws" - read_only: true clp-config-readonly: &volume_clp_config_readonly type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" @@ -84,10 +79,10 @@ services: RESULT_BACKEND: >- redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams" command: [ From b788c3350cdb4bf6f1f3152fe2184b9c4c841392 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:30:23 -0400 Subject: [PATCH 326/408] alphabetize mounts --- tools/deployment/package/docker-compose.base.yaml | 2 +- tools/deployment/package/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 542bc31adb..75594e6d29 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -267,11 +267,11 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - *volume_logs_input_readonly + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" - - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index bc283ed7cd..b804c015e0 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -83,8 +83,8 @@ services: - *volume_clp_logs - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" command: [ "python3", "-u", From 6fc219cbfc6c7bd10b7998dc25112608755cc58b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:40:25 -0400 Subject: [PATCH 327/408] Do not map logs input dir when CLP_LOGS_INPUT_DIR_HOST is undefined --- tools/deployment/package/docker-compose.base.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 75594e6d29..03a6aabb6e 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -39,11 +39,6 @@ x-volume-definitions: type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}" target: "/var/log" - logs-input-readonly: &volume_logs_input_readonly - type: "bind" - source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" - target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - read_only: true volumes: # Dummy volume to use when a bind mount is not desired. @@ -233,8 +228,8 @@ services: volumes: - *volume_clp_config_readonly - *volume_clp_logs - - *volume_logs_input_readonly - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -266,8 +261,8 @@ services: volumes: - *volume_clp_config_readonly - *volume_clp_logs - - *volume_logs_input_readonly - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ @@ -306,7 +301,7 @@ services: published: "${CLP_WEBUI_PORT:-4000}" target: 4000 volumes: - - *volume_aws_config_readonly + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - type: "bind" source: "./var/www/webui/client/settings.json" From 09261802904ea7e984eccfaa3e8dcaf255900d33 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 11:45:13 -0400 Subject: [PATCH 328/408] lint --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- components/clp-package-utils/clp_package_utils/general.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index b3c5313f70..92ab38a7a2 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -41,6 +41,7 @@ check_docker_dependencies, CONTAINER_CLP_HOME, DockerComposeProjectNotRunningError, + DockerDependencyError, dump_shared_container_config, generate_docker_compose_container_config, get_clp_home, @@ -50,7 +51,6 @@ validate_redis_config, validate_results_cache_config, validate_webui_config, - DockerDependencyError, ) LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 58ff00ea6b..8dc3833da9 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -181,9 +181,7 @@ def check_docker_dependencies(should_compose_project_be_running: bool, project_n stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: - raise DockerNotAvailableError( - "docker is not installed or available on the path", e - ) from e + raise DockerNotAvailableError("docker is not installed or available on the path", e) from e is_running = _is_docker_compose_project_running(project_name) From e2daf327c404a3f984e228fafba85c70dffcb8d3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 12:09:15 -0400 Subject: [PATCH 329/408] lint --- tools/deployment/package/docker-compose.base.yaml | 14 ++++++++------ tools/deployment/package/docker-compose.yaml | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 03a6aabb6e..35370f117b 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -217,14 +217,15 @@ services: hostname: "compression_scheduler" stop_grace_period: "300s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ + ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}" volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -249,15 +250,16 @@ services: hostname: "compression_worker" stop_grace_period: "60s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ + :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/compression_worker" CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}" volumes: - *volume_clp_config_readonly - *volume_clp_logs diff --git a/tools/deployment/package/docker-compose.yaml b/tools/deployment/package/docker-compose.yaml index b804c015e0..8674218473 100644 --- a/tools/deployment/package/docker-compose.yaml +++ b/tools/deployment/package/docker-compose.yaml @@ -32,14 +32,15 @@ services: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ + ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_QUERY:-0}" volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -69,15 +70,16 @@ services: <<: *service_defaults hostname: "query_worker" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ + ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/query_worker" CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_QUERY:-0}" volumes: - *volume_clp_config_readonly - *volume_clp_logs From 49561062937137d4ebac10641f34cffae54eee43 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 12:18:56 -0400 Subject: [PATCH 330/408] fix: Add AWS config directory mount to garbage collector --- tools/deployment/package/docker-compose.base.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/deployment/package/docker-compose.base.yaml b/tools/deployment/package/docker-compose.base.yaml index 35370f117b..f53d71f1e1 100644 --- a/tools/deployment/package/docker-compose.base.yaml +++ b/tools/deployment/package/docker-compose.base.yaml @@ -348,6 +348,7 @@ services: - *volume_clp_config_readonly - *volume_clp_logs - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" depends_on: db-table-creator: From 91cacb01891bda2ab30faa5bdd4aed10984ce864 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 21 Oct 2025 12:27:50 -0400 Subject: [PATCH 331/408] fix(clp-package-utils): Add return type annotations to custom exception constructors. --- components/clp-package-utils/clp_package_utils/general.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 8dc3833da9..e98e0b9e0c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -56,7 +56,7 @@ class DockerDependencyError(OSError): class DockerNotAvailableError(DockerDependencyError): """Raised when Docker or Docker Compose is unavailable.""" - def __init__(self, base_message: str, process_error: subprocess.CalledProcessError): + def __init__(self, base_message: str, process_error: subprocess.CalledProcessError) -> None: message = base_message output_chunks: list[str] = [] for stream in (process_error.output, process_error.stderr): @@ -77,14 +77,14 @@ def __init__(self, base_message: str, process_error: subprocess.CalledProcessErr class DockerComposeProjectNotRunningError(DockerDependencyError): """Raised when an expected Docker Compose project is not running.""" - def __init__(self, project_name: str): + def __init__(self, project_name: str) -> None: super().__init__(errno.ESRCH, f"Docker Compose project '{project_name}' is not running.") class DockerComposeProjectAlreadyRunningError(DockerDependencyError): """Raised when a Docker Compose project is already running but should not be.""" - def __init__(self, project_name: str): + def __init__(self, project_name: str) -> None: super().__init__( errno.EEXIST, f"Docker Compose project '{project_name}' is already running." ) From a6598e64c33845abc9426134649d893c70fa40fe Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 21 Oct 2025 16:43:52 -0400 Subject: [PATCH 332/408] Adapt spider docker file --- .../package/docker-compose.spider.base.yaml | 78 +++++++++---------- .../package/docker-compose.spider.yaml | 37 ++++----- 2 files changed, 53 insertions(+), 62 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index 8319534b56..48a34dbd39 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -30,11 +30,6 @@ x-healthcheck-defaults: &healthcheck_defaults # Common volume definitions. x-volume-definitions: - aws-config-readonly: &volume_aws_config_readonly - type: "bind" - source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" - target: "/.aws" - read_only: true clp-config-readonly: &volume_clp_config_readonly type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" @@ -44,11 +39,6 @@ x-volume-definitions: type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}" target: "/var/log" - logs-input-readonly: &volume_logs_input_readonly - type: "bind" - source: "${CLP_LOGS_INPUT_DIR_HOST:-/}" - target: "${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - read_only: true volumes: # Dummy volume to use when a bind mount is not desired. @@ -67,9 +57,9 @@ services: user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: MYSQL_DATABASE: "${CLP_DB_NAME:-clp-db}" - MYSQL_PASSWORD: "${CLP_DB_PASS:?error}" - MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?error}" - MYSQL_USER: "${CLP_DB_USER:?error}" + MYSQL_PASSWORD: "${CLP_DB_PASS:?Please set a value.}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?Please set a value.}" + MYSQL_USER: "${CLP_DB_USER:?Please set a value.}" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" published: "${CLP_DB_PORT:-3306}" @@ -92,16 +82,16 @@ services: "mysqladmin", "ping", "--silent", "-h", "127.0.0.1", - "-u", "${CLP_DB_USER:?error}", - "--password=${CLP_DB_PASS:?error}" + "-u", "${CLP_DB_USER:?Please set a value.}", + "--password=${CLP_DB_PASS:?Please set a value.}" ] db-table-creator: <<: *service_defaults hostname: "db_table_creator" environment: - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly @@ -140,16 +130,16 @@ services: <<: *service_defaults hostname: "compression_scheduler" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" - CLP_LOGS_DIR: "/var/log" + SPIDER_LOG_FILE: "/var/log/compression_scheduler.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - - *volume_logs_input_readonly + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -162,24 +152,24 @@ services: compression-worker: <<: *service_defaults - container_name: "compression_worker" + hostname: "compression_worker" + stop_grace_period: "60s" environment: CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/compression_worker" - CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" - LD_LIBRARY_PATH: "/opt/clp/lib" PYTHONPATH: "/opt/clp/lib/python3/site-packages" + SPIDER_LOG_DIR: "/var/log/compression_worker" volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - - *volume_logs_input_readonly - - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/archives" - - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:\ -${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ + :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ + :${CLP_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/staged-archives" - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" @@ -200,8 +190,10 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" <<: *service_defaults hostname: "webui" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + AWS_ACCESS_KEY_ID: "${CLP_STREAM_OUTPUT_AWS_ACCESS_KEY_ID:-}" + AWS_SECRET_ACCESS_KEY: "${CLP_STREAM_OUTPUT_AWS_SECRET_ACCESS_KEY:-}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" HOST: "0.0.0.0" NODE_ENV: "production" NODE_PATH: "/opt/clp/var/www/webui/server/node_modules" @@ -212,10 +204,8 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" published: "${CLP_WEBUI_PORT:-4000}" target: 4000 volumes: - - *volume_aws_config_readonly - - type: "bind" - source: "${CLP_STREAM_OUTPUT_DIR_HOST:-./var/data/streams}" - target: "/var/data/streams" + - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - type: "bind" source: "./var/www/webui/client/settings.json" target: "/opt/clp/var/www/webui/client/settings.json" @@ -245,18 +235,22 @@ ${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-/tmp}/var/data/staged-archives" garbage-collector: <<: *service_defaults hostname: "garbage_collector" + deploy: + # Value must be either 0 or 1. Set to 0 to disable the garbage collector. + replicas: "${CLP_GARBAGE_COLLECTOR_ENABLED:-1}" environment: - CLP_DB_PASS: "${CLP_DB_PASS}" - CLP_DB_USER: "${CLP_DB_USER}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly - - type: "bind" - source: "${CLP_LOGS_DIR_HOST:-./var/log}/garbage_collector" - target: "/var/log/garbage_collector" + - *volume_clp_logs + - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" depends_on: db-table-creator: condition: "service_completed_successfully" diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index 9c11905e24..9fa6dde1d4 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -2,7 +2,7 @@ name: "clp-package" include: ["docker-compose.spider.base.yaml"] -# Below x-* definitions are duplicated from docker-compose.base.yaml. Refer to that file for +# The `x-*` definitions below are duplicated from docker-compose.base.yaml. Refer to that file for # documentation. x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" @@ -17,11 +17,6 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: "60s" timeout: "2s" x-volume-definitions: - aws-config-readonly: &volume_aws_config_readonly - type: "bind" - source: "${CLP_AWS_CONFIG_DIR_HOST:-~/.aws}" - target: "/.aws" - read_only: true clp-config-readonly: &volume_clp_config_readonly type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yml" @@ -40,8 +35,8 @@ services: hostname: "queue" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: - RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?error}" - RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?error}" + RABBITMQ_DEFAULT_PASS: "${CLP_QUEUE_PASS:?Please set a value.}" + RABBITMQ_DEFAULT_USER: "${CLP_QUEUE_USER:?Please set a value.}" RABBITMQ_LOGS: "/var/log/rabbitmq/rabbitmq.log" ports: - host_ip: "${CLP_QUEUE_HOST:-127.0.0.1}" @@ -81,7 +76,7 @@ services: command: [ "redis-server", "/usr/local/etc/redis/redis.conf", - "--requirepass", "${CLP_REDIS_PASS:?error}" + "--requirepass", "${CLP_REDIS_PASS:?Please set a value.}" ] healthcheck: <<: *healthcheck_defaults @@ -90,7 +85,7 @@ services: "redis-cli", "-h", "127.0.0.1", "-p", "6379", - "-a", "${CLP_REDIS_PASS:?error}", + "-a", "${CLP_REDIS_PASS:?Please set a value.}", "PING" ] @@ -146,14 +141,15 @@ services: <<: *service_defaults hostname: "query_scheduler" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" - CLP_DB_PASS: "${CLP_DB_PASS:?error}" - CLP_DB_USER: "${CLP_DB_USER:?error}" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ + ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_QUERY:-0}" volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -183,22 +179,23 @@ services: <<: *service_defaults hostname: "query_worker" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?error}:${CLP_QUEUE_PASS:?error}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ + ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/query_worker" CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: >- - redis://default:${CLP_REDIS_PASS:?error}@redis:6379/${CLP_REDIS_BACKEND_DB_QUERY:-0} + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_QUERY:-0}" volumes: - - *volume_aws_config_readonly - *volume_clp_config_readonly - *volume_clp_logs - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" - - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams" + - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" command: [ "python3", "-u", From 2be2d5c34b08065c13fb772789681ecf37d60818 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 21 Oct 2025 17:11:18 -0400 Subject: [PATCH 333/408] Fix docker file --- tools/deployment/package/docker-compose.spider.base.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index 48a34dbd39..bb05718312 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -109,9 +109,7 @@ services: spider-scheduler: <<: *service_defaults - container_name: "spider_scheduler" - environment: - LD_LIBRARY_PATH: "/opt/clp/lib" + hostname: "spider_scheduler" depends_on: db-table-creator: condition: "service_completed_successfully" From 36b0f9a380e2bd4bdbe14b0e858a131366b66b7a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 21 Oct 2025 17:46:00 -0400 Subject: [PATCH 334/408] Fix docker file log env --- tools/deployment/package/docker-compose.spider.base.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index bb05718312..935f35a8f3 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -110,6 +110,8 @@ services: spider-scheduler: <<: *service_defaults hostname: "spider_scheduler" + environment: + SPIDER_LOG_FILE: "/var/log/compression_scheduler.log" depends_on: db-table-creator: condition: "service_completed_successfully" @@ -131,7 +133,7 @@ services: CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}" - SPIDER_LOG_FILE: "/var/log/compression_scheduler.log" + CLP_LOGS_DIR: "/var/log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: - *volume_clp_config_readonly From fb8c8e03832d35dac09227dfbda3656ba90a1e62 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 23 Oct 2025 13:18:49 -0400 Subject: [PATCH 335/408] Parse spider related config selectively --- components/clp-package-utils/clp_package_utils/controller.py | 5 +++-- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index f2276d3c95..70b8afbfa1 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -787,8 +787,9 @@ def _set_up_env(self) -> None: env_vars |= self._set_up_env_for_database() env_vars |= self._set_up_env_for_queue() env_vars |= self._set_up_env_for_redis() - env_vars |= self._set_up_env_for_spider_db() - env_vars |= self._set_up_env_for_spider_scheduler() + if self._clp_config.compression_scheduler.type == OrchestrationType.spider: + env_vars |= self._set_up_env_for_spider_db() + env_vars |= self._set_up_env_for_spider_scheduler() env_vars |= self._set_up_env_for_results_cache() env_vars |= self._set_up_env_for_compression_scheduler() env_vars |= self._set_up_env_for_query_scheduler() diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 4225241f41..24c362e3e5 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -54,7 +54,8 @@ def main(argv): validate_and_load_db_credentials_file(clp_config, clp_home, True) validate_and_load_queue_credentials_file(clp_config, clp_home, True) validate_and_load_redis_credentials_file(clp_config, clp_home, True) - validate_and_load_spider_db_credentials_file(clp_config, clp_home, True) + if clp_config.compression_scheduler.type == OrchestrationType.spider: + validate_and_load_spider_db_credentials_file(clp_config, clp_home, True) validate_logs_input_config(clp_config) validate_output_storage_config(clp_config) validate_retention_config(clp_config) From cd2b8c20fb4155a95442a9a475433a41551c02f0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 23 Oct 2025 13:28:30 -0400 Subject: [PATCH 336/408] Remove unused doc before merge main --- docs/src/dev-docs/design-docker-compose.md | 199 --------------------- 1 file changed, 199 deletions(-) delete mode 100644 docs/src/dev-docs/design-docker-compose.md diff --git a/docs/src/dev-docs/design-docker-compose.md b/docs/src/dev-docs/design-docker-compose.md deleted file mode 100644 index 9bec8bb49b..0000000000 --- a/docs/src/dev-docs/design-docker-compose.md +++ /dev/null @@ -1,199 +0,0 @@ -# Docker Compose design - -This document explains the technical details of CLP's Docker Compose implementation. - -## Overview - -The Docker Compose implementation follows a controller architecture with a `BaseController` abstract -class and a `DockerComposeController` implementation. - -## Architecture - -### Controller Pattern - -The orchestration uses a controller pattern: - -* `BaseController` (abstract): Defines the interface for provisioning and managing CLP components. -* `DockerComposeController`: Implements Docker Compose-specific logic. - -## Initialization - -The controller performs these initialization steps: - -1. **Provisioning**: Provisions all components and generates component specific configuration - variables. -2. **Configuration Transformation**: The `transform_for_container()` method in `CLPConfig` adapts - configurations for containerized environments -3. **Environment Generation**: Creates a `.env` file with necessary Docker Compose variables - -### Configuration Transformation - -The `transform_for_container()` method in the `CLPConfig` class and related component classes -adapts the configuration for containerized environments by: - -1. Converting host paths to container paths -2. Updating service hostnames to match Docker Compose service names -3. Setting appropriate ports for container communication - -### Environment Variables - -The controller generates a comprehensive set of environment variables that are written to a `.env` -file, including: - -* Component-specific settings (ports, logging levels, concurrency) -* Credentials for database, queue, and Redis services -* Paths for data, logs, archives, and streams -* AWS credentials when needed - -## Deployment Process - -The `start-clp.sh` script executes the `start_clp.py` Python script to orchestrate the deployment. - -### Deployment Types - -CLP supports two deployment types determined by the `package.query_engine` configuration setting: - -1. **BASE**: For deployments using [Presto][presto-integration] as the query engine. Uses only - `docker-compose.base.yaml`. -2. **FULL**: For deployments using CLP's native query engine. Uses both compose files. - -## Docker Compose Files - -The Docker Compose setup uses two files: - -* `docker-compose.base.yaml`: Defines base services for all deployment types, excluding Celery - scheduler and worker components to allow separate Presto [integration][presto-integration]. -* `docker-compose.yaml`: Extends the base file with additional services for complete deployments - -Each file defines services with: - -* Service dependencies via `depends_on` -* Health checks for critical services -* Volume binding mounts for persistent data -* Network configuration -* User permissions - -## Service architecture - -The Docker Compose setup includes the following services: - -:::{mermaid} -graph LR - %% Services - database["database (MySQL)"] - queue["queue (RabbitMQ)"] - redis["redis (Redis)"] - results_cache["results-cache (MongoDB)"] - compression_scheduler["compression-scheduler"] - query_scheduler["query-scheduler"] - compression_worker["compression-worker"] - query_worker["query-worker"] - reducer["reducer"] - webui["webui"] - garbage_collector["garbage-collector"] - - %% One-time jobs - db_table_creator["db-table-creator"] - results_cache_indices_creator["results-cache-indices-creator"] - - %% Dependencies - database -->|healthy| db_table_creator - results_cache -->|healthy| results_cache_indices_creator - db_table_creator -->|completed_successfully| compression_scheduler - queue -->|healthy| compression_scheduler - redis -->|healthy| compression_scheduler - db_table_creator -->|completed_successfully| query_scheduler - queue -->|healthy| query_scheduler - redis -->|healthy| query_scheduler - query_scheduler -->|healthy| reducer - results_cache_indices_creator -->|completed_successfully| reducer - db_table_creator -->|completed_successfully| webui - results_cache_indices_creator -->|completed_successfully| webui - db_table_creator -->|completed_successfully| garbage_collector - results_cache_indices_creator -->|completed_successfully| garbage_collector - - subgraph Databases - database - queue - redis - results_cache - end - - subgraph DB Migration Jobs - db_table_creator - results_cache_indices_creator - end - - subgraph Schedulers - compression_scheduler - query_scheduler - end - - subgraph Workers - compression_worker - query_worker - reducer - end - - subgraph UI & Management - webui - garbage_collector - end -::: - -### Services overview - -The CLP package is composed of several service components. The tables below list the services and -their functions. - -:::{table} Services -:align: left - -| Service | Description | -|-----------------------|-----------------------------------------------------------------| -| database | Database for archive metadata, compression jobs, and query jobs | -| queue | Task queue for schedulers | -| redis | Task result storage for workers | -| compression_scheduler | Scheduler for compression jobs | -| query_scheduler | Scheduler for search/aggregation jobs | -| results_cache | Storage for the workers to return search results to the UI | -| compression_worker | Worker processes for compression jobs | -| query_worker | Worker processes for search/aggregation jobs | -| reducer | Reducers for performing the final stages of aggregation jobs | -| webui | Web server for the UI | -| garbage_collector | Background process for retention control | -::: - -### One-time initialization jobs - -We also set up short-lived run-once "services" to initialize some services listed above. - -:::{table} Initialization jobs -:align: left - -| Job | Description | -|-------------------------------|---------------------------------------------------------| -| db-table-creator | Initializes database tables | -| results-cache-indices-creator | Initializes single-node replica set and sets up indices | -::: - -## Troubleshooting - -If you encounter issues with the Docker Compose deployment: - -1. Check service status: - ```bash - docker compose ps - ``` - -2. View service logs: - ```bash - docker compose logs - ``` - -3. Validate configuration: - ```bash - docker compose config - ``` - -[presto-integration]: ../user-docs/guides-using-presto.md From a17b9b28355de2ca9e61ca8b36c5473a816b299e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 23 Oct 2025 13:34:38 -0400 Subject: [PATCH 337/408] Update spider docker file --- .../package/docker-compose.spider.base.yaml | 48 +++++++++++++------ .../package/docker-compose.spider.yaml | 12 ++--- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index 935f35a8f3..4e6468c39a 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -5,7 +5,7 @@ x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" - stop_grace_period: "3s" + stop_grace_period: "60s" user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" # Common healthcheck defaults. @@ -48,11 +48,10 @@ volumes: type: "tmpfs" size: 0 - services: database: <<: *service_defaults - image: "${CLP_DB_IMAGE:-mariadb:10-jammy}" + image: "${CLP_DB_CONTAINER_IMAGE_REF:-mariadb:10-jammy}" hostname: "database" user: "${CLP_THIRD_PARTY_SERVICE_UID_GID:-1000:1000}" environment: @@ -106,7 +105,6 @@ services: "--storage-engine", "${CLP_PACKAGE_STORAGE_ENGINE:-clp}", ] - spider-scheduler: <<: *service_defaults hostname: "spider_scheduler" @@ -153,19 +151,39 @@ services: compression-worker: <<: *service_defaults hostname: "compression_worker" - stop_grace_period: "60s" environment: + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ + :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/compression_worker" + CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - SPIDER_LOG_DIR: "/var/log/compression_worker" + RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ + /${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}" volumes: - *volume_clp_config_readonly - *volume_clp_logs - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" + + # NOTE: Only one of `CLP_ARCHIVE_OUTPUT_DIR_HOST` and `CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST` are + # set at a time, but since `./var/data` on the host is mounted into the container and both + # `CLP_ARCHIVE_OUTPUT_DIR_HOST` and `CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST` default to + # directories under `./var/data`, we need to use a hack to avoid having Docker create the + # unset directory on the host (as root). + # + # For example, let's say we use the following as the mount for staged archives: + # "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/staged-archives". If + # `CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST` is unset, Docker will create + # `/var/data/staged-archives` in the container, but it will also create + # `./var/data/staged-archives` on the host as root; this is because `/var/data` in the + # container is bind mounted to `./var/data` on the host. + # + # The hack to avoid this is if one of `CLP_ARCHIVE_OUTPUT_DIR_HOST` or + # `CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST` is unset, we set the target for the corresponding mount + # to a path that's not under `/var/data` in the container. - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ :${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:+/tmp}/var/data/archives" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}\ @@ -173,17 +191,17 @@ services: - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" - depends_on: - db-table-creator: - condition: "service_completed_successfully" command: [ "python3", "-u", - "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.compress", + "worker", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--storage-url", "${SPIDER_DB_URL}", - # NOTE: Leave host to spider scheduler's host. This only affects task placement. - "--host", "${SPIDER_SCHEDULER_HOST}", + "--loglevel", "WARNING", + "-f", "/var/log/compression_worker/worker.log", + "-Q", "compression", + "-n", "compression-worker" ] webui: @@ -204,7 +222,7 @@ services: published: "${CLP_WEBUI_PORT:-4000}" target: 4000 volumes: - - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" + - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/.aws:ro" - "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams" - type: "bind" source: "./var/www/webui/client/settings.json" @@ -242,7 +260,7 @@ services: CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_HOME: "/opt/clp" - CLP_LOGGING_LEVEL: "${CLP_GC_LOGGING_LEVEL:-INFO}" + CLP_LOGGING_LEVEL: "${CLP_GARBAGE_COLLECTOR_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/garbage_collector" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: diff --git a/tools/deployment/package/docker-compose.spider.yaml b/tools/deployment/package/docker-compose.spider.yaml index 9fa6dde1d4..9f6fb728f9 100644 --- a/tools/deployment/package/docker-compose.spider.yaml +++ b/tools/deployment/package/docker-compose.spider.yaml @@ -8,7 +8,7 @@ x-service-defaults: &service_defaults image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" logging: driver: "local" - stop_grace_period: "3s" + stop_grace_period: "60s" user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" x-healthcheck-defaults: &healthcheck_defaults interval: "30s" @@ -136,13 +136,13 @@ services: "--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}", ] - query-scheduler: <<: *service_defaults hostname: "query_scheduler" + stop_grace_period: "10s" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ - ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ + :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}" @@ -179,8 +179,8 @@ services: <<: *service_defaults hostname: "query_worker" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}:\ - ${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" + BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ + :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}" From 2888b6343881efb1d64f14d919ccd3f09390562c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 23 Oct 2025 14:15:41 -0400 Subject: [PATCH 338/408] Fix docker file --- .../package/docker-compose.spider.base.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index 4e6468c39a..52998ad312 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -127,6 +127,7 @@ services: compression-scheduler: <<: *service_defaults hostname: "compression_scheduler" + stop_grace_period: "300s" environment: CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" @@ -152,16 +153,12 @@ services: <<: *service_defaults hostname: "compression_worker" environment: - BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\ - :${CLP_QUEUE_PASS:?Please set a value.}@queue:5672" CLP_CONFIG_PATH: "/etc/clp-config.yml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/compression_worker" - CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log" PYTHONPATH: "/opt/clp/lib/python3/site-packages" - RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\ - /${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}" + SPIDER_LOG_DIR: "/var/log/compression_worker" volumes: - *volume_clp_config_readonly - *volume_clp_logs @@ -194,14 +191,11 @@ services: command: [ "python3", "-u", - "/opt/clp/lib/python3/site-packages/bin/celery", - "-A", "job_orchestration.executor.compress", - "worker", + "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--loglevel", "WARNING", - "-f", "/var/log/compression_worker/worker.log", - "-Q", "compression", - "-n", "compression-worker" + "--storage-url", "${SPIDER_DB_URL}", + # NOTE: Leave host to spider scheduler's host. This only affects task placement. + "--host", "${SPIDER_SCHEDULER_HOST}", ] webui: From 16a8f2910ab0fe26da0f5d5299a48963636c1b0b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 23 Oct 2025 14:33:36 -0400 Subject: [PATCH 339/408] Fix merge --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 70b8afbfa1..691f0a7bc7 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -669,7 +669,7 @@ def start(self) -> None: cmd += ["--file", "docker-compose.spider.base.yaml"] else: cmd += ["--file", "docker-compose.base.yaml"] - if self._clp_config.compression_scheduler.type == OrchestrationType.spider: + elif self._clp_config.compression_scheduler.type == OrchestrationType.spider: cmd += ["--file", "docker-compose.spider.yaml"] cmd += ["up", "--detach", "--wait"] subprocess.run( From 4879aa15f0e08b3ba15ff4a85ba600c12605660e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 23 Oct 2025 15:08:12 -0400 Subject: [PATCH 340/408] Fix spider docker merge --- tools/deployment/package/docker-compose.spider.base.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/deployment/package/docker-compose.spider.base.yaml b/tools/deployment/package/docker-compose.spider.base.yaml index 52998ad312..8b4a6a806d 100644 --- a/tools/deployment/package/docker-compose.spider.base.yaml +++ b/tools/deployment/package/docker-compose.spider.base.yaml @@ -188,6 +188,9 @@ services: - type: "bind" source: "${CLP_DATA_DIR_HOST:-./var/data}" target: "/var/data" + depends_on: + db-table-creator: + condition: "service_completed_successfully" command: [ "python3", "-u", From a5871874620cb406f491c8ce0422173ab1873e92 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 27 Oct 2025 21:11:43 -0400 Subject: [PATCH 341/408] Fix merge --- components/clp-package-utils/clp_package_utils/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 7c2ffd97f5..89905bfa83 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -798,7 +798,6 @@ def start(self) -> None: cmd += ["--file", "docker-compose.base.yaml"] elif self._clp_config.compression_scheduler.type == OrchestrationType.spider: cmd += ["--file", "docker-compose.spider.yaml"] - cmd += ["up", "--detach", "--wait"] if self._clp_config.mcp_server is not None: cmd += ["--profile", "mcp"] From 551c309446cf03b1ec9a3dc3b164d11b24274e69 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 3 Nov 2025 10:49:32 -0500 Subject: [PATCH 342/408] Use bytes --- .../executor/compress/spider_compress.py | 16 ++++++++-------- .../compress/task_manager/spider_task_manager.py | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index fa09549134..beabc59b41 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -15,10 +15,10 @@ def compress( job_id: Int64, task_id: Int64, tag_ids: list[Int64], - clp_io_config_json: list[Int8], - paths_to_compress_json: list[Int8], - clp_metadata_db_connection_config_json: list[Int8], -) -> list[Int8]: + clp_io_config_json: bytes, + paths_to_compress_json: bytes, + clp_metadata_db_connection_config_json: bytes, +) -> bytes: """ Compresses files using the general compression entry point. :param _: Spider's task context. Not used in the function. @@ -40,11 +40,11 @@ def compress( int(job_id), int(task_id), [int(tag_id) for tag_id in tag_ids], - int8_list_to_utf8_str(clp_io_config_json), - int8_list_to_utf8_str(paths_to_compress_json), - json.loads(int8_list_to_utf8_str(clp_metadata_db_connection_config_json)), + clp_io_config_json.decode("utf-8"), + paths_to_compress_json.decode("utf-8"), + json.loads(clp_metadata_db_connection_config_json.decode("utf-8")), logger, ) ) - return utf8_str_to_int8_list(result_as_json_str) + return result_as_json_str.encode("utf-8") diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index 9c68ad4f5e..f112858e35 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -24,10 +24,10 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None return None if not isinstance(job_results, tuple): return [ - CompressionTaskResult.model_validate_json(int8_list_to_utf8_str(job_results)) + CompressionTaskResult.model_validate_json(job_results.decode("utf-8")) ] return [ - CompressionTaskResult.model_validate_json(int8_list_to_utf8_str(task_result)) + CompressionTaskResult.model_validate_json(task_result.decode("utf-8")) for task_result in job_results ] @@ -43,10 +43,10 @@ def submit(self, task_params: list[dict[str, Any]]) -> TaskManager.ResultHandle: job_args.append(spider_py.Int64(task_param["job_id"])) job_args.append(spider_py.Int64(task_param["task_id"])) job_args.append([spider_py.Int64(tag_id) for tag_id in task_param["tag_ids"]]) - job_args.append(utf8_str_to_int8_list(task_param["clp_io_config_json"])) - job_args.append(utf8_str_to_int8_list(task_param["paths_to_compress_json"])) + job_args.append(task_param["clp_io_config_json"].encode("utf-8")) + job_args.append(task_param["paths_to_compress_json"].encode("utf-8")) job_args.append( - utf8_str_to_int8_list(json.dumps(task_param["clp_metadata_db_connection_config"])) + json.dumps(task_param["clp_metadata_db_connection_config"]).encode("utf-8") ) submitted_job = self._driver.submit_jobs([job], [job_args])[0] return SpiderTaskManager.ResultHandle(submitted_job) From ac660b21cadb3314ead540af66ff16b32279609d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 13:52:32 -0500 Subject: [PATCH 343/408] Update spider library --- taskfiles/deps/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 118ec1736e..15fea8d1b1 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -102,8 +102,8 @@ tasks: - task: "yscope-dev-utils:cmake:install-remote-tar" vars: CMAKE_PACKAGE_NAME: "spider" - TAR_SHA256: "43478b6da99aa2df50c960573ec68c6fcf67cdfbab39b5e314676218aa93b959" - TAR_URL: "https://github.com/y-scope/spider/archive/6a7bad3.tar.gz" + TAR_SHA256: "c4ce1315a6788c34e7986e40b501adc3163dcd178865373e7cbc0684321a513c" + TAR_URL: "https://github.com/y-scope/spider/archive/b34204b.tar.gz" WORK_DIR: "{{.G_SPIDER_BUILD_DIR}}" CMAKE_JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" CMAKE_GEN_ARGS: From 9909731a75c0314cde8997e2faac06a1f73e6584 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 13:53:41 -0500 Subject: [PATCH 344/408] Bump spider-py version --- components/job-orchestration/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/pyproject.toml b/components/job-orchestration/pyproject.toml index 09294c2ed1..9e173e9b3e 100644 --- a/components/job-orchestration/pyproject.toml +++ b/components/job-orchestration/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "pydantic>=2.12.3", "pymongo>=4.15.3", "PyYAML>=6.0.3", - "yscope-spider-py==0.1.0", + "yscope-spider-py==0.2.0", ] [build-system] From 37e9639219d9764881836f40a7dc53e60c321454 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 21:28:23 -0500 Subject: [PATCH 345/408] Update mariadb version --- components/clp-py-utils/pyproject.toml | 4 +- components/clp-py-utils/uv.lock | 666 +++++++++++--------- components/job-orchestration/pyproject.toml | 4 +- components/job-orchestration/uv.lock | 561 +++++++++-------- 4 files changed, 640 insertions(+), 595 deletions(-) diff --git a/components/clp-py-utils/pyproject.toml b/components/clp-py-utils/pyproject.toml index 2a6ea05246..84fa617174 100644 --- a/components/clp-py-utils/pyproject.toml +++ b/components/clp-py-utils/pyproject.toml @@ -7,9 +7,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "boto3>=1.40.55", - # mariadb version must be compatible with libmariadev installed in runtime env. - # See https://mariadb.com/docs/server/connect/programming-languages/python/install/#Dependencies - "mariadb>=1.0.11,<1.1.dev0", + "mariadb>=1.1.14", "mysql-connector-python>=9.4.0", "pydantic>=2.12.3", "python-Levenshtein>=0.27.1", diff --git a/components/clp-py-utils/uv.lock b/components/clp-py-utils/uv.lock index ff976ee6f7..a063a5e468 100644 --- a/components/clp-py-utils/uv.lock +++ b/components/clp-py-utils/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" [[package]] @@ -13,30 +13,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.40.55" +version = "1.40.74" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/50/d8/a279c054e0c9731172f05b3d118f3ffc9d74806657f84fc0c93c42d1bb5d/boto3-1.40.55.tar.gz", hash = "sha256:27e35b4fa9edd414ce06c1a748bf57cacd8203271847d93fc1053e4a4ec6e1a9", size = 111590, upload-time = "2025-10-17T19:34:56.753Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/37/0db5fc46548b347255310893f1a47971a1d8eb0dbc46dfb5ace8a1e7d45e/boto3-1.40.74.tar.gz", hash = "sha256:484e46bf394b03a7c31b34f90945ebe1390cb1e2ac61980d128a9079beac87d4", size = 111592, upload-time = "2025-11-14T20:29:10.991Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/8c/559c6145d857ed953536a83f3a94915bbd5d3d2d406db1abf8bf40be7645/boto3-1.40.55-py3-none-any.whl", hash = "sha256:2e30f5a0d49e107b8a5c0c487891afd300bfa410e1d918bf187ae45ac3839332", size = 139322, upload-time = "2025-10-17T19:34:55.028Z" }, + { url = "https://files.pythonhosted.org/packages/d2/08/c52751748762901c0ca3c3019e3aa950010217f0fdf9940ebe68e6bb2f5a/boto3-1.40.74-py3-none-any.whl", hash = "sha256:41fc8844b37ae27b24bcabf8369769df246cc12c09453988d0696ad06d6aa9ef", size = 139360, upload-time = "2025-11-14T20:29:09.477Z" }, ] [[package]] name = "botocore" -version = "1.40.55" +version = "1.40.74" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a4/92/dce4842b2e215d213d34b064fcdd13c6a782c43344e77336bcde586e9229/botocore-1.40.55.tar.gz", hash = "sha256:79b6472e2de92b3519d44fc1eec8c5feced7f99a0d10fdea6dc93133426057c1", size = 14446917, upload-time = "2025-10-17T19:34:47.44Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/dc/0412505f05286f282a75bb0c650e525ddcfaf3f6f1a05cd8e99d32a2db06/botocore-1.40.74.tar.gz", hash = "sha256:57de0b9ffeada06015b3c7e5186c77d0692b210d9e5efa294f3214df97e2f8ee", size = 14452479, upload-time = "2025-11-14T20:29:00.949Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/21/30/f13bbc36e83b78777ff1abf50a084efcc3336b808e76560d8c5a0c9219e0/botocore-1.40.55-py3-none-any.whl", hash = "sha256:cdc38f7a4ddb30a2cd1cdd4fabde2a5a16e41b5a642292e1c30de5c4e46f5d44", size = 14116107, upload-time = "2025-10-17T19:34:44.398Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a2/306dec16e3c84f3ca7aaead0084358c1c7fbe6501f6160844cbc93bc871e/botocore-1.40.74-py3-none-any.whl", hash = "sha256:f39f5763e35e75f0bd91212b7b36120b1536203e8003cd952ef527db79702b15", size = 14117911, upload-time = "2025-11-14T20:28:58.153Z" }, ] [[package]] @@ -58,7 +58,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "boto3", specifier = ">=1.40.55" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "python-levenshtein", specifier = ">=0.27.1" }, @@ -82,6 +82,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/91/ae2eb6b7979e2f9b035a9f612cf70f1bf54aad4e1d125129bef1eae96f19/greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d", size = 584358, upload-time = "2025-08-07T13:18:23.708Z" }, { url = "https://files.pythonhosted.org/packages/f7/85/433de0c9c0252b22b16d413c9407e6cb3b41df7389afc366ca204dbc1393/greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5", size = 1113550, upload-time = "2025-08-07T13:42:37.467Z" }, { url = "https://files.pythonhosted.org/packages/a1/8d/88f3ebd2bc96bf7747093696f4335a0a8a4c5acfcf1b757717c0d2474ba3/greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f", size = 1137126, upload-time = "2025-08-07T13:18:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/f1/29/74242b7d72385e29bcc5563fba67dad94943d7cd03552bac320d597f29b2/greenlet-3.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f47617f698838ba98f4ff4189aef02e7343952df3a615f847bb575c3feb177a7", size = 1544904, upload-time = "2025-11-04T12:42:04.763Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e2/1572b8eeab0f77df5f6729d6ab6b141e4a84ee8eb9bc8c1e7918f94eda6d/greenlet-3.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af41be48a4f60429d5cad9d22175217805098a9ef7c40bfef44f7669fb9d74d8", size = 1611228, upload-time = "2025-11-04T12:42:08.423Z" }, { url = "https://files.pythonhosted.org/packages/d6/6f/b60b0291d9623c496638c582297ead61f43c4b72eef5e9c926ef4565ec13/greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c", size = 298654, upload-time = "2025-08-07T13:50:00.469Z" }, { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, @@ -91,6 +93,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, { url = "https://files.pythonhosted.org/packages/3f/cc/b07000438a29ac5cfb2194bfc128151d52f333cee74dd7dfe3fb733fc16c/greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa", size = 1142073, upload-time = "2025-08-07T13:18:21.737Z" }, + { url = "https://files.pythonhosted.org/packages/67/24/28a5b2fa42d12b3d7e5614145f0bd89714c34c08be6aabe39c14dd52db34/greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c", size = 1548385, upload-time = "2025-11-04T12:42:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/6a/05/03f2f0bdd0b0ff9a4f7b99333d57b53a7709c27723ec8123056b084e69cd/greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5", size = 1613329, upload-time = "2025-11-04T12:42:12.928Z" }, { url = "https://files.pythonhosted.org/packages/d8/0f/30aef242fcab550b0b3520b8e3561156857c94288f0332a79928c31a52cf/greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9", size = 299100, upload-time = "2025-08-07T13:44:12.287Z" }, { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, @@ -100,6 +104,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/27/45/80935968b53cfd3f33cf99ea5f08227f2646e044568c9b1555b58ffd61c2/greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0", size = 1564846, upload-time = "2025-11-04T12:42:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/69/02/b7c30e5e04752cb4db6202a3858b149c0710e5453b71a3b2aec5d78a1aab/greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d", size = 1633814, upload-time = "2025-11-04T12:42:17.175Z" }, { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, @@ -109,6 +115,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, { url = "https://files.pythonhosted.org/packages/a2/15/0d5e4e1a66fab130d98168fe984c509249c833c1a3c16806b90f253ce7b9/greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae", size = 1149210, upload-time = "2025-08-07T13:18:24.072Z" }, + { url = "https://files.pythonhosted.org/packages/1c/53/f9c440463b3057485b8594d7a638bed53ba531165ef0ca0e6c364b5cc807/greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b", size = 1564759, upload-time = "2025-11-04T12:42:19.395Z" }, + { url = "https://files.pythonhosted.org/packages/47/e4/3bb4240abdd0a8d23f4f88adec746a3099f0d86bfedb623f063b2e3b4df0/greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929", size = 1634288, upload-time = "2025-11-04T12:42:21.174Z" }, { url = "https://files.pythonhosted.org/packages/0b/55/2321e43595e6801e105fcfdee02b34c0f996eb71e6ddffca6b10b7e1d771/greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b", size = 299685, upload-time = "2025-08-07T13:24:38.824Z" }, { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, @@ -116,6 +124,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, + { url = "https://files.pythonhosted.org/packages/23/6e/74407aed965a4ab6ddd93a7ded3180b730d281c77b765788419484cdfeef/greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269", size = 1612508, upload-time = "2025-11-04T12:42:23.427Z" }, + { url = "https://files.pythonhosted.org/packages/0d/da/343cd760ab2f92bac1845ca07ee3faea9fe52bee65f7bcb19f16ad7de08b/greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681", size = 1680760, upload-time = "2025-11-04T12:42:25.341Z" }, { url = "https://files.pythonhosted.org/packages/e3/a5/6ddab2b4c112be95601c13428db1d8b6608a8b6039816f2ba09c346c08fc/greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01", size = 303425, upload-time = "2025-08-07T13:32:27.59Z" }, ] @@ -130,129 +140,164 @@ wheels = [ [[package]] name = "levenshtein" -version = "0.27.1" +version = "0.27.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "rapidfuzz" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7e/b3/b5f8011483ba9083a0bc74c4d58705e9cf465fbe55c948a1b1357d0a2aa8/levenshtein-0.27.1.tar.gz", hash = "sha256:3e18b73564cfc846eec94dd13fab6cb006b5d2e0cc56bad1fd7d5585881302e3", size = 382571, upload-time = "2025-03-02T19:44:56.148Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/56/dcf68853b062e3b94bdc3d011cc4198779abc5b9dc134146a062920ce2e2/levenshtein-0.27.3.tar.gz", hash = "sha256:1ac326b2c84215795163d8a5af471188918b8797b4953ec87aaba22c9c1f9fc0", size = 393269, upload-time = "2025-11-01T12:14:31.04Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/b1/9906a75b98dd9c008015a72d7658be53851e361a35492631edf1b1f334ab/levenshtein-0.27.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13d6f617cb6fe63714c4794861cfaacd398db58a292f930edb7f12aad931dace", size = 174542, upload-time = "2025-03-02T19:42:24.364Z" }, - { url = "https://files.pythonhosted.org/packages/3b/57/e26e0164a93fb045316856603111d95538cac8224a3709e4ac96a6bb74f3/levenshtein-0.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca9d54d41075e130c390e61360bec80f116b62d6ae973aec502e77e921e95334", size = 156367, upload-time = "2025-03-02T19:42:26.65Z" }, - { url = "https://files.pythonhosted.org/packages/6d/dd/92fcb71d48c1fe69c46c211156adafb8175037dc63e80e970106aef3f9d5/levenshtein-0.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de1f822b5c9a20d10411f779dfd7181ce3407261436f8470008a98276a9d07f", size = 152189, upload-time = "2025-03-02T19:42:28.533Z" }, - { url = "https://files.pythonhosted.org/packages/5e/23/3f331f5fbfa93634126439cfc8c01b31f7ef1fbedb81663581e27a69da4d/levenshtein-0.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81270392c2e45d1a7e1b3047c3a272d5e28bb4f1eff0137637980064948929b7", size = 184271, upload-time = "2025-03-02T19:42:30.525Z" }, - { url = "https://files.pythonhosted.org/packages/5a/76/d6ac541a1a80bdc5c98584a6a2d2301e677af4cb2e4092247207791b56a6/levenshtein-0.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d30c3ea23a94dddd56dbe323e1fa8a29ceb24da18e2daa8d0abf78b269a5ad1", size = 185078, upload-time = "2025-03-02T19:42:32.531Z" }, - { url = "https://files.pythonhosted.org/packages/2d/ed/d0c5abe8cfcf6a7f2a4197e889e12b7a0c2145a0ef3354b1c000bf367305/levenshtein-0.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3e0bea76695b9045bbf9ad5f67ad4cc01c11f783368f34760e068f19b6a6bc", size = 161505, upload-time = "2025-03-02T19:42:34.641Z" }, - { url = "https://files.pythonhosted.org/packages/f3/28/a5b78e1818211bc6407590876bbdcc6d79671e529a0c186780492c1f2136/levenshtein-0.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cdd190e468a68c31a5943368a5eaf4e130256a8707886d23ab5906a0cb98a43c", size = 246968, upload-time = "2025-03-02T19:42:36.195Z" }, - { url = "https://files.pythonhosted.org/packages/77/7f/981b903583956cb67b33bed39d9840ab5e4c7062bceec564b7bf2c3f6f49/levenshtein-0.27.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7c3121314bb4b676c011c33f6a0ebb462cfdcf378ff383e6f9e4cca5618d0ba7", size = 1116000, upload-time = "2025-03-02T19:42:38.292Z" }, - { url = "https://files.pythonhosted.org/packages/75/1d/c4be47d5f436fd310373c5ebdf05828c1d95be9a44c3e94f29c40937b30c/levenshtein-0.27.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f8ef378c873efcc5e978026b69b45342d841cd7a2f273447324f1c687cc4dc37", size = 1401162, upload-time = "2025-03-02T19:42:40.496Z" }, - { url = "https://files.pythonhosted.org/packages/91/e4/0b107676efe3ecd5fada1ed3a3bbddd4c829e2ef34e980b76374c116235b/levenshtein-0.27.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ff18d78c5c16bea20876425e1bf5af56c25918fb01bc0f2532db1317d4c0e157", size = 1225141, upload-time = "2025-03-02T19:42:42.636Z" }, - { url = "https://files.pythonhosted.org/packages/29/f0/f3f88d766fdbb1d39fe98dc5527223bae099444e501550ae088c47ddd97b/levenshtein-0.27.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:13412ff805afbfe619d070280d1a76eb4198c60c5445cd5478bd4c7055bb3d51", size = 1419707, upload-time = "2025-03-02T19:42:44.69Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1c/f51ac1db4064a85effa50df240250e413f428164301d836c312baf09381e/levenshtein-0.27.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2adb9f263557f7fb13e19eb2f34595d86929a44c250b2fca6e9b65971e51e20", size = 1189284, upload-time = "2025-03-02T19:42:46.098Z" }, - { url = "https://files.pythonhosted.org/packages/e0/67/5ace76bc964b93ed6203a9f8c4dcde1a50e336468f7da3a21dd29febaf46/levenshtein-0.27.1-cp310-cp310-win32.whl", hash = "sha256:6278a33d2e0e909d8829b5a72191419c86dd3bb45b82399c7efc53dabe870c35", size = 88036, upload-time = "2025-03-02T19:42:47.869Z" }, - { url = "https://files.pythonhosted.org/packages/06/e0/d9737dbbe85842ddb300cb7974fc065edc56ec647652863f95ac1977d378/levenshtein-0.27.1-cp310-cp310-win_amd64.whl", hash = "sha256:5b602b8428ee5dc88432a55c5303a739ee2be7c15175bd67c29476a9d942f48e", size = 99922, upload-time = "2025-03-02T19:42:49.431Z" }, - { url = "https://files.pythonhosted.org/packages/27/b8/13e22789ab700db0da98f973a508643dbe2d25bd0fb5dc53239e0e2852c1/levenshtein-0.27.1-cp310-cp310-win_arm64.whl", hash = "sha256:48334081fddaa0c259ba01ee898640a2cf8ede62e5f7e25fefece1c64d34837f", size = 87846, upload-time = "2025-03-02T19:42:50.665Z" }, - { url = "https://files.pythonhosted.org/packages/22/84/110136e740655779aceb0da2399977362f21b2dbf3ea3646557f9c2237c4/levenshtein-0.27.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2e6f1760108319a108dceb2f02bc7cdb78807ad1f9c673c95eaa1d0fe5dfcaae", size = 174555, upload-time = "2025-03-02T19:42:51.781Z" }, - { url = "https://files.pythonhosted.org/packages/19/5b/176d96959f5c5969f356d8856f8e20d2e72f7e4879f6d1cda8e5c2ac2614/levenshtein-0.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c4ed8400d94ab348099395e050b8ed9dd6a5d6b5b9e75e78b2b3d0b5f5b10f38", size = 156286, upload-time = "2025-03-02T19:42:53.106Z" }, - { url = "https://files.pythonhosted.org/packages/2a/2d/a75abaafc8a46b0dc52ab14dc96708989a31799a02a4914f9210c3415f04/levenshtein-0.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7826efe51be8ff58bc44a633e022fdd4b9fc07396375a6dbc4945a3bffc7bf8f", size = 152413, upload-time = "2025-03-02T19:42:55.129Z" }, - { url = "https://files.pythonhosted.org/packages/9a/5f/533f4adf964b10817a1d0ecca978b3542b3b9915c96172d20162afe18bed/levenshtein-0.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff5afb78719659d353055863c7cb31599fbea6865c0890b2d840ee40214b3ddb", size = 184236, upload-time = "2025-03-02T19:42:56.427Z" }, - { url = "https://files.pythonhosted.org/packages/02/79/e698623795e36e0d166a3aa1eac6fe1e446cac3a5c456664a95c351571d1/levenshtein-0.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:201dafd5c004cd52018560cf3213da799534d130cf0e4db839b51f3f06771de0", size = 185502, upload-time = "2025-03-02T19:42:57.596Z" }, - { url = "https://files.pythonhosted.org/packages/ac/94/76b64762f4af6e20bbab79713c4c48783240e6e502b2f52e5037ddda688a/levenshtein-0.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5ddd59f3cfaec216811ee67544779d9e2d6ed33f79337492a248245d6379e3d", size = 161749, upload-time = "2025-03-02T19:42:59.222Z" }, - { url = "https://files.pythonhosted.org/packages/56/d0/d10eff9224c94a478078a469aaeb43471fdeddad035f443091224c7544b8/levenshtein-0.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6afc241d27ecf5b921063b796812c55b0115423ca6fa4827aa4b1581643d0a65", size = 246686, upload-time = "2025-03-02T19:43:00.454Z" }, - { url = "https://files.pythonhosted.org/packages/b2/8a/ebbeff74461da3230d00e8a8197480a2ea1a9bbb7dbc273214d7ea3896cb/levenshtein-0.27.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee2e766277cceb8ca9e584ea03b8dc064449ba588d3e24c1923e4b07576db574", size = 1116616, upload-time = "2025-03-02T19:43:02.431Z" }, - { url = "https://files.pythonhosted.org/packages/1d/9b/e7323684f833ede13113fba818c3afe665a78b47d720afdeb2e530c1ecb3/levenshtein-0.27.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:920b23d6109453913ce78ec451bc402ff19d020ee8be4722e9d11192ec2fac6f", size = 1401483, upload-time = "2025-03-02T19:43:04.62Z" }, - { url = "https://files.pythonhosted.org/packages/ef/1d/9b6ab30ff086a33492d6f7de86a07050b15862ccf0d9feeccfbe26af52d8/levenshtein-0.27.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:560d7edba126e2eea3ac3f2f12e7bd8bc9c6904089d12b5b23b6dfa98810b209", size = 1225805, upload-time = "2025-03-02T19:43:06.734Z" }, - { url = "https://files.pythonhosted.org/packages/1b/07/ae2f31e87ff65ba4857e25192646f1f3c8cca83c2ac1c27e551215b7e1b6/levenshtein-0.27.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:8d5362b6c7aa4896dc0cb1e7470a4ad3c06124e0af055dda30d81d3c5549346b", size = 1419860, upload-time = "2025-03-02T19:43:08.084Z" }, - { url = "https://files.pythonhosted.org/packages/43/d2/dfcc5c22c07bab9be99f3f47a907be583bcd37bfd2eec57a205e59671019/levenshtein-0.27.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:65ba880815b0f80a80a293aeebac0fab8069d03ad2d6f967a886063458f9d7a1", size = 1188823, upload-time = "2025-03-02T19:43:09.592Z" }, - { url = "https://files.pythonhosted.org/packages/8b/96/713335623f8ab50eba0627c8685618dc3a985aedaaea9f492986b9443551/levenshtein-0.27.1-cp311-cp311-win32.whl", hash = "sha256:fcc08effe77fec0bc5b0f6f10ff20b9802b961c4a69047b5499f383119ddbe24", size = 88156, upload-time = "2025-03-02T19:43:11.442Z" }, - { url = "https://files.pythonhosted.org/packages/aa/ae/444d6e8ba9a35379a56926716f18bb2e77c6cf69e5324521fbe6885f14f6/levenshtein-0.27.1-cp311-cp311-win_amd64.whl", hash = "sha256:0ed402d8902be7df212ac598fc189f9b2d520817fdbc6a05e2ce44f7f3ef6857", size = 100399, upload-time = "2025-03-02T19:43:13.066Z" }, - { url = "https://files.pythonhosted.org/packages/80/c0/ff226897a238a2deb2ca2c00d658755a1aa01884b0ddc8f5d406cb5f2b0d/levenshtein-0.27.1-cp311-cp311-win_arm64.whl", hash = "sha256:7fdaab29af81a8eb981043737f42450efca64b9761ca29385487b29c506da5b5", size = 88033, upload-time = "2025-03-02T19:43:14.211Z" }, - { url = "https://files.pythonhosted.org/packages/0d/73/84a7126b9e6441c2547f1fbfd65f3c15c387d1fc04e0dd1d025a12107771/levenshtein-0.27.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25fb540d8c55d1dc7bdc59b7de518ea5ed9df92eb2077e74bcb9bb6de7b06f69", size = 173953, upload-time = "2025-03-02T19:43:16.029Z" }, - { url = "https://files.pythonhosted.org/packages/8f/5c/06c01870c0cf336f9f29397bbfbfbbfd3a59918868716e7bb15828e89367/levenshtein-0.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f09cfab6387e9c908c7b37961c045e8e10eb9b7ec4a700367f8e080ee803a562", size = 156399, upload-time = "2025-03-02T19:43:17.233Z" }, - { url = "https://files.pythonhosted.org/packages/c7/4a/c1d3f27ec8b3fff5a96617251bf3f61c67972869ac0a0419558fc3e2cbe6/levenshtein-0.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dafa29c0e616f322b574e0b2aeb5b1ff2f8d9a1a6550f22321f3bd9bb81036e3", size = 151061, upload-time = "2025-03-02T19:43:18.414Z" }, - { url = "https://files.pythonhosted.org/packages/4d/8f/2521081e9a265891edf46aa30e1b59c1f347a452aed4c33baafbec5216fa/levenshtein-0.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be7a7642ea64392fa1e6ef7968c2e50ef2152c60948f95d0793361ed97cf8a6f", size = 183119, upload-time = "2025-03-02T19:43:19.975Z" }, - { url = "https://files.pythonhosted.org/packages/1f/a0/a63e3bce6376127596d04be7f57e672d2f3d5f540265b1e30b9dd9b3c5a9/levenshtein-0.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:060b48c45ed54bcea9582ce79c6365b20a1a7473767e0b3d6be712fa3a22929c", size = 185352, upload-time = "2025-03-02T19:43:21.424Z" }, - { url = "https://files.pythonhosted.org/packages/17/8c/8352e992063952b38fb61d49bad8d193a4a713e7eeceb3ae74b719d7863d/levenshtein-0.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:712f562c5e64dd0398d3570fe99f8fbb88acec7cc431f101cb66c9d22d74c542", size = 159879, upload-time = "2025-03-02T19:43:22.792Z" }, - { url = "https://files.pythonhosted.org/packages/69/b4/564866e2038acf47c3de3e9292fc7fc7cc18d2593fedb04f001c22ac6e15/levenshtein-0.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6141ad65cab49aa4527a3342d76c30c48adb2393b6cdfeca65caae8d25cb4b8", size = 245005, upload-time = "2025-03-02T19:43:24.069Z" }, - { url = "https://files.pythonhosted.org/packages/ba/f9/7367f87e3a6eed282f3654ec61a174b4d1b78a7a73f2cecb91f0ab675153/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:799b8d73cda3265331116f62932f553804eae16c706ceb35aaf16fc2a704791b", size = 1116865, upload-time = "2025-03-02T19:43:25.4Z" }, - { url = "https://files.pythonhosted.org/packages/f5/02/b5b3bfb4b4cd430e9d110bad2466200d51c6061dae7c5a64e36047c8c831/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ec99871d98e517e1cc4a15659c62d6ea63ee5a2d72c5ddbebd7bae8b9e2670c8", size = 1401723, upload-time = "2025-03-02T19:43:28.099Z" }, - { url = "https://files.pythonhosted.org/packages/ef/69/b93bccd093b3f06a99e67e11ebd6e100324735dc2834958ba5852a1b9fed/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8799164e1f83588dbdde07f728ea80796ea72196ea23484d78d891470241b222", size = 1226276, upload-time = "2025-03-02T19:43:30.192Z" }, - { url = "https://files.pythonhosted.org/packages/ab/32/37dd1bc5ce866c136716619e6f7081d7078d7dd1c1da7025603dcfd9cf5f/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:583943813898326516ab451a83f734c6f07488cda5c361676150d3e3e8b47927", size = 1420132, upload-time = "2025-03-02T19:43:33.322Z" }, - { url = "https://files.pythonhosted.org/packages/4b/08/f3bc828dd9f0f8433b26f37c4fceab303186ad7b9b70819f2ccb493d99fc/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5bb22956af44bb4eade93546bf95be610c8939b9a9d4d28b2dfa94abf454fed7", size = 1189144, upload-time = "2025-03-02T19:43:34.814Z" }, - { url = "https://files.pythonhosted.org/packages/2d/54/5ecd89066cf579223d504abe3ac37ba11f63b01a19fd12591083acc00eb6/levenshtein-0.27.1-cp312-cp312-win32.whl", hash = "sha256:d9099ed1bcfa7ccc5540e8ad27b5dc6f23d16addcbe21fdd82af6440f4ed2b6d", size = 88279, upload-time = "2025-03-02T19:43:38.86Z" }, - { url = "https://files.pythonhosted.org/packages/53/79/4f8fabcc5aca9305b494d1d6c7a98482e90a855e0050ae9ff5d7bf4ab2c6/levenshtein-0.27.1-cp312-cp312-win_amd64.whl", hash = "sha256:7f071ecdb50aa6c15fd8ae5bcb67e9da46ba1df7bba7c6bf6803a54c7a41fd96", size = 100659, upload-time = "2025-03-02T19:43:40.082Z" }, - { url = "https://files.pythonhosted.org/packages/cb/81/f8e4c0f571c2aac2e0c56a6e0e41b679937a2b7013e79415e4aef555cff0/levenshtein-0.27.1-cp312-cp312-win_arm64.whl", hash = "sha256:83b9033a984ccace7703f35b688f3907d55490182fd39b33a8e434d7b2e249e6", size = 88168, upload-time = "2025-03-02T19:43:41.42Z" }, - { url = "https://files.pythonhosted.org/packages/c6/d3/30485fb9aee848542ee2d01aba85106a7f5da982ebeeffc619f70ea593c7/levenshtein-0.27.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ab00c2cae2889166afb7e1af64af2d4e8c1b126f3902d13ef3740df00e54032d", size = 173397, upload-time = "2025-03-02T19:43:42.553Z" }, - { url = "https://files.pythonhosted.org/packages/df/9f/40a81c54cfe74b22737710e654bd25ad934a675f737b60b24f84099540e0/levenshtein-0.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c27e00bc7527e282f7c437817081df8da4eb7054e7ef9055b851fa3947896560", size = 155787, upload-time = "2025-03-02T19:43:43.864Z" }, - { url = "https://files.pythonhosted.org/packages/df/98/915f4e24e21982b6eca2c0203546c160f4a83853fa6a2ac6e2b208a54afc/levenshtein-0.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5b07de42bfc051136cc8e7f1e7ba2cb73666aa0429930f4218efabfdc5837ad", size = 150013, upload-time = "2025-03-02T19:43:45.134Z" }, - { url = "https://files.pythonhosted.org/packages/80/93/9b0773107580416b9de14bf6a12bd1dd2b2964f7a9f6fb0e40723e1f0572/levenshtein-0.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb11ad3c9dae3063405aa50d9c96923722ab17bb606c776b6817d70b51fd7e07", size = 181234, upload-time = "2025-03-02T19:43:47.125Z" }, - { url = "https://files.pythonhosted.org/packages/91/b1/3cd4f69af32d40de14808142cc743af3a1b737b25571bd5e8d2f46b885e0/levenshtein-0.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c5986fb46cb0c063305fd45b0a79924abf2959a6d984bbac2b511d3ab259f3f", size = 183697, upload-time = "2025-03-02T19:43:48.412Z" }, - { url = "https://files.pythonhosted.org/packages/bb/65/b691e502c6463f6965b7e0d8d84224c188aa35b53fbc85853c72a0e436c9/levenshtein-0.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75191e469269ddef2859bc64c4a8cfd6c9e063302766b5cb7e1e67f38cc7051a", size = 159964, upload-time = "2025-03-02T19:43:49.704Z" }, - { url = "https://files.pythonhosted.org/packages/0f/c0/89a922a47306a475fb6d8f2ab08668f143d3dc7dea4c39d09e46746e031c/levenshtein-0.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51b3a7b2266933babc04e4d9821a495142eebd6ef709f90e24bc532b52b81385", size = 244759, upload-time = "2025-03-02T19:43:51.733Z" }, - { url = "https://files.pythonhosted.org/packages/b4/93/30283c6e69a6556b02e0507c88535df9613179f7b44bc49cdb4bc5e889a3/levenshtein-0.27.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bbac509794afc3e2a9e73284c9e3d0aab5b1d928643f42b172969c3eefa1f2a3", size = 1115955, upload-time = "2025-03-02T19:43:53.739Z" }, - { url = "https://files.pythonhosted.org/packages/0b/cf/7e19ea2c23671db02fbbe5a5a4aeafd1d471ee573a6251ae17008458c434/levenshtein-0.27.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8d68714785178347ecb272b94e85cbf7e638165895c4dd17ab57e7742d8872ec", size = 1400921, upload-time = "2025-03-02T19:43:55.146Z" }, - { url = "https://files.pythonhosted.org/packages/e3/f7/fb42bfe2f3b46ef91f0fc6fa217b44dbeb4ef8c72a9c1917bbbe1cafc0f8/levenshtein-0.27.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:8ee74ee31a5ab8f61cd6c6c6e9ade4488dde1285f3c12207afc018393c9b8d14", size = 1225037, upload-time = "2025-03-02T19:43:56.7Z" }, - { url = "https://files.pythonhosted.org/packages/74/25/c86f8874ac7b0632b172d0d1622ed3ab9608a7f8fe85d41d632b16f5948e/levenshtein-0.27.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f2441b6365453ec89640b85344afd3d602b0d9972840b693508074c613486ce7", size = 1420601, upload-time = "2025-03-02T19:43:58.383Z" }, - { url = "https://files.pythonhosted.org/packages/20/fe/ebfbaadcd90ea7dfde987ae95b5c11dc27c2c5d55a2c4ccbbe4e18a8af7b/levenshtein-0.27.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a9be39640a46d8a0f9be729e641651d16a62b2c07d3f4468c36e1cc66b0183b9", size = 1188241, upload-time = "2025-03-02T19:44:00.976Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1a/aa6b07316e10781a6c5a5a8308f9bdc22213dc3911b959daa6d7ff654fc6/levenshtein-0.27.1-cp313-cp313-win32.whl", hash = "sha256:a520af67d976761eb6580e7c026a07eb8f74f910f17ce60e98d6e492a1f126c7", size = 88103, upload-time = "2025-03-02T19:44:02.42Z" }, - { url = "https://files.pythonhosted.org/packages/9d/7b/9bbfd417f80f1047a28d0ea56a9b38b9853ba913b84dd5998785c5f98541/levenshtein-0.27.1-cp313-cp313-win_amd64.whl", hash = "sha256:7dd60aa49c2d8d23e0ef6452c8329029f5d092f386a177e3385d315cabb78f2a", size = 100579, upload-time = "2025-03-02T19:44:04.142Z" }, - { url = "https://files.pythonhosted.org/packages/8b/01/5f3ff775db7340aa378b250e2a31e6b4b038809a24ff0a3636ef20c7ca31/levenshtein-0.27.1-cp313-cp313-win_arm64.whl", hash = "sha256:149cd4f0baf5884ac5df625b7b0d281721b15de00f447080e38f5188106e1167", size = 87933, upload-time = "2025-03-02T19:44:05.364Z" }, - { url = "https://files.pythonhosted.org/packages/25/ed/37e2d1f5e690d7376cd7e8bdd19411479ff352a3df9ab5f845dd680ef779/levenshtein-0.27.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c92a222ab95b8d903eae6d5e7d51fe6c999be021b647715c18d04d0b0880f463", size = 170482, upload-time = "2025-03-02T19:44:30.177Z" }, - { url = "https://files.pythonhosted.org/packages/6d/9f/30b1144b9d1da74743e7d7cdf47575b7013c9767e608c7454dbd318aacd2/levenshtein-0.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:71afc36b4ee950fa1140aff22ffda9e5e23280285858e1303260dbb2eabf342d", size = 153106, upload-time = "2025-03-02T19:44:31.489Z" }, - { url = "https://files.pythonhosted.org/packages/b1/c5/18d0bec94a166cebaefa3db4beab9a7e0d75412b52e9626f5dce1ca8d149/levenshtein-0.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b1daeebfc148a571f09cfe18c16911ea1eaaa9e51065c5f7e7acbc4b866afa", size = 150984, upload-time = "2025-03-02T19:44:32.697Z" }, - { url = "https://files.pythonhosted.org/packages/55/b4/4b80eb0c96caabdb683256cac9cc2cc9a73dee8ea80ab7cc3ee8aebd603f/levenshtein-0.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:105edcb14797d95c77f69bad23104314715a64cafbf4b0e79d354a33d7b54d8d", size = 158673, upload-time = "2025-03-02T19:44:33.998Z" }, - { url = "https://files.pythonhosted.org/packages/81/14/a43daefbc6d5e5561176150363cbac73003795b85ae136ffd4d0691af3fb/levenshtein-0.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9c58fb1ef8bdc8773d705fbacf628e12c3bb63ee4d065dda18a76e86042444a", size = 244419, upload-time = "2025-03-02T19:44:35.317Z" }, - { url = "https://files.pythonhosted.org/packages/d0/55/34f133f4f0998d7335bd96b9d315dc888b118e48e999c3d2c621b84965b9/levenshtein-0.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e52270591854af67217103955a36bd7436b57c801e3354e73ba44d689ed93697", size = 97932, upload-time = "2025-03-02T19:44:36.701Z" }, - { url = "https://files.pythonhosted.org/packages/7d/44/c5955d0b6830925559b00617d80c9f6e03a9b00c451835ee4da7010e71cd/levenshtein-0.27.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:909b7b6bce27a4ec90576c9a9bd9af5a41308dfecf364b410e80b58038277bbe", size = 170533, upload-time = "2025-03-02T19:44:38.096Z" }, - { url = "https://files.pythonhosted.org/packages/e7/3f/858572d68b33e13a9c154b99f153317efe68381bf63cc4e986e820935fc3/levenshtein-0.27.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d193a7f97b8c6a350e36ec58e41a627c06fa4157c3ce4b2b11d90cfc3c2ebb8f", size = 153119, upload-time = "2025-03-02T19:44:39.388Z" }, - { url = "https://files.pythonhosted.org/packages/d1/60/2bd8d001ea4eb53ca16faa7a649d56005ba22b1bcc2a4f1617ab27ed7e48/levenshtein-0.27.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:614be316e3c06118705fae1f717f9072d35108e5fd4e66a7dd0e80356135340b", size = 149576, upload-time = "2025-03-02T19:44:40.617Z" }, - { url = "https://files.pythonhosted.org/packages/e4/db/0580797e1e4ac26cf67761a235b29b49f62d2b175dbbc609882f2aecd4e4/levenshtein-0.27.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31fc0a5bb070722bdabb6f7e14955a294a4a968c68202d294699817f21545d22", size = 157445, upload-time = "2025-03-02T19:44:41.901Z" }, - { url = "https://files.pythonhosted.org/packages/f4/de/9c171c96d1f15c900086d7212b5543a85539e767689fc4933d14048ba1ec/levenshtein-0.27.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9415aa5257227af543be65768a80c7a75e266c3c818468ce6914812f88f9c3df", size = 243141, upload-time = "2025-03-02T19:44:43.228Z" }, - { url = "https://files.pythonhosted.org/packages/dc/1e/408fd10217eac0e43aea0604be22b4851a09e03d761d44d4ea12089dd70e/levenshtein-0.27.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7987ef006a3cf56a4532bd4c90c2d3b7b4ca9ad3bf8ae1ee5713c4a3bdfda913", size = 98045, upload-time = "2025-03-02T19:44:44.527Z" }, + { url = "https://files.pythonhosted.org/packages/ab/07/e8d04ec84fae72f0a75a2c46f897fe2abb82a657707a902a414faa5f8a72/levenshtein-0.27.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d61eff70799fd5e710625da8a13e5adabd62bfd9f70abb9c531af6cad458cd27", size = 171954, upload-time = "2025-11-01T12:12:40.151Z" }, + { url = "https://files.pythonhosted.org/packages/8d/13/606682ad2a7f0c01178cbc1f8de1b53d86e5dd8a03983c8feb8a6f403e76/levenshtein-0.27.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:477efed87edf72ad0d3870038479ed2f63020a42e69c6a38a32a550e51f8e70e", size = 158414, upload-time = "2025-11-01T12:12:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/ce/c5/9627e1fc5cbfaff7fbf2e95aaf29340929ff2e92ae2d185b967a36942262/levenshtein-0.27.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ef99b9827d7d1100fc4398ac5522bd56766b894561c0cbdea0a01b93f24e642", size = 133822, upload-time = "2025-11-01T12:12:43.243Z" }, + { url = "https://files.pythonhosted.org/packages/32/88/9e24a51b99b3dd6b3706a94bd258b2254edab5392e92c2e6d9b0773eba8f/levenshtein-0.27.3-cp310-cp310-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9091e8ca9fff6088836abf372f8871fb480e44603defa526e1c3ae2f1d70acc5", size = 114383, upload-time = "2025-11-01T12:12:44.4Z" }, + { url = "https://files.pythonhosted.org/packages/4c/95/9a11eb769bad0583712e2772e90ef92929d4ff4931fbb34efe79a0bff493/levenshtein-0.27.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6ffdb2329712c5595eda3532a4f701f87f6c73a0f7aaac240681bf0b54310d63", size = 153061, upload-time = "2025-11-01T12:12:46.215Z" }, + { url = "https://files.pythonhosted.org/packages/b3/86/47387ed38df23ed3a6640032cdca97367eacb2a2d2075d97d6e88f43b40e/levenshtein-0.27.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:35856330eac1b968b45a5abbc4a3d14279bd9d1224be727cb1aac9ac4928a419", size = 1115566, upload-time = "2025-11-01T12:12:47.965Z" }, + { url = "https://files.pythonhosted.org/packages/dc/17/ed94dadabdf7e86940f6179238312a6750688f44565a4eb19ae5a87ce8a8/levenshtein-0.27.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:5377e237f6a13f5b0618621cca7992848993470c011716c3ad09cdf19c3b13ab", size = 1007140, upload-time = "2025-11-01T12:12:49.283Z" }, + { url = "https://files.pythonhosted.org/packages/52/25/c971c043aec0994c5600789d2bf4c183e2f389ee21559bb46a06c6f46ec2/levenshtein-0.27.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e30614186eb5c43833b62ae7d893a116b88373eec8cf3f3d62ba51aa5962d8ea", size = 1185316, upload-time = "2025-11-01T12:12:50.849Z" }, + { url = "https://files.pythonhosted.org/packages/3c/54/2a1a1af73470cd6ca0d709efb1786fe4651eee9a3cb5b767903defb4fe9c/levenshtein-0.27.3-cp310-cp310-win32.whl", hash = "sha256:5499342fd6b003bd5abc28790c7b333884838f7fd8c50570a6520bbaf5e2a35b", size = 84312, upload-time = "2025-11-01T12:12:52.366Z" }, + { url = "https://files.pythonhosted.org/packages/10/15/50f508790a7b7e0d6258ec85add62c257ab27ca70e5e8a1bae8350305932/levenshtein-0.27.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e2792730388bec6a85d4d3e3a9b53b8a4b509722bea1a78a39a1a0a7d8f0e13", size = 94376, upload-time = "2025-11-01T12:12:53.361Z" }, + { url = "https://files.pythonhosted.org/packages/9a/3f/ca3e54e5144695cc8a34601d275fabfc97c2ab9b824cbe0b49a0173a0575/levenshtein-0.27.3-cp310-cp310-win_arm64.whl", hash = "sha256:8a2a274b55562a49c6e9dadb16d05f6c27ffa98906b55d5c122893457ca6e464", size = 87216, upload-time = "2025-11-01T12:12:54.674Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fd/42e28a86e2f04a2e064faa1eab7d81a35fb111212b508ce7e450f839943d/levenshtein-0.27.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:245b6ffb6e1b0828cafbce35c500cb3265d0962c121d090669f177968c5a2980", size = 172216, upload-time = "2025-11-01T12:12:55.727Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f4/fe665c8e5d8ebe4266807e43af72db9d4f84d4f513ea86eacca3aaf5f77b/levenshtein-0.27.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8f44c98fa23f489eb7b2ad87d5dd24b6a784434bb5edb73f6b0513309c949690", size = 158616, upload-time = "2025-11-01T12:12:56.99Z" }, + { url = "https://files.pythonhosted.org/packages/22/46/9998bc56729444e350c083635b94c3eae97218b8a618cdc89f6825eec08c/levenshtein-0.27.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f5f85a1fc96dfc147bba82b4c67d6346ea26c27ef77a6a9de689118e26dddbe", size = 134222, upload-time = "2025-11-01T12:12:58.437Z" }, + { url = "https://files.pythonhosted.org/packages/19/09/914b3fc22c083728904f8dc7876a2a90a602b4769f27f5320176cbd6f781/levenshtein-0.27.3-cp311-cp311-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:18ceddd38d0e990d2c1c9b72f3e191dace87e2f8f0446207ce9e9cd2bfdfc8a1", size = 114902, upload-time = "2025-11-01T12:12:59.645Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ee/f361bfa5afe24698fb07ae7811e00c2984131023c7688299dea4fd3f2f4c/levenshtein-0.27.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:222b81adca29ee4128183328c6e1b25a48c817d14a008ab49e74be9df963b293", size = 153562, upload-time = "2025-11-01T12:13:00.745Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4f/614d0ab9777ebb91895ce1c9390ec2f244f53f7ddf7e29f36b0ca33f3841/levenshtein-0.27.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee3769ab6e89c24f901e6b7004100630e86721464d7d0384860a322d7953d3a5", size = 1115732, upload-time = "2025-11-01T12:13:02.219Z" }, + { url = "https://files.pythonhosted.org/packages/24/d9/f33c4e35399349ec2eb7be53ed49459bf6e59c31668868c89cf6f7964029/levenshtein-0.27.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:03eba8fda9f3f2b4b0760263fa20b20a90ab00cbeeab4d0d9d899b4f77912b0a", size = 1009023, upload-time = "2025-11-01T12:13:03.954Z" }, + { url = "https://files.pythonhosted.org/packages/2e/63/e8803a6d71488334c100afc79a98efc8cf0086ad29ee7f1d083f7f2c584d/levenshtein-0.27.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c870b19e2d5c7bc7f16213cc10312b82d873a4d46e1c6d51857a12ef39a76552", size = 1185850, upload-time = "2025-11-01T12:13:05.341Z" }, + { url = "https://files.pythonhosted.org/packages/09/55/a6a815ef76a6d5f7a2ee4e1edc8e8f1f935b9fa278634cc687af19b86de9/levenshtein-0.27.3-cp311-cp311-win32.whl", hash = "sha256:1987622e9b8ba2ae47dc27469291da1f58462660fa34f4358e9d9c1830fb1355", size = 84375, upload-time = "2025-11-01T12:13:06.647Z" }, + { url = "https://files.pythonhosted.org/packages/e5/36/cf4c36ffe91994e772b682ff4c3cb721bd50ac05d4a887baa35f4d3b2268/levenshtein-0.27.3-cp311-cp311-win_amd64.whl", hash = "sha256:a2b2aa81851e01bb09667b07e80c3fbf0f5a7c6ee9cd80caf43cce705e65832a", size = 94598, upload-time = "2025-11-01T12:13:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/92/4b/43e820c3a13033908925eae8614ad7c0be1e5868836770565174012158c0/levenshtein-0.27.3-cp311-cp311-win_arm64.whl", hash = "sha256:a084b335c54def1aef9a594b7163faa44dd00056323808bab783f43d8e4c1395", size = 87133, upload-time = "2025-11-01T12:13:08.701Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8e/3be9d8e0245704e3af5258fb6cb157c3d59902e1351e95edf6ed8a8c0434/levenshtein-0.27.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2de7f095b0ca8e44de9de986ccba661cd0dec3511c751b499e76b60da46805e9", size = 169622, upload-time = "2025-11-01T12:13:10.026Z" }, + { url = "https://files.pythonhosted.org/packages/a6/42/a2b2fda5e8caf6ecd5aac142f946a77574a3961e65da62c12fd7e48e5cb1/levenshtein-0.27.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9b8b29e5d5145a3c958664c85151b1bb4b26e4ca764380b947e6a96a321217c", size = 159183, upload-time = "2025-11-01T12:13:11.197Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c4/f083fabbd61c449752df1746533538f4a8629e8811931b52f66e6c4290ad/levenshtein-0.27.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc975465a51b1c5889eadee1a583b81fba46372b4b22df28973e49e8ddb8f54a", size = 133120, upload-time = "2025-11-01T12:13:12.363Z" }, + { url = "https://files.pythonhosted.org/packages/4e/e5/b6421e04cb0629615b8efd6d4d167dd2b1afb5097b87bb83cd992004dcca/levenshtein-0.27.3-cp312-cp312-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:57573ed885118554770979fdee584071b66103f6d50beddeabb54607a1213d81", size = 114988, upload-time = "2025-11-01T12:13:13.486Z" }, + { url = "https://files.pythonhosted.org/packages/e5/77/39ee0e8d3028e90178e1031530ccc98563f8f2f0d905ec784669dcf0fa90/levenshtein-0.27.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23aff800a6dd5d91bb3754a6092085aa7ad46b28e497682c155c74f681cfaa2d", size = 153346, upload-time = "2025-11-01T12:13:14.744Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/c0f367bbd260dbd7a4e134fd21f459e0f5eac43deac507952b46a1d8a93a/levenshtein-0.27.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c08a952432b8ad9dccb145f812176db94c52cda732311ddc08d29fd3bf185b0a", size = 1114538, upload-time = "2025-11-01T12:13:15.851Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ef/ae71433f7b4db0bd2af7974785e36cdec899919203fb82e647c5a6109c07/levenshtein-0.27.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:3bfcb2d78ab9cc06a1e75da8fcfb7a430fe513d66cfe54c07e50f32805e5e6db", size = 1009734, upload-time = "2025-11-01T12:13:17.212Z" }, + { url = "https://files.pythonhosted.org/packages/27/dc/62c28b812dcb0953fc32ab7adf3d0e814e43c8560bb28d9269a44d874adf/levenshtein-0.27.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba7235f6dcb31a217247468295e2dd4c6c1d3ac81629dc5d355d93e1a5f4c185", size = 1185581, upload-time = "2025-11-01T12:13:18.661Z" }, + { url = "https://files.pythonhosted.org/packages/56/e8/2e7ab9c565793220edb8e5432f9a846386a157075bdd032a90e9585bce38/levenshtein-0.27.3-cp312-cp312-win32.whl", hash = "sha256:ea80d70f1d18c161a209be556b9094968627cbaae620e102459ef9c320a98cbb", size = 84660, upload-time = "2025-11-01T12:13:19.87Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a6/907a1fc8587dc91c40156973e09d106ab064c06eb28dc4700ba0fe54d654/levenshtein-0.27.3-cp312-cp312-win_amd64.whl", hash = "sha256:fbaa1219d9b2d955339a37e684256a861e9274a3fe3a6ee1b8ea8724c3231ed9", size = 94909, upload-time = "2025-11-01T12:13:21.323Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d6/e04f0ddf6a71df3cdd1817b71703490ac874601ed460b2af172d3752c321/levenshtein-0.27.3-cp312-cp312-win_arm64.whl", hash = "sha256:2edbaa84f887ea1d9d8e4440af3fdda44769a7855d581c6248d7ee51518402a8", size = 87358, upload-time = "2025-11-01T12:13:22.393Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f2/162e9ea7490b36bbf05776c8e3a8114c75aa78546ddda8e8f36731db3da6/levenshtein-0.27.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e55aa9f9453fd89d4a9ff1f3c4a650b307d5f61a7eed0568a52fbd2ff2eba107", size = 169230, upload-time = "2025-11-01T12:13:23.735Z" }, + { url = "https://files.pythonhosted.org/packages/01/2d/7316ba7f94e3d60e89bd120526bc71e4812866bb7162767a2a10f73f72c5/levenshtein-0.27.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ae4d484453c48939ecd01c5c213530c68dd5cd6e5090f0091ef69799ec7a8a9f", size = 158643, upload-time = "2025-11-01T12:13:25.549Z" }, + { url = "https://files.pythonhosted.org/packages/5e/87/85433cb1e51c45016f061d96fea3106b6969f700e2cbb56c15de82d0deeb/levenshtein-0.27.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d18659832567ee387b266be390da0de356a3aa6cf0e8bc009b6042d8188e131f", size = 132881, upload-time = "2025-11-01T12:13:26.822Z" }, + { url = "https://files.pythonhosted.org/packages/40/1c/3ce66c9a7da169a43dd89146d69df9dec935e6f86c70c6404f48d1291d2c/levenshtein-0.27.3-cp313-cp313-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027b3d142cc8ea2ab4e60444d7175f65a94dde22a54382b2f7b47cc24936eb53", size = 114650, upload-time = "2025-11-01T12:13:28.382Z" }, + { url = "https://files.pythonhosted.org/packages/73/60/7138e98884ca105c76ef192f5b43165d6eac6f32b432853ebe9f09ee50c9/levenshtein-0.27.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffdca6989368cc64f347f0423c528520f12775b812e170a0eb0c10e4c9b0f3ff", size = 153127, upload-time = "2025-11-01T12:13:29.781Z" }, + { url = "https://files.pythonhosted.org/packages/df/8f/664ac8b83026d7d1382866b68babae17e92b7b6ff8dc3c6205c0066b8ce1/levenshtein-0.27.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fa00ab389386032b02a1c9050ec3c6aa824d2bbcc692548fdc44a46b71c058c6", size = 1114602, upload-time = "2025-11-01T12:13:31.651Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c8/8905d96cf2d7ed6af7eb39a8be0925ef335729473c1e9d1f56230ecaffc5/levenshtein-0.27.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:691c9003c6c481b899a5c2f72e8ce05a6d956a9668dc75f2a3ce9f4381a76dc6", size = 1008036, upload-time = "2025-11-01T12:13:33.006Z" }, + { url = "https://files.pythonhosted.org/packages/c7/57/01c37608121380a6357a297625562adad1c1fc8058d4f62279b735108927/levenshtein-0.27.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:12f7fc8bf0c24492fe97905348e020b55b9fc6dbaab7cd452566d1a466cb5e15", size = 1185338, upload-time = "2025-11-01T12:13:34.452Z" }, + { url = "https://files.pythonhosted.org/packages/dd/57/bceab41d40b58dee7927a8d1d18ed3bff7c95c5e530fb60093ce741a8c26/levenshtein-0.27.3-cp313-cp313-win32.whl", hash = "sha256:9f4872e4e19ee48eed39f214eea4eca42e5ef303f8a4a488d8312370674dbf3a", size = 84562, upload-time = "2025-11-01T12:13:35.858Z" }, + { url = "https://files.pythonhosted.org/packages/42/1d/74f1ff589bb687d0cad2bbdceef208dc070f56d1e38a3831da8c00bf13bb/levenshtein-0.27.3-cp313-cp313-win_amd64.whl", hash = "sha256:83aa2422e9a9af2c9d3e56a53e3e8de6bae58d1793628cae48c4282577c5c2c6", size = 94658, upload-time = "2025-11-01T12:13:36.963Z" }, + { url = "https://files.pythonhosted.org/packages/21/3c/22c86d3c8f254141096fd6089d2e9fdf98b1472c7a5d79d36d3557ec2d83/levenshtein-0.27.3-cp313-cp313-win_arm64.whl", hash = "sha256:d4adaf1edbcf38c3f2e290b52f4dcb5c6deff20308c26ef1127a106bc2d23e9f", size = 86929, upload-time = "2025-11-01T12:13:37.997Z" }, + { url = "https://files.pythonhosted.org/packages/0e/bc/9b7cf1b5fa098b86844d42de22549304699deff309c5c9e28b9a3fc4076a/levenshtein-0.27.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:272e24764b8210337b65a1cfd69ce40df5d2de1a3baf1234e7f06d2826ba2e7a", size = 170360, upload-time = "2025-11-01T12:13:39.019Z" }, + { url = "https://files.pythonhosted.org/packages/dc/95/997f2c83bd4712426bf0de8143b5e4403c7ebbafb5d1271983e774de3ae7/levenshtein-0.27.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:329a8e748a4e14d56daaa11f07bce3fde53385d05bad6b3f6dd9ee7802cdc915", size = 159098, upload-time = "2025-11-01T12:13:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/123c3316ae2f72c73be4fba9756924af015da4c0e5b12804f5753c0ee511/levenshtein-0.27.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5fea1a9c6b9cc8729e467e2174b4359ff6bac27356bb5f31898e596b4ce133a", size = 136655, upload-time = "2025-11-01T12:13:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/45/72/a3180d437736b1b9eacc3100be655a756deafb91de47c762d40eb45a9d91/levenshtein-0.27.3-cp313-cp313t-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3a61aa825819b6356555091d8a575d1235bd9c3753a68316a261af4856c3b487", size = 117511, upload-time = "2025-11-01T12:13:42.647Z" }, + { url = "https://files.pythonhosted.org/packages/61/f9/ba7c546a4b99347938e6661104064ab6a3651c601d59f241ffdc37510ecc/levenshtein-0.27.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51de7a514e8183f0a82f2947d01b014d2391426543b1c076bf5a26328cec4e4", size = 155656, upload-time = "2025-11-01T12:13:44.208Z" }, + { url = "https://files.pythonhosted.org/packages/42/cd/5edd6e1e02c3e47c8121761756dd0f85f816b636f25509118b687e6b0f96/levenshtein-0.27.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53cbf726d6e92040c9be7e594d959d496bd62597ea48eba9d96105898acbeafe", size = 1116689, upload-time = "2025-11-01T12:13:45.485Z" }, + { url = "https://files.pythonhosted.org/packages/95/67/25ca0119e0c6ec17226c72638f48ef8887124597ac48ad5da111c0b3a825/levenshtein-0.27.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:191b358afead8561c4fcfed22f83c13bb6c8da5f5789e277f0c5aa1c45ca612f", size = 1003166, upload-time = "2025-11-01T12:13:47.126Z" }, + { url = "https://files.pythonhosted.org/packages/45/64/ab216f3fb3cef1ee7e222665537f9340d828ef84c99409ba31f2ef2a3947/levenshtein-0.27.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ba1318d0635b834b8f0397014a7c43f007e65fce396a47614780c881bdff828b", size = 1189362, upload-time = "2025-11-01T12:13:48.627Z" }, + { url = "https://files.pythonhosted.org/packages/31/58/b150034858de0899a5a222974b6710618ebc0779a0695df070f7ab559a0b/levenshtein-0.27.3-cp313-cp313t-win32.whl", hash = "sha256:8dd9e1db6c3b35567043e155a686e4827c4aa28a594bd81e3eea84d3a1bd5875", size = 86149, upload-time = "2025-11-01T12:13:50.588Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c4/bbe46a11073641450200e6a604b3b62d311166e8061c492612a40e560e85/levenshtein-0.27.3-cp313-cp313t-win_amd64.whl", hash = "sha256:7813ecdac7a6223264ebfea0c8d69959c43d21a99694ef28018d22c4265c2af6", size = 96685, upload-time = "2025-11-01T12:13:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/23/65/30b362ad9bfc1085741776a08b6ddee3f434e9daac2920daaee2e26271bf/levenshtein-0.27.3-cp313-cp313t-win_arm64.whl", hash = "sha256:8f05a0d23d13a6f802c7af595d0e43f5b9b98b6ed390cec7a35cb5d6693b882b", size = 88538, upload-time = "2025-11-01T12:13:52.757Z" }, + { url = "https://files.pythonhosted.org/packages/f3/e1/2f705da403f865a5fa3449b155738dc9c53021698fd6926253a9af03180b/levenshtein-0.27.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a6728bfae9a86002f0223576675fc7e2a6e7735da47185a1d13d1eaaa73dd4be", size = 169457, upload-time = "2025-11-01T12:13:53.778Z" }, + { url = "https://files.pythonhosted.org/packages/76/2c/bb6ef359e007fe7b6b3195b68a94f4dd3ecd1885ee337ee8fbd4df55996f/levenshtein-0.27.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8e5037c4a6f97a238e24aad6f98a1e984348b7931b1b04b6bd02bd4f8238150d", size = 158680, upload-time = "2025-11-01T12:13:55.005Z" }, + { url = "https://files.pythonhosted.org/packages/51/7b/de1999f4cf1cfebc3fbbf03a6d58498952d6560d9798af4b0a566e6b6f30/levenshtein-0.27.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6cf5ecf9026bf24cf66ad019c6583f50058fae3e1b3c20e8812455b55d597f1", size = 133167, upload-time = "2025-11-01T12:13:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/aaa7f3a0a8ae8744b284043653652db3d7d93595517f9ed8158c03287692/levenshtein-0.27.3-cp314-cp314-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9285084bd2fc19adb47dab54ed4a71f57f78fe0d754e4a01e3c75409a25aed24", size = 114530, upload-time = "2025-11-01T12:13:57.883Z" }, + { url = "https://files.pythonhosted.org/packages/29/ce/ed422816fb30ffa3bc11597b30d5deca06b4a1388707a04215da73c65b53/levenshtein-0.27.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce3bbbe92172a08b599d79956182c6b7ab6ec8d4adbe7237417a363b968ad87b", size = 153325, upload-time = "2025-11-01T12:13:59.318Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5a/a225477a0bda154f19f1c07a5e35500d631ae25dfd620b479027d79f0d4c/levenshtein-0.27.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9dac48fab9d166ca90e12fb6cf6c7c8eb9c41aacf7136584411e20f7f136f745", size = 1114956, upload-time = "2025-11-01T12:14:00.543Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c4/a1be1040f3cce516a5e2be68453fd0c32ac63b2e9d31f476723fd8002c09/levenshtein-0.27.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d37a83722dc5326c93d17078e926c4732dc4f3488dc017c6839e34cd16af92b7", size = 1007610, upload-time = "2025-11-01T12:14:02.036Z" }, + { url = "https://files.pythonhosted.org/packages/86/d7/6f50e8a307e0c2befd819b481eb3a4c2eacab3dd8101982423003fac8ea3/levenshtein-0.27.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3466cb8294ce586e49dd467560a153ab8d296015c538223f149f9aefd3d9f955", size = 1185379, upload-time = "2025-11-01T12:14:03.385Z" }, + { url = "https://files.pythonhosted.org/packages/6b/e5/5d8fb1b3ebd5735f53221bf95c923066bcfc132234925820128f7eee5b47/levenshtein-0.27.3-cp314-cp314-win32.whl", hash = "sha256:c848bf2457b268672b7e9e73b44f18f49856420ac50b2564cf115a6e4ef82688", size = 86328, upload-time = "2025-11-01T12:14:04.74Z" }, + { url = "https://files.pythonhosted.org/packages/30/82/8a9ccbdb4e38bd4d516f2804999dccb8cb4bcb4e33f52851735da0c73ea7/levenshtein-0.27.3-cp314-cp314-win_amd64.whl", hash = "sha256:742633f024362a4ed6ef9d7e75d68f74b041ae738985fcf55a0e6d1d4cade438", size = 96640, upload-time = "2025-11-01T12:14:06.24Z" }, + { url = "https://files.pythonhosted.org/packages/14/86/f9d15919f59f5d92c6baa500315e1fa0143a39d811427b83c54f038267ca/levenshtein-0.27.3-cp314-cp314-win_arm64.whl", hash = "sha256:9eed6851224b19e8d588ddb8eb8a4ae3c2dcabf3d1213985f0b94a67e517b1df", size = 89689, upload-time = "2025-11-01T12:14:07.379Z" }, + { url = "https://files.pythonhosted.org/packages/ed/f6/10f44975ae6dc3047b2cd260e3d4c3a5258b8d10690a42904115de24fc51/levenshtein-0.27.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:77de69a345c76227b51a4521cd85442eb3da54c7eb6a06663a20c058fc49e683", size = 170518, upload-time = "2025-11-01T12:14:09.196Z" }, + { url = "https://files.pythonhosted.org/packages/08/07/fa294a145a0c99a814a9a807614962c1ee0f5749ca691645980462027d5d/levenshtein-0.27.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:eba2756dc1f5b962b0ff80e49abb2153d5e809cc5e7fa5e85be9410ce474795d", size = 159097, upload-time = "2025-11-01T12:14:10.404Z" }, + { url = "https://files.pythonhosted.org/packages/ae/50/24bdf37813fc30f293e53b46022b091144f4737a6a66663d2235b311bb98/levenshtein-0.27.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c8fcb498287e971d84260f67808ff1a06b3f6212d80fea75cf5155db80606ff", size = 136650, upload-time = "2025-11-01T12:14:11.579Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a9/0399c7a190b277cdea3acc801129d9d30da57c3fa79519e7b8c3f080d86c/levenshtein-0.27.3-cp314-cp314t-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f067092c67464faab13e00a5c1a80da93baca8955d4d49579861400762e35591", size = 117515, upload-time = "2025-11-01T12:14:12.877Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a4/1c27533e97578b385a4b8079abe8d1ce2e514717c761efbe4bf7bbd0ac2e/levenshtein-0.27.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92415f32c68491203f2855d05eef3277d376182d014cf0859c013c89f277fbbf", size = 155711, upload-time = "2025-11-01T12:14:13.985Z" }, + { url = "https://files.pythonhosted.org/packages/50/35/bbc26638394a72b1e31a685ec251c995ee66a630c7e5c86f98770928b632/levenshtein-0.27.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef61eeaf1e0a42d7d947978d981fe4b9426b98b3dd8c1582c535f10dee044c3f", size = 1116692, upload-time = "2025-11-01T12:14:15.359Z" }, + { url = "https://files.pythonhosted.org/packages/cd/83/32fcf28b388f8dc6c36b54552b9bae289dab07d43df104893158c834cbcc/levenshtein-0.27.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:103bb2e9049d1aa0d1216dd09c1c9106ecfe7541bbdc1a0490b9357d42eec8f2", size = 1003167, upload-time = "2025-11-01T12:14:17.469Z" }, + { url = "https://files.pythonhosted.org/packages/d1/79/1fbf2877ec4b819f373a32ebe3c48a61ee810693593a6015108b0be97b78/levenshtein-0.27.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6a64ddd1986b2a4c468b09544382287315c53585eb067f6e200c337741e057ee", size = 1189417, upload-time = "2025-11-01T12:14:19.081Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/dad4e09f1f7459c64172e48e40ed2baf3aa92d38205bcbd1b4ff00853701/levenshtein-0.27.3-cp314-cp314t-win32.whl", hash = "sha256:957244f27dc284ccb030a8b77b8a00deb7eefdcd70052a4b1d96f375780ae9dc", size = 88144, upload-time = "2025-11-01T12:14:20.667Z" }, + { url = "https://files.pythonhosted.org/packages/c0/61/cd51dc8b8a382e17c559a9812734c3a9afc2dab7d36253516335ee16ae50/levenshtein-0.27.3-cp314-cp314t-win_amd64.whl", hash = "sha256:ccd7eaa6d8048c3ec07c93cfbcdefd4a3ae8c6aca3a370f2023ee69341e5f076", size = 98516, upload-time = "2025-11-01T12:14:21.786Z" }, + { url = "https://files.pythonhosted.org/packages/27/5e/3fb67e882c1fee01ebb7abc1c0a6669e5ff8acd060e93bfe7229e9ce6e4f/levenshtein-0.27.3-cp314-cp314t-win_arm64.whl", hash = "sha256:1d8520b89b7a27bb5aadbcc156715619bcbf556a8ac46ad932470945dca6e1bd", size = 91020, upload-time = "2025-11-01T12:14:22.944Z" }, + { url = "https://files.pythonhosted.org/packages/b4/bc/21983893d3f40c6990e2e51c02dd48cfca350a36214be90d7c58f5f85896/levenshtein-0.27.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d2d7d22b6117a143f0cf101fe18a3ca90bd949fc33716a42d6165b9768d4a78c", size = 166073, upload-time = "2025-11-01T12:14:24.436Z" }, + { url = "https://files.pythonhosted.org/packages/ef/bb/52deb821ebf0cfc61baf7c9ebc5601649cfbfdaaaf156867786d1c5332d5/levenshtein-0.27.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:a55e7a2f317abd28576636e1f840fd268261f447c496a8481a9997a5ce889c59", size = 153629, upload-time = "2025-11-01T12:14:25.623Z" }, + { url = "https://files.pythonhosted.org/packages/60/0c/b72e6e2d16efd57c143785a30370ca50c2e355a9d0d678edb1c024865447/levenshtein-0.27.3-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55fa5f11952c38186bd4719e936eb4595b3d519218634924928787c36840256c", size = 130242, upload-time = "2025-11-01T12:14:26.926Z" }, + { url = "https://files.pythonhosted.org/packages/b5/b0/0aafad0dab03a58fd507773d3ff94ec13efdd3772ba217f85366213ab7ae/levenshtein-0.27.3-pp311-pypy311_pp73-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:559d3588e6766134d95f84f830cf40166360e1769d253f5f83474bff10a24341", size = 150655, upload-time = "2025-11-01T12:14:28.034Z" }, + { url = "https://files.pythonhosted.org/packages/b7/77/42dbcbafe9e0b0eb14cb6b08378c8c3bdc563ee34ee58f62e708e7f8956e/levenshtein-0.27.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:82d40da143c1b9e27adcd34a33dfcc4a0761aa717c5f618b9c6f57dec5d7a958", size = 92370, upload-time = "2025-11-01T12:14:29.143Z" }, ] [[package]] name = "mariadb" -version = "1.0.11" +version = "1.1.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/0a/ce5724f852f2937c6955bcea09659aa2d85d487df1c9de6711344b71527d/mariadb-1.0.11.zip", hash = "sha256:76916c892bc936c5b0f36e25a1411f651a7b7ce978992ae3a8de7e283efdacbf", size = 85926, upload-time = "2022-04-12T19:33:17.988Z" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/ba/cedef19833be88e07bfff11964441cda8a998f1628dd3b2fa3e7751d36e0/mariadb-1.1.14.tar.gz", hash = "sha256:e6d702a53eccf20922e47f2f45cfb5c7a0c2c6c0a46e4ee2d8a80d0ff4a52f34", size = 111715, upload-time = "2025-10-07T06:45:48.017Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/4a/6a6b7ad6a7b8156b41d0a6d849debe804f9d8696288ed2c6c31a9654357c/mariadb-1.0.11-cp310-cp310-win32.whl", hash = "sha256:4a583a80059d11f1895a2c93b1b7110948e87f0256da3e3222939a2530f0518e", size = 159743, upload-time = "2022-04-12T19:33:04.712Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a7/10740ceec1ce9d57f4bc3614e55efe2e72ae284e8c8d32eacabfbd7ad6cc/mariadb-1.0.11-cp310-cp310-win_amd64.whl", hash = "sha256:63c5c7cf99335e5c961e1d65a323576c9cb834e1a6a8084a6a8b4ffd85ca6213", size = 177316, upload-time = "2022-04-12T19:33:06.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/bf/5e1a3a5be297c3a679e08f5359165491508fbfb64faf854dc1d626cea9c0/mariadb-1.1.14-cp310-cp310-win32.whl", hash = "sha256:4c7f33578da163a1b79929aae241f5f981d7b9d5a94d89e589aad7ec58e313ea", size = 185064, upload-time = "2025-10-07T06:45:24.858Z" }, + { url = "https://files.pythonhosted.org/packages/31/30/3a61991c13cb8257f5db64aca12bafaa3d811d407e1fae019139fd17c99b/mariadb-1.1.14-cp310-cp310-win_amd64.whl", hash = "sha256:3f6fdc4ded5e0500a6a29bf0c8bf1be94189dcef5a8d5e0e154a4b3456f86bcc", size = 202020, upload-time = "2025-10-07T06:45:27.785Z" }, + { url = "https://files.pythonhosted.org/packages/56/aa/a7b3c66b2792e8319ec9157d63851ff2e0b26496a05044e22b50a012a05e/mariadb-1.1.14-cp311-cp311-win32.whl", hash = "sha256:932a95016b7e9b8d78893aa5ee608e74199e3c6dd607dbe5e4da2010a4f67b88", size = 185061, upload-time = "2025-10-07T06:45:29.964Z" }, + { url = "https://files.pythonhosted.org/packages/54/04/ea2374867756b4082764484bc8b82e1798d94f171bcc914e08c60d640f8f/mariadb-1.1.14-cp311-cp311-win_amd64.whl", hash = "sha256:55ddbe5272c292cbcb2968d87681b5d2b327e65646a015e324b8eeb804d14531", size = 202016, upload-time = "2025-10-07T06:45:32.151Z" }, + { url = "https://files.pythonhosted.org/packages/00/04/659a8d30513700b5921ec96bddc07f550016c045fcbeb199d8cd18476ecc/mariadb-1.1.14-cp312-cp312-win32.whl", hash = "sha256:98d552a8bb599eceaa88f65002ad00bd88aeed160592c273a7e5c1d79ab733dd", size = 185266, upload-time = "2025-10-07T06:45:34.164Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a9/8f210291bc5fc044e20497454f40d35b3bab326e2cab6fccdc38121cb2c1/mariadb-1.1.14-cp312-cp312-win_amd64.whl", hash = "sha256:685a1ad2a24fd0aae1c4416fe0ac794adc84ab9209c8d0c57078f770d39731db", size = 202112, upload-time = "2025-10-07T06:45:35.824Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/51130048bcce038bb859978250515f1aad90e9c4d273630a704e0a8b1ae1/mariadb-1.1.14-cp313-cp313-win32.whl", hash = "sha256:3d2c795cde606f4e12c0d73282b062433f414cae035675b0d81f2d65c9b79ac5", size = 185221, upload-time = "2025-10-07T06:45:37.848Z" }, + { url = "https://files.pythonhosted.org/packages/af/23/e952a7e442913abd8079cd27b80b69474895c93b3727fad41c7642a80c62/mariadb-1.1.14-cp313-cp313-win_amd64.whl", hash = "sha256:7fd603c5cf23c47ef0d28fdc2b4b79919ee7f75d00ed070d3cd1054dcf816aeb", size = 202121, upload-time = "2025-10-07T06:45:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f2/7059f83543a4264b98777d7cc8aa203e1ca6a13a461f730d1c97f29628d4/mariadb-1.1.14-cp314-cp314-win32.whl", hash = "sha256:1a50b4612c0dd5b69690cebb34cef552a7f64dcadeb5aa91d70cd99bf01bc5b3", size = 190620, upload-time = "2025-10-07T06:45:41.349Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7c/7e094b0b396d742494f6346f2ffa9709e429970b0461aca50526f5f02f12/mariadb-1.1.14-cp314-cp314-win_amd64.whl", hash = "sha256:6659725837e48fa6af05e20128fb525029f706f1921d5dbf639a25b2f80b9f93", size = 206263, upload-time = "2025-10-07T06:45:43.227Z" }, ] [[package]] name = "mysql-connector-python" -version = "9.4.0" +version = "9.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/33/b332b001bc8c5ee09255a0d4b09a254da674450edd6a3e5228b245ca82a0/mysql_connector_python-9.5.0.tar.gz", hash = "sha256:92fb924285a86d8c146ebd63d94f9eaefa548da7813bc46271508fdc6cc1d596", size = 12251077, upload-time = "2025-10-22T09:05:45.423Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/5d/30210fcf7ba98d1e03de0c47a58218ab5313d82f2e01ae53b47f45c36b9d/mysql_connector_python-9.5.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:77d14c9fde90726de22443e8c5ba0912a4ebb632cc1ade52a349dacbac47b140", size = 17579085, upload-time = "2025-10-22T09:01:27.388Z" }, + { url = "https://files.pythonhosted.org/packages/77/92/ea79a0875436665330a81e82b4b73a6d52aebcfb1cf4d97f4ad4bd4dedf5/mysql_connector_python-9.5.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:4d603b55de310b9689bb3cb5e57fe97e98756e36d62f8f308f132f2c724f62b8", size = 18445098, upload-time = "2025-10-22T09:01:29.721Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f2/4578b5093f46985c659035e880e70e8b0bed44d4a59ad4e83df5d49b9c69/mysql_connector_python-9.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:48ffa71ba748afaae5c45ed9a085a72604368ce611fe81c3fdc146ef60181d51", size = 33660118, upload-time = "2025-10-22T09:01:32.048Z" }, + { url = "https://files.pythonhosted.org/packages/c5/60/63135610ae0cee1260ce64874c1ddbf08e7fb560c21a3d9cce88b0ddc266/mysql_connector_python-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:77c71df48293d3c08713ff7087cf483804c8abf41a4bb4aefea7317b752c8e9a", size = 34096212, upload-time = "2025-10-22T09:01:36.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b1/78dc693552cfbb45076b3638ca4c402fae52209af8f276370d02d78367a0/mysql_connector_python-9.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:4f8d2d9d586c34dc9508a44d19cf30ccafabbbd12d7f8ab58da3af118636843c", size = 16512395, upload-time = "2025-10-22T09:01:38.602Z" }, + { url = "https://files.pythonhosted.org/packages/05/03/77347d58b0027ce93a41858477e08422e498c6ebc24348b1f725ed7a67ae/mysql_connector_python-9.5.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:653e70cd10cf2d18dd828fae58dff5f0f7a5cf7e48e244f2093314dddf84a4b9", size = 17578984, upload-time = "2025-10-22T09:01:41.213Z" }, + { url = "https://files.pythonhosted.org/packages/a5/bb/0f45c7ee55ebc56d6731a593d85c0e7f25f83af90a094efebfd5be9fe010/mysql_connector_python-9.5.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:5add93f60b3922be71ea31b89bc8a452b876adbb49262561bd559860dae96b3f", size = 18445067, upload-time = "2025-10-22T09:01:43.215Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ec/054de99d4aa50d851a37edca9039280f7194cc1bfd30aab38f5bd6977ebe/mysql_connector_python-9.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:20950a5e44896c03e3dc93ceb3a5e9b48c9acae18665ca6e13249b3fe5b96811", size = 33668029, upload-time = "2025-10-22T09:01:45.74Z" }, + { url = "https://files.pythonhosted.org/packages/90/a2/e6095dc3a7ad5c959fe4a65681db63af131f572e57cdffcc7816bc84e3ad/mysql_connector_python-9.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7fdd3205b9242c284019310fa84437f3357b13f598e3f9b5d80d337d4a6406b8", size = 34101687, upload-time = "2025-10-22T09:01:48.462Z" }, + { url = "https://files.pythonhosted.org/packages/9c/88/bc13c33fca11acaf808bd1809d8602d78f5bb84f7b1e7b1a288c383a14fd/mysql_connector_python-9.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:c021d8b0830958b28712c70c53b206b4cf4766948dae201ea7ca588a186605e0", size = 16511749, upload-time = "2025-10-22T09:01:51.032Z" }, + { url = "https://files.pythonhosted.org/packages/02/89/167ebee82f4b01ba7339c241c3cc2518886a2be9f871770a1efa81b940a0/mysql_connector_python-9.5.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a72c2ef9d50b84f3c567c31b3bf30901af740686baa2a4abead5f202e0b7ea61", size = 17581904, upload-time = "2025-10-22T09:01:53.21Z" }, + { url = "https://files.pythonhosted.org/packages/67/46/630ca969ce10b30fdc605d65dab4a6157556d8cc3b77c724f56c2d83cb79/mysql_connector_python-9.5.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bd9ba5a946cfd3b3b2688a75135357e862834b0321ed936fd968049be290872b", size = 18448195, upload-time = "2025-10-22T09:01:55.378Z" }, + { url = "https://files.pythonhosted.org/packages/f6/87/4c421f41ad169d8c9065ad5c46673c7af889a523e4899c1ac1d6bfd37262/mysql_connector_python-9.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5ef7accbdf8b5f6ec60d2a1550654b7e27e63bf6f7b04020d5fb4191fb02bc4d", size = 33668638, upload-time = "2025-10-22T09:01:57.896Z" }, + { url = "https://files.pythonhosted.org/packages/a6/01/67cf210d50bfefbb9224b9a5c465857c1767388dade1004c903c8e22a991/mysql_connector_python-9.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a6e0a4a0274d15e3d4c892ab93f58f46431222117dba20608178dfb2cc4d5fd8", size = 34102899, upload-time = "2025-10-22T09:02:00.291Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ef/3d1a67d503fff38cc30e11d111cf28f0976987fb175f47b10d44494e1080/mysql_connector_python-9.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:b6c69cb37600b7e22f476150034e2afbd53342a175e20aea887f8158fc5e3ff6", size = 16512684, upload-time = "2025-10-22T09:02:02.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/18/f221aeac49ce94ac119a427afbd51fe1629d48745b571afc0de49647b528/mysql_connector_python-9.5.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:1f5f7346b0d5edb2e994c1bd77b3f5eed88b0ca368ad6788d1012c7e56d7bf68", size = 17581933, upload-time = "2025-10-22T09:02:04.396Z" }, + { url = "https://files.pythonhosted.org/packages/de/8e/14d44db7353350006a12e46d61c3a995bba06acd7547fc78f9bb32611e0c/mysql_connector_python-9.5.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:07bf52591b4215cb4318b4617c327a6d84c31978c11e3255f01a627bcda2618e", size = 18448446, upload-time = "2025-10-22T09:02:06.399Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f5/ab306f292a99bff3544ff44ad53661a031dc1a11e5b1ad64b9e5b5290ef9/mysql_connector_python-9.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:8972c1f960b30d487f34f9125ec112ea2b3200bd02c53e5e32ee7a43be6d64c1", size = 33668933, upload-time = "2025-10-22T09:02:08.785Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/d146d2642552ebb5811cf551f06aca7da536c80b18fb6c75bdbc29723388/mysql_connector_python-9.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:f6d32d7aa514d2f6f8709ba1e018314f82ab2acea2e6af30d04c1906fe9171b9", size = 34103214, upload-time = "2025-10-22T09:02:11.657Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f8/5e88e5eda1fe58f7d146b73744f691d85dce76fb42e7ce3de53e49911da3/mysql_connector_python-9.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:edd47048eb65c196b28aa9d2c0c6a017d8ca084a9a7041cd317301c829eb5a05", size = 16512689, upload-time = "2025-10-22T09:02:14.167Z" }, + { url = "https://files.pythonhosted.org/packages/14/42/52bef145028af1b8e633eb77773278a04b2cd9f824117209aba093018445/mysql_connector_python-9.5.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:6effda35df1a96d9a096f04468d40f2324ea36b34d0e9632e81daae8be97b308", size = 17581903, upload-time = "2025-10-22T09:02:16.441Z" }, + { url = "https://files.pythonhosted.org/packages/f4/a6/bd800b42bde86bf2e9468dfabcbd7538c66daff9d1a9fc97d2cc897f96fa/mysql_connector_python-9.5.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:fd057bd042464eedbf5337d1ceea7f2a4ab075a1cf6d1d62ffd5184966a656dd", size = 18448394, upload-time = "2025-10-22T09:02:18.436Z" }, + { url = "https://files.pythonhosted.org/packages/4a/21/a1a3247775d0dfee094499cb915560755eaa1013ac3b03e34a98b0e16e49/mysql_connector_python-9.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:2797dd7bbefb1d1669d984cfb284ea6b34401bbd9c1b3bf84e646d0bd3a82197", size = 33669845, upload-time = "2025-10-22T09:02:20.966Z" }, + { url = "https://files.pythonhosted.org/packages/58/b7/dcab48349ab8abafd6f40f113101549e0cf107e43dd9c7e1fae79799604b/mysql_connector_python-9.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:a5fff063ed48281b7374a4da6b9ef4293d390c153f79b1589ee547ea08c92310", size = 34104103, upload-time = "2025-10-22T09:02:23.469Z" }, + { url = "https://files.pythonhosted.org/packages/21/3a/be129764fe5f5cd89a5aa3f58e7a7471284715f4af71097a980d24ebec0a/mysql_connector_python-9.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:56104693478fd447886c470a6d0558ded0fe2577df44c18232a6af6a2bbdd3e9", size = 17001255, upload-time = "2025-10-22T09:02:25.765Z" }, + { url = "https://files.pythonhosted.org/packages/95/e1/45373c06781340c7b74fe9b88b85278ac05321889a307eaa5be079a997d4/mysql_connector_python-9.5.0-py2.py3-none-any.whl", hash = "sha256:ace137b88eb6fdafa1e5b2e03ac76ce1b8b1844b3a4af1192a02ae7c1a45bdee", size = 479047, upload-time = "2025-10-22T09:02:27.809Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/77/2b45e6460d05b1f1b7a4c8eb79a50440b4417971973bb78c9ef6cad630a6/mysql_connector_python-9.4.0.tar.gz", hash = "sha256:d111360332ae78933daf3d48ff497b70739aa292ab0017791a33e826234e743b", size = 12185532, upload-time = "2025-07-22T08:02:05.788Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/ef/1a35d9ebfaf80cf5aa238be471480e16a69a494d276fb07b889dc9a5cfc3/mysql_connector_python-9.4.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3c2603e00516cf4208c6266e85c5c87d5f4d0ac79768106d50de42ccc8414c05", size = 17501678, upload-time = "2025-07-22T07:57:23.237Z" }, - { url = "https://files.pythonhosted.org/packages/3c/39/09ae7082c77a978f2d72d94856e2e57906165c645693bc3a940bcad3a32d/mysql_connector_python-9.4.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:47884fcb050112b8bef3458e17eac47cc81a6cbbf3524e3456146c949772d9b4", size = 18369526, upload-time = "2025-07-22T07:57:27.569Z" }, - { url = "https://files.pythonhosted.org/packages/40/56/1bea00f5129550bcd0175781b9cd467e865d4aea4a6f38f700f34d95dcb8/mysql_connector_python-9.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f14b6936cd326e212fc9ab5f666dea3efea654f0cb644460334e60e22986e735", size = 33508525, upload-time = "2025-07-22T07:57:32.935Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ec/86dfefd3e6c0fca13085bc28b7f9baae3fce9f6af243d8693729f6b5063c/mysql_connector_python-9.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0f5ad70355720e64b72d7c068e858c9fd1f69b671d9575f857f235a10f878939", size = 33911834, upload-time = "2025-07-22T07:57:38.203Z" }, - { url = "https://files.pythonhosted.org/packages/2c/11/6907d53349b11478f72c8f22e38368d18262fbffc27e0f30e365d76dad93/mysql_connector_python-9.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:7106670abce510e440d393e27fc3602b8cf21e7a8a80216cc9ad9a68cd2e4595", size = 16393044, upload-time = "2025-07-22T07:57:42.053Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0c/4365a802129be9fa63885533c38be019f1c6b6f5bcf8844ac53902314028/mysql_connector_python-9.4.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:7df1a8ddd182dd8adc914f6dc902a986787bf9599705c29aca7b2ce84e79d361", size = 17501627, upload-time = "2025-07-22T07:57:45.416Z" }, - { url = "https://files.pythonhosted.org/packages/c0/bf/ca596c00d7a6eaaf8ef2f66c9b23cd312527f483073c43ffac7843049cb4/mysql_connector_python-9.4.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:3892f20472e13e63b1fb4983f454771dd29f211b09724e69a9750e299542f2f8", size = 18369494, upload-time = "2025-07-22T07:57:49.714Z" }, - { url = "https://files.pythonhosted.org/packages/25/14/6510a11ed9f80d77f743dc207773092c4ab78d5efa454b39b48480315d85/mysql_connector_python-9.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d3e87142103d71c4df647ece30f98e85e826652272ed1c74822b56f6acdc38e7", size = 33516187, upload-time = "2025-07-22T07:57:55.294Z" }, - { url = "https://files.pythonhosted.org/packages/16/a8/4f99d80f1cf77733ce9a44b6adb7f0dd7079e7afa51ca4826515ef0c3e16/mysql_connector_python-9.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:b27fcd403436fe83bafb2fe7fcb785891e821e639275c4ad3b3bd1e25f533206", size = 33917818, upload-time = "2025-07-22T07:58:00.523Z" }, - { url = "https://files.pythonhosted.org/packages/15/9c/127f974ca9d5ee25373cb5433da06bb1f36e05f2a6b7436da1fe9c6346b0/mysql_connector_python-9.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd6ff5afb9c324b0bbeae958c93156cce4168c743bf130faf224d52818d1f0ee", size = 16392378, upload-time = "2025-07-22T07:58:04.669Z" }, - { url = "https://files.pythonhosted.org/packages/03/7c/a543fb17c2dfa6be8548dfdc5879a0c7924cd5d1c79056c48472bb8fe858/mysql_connector_python-9.4.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4efa3898a24aba6a4bfdbf7c1f5023c78acca3150d72cc91199cca2ccd22f76f", size = 17503693, upload-time = "2025-07-22T07:58:08.96Z" }, - { url = "https://files.pythonhosted.org/packages/cb/6e/c22fbee05f5cfd6ba76155b6d45f6261d8d4c1e36e23de04e7f25fbd01a4/mysql_connector_python-9.4.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:665c13e7402235162e5b7a2bfdee5895192121b64ea455c90a81edac6a48ede5", size = 18371987, upload-time = "2025-07-22T07:58:13.273Z" }, - { url = "https://files.pythonhosted.org/packages/b4/fd/f426f5f35a3d3180c7f84d1f96b4631be2574df94ca1156adab8618b236c/mysql_connector_python-9.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:815aa6cad0f351c1223ef345781a538f2e5e44ef405fdb3851eb322bd9c4ca2b", size = 33516214, upload-time = "2025-07-22T07:58:18.967Z" }, - { url = "https://files.pythonhosted.org/packages/45/5a/1b053ae80b43cd3ccebc4bb99a98826969b3b0f8adebdcc2530750ad76ed/mysql_connector_python-9.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b3436a2c8c0ec7052932213e8d01882e6eb069dbab33402e685409084b133a1c", size = 33918565, upload-time = "2025-07-22T07:58:25.28Z" }, - { url = "https://files.pythonhosted.org/packages/cb/69/36b989de675d98ba8ff7d45c96c30c699865c657046f2e32db14e78f13d9/mysql_connector_python-9.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:57b0c224676946b70548c56798d5023f65afa1ba5b8ac9f04a143d27976c7029", size = 16392563, upload-time = "2025-07-22T07:58:29.623Z" }, - { url = "https://files.pythonhosted.org/packages/79/e2/13036479cd1070d1080cee747de6c96bd6fbb021b736dd3ccef2b19016c8/mysql_connector_python-9.4.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:fde3bbffb5270a4b02077029914e6a9d2ec08f67d8375b4111432a2778e7540b", size = 17503749, upload-time = "2025-07-22T07:58:33.649Z" }, - { url = "https://files.pythonhosted.org/packages/31/df/b89e6551b91332716d384dcc3223e1f8065902209dcd9e477a3df80154f7/mysql_connector_python-9.4.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:25f77ad7d845df3b5a5a3a6a8d1fed68248dc418a6938a371d1ddaaab6b9a8e3", size = 18372145, upload-time = "2025-07-22T07:58:37.384Z" }, - { url = "https://files.pythonhosted.org/packages/07/bd/af0de40a01d5cb4df19318cc018e64666f2b7fa89bffa1ab5b35337aae2c/mysql_connector_python-9.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:227dd420c71e6d4788d52d98f298e563f16b6853577e5ade4bd82d644257c812", size = 33516503, upload-time = "2025-07-22T07:58:41.987Z" }, - { url = "https://files.pythonhosted.org/packages/d1/9b/712053216fcbe695e519ecb1035ffd767c2de9f51ccba15078537c99d6fa/mysql_connector_python-9.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5163381a312d38122eded2197eb5cd7ccf1a5c5881d4e7a6de10d6ea314d088e", size = 33918904, upload-time = "2025-07-22T07:58:46.796Z" }, - { url = "https://files.pythonhosted.org/packages/64/15/cbd996d425c59811849f3c1d1b1dae089a1ae18c4acd4d8de2b847b772df/mysql_connector_python-9.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:c727cb1f82b40c9aaa7a15ab5cf0a7f87c5d8dce32eab5ff2530a4aa6054e7df", size = 16392566, upload-time = "2025-07-22T07:58:50.223Z" }, - { url = "https://files.pythonhosted.org/packages/36/34/b6165e15fd45a8deb00932d8e7d823de7650270873b4044c4db6688e1d8f/mysql_connector_python-9.4.0-py2.py3-none-any.whl", hash = "sha256:56e679169c704dab279b176fab2a9ee32d2c632a866c0f7cd48a8a1e2cf802c4", size = 406574, upload-time = "2025-07-22T07:59:08.394Z" }, + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] [[package]] name = "pydantic" -version = "2.12.3" +version = "2.12.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -260,123 +305,127 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/1e/4f0a3233767010308f2fd6bd0814597e3f63f1dc98304a9112b8759df4ff/pydantic-2.12.3.tar.gz", hash = "sha256:1da1c82b0fc140bb0103bc1441ffe062154c8d38491189751ee00fd8ca65ce74", size = 819383, upload-time = "2025-10-17T15:04:21.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/6b/83661fa77dcefa195ad5f8cd9af3d1a7450fd57cc883ad04d65446ac2029/pydantic-2.12.3-py3-none-any.whl", hash = "sha256:6986454a854bc3bc6e5443e1369e06a3a456af9d339eda45510f517d9ea5c6bf", size = 462431, upload-time = "2025-10-17T15:04:19.346Z" }, + { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" }, ] [[package]] name = "pydantic-core" -version = "2.41.4" +version = "2.41.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/18/d0944e8eaaa3efd0a91b0f1fc537d3be55ad35091b6a87638211ba691964/pydantic_core-2.41.4.tar.gz", hash = "sha256:70e47929a9d4a1905a67e4b687d5946026390568a8e952b92824118063cee4d5", size = 457557, upload-time = "2025-10-14T10:23:47.909Z" } +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/3d/9b8ca77b0f76fcdbf8bc6b72474e264283f461284ca84ac3fde570c6c49a/pydantic_core-2.41.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2442d9a4d38f3411f22eb9dd0912b7cbf4b7d5b6c92c4173b75d3e1ccd84e36e", size = 2111197, upload-time = "2025-10-14T10:19:43.303Z" }, - { url = "https://files.pythonhosted.org/packages/59/92/b7b0fe6ed4781642232755cb7e56a86e2041e1292f16d9ae410a0ccee5ac/pydantic_core-2.41.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:30a9876226dda131a741afeab2702e2d127209bde3c65a2b8133f428bc5d006b", size = 1917909, upload-time = "2025-10-14T10:19:45.194Z" }, - { url = "https://files.pythonhosted.org/packages/52/8c/3eb872009274ffa4fb6a9585114e161aa1a0915af2896e2d441642929fe4/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d55bbac04711e2980645af68b97d445cdbcce70e5216de444a6c4b6943ebcccd", size = 1969905, upload-time = "2025-10-14T10:19:46.567Z" }, - { url = "https://files.pythonhosted.org/packages/f4/21/35adf4a753bcfaea22d925214a0c5b880792e3244731b3f3e6fec0d124f7/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e1d778fb7849a42d0ee5927ab0f7453bf9f85eef8887a546ec87db5ddb178945", size = 2051938, upload-time = "2025-10-14T10:19:48.237Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d0/cdf7d126825e36d6e3f1eccf257da8954452934ede275a8f390eac775e89/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b65077a4693a98b90ec5ad8f203ad65802a1b9b6d4a7e48066925a7e1606706", size = 2250710, upload-time = "2025-10-14T10:19:49.619Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1c/af1e6fd5ea596327308f9c8d1654e1285cc3d8de0d584a3c9d7705bf8a7c/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62637c769dee16eddb7686bf421be48dfc2fae93832c25e25bc7242e698361ba", size = 2367445, upload-time = "2025-10-14T10:19:51.269Z" }, - { url = "https://files.pythonhosted.org/packages/d3/81/8cece29a6ef1b3a92f956ea6da6250d5b2d2e7e4d513dd3b4f0c7a83dfea/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfe3aa529c8f501babf6e502936b9e8d4698502b2cfab41e17a028d91b1ac7b", size = 2072875, upload-time = "2025-10-14T10:19:52.671Z" }, - { url = "https://files.pythonhosted.org/packages/e3/37/a6a579f5fc2cd4d5521284a0ab6a426cc6463a7b3897aeb95b12f1ba607b/pydantic_core-2.41.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca2322da745bf2eeb581fc9ea3bbb31147702163ccbcbf12a3bb630e4bf05e1d", size = 2191329, upload-time = "2025-10-14T10:19:54.214Z" }, - { url = "https://files.pythonhosted.org/packages/ae/03/505020dc5c54ec75ecba9f41119fd1e48f9e41e4629942494c4a8734ded1/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e8cd3577c796be7231dcf80badcf2e0835a46665eaafd8ace124d886bab4d700", size = 2151658, upload-time = "2025-10-14T10:19:55.843Z" }, - { url = "https://files.pythonhosted.org/packages/cb/5d/2c0d09fb53aa03bbd2a214d89ebfa6304be7df9ed86ee3dc7770257f41ee/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:1cae8851e174c83633f0833e90636832857297900133705ee158cf79d40f03e6", size = 2316777, upload-time = "2025-10-14T10:19:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/ea/4b/c2c9c8f5e1f9c864b57d08539d9d3db160e00491c9f5ee90e1bfd905e644/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a26d950449aae348afe1ac8be5525a00ae4235309b729ad4d3399623125b43c9", size = 2320705, upload-time = "2025-10-14T10:19:59.016Z" }, - { url = "https://files.pythonhosted.org/packages/28/c3/a74c1c37f49c0a02c89c7340fafc0ba816b29bd495d1a31ce1bdeacc6085/pydantic_core-2.41.4-cp310-cp310-win32.whl", hash = "sha256:0cf2a1f599efe57fa0051312774280ee0f650e11152325e41dfd3018ef2c1b57", size = 1975464, upload-time = "2025-10-14T10:20:00.581Z" }, - { url = "https://files.pythonhosted.org/packages/d6/23/5dd5c1324ba80303368f7569e2e2e1a721c7d9eb16acb7eb7b7f85cb1be2/pydantic_core-2.41.4-cp310-cp310-win_amd64.whl", hash = "sha256:a8c2e340d7e454dc3340d3d2e8f23558ebe78c98aa8f68851b04dcb7bc37abdc", size = 2024497, upload-time = "2025-10-14T10:20:03.018Z" }, - { url = "https://files.pythonhosted.org/packages/62/4c/f6cbfa1e8efacd00b846764e8484fe173d25b8dab881e277a619177f3384/pydantic_core-2.41.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:28ff11666443a1a8cf2a044d6a545ebffa8382b5f7973f22c36109205e65dc80", size = 2109062, upload-time = "2025-10-14T10:20:04.486Z" }, - { url = "https://files.pythonhosted.org/packages/21/f8/40b72d3868896bfcd410e1bd7e516e762d326201c48e5b4a06446f6cf9e8/pydantic_core-2.41.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61760c3925d4633290292bad462e0f737b840508b4f722247d8729684f6539ae", size = 1916301, upload-time = "2025-10-14T10:20:06.857Z" }, - { url = "https://files.pythonhosted.org/packages/94/4d/d203dce8bee7faeca791671c88519969d98d3b4e8f225da5b96dad226fc8/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eae547b7315d055b0de2ec3965643b0ab82ad0106a7ffd29615ee9f266a02827", size = 1968728, upload-time = "2025-10-14T10:20:08.353Z" }, - { url = "https://files.pythonhosted.org/packages/65/f5/6a66187775df87c24d526985b3a5d78d861580ca466fbd9d4d0e792fcf6c/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef9ee5471edd58d1fcce1c80ffc8783a650e3e3a193fe90d52e43bb4d87bff1f", size = 2050238, upload-time = "2025-10-14T10:20:09.766Z" }, - { url = "https://files.pythonhosted.org/packages/5e/b9/78336345de97298cf53236b2f271912ce11f32c1e59de25a374ce12f9cce/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15dd504af121caaf2c95cb90c0ebf71603c53de98305621b94da0f967e572def", size = 2249424, upload-time = "2025-10-14T10:20:11.732Z" }, - { url = "https://files.pythonhosted.org/packages/99/bb/a4584888b70ee594c3d374a71af5075a68654d6c780369df269118af7402/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a926768ea49a8af4d36abd6a8968b8790f7f76dd7cbd5a4c180db2b4ac9a3a2", size = 2366047, upload-time = "2025-10-14T10:20:13.647Z" }, - { url = "https://files.pythonhosted.org/packages/5f/8d/17fc5de9d6418e4d2ae8c675f905cdafdc59d3bf3bf9c946b7ab796a992a/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916b9b7d134bff5440098a4deb80e4cb623e68974a87883299de9124126c2a8", size = 2071163, upload-time = "2025-10-14T10:20:15.307Z" }, - { url = "https://files.pythonhosted.org/packages/54/e7/03d2c5c0b8ed37a4617430db68ec5e7dbba66358b629cd69e11b4d564367/pydantic_core-2.41.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cf90535979089df02e6f17ffd076f07237efa55b7343d98760bde8743c4b265", size = 2190585, upload-time = "2025-10-14T10:20:17.3Z" }, - { url = "https://files.pythonhosted.org/packages/be/fc/15d1c9fe5ad9266a5897d9b932b7f53d7e5cfc800573917a2c5d6eea56ec/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7533c76fa647fade2d7ec75ac5cc079ab3f34879626dae5689b27790a6cf5a5c", size = 2150109, upload-time = "2025-10-14T10:20:19.143Z" }, - { url = "https://files.pythonhosted.org/packages/26/ef/e735dd008808226c83ba56972566138665b71477ad580fa5a21f0851df48/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:37e516bca9264cbf29612539801ca3cd5d1be465f940417b002905e6ed79d38a", size = 2315078, upload-time = "2025-10-14T10:20:20.742Z" }, - { url = "https://files.pythonhosted.org/packages/90/00/806efdcf35ff2ac0f938362350cd9827b8afb116cc814b6b75cf23738c7c/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0c19cb355224037c83642429b8ce261ae108e1c5fbf5c028bac63c77b0f8646e", size = 2318737, upload-time = "2025-10-14T10:20:22.306Z" }, - { url = "https://files.pythonhosted.org/packages/41/7e/6ac90673fe6cb36621a2283552897838c020db343fa86e513d3f563b196f/pydantic_core-2.41.4-cp311-cp311-win32.whl", hash = "sha256:09c2a60e55b357284b5f31f5ab275ba9f7f70b7525e18a132ec1f9160b4f1f03", size = 1974160, upload-time = "2025-10-14T10:20:23.817Z" }, - { url = "https://files.pythonhosted.org/packages/e0/9d/7c5e24ee585c1f8b6356e1d11d40ab807ffde44d2db3b7dfd6d20b09720e/pydantic_core-2.41.4-cp311-cp311-win_amd64.whl", hash = "sha256:711156b6afb5cb1cb7c14a2cc2c4a8b4c717b69046f13c6b332d8a0a8f41ca3e", size = 2021883, upload-time = "2025-10-14T10:20:25.48Z" }, - { url = "https://files.pythonhosted.org/packages/33/90/5c172357460fc28b2871eb4a0fb3843b136b429c6fa827e4b588877bf115/pydantic_core-2.41.4-cp311-cp311-win_arm64.whl", hash = "sha256:6cb9cf7e761f4f8a8589a45e49ed3c0d92d1d696a45a6feaee8c904b26efc2db", size = 1968026, upload-time = "2025-10-14T10:20:27.039Z" }, - { url = "https://files.pythonhosted.org/packages/e9/81/d3b3e95929c4369d30b2a66a91db63c8ed0a98381ae55a45da2cd1cc1288/pydantic_core-2.41.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ab06d77e053d660a6faaf04894446df7b0a7e7aba70c2797465a0a1af00fc887", size = 2099043, upload-time = "2025-10-14T10:20:28.561Z" }, - { url = "https://files.pythonhosted.org/packages/58/da/46fdac49e6717e3a94fc9201403e08d9d61aa7a770fab6190b8740749047/pydantic_core-2.41.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c53ff33e603a9c1179a9364b0a24694f183717b2e0da2b5ad43c316c956901b2", size = 1910699, upload-time = "2025-10-14T10:20:30.217Z" }, - { url = "https://files.pythonhosted.org/packages/1e/63/4d948f1b9dd8e991a5a98b77dd66c74641f5f2e5225fee37994b2e07d391/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:304c54176af2c143bd181d82e77c15c41cbacea8872a2225dd37e6544dce9999", size = 1952121, upload-time = "2025-10-14T10:20:32.246Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a7/e5fc60a6f781fc634ecaa9ecc3c20171d238794cef69ae0af79ac11b89d7/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:025ba34a4cf4fb32f917d5d188ab5e702223d3ba603be4d8aca2f82bede432a4", size = 2041590, upload-time = "2025-10-14T10:20:34.332Z" }, - { url = "https://files.pythonhosted.org/packages/70/69/dce747b1d21d59e85af433428978a1893c6f8a7068fa2bb4a927fba7a5ff/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9f5f30c402ed58f90c70e12eff65547d3ab74685ffe8283c719e6bead8ef53f", size = 2219869, upload-time = "2025-10-14T10:20:35.965Z" }, - { url = "https://files.pythonhosted.org/packages/83/6a/c070e30e295403bf29c4df1cb781317b6a9bac7cd07b8d3acc94d501a63c/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd96e5d15385d301733113bcaa324c8bcf111275b7675a9c6e88bfb19fc05e3b", size = 2345169, upload-time = "2025-10-14T10:20:37.627Z" }, - { url = "https://files.pythonhosted.org/packages/f0/83/06d001f8043c336baea7fd202a9ac7ad71f87e1c55d8112c50b745c40324/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98f348cbb44fae6e9653c1055db7e29de67ea6a9ca03a5fa2c2e11a47cff0e47", size = 2070165, upload-time = "2025-10-14T10:20:39.246Z" }, - { url = "https://files.pythonhosted.org/packages/14/0a/e567c2883588dd12bcbc110232d892cf385356f7c8a9910311ac997ab715/pydantic_core-2.41.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec22626a2d14620a83ca583c6f5a4080fa3155282718b6055c2ea48d3ef35970", size = 2189067, upload-time = "2025-10-14T10:20:41.015Z" }, - { url = "https://files.pythonhosted.org/packages/f4/1d/3d9fca34273ba03c9b1c5289f7618bc4bd09c3ad2289b5420481aa051a99/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a95d4590b1f1a43bf33ca6d647b990a88f4a3824a8c4572c708f0b45a5290ed", size = 2132997, upload-time = "2025-10-14T10:20:43.106Z" }, - { url = "https://files.pythonhosted.org/packages/52/70/d702ef7a6cd41a8afc61f3554922b3ed8d19dd54c3bd4bdbfe332e610827/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:f9672ab4d398e1b602feadcffcdd3af44d5f5e6ddc15bc7d15d376d47e8e19f8", size = 2307187, upload-time = "2025-10-14T10:20:44.849Z" }, - { url = "https://files.pythonhosted.org/packages/68/4c/c06be6e27545d08b802127914156f38d10ca287a9e8489342793de8aae3c/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:84d8854db5f55fead3b579f04bda9a36461dab0730c5d570e1526483e7bb8431", size = 2305204, upload-time = "2025-10-14T10:20:46.781Z" }, - { url = "https://files.pythonhosted.org/packages/b0/e5/35ae4919bcd9f18603419e23c5eaf32750224a89d41a8df1a3704b69f77e/pydantic_core-2.41.4-cp312-cp312-win32.whl", hash = "sha256:9be1c01adb2ecc4e464392c36d17f97e9110fbbc906bcbe1c943b5b87a74aabd", size = 1972536, upload-time = "2025-10-14T10:20:48.39Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c2/49c5bb6d2a49eb2ee3647a93e3dae7080c6409a8a7558b075027644e879c/pydantic_core-2.41.4-cp312-cp312-win_amd64.whl", hash = "sha256:d682cf1d22bab22a5be08539dca3d1593488a99998f9f412137bc323179067ff", size = 2031132, upload-time = "2025-10-14T10:20:50.421Z" }, - { url = "https://files.pythonhosted.org/packages/06/23/936343dbcba6eec93f73e95eb346810fc732f71ba27967b287b66f7b7097/pydantic_core-2.41.4-cp312-cp312-win_arm64.whl", hash = "sha256:833eebfd75a26d17470b58768c1834dfc90141b7afc6eb0429c21fc5a21dcfb8", size = 1969483, upload-time = "2025-10-14T10:20:52.35Z" }, - { url = "https://files.pythonhosted.org/packages/13/d0/c20adabd181a029a970738dfe23710b52a31f1258f591874fcdec7359845/pydantic_core-2.41.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:85e050ad9e5f6fe1004eec65c914332e52f429bc0ae12d6fa2092407a462c746", size = 2105688, upload-time = "2025-10-14T10:20:54.448Z" }, - { url = "https://files.pythonhosted.org/packages/00/b6/0ce5c03cec5ae94cca220dfecddc453c077d71363b98a4bbdb3c0b22c783/pydantic_core-2.41.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7393f1d64792763a48924ba31d1e44c2cfbc05e3b1c2c9abb4ceeadd912cced", size = 1910807, upload-time = "2025-10-14T10:20:56.115Z" }, - { url = "https://files.pythonhosted.org/packages/68/3e/800d3d02c8beb0b5c069c870cbb83799d085debf43499c897bb4b4aaff0d/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94dab0940b0d1fb28bcab847adf887c66a27a40291eedf0b473be58761c9799a", size = 1956669, upload-time = "2025-10-14T10:20:57.874Z" }, - { url = "https://files.pythonhosted.org/packages/60/a4/24271cc71a17f64589be49ab8bd0751f6a0a03046c690df60989f2f95c2c/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:de7c42f897e689ee6f9e93c4bec72b99ae3b32a2ade1c7e4798e690ff5246e02", size = 2051629, upload-time = "2025-10-14T10:21:00.006Z" }, - { url = "https://files.pythonhosted.org/packages/68/de/45af3ca2f175d91b96bfb62e1f2d2f1f9f3b14a734afe0bfeff079f78181/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:664b3199193262277b8b3cd1e754fb07f2c6023289c815a1e1e8fb415cb247b1", size = 2224049, upload-time = "2025-10-14T10:21:01.801Z" }, - { url = "https://files.pythonhosted.org/packages/af/8f/ae4e1ff84672bf869d0a77af24fd78387850e9497753c432875066b5d622/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95b253b88f7d308b1c0b417c4624f44553ba4762816f94e6986819b9c273fb2", size = 2342409, upload-time = "2025-10-14T10:21:03.556Z" }, - { url = "https://files.pythonhosted.org/packages/18/62/273dd70b0026a085c7b74b000394e1ef95719ea579c76ea2f0cc8893736d/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1351f5bbdbbabc689727cb91649a00cb9ee7203e0a6e54e9f5ba9e22e384b84", size = 2069635, upload-time = "2025-10-14T10:21:05.385Z" }, - { url = "https://files.pythonhosted.org/packages/30/03/cf485fff699b4cdaea469bc481719d3e49f023241b4abb656f8d422189fc/pydantic_core-2.41.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1affa4798520b148d7182da0615d648e752de4ab1a9566b7471bc803d88a062d", size = 2194284, upload-time = "2025-10-14T10:21:07.122Z" }, - { url = "https://files.pythonhosted.org/packages/f9/7e/c8e713db32405dfd97211f2fc0a15d6bf8adb7640f3d18544c1f39526619/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7b74e18052fea4aa8dea2fb7dbc23d15439695da6cbe6cfc1b694af1115df09d", size = 2137566, upload-time = "2025-10-14T10:21:08.981Z" }, - { url = "https://files.pythonhosted.org/packages/04/f7/db71fd4cdccc8b75990f79ccafbbd66757e19f6d5ee724a6252414483fb4/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:285b643d75c0e30abda9dc1077395624f314a37e3c09ca402d4015ef5979f1a2", size = 2316809, upload-time = "2025-10-14T10:21:10.805Z" }, - { url = "https://files.pythonhosted.org/packages/76/63/a54973ddb945f1bca56742b48b144d85c9fc22f819ddeb9f861c249d5464/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f52679ff4218d713b3b33f88c89ccbf3a5c2c12ba665fb80ccc4192b4608dbab", size = 2311119, upload-time = "2025-10-14T10:21:12.583Z" }, - { url = "https://files.pythonhosted.org/packages/f8/03/5d12891e93c19218af74843a27e32b94922195ded2386f7b55382f904d2f/pydantic_core-2.41.4-cp313-cp313-win32.whl", hash = "sha256:ecde6dedd6fff127c273c76821bb754d793be1024bc33314a120f83a3c69460c", size = 1981398, upload-time = "2025-10-14T10:21:14.584Z" }, - { url = "https://files.pythonhosted.org/packages/be/d8/fd0de71f39db91135b7a26996160de71c073d8635edfce8b3c3681be0d6d/pydantic_core-2.41.4-cp313-cp313-win_amd64.whl", hash = "sha256:d081a1f3800f05409ed868ebb2d74ac39dd0c1ff6c035b5162356d76030736d4", size = 2030735, upload-time = "2025-10-14T10:21:16.432Z" }, - { url = "https://files.pythonhosted.org/packages/72/86/c99921c1cf6650023c08bfab6fe2d7057a5142628ef7ccfa9921f2dda1d5/pydantic_core-2.41.4-cp313-cp313-win_arm64.whl", hash = "sha256:f8e49c9c364a7edcbe2a310f12733aad95b022495ef2a8d653f645e5d20c1564", size = 1973209, upload-time = "2025-10-14T10:21:18.213Z" }, - { url = "https://files.pythonhosted.org/packages/36/0d/b5706cacb70a8414396efdda3d72ae0542e050b591119e458e2490baf035/pydantic_core-2.41.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ed97fd56a561f5eb5706cebe94f1ad7c13b84d98312a05546f2ad036bafe87f4", size = 1877324, upload-time = "2025-10-14T10:21:20.363Z" }, - { url = "https://files.pythonhosted.org/packages/de/2d/cba1fa02cfdea72dfb3a9babb067c83b9dff0bbcb198368e000a6b756ea7/pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a870c307bf1ee91fc58a9a61338ff780d01bfae45922624816878dce784095d2", size = 1884515, upload-time = "2025-10-14T10:21:22.339Z" }, - { url = "https://files.pythonhosted.org/packages/07/ea/3df927c4384ed9b503c9cc2d076cf983b4f2adb0c754578dfb1245c51e46/pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d25e97bc1f5f8f7985bdc2335ef9e73843bb561eb1fa6831fdfc295c1c2061cf", size = 2042819, upload-time = "2025-10-14T10:21:26.683Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ee/df8e871f07074250270a3b1b82aad4cd0026b588acd5d7d3eb2fcb1471a3/pydantic_core-2.41.4-cp313-cp313t-win_amd64.whl", hash = "sha256:d405d14bea042f166512add3091c1af40437c2e7f86988f3915fabd27b1e9cd2", size = 1995866, upload-time = "2025-10-14T10:21:28.951Z" }, - { url = "https://files.pythonhosted.org/packages/fc/de/b20f4ab954d6d399499c33ec4fafc46d9551e11dc1858fb7f5dca0748ceb/pydantic_core-2.41.4-cp313-cp313t-win_arm64.whl", hash = "sha256:19f3684868309db5263a11bace3c45d93f6f24afa2ffe75a647583df22a2ff89", size = 1970034, upload-time = "2025-10-14T10:21:30.869Z" }, - { url = "https://files.pythonhosted.org/packages/54/28/d3325da57d413b9819365546eb9a6e8b7cbd9373d9380efd5f74326143e6/pydantic_core-2.41.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e9205d97ed08a82ebb9a307e92914bb30e18cdf6f6b12ca4bedadb1588a0bfe1", size = 2102022, upload-time = "2025-10-14T10:21:32.809Z" }, - { url = "https://files.pythonhosted.org/packages/9e/24/b58a1bc0d834bf1acc4361e61233ee217169a42efbdc15a60296e13ce438/pydantic_core-2.41.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:82df1f432b37d832709fbcc0e24394bba04a01b6ecf1ee87578145c19cde12ac", size = 1905495, upload-time = "2025-10-14T10:21:34.812Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a4/71f759cc41b7043e8ecdaab81b985a9b6cad7cec077e0b92cff8b71ecf6b/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3b4cc4539e055cfa39a3763c939f9d409eb40e85813257dcd761985a108554", size = 1956131, upload-time = "2025-10-14T10:21:36.924Z" }, - { url = "https://files.pythonhosted.org/packages/b0/64/1e79ac7aa51f1eec7c4cda8cbe456d5d09f05fdd68b32776d72168d54275/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b1eb1754fce47c63d2ff57fdb88c351a6c0150995890088b33767a10218eaa4e", size = 2052236, upload-time = "2025-10-14T10:21:38.927Z" }, - { url = "https://files.pythonhosted.org/packages/e9/e3/a3ffc363bd4287b80f1d43dc1c28ba64831f8dfc237d6fec8f2661138d48/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6ab5ab30ef325b443f379ddb575a34969c333004fca5a1daa0133a6ffaad616", size = 2223573, upload-time = "2025-10-14T10:21:41.574Z" }, - { url = "https://files.pythonhosted.org/packages/28/27/78814089b4d2e684a9088ede3790763c64693c3d1408ddc0a248bc789126/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:31a41030b1d9ca497634092b46481b937ff9397a86f9f51bd41c4767b6fc04af", size = 2342467, upload-time = "2025-10-14T10:21:44.018Z" }, - { url = "https://files.pythonhosted.org/packages/92/97/4de0e2a1159cb85ad737e03306717637842c88c7fd6d97973172fb183149/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a44ac1738591472c3d020f61c6df1e4015180d6262ebd39bf2aeb52571b60f12", size = 2063754, upload-time = "2025-10-14T10:21:46.466Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/8cb90ce4b9efcf7ae78130afeb99fd1c86125ccdf9906ef64b9d42f37c25/pydantic_core-2.41.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d72f2b5e6e82ab8f94ea7d0d42f83c487dc159c5240d8f83beae684472864e2d", size = 2196754, upload-time = "2025-10-14T10:21:48.486Z" }, - { url = "https://files.pythonhosted.org/packages/34/3b/ccdc77af9cd5082723574a1cc1bcae7a6acacc829d7c0a06201f7886a109/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c4d1e854aaf044487d31143f541f7aafe7b482ae72a022c664b2de2e466ed0ad", size = 2137115, upload-time = "2025-10-14T10:21:50.63Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ba/e7c7a02651a8f7c52dc2cff2b64a30c313e3b57c7d93703cecea76c09b71/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b568af94267729d76e6ee5ececda4e283d07bbb28e8148bb17adad93d025d25a", size = 2317400, upload-time = "2025-10-14T10:21:52.959Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ba/6c533a4ee8aec6b812c643c49bb3bd88d3f01e3cebe451bb85512d37f00f/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6d55fb8b1e8929b341cc313a81a26e0d48aa3b519c1dbaadec3a6a2b4fcad025", size = 2312070, upload-time = "2025-10-14T10:21:55.419Z" }, - { url = "https://files.pythonhosted.org/packages/22/ae/f10524fcc0ab8d7f96cf9a74c880243576fd3e72bd8ce4f81e43d22bcab7/pydantic_core-2.41.4-cp314-cp314-win32.whl", hash = "sha256:5b66584e549e2e32a1398df11da2e0a7eff45d5c2d9db9d5667c5e6ac764d77e", size = 1982277, upload-time = "2025-10-14T10:21:57.474Z" }, - { url = "https://files.pythonhosted.org/packages/b4/dc/e5aa27aea1ad4638f0c3fb41132f7eb583bd7420ee63204e2d4333a3bbf9/pydantic_core-2.41.4-cp314-cp314-win_amd64.whl", hash = "sha256:557a0aab88664cc552285316809cab897716a372afaf8efdbef756f8b890e894", size = 2024608, upload-time = "2025-10-14T10:21:59.557Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/51d89cc2612bd147198e120a13f150afbf0bcb4615cddb049ab10b81b79e/pydantic_core-2.41.4-cp314-cp314-win_arm64.whl", hash = "sha256:3f1ea6f48a045745d0d9f325989d8abd3f1eaf47dd00485912d1a3a63c623a8d", size = 1967614, upload-time = "2025-10-14T10:22:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c2/472f2e31b95eff099961fa050c376ab7156a81da194f9edb9f710f68787b/pydantic_core-2.41.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6c1fe4c5404c448b13188dd8bd2ebc2bdd7e6727fa61ff481bcc2cca894018da", size = 1876904, upload-time = "2025-10-14T10:22:04.062Z" }, - { url = "https://files.pythonhosted.org/packages/4a/07/ea8eeb91173807ecdae4f4a5f4b150a520085b35454350fc219ba79e66a3/pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:523e7da4d43b113bf8e7b49fa4ec0c35bf4fe66b2230bfc5c13cc498f12c6c3e", size = 1882538, upload-time = "2025-10-14T10:22:06.39Z" }, - { url = "https://files.pythonhosted.org/packages/1e/29/b53a9ca6cd366bfc928823679c6a76c7a4c69f8201c0ba7903ad18ebae2f/pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5729225de81fb65b70fdb1907fcf08c75d498f4a6f15af005aabb1fdadc19dfa", size = 2041183, upload-time = "2025-10-14T10:22:08.812Z" }, - { url = "https://files.pythonhosted.org/packages/c7/3d/f8c1a371ceebcaf94d6dd2d77c6cf4b1c078e13a5837aee83f760b4f7cfd/pydantic_core-2.41.4-cp314-cp314t-win_amd64.whl", hash = "sha256:de2cfbb09e88f0f795fd90cf955858fc2c691df65b1f21f0aa00b99f3fbc661d", size = 1993542, upload-time = "2025-10-14T10:22:11.332Z" }, - { url = "https://files.pythonhosted.org/packages/8a/ac/9fc61b4f9d079482a290afe8d206b8f490e9fd32d4fc03ed4fc698214e01/pydantic_core-2.41.4-cp314-cp314t-win_arm64.whl", hash = "sha256:d34f950ae05a83e0ede899c595f312ca976023ea1db100cd5aa188f7005e3ab0", size = 1973897, upload-time = "2025-10-14T10:22:13.444Z" }, - { url = "https://files.pythonhosted.org/packages/b0/12/5ba58daa7f453454464f92b3ca7b9d7c657d8641c48e370c3ebc9a82dd78/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:a1b2cfec3879afb742a7b0bcfa53e4f22ba96571c9e54d6a3afe1052d17d843b", size = 2122139, upload-time = "2025-10-14T10:22:47.288Z" }, - { url = "https://files.pythonhosted.org/packages/21/fb/6860126a77725c3108baecd10fd3d75fec25191d6381b6eb2ac660228eac/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:d175600d975b7c244af6eb9c9041f10059f20b8bbffec9e33fdd5ee3f67cdc42", size = 1936674, upload-time = "2025-10-14T10:22:49.555Z" }, - { url = "https://files.pythonhosted.org/packages/de/be/57dcaa3ed595d81f8757e2b44a38240ac5d37628bce25fb20d02c7018776/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f184d657fa4947ae5ec9c47bd7e917730fa1cbb78195037e32dcbab50aca5ee", size = 1956398, upload-time = "2025-10-14T10:22:52.19Z" }, - { url = "https://files.pythonhosted.org/packages/2f/1d/679a344fadb9695f1a6a294d739fbd21d71fa023286daeea8c0ed49e7c2b/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed810568aeffed3edc78910af32af911c835cc39ebbfacd1f0ab5dd53028e5c", size = 2138674, upload-time = "2025-10-14T10:22:54.499Z" }, - { url = "https://files.pythonhosted.org/packages/c4/48/ae937e5a831b7c0dc646b2ef788c27cd003894882415300ed21927c21efa/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:4f5d640aeebb438517150fdeec097739614421900e4a08db4a3ef38898798537", size = 2112087, upload-time = "2025-10-14T10:22:56.818Z" }, - { url = "https://files.pythonhosted.org/packages/5e/db/6db8073e3d32dae017da7e0d16a9ecb897d0a4d92e00634916e486097961/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:4a9ab037b71927babc6d9e7fc01aea9e66dc2a4a34dff06ef0724a4049629f94", size = 1920387, upload-time = "2025-10-14T10:22:59.342Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c1/dd3542d072fcc336030d66834872f0328727e3b8de289c662faa04aa270e/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4dab9484ec605c3016df9ad4fd4f9a390bc5d816a3b10c6550f8424bb80b18c", size = 1951495, upload-time = "2025-10-14T10:23:02.089Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c6/db8d13a1f8ab3f1eb08c88bd00fd62d44311e3456d1e85c0e59e0a0376e7/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8a5028425820731d8c6c098ab642d7b8b999758e24acae03ed38a66eca8335", size = 2139008, upload-time = "2025-10-14T10:23:04.539Z" }, - { url = "https://files.pythonhosted.org/packages/5d/d4/912e976a2dd0b49f31c98a060ca90b353f3b73ee3ea2fd0030412f6ac5ec/pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e5ab4fc177dd41536b3c32b2ea11380dd3d4619a385860621478ac2d25ceb00", size = 2106739, upload-time = "2025-10-14T10:23:06.934Z" }, - { url = "https://files.pythonhosted.org/packages/71/f0/66ec5a626c81eba326072d6ee2b127f8c139543f1bf609b4842978d37833/pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d88d0054d3fa11ce936184896bed3c1c5441d6fa483b498fac6a5d0dd6f64a9", size = 1932549, upload-time = "2025-10-14T10:23:09.24Z" }, - { url = "https://files.pythonhosted.org/packages/c4/af/625626278ca801ea0a658c2dcf290dc9f21bb383098e99e7c6a029fccfc0/pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b2a054a8725f05b4b6503357e0ac1c4e8234ad3b0c2ac130d6ffc66f0e170e2", size = 2135093, upload-time = "2025-10-14T10:23:11.626Z" }, - { url = "https://files.pythonhosted.org/packages/20/f6/2fba049f54e0f4975fef66be654c597a1d005320fa141863699180c7697d/pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0d9db5a161c99375a0c68c058e227bee1d89303300802601d76a3d01f74e258", size = 2187971, upload-time = "2025-10-14T10:23:14.437Z" }, - { url = "https://files.pythonhosted.org/packages/0e/80/65ab839a2dfcd3b949202f9d920c34f9de5a537c3646662bdf2f7d999680/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6273ea2c8ffdac7b7fda2653c49682db815aebf4a89243a6feccf5e36c18c347", size = 2147939, upload-time = "2025-10-14T10:23:16.831Z" }, - { url = "https://files.pythonhosted.org/packages/44/58/627565d3d182ce6dfda18b8e1c841eede3629d59c9d7cbc1e12a03aeb328/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:4c973add636efc61de22530b2ef83a65f39b6d6f656df97f678720e20de26caa", size = 2311400, upload-time = "2025-10-14T10:23:19.234Z" }, - { url = "https://files.pythonhosted.org/packages/24/06/8a84711162ad5a5f19a88cead37cca81b4b1f294f46260ef7334ae4f24d3/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b69d1973354758007f46cf2d44a4f3d0933f10b6dc9bf15cf1356e037f6f731a", size = 2316840, upload-time = "2025-10-14T10:23:21.738Z" }, - { url = "https://files.pythonhosted.org/packages/aa/8b/b7bb512a4682a2f7fbfae152a755d37351743900226d29bd953aaf870eaa/pydantic_core-2.41.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3619320641fd212aaf5997b6ca505e97540b7e16418f4a241f44cdf108ffb50d", size = 2149135, upload-time = "2025-10-14T10:23:24.379Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7d/138e902ed6399b866f7cfe4435d22445e16fff888a1c00560d9dc79a780f/pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:491535d45cd7ad7e4a2af4a5169b0d07bebf1adfd164b0368da8aa41e19907a5", size = 2104721, upload-time = "2025-10-14T10:23:26.906Z" }, - { url = "https://files.pythonhosted.org/packages/47/13/0525623cf94627f7b53b4c2034c81edc8491cbfc7c28d5447fa318791479/pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:54d86c0cada6aba4ec4c047d0e348cbad7063b87ae0f005d9f8c9ad04d4a92a2", size = 1931608, upload-time = "2025-10-14T10:23:29.306Z" }, - { url = "https://files.pythonhosted.org/packages/d6/f9/744bc98137d6ef0a233f808bfc9b18cf94624bf30836a18d3b05d08bf418/pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eca1124aced216b2500dc2609eade086d718e8249cb9696660ab447d50a758bd", size = 2132986, upload-time = "2025-10-14T10:23:32.057Z" }, - { url = "https://files.pythonhosted.org/packages/17/c8/629e88920171173f6049386cc71f893dff03209a9ef32b4d2f7e7c264bcf/pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c9024169becccf0cb470ada03ee578d7348c119a0d42af3dcf9eda96e3a247c", size = 2187516, upload-time = "2025-10-14T10:23:34.871Z" }, - { url = "https://files.pythonhosted.org/packages/2e/0f/4f2734688d98488782218ca61bcc118329bf5de05bb7fe3adc7dd79b0b86/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:26895a4268ae5a2849269f4991cdc97236e4b9c010e51137becf25182daac405", size = 2146146, upload-time = "2025-10-14T10:23:37.342Z" }, - { url = "https://files.pythonhosted.org/packages/ed/f2/ab385dbd94a052c62224b99cf99002eee99dbec40e10006c78575aead256/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:ca4df25762cf71308c446e33c9b1fdca2923a3f13de616e2a949f38bf21ff5a8", size = 2311296, upload-time = "2025-10-14T10:23:40.145Z" }, - { url = "https://files.pythonhosted.org/packages/fc/8e/e4f12afe1beeb9823bba5375f8f258df0cc61b056b0195fb1cf9f62a1a58/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5a28fcedd762349519276c36634e71853b4541079cab4acaaac60c4421827308", size = 2315386, upload-time = "2025-10-14T10:23:42.624Z" }, - { url = "https://files.pythonhosted.org/packages/48/f7/925f65d930802e3ea2eb4d5afa4cb8730c8dc0d2cb89a59dc4ed2fcb2d74/pydantic_core-2.41.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c173ddcd86afd2535e2b695217e82191580663a1d1928239f877f5a1649ef39f", size = 2147775, upload-time = "2025-10-14T10:23:45.406Z" }, + { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" }, + { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" }, + { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" }, + { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" }, + { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" }, + { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" }, + { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" }, + { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" }, + { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" }, + { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" }, + { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" }, + { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" }, + { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] [[package]] @@ -393,14 +442,14 @@ wheels = [ [[package]] name = "python-levenshtein" -version = "0.27.1" +version = "0.27.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "levenshtein" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/f6/d865a565b7eeef4b5f9a18accafb03d5730c712420fc84a3a40555f7ea6b/python_levenshtein-0.27.1.tar.gz", hash = "sha256:3a5314a011016d373d309a68e875fd029caaa692ad3f32e78319299648045f11", size = 12326, upload-time = "2025-03-02T19:47:25.641Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/b4/36eda4188dd19d3cb53d8a8749d7520bd23dfe1c1f44e56ea9dcd0232274/python_levenshtein-0.27.3.tar.gz", hash = "sha256:27dc2d65aeb62a7d6852388f197073296370779286c0860b087357f3b8129a62", size = 12446, upload-time = "2025-11-01T12:54:59.712Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/95/8c8fd923b0a702388da4f9e0368f490d123cc5224279e6a083984304a15e/python_levenshtein-0.27.1-py3-none-any.whl", hash = "sha256:e1a4bc2a70284b2ebc4c505646142fecd0f831e49aa04ed972995895aec57396", size = 9426, upload-time = "2025-03-02T19:47:24.801Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/26e3cca2589252ceabf964ba81514e6f48556553c9c2766e1a0fdceec696/python_levenshtein-0.27.3-py3-none-any.whl", hash = "sha256:5d6168a8e8befb25abf04d2952368a446722be10e8ced218d0dc4fd3703a43a1", size = 9504, upload-time = "2025-11-01T12:54:58.933Z" }, ] [[package]] @@ -469,101 +518,92 @@ wheels = [ [[package]] name = "rapidfuzz" -version = "3.14.1" +version = "3.14.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ed/fc/a98b616db9a42dcdda7c78c76bdfdf6fe290ac4c5ffbb186f73ec981ad5b/rapidfuzz-3.14.1.tar.gz", hash = "sha256:b02850e7f7152bd1edff27e9d584505b84968cacedee7a734ec4050c655a803c", size = 57869570, upload-time = "2025-09-08T21:08:15.922Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/28/9d808fe62375b9aab5ba92fa9b29371297b067c2790b2d7cda648b1e2f8d/rapidfuzz-3.14.3.tar.gz", hash = "sha256:2491937177868bc4b1e469087601d53f925e8d270ccc21e07404b4b5814b7b5f", size = 57863900, upload-time = "2025-11-01T11:54:52.321Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/b9/4e35178f405a1a95abd37cce4dc09d4a5bbc5e098687680b5ba796d3115b/rapidfuzz-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:489440e4b5eea0d150a31076eb183bed0ec84f934df206c72ae4fc3424501758", size = 1939645, upload-time = "2025-09-08T21:05:16.569Z" }, - { url = "https://files.pythonhosted.org/packages/51/af/fd7b8662a3b6952559af322dcf1c9d4eb5ec6be2697c30ae8ed3c44876ca/rapidfuzz-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eff22cc938c3f74d194df03790a6c3325d213b28cf65cdefd6fdeae759b745d5", size = 1393620, upload-time = "2025-09-08T21:05:18.598Z" }, - { url = "https://files.pythonhosted.org/packages/c5/5b/5715445e29c1c6ba364b3d27278da3fdffb18d9147982e977c6638dcecbf/rapidfuzz-3.14.1-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0307f018b16feaa36074bcec2496f6f120af151a098910296e72e233232a62f", size = 1387721, upload-time = "2025-09-08T21:05:20.408Z" }, - { url = "https://files.pythonhosted.org/packages/19/49/83a14a6a90982b090257c4b2e96b9b9c423a89012b8504d5a14d92a4f8c2/rapidfuzz-3.14.1-cp310-cp310-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bc133652da143aca1ab72de235446432888b2b7f44ee332d006f8207967ecb8a", size = 1694545, upload-time = "2025-09-08T21:05:22.137Z" }, - { url = "https://files.pythonhosted.org/packages/99/f7/94618fcaaac8c04abf364f405c6811a02bc9edef209f276dc513a9a50f7c/rapidfuzz-3.14.1-cp310-cp310-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e9e71b3fe7e4a1590843389a90fe2a8684649fc74b9b7446e17ee504ddddb7de", size = 2237075, upload-time = "2025-09-08T21:05:23.637Z" }, - { url = "https://files.pythonhosted.org/packages/58/f6/a5ee2db25f36b0e5e06502fb77449b7718cd9f92ad36d598e669ba91db7b/rapidfuzz-3.14.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c51519eb2f20b52eba6fc7d857ae94acc6c2a1f5d0f2d794b9d4977cdc29dd7", size = 3168778, upload-time = "2025-09-08T21:05:25.508Z" }, - { url = "https://files.pythonhosted.org/packages/0f/e8/c9620e358805c099e6755b7d2827b1e711b5e61914d6112ce2faa2c2af79/rapidfuzz-3.14.1-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:fe87d94602624f8f25fff9a0a7b47f33756c4d9fc32b6d3308bb142aa483b8a4", size = 1223827, upload-time = "2025-09-08T21:05:27.299Z" }, - { url = "https://files.pythonhosted.org/packages/84/08/24916c3c3d55d6236474c9da0a595641d0013d3604de0625e8a8974371c3/rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2d665380503a575dda52eb712ea521f789e8f8fd629c7a8e6c0f8ff480febc78", size = 2408366, upload-time = "2025-09-08T21:05:28.808Z" }, - { url = "https://files.pythonhosted.org/packages/40/d4/4152e8821b5c548443a6c46568fccef13de5818a5ab370d553ea3d5955b3/rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0f0dd022b8a7cbf3c891f6de96a80ab6a426f1069a085327816cea749e096c2", size = 2530148, upload-time = "2025-09-08T21:05:30.782Z" }, - { url = "https://files.pythonhosted.org/packages/bd/af/6587c6d590abe232c530ad43fbfbcaec899bff7204e237f1fd21e2e44b81/rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf1ba22d36858b265c95cd774ba7fe8991e80a99cd86fe4f388605b01aee81a3", size = 2810628, upload-time = "2025-09-08T21:05:32.844Z" }, - { url = "https://files.pythonhosted.org/packages/d7/90/a99e6cfd90feb9d770654f1f39321099bbbf7f85d2832f2ef48d3f4ebc5f/rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:ca1c1494ac9f9386d37f0e50cbaf4d07d184903aed7691549df1b37e9616edc9", size = 3314406, upload-time = "2025-09-08T21:05:34.585Z" }, - { url = "https://files.pythonhosted.org/packages/5f/b3/eba5a6c217200fd1d3615997930a9e5db6a74e3002b7867b54545f9b5cbb/rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9e4b12e921b0fa90d7c2248742a536f21eae5562174090b83edd0b4ab8b557d7", size = 4280030, upload-time = "2025-09-08T21:05:36.646Z" }, - { url = "https://files.pythonhosted.org/packages/04/6f/d2e060a2094cfb7f3cd487c376e098abb22601e0eea178e51a59ce0a3158/rapidfuzz-3.14.1-cp310-cp310-win32.whl", hash = "sha256:5e1c1f2292baa4049535b07e9e81feb29e3650d2ba35ee491e64aca7ae4cb15e", size = 1727070, upload-time = "2025-09-08T21:05:38.57Z" }, - { url = "https://files.pythonhosted.org/packages/73/0a/ca231464ec689f2aabf9547a52cbc76a10affe960bddde8660699ba3de33/rapidfuzz-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:59a8694beb9a13c4090ab3d1712cabbd896c6949706d1364e2a2e1713c413760", size = 1545335, upload-time = "2025-09-08T21:05:40.22Z" }, - { url = "https://files.pythonhosted.org/packages/59/c5/1e0b17f20fd3d701470548a6db8f36d589fb1a8a65d3828968547d987486/rapidfuzz-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:e94cee93faa792572c574a615abe12912124b4ffcf55876b72312914ab663345", size = 816960, upload-time = "2025-09-08T21:05:42.225Z" }, - { url = "https://files.pythonhosted.org/packages/5c/c7/c3c860d512606225c11c8ee455b4dc0b0214dbcfac90a2c22dddf55320f3/rapidfuzz-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4d976701060886a791c8a9260b1d4139d14c1f1e9a6ab6116b45a1acf3baff67", size = 1938398, upload-time = "2025-09-08T21:05:44.031Z" }, - { url = "https://files.pythonhosted.org/packages/c0/f3/67f5c5cd4d728993c48c1dcb5da54338d77c03c34b4903cc7839a3b89faf/rapidfuzz-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e6ba7e6eb2ab03870dcab441d707513db0b4264c12fba7b703e90e8b4296df2", size = 1392819, upload-time = "2025-09-08T21:05:45.549Z" }, - { url = "https://files.pythonhosted.org/packages/d5/06/400d44842f4603ce1bebeaeabe776f510e329e7dbf6c71b6f2805e377889/rapidfuzz-3.14.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e532bf46de5fd3a1efde73a16a4d231d011bce401c72abe3c6ecf9de681003f", size = 1391798, upload-time = "2025-09-08T21:05:47.044Z" }, - { url = "https://files.pythonhosted.org/packages/90/97/a6944955713b47d88e8ca4305ca7484940d808c4e6c4e28b6fa0fcbff97e/rapidfuzz-3.14.1-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f9b6a6fb8ed9b951e5f3b82c1ce6b1665308ec1a0da87f799b16e24fc59e4662", size = 1699136, upload-time = "2025-09-08T21:05:48.919Z" }, - { url = "https://files.pythonhosted.org/packages/a8/1e/f311a5c95ddf922db6dd8666efeceb9ac69e1319ed098ac80068a4041732/rapidfuzz-3.14.1-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b6ac3f9810949caef0e63380b11a3c32a92f26bacb9ced5e32c33560fcdf8d1", size = 2236238, upload-time = "2025-09-08T21:05:50.844Z" }, - { url = "https://files.pythonhosted.org/packages/85/27/e14e9830255db8a99200f7111b158ddef04372cf6332a415d053fe57cc9c/rapidfuzz-3.14.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e52e4c34fd567f77513e886b66029c1ae02f094380d10eba18ba1c68a46d8b90", size = 3183685, upload-time = "2025-09-08T21:05:52.362Z" }, - { url = "https://files.pythonhosted.org/packages/61/b2/42850c9616ddd2887904e5dd5377912cbabe2776fdc9fd4b25e6e12fba32/rapidfuzz-3.14.1-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:2ef72e41b1a110149f25b14637f1cedea6df192462120bea3433980fe9d8ac05", size = 1231523, upload-time = "2025-09-08T21:05:53.927Z" }, - { url = "https://files.pythonhosted.org/packages/de/b5/6b90ed7127a1732efef39db46dd0afc911f979f215b371c325a2eca9cb15/rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fb654a35b373d712a6b0aa2a496b2b5cdd9d32410cfbaecc402d7424a90ba72a", size = 2415209, upload-time = "2025-09-08T21:05:55.422Z" }, - { url = "https://files.pythonhosted.org/packages/70/60/af51c50d238c82f2179edc4b9f799cc5a50c2c0ebebdcfaa97ded7d02978/rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2b2c12e5b9eb8fe9a51b92fe69e9ca362c0970e960268188a6d295e1dec91e6d", size = 2532957, upload-time = "2025-09-08T21:05:57.048Z" }, - { url = "https://files.pythonhosted.org/packages/50/92/29811d2ba7c984251a342c4f9ccc7cc4aa09d43d800af71510cd51c36453/rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4f069dec5c450bd987481e752f0a9979e8fdf8e21e5307f5058f5c4bb162fa56", size = 2815720, upload-time = "2025-09-08T21:05:58.618Z" }, - { url = "https://files.pythonhosted.org/packages/78/69/cedcdee16a49e49d4985eab73b59447f211736c5953a58f1b91b6c53a73f/rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4d0d9163725b7ad37a8c46988cae9ebab255984db95ad01bf1987ceb9e3058dd", size = 3323704, upload-time = "2025-09-08T21:06:00.576Z" }, - { url = "https://files.pythonhosted.org/packages/76/3e/5a3f9a5540f18e0126e36f86ecf600145344acb202d94b63ee45211a18b8/rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db656884b20b213d846f6bc990c053d1f4a60e6d4357f7211775b02092784ca1", size = 4287341, upload-time = "2025-09-08T21:06:02.301Z" }, - { url = "https://files.pythonhosted.org/packages/46/26/45db59195929dde5832852c9de8533b2ac97dcc0d852d1f18aca33828122/rapidfuzz-3.14.1-cp311-cp311-win32.whl", hash = "sha256:4b42f7b9c58cbcfbfaddc5a6278b4ca3b6cd8983e7fd6af70ca791dff7105fb9", size = 1726574, upload-time = "2025-09-08T21:06:04.357Z" }, - { url = "https://files.pythonhosted.org/packages/01/5c/a4caf76535f35fceab25b2aaaed0baecf15b3d1fd40746f71985d20f8c4b/rapidfuzz-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:e5847f30d7d4edefe0cb37294d956d3495dd127c1c56e9128af3c2258a520bb4", size = 1547124, upload-time = "2025-09-08T21:06:06.002Z" }, - { url = "https://files.pythonhosted.org/packages/c6/66/aa93b52f95a314584d71fa0b76df00bdd4158aafffa76a350f1ae416396c/rapidfuzz-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:5087d8ad453092d80c042a08919b1cb20c8ad6047d772dc9312acd834da00f75", size = 816958, upload-time = "2025-09-08T21:06:07.509Z" }, - { url = "https://files.pythonhosted.org/packages/df/77/2f4887c9b786f203e50b816c1cde71f96642f194e6fa752acfa042cf53fd/rapidfuzz-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:809515194f628004aac1b1b280c3734c5ea0ccbd45938c9c9656a23ae8b8f553", size = 1932216, upload-time = "2025-09-08T21:06:09.342Z" }, - { url = "https://files.pythonhosted.org/packages/de/bd/b5e445d156cb1c2a87d36d8da53daf4d2a1d1729b4851660017898b49aa0/rapidfuzz-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0afcf2d6cb633d0d4260d8df6a40de2d9c93e9546e2c6b317ab03f89aa120ad7", size = 1393414, upload-time = "2025-09-08T21:06:10.959Z" }, - { url = "https://files.pythonhosted.org/packages/de/bd/98d065dd0a4479a635df855616980eaae1a1a07a876db9400d421b5b6371/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c1c3d07d53dcafee10599da8988d2b1f39df236aee501ecbd617bd883454fcd", size = 1377194, upload-time = "2025-09-08T21:06:12.471Z" }, - { url = "https://files.pythonhosted.org/packages/d3/8a/1265547b771128b686f3c431377ff1db2fa073397ed082a25998a7b06d4e/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e9ee3e1eb0a027717ee72fe34dc9ac5b3e58119f1bd8dd15bc19ed54ae3e62b", size = 1669573, upload-time = "2025-09-08T21:06:14.016Z" }, - { url = "https://files.pythonhosted.org/packages/a8/57/e73755c52fb451f2054196404ccc468577f8da023b3a48c80bce29ee5d4a/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:70c845b64a033a20c44ed26bc890eeb851215148cc3e696499f5f65529afb6cb", size = 2217833, upload-time = "2025-09-08T21:06:15.666Z" }, - { url = "https://files.pythonhosted.org/packages/20/14/7399c18c460e72d1b754e80dafc9f65cb42a46cc8f29cd57d11c0c4acc94/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26db0e815213d04234298dea0d884d92b9cb8d4ba954cab7cf67a35853128a33", size = 3159012, upload-time = "2025-09-08T21:06:17.631Z" }, - { url = "https://files.pythonhosted.org/packages/f8/5e/24f0226ddb5440cabd88605d2491f99ae3748a6b27b0bc9703772892ced7/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:6ad3395a416f8b126ff11c788531f157c7debeb626f9d897c153ff8980da10fb", size = 1227032, upload-time = "2025-09-08T21:06:21.06Z" }, - { url = "https://files.pythonhosted.org/packages/40/43/1d54a4ad1a5fac2394d5f28a3108e2bf73c26f4f23663535e3139cfede9b/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:61c5b9ab6f730e6478aa2def566223712d121c6f69a94c7cc002044799442afd", size = 2395054, upload-time = "2025-09-08T21:06:23.482Z" }, - { url = "https://files.pythonhosted.org/packages/0c/71/e9864cd5b0f086c4a03791f5dfe0155a1b132f789fe19b0c76fbabd20513/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:13e0ea3d0c533969158727d1bb7a08c2cc9a816ab83f8f0dcfde7e38938ce3e6", size = 2524741, upload-time = "2025-09-08T21:06:26.825Z" }, - { url = "https://files.pythonhosted.org/packages/b2/0c/53f88286b912faf4a3b2619a60df4f4a67bd0edcf5970d7b0c1143501f0c/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6325ca435b99f4001aac919ab8922ac464999b100173317defb83eae34e82139", size = 2785311, upload-time = "2025-09-08T21:06:29.471Z" }, - { url = "https://files.pythonhosted.org/packages/53/9a/229c26dc4f91bad323f07304ee5ccbc28f0d21c76047a1e4f813187d0bad/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:07a9fad3247e68798424bdc116c1094e88ecfabc17b29edf42a777520347648e", size = 3303630, upload-time = "2025-09-08T21:06:31.094Z" }, - { url = "https://files.pythonhosted.org/packages/05/de/20e330d6d58cbf83da914accd9e303048b7abae2f198886f65a344b69695/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8ff5dbe78db0a10c1f916368e21d328935896240f71f721e073cf6c4c8cdedd", size = 4262364, upload-time = "2025-09-08T21:06:32.877Z" }, - { url = "https://files.pythonhosted.org/packages/1f/10/2327f83fad3534a8d69fe9cd718f645ec1fe828b60c0e0e97efc03bf12f8/rapidfuzz-3.14.1-cp312-cp312-win32.whl", hash = "sha256:9c83270e44a6ae7a39fc1d7e72a27486bccc1fa5f34e01572b1b90b019e6b566", size = 1711927, upload-time = "2025-09-08T21:06:34.669Z" }, - { url = "https://files.pythonhosted.org/packages/78/8d/199df0370133fe9f35bc72f3c037b53c93c5c1fc1e8d915cf7c1f6bb8557/rapidfuzz-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:e06664c7fdb51c708e082df08a6888fce4c5c416d7e3cc2fa66dd80eb76a149d", size = 1542045, upload-time = "2025-09-08T21:06:36.364Z" }, - { url = "https://files.pythonhosted.org/packages/b3/c6/cc5d4bd1b16ea2657c80b745d8b1c788041a31fad52e7681496197b41562/rapidfuzz-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:6c7c26025f7934a169a23dafea6807cfc3fb556f1dd49229faf2171e5d8101cc", size = 813170, upload-time = "2025-09-08T21:06:38.001Z" }, - { url = "https://files.pythonhosted.org/packages/0d/f2/0024cc8eead108c4c29337abe133d72ddf3406ce9bbfbcfc110414a7ea07/rapidfuzz-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8d69f470d63ee824132ecd80b1974e1d15dd9df5193916901d7860cef081a260", size = 1926515, upload-time = "2025-09-08T21:06:39.834Z" }, - { url = "https://files.pythonhosted.org/packages/12/ae/6cb211f8930bea20fa989b23f31ee7f92940caaf24e3e510d242a1b28de4/rapidfuzz-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6f571d20152fc4833b7b5e781b36d5e4f31f3b5a596a3d53cf66a1bd4436b4f4", size = 1388431, upload-time = "2025-09-08T21:06:41.73Z" }, - { url = "https://files.pythonhosted.org/packages/39/88/bfec24da0607c39e5841ced5594ea1b907d20f83adf0e3ee87fa454a425b/rapidfuzz-3.14.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61d77e09b2b6bc38228f53b9ea7972a00722a14a6048be9a3672fb5cb08bad3a", size = 1375664, upload-time = "2025-09-08T21:06:43.737Z" }, - { url = "https://files.pythonhosted.org/packages/f4/43/9f282ba539e404bdd7052c7371d3aaaa1a9417979d2a1d8332670c7f385a/rapidfuzz-3.14.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b41d95ef86a6295d353dc3bb6c80550665ba2c3bef3a9feab46074d12a9af8f", size = 1668113, upload-time = "2025-09-08T21:06:45.758Z" }, - { url = "https://files.pythonhosted.org/packages/7f/2f/0b3153053b1acca90969eb0867922ac8515b1a8a48706a3215c2db60e87c/rapidfuzz-3.14.1-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0591df2e856ad583644b40a2b99fb522f93543c65e64b771241dda6d1cfdc96b", size = 2212875, upload-time = "2025-09-08T21:06:47.447Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/623001dddc518afaa08ed1fbbfc4005c8692b7a32b0f08b20c506f17a770/rapidfuzz-3.14.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f277801f55b2f3923ef2de51ab94689a0671a4524bf7b611de979f308a54cd6f", size = 3161181, upload-time = "2025-09-08T21:06:49.179Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b7/d8404ed5ad56eb74463e5ebf0a14f0019d7eb0e65e0323f709fe72e0884c/rapidfuzz-3.14.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:893fdfd4f66ebb67f33da89eb1bd1674b7b30442fdee84db87f6cb9074bf0ce9", size = 1225495, upload-time = "2025-09-08T21:06:51.056Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6c/b96af62bc7615d821e3f6b47563c265fd7379d7236dfbc1cbbcce8beb1d2/rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fe2651258c1f1afa9b66f44bf82f639d5f83034f9804877a1bbbae2120539ad1", size = 2396294, upload-time = "2025-09-08T21:06:53.063Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b7/c60c9d22a7debed8b8b751f506a4cece5c22c0b05e47a819d6b47bc8c14e/rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:ace21f7a78519d8e889b1240489cd021c5355c496cb151b479b741a4c27f0a25", size = 2529629, upload-time = "2025-09-08T21:06:55.188Z" }, - { url = "https://files.pythonhosted.org/packages/25/94/a9ec7ccb28381f14de696ffd51c321974762f137679df986f5375d35264f/rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:cb5acf24590bc5e57027283b015950d713f9e4d155fda5cfa71adef3b3a84502", size = 2782960, upload-time = "2025-09-08T21:06:57.339Z" }, - { url = "https://files.pythonhosted.org/packages/68/80/04e5276d223060eca45250dbf79ea39940c0be8b3083661d58d57572c2c5/rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:67ea46fa8cc78174bad09d66b9a4b98d3068e85de677e3c71ed931a1de28171f", size = 3298427, upload-time = "2025-09-08T21:06:59.319Z" }, - { url = "https://files.pythonhosted.org/packages/4a/63/24759b2a751562630b244e68ccaaf7a7525c720588fcc77c964146355aee/rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:44e741d785de57d1a7bae03599c1cbc7335d0b060a35e60c44c382566e22782e", size = 4267736, upload-time = "2025-09-08T21:07:01.31Z" }, - { url = "https://files.pythonhosted.org/packages/18/a4/73f1b1f7f44d55f40ffbffe85e529eb9d7e7f7b2ffc0931760eadd163995/rapidfuzz-3.14.1-cp313-cp313-win32.whl", hash = "sha256:b1fe6001baa9fa36bcb565e24e88830718f6c90896b91ceffcb48881e3adddbc", size = 1710515, upload-time = "2025-09-08T21:07:03.16Z" }, - { url = "https://files.pythonhosted.org/packages/6a/8b/a8fe5a6ee4d06fd413aaa9a7e0a23a8630c4b18501509d053646d18c2aa7/rapidfuzz-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:83b8cc6336709fa5db0579189bfd125df280a554af544b2dc1c7da9cdad7e44d", size = 1540081, upload-time = "2025-09-08T21:07:05.401Z" }, - { url = "https://files.pythonhosted.org/packages/ac/fe/4b0ac16c118a2367d85450b45251ee5362661e9118a1cef88aae1765ffff/rapidfuzz-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:cf75769662eadf5f9bd24e865c19e5ca7718e879273dce4e7b3b5824c4da0eb4", size = 812725, upload-time = "2025-09-08T21:07:07.148Z" }, - { url = "https://files.pythonhosted.org/packages/e2/cb/1ad9a76d974d153783f8e0be8dbe60ec46488fac6e519db804e299e0da06/rapidfuzz-3.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d937dbeda71c921ef6537c6d41a84f1b8112f107589c9977059de57a1d726dd6", size = 1945173, upload-time = "2025-09-08T21:07:08.893Z" }, - { url = "https://files.pythonhosted.org/packages/d9/61/959ed7460941d8a81cbf6552b9c45564778a36cf5e5aa872558b30fc02b2/rapidfuzz-3.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7a2d80cc1a4fcc7e259ed4f505e70b36433a63fa251f1bb69ff279fe376c5efd", size = 1413949, upload-time = "2025-09-08T21:07:11.033Z" }, - { url = "https://files.pythonhosted.org/packages/7b/a0/f46fca44457ca1f25f23cc1f06867454fc3c3be118cd10b552b0ab3e58a2/rapidfuzz-3.14.1-cp313-cp313t-win32.whl", hash = "sha256:40875e0c06f1a388f1cab3885744f847b557e0b1642dfc31ff02039f9f0823ef", size = 1760666, upload-time = "2025-09-08T21:07:12.884Z" }, - { url = "https://files.pythonhosted.org/packages/9b/d0/7a5d9c04446f8b66882b0fae45b36a838cf4d31439b5d1ab48a9d17c8e57/rapidfuzz-3.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:876dc0c15552f3d704d7fb8d61bdffc872ff63bedf683568d6faad32e51bbce8", size = 1579760, upload-time = "2025-09-08T21:07:14.718Z" }, - { url = "https://files.pythonhosted.org/packages/4e/aa/2c03ae112320d0746f2c869cae68c413f3fe3b6403358556f2b747559723/rapidfuzz-3.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:61458e83b0b3e2abc3391d0953c47d6325e506ba44d6a25c869c4401b3bc222c", size = 832088, upload-time = "2025-09-08T21:07:17.03Z" }, - { url = "https://files.pythonhosted.org/packages/d6/36/53debca45fbe693bd6181fb05b6a2fd561c87669edb82ec0d7c1961a43f0/rapidfuzz-3.14.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e84d9a844dc2e4d5c4cabd14c096374ead006583304333c14a6fbde51f612a44", size = 1926336, upload-time = "2025-09-08T21:07:18.809Z" }, - { url = "https://files.pythonhosted.org/packages/ae/32/b874f48609665fcfeaf16cbaeb2bbc210deef2b88e996c51cfc36c3eb7c3/rapidfuzz-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:40301b93b99350edcd02dbb22e37ca5f2a75d0db822e9b3c522da451a93d6f27", size = 1389653, upload-time = "2025-09-08T21:07:20.667Z" }, - { url = "https://files.pythonhosted.org/packages/97/25/f6c5a1ff4ec11edadacb270e70b8415f51fa2f0d5730c2c552b81651fbe3/rapidfuzz-3.14.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fedd5097a44808dddf341466866e5c57a18a19a336565b4ff50aa8f09eb528f6", size = 1380911, upload-time = "2025-09-08T21:07:22.584Z" }, - { url = "https://files.pythonhosted.org/packages/d8/f3/d322202ef8fab463759b51ebfaa33228100510c82e6153bd7a922e150270/rapidfuzz-3.14.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e3e61c9e80d8c26709d8aa5c51fdd25139c81a4ab463895f8a567f8347b0548", size = 1673515, upload-time = "2025-09-08T21:07:24.417Z" }, - { url = "https://files.pythonhosted.org/packages/8d/b9/6b2a97f4c6be96cac3749f32301b8cdf751ce5617b1c8934c96586a0662b/rapidfuzz-3.14.1-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da011a373722fac6e64687297a1d17dc8461b82cb12c437845d5a5b161bc24b9", size = 2219394, upload-time = "2025-09-08T21:07:26.402Z" }, - { url = "https://files.pythonhosted.org/packages/11/bf/afb76adffe4406e6250f14ce48e60a7eb05d4624945bd3c044cfda575fbc/rapidfuzz-3.14.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5967d571243cfb9ad3710e6e628ab68c421a237b76e24a67ac22ee0ff12784d6", size = 3163582, upload-time = "2025-09-08T21:07:28.878Z" }, - { url = "https://files.pythonhosted.org/packages/42/34/e6405227560f61e956cb4c5de653b0f874751c5ada658d3532d6c1df328e/rapidfuzz-3.14.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:474f416cbb9099676de54aa41944c154ba8d25033ee460f87bb23e54af6d01c9", size = 1221116, upload-time = "2025-09-08T21:07:30.8Z" }, - { url = "https://files.pythonhosted.org/packages/55/e6/5b757e2e18de384b11d1daf59608453f0baf5d5d8d1c43e1a964af4dc19a/rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae2d57464b59297f727c4e201ea99ec7b13935f1f056c753e8103da3f2fc2404", size = 2402670, upload-time = "2025-09-08T21:07:32.702Z" }, - { url = "https://files.pythonhosted.org/packages/43/c4/d753a415fe54531aa882e288db5ed77daaa72e05c1a39e1cbac00d23024f/rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:57047493a1f62f11354c7143c380b02f1b355c52733e6b03adb1cb0fe8fb8816", size = 2521659, upload-time = "2025-09-08T21:07:35.218Z" }, - { url = "https://files.pythonhosted.org/packages/cd/28/d4e7fe1515430db98f42deb794c7586a026d302fe70f0216b638d89cf10f/rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4acc20776f225ee37d69517a237c090b9fa7e0836a0b8bc58868e9168ba6ef6f", size = 2788552, upload-time = "2025-09-08T21:07:37.188Z" }, - { url = "https://files.pythonhosted.org/packages/4f/00/eab05473af7a2cafb4f3994bc6bf408126b8eec99a569aac6254ac757db4/rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4373f914ff524ee0146919dea96a40a8200ab157e5a15e777a74a769f73d8a4a", size = 3306261, upload-time = "2025-09-08T21:07:39.624Z" }, - { url = "https://files.pythonhosted.org/packages/d1/31/2feb8dfcfcff6508230cd2ccfdde7a8bf988c6fda142fe9ce5d3eb15704d/rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:37017b84953927807847016620d61251fe236bd4bcb25e27b6133d955bb9cafb", size = 4269522, upload-time = "2025-09-08T21:07:41.663Z" }, - { url = "https://files.pythonhosted.org/packages/a3/99/250538d73c8fbab60597c3d131a11ef2a634d38b44296ca11922794491ac/rapidfuzz-3.14.1-cp314-cp314-win32.whl", hash = "sha256:c8d1dd1146539e093b84d0805e8951475644af794ace81d957ca612e3eb31598", size = 1745018, upload-time = "2025-09-08T21:07:44.313Z" }, - { url = "https://files.pythonhosted.org/packages/c5/15/d50839d20ad0743aded25b08a98ffb872f4bfda4e310bac6c111fcf6ea1f/rapidfuzz-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:f51c7571295ea97387bac4f048d73cecce51222be78ed808263b45c79c40a440", size = 1587666, upload-time = "2025-09-08T21:07:46.917Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ff/d73fec989213fb6f0b6f15ee4bbdf2d88b0686197951a06b036111cd1c7d/rapidfuzz-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:01eab10ec90912d7d28b3f08f6c91adbaf93458a53f849ff70776ecd70dd7a7a", size = 835780, upload-time = "2025-09-08T21:07:49.256Z" }, - { url = "https://files.pythonhosted.org/packages/b7/e7/f0a242687143cebd33a1fb165226b73bd9496d47c5acfad93de820a18fa8/rapidfuzz-3.14.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:60879fcae2f7618403c4c746a9a3eec89327d73148fb6e89a933b78442ff0669", size = 1945182, upload-time = "2025-09-08T21:07:51.84Z" }, - { url = "https://files.pythonhosted.org/packages/96/29/ca8a3f8525e3d0e7ab49cb927b5fb4a54855f794c9ecd0a0b60a6c96a05f/rapidfuzz-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f94d61e44db3fc95a74006a394257af90fa6e826c900a501d749979ff495d702", size = 1413946, upload-time = "2025-09-08T21:07:53.702Z" }, - { url = "https://files.pythonhosted.org/packages/b5/ef/6fd10aa028db19c05b4ac7fe77f5613e4719377f630c709d89d7a538eea2/rapidfuzz-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:93b6294a3ffab32a9b5f9b5ca048fa0474998e7e8bb0f2d2b5e819c64cb71ec7", size = 1795851, upload-time = "2025-09-08T21:07:55.76Z" }, - { url = "https://files.pythonhosted.org/packages/e4/30/acd29ebd906a50f9e0f27d5f82a48cf5e8854637b21489bd81a2459985cf/rapidfuzz-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:6cb56b695421538fdbe2c0c85888b991d833b8637d2f2b41faa79cea7234c000", size = 1626748, upload-time = "2025-09-08T21:07:58.166Z" }, - { url = "https://files.pythonhosted.org/packages/c1/f4/dfc7b8c46b1044a47f7ca55deceb5965985cff3193906cb32913121e6652/rapidfuzz-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7cd312c380d3ce9d35c3ec9726b75eee9da50e8a38e89e229a03db2262d3d96b", size = 853771, upload-time = "2025-09-08T21:08:00.816Z" }, - { url = "https://files.pythonhosted.org/packages/6d/10/0ed838b296fdac08ecbaa3a220fb4f1d887ff41b0be44fe8eade45bb650e/rapidfuzz-3.14.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:673ce55a9be5b772dade911909e42382c0828b8a50ed7f9168763fa6b9f7054d", size = 1860246, upload-time = "2025-09-08T21:08:02.762Z" }, - { url = "https://files.pythonhosted.org/packages/a4/70/a08f4a86387dec97508ead51cc7a4b3130d4e62ac0eae938a6d8e1feff14/rapidfuzz-3.14.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:45c62ada1980ebf4c64c4253993cc8daa018c63163f91db63bb3af69cb74c2e3", size = 1336749, upload-time = "2025-09-08T21:08:04.783Z" }, - { url = "https://files.pythonhosted.org/packages/d4/39/c12f76f69184bcfb9977d6404b2c5dac7dd4d70ee6803e61556e539d0097/rapidfuzz-3.14.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4d51efb29c0df0d4f7f64f672a7624c2146527f0745e3572098d753676538800", size = 1512629, upload-time = "2025-09-08T21:08:06.697Z" }, - { url = "https://files.pythonhosted.org/packages/05/c7/1b17347e30f2b50dd976c54641aa12003569acb1bdaabf45a5cc6f471c58/rapidfuzz-3.14.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4a21ccdf1bd7d57a1009030527ba8fae1c74bf832d0a08f6b67de8f5c506c96f", size = 1862602, upload-time = "2025-09-08T21:08:09.088Z" }, - { url = "https://files.pythonhosted.org/packages/09/cf/95d0dacac77eda22499991bd5f304c77c5965fb27348019a48ec3fe4a3f6/rapidfuzz-3.14.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:589fb0af91d3aff318750539c832ea1100dbac2c842fde24e42261df443845f6", size = 1339548, upload-time = "2025-09-08T21:08:11.059Z" }, - { url = "https://files.pythonhosted.org/packages/b6/58/f515c44ba8c6fa5daa35134b94b99661ced852628c5505ead07b905c3fc7/rapidfuzz-3.14.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a4f18092db4825f2517d135445015b40033ed809a41754918a03ef062abe88a0", size = 1513859, upload-time = "2025-09-08T21:08:13.07Z" }, + { url = "https://files.pythonhosted.org/packages/69/d1/0efa42a602ed466d3ca1c462eed5d62015c3fd2a402199e2c4b87aa5aa25/rapidfuzz-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b9fcd4d751a4fffa17aed1dde41647923c72c74af02459ad1222e3b0022da3a1", size = 1952376, upload-time = "2025-11-01T11:52:29.175Z" }, + { url = "https://files.pythonhosted.org/packages/be/00/37a169bb28b23850a164e6624b1eb299e1ad73c9e7c218ee15744e68d628/rapidfuzz-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4ad73afb688b36864a8d9b7344a9cf6da186c471e5790cbf541a635ee0f457f2", size = 1390903, upload-time = "2025-11-01T11:52:31.239Z" }, + { url = "https://files.pythonhosted.org/packages/3c/91/b37207cbbdb6eaafac3da3f55ea85287b27745cb416e75e15769b7d8abe8/rapidfuzz-3.14.3-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5fb2d978a601820d2cfd111e2c221a9a7bfdf84b41a3ccbb96ceef29f2f1ac7", size = 1385655, upload-time = "2025-11-01T11:52:32.852Z" }, + { url = "https://files.pythonhosted.org/packages/f2/bb/ca53e518acf43430be61f23b9c5987bd1e01e74fcb7a9ee63e00f597aefb/rapidfuzz-3.14.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d83b8b712fa37e06d59f29a4b49e2e9e8635e908fbc21552fe4d1163db9d2a1", size = 3164708, upload-time = "2025-11-01T11:52:34.618Z" }, + { url = "https://files.pythonhosted.org/packages/df/e1/7667bf2db3e52adb13cb933dd4a6a2efc66045d26fa150fc0feb64c26d61/rapidfuzz-3.14.3-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:dc8c07801df5206b81ed6bd6c35cb520cf9b6c64b9b0d19d699f8633dc942897", size = 1221106, upload-time = "2025-11-01T11:52:36.069Z" }, + { url = "https://files.pythonhosted.org/packages/05/8a/84d9f2d46a2c8eb2ccae81747c4901fa10fe4010aade2d57ce7b4b8e02ec/rapidfuzz-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c71ce6d4231e5ef2e33caa952bfe671cb9fd42e2afb11952df9fad41d5c821f9", size = 2406048, upload-time = "2025-11-01T11:52:37.936Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a9/a0b7b7a1b81a020c034eb67c8e23b7e49f920004e295378de3046b0d99e1/rapidfuzz-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:0e38828d1381a0cceb8a4831212b2f673d46f5129a1897b0451c883eaf4a1747", size = 2527020, upload-time = "2025-11-01T11:52:39.657Z" }, + { url = "https://files.pythonhosted.org/packages/b4/bc/416df7d108b99b4942ba04dd4cf73c45c3aadb3ef003d95cad78b1d12eb9/rapidfuzz-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:da2a007434323904719158e50f3076a4dadb176ce43df28ed14610c773cc9825", size = 4273958, upload-time = "2025-11-01T11:52:41.017Z" }, + { url = "https://files.pythonhosted.org/packages/81/d0/b81e041c17cd475002114e0ab8800e4305e60837882cb376a621e520d70f/rapidfuzz-3.14.3-cp310-cp310-win32.whl", hash = "sha256:fce3152f94afcfd12f3dd8cf51e48fa606e3cb56719bccebe3b401f43d0714f9", size = 1725043, upload-time = "2025-11-01T11:52:42.465Z" }, + { url = "https://files.pythonhosted.org/packages/09/6b/64ad573337d81d64bc78a6a1df53a72a71d54d43d276ce0662c2e95a1f35/rapidfuzz-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:37d3c653af15cd88592633e942f5407cb4c64184efab163c40fcebad05f25141", size = 1542273, upload-time = "2025-11-01T11:52:44.005Z" }, + { url = "https://files.pythonhosted.org/packages/f4/5e/faf76e259bc15808bc0b86028f510215c3d755b6c3a3911113079485e561/rapidfuzz-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:cc594bbcd3c62f647dfac66800f307beaee56b22aaba1c005e9c4c40ed733923", size = 814875, upload-time = "2025-11-01T11:52:45.405Z" }, + { url = "https://files.pythonhosted.org/packages/76/25/5b0a33ad3332ee1213068c66f7c14e9e221be90bab434f0cb4defa9d6660/rapidfuzz-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dea2d113e260a5da0c4003e0a5e9fdf24a9dc2bb9eaa43abd030a1e46ce7837d", size = 1953885, upload-time = "2025-11-01T11:52:47.75Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ab/f1181f500c32c8fcf7c966f5920c7e56b9b1d03193386d19c956505c312d/rapidfuzz-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e6c31a4aa68cfa75d7eede8b0ed24b9e458447db604c2db53f358be9843d81d3", size = 1390200, upload-time = "2025-11-01T11:52:49.491Z" }, + { url = "https://files.pythonhosted.org/packages/14/2a/0f2de974ececad873865c6bb3ea3ad07c976ac293d5025b2d73325aac1d4/rapidfuzz-3.14.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02821366d928e68ddcb567fed8723dad7ea3a979fada6283e6914d5858674850", size = 1389319, upload-time = "2025-11-01T11:52:51.224Z" }, + { url = "https://files.pythonhosted.org/packages/ed/69/309d8f3a0bb3031fd9b667174cc4af56000645298af7c2931be5c3d14bb4/rapidfuzz-3.14.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfe8df315ab4e6db4e1be72c5170f8e66021acde22cd2f9d04d2058a9fd8162e", size = 3178495, upload-time = "2025-11-01T11:52:53.005Z" }, + { url = "https://files.pythonhosted.org/packages/10/b7/f9c44a99269ea5bf6fd6a40b84e858414b6e241288b9f2b74af470d222b1/rapidfuzz-3.14.3-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:769f31c60cd79420188fcdb3c823227fc4a6deb35cafec9d14045c7f6743acae", size = 1228443, upload-time = "2025-11-01T11:52:54.991Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0a/3b3137abac7f19c9220e14cd7ce993e35071a7655e7ef697785a3edfea1a/rapidfuzz-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:54fa03062124e73086dae66a3451c553c1e20a39c077fd704dc7154092c34c63", size = 2411998, upload-time = "2025-11-01T11:52:56.629Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b6/983805a844d44670eaae63831024cdc97ada4e9c62abc6b20703e81e7f9b/rapidfuzz-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:834d1e818005ed0d4ae38f6b87b86fad9b0a74085467ece0727d20e15077c094", size = 2530120, upload-time = "2025-11-01T11:52:58.298Z" }, + { url = "https://files.pythonhosted.org/packages/b4/cc/2c97beb2b1be2d7595d805682472f1b1b844111027d5ad89b65e16bdbaaa/rapidfuzz-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:948b00e8476a91f510dd1ec07272efc7d78c275d83b630455559671d4e33b678", size = 4283129, upload-time = "2025-11-01T11:53:00.188Z" }, + { url = "https://files.pythonhosted.org/packages/4d/03/2f0e5e94941045aefe7eafab72320e61285c07b752df9884ce88d6b8b835/rapidfuzz-3.14.3-cp311-cp311-win32.whl", hash = "sha256:43d0305c36f504232f18ea04e55f2059bb89f169d3119c4ea96a0e15b59e2a91", size = 1724224, upload-time = "2025-11-01T11:53:02.149Z" }, + { url = "https://files.pythonhosted.org/packages/cf/99/5fa23e204435803875daefda73fd61baeabc3c36b8fc0e34c1705aab8c7b/rapidfuzz-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:ef6bf930b947bd0735c550683939a032090f1d688dfd8861d6b45307b96fd5c5", size = 1544259, upload-time = "2025-11-01T11:53:03.66Z" }, + { url = "https://files.pythonhosted.org/packages/48/35/d657b85fcc615a42661b98ac90ce8e95bd32af474603a105643963749886/rapidfuzz-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:f3eb0ff3b75d6fdccd40b55e7414bb859a1cda77c52762c9c82b85569f5088e7", size = 814734, upload-time = "2025-11-01T11:53:05.008Z" }, + { url = "https://files.pythonhosted.org/packages/fa/8e/3c215e860b458cfbedb3ed73bc72e98eb7e0ed72f6b48099604a7a3260c2/rapidfuzz-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:685c93ea961d135893b5984a5a9851637d23767feabe414ec974f43babbd8226", size = 1945306, upload-time = "2025-11-01T11:53:06.452Z" }, + { url = "https://files.pythonhosted.org/packages/36/d9/31b33512015c899f4a6e6af64df8dfe8acddf4c8b40a4b3e0e6e1bcd00e5/rapidfuzz-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fa7c8f26f009f8c673fbfb443792f0cf8cf50c4e18121ff1e285b5e08a94fbdb", size = 1390788, upload-time = "2025-11-01T11:53:08.721Z" }, + { url = "https://files.pythonhosted.org/packages/a9/67/2ee6f8de6e2081ccd560a571d9c9063184fe467f484a17fa90311a7f4a2e/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57f878330c8d361b2ce76cebb8e3e1dc827293b6abf404e67d53260d27b5d941", size = 1374580, upload-time = "2025-11-01T11:53:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/30/83/80d22997acd928eda7deadc19ccd15883904622396d6571e935993e0453a/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c5f545f454871e6af05753a0172849c82feaf0f521c5ca62ba09e1b382d6382", size = 3154947, upload-time = "2025-11-01T11:53:12.093Z" }, + { url = "https://files.pythonhosted.org/packages/5b/cf/9f49831085a16384695f9fb096b99662f589e30b89b4a589a1ebc1a19d34/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:07aa0b5d8863e3151e05026a28e0d924accf0a7a3b605da978f0359bb804df43", size = 1223872, upload-time = "2025-11-01T11:53:13.664Z" }, + { url = "https://files.pythonhosted.org/packages/c8/0f/41ee8034e744b871c2e071ef0d360686f5ccfe5659f4fd96c3ec406b3c8b/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73b07566bc7e010e7b5bd490fb04bb312e820970180df6b5655e9e6224c137db", size = 2392512, upload-time = "2025-11-01T11:53:15.109Z" }, + { url = "https://files.pythonhosted.org/packages/da/86/280038b6b0c2ccec54fb957c732ad6b41cc1fd03b288d76545b9cf98343f/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6de00eb84c71476af7d3110cf25d8fe7c792d7f5fa86764ef0b4ca97e78ca3ed", size = 2521398, upload-time = "2025-11-01T11:53:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7b/05c26f939607dca0006505e3216248ae2de631e39ef94dd63dbbf0860021/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d7843a1abf0091773a530636fdd2a49a41bcae22f9910b86b4f903e76ddc82dc", size = 4259416, upload-time = "2025-11-01T11:53:19.34Z" }, + { url = "https://files.pythonhosted.org/packages/40/eb/9e3af4103d91788f81111af1b54a28de347cdbed8eaa6c91d5e98a889aab/rapidfuzz-3.14.3-cp312-cp312-win32.whl", hash = "sha256:dea97ac3ca18cd3ba8f3d04b5c1fe4aa60e58e8d9b7793d3bd595fdb04128d7a", size = 1709527, upload-time = "2025-11-01T11:53:20.949Z" }, + { url = "https://files.pythonhosted.org/packages/b8/63/d06ecce90e2cf1747e29aeab9f823d21e5877a4c51b79720b2d3be7848f8/rapidfuzz-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:b5100fd6bcee4d27f28f4e0a1c6b5127bc8ba7c2a9959cad9eab0bf4a7ab3329", size = 1538989, upload-time = "2025-11-01T11:53:22.428Z" }, + { url = "https://files.pythonhosted.org/packages/fc/6d/beee32dcda64af8128aab3ace2ccb33d797ed58c434c6419eea015fec779/rapidfuzz-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:4e49c9e992bc5fc873bd0fff7ef16a4405130ec42f2ce3d2b735ba5d3d4eb70f", size = 811161, upload-time = "2025-11-01T11:53:23.811Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/0d94d09646853bd26978cb3a7541b6233c5760687777fa97da8de0d9a6ac/rapidfuzz-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dbcb726064b12f356bf10fffdb6db4b6dce5390b23627c08652b3f6e49aa56ae", size = 1939646, upload-time = "2025-11-01T11:53:25.292Z" }, + { url = "https://files.pythonhosted.org/packages/b6/eb/f96aefc00f3bbdbab9c0657363ea8437a207d7545ac1c3789673e05d80bd/rapidfuzz-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1704fc70d214294e554a2421b473779bcdeef715881c5e927dc0f11e1692a0ff", size = 1385512, upload-time = "2025-11-01T11:53:27.594Z" }, + { url = "https://files.pythonhosted.org/packages/26/34/71c4f7749c12ee223dba90017a5947e8f03731a7cc9f489b662a8e9e643d/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc65e72790ddfd310c2c8912b45106e3800fefe160b0c2ef4d6b6fec4e826457", size = 1373571, upload-time = "2025-11-01T11:53:29.096Z" }, + { url = "https://files.pythonhosted.org/packages/32/00/ec8597a64f2be301ce1ee3290d067f49f6a7afb226b67d5f15b56d772ba5/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e38c1305cffae8472572a0584d4ffc2f130865586a81038ca3965301f7c97c", size = 3156759, upload-time = "2025-11-01T11:53:30.777Z" }, + { url = "https://files.pythonhosted.org/packages/61/d5/b41eeb4930501cc899d5a9a7b5c9a33d85a670200d7e81658626dcc0ecc0/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:e195a77d06c03c98b3fc06b8a28576ba824392ce40de8c708f96ce04849a052e", size = 1222067, upload-time = "2025-11-01T11:53:32.334Z" }, + { url = "https://files.pythonhosted.org/packages/2a/7d/6d9abb4ffd1027c6ed837b425834f3bed8344472eb3a503ab55b3407c721/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b7ef2f4b8583a744338a18f12c69693c194fb6777c0e9ada98cd4d9e8f09d10", size = 2394775, upload-time = "2025-11-01T11:53:34.24Z" }, + { url = "https://files.pythonhosted.org/packages/15/ce/4f3ab4c401c5a55364da1ffff8cc879fc97b4e5f4fa96033827da491a973/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a2135b138bcdcb4c3742d417f215ac2d8c2b87bde15b0feede231ae95f09ec41", size = 2526123, upload-time = "2025-11-01T11:53:35.779Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4b/54f804975376a328f57293bd817c12c9036171d15cf7292032e3f5820b2d/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33a325ed0e8e1aa20c3e75f8ab057a7b248fdea7843c2a19ade0008906c14af0", size = 4262874, upload-time = "2025-11-01T11:53:37.866Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b6/958db27d8a29a50ee6edd45d33debd3ce732e7209183a72f57544cd5fe22/rapidfuzz-3.14.3-cp313-cp313-win32.whl", hash = "sha256:8383b6d0d92f6cd008f3c9216535be215a064b2cc890398a678b56e6d280cb63", size = 1707972, upload-time = "2025-11-01T11:53:39.442Z" }, + { url = "https://files.pythonhosted.org/packages/07/75/fde1f334b0cec15b5946d9f84d73250fbfcc73c236b4bc1b25129d90876b/rapidfuzz-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:e6b5e3036976f0fde888687d91be86d81f9ac5f7b02e218913c38285b756be6c", size = 1537011, upload-time = "2025-11-01T11:53:40.92Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d7/d83fe001ce599dc7ead57ba1debf923dc961b6bdce522b741e6b8c82f55c/rapidfuzz-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:7ba009977601d8b0828bfac9a110b195b3e4e79b350dcfa48c11269a9f1918a0", size = 810744, upload-time = "2025-11-01T11:53:42.723Z" }, + { url = "https://files.pythonhosted.org/packages/92/13/a486369e63ff3c1a58444d16b15c5feb943edd0e6c28a1d7d67cb8946b8f/rapidfuzz-3.14.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a0a28add871425c2fe94358c6300bbeb0bc2ed828ca003420ac6825408f5a424", size = 1967702, upload-time = "2025-11-01T11:53:44.554Z" }, + { url = "https://files.pythonhosted.org/packages/f1/82/efad25e260b7810f01d6b69122685e355bed78c94a12784bac4e0beb2afb/rapidfuzz-3.14.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:010e12e2411a4854b0434f920e72b717c43f8ec48d57e7affe5c42ecfa05dd0e", size = 1410702, upload-time = "2025-11-01T11:53:46.066Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1a/34c977b860cde91082eae4a97ae503f43e0d84d4af301d857679b66f9869/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cfc3d57abd83c734d1714ec39c88a34dd69c85474918ebc21296f1e61eb5ca8", size = 1382337, upload-time = "2025-11-01T11:53:47.62Z" }, + { url = "https://files.pythonhosted.org/packages/88/74/f50ea0e24a5880a9159e8fd256b84d8f4634c2f6b4f98028bdd31891d907/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89acb8cbb52904f763e5ac238083b9fc193bed8d1f03c80568b20e4cef43a519", size = 3165563, upload-time = "2025-11-01T11:53:49.216Z" }, + { url = "https://files.pythonhosted.org/packages/e8/7a/e744359404d7737049c26099423fc54bcbf303de5d870d07d2fb1410f567/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_31_armv7l.whl", hash = "sha256:7d9af908c2f371bfb9c985bd134e295038e3031e666e4b2ade1e7cb7f5af2f1a", size = 1214727, upload-time = "2025-11-01T11:53:50.883Z" }, + { url = "https://files.pythonhosted.org/packages/d3/2e/87adfe14ce75768ec6c2b8acd0e05e85e84be4be5e3d283cdae360afc4fe/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1f1925619627f8798f8c3a391d81071336942e5fe8467bc3c567f982e7ce2897", size = 2403349, upload-time = "2025-11-01T11:53:52.322Z" }, + { url = "https://files.pythonhosted.org/packages/70/17/6c0b2b2bff9c8b12e12624c07aa22e922b0c72a490f180fa9183d1ef2c75/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:152555187360978119e98ce3e8263d70dd0c40c7541193fc302e9b7125cf8f58", size = 2507596, upload-time = "2025-11-01T11:53:53.835Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d1/87852a7cbe4da7b962174c749a47433881a63a817d04f3e385ea9babcd9e/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:52619d25a09546b8db078981ca88939d72caa6b8701edd8b22e16482a38e799f", size = 4273595, upload-time = "2025-11-01T11:53:55.961Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ab/1d0354b7d1771a28fa7fe089bc23acec2bdd3756efa2419f463e3ed80e16/rapidfuzz-3.14.3-cp313-cp313t-win32.whl", hash = "sha256:489ce98a895c98cad284f0a47960c3e264c724cb4cfd47a1430fa091c0c25204", size = 1757773, upload-time = "2025-11-01T11:53:57.628Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0c/71ef356adc29e2bdf74cd284317b34a16b80258fa0e7e242dd92cc1e6d10/rapidfuzz-3.14.3-cp313-cp313t-win_amd64.whl", hash = "sha256:656e52b054d5b5c2524169240e50cfa080b04b1c613c5f90a2465e84888d6f15", size = 1576797, upload-time = "2025-11-01T11:53:59.455Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d2/0e64fc27bb08d4304aa3d11154eb5480bcf5d62d60140a7ee984dc07468a/rapidfuzz-3.14.3-cp313-cp313t-win_arm64.whl", hash = "sha256:c7e40c0a0af02ad6e57e89f62bef8604f55a04ecae90b0ceeda591bbf5923317", size = 829940, upload-time = "2025-11-01T11:54:01.1Z" }, + { url = "https://files.pythonhosted.org/packages/32/6f/1b88aaeade83abc5418788f9e6b01efefcd1a69d65ded37d89cd1662be41/rapidfuzz-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:442125473b247227d3f2de807a11da6c08ccf536572d1be943f8e262bae7e4ea", size = 1942086, upload-time = "2025-11-01T11:54:02.592Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2c/b23861347436cb10f46c2bd425489ec462790faaa360a54a7ede5f78de88/rapidfuzz-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ec0c8c0c3d4f97ced46b2e191e883f8c82dbbf6d5ebc1842366d7eff13cd5a6", size = 1386993, upload-time = "2025-11-01T11:54:04.12Z" }, + { url = "https://files.pythonhosted.org/packages/83/86/5d72e2c060aa1fbdc1f7362d938f6b237dff91f5b9fc5dd7cc297e112250/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2dc37bc20272f388b8c3a4eba4febc6e77e50a8f450c472def4751e7678f55e4", size = 1379126, upload-time = "2025-11-01T11:54:05.777Z" }, + { url = "https://files.pythonhosted.org/packages/c9/bc/ef2cee3e4d8b3fc22705ff519f0d487eecc756abdc7c25d53686689d6cf2/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dee362e7e79bae940a5e2b3f6d09c6554db6a4e301cc68343886c08be99844f1", size = 3159304, upload-time = "2025-11-01T11:54:07.351Z" }, + { url = "https://files.pythonhosted.org/packages/a0/36/dc5f2f62bbc7bc90be1f75eeaf49ed9502094bb19290dfb4747317b17f12/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:4b39921df948388a863f0e267edf2c36302983459b021ab928d4b801cbe6a421", size = 1218207, upload-time = "2025-11-01T11:54:09.641Z" }, + { url = "https://files.pythonhosted.org/packages/df/7e/8f4be75c1bc62f47edf2bbbe2370ee482fae655ebcc4718ac3827ead3904/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:beda6aa9bc44d1d81242e7b291b446be352d3451f8217fcb068fc2933927d53b", size = 2401245, upload-time = "2025-11-01T11:54:11.543Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/f7c92759e1bb188dd05b80d11c630ba59b8d7856657baf454ff56059c2ab/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:6a014ba09657abfcfeed64b7d09407acb29af436d7fc075b23a298a7e4a6b41c", size = 2518308, upload-time = "2025-11-01T11:54:13.134Z" }, + { url = "https://files.pythonhosted.org/packages/c7/ac/85820f70fed5ecb5f1d9a55f1e1e2090ef62985ef41db289b5ac5ec56e28/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:32eeafa3abce138bb725550c0e228fc7eaeec7059aa8093d9cbbec2b58c2371a", size = 4265011, upload-time = "2025-11-01T11:54:15.087Z" }, + { url = "https://files.pythonhosted.org/packages/46/a9/616930721ea9835c918af7cde22bff17f9db3639b0c1a7f96684be7f5630/rapidfuzz-3.14.3-cp314-cp314-win32.whl", hash = "sha256:adb44d996fc610c7da8c5048775b21db60dd63b1548f078e95858c05c86876a3", size = 1742245, upload-time = "2025-11-01T11:54:17.19Z" }, + { url = "https://files.pythonhosted.org/packages/06/8a/f2fa5e9635b1ccafda4accf0e38246003f69982d7c81f2faa150014525a4/rapidfuzz-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:f3d15d8527e2b293e38ce6e437631af0708df29eafd7c9fc48210854c94472f9", size = 1584856, upload-time = "2025-11-01T11:54:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/ef/97/09e20663917678a6d60d8e0e29796db175b1165e2079830430342d5298be/rapidfuzz-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:576e4b9012a67e0bf54fccb69a7b6c94d4e86a9540a62f1a5144977359133583", size = 833490, upload-time = "2025-11-01T11:54:20.753Z" }, + { url = "https://files.pythonhosted.org/packages/03/1b/6b6084576ba87bf21877c77218a0c97ba98cb285b0c02eaaee3acd7c4513/rapidfuzz-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:cec3c0da88562727dd5a5a364bd9efeb535400ff0bfb1443156dd139a1dd7b50", size = 1968658, upload-time = "2025-11-01T11:54:22.25Z" }, + { url = "https://files.pythonhosted.org/packages/38/c0/fb02a0db80d95704b0a6469cc394e8c38501abf7e1c0b2afe3261d1510c2/rapidfuzz-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d1fa009f8b1100e4880868137e7bf0501422898f7674f2adcd85d5a67f041296", size = 1410742, upload-time = "2025-11-01T11:54:23.863Z" }, + { url = "https://files.pythonhosted.org/packages/a4/72/3fbf12819fc6afc8ec75a45204013b40979d068971e535a7f3512b05e765/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b86daa7419b5e8b180690efd1fdbac43ff19230803282521c5b5a9c83977655", size = 1382810, upload-time = "2025-11-01T11:54:25.571Z" }, + { url = "https://files.pythonhosted.org/packages/0f/18/0f1991d59bb7eee28922a00f79d83eafa8c7bfb4e8edebf4af2a160e7196/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7bd1816db05d6c5ffb3a4df0a2b7b56fb8c81ef584d08e37058afa217da91b1", size = 3166349, upload-time = "2025-11-01T11:54:27.195Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f0/baa958b1989c8f88c78bbb329e969440cf330b5a01a982669986495bb980/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:33da4bbaf44e9755b0ce192597f3bde7372fe2e381ab305f41b707a95ac57aa7", size = 1214994, upload-time = "2025-11-01T11:54:28.821Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a0/cd12ec71f9b2519a3954febc5740291cceabc64c87bc6433afcb36259f3b/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3fecce764cf5a991ee2195a844196da840aba72029b2612f95ac68a8b74946bf", size = 2403919, upload-time = "2025-11-01T11:54:30.393Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ce/019bd2176c1644098eced4f0595cb4b3ef52e4941ac9a5854f209d0a6e16/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ecd7453e02cf072258c3a6b8e930230d789d5d46cc849503729f9ce475d0e785", size = 2508346, upload-time = "2025-11-01T11:54:32.048Z" }, + { url = "https://files.pythonhosted.org/packages/23/f8/be16c68e2c9e6c4f23e8f4adbb7bccc9483200087ed28ff76c5312da9b14/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea188aa00e9bcae8c8411f006a5f2f06c4607a02f24eab0d8dc58566aa911f35", size = 4274105, upload-time = "2025-11-01T11:54:33.701Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d1/5ab148e03f7e6ec8cd220ccf7af74d3aaa4de26dd96df58936beb7cba820/rapidfuzz-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:7ccbf68100c170e9a0581accbe9291850936711548c6688ce3bfb897b8c589ad", size = 1793465, upload-time = "2025-11-01T11:54:35.331Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/433b2d98e97abd9fff1c470a109b311669f44cdec8d0d5aa250aceaed1fb/rapidfuzz-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:9ec02e62ae765a318d6de38df609c57fc6dacc65c0ed1fd489036834fd8a620c", size = 1623491, upload-time = "2025-11-01T11:54:38.085Z" }, + { url = "https://files.pythonhosted.org/packages/e2/f6/e2176eb94f94892441bce3ddc514c179facb65db245e7ce3356965595b19/rapidfuzz-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:e805e52322ae29aa945baf7168b6c898120fbc16d2b8f940b658a5e9e3999253", size = 851487, upload-time = "2025-11-01T11:54:40.176Z" }, + { url = "https://files.pythonhosted.org/packages/c9/33/b5bd6475c7c27164b5becc9b0e3eb978f1e3640fea590dd3dced6006ee83/rapidfuzz-3.14.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7cf174b52cb3ef5d49e45d0a1133b7e7d0ecf770ed01f97ae9962c5c91d97d23", size = 1888499, upload-time = "2025-11-01T11:54:42.094Z" }, + { url = "https://files.pythonhosted.org/packages/30/d2/89d65d4db4bb931beade9121bc71ad916b5fa9396e807d11b33731494e8e/rapidfuzz-3.14.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:442cba39957a008dfc5bdef21a9c3f4379e30ffb4e41b8555dbaf4887eca9300", size = 1336747, upload-time = "2025-11-01T11:54:43.957Z" }, + { url = "https://files.pythonhosted.org/packages/85/33/cd87d92b23f0b06e8914a61cea6850c6d495ca027f669fab7a379041827a/rapidfuzz-3.14.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1faa0f8f76ba75fd7b142c984947c280ef6558b5067af2ae9b8729b0a0f99ede", size = 1352187, upload-time = "2025-11-01T11:54:45.518Z" }, + { url = "https://files.pythonhosted.org/packages/22/20/9d30b4a1ab26aac22fff17d21dec7e9089ccddfe25151d0a8bb57001dc3d/rapidfuzz-3.14.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e6eefec45625c634926a9fd46c9e4f31118ac8f3156fff9494422cee45207e6", size = 3101472, upload-time = "2025-11-01T11:54:47.255Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ad/fa2d3e5c29a04ead7eaa731c7cd1f30f9ec3c77b3a578fdf90280797cbcb/rapidfuzz-3.14.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56fefb4382bb12250f164250240b9dd7772e41c5c8ae976fd598a32292449cc5", size = 1511361, upload-time = "2025-11-01T11:54:49.057Z" }, ] [[package]] diff --git a/components/job-orchestration/pyproject.toml b/components/job-orchestration/pyproject.toml index 30a5c9cbe2..2f34568a27 100644 --- a/components/job-orchestration/pyproject.toml +++ b/components/job-orchestration/pyproject.toml @@ -8,9 +8,7 @@ requires-python = ">=3.10" dependencies = [ "Brotli>=1.1.0", "celery[redis]>=5.5.3", - # mariadb version must be compatible with libmariadev installed in runtime env. - # See https://mariadb.com/docs/server/connect/programming-languages/python/install/#Dependencies - "mariadb>=1.0.11,<1.1.dev0", + "mariadb>=1.1.14", "msgpack>=1.1.2", "mysql-connector-python>=9.4.0", "pika>=1.3.2", diff --git a/components/job-orchestration/uv.lock b/components/job-orchestration/uv.lock index b0476ccf4b..31532efff2 100644 --- a/components/job-orchestration/uv.lock +++ b/components/job-orchestration/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" [[package]] @@ -43,72 +43,60 @@ wheels = [ [[package]] name = "brotli" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270, upload-time = "2023-09-07T14:05:41.643Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/3a/dbf4fb970c1019a57b5e492e1e0eae745d32e59ba4d6161ab5422b08eefe/Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752", size = 873045, upload-time = "2023-09-07T14:03:16.894Z" }, - { url = "https://files.pythonhosted.org/packages/dd/11/afc14026ea7f44bd6eb9316d800d439d092c8d508752055ce8d03086079a/Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9", size = 446218, upload-time = "2023-09-07T14:03:18.917Z" }, - { url = "https://files.pythonhosted.org/packages/36/83/7545a6e7729db43cb36c4287ae388d6885c85a86dd251768a47015dfde32/Brotli-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3", size = 2903872, upload-time = "2023-09-07T14:03:20.398Z" }, - { url = "https://files.pythonhosted.org/packages/32/23/35331c4d9391fcc0f29fd9bec2c76e4b4eeab769afbc4b11dd2e1098fb13/Brotli-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d", size = 2941254, upload-time = "2023-09-07T14:03:21.914Z" }, - { url = "https://files.pythonhosted.org/packages/3b/24/1671acb450c902edb64bd765d73603797c6c7280a9ada85a195f6b78c6e5/Brotli-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e", size = 2857293, upload-time = "2023-09-07T14:03:24Z" }, - { url = "https://files.pythonhosted.org/packages/d5/00/40f760cc27007912b327fe15bf6bfd8eaecbe451687f72a8abc587d503b3/Brotli-1.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da", size = 3002385, upload-time = "2023-09-07T14:03:26.248Z" }, - { url = "https://files.pythonhosted.org/packages/b8/cb/8aaa83f7a4caa131757668c0fb0c4b6384b09ffa77f2fba9570d87ab587d/Brotli-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80", size = 2911104, upload-time = "2023-09-07T14:03:27.849Z" }, - { url = "https://files.pythonhosted.org/packages/bc/c4/65456561d89d3c49f46b7fbeb8fe6e449f13bdc8ea7791832c5d476b2faf/Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d", size = 2809981, upload-time = "2023-09-07T14:03:29.92Z" }, - { url = "https://files.pythonhosted.org/packages/05/1b/cf49528437bae28abce5f6e059f0d0be6fecdcc1d3e33e7c54b3ca498425/Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0", size = 2935297, upload-time = "2023-09-07T14:03:32.035Z" }, - { url = "https://files.pythonhosted.org/packages/81/ff/190d4af610680bf0c5a09eb5d1eac6e99c7c8e216440f9c7cfd42b7adab5/Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e", size = 2930735, upload-time = "2023-09-07T14:03:33.801Z" }, - { url = "https://files.pythonhosted.org/packages/80/7d/f1abbc0c98f6e09abd3cad63ec34af17abc4c44f308a7a539010f79aae7a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c", size = 2933107, upload-time = "2024-10-18T12:32:09.016Z" }, - { url = "https://files.pythonhosted.org/packages/34/ce/5a5020ba48f2b5a4ad1c0522d095ad5847a0be508e7d7569c8630ce25062/Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1", size = 2845400, upload-time = "2024-10-18T12:32:11.134Z" }, - { url = "https://files.pythonhosted.org/packages/44/89/fa2c4355ab1eecf3994e5a0a7f5492c6ff81dfcb5f9ba7859bd534bb5c1a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2", size = 3031985, upload-time = "2024-10-18T12:32:12.813Z" }, - { url = "https://files.pythonhosted.org/packages/af/a4/79196b4a1674143d19dca400866b1a4d1a089040df7b93b88ebae81f3447/Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec", size = 2927099, upload-time = "2024-10-18T12:32:14.733Z" }, - { url = "https://files.pythonhosted.org/packages/e9/54/1c0278556a097f9651e657b873ab08f01b9a9ae4cac128ceb66427d7cd20/Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2", size = 333172, upload-time = "2023-09-07T14:03:35.212Z" }, - { url = "https://files.pythonhosted.org/packages/f7/65/b785722e941193fd8b571afd9edbec2a9b838ddec4375d8af33a50b8dab9/Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128", size = 357255, upload-time = "2023-09-07T14:03:36.447Z" }, - { url = "https://files.pythonhosted.org/packages/96/12/ad41e7fadd5db55459c4c401842b47f7fee51068f86dd2894dd0dcfc2d2a/Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc", size = 873068, upload-time = "2023-09-07T14:03:37.779Z" }, - { url = "https://files.pythonhosted.org/packages/95/4e/5afab7b2b4b61a84e9c75b17814198ce515343a44e2ed4488fac314cd0a9/Brotli-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6", size = 446244, upload-time = "2023-09-07T14:03:39.223Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e6/f305eb61fb9a8580c525478a4a34c5ae1a9bcb12c3aee619114940bc513d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd", size = 2906500, upload-time = "2023-09-07T14:03:40.858Z" }, - { url = "https://files.pythonhosted.org/packages/3e/4f/af6846cfbc1550a3024e5d3775ede1e00474c40882c7bf5b37a43ca35e91/Brotli-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf", size = 2943950, upload-time = "2023-09-07T14:03:42.896Z" }, - { url = "https://files.pythonhosted.org/packages/b3/e7/ca2993c7682d8629b62630ebf0d1f3bb3d579e667ce8e7ca03a0a0576a2d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61", size = 2918527, upload-time = "2023-09-07T14:03:44.552Z" }, - { url = "https://files.pythonhosted.org/packages/b3/96/da98e7bedc4c51104d29cc61e5f449a502dd3dbc211944546a4cc65500d3/Brotli-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327", size = 2845489, upload-time = "2023-09-07T14:03:46.594Z" }, - { url = "https://files.pythonhosted.org/packages/e8/ef/ccbc16947d6ce943a7f57e1a40596c75859eeb6d279c6994eddd69615265/Brotli-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd", size = 2914080, upload-time = "2023-09-07T14:03:48.204Z" }, - { url = "https://files.pythonhosted.org/packages/80/d6/0bd38d758d1afa62a5524172f0b18626bb2392d717ff94806f741fcd5ee9/Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9", size = 2813051, upload-time = "2023-09-07T14:03:50.348Z" }, - { url = "https://files.pythonhosted.org/packages/14/56/48859dd5d129d7519e001f06dcfbb6e2cf6db92b2702c0c2ce7d97e086c1/Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265", size = 2938172, upload-time = "2023-09-07T14:03:52.395Z" }, - { url = "https://files.pythonhosted.org/packages/3d/77/a236d5f8cd9e9f4348da5acc75ab032ab1ab2c03cc8f430d24eea2672888/Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8", size = 2933023, upload-time = "2023-09-07T14:03:53.96Z" }, - { url = "https://files.pythonhosted.org/packages/f1/87/3b283efc0f5cb35f7f84c0c240b1e1a1003a5e47141a4881bf87c86d0ce2/Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f", size = 2935871, upload-time = "2024-10-18T12:32:16.688Z" }, - { url = "https://files.pythonhosted.org/packages/f3/eb/2be4cc3e2141dc1a43ad4ca1875a72088229de38c68e842746b342667b2a/Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757", size = 2847784, upload-time = "2024-10-18T12:32:18.459Z" }, - { url = "https://files.pythonhosted.org/packages/66/13/b58ddebfd35edde572ccefe6890cf7c493f0c319aad2a5badee134b4d8ec/Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0", size = 3034905, upload-time = "2024-10-18T12:32:20.192Z" }, - { url = "https://files.pythonhosted.org/packages/84/9c/bc96b6c7db824998a49ed3b38e441a2cae9234da6fa11f6ed17e8cf4f147/Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b", size = 2929467, upload-time = "2024-10-18T12:32:21.774Z" }, - { url = "https://files.pythonhosted.org/packages/e7/71/8f161dee223c7ff7fea9d44893fba953ce97cf2c3c33f78ba260a91bcff5/Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50", size = 333169, upload-time = "2023-09-07T14:03:55.404Z" }, - { url = "https://files.pythonhosted.org/packages/02/8a/fece0ee1057643cb2a5bbf59682de13f1725f8482b2c057d4e799d7ade75/Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1", size = 357253, upload-time = "2023-09-07T14:03:56.643Z" }, - { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693, upload-time = "2024-10-18T12:32:23.824Z" }, - { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489, upload-time = "2024-10-18T12:32:25.641Z" }, - { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081, upload-time = "2023-09-07T14:03:57.967Z" }, - { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244, upload-time = "2023-09-07T14:03:59.319Z" }, - { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505, upload-time = "2023-09-07T14:04:01.327Z" }, - { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152, upload-time = "2023-09-07T14:04:03.033Z" }, - { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252, upload-time = "2023-09-07T14:04:04.675Z" }, - { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955, upload-time = "2023-09-07T14:04:06.585Z" }, - { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304, upload-time = "2023-09-07T14:04:08.668Z" }, - { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452, upload-time = "2023-09-07T14:04:10.736Z" }, - { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751, upload-time = "2023-09-07T14:04:12.875Z" }, - { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757, upload-time = "2023-09-07T14:04:14.551Z" }, - { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146, upload-time = "2024-10-18T12:32:27.257Z" }, - { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055, upload-time = "2024-10-18T12:32:29.376Z" }, - { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102, upload-time = "2024-10-18T12:32:31.371Z" }, - { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029, upload-time = "2024-10-18T12:32:33.293Z" }, - { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276, upload-time = "2023-09-07T14:04:16.49Z" }, - { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255, upload-time = "2023-09-07T14:04:17.83Z" }, - { url = "https://files.pythonhosted.org/packages/0a/9f/fb37bb8ffc52a8da37b1c03c459a8cd55df7a57bdccd8831d500e994a0ca/Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5", size = 815681, upload-time = "2024-10-18T12:32:34.942Z" }, - { url = "https://files.pythonhosted.org/packages/06/b3/dbd332a988586fefb0aa49c779f59f47cae76855c2d00f450364bb574cac/Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8", size = 422475, upload-time = "2024-10-18T12:32:36.485Z" }, - { url = "https://files.pythonhosted.org/packages/bb/80/6aaddc2f63dbcf2d93c2d204e49c11a9ec93a8c7c63261e2b4bd35198283/Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f", size = 2906173, upload-time = "2024-10-18T12:32:37.978Z" }, - { url = "https://files.pythonhosted.org/packages/ea/1d/e6ca79c96ff5b641df6097d299347507d39a9604bde8915e76bf026d6c77/Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648", size = 2943803, upload-time = "2024-10-18T12:32:39.606Z" }, - { url = "https://files.pythonhosted.org/packages/ac/a3/d98d2472e0130b7dd3acdbb7f390d478123dbf62b7d32bda5c830a96116d/Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0", size = 2918946, upload-time = "2024-10-18T12:32:41.679Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a5/c69e6d272aee3e1423ed005d8915a7eaa0384c7de503da987f2d224d0721/Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089", size = 2845707, upload-time = "2024-10-18T12:32:43.478Z" }, - { url = "https://files.pythonhosted.org/packages/58/9f/4149d38b52725afa39067350696c09526de0125ebfbaab5acc5af28b42ea/Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368", size = 2936231, upload-time = "2024-10-18T12:32:45.224Z" }, - { url = "https://files.pythonhosted.org/packages/5a/5a/145de884285611838a16bebfdb060c231c52b8f84dfbe52b852a15780386/Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c", size = 2848157, upload-time = "2024-10-18T12:32:46.894Z" }, - { url = "https://files.pythonhosted.org/packages/50/ae/408b6bfb8525dadebd3b3dd5b19d631da4f7d46420321db44cd99dcf2f2c/Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284", size = 3035122, upload-time = "2024-10-18T12:32:48.844Z" }, - { url = "https://files.pythonhosted.org/packages/af/85/a94e5cfaa0ca449d8f91c3d6f78313ebf919a0dbd55a100c711c6e9655bc/Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7", size = 2930206, upload-time = "2024-10-18T12:32:51.198Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f0/a61d9262cd01351df22e57ad7c34f66794709acab13f34be2675f45bf89d/Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0", size = 333804, upload-time = "2024-10-18T12:32:52.661Z" }, - { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517, upload-time = "2024-10-18T12:32:54.066Z" }, +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/16/c92ca344d646e71a43b8bb353f0a6490d7f6e06210f8554c8f874e454285/brotli-1.2.0.tar.gz", hash = "sha256:e310f77e41941c13340a95976fe66a8a95b01e783d430eeaf7a2f87e0a57dd0a", size = 7388632, upload-time = "2025-11-05T18:39:42.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/10/a090475284fc4a71aed40a96f32e44a7fe5bda39687353dd977720b211b6/brotli-1.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b90b767916ac44e93a8e28ce6adf8d551e43affb512f2377c732d486ac6514e", size = 863089, upload-time = "2025-11-05T18:38:01.181Z" }, + { url = "https://files.pythonhosted.org/packages/03/41/17416630e46c07ac21e378c3464815dd2e120b441e641bc516ac32cc51d2/brotli-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6be67c19e0b0c56365c6a76e393b932fb0e78b3b56b711d180dd7013cb1fd984", size = 445442, upload-time = "2025-11-05T18:38:02.434Z" }, + { url = "https://files.pythonhosted.org/packages/24/31/90cc06584deb5d4fcafc0985e37741fc6b9717926a78674bbb3ce018957e/brotli-1.2.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0bbd5b5ccd157ae7913750476d48099aaf507a79841c0d04a9db4415b14842de", size = 1532658, upload-time = "2025-11-05T18:38:03.588Z" }, + { url = "https://files.pythonhosted.org/packages/62/17/33bf0c83bcbc96756dfd712201d87342732fad70bb3472c27e833a44a4f9/brotli-1.2.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3f3c908bcc404c90c77d5a073e55271a0a498f4e0756e48127c35d91cf155947", size = 1631241, upload-time = "2025-11-05T18:38:04.582Z" }, + { url = "https://files.pythonhosted.org/packages/48/10/f47854a1917b62efe29bc98ac18e5d4f71df03f629184575b862ef2e743b/brotli-1.2.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1b557b29782a643420e08d75aea889462a4a8796e9a6cf5621ab05a3f7da8ef2", size = 1424307, upload-time = "2025-11-05T18:38:05.587Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b7/f88eb461719259c17483484ea8456925ee057897f8e64487d76e24e5e38d/brotli-1.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81da1b229b1889f25adadc929aeb9dbc4e922bd18561b65b08dd9343cfccca84", size = 1488208, upload-time = "2025-11-05T18:38:06.613Z" }, + { url = "https://files.pythonhosted.org/packages/26/59/41bbcb983a0c48b0b8004203e74706c6b6e99a04f3c7ca6f4f41f364db50/brotli-1.2.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ff09cd8c5eec3b9d02d2408db41be150d8891c5566addce57513bf546e3d6c6d", size = 1597574, upload-time = "2025-11-05T18:38:07.838Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e6/8c89c3bdabbe802febb4c5c6ca224a395e97913b5df0dff11b54f23c1788/brotli-1.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a1778532b978d2536e79c05dac2d8cd857f6c55cd0c95ace5b03740824e0e2f1", size = 1492109, upload-time = "2025-11-05T18:38:08.816Z" }, + { url = "https://files.pythonhosted.org/packages/ed/9a/4b19d4310b2dbd545c0c33f176b0528fa68c3cd0754e34b2f2bcf56548ae/brotli-1.2.0-cp310-cp310-win32.whl", hash = "sha256:b232029d100d393ae3c603c8ffd7e3fe6f798c5e28ddca5feabb8e8fdb732997", size = 334461, upload-time = "2025-11-05T18:38:10.729Z" }, + { url = "https://files.pythonhosted.org/packages/ac/39/70981d9f47705e3c2b95c0847dfa3e7a37aa3b7c6030aedc4873081ed005/brotli-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:ef87b8ab2704da227e83a246356a2b179ef826f550f794b2c52cddb4efbd0196", size = 369035, upload-time = "2025-11-05T18:38:11.827Z" }, + { url = "https://files.pythonhosted.org/packages/7a/ef/f285668811a9e1ddb47a18cb0b437d5fc2760d537a2fe8a57875ad6f8448/brotli-1.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:15b33fe93cedc4caaff8a0bd1eb7e3dab1c61bb22a0bf5bdfdfd97cd7da79744", size = 863110, upload-time = "2025-11-05T18:38:12.978Z" }, + { url = "https://files.pythonhosted.org/packages/50/62/a3b77593587010c789a9d6eaa527c79e0848b7b860402cc64bc0bc28a86c/brotli-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:898be2be399c221d2671d29eed26b6b2713a02c2119168ed914e7d00ceadb56f", size = 445438, upload-time = "2025-11-05T18:38:14.208Z" }, + { url = "https://files.pythonhosted.org/packages/cd/e1/7fadd47f40ce5549dc44493877db40292277db373da5053aff181656e16e/brotli-1.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350c8348f0e76fff0a0fd6c26755d2653863279d086d3aa2c290a6a7251135dd", size = 1534420, upload-time = "2025-11-05T18:38:15.111Z" }, + { url = "https://files.pythonhosted.org/packages/12/8b/1ed2f64054a5a008a4ccd2f271dbba7a5fb1a3067a99f5ceadedd4c1d5a7/brotli-1.2.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e1ad3fda65ae0d93fec742a128d72e145c9c7a99ee2fcd667785d99eb25a7fe", size = 1632619, upload-time = "2025-11-05T18:38:16.094Z" }, + { url = "https://files.pythonhosted.org/packages/89/5a/7071a621eb2d052d64efd5da2ef55ecdac7c3b0c6e4f9d519e9c66d987ef/brotli-1.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:40d918bce2b427a0c4ba189df7a006ac0c7277c180aee4617d99e9ccaaf59e6a", size = 1426014, upload-time = "2025-11-05T18:38:17.177Z" }, + { url = "https://files.pythonhosted.org/packages/26/6d/0971a8ea435af5156acaaccec1a505f981c9c80227633851f2810abd252a/brotli-1.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2a7f1d03727130fc875448b65b127a9ec5d06d19d0148e7554384229706f9d1b", size = 1489661, upload-time = "2025-11-05T18:38:18.41Z" }, + { url = "https://files.pythonhosted.org/packages/f3/75/c1baca8b4ec6c96a03ef8230fab2a785e35297632f402ebb1e78a1e39116/brotli-1.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9c79f57faa25d97900bfb119480806d783fba83cd09ee0b33c17623935b05fa3", size = 1599150, upload-time = "2025-11-05T18:38:19.792Z" }, + { url = "https://files.pythonhosted.org/packages/0d/1a/23fcfee1c324fd48a63d7ebf4bac3a4115bdb1b00e600f80f727d850b1ae/brotli-1.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:844a8ceb8483fefafc412f85c14f2aae2fb69567bf2a0de53cdb88b73e7c43ae", size = 1493505, upload-time = "2025-11-05T18:38:20.913Z" }, + { url = "https://files.pythonhosted.org/packages/36/e5/12904bbd36afeef53d45a84881a4810ae8810ad7e328a971ebbfd760a0b3/brotli-1.2.0-cp311-cp311-win32.whl", hash = "sha256:aa47441fa3026543513139cb8926a92a8e305ee9c71a6209ef7a97d91640ea03", size = 334451, upload-time = "2025-11-05T18:38:21.94Z" }, + { url = "https://files.pythonhosted.org/packages/02/8b/ecb5761b989629a4758c394b9301607a5880de61ee2ee5fe104b87149ebc/brotli-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:022426c9e99fd65d9475dce5c195526f04bb8be8907607e27e747893f6ee3e24", size = 369035, upload-time = "2025-11-05T18:38:22.941Z" }, + { url = "https://files.pythonhosted.org/packages/11/ee/b0a11ab2315c69bb9b45a2aaed022499c9c24a205c3a49c3513b541a7967/brotli-1.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:35d382625778834a7f3061b15423919aa03e4f5da34ac8e02c074e4b75ab4f84", size = 861543, upload-time = "2025-11-05T18:38:24.183Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2f/29c1459513cd35828e25531ebfcbf3e92a5e49f560b1777a9af7203eb46e/brotli-1.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7a61c06b334bd99bc5ae84f1eeb36bfe01400264b3c352f968c6e30a10f9d08b", size = 444288, upload-time = "2025-11-05T18:38:25.139Z" }, + { url = "https://files.pythonhosted.org/packages/3d/6f/feba03130d5fceadfa3a1bb102cb14650798c848b1df2a808356f939bb16/brotli-1.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acec55bb7c90f1dfc476126f9711a8e81c9af7fb617409a9ee2953115343f08d", size = 1528071, upload-time = "2025-11-05T18:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/2b/38/f3abb554eee089bd15471057ba85f47e53a44a462cfce265d9bf7088eb09/brotli-1.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:260d3692396e1895c5034f204f0db022c056f9e2ac841593a4cf9426e2a3faca", size = 1626913, upload-time = "2025-11-05T18:38:27.284Z" }, + { url = "https://files.pythonhosted.org/packages/03/a7/03aa61fbc3c5cbf99b44d158665f9b0dd3d8059be16c460208d9e385c837/brotli-1.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:072e7624b1fc4d601036ab3f4f27942ef772887e876beff0301d261210bca97f", size = 1419762, upload-time = "2025-11-05T18:38:28.295Z" }, + { url = "https://files.pythonhosted.org/packages/21/1b/0374a89ee27d152a5069c356c96b93afd1b94eae83f1e004b57eb6ce2f10/brotli-1.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adedc4a67e15327dfdd04884873c6d5a01d3e3b6f61406f99b1ed4865a2f6d28", size = 1484494, upload-time = "2025-11-05T18:38:29.29Z" }, + { url = "https://files.pythonhosted.org/packages/cf/57/69d4fe84a67aef4f524dcd075c6eee868d7850e85bf01d778a857d8dbe0a/brotli-1.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7a47ce5c2288702e09dc22a44d0ee6152f2c7eda97b3c8482d826a1f3cfc7da7", size = 1593302, upload-time = "2025-11-05T18:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/d5/3b/39e13ce78a8e9a621c5df3aeb5fd181fcc8caba8c48a194cd629771f6828/brotli-1.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:af43b8711a8264bb4e7d6d9a6d004c3a2019c04c01127a868709ec29962b6036", size = 1487913, upload-time = "2025-11-05T18:38:31.618Z" }, + { url = "https://files.pythonhosted.org/packages/62/28/4d00cb9bd76a6357a66fcd54b4b6d70288385584063f4b07884c1e7286ac/brotli-1.2.0-cp312-cp312-win32.whl", hash = "sha256:e99befa0b48f3cd293dafeacdd0d191804d105d279e0b387a32054c1180f3161", size = 334362, upload-time = "2025-11-05T18:38:32.939Z" }, + { url = "https://files.pythonhosted.org/packages/1c/4e/bc1dcac9498859d5e353c9b153627a3752868a9d5f05ce8dedd81a2354ab/brotli-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:b35c13ce241abdd44cb8ca70683f20c0c079728a36a996297adb5334adfc1c44", size = 369115, upload-time = "2025-11-05T18:38:33.765Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d4/4ad5432ac98c73096159d9ce7ffeb82d151c2ac84adcc6168e476bb54674/brotli-1.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9e5825ba2c9998375530504578fd4d5d1059d09621a02065d1b6bfc41a8e05ab", size = 861523, upload-time = "2025-11-05T18:38:34.67Z" }, + { url = "https://files.pythonhosted.org/packages/91/9f/9cc5bd03ee68a85dc4bc89114f7067c056a3c14b3d95f171918c088bf88d/brotli-1.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0cf8c3b8ba93d496b2fae778039e2f5ecc7cff99df84df337ca31d8f2252896c", size = 444289, upload-time = "2025-11-05T18:38:35.6Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b6/fe84227c56a865d16a6614e2c4722864b380cb14b13f3e6bef441e73a85a/brotli-1.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8565e3cdc1808b1a34714b553b262c5de5fbda202285782173ec137fd13709f", size = 1528076, upload-time = "2025-11-05T18:38:36.639Z" }, + { url = "https://files.pythonhosted.org/packages/55/de/de4ae0aaca06c790371cf6e7ee93a024f6b4bb0568727da8c3de112e726c/brotli-1.2.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:26e8d3ecb0ee458a9804f47f21b74845cc823fd1bb19f02272be70774f56e2a6", size = 1626880, upload-time = "2025-11-05T18:38:37.623Z" }, + { url = "https://files.pythonhosted.org/packages/5f/16/a1b22cbea436642e071adcaf8d4b350a2ad02f5e0ad0da879a1be16188a0/brotli-1.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67a91c5187e1eec76a61625c77a6c8c785650f5b576ca732bd33ef58b0dff49c", size = 1419737, upload-time = "2025-11-05T18:38:38.729Z" }, + { url = "https://files.pythonhosted.org/packages/46/63/c968a97cbb3bdbf7f974ef5a6ab467a2879b82afbc5ffb65b8acbb744f95/brotli-1.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4ecdb3b6dc36e6d6e14d3a1bdc6c1057c8cbf80db04031d566eb6080ce283a48", size = 1484440, upload-time = "2025-11-05T18:38:39.916Z" }, + { url = "https://files.pythonhosted.org/packages/06/9d/102c67ea5c9fc171f423e8399e585dabea29b5bc79b05572891e70013cdd/brotli-1.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3e1b35d56856f3ed326b140d3c6d9db91740f22e14b06e840fe4bb1923439a18", size = 1593313, upload-time = "2025-11-05T18:38:41.24Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/9526d14fa6b87bc827ba1755a8440e214ff90de03095cacd78a64abe2b7d/brotli-1.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54a50a9dad16b32136b2241ddea9e4df159b41247b2ce6aac0b3276a66a8f1e5", size = 1487945, upload-time = "2025-11-05T18:38:42.277Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e8/3fe1ffed70cbef83c5236166acaed7bb9c766509b157854c80e2f766b38c/brotli-1.2.0-cp313-cp313-win32.whl", hash = "sha256:1b1d6a4efedd53671c793be6dd760fcf2107da3a52331ad9ea429edf0902f27a", size = 334368, upload-time = "2025-11-05T18:38:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/ff/91/e739587be970a113b37b821eae8097aac5a48e5f0eca438c22e4c7dd8648/brotli-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:b63daa43d82f0cdabf98dee215b375b4058cce72871fd07934f179885aad16e8", size = 369116, upload-time = "2025-11-05T18:38:44.609Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/298c2ddf786bb7347a1cd71d63a347a79e5712a7c0cba9e3c3458ebd976f/brotli-1.2.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6c12dad5cd04530323e723787ff762bac749a7b256a5bece32b2243dd5c27b21", size = 863080, upload-time = "2025-11-05T18:38:45.503Z" }, + { url = "https://files.pythonhosted.org/packages/84/0c/aac98e286ba66868b2b3b50338ffbd85a35c7122e9531a73a37a29763d38/brotli-1.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3219bd9e69868e57183316ee19c84e03e8f8b5a1d1f2667e1aa8c2f91cb061ac", size = 445453, upload-time = "2025-11-05T18:38:46.433Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f1/0ca1f3f99ae300372635ab3fe2f7a79fa335fee3d874fa7f9e68575e0e62/brotli-1.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:963a08f3bebd8b75ac57661045402da15991468a621f014be54e50f53a58d19e", size = 1528168, upload-time = "2025-11-05T18:38:47.371Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a6/2ebfc8f766d46df8d3e65b880a2e220732395e6d7dc312c1e1244b0f074a/brotli-1.2.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9322b9f8656782414b37e6af884146869d46ab85158201d82bab9abbcb971dc7", size = 1627098, upload-time = "2025-11-05T18:38:48.385Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2f/0976d5b097ff8a22163b10617f76b2557f15f0f39d6a0fe1f02b1a53e92b/brotli-1.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cf9cba6f5b78a2071ec6fb1e7bd39acf35071d90a81231d67e92d637776a6a63", size = 1419861, upload-time = "2025-11-05T18:38:49.372Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/d76df7176a2ce7616ff94c1fb72d307c9a30d2189fe877f3dd99af00ea5a/brotli-1.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7547369c4392b47d30a3467fe8c3330b4f2e0f7730e45e3103d7d636678a808b", size = 1484594, upload-time = "2025-11-05T18:38:50.655Z" }, + { url = "https://files.pythonhosted.org/packages/d3/93/14cf0b1216f43df5609f5b272050b0abd219e0b54ea80b47cef9867b45e7/brotli-1.2.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:fc1530af5c3c275b8524f2e24841cbe2599d74462455e9bae5109e9ff42e9361", size = 1593455, upload-time = "2025-11-05T18:38:51.624Z" }, + { url = "https://files.pythonhosted.org/packages/b3/73/3183c9e41ca755713bdf2cc1d0810df742c09484e2e1ddd693bee53877c1/brotli-1.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d2d085ded05278d1c7f65560aae97b3160aeb2ea2c0b3e26204856beccb60888", size = 1488164, upload-time = "2025-11-05T18:38:53.079Z" }, + { url = "https://files.pythonhosted.org/packages/64/6a/0c78d8f3a582859236482fd9fa86a65a60328a00983006bcf6d83b7b2253/brotli-1.2.0-cp314-cp314-win32.whl", hash = "sha256:832c115a020e463c2f67664560449a7bea26b0c1fdd690352addad6d0a08714d", size = 339280, upload-time = "2025-11-05T18:38:54.02Z" }, + { url = "https://files.pythonhosted.org/packages/f5/10/56978295c14794b2c12007b07f3e41ba26acda9257457d7085b0bb3bb90c/brotli-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e7c0af964e0b4e3412a0ebf341ea26ec767fa0b4cf81abb5e897c9338b5ad6a3", size = 375639, upload-time = "2025-11-05T18:38:55.67Z" }, ] [[package]] @@ -223,14 +211,14 @@ dependencies = [ requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, { name = "celery", extras = ["redis"], specifier = ">=5.5.3" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "msgpack", specifier = ">=1.1.2" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pika", specifier = ">=1.3.2" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "pymongo", specifier = ">=4.15.3" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "yscope-spider-py", specifier = "==0.1.0" }, + { name = "yscope-spider-py", specifier = "==0.2.0" }, ] [[package]] @@ -255,21 +243,32 @@ redis = [ [[package]] name = "lark" -version = "1.3.0" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/37/a13baf0135f348af608c667633cbe5d13aa2c5c15a56ae9ad3e6cba45ae3/lark-1.3.0.tar.gz", hash = "sha256:9a3839d0ca5e1faf7cfa3460e420e859b66bcbde05b634e73c369c8244c5fa48", size = 259551, upload-time = "2025-09-22T13:45:05.072Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/3e/1c6b43277de64fc3c0333b0e72ab7b52ddaaea205210d60d9b9f83c3d0c7/lark-1.3.0-py3-none-any.whl", hash = "sha256:80661f261fb2584a9828a097a2432efd575af27d20be0fd35d17f0fe37253831", size = 113002, upload-time = "2025-09-22T13:45:03.747Z" }, + { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, ] [[package]] name = "mariadb" -version = "1.0.11" +version = "1.1.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/0a/ce5724f852f2937c6955bcea09659aa2d85d487df1c9de6711344b71527d/mariadb-1.0.11.zip", hash = "sha256:76916c892bc936c5b0f36e25a1411f651a7b7ce978992ae3a8de7e283efdacbf", size = 85926, upload-time = "2022-04-12T19:33:17.988Z" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/ba/cedef19833be88e07bfff11964441cda8a998f1628dd3b2fa3e7751d36e0/mariadb-1.1.14.tar.gz", hash = "sha256:e6d702a53eccf20922e47f2f45cfb5c7a0c2c6c0a46e4ee2d8a80d0ff4a52f34", size = 111715, upload-time = "2025-10-07T06:45:48.017Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/4a/6a6b7ad6a7b8156b41d0a6d849debe804f9d8696288ed2c6c31a9654357c/mariadb-1.0.11-cp310-cp310-win32.whl", hash = "sha256:4a583a80059d11f1895a2c93b1b7110948e87f0256da3e3222939a2530f0518e", size = 159743, upload-time = "2022-04-12T19:33:04.712Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a7/10740ceec1ce9d57f4bc3614e55efe2e72ae284e8c8d32eacabfbd7ad6cc/mariadb-1.0.11-cp310-cp310-win_amd64.whl", hash = "sha256:63c5c7cf99335e5c961e1d65a323576c9cb834e1a6a8084a6a8b4ffd85ca6213", size = 177316, upload-time = "2022-04-12T19:33:06.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/bf/5e1a3a5be297c3a679e08f5359165491508fbfb64faf854dc1d626cea9c0/mariadb-1.1.14-cp310-cp310-win32.whl", hash = "sha256:4c7f33578da163a1b79929aae241f5f981d7b9d5a94d89e589aad7ec58e313ea", size = 185064, upload-time = "2025-10-07T06:45:24.858Z" }, + { url = "https://files.pythonhosted.org/packages/31/30/3a61991c13cb8257f5db64aca12bafaa3d811d407e1fae019139fd17c99b/mariadb-1.1.14-cp310-cp310-win_amd64.whl", hash = "sha256:3f6fdc4ded5e0500a6a29bf0c8bf1be94189dcef5a8d5e0e154a4b3456f86bcc", size = 202020, upload-time = "2025-10-07T06:45:27.785Z" }, + { url = "https://files.pythonhosted.org/packages/56/aa/a7b3c66b2792e8319ec9157d63851ff2e0b26496a05044e22b50a012a05e/mariadb-1.1.14-cp311-cp311-win32.whl", hash = "sha256:932a95016b7e9b8d78893aa5ee608e74199e3c6dd607dbe5e4da2010a4f67b88", size = 185061, upload-time = "2025-10-07T06:45:29.964Z" }, + { url = "https://files.pythonhosted.org/packages/54/04/ea2374867756b4082764484bc8b82e1798d94f171bcc914e08c60d640f8f/mariadb-1.1.14-cp311-cp311-win_amd64.whl", hash = "sha256:55ddbe5272c292cbcb2968d87681b5d2b327e65646a015e324b8eeb804d14531", size = 202016, upload-time = "2025-10-07T06:45:32.151Z" }, + { url = "https://files.pythonhosted.org/packages/00/04/659a8d30513700b5921ec96bddc07f550016c045fcbeb199d8cd18476ecc/mariadb-1.1.14-cp312-cp312-win32.whl", hash = "sha256:98d552a8bb599eceaa88f65002ad00bd88aeed160592c273a7e5c1d79ab733dd", size = 185266, upload-time = "2025-10-07T06:45:34.164Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a9/8f210291bc5fc044e20497454f40d35b3bab326e2cab6fccdc38121cb2c1/mariadb-1.1.14-cp312-cp312-win_amd64.whl", hash = "sha256:685a1ad2a24fd0aae1c4416fe0ac794adc84ab9209c8d0c57078f770d39731db", size = 202112, upload-time = "2025-10-07T06:45:35.824Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/51130048bcce038bb859978250515f1aad90e9c4d273630a704e0a8b1ae1/mariadb-1.1.14-cp313-cp313-win32.whl", hash = "sha256:3d2c795cde606f4e12c0d73282b062433f414cae035675b0d81f2d65c9b79ac5", size = 185221, upload-time = "2025-10-07T06:45:37.848Z" }, + { url = "https://files.pythonhosted.org/packages/af/23/e952a7e442913abd8079cd27b80b69474895c93b3727fad41c7642a80c62/mariadb-1.1.14-cp313-cp313-win_amd64.whl", hash = "sha256:7fd603c5cf23c47ef0d28fdc2b4b79919ee7f75d00ed070d3cd1054dcf816aeb", size = 202121, upload-time = "2025-10-07T06:45:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f2/7059f83543a4264b98777d7cc8aa203e1ca6a13a461f730d1c97f29628d4/mariadb-1.1.14-cp314-cp314-win32.whl", hash = "sha256:1a50b4612c0dd5b69690cebb34cef552a7f64dcadeb5aa91d70cd99bf01bc5b3", size = 190620, upload-time = "2025-10-07T06:45:41.349Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7c/7e094b0b396d742494f6346f2ffa9709e429970b0461aca50526f5f02f12/mariadb-1.1.14-cp314-cp314-win_amd64.whl", hash = "sha256:6659725837e48fa6af05e20128fb525029f706f1921d5dbf639a25b2f80b9f93", size = 206263, upload-time = "2025-10-07T06:45:43.227Z" }, ] [[package]] @@ -348,31 +347,36 @@ wheels = [ [[package]] name = "mysql-connector-python" -version = "9.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/77/2b45e6460d05b1f1b7a4c8eb79a50440b4417971973bb78c9ef6cad630a6/mysql_connector_python-9.4.0.tar.gz", hash = "sha256:d111360332ae78933daf3d48ff497b70739aa292ab0017791a33e826234e743b", size = 12185532, upload-time = "2025-07-22T08:02:05.788Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/ef/1a35d9ebfaf80cf5aa238be471480e16a69a494d276fb07b889dc9a5cfc3/mysql_connector_python-9.4.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3c2603e00516cf4208c6266e85c5c87d5f4d0ac79768106d50de42ccc8414c05", size = 17501678, upload-time = "2025-07-22T07:57:23.237Z" }, - { url = "https://files.pythonhosted.org/packages/3c/39/09ae7082c77a978f2d72d94856e2e57906165c645693bc3a940bcad3a32d/mysql_connector_python-9.4.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:47884fcb050112b8bef3458e17eac47cc81a6cbbf3524e3456146c949772d9b4", size = 18369526, upload-time = "2025-07-22T07:57:27.569Z" }, - { url = "https://files.pythonhosted.org/packages/40/56/1bea00f5129550bcd0175781b9cd467e865d4aea4a6f38f700f34d95dcb8/mysql_connector_python-9.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f14b6936cd326e212fc9ab5f666dea3efea654f0cb644460334e60e22986e735", size = 33508525, upload-time = "2025-07-22T07:57:32.935Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ec/86dfefd3e6c0fca13085bc28b7f9baae3fce9f6af243d8693729f6b5063c/mysql_connector_python-9.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0f5ad70355720e64b72d7c068e858c9fd1f69b671d9575f857f235a10f878939", size = 33911834, upload-time = "2025-07-22T07:57:38.203Z" }, - { url = "https://files.pythonhosted.org/packages/2c/11/6907d53349b11478f72c8f22e38368d18262fbffc27e0f30e365d76dad93/mysql_connector_python-9.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:7106670abce510e440d393e27fc3602b8cf21e7a8a80216cc9ad9a68cd2e4595", size = 16393044, upload-time = "2025-07-22T07:57:42.053Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0c/4365a802129be9fa63885533c38be019f1c6b6f5bcf8844ac53902314028/mysql_connector_python-9.4.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:7df1a8ddd182dd8adc914f6dc902a986787bf9599705c29aca7b2ce84e79d361", size = 17501627, upload-time = "2025-07-22T07:57:45.416Z" }, - { url = "https://files.pythonhosted.org/packages/c0/bf/ca596c00d7a6eaaf8ef2f66c9b23cd312527f483073c43ffac7843049cb4/mysql_connector_python-9.4.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:3892f20472e13e63b1fb4983f454771dd29f211b09724e69a9750e299542f2f8", size = 18369494, upload-time = "2025-07-22T07:57:49.714Z" }, - { url = "https://files.pythonhosted.org/packages/25/14/6510a11ed9f80d77f743dc207773092c4ab78d5efa454b39b48480315d85/mysql_connector_python-9.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d3e87142103d71c4df647ece30f98e85e826652272ed1c74822b56f6acdc38e7", size = 33516187, upload-time = "2025-07-22T07:57:55.294Z" }, - { url = "https://files.pythonhosted.org/packages/16/a8/4f99d80f1cf77733ce9a44b6adb7f0dd7079e7afa51ca4826515ef0c3e16/mysql_connector_python-9.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:b27fcd403436fe83bafb2fe7fcb785891e821e639275c4ad3b3bd1e25f533206", size = 33917818, upload-time = "2025-07-22T07:58:00.523Z" }, - { url = "https://files.pythonhosted.org/packages/15/9c/127f974ca9d5ee25373cb5433da06bb1f36e05f2a6b7436da1fe9c6346b0/mysql_connector_python-9.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd6ff5afb9c324b0bbeae958c93156cce4168c743bf130faf224d52818d1f0ee", size = 16392378, upload-time = "2025-07-22T07:58:04.669Z" }, - { url = "https://files.pythonhosted.org/packages/03/7c/a543fb17c2dfa6be8548dfdc5879a0c7924cd5d1c79056c48472bb8fe858/mysql_connector_python-9.4.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4efa3898a24aba6a4bfdbf7c1f5023c78acca3150d72cc91199cca2ccd22f76f", size = 17503693, upload-time = "2025-07-22T07:58:08.96Z" }, - { url = "https://files.pythonhosted.org/packages/cb/6e/c22fbee05f5cfd6ba76155b6d45f6261d8d4c1e36e23de04e7f25fbd01a4/mysql_connector_python-9.4.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:665c13e7402235162e5b7a2bfdee5895192121b64ea455c90a81edac6a48ede5", size = 18371987, upload-time = "2025-07-22T07:58:13.273Z" }, - { url = "https://files.pythonhosted.org/packages/b4/fd/f426f5f35a3d3180c7f84d1f96b4631be2574df94ca1156adab8618b236c/mysql_connector_python-9.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:815aa6cad0f351c1223ef345781a538f2e5e44ef405fdb3851eb322bd9c4ca2b", size = 33516214, upload-time = "2025-07-22T07:58:18.967Z" }, - { url = "https://files.pythonhosted.org/packages/45/5a/1b053ae80b43cd3ccebc4bb99a98826969b3b0f8adebdcc2530750ad76ed/mysql_connector_python-9.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b3436a2c8c0ec7052932213e8d01882e6eb069dbab33402e685409084b133a1c", size = 33918565, upload-time = "2025-07-22T07:58:25.28Z" }, - { url = "https://files.pythonhosted.org/packages/cb/69/36b989de675d98ba8ff7d45c96c30c699865c657046f2e32db14e78f13d9/mysql_connector_python-9.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:57b0c224676946b70548c56798d5023f65afa1ba5b8ac9f04a143d27976c7029", size = 16392563, upload-time = "2025-07-22T07:58:29.623Z" }, - { url = "https://files.pythonhosted.org/packages/79/e2/13036479cd1070d1080cee747de6c96bd6fbb021b736dd3ccef2b19016c8/mysql_connector_python-9.4.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:fde3bbffb5270a4b02077029914e6a9d2ec08f67d8375b4111432a2778e7540b", size = 17503749, upload-time = "2025-07-22T07:58:33.649Z" }, - { url = "https://files.pythonhosted.org/packages/31/df/b89e6551b91332716d384dcc3223e1f8065902209dcd9e477a3df80154f7/mysql_connector_python-9.4.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:25f77ad7d845df3b5a5a3a6a8d1fed68248dc418a6938a371d1ddaaab6b9a8e3", size = 18372145, upload-time = "2025-07-22T07:58:37.384Z" }, - { url = "https://files.pythonhosted.org/packages/07/bd/af0de40a01d5cb4df19318cc018e64666f2b7fa89bffa1ab5b35337aae2c/mysql_connector_python-9.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:227dd420c71e6d4788d52d98f298e563f16b6853577e5ade4bd82d644257c812", size = 33516503, upload-time = "2025-07-22T07:58:41.987Z" }, - { url = "https://files.pythonhosted.org/packages/d1/9b/712053216fcbe695e519ecb1035ffd767c2de9f51ccba15078537c99d6fa/mysql_connector_python-9.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5163381a312d38122eded2197eb5cd7ccf1a5c5881d4e7a6de10d6ea314d088e", size = 33918904, upload-time = "2025-07-22T07:58:46.796Z" }, - { url = "https://files.pythonhosted.org/packages/64/15/cbd996d425c59811849f3c1d1b1dae089a1ae18c4acd4d8de2b847b772df/mysql_connector_python-9.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:c727cb1f82b40c9aaa7a15ab5cf0a7f87c5d8dce32eab5ff2530a4aa6054e7df", size = 16392566, upload-time = "2025-07-22T07:58:50.223Z" }, - { url = "https://files.pythonhosted.org/packages/36/34/b6165e15fd45a8deb00932d8e7d823de7650270873b4044c4db6688e1d8f/mysql_connector_python-9.4.0-py2.py3-none-any.whl", hash = "sha256:56e679169c704dab279b176fab2a9ee32d2c632a866c0f7cd48a8a1e2cf802c4", size = 406574, upload-time = "2025-07-22T07:59:08.394Z" }, +version = "9.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/33/b332b001bc8c5ee09255a0d4b09a254da674450edd6a3e5228b245ca82a0/mysql_connector_python-9.5.0.tar.gz", hash = "sha256:92fb924285a86d8c146ebd63d94f9eaefa548da7813bc46271508fdc6cc1d596", size = 12251077, upload-time = "2025-10-22T09:05:45.423Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/5d/30210fcf7ba98d1e03de0c47a58218ab5313d82f2e01ae53b47f45c36b9d/mysql_connector_python-9.5.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:77d14c9fde90726de22443e8c5ba0912a4ebb632cc1ade52a349dacbac47b140", size = 17579085, upload-time = "2025-10-22T09:01:27.388Z" }, + { url = "https://files.pythonhosted.org/packages/77/92/ea79a0875436665330a81e82b4b73a6d52aebcfb1cf4d97f4ad4bd4dedf5/mysql_connector_python-9.5.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:4d603b55de310b9689bb3cb5e57fe97e98756e36d62f8f308f132f2c724f62b8", size = 18445098, upload-time = "2025-10-22T09:01:29.721Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f2/4578b5093f46985c659035e880e70e8b0bed44d4a59ad4e83df5d49b9c69/mysql_connector_python-9.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:48ffa71ba748afaae5c45ed9a085a72604368ce611fe81c3fdc146ef60181d51", size = 33660118, upload-time = "2025-10-22T09:01:32.048Z" }, + { url = "https://files.pythonhosted.org/packages/c5/60/63135610ae0cee1260ce64874c1ddbf08e7fb560c21a3d9cce88b0ddc266/mysql_connector_python-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:77c71df48293d3c08713ff7087cf483804c8abf41a4bb4aefea7317b752c8e9a", size = 34096212, upload-time = "2025-10-22T09:01:36.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b1/78dc693552cfbb45076b3638ca4c402fae52209af8f276370d02d78367a0/mysql_connector_python-9.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:4f8d2d9d586c34dc9508a44d19cf30ccafabbbd12d7f8ab58da3af118636843c", size = 16512395, upload-time = "2025-10-22T09:01:38.602Z" }, + { url = "https://files.pythonhosted.org/packages/05/03/77347d58b0027ce93a41858477e08422e498c6ebc24348b1f725ed7a67ae/mysql_connector_python-9.5.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:653e70cd10cf2d18dd828fae58dff5f0f7a5cf7e48e244f2093314dddf84a4b9", size = 17578984, upload-time = "2025-10-22T09:01:41.213Z" }, + { url = "https://files.pythonhosted.org/packages/a5/bb/0f45c7ee55ebc56d6731a593d85c0e7f25f83af90a094efebfd5be9fe010/mysql_connector_python-9.5.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:5add93f60b3922be71ea31b89bc8a452b876adbb49262561bd559860dae96b3f", size = 18445067, upload-time = "2025-10-22T09:01:43.215Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ec/054de99d4aa50d851a37edca9039280f7194cc1bfd30aab38f5bd6977ebe/mysql_connector_python-9.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:20950a5e44896c03e3dc93ceb3a5e9b48c9acae18665ca6e13249b3fe5b96811", size = 33668029, upload-time = "2025-10-22T09:01:45.74Z" }, + { url = "https://files.pythonhosted.org/packages/90/a2/e6095dc3a7ad5c959fe4a65681db63af131f572e57cdffcc7816bc84e3ad/mysql_connector_python-9.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7fdd3205b9242c284019310fa84437f3357b13f598e3f9b5d80d337d4a6406b8", size = 34101687, upload-time = "2025-10-22T09:01:48.462Z" }, + { url = "https://files.pythonhosted.org/packages/9c/88/bc13c33fca11acaf808bd1809d8602d78f5bb84f7b1e7b1a288c383a14fd/mysql_connector_python-9.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:c021d8b0830958b28712c70c53b206b4cf4766948dae201ea7ca588a186605e0", size = 16511749, upload-time = "2025-10-22T09:01:51.032Z" }, + { url = "https://files.pythonhosted.org/packages/02/89/167ebee82f4b01ba7339c241c3cc2518886a2be9f871770a1efa81b940a0/mysql_connector_python-9.5.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a72c2ef9d50b84f3c567c31b3bf30901af740686baa2a4abead5f202e0b7ea61", size = 17581904, upload-time = "2025-10-22T09:01:53.21Z" }, + { url = "https://files.pythonhosted.org/packages/67/46/630ca969ce10b30fdc605d65dab4a6157556d8cc3b77c724f56c2d83cb79/mysql_connector_python-9.5.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bd9ba5a946cfd3b3b2688a75135357e862834b0321ed936fd968049be290872b", size = 18448195, upload-time = "2025-10-22T09:01:55.378Z" }, + { url = "https://files.pythonhosted.org/packages/f6/87/4c421f41ad169d8c9065ad5c46673c7af889a523e4899c1ac1d6bfd37262/mysql_connector_python-9.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5ef7accbdf8b5f6ec60d2a1550654b7e27e63bf6f7b04020d5fb4191fb02bc4d", size = 33668638, upload-time = "2025-10-22T09:01:57.896Z" }, + { url = "https://files.pythonhosted.org/packages/a6/01/67cf210d50bfefbb9224b9a5c465857c1767388dade1004c903c8e22a991/mysql_connector_python-9.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a6e0a4a0274d15e3d4c892ab93f58f46431222117dba20608178dfb2cc4d5fd8", size = 34102899, upload-time = "2025-10-22T09:02:00.291Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ef/3d1a67d503fff38cc30e11d111cf28f0976987fb175f47b10d44494e1080/mysql_connector_python-9.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:b6c69cb37600b7e22f476150034e2afbd53342a175e20aea887f8158fc5e3ff6", size = 16512684, upload-time = "2025-10-22T09:02:02.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/18/f221aeac49ce94ac119a427afbd51fe1629d48745b571afc0de49647b528/mysql_connector_python-9.5.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:1f5f7346b0d5edb2e994c1bd77b3f5eed88b0ca368ad6788d1012c7e56d7bf68", size = 17581933, upload-time = "2025-10-22T09:02:04.396Z" }, + { url = "https://files.pythonhosted.org/packages/de/8e/14d44db7353350006a12e46d61c3a995bba06acd7547fc78f9bb32611e0c/mysql_connector_python-9.5.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:07bf52591b4215cb4318b4617c327a6d84c31978c11e3255f01a627bcda2618e", size = 18448446, upload-time = "2025-10-22T09:02:06.399Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f5/ab306f292a99bff3544ff44ad53661a031dc1a11e5b1ad64b9e5b5290ef9/mysql_connector_python-9.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:8972c1f960b30d487f34f9125ec112ea2b3200bd02c53e5e32ee7a43be6d64c1", size = 33668933, upload-time = "2025-10-22T09:02:08.785Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/d146d2642552ebb5811cf551f06aca7da536c80b18fb6c75bdbc29723388/mysql_connector_python-9.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:f6d32d7aa514d2f6f8709ba1e018314f82ab2acea2e6af30d04c1906fe9171b9", size = 34103214, upload-time = "2025-10-22T09:02:11.657Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f8/5e88e5eda1fe58f7d146b73744f691d85dce76fb42e7ce3de53e49911da3/mysql_connector_python-9.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:edd47048eb65c196b28aa9d2c0c6a017d8ca084a9a7041cd317301c829eb5a05", size = 16512689, upload-time = "2025-10-22T09:02:14.167Z" }, + { url = "https://files.pythonhosted.org/packages/14/42/52bef145028af1b8e633eb77773278a04b2cd9f824117209aba093018445/mysql_connector_python-9.5.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:6effda35df1a96d9a096f04468d40f2324ea36b34d0e9632e81daae8be97b308", size = 17581903, upload-time = "2025-10-22T09:02:16.441Z" }, + { url = "https://files.pythonhosted.org/packages/f4/a6/bd800b42bde86bf2e9468dfabcbd7538c66daff9d1a9fc97d2cc897f96fa/mysql_connector_python-9.5.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:fd057bd042464eedbf5337d1ceea7f2a4ab075a1cf6d1d62ffd5184966a656dd", size = 18448394, upload-time = "2025-10-22T09:02:18.436Z" }, + { url = "https://files.pythonhosted.org/packages/4a/21/a1a3247775d0dfee094499cb915560755eaa1013ac3b03e34a98b0e16e49/mysql_connector_python-9.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:2797dd7bbefb1d1669d984cfb284ea6b34401bbd9c1b3bf84e646d0bd3a82197", size = 33669845, upload-time = "2025-10-22T09:02:20.966Z" }, + { url = "https://files.pythonhosted.org/packages/58/b7/dcab48349ab8abafd6f40f113101549e0cf107e43dd9c7e1fae79799604b/mysql_connector_python-9.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:a5fff063ed48281b7374a4da6b9ef4293d390c153f79b1589ee547ea08c92310", size = 34104103, upload-time = "2025-10-22T09:02:23.469Z" }, + { url = "https://files.pythonhosted.org/packages/21/3a/be129764fe5f5cd89a5aa3f58e7a7471284715f4af71097a980d24ebec0a/mysql_connector_python-9.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:56104693478fd447886c470a6d0558ded0fe2577df44c18232a6af6a2bbdd3e9", size = 17001255, upload-time = "2025-10-22T09:02:25.765Z" }, + { url = "https://files.pythonhosted.org/packages/95/e1/45373c06781340c7b74fe9b88b85278ac05321889a307eaa5be079a997d4/mysql_connector_python-9.5.0-py2.py3-none-any.whl", hash = "sha256:ace137b88eb6fdafa1e5b2e03ac76ce1b8b1844b3a4af1192a02ae7c1a45bdee", size = 479047, upload-time = "2025-10-22T09:02:27.809Z" }, ] [[package]] @@ -407,7 +411,7 @@ wheels = [ [[package]] name = "pydantic" -version = "2.12.3" +version = "2.12.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -415,194 +419,198 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/1e/4f0a3233767010308f2fd6bd0814597e3f63f1dc98304a9112b8759df4ff/pydantic-2.12.3.tar.gz", hash = "sha256:1da1c82b0fc140bb0103bc1441ffe062154c8d38491189751ee00fd8ca65ce74", size = 819383, upload-time = "2025-10-17T15:04:21.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/6b/83661fa77dcefa195ad5f8cd9af3d1a7450fd57cc883ad04d65446ac2029/pydantic-2.12.3-py3-none-any.whl", hash = "sha256:6986454a854bc3bc6e5443e1369e06a3a456af9d339eda45510f517d9ea5c6bf", size = 462431, upload-time = "2025-10-17T15:04:19.346Z" }, + { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" }, ] [[package]] name = "pydantic-core" -version = "2.41.4" +version = "2.41.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/18/d0944e8eaaa3efd0a91b0f1fc537d3be55ad35091b6a87638211ba691964/pydantic_core-2.41.4.tar.gz", hash = "sha256:70e47929a9d4a1905a67e4b687d5946026390568a8e952b92824118063cee4d5", size = 457557, upload-time = "2025-10-14T10:23:47.909Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/3d/9b8ca77b0f76fcdbf8bc6b72474e264283f461284ca84ac3fde570c6c49a/pydantic_core-2.41.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2442d9a4d38f3411f22eb9dd0912b7cbf4b7d5b6c92c4173b75d3e1ccd84e36e", size = 2111197, upload-time = "2025-10-14T10:19:43.303Z" }, - { url = "https://files.pythonhosted.org/packages/59/92/b7b0fe6ed4781642232755cb7e56a86e2041e1292f16d9ae410a0ccee5ac/pydantic_core-2.41.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:30a9876226dda131a741afeab2702e2d127209bde3c65a2b8133f428bc5d006b", size = 1917909, upload-time = "2025-10-14T10:19:45.194Z" }, - { url = "https://files.pythonhosted.org/packages/52/8c/3eb872009274ffa4fb6a9585114e161aa1a0915af2896e2d441642929fe4/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d55bbac04711e2980645af68b97d445cdbcce70e5216de444a6c4b6943ebcccd", size = 1969905, upload-time = "2025-10-14T10:19:46.567Z" }, - { url = "https://files.pythonhosted.org/packages/f4/21/35adf4a753bcfaea22d925214a0c5b880792e3244731b3f3e6fec0d124f7/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e1d778fb7849a42d0ee5927ab0f7453bf9f85eef8887a546ec87db5ddb178945", size = 2051938, upload-time = "2025-10-14T10:19:48.237Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d0/cdf7d126825e36d6e3f1eccf257da8954452934ede275a8f390eac775e89/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b65077a4693a98b90ec5ad8f203ad65802a1b9b6d4a7e48066925a7e1606706", size = 2250710, upload-time = "2025-10-14T10:19:49.619Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1c/af1e6fd5ea596327308f9c8d1654e1285cc3d8de0d584a3c9d7705bf8a7c/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62637c769dee16eddb7686bf421be48dfc2fae93832c25e25bc7242e698361ba", size = 2367445, upload-time = "2025-10-14T10:19:51.269Z" }, - { url = "https://files.pythonhosted.org/packages/d3/81/8cece29a6ef1b3a92f956ea6da6250d5b2d2e7e4d513dd3b4f0c7a83dfea/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfe3aa529c8f501babf6e502936b9e8d4698502b2cfab41e17a028d91b1ac7b", size = 2072875, upload-time = "2025-10-14T10:19:52.671Z" }, - { url = "https://files.pythonhosted.org/packages/e3/37/a6a579f5fc2cd4d5521284a0ab6a426cc6463a7b3897aeb95b12f1ba607b/pydantic_core-2.41.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca2322da745bf2eeb581fc9ea3bbb31147702163ccbcbf12a3bb630e4bf05e1d", size = 2191329, upload-time = "2025-10-14T10:19:54.214Z" }, - { url = "https://files.pythonhosted.org/packages/ae/03/505020dc5c54ec75ecba9f41119fd1e48f9e41e4629942494c4a8734ded1/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e8cd3577c796be7231dcf80badcf2e0835a46665eaafd8ace124d886bab4d700", size = 2151658, upload-time = "2025-10-14T10:19:55.843Z" }, - { url = "https://files.pythonhosted.org/packages/cb/5d/2c0d09fb53aa03bbd2a214d89ebfa6304be7df9ed86ee3dc7770257f41ee/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:1cae8851e174c83633f0833e90636832857297900133705ee158cf79d40f03e6", size = 2316777, upload-time = "2025-10-14T10:19:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/ea/4b/c2c9c8f5e1f9c864b57d08539d9d3db160e00491c9f5ee90e1bfd905e644/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a26d950449aae348afe1ac8be5525a00ae4235309b729ad4d3399623125b43c9", size = 2320705, upload-time = "2025-10-14T10:19:59.016Z" }, - { url = "https://files.pythonhosted.org/packages/28/c3/a74c1c37f49c0a02c89c7340fafc0ba816b29bd495d1a31ce1bdeacc6085/pydantic_core-2.41.4-cp310-cp310-win32.whl", hash = "sha256:0cf2a1f599efe57fa0051312774280ee0f650e11152325e41dfd3018ef2c1b57", size = 1975464, upload-time = "2025-10-14T10:20:00.581Z" }, - { url = "https://files.pythonhosted.org/packages/d6/23/5dd5c1324ba80303368f7569e2e2e1a721c7d9eb16acb7eb7b7f85cb1be2/pydantic_core-2.41.4-cp310-cp310-win_amd64.whl", hash = "sha256:a8c2e340d7e454dc3340d3d2e8f23558ebe78c98aa8f68851b04dcb7bc37abdc", size = 2024497, upload-time = "2025-10-14T10:20:03.018Z" }, - { url = "https://files.pythonhosted.org/packages/62/4c/f6cbfa1e8efacd00b846764e8484fe173d25b8dab881e277a619177f3384/pydantic_core-2.41.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:28ff11666443a1a8cf2a044d6a545ebffa8382b5f7973f22c36109205e65dc80", size = 2109062, upload-time = "2025-10-14T10:20:04.486Z" }, - { url = "https://files.pythonhosted.org/packages/21/f8/40b72d3868896bfcd410e1bd7e516e762d326201c48e5b4a06446f6cf9e8/pydantic_core-2.41.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61760c3925d4633290292bad462e0f737b840508b4f722247d8729684f6539ae", size = 1916301, upload-time = "2025-10-14T10:20:06.857Z" }, - { url = "https://files.pythonhosted.org/packages/94/4d/d203dce8bee7faeca791671c88519969d98d3b4e8f225da5b96dad226fc8/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eae547b7315d055b0de2ec3965643b0ab82ad0106a7ffd29615ee9f266a02827", size = 1968728, upload-time = "2025-10-14T10:20:08.353Z" }, - { url = "https://files.pythonhosted.org/packages/65/f5/6a66187775df87c24d526985b3a5d78d861580ca466fbd9d4d0e792fcf6c/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef9ee5471edd58d1fcce1c80ffc8783a650e3e3a193fe90d52e43bb4d87bff1f", size = 2050238, upload-time = "2025-10-14T10:20:09.766Z" }, - { url = "https://files.pythonhosted.org/packages/5e/b9/78336345de97298cf53236b2f271912ce11f32c1e59de25a374ce12f9cce/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15dd504af121caaf2c95cb90c0ebf71603c53de98305621b94da0f967e572def", size = 2249424, upload-time = "2025-10-14T10:20:11.732Z" }, - { url = "https://files.pythonhosted.org/packages/99/bb/a4584888b70ee594c3d374a71af5075a68654d6c780369df269118af7402/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a926768ea49a8af4d36abd6a8968b8790f7f76dd7cbd5a4c180db2b4ac9a3a2", size = 2366047, upload-time = "2025-10-14T10:20:13.647Z" }, - { url = "https://files.pythonhosted.org/packages/5f/8d/17fc5de9d6418e4d2ae8c675f905cdafdc59d3bf3bf9c946b7ab796a992a/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916b9b7d134bff5440098a4deb80e4cb623e68974a87883299de9124126c2a8", size = 2071163, upload-time = "2025-10-14T10:20:15.307Z" }, - { url = "https://files.pythonhosted.org/packages/54/e7/03d2c5c0b8ed37a4617430db68ec5e7dbba66358b629cd69e11b4d564367/pydantic_core-2.41.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cf90535979089df02e6f17ffd076f07237efa55b7343d98760bde8743c4b265", size = 2190585, upload-time = "2025-10-14T10:20:17.3Z" }, - { url = "https://files.pythonhosted.org/packages/be/fc/15d1c9fe5ad9266a5897d9b932b7f53d7e5cfc800573917a2c5d6eea56ec/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7533c76fa647fade2d7ec75ac5cc079ab3f34879626dae5689b27790a6cf5a5c", size = 2150109, upload-time = "2025-10-14T10:20:19.143Z" }, - { url = "https://files.pythonhosted.org/packages/26/ef/e735dd008808226c83ba56972566138665b71477ad580fa5a21f0851df48/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:37e516bca9264cbf29612539801ca3cd5d1be465f940417b002905e6ed79d38a", size = 2315078, upload-time = "2025-10-14T10:20:20.742Z" }, - { url = "https://files.pythonhosted.org/packages/90/00/806efdcf35ff2ac0f938362350cd9827b8afb116cc814b6b75cf23738c7c/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0c19cb355224037c83642429b8ce261ae108e1c5fbf5c028bac63c77b0f8646e", size = 2318737, upload-time = "2025-10-14T10:20:22.306Z" }, - { url = "https://files.pythonhosted.org/packages/41/7e/6ac90673fe6cb36621a2283552897838c020db343fa86e513d3f563b196f/pydantic_core-2.41.4-cp311-cp311-win32.whl", hash = "sha256:09c2a60e55b357284b5f31f5ab275ba9f7f70b7525e18a132ec1f9160b4f1f03", size = 1974160, upload-time = "2025-10-14T10:20:23.817Z" }, - { url = "https://files.pythonhosted.org/packages/e0/9d/7c5e24ee585c1f8b6356e1d11d40ab807ffde44d2db3b7dfd6d20b09720e/pydantic_core-2.41.4-cp311-cp311-win_amd64.whl", hash = "sha256:711156b6afb5cb1cb7c14a2cc2c4a8b4c717b69046f13c6b332d8a0a8f41ca3e", size = 2021883, upload-time = "2025-10-14T10:20:25.48Z" }, - { url = "https://files.pythonhosted.org/packages/33/90/5c172357460fc28b2871eb4a0fb3843b136b429c6fa827e4b588877bf115/pydantic_core-2.41.4-cp311-cp311-win_arm64.whl", hash = "sha256:6cb9cf7e761f4f8a8589a45e49ed3c0d92d1d696a45a6feaee8c904b26efc2db", size = 1968026, upload-time = "2025-10-14T10:20:27.039Z" }, - { url = "https://files.pythonhosted.org/packages/e9/81/d3b3e95929c4369d30b2a66a91db63c8ed0a98381ae55a45da2cd1cc1288/pydantic_core-2.41.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ab06d77e053d660a6faaf04894446df7b0a7e7aba70c2797465a0a1af00fc887", size = 2099043, upload-time = "2025-10-14T10:20:28.561Z" }, - { url = "https://files.pythonhosted.org/packages/58/da/46fdac49e6717e3a94fc9201403e08d9d61aa7a770fab6190b8740749047/pydantic_core-2.41.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c53ff33e603a9c1179a9364b0a24694f183717b2e0da2b5ad43c316c956901b2", size = 1910699, upload-time = "2025-10-14T10:20:30.217Z" }, - { url = "https://files.pythonhosted.org/packages/1e/63/4d948f1b9dd8e991a5a98b77dd66c74641f5f2e5225fee37994b2e07d391/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:304c54176af2c143bd181d82e77c15c41cbacea8872a2225dd37e6544dce9999", size = 1952121, upload-time = "2025-10-14T10:20:32.246Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a7/e5fc60a6f781fc634ecaa9ecc3c20171d238794cef69ae0af79ac11b89d7/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:025ba34a4cf4fb32f917d5d188ab5e702223d3ba603be4d8aca2f82bede432a4", size = 2041590, upload-time = "2025-10-14T10:20:34.332Z" }, - { url = "https://files.pythonhosted.org/packages/70/69/dce747b1d21d59e85af433428978a1893c6f8a7068fa2bb4a927fba7a5ff/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9f5f30c402ed58f90c70e12eff65547d3ab74685ffe8283c719e6bead8ef53f", size = 2219869, upload-time = "2025-10-14T10:20:35.965Z" }, - { url = "https://files.pythonhosted.org/packages/83/6a/c070e30e295403bf29c4df1cb781317b6a9bac7cd07b8d3acc94d501a63c/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd96e5d15385d301733113bcaa324c8bcf111275b7675a9c6e88bfb19fc05e3b", size = 2345169, upload-time = "2025-10-14T10:20:37.627Z" }, - { url = "https://files.pythonhosted.org/packages/f0/83/06d001f8043c336baea7fd202a9ac7ad71f87e1c55d8112c50b745c40324/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98f348cbb44fae6e9653c1055db7e29de67ea6a9ca03a5fa2c2e11a47cff0e47", size = 2070165, upload-time = "2025-10-14T10:20:39.246Z" }, - { url = "https://files.pythonhosted.org/packages/14/0a/e567c2883588dd12bcbc110232d892cf385356f7c8a9910311ac997ab715/pydantic_core-2.41.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec22626a2d14620a83ca583c6f5a4080fa3155282718b6055c2ea48d3ef35970", size = 2189067, upload-time = "2025-10-14T10:20:41.015Z" }, - { url = "https://files.pythonhosted.org/packages/f4/1d/3d9fca34273ba03c9b1c5289f7618bc4bd09c3ad2289b5420481aa051a99/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a95d4590b1f1a43bf33ca6d647b990a88f4a3824a8c4572c708f0b45a5290ed", size = 2132997, upload-time = "2025-10-14T10:20:43.106Z" }, - { url = "https://files.pythonhosted.org/packages/52/70/d702ef7a6cd41a8afc61f3554922b3ed8d19dd54c3bd4bdbfe332e610827/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:f9672ab4d398e1b602feadcffcdd3af44d5f5e6ddc15bc7d15d376d47e8e19f8", size = 2307187, upload-time = "2025-10-14T10:20:44.849Z" }, - { url = "https://files.pythonhosted.org/packages/68/4c/c06be6e27545d08b802127914156f38d10ca287a9e8489342793de8aae3c/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:84d8854db5f55fead3b579f04bda9a36461dab0730c5d570e1526483e7bb8431", size = 2305204, upload-time = "2025-10-14T10:20:46.781Z" }, - { url = "https://files.pythonhosted.org/packages/b0/e5/35ae4919bcd9f18603419e23c5eaf32750224a89d41a8df1a3704b69f77e/pydantic_core-2.41.4-cp312-cp312-win32.whl", hash = "sha256:9be1c01adb2ecc4e464392c36d17f97e9110fbbc906bcbe1c943b5b87a74aabd", size = 1972536, upload-time = "2025-10-14T10:20:48.39Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c2/49c5bb6d2a49eb2ee3647a93e3dae7080c6409a8a7558b075027644e879c/pydantic_core-2.41.4-cp312-cp312-win_amd64.whl", hash = "sha256:d682cf1d22bab22a5be08539dca3d1593488a99998f9f412137bc323179067ff", size = 2031132, upload-time = "2025-10-14T10:20:50.421Z" }, - { url = "https://files.pythonhosted.org/packages/06/23/936343dbcba6eec93f73e95eb346810fc732f71ba27967b287b66f7b7097/pydantic_core-2.41.4-cp312-cp312-win_arm64.whl", hash = "sha256:833eebfd75a26d17470b58768c1834dfc90141b7afc6eb0429c21fc5a21dcfb8", size = 1969483, upload-time = "2025-10-14T10:20:52.35Z" }, - { url = "https://files.pythonhosted.org/packages/13/d0/c20adabd181a029a970738dfe23710b52a31f1258f591874fcdec7359845/pydantic_core-2.41.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:85e050ad9e5f6fe1004eec65c914332e52f429bc0ae12d6fa2092407a462c746", size = 2105688, upload-time = "2025-10-14T10:20:54.448Z" }, - { url = "https://files.pythonhosted.org/packages/00/b6/0ce5c03cec5ae94cca220dfecddc453c077d71363b98a4bbdb3c0b22c783/pydantic_core-2.41.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7393f1d64792763a48924ba31d1e44c2cfbc05e3b1c2c9abb4ceeadd912cced", size = 1910807, upload-time = "2025-10-14T10:20:56.115Z" }, - { url = "https://files.pythonhosted.org/packages/68/3e/800d3d02c8beb0b5c069c870cbb83799d085debf43499c897bb4b4aaff0d/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94dab0940b0d1fb28bcab847adf887c66a27a40291eedf0b473be58761c9799a", size = 1956669, upload-time = "2025-10-14T10:20:57.874Z" }, - { url = "https://files.pythonhosted.org/packages/60/a4/24271cc71a17f64589be49ab8bd0751f6a0a03046c690df60989f2f95c2c/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:de7c42f897e689ee6f9e93c4bec72b99ae3b32a2ade1c7e4798e690ff5246e02", size = 2051629, upload-time = "2025-10-14T10:21:00.006Z" }, - { url = "https://files.pythonhosted.org/packages/68/de/45af3ca2f175d91b96bfb62e1f2d2f1f9f3b14a734afe0bfeff079f78181/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:664b3199193262277b8b3cd1e754fb07f2c6023289c815a1e1e8fb415cb247b1", size = 2224049, upload-time = "2025-10-14T10:21:01.801Z" }, - { url = "https://files.pythonhosted.org/packages/af/8f/ae4e1ff84672bf869d0a77af24fd78387850e9497753c432875066b5d622/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95b253b88f7d308b1c0b417c4624f44553ba4762816f94e6986819b9c273fb2", size = 2342409, upload-time = "2025-10-14T10:21:03.556Z" }, - { url = "https://files.pythonhosted.org/packages/18/62/273dd70b0026a085c7b74b000394e1ef95719ea579c76ea2f0cc8893736d/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1351f5bbdbbabc689727cb91649a00cb9ee7203e0a6e54e9f5ba9e22e384b84", size = 2069635, upload-time = "2025-10-14T10:21:05.385Z" }, - { url = "https://files.pythonhosted.org/packages/30/03/cf485fff699b4cdaea469bc481719d3e49f023241b4abb656f8d422189fc/pydantic_core-2.41.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1affa4798520b148d7182da0615d648e752de4ab1a9566b7471bc803d88a062d", size = 2194284, upload-time = "2025-10-14T10:21:07.122Z" }, - { url = "https://files.pythonhosted.org/packages/f9/7e/c8e713db32405dfd97211f2fc0a15d6bf8adb7640f3d18544c1f39526619/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7b74e18052fea4aa8dea2fb7dbc23d15439695da6cbe6cfc1b694af1115df09d", size = 2137566, upload-time = "2025-10-14T10:21:08.981Z" }, - { url = "https://files.pythonhosted.org/packages/04/f7/db71fd4cdccc8b75990f79ccafbbd66757e19f6d5ee724a6252414483fb4/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:285b643d75c0e30abda9dc1077395624f314a37e3c09ca402d4015ef5979f1a2", size = 2316809, upload-time = "2025-10-14T10:21:10.805Z" }, - { url = "https://files.pythonhosted.org/packages/76/63/a54973ddb945f1bca56742b48b144d85c9fc22f819ddeb9f861c249d5464/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f52679ff4218d713b3b33f88c89ccbf3a5c2c12ba665fb80ccc4192b4608dbab", size = 2311119, upload-time = "2025-10-14T10:21:12.583Z" }, - { url = "https://files.pythonhosted.org/packages/f8/03/5d12891e93c19218af74843a27e32b94922195ded2386f7b55382f904d2f/pydantic_core-2.41.4-cp313-cp313-win32.whl", hash = "sha256:ecde6dedd6fff127c273c76821bb754d793be1024bc33314a120f83a3c69460c", size = 1981398, upload-time = "2025-10-14T10:21:14.584Z" }, - { url = "https://files.pythonhosted.org/packages/be/d8/fd0de71f39db91135b7a26996160de71c073d8635edfce8b3c3681be0d6d/pydantic_core-2.41.4-cp313-cp313-win_amd64.whl", hash = "sha256:d081a1f3800f05409ed868ebb2d74ac39dd0c1ff6c035b5162356d76030736d4", size = 2030735, upload-time = "2025-10-14T10:21:16.432Z" }, - { url = "https://files.pythonhosted.org/packages/72/86/c99921c1cf6650023c08bfab6fe2d7057a5142628ef7ccfa9921f2dda1d5/pydantic_core-2.41.4-cp313-cp313-win_arm64.whl", hash = "sha256:f8e49c9c364a7edcbe2a310f12733aad95b022495ef2a8d653f645e5d20c1564", size = 1973209, upload-time = "2025-10-14T10:21:18.213Z" }, - { url = "https://files.pythonhosted.org/packages/36/0d/b5706cacb70a8414396efdda3d72ae0542e050b591119e458e2490baf035/pydantic_core-2.41.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ed97fd56a561f5eb5706cebe94f1ad7c13b84d98312a05546f2ad036bafe87f4", size = 1877324, upload-time = "2025-10-14T10:21:20.363Z" }, - { url = "https://files.pythonhosted.org/packages/de/2d/cba1fa02cfdea72dfb3a9babb067c83b9dff0bbcb198368e000a6b756ea7/pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a870c307bf1ee91fc58a9a61338ff780d01bfae45922624816878dce784095d2", size = 1884515, upload-time = "2025-10-14T10:21:22.339Z" }, - { url = "https://files.pythonhosted.org/packages/07/ea/3df927c4384ed9b503c9cc2d076cf983b4f2adb0c754578dfb1245c51e46/pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d25e97bc1f5f8f7985bdc2335ef9e73843bb561eb1fa6831fdfc295c1c2061cf", size = 2042819, upload-time = "2025-10-14T10:21:26.683Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ee/df8e871f07074250270a3b1b82aad4cd0026b588acd5d7d3eb2fcb1471a3/pydantic_core-2.41.4-cp313-cp313t-win_amd64.whl", hash = "sha256:d405d14bea042f166512add3091c1af40437c2e7f86988f3915fabd27b1e9cd2", size = 1995866, upload-time = "2025-10-14T10:21:28.951Z" }, - { url = "https://files.pythonhosted.org/packages/fc/de/b20f4ab954d6d399499c33ec4fafc46d9551e11dc1858fb7f5dca0748ceb/pydantic_core-2.41.4-cp313-cp313t-win_arm64.whl", hash = "sha256:19f3684868309db5263a11bace3c45d93f6f24afa2ffe75a647583df22a2ff89", size = 1970034, upload-time = "2025-10-14T10:21:30.869Z" }, - { url = "https://files.pythonhosted.org/packages/54/28/d3325da57d413b9819365546eb9a6e8b7cbd9373d9380efd5f74326143e6/pydantic_core-2.41.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e9205d97ed08a82ebb9a307e92914bb30e18cdf6f6b12ca4bedadb1588a0bfe1", size = 2102022, upload-time = "2025-10-14T10:21:32.809Z" }, - { url = "https://files.pythonhosted.org/packages/9e/24/b58a1bc0d834bf1acc4361e61233ee217169a42efbdc15a60296e13ce438/pydantic_core-2.41.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:82df1f432b37d832709fbcc0e24394bba04a01b6ecf1ee87578145c19cde12ac", size = 1905495, upload-time = "2025-10-14T10:21:34.812Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a4/71f759cc41b7043e8ecdaab81b985a9b6cad7cec077e0b92cff8b71ecf6b/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3b4cc4539e055cfa39a3763c939f9d409eb40e85813257dcd761985a108554", size = 1956131, upload-time = "2025-10-14T10:21:36.924Z" }, - { url = "https://files.pythonhosted.org/packages/b0/64/1e79ac7aa51f1eec7c4cda8cbe456d5d09f05fdd68b32776d72168d54275/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b1eb1754fce47c63d2ff57fdb88c351a6c0150995890088b33767a10218eaa4e", size = 2052236, upload-time = "2025-10-14T10:21:38.927Z" }, - { url = "https://files.pythonhosted.org/packages/e9/e3/a3ffc363bd4287b80f1d43dc1c28ba64831f8dfc237d6fec8f2661138d48/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6ab5ab30ef325b443f379ddb575a34969c333004fca5a1daa0133a6ffaad616", size = 2223573, upload-time = "2025-10-14T10:21:41.574Z" }, - { url = "https://files.pythonhosted.org/packages/28/27/78814089b4d2e684a9088ede3790763c64693c3d1408ddc0a248bc789126/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:31a41030b1d9ca497634092b46481b937ff9397a86f9f51bd41c4767b6fc04af", size = 2342467, upload-time = "2025-10-14T10:21:44.018Z" }, - { url = "https://files.pythonhosted.org/packages/92/97/4de0e2a1159cb85ad737e03306717637842c88c7fd6d97973172fb183149/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a44ac1738591472c3d020f61c6df1e4015180d6262ebd39bf2aeb52571b60f12", size = 2063754, upload-time = "2025-10-14T10:21:46.466Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/8cb90ce4b9efcf7ae78130afeb99fd1c86125ccdf9906ef64b9d42f37c25/pydantic_core-2.41.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d72f2b5e6e82ab8f94ea7d0d42f83c487dc159c5240d8f83beae684472864e2d", size = 2196754, upload-time = "2025-10-14T10:21:48.486Z" }, - { url = "https://files.pythonhosted.org/packages/34/3b/ccdc77af9cd5082723574a1cc1bcae7a6acacc829d7c0a06201f7886a109/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c4d1e854aaf044487d31143f541f7aafe7b482ae72a022c664b2de2e466ed0ad", size = 2137115, upload-time = "2025-10-14T10:21:50.63Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ba/e7c7a02651a8f7c52dc2cff2b64a30c313e3b57c7d93703cecea76c09b71/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b568af94267729d76e6ee5ececda4e283d07bbb28e8148bb17adad93d025d25a", size = 2317400, upload-time = "2025-10-14T10:21:52.959Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ba/6c533a4ee8aec6b812c643c49bb3bd88d3f01e3cebe451bb85512d37f00f/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6d55fb8b1e8929b341cc313a81a26e0d48aa3b519c1dbaadec3a6a2b4fcad025", size = 2312070, upload-time = "2025-10-14T10:21:55.419Z" }, - { url = "https://files.pythonhosted.org/packages/22/ae/f10524fcc0ab8d7f96cf9a74c880243576fd3e72bd8ce4f81e43d22bcab7/pydantic_core-2.41.4-cp314-cp314-win32.whl", hash = "sha256:5b66584e549e2e32a1398df11da2e0a7eff45d5c2d9db9d5667c5e6ac764d77e", size = 1982277, upload-time = "2025-10-14T10:21:57.474Z" }, - { url = "https://files.pythonhosted.org/packages/b4/dc/e5aa27aea1ad4638f0c3fb41132f7eb583bd7420ee63204e2d4333a3bbf9/pydantic_core-2.41.4-cp314-cp314-win_amd64.whl", hash = "sha256:557a0aab88664cc552285316809cab897716a372afaf8efdbef756f8b890e894", size = 2024608, upload-time = "2025-10-14T10:21:59.557Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/51d89cc2612bd147198e120a13f150afbf0bcb4615cddb049ab10b81b79e/pydantic_core-2.41.4-cp314-cp314-win_arm64.whl", hash = "sha256:3f1ea6f48a045745d0d9f325989d8abd3f1eaf47dd00485912d1a3a63c623a8d", size = 1967614, upload-time = "2025-10-14T10:22:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c2/472f2e31b95eff099961fa050c376ab7156a81da194f9edb9f710f68787b/pydantic_core-2.41.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6c1fe4c5404c448b13188dd8bd2ebc2bdd7e6727fa61ff481bcc2cca894018da", size = 1876904, upload-time = "2025-10-14T10:22:04.062Z" }, - { url = "https://files.pythonhosted.org/packages/4a/07/ea8eeb91173807ecdae4f4a5f4b150a520085b35454350fc219ba79e66a3/pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:523e7da4d43b113bf8e7b49fa4ec0c35bf4fe66b2230bfc5c13cc498f12c6c3e", size = 1882538, upload-time = "2025-10-14T10:22:06.39Z" }, - { url = "https://files.pythonhosted.org/packages/1e/29/b53a9ca6cd366bfc928823679c6a76c7a4c69f8201c0ba7903ad18ebae2f/pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5729225de81fb65b70fdb1907fcf08c75d498f4a6f15af005aabb1fdadc19dfa", size = 2041183, upload-time = "2025-10-14T10:22:08.812Z" }, - { url = "https://files.pythonhosted.org/packages/c7/3d/f8c1a371ceebcaf94d6dd2d77c6cf4b1c078e13a5837aee83f760b4f7cfd/pydantic_core-2.41.4-cp314-cp314t-win_amd64.whl", hash = "sha256:de2cfbb09e88f0f795fd90cf955858fc2c691df65b1f21f0aa00b99f3fbc661d", size = 1993542, upload-time = "2025-10-14T10:22:11.332Z" }, - { url = "https://files.pythonhosted.org/packages/8a/ac/9fc61b4f9d079482a290afe8d206b8f490e9fd32d4fc03ed4fc698214e01/pydantic_core-2.41.4-cp314-cp314t-win_arm64.whl", hash = "sha256:d34f950ae05a83e0ede899c595f312ca976023ea1db100cd5aa188f7005e3ab0", size = 1973897, upload-time = "2025-10-14T10:22:13.444Z" }, - { url = "https://files.pythonhosted.org/packages/b0/12/5ba58daa7f453454464f92b3ca7b9d7c657d8641c48e370c3ebc9a82dd78/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:a1b2cfec3879afb742a7b0bcfa53e4f22ba96571c9e54d6a3afe1052d17d843b", size = 2122139, upload-time = "2025-10-14T10:22:47.288Z" }, - { url = "https://files.pythonhosted.org/packages/21/fb/6860126a77725c3108baecd10fd3d75fec25191d6381b6eb2ac660228eac/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:d175600d975b7c244af6eb9c9041f10059f20b8bbffec9e33fdd5ee3f67cdc42", size = 1936674, upload-time = "2025-10-14T10:22:49.555Z" }, - { url = "https://files.pythonhosted.org/packages/de/be/57dcaa3ed595d81f8757e2b44a38240ac5d37628bce25fb20d02c7018776/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f184d657fa4947ae5ec9c47bd7e917730fa1cbb78195037e32dcbab50aca5ee", size = 1956398, upload-time = "2025-10-14T10:22:52.19Z" }, - { url = "https://files.pythonhosted.org/packages/2f/1d/679a344fadb9695f1a6a294d739fbd21d71fa023286daeea8c0ed49e7c2b/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed810568aeffed3edc78910af32af911c835cc39ebbfacd1f0ab5dd53028e5c", size = 2138674, upload-time = "2025-10-14T10:22:54.499Z" }, - { url = "https://files.pythonhosted.org/packages/c4/48/ae937e5a831b7c0dc646b2ef788c27cd003894882415300ed21927c21efa/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:4f5d640aeebb438517150fdeec097739614421900e4a08db4a3ef38898798537", size = 2112087, upload-time = "2025-10-14T10:22:56.818Z" }, - { url = "https://files.pythonhosted.org/packages/5e/db/6db8073e3d32dae017da7e0d16a9ecb897d0a4d92e00634916e486097961/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:4a9ab037b71927babc6d9e7fc01aea9e66dc2a4a34dff06ef0724a4049629f94", size = 1920387, upload-time = "2025-10-14T10:22:59.342Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c1/dd3542d072fcc336030d66834872f0328727e3b8de289c662faa04aa270e/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4dab9484ec605c3016df9ad4fd4f9a390bc5d816a3b10c6550f8424bb80b18c", size = 1951495, upload-time = "2025-10-14T10:23:02.089Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c6/db8d13a1f8ab3f1eb08c88bd00fd62d44311e3456d1e85c0e59e0a0376e7/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8a5028425820731d8c6c098ab642d7b8b999758e24acae03ed38a66eca8335", size = 2139008, upload-time = "2025-10-14T10:23:04.539Z" }, - { url = "https://files.pythonhosted.org/packages/5d/d4/912e976a2dd0b49f31c98a060ca90b353f3b73ee3ea2fd0030412f6ac5ec/pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e5ab4fc177dd41536b3c32b2ea11380dd3d4619a385860621478ac2d25ceb00", size = 2106739, upload-time = "2025-10-14T10:23:06.934Z" }, - { url = "https://files.pythonhosted.org/packages/71/f0/66ec5a626c81eba326072d6ee2b127f8c139543f1bf609b4842978d37833/pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d88d0054d3fa11ce936184896bed3c1c5441d6fa483b498fac6a5d0dd6f64a9", size = 1932549, upload-time = "2025-10-14T10:23:09.24Z" }, - { url = "https://files.pythonhosted.org/packages/c4/af/625626278ca801ea0a658c2dcf290dc9f21bb383098e99e7c6a029fccfc0/pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b2a054a8725f05b4b6503357e0ac1c4e8234ad3b0c2ac130d6ffc66f0e170e2", size = 2135093, upload-time = "2025-10-14T10:23:11.626Z" }, - { url = "https://files.pythonhosted.org/packages/20/f6/2fba049f54e0f4975fef66be654c597a1d005320fa141863699180c7697d/pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0d9db5a161c99375a0c68c058e227bee1d89303300802601d76a3d01f74e258", size = 2187971, upload-time = "2025-10-14T10:23:14.437Z" }, - { url = "https://files.pythonhosted.org/packages/0e/80/65ab839a2dfcd3b949202f9d920c34f9de5a537c3646662bdf2f7d999680/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6273ea2c8ffdac7b7fda2653c49682db815aebf4a89243a6feccf5e36c18c347", size = 2147939, upload-time = "2025-10-14T10:23:16.831Z" }, - { url = "https://files.pythonhosted.org/packages/44/58/627565d3d182ce6dfda18b8e1c841eede3629d59c9d7cbc1e12a03aeb328/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:4c973add636efc61de22530b2ef83a65f39b6d6f656df97f678720e20de26caa", size = 2311400, upload-time = "2025-10-14T10:23:19.234Z" }, - { url = "https://files.pythonhosted.org/packages/24/06/8a84711162ad5a5f19a88cead37cca81b4b1f294f46260ef7334ae4f24d3/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b69d1973354758007f46cf2d44a4f3d0933f10b6dc9bf15cf1356e037f6f731a", size = 2316840, upload-time = "2025-10-14T10:23:21.738Z" }, - { url = "https://files.pythonhosted.org/packages/aa/8b/b7bb512a4682a2f7fbfae152a755d37351743900226d29bd953aaf870eaa/pydantic_core-2.41.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3619320641fd212aaf5997b6ca505e97540b7e16418f4a241f44cdf108ffb50d", size = 2149135, upload-time = "2025-10-14T10:23:24.379Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7d/138e902ed6399b866f7cfe4435d22445e16fff888a1c00560d9dc79a780f/pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:491535d45cd7ad7e4a2af4a5169b0d07bebf1adfd164b0368da8aa41e19907a5", size = 2104721, upload-time = "2025-10-14T10:23:26.906Z" }, - { url = "https://files.pythonhosted.org/packages/47/13/0525623cf94627f7b53b4c2034c81edc8491cbfc7c28d5447fa318791479/pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:54d86c0cada6aba4ec4c047d0e348cbad7063b87ae0f005d9f8c9ad04d4a92a2", size = 1931608, upload-time = "2025-10-14T10:23:29.306Z" }, - { url = "https://files.pythonhosted.org/packages/d6/f9/744bc98137d6ef0a233f808bfc9b18cf94624bf30836a18d3b05d08bf418/pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eca1124aced216b2500dc2609eade086d718e8249cb9696660ab447d50a758bd", size = 2132986, upload-time = "2025-10-14T10:23:32.057Z" }, - { url = "https://files.pythonhosted.org/packages/17/c8/629e88920171173f6049386cc71f893dff03209a9ef32b4d2f7e7c264bcf/pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c9024169becccf0cb470ada03ee578d7348c119a0d42af3dcf9eda96e3a247c", size = 2187516, upload-time = "2025-10-14T10:23:34.871Z" }, - { url = "https://files.pythonhosted.org/packages/2e/0f/4f2734688d98488782218ca61bcc118329bf5de05bb7fe3adc7dd79b0b86/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:26895a4268ae5a2849269f4991cdc97236e4b9c010e51137becf25182daac405", size = 2146146, upload-time = "2025-10-14T10:23:37.342Z" }, - { url = "https://files.pythonhosted.org/packages/ed/f2/ab385dbd94a052c62224b99cf99002eee99dbec40e10006c78575aead256/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:ca4df25762cf71308c446e33c9b1fdca2923a3f13de616e2a949f38bf21ff5a8", size = 2311296, upload-time = "2025-10-14T10:23:40.145Z" }, - { url = "https://files.pythonhosted.org/packages/fc/8e/e4f12afe1beeb9823bba5375f8f258df0cc61b056b0195fb1cf9f62a1a58/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5a28fcedd762349519276c36634e71853b4541079cab4acaaac60c4421827308", size = 2315386, upload-time = "2025-10-14T10:23:42.624Z" }, - { url = "https://files.pythonhosted.org/packages/48/f7/925f65d930802e3ea2eb4d5afa4cb8730c8dc0d2cb89a59dc4ed2fcb2d74/pydantic_core-2.41.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c173ddcd86afd2535e2b695217e82191580663a1d1928239f877f5a1649ef39f", size = 2147775, upload-time = "2025-10-14T10:23:45.406Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" }, + { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" }, + { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" }, + { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" }, + { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" }, + { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" }, + { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" }, + { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" }, + { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" }, + { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" }, + { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" }, + { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" }, + { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] [[package]] name = "pymongo" -version = "4.15.3" +version = "4.15.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dnspython" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/7b/a709c85dc716eb85b69f71a4bb375cf1e72758a7e872103f27551243319c/pymongo-4.15.3.tar.gz", hash = "sha256:7a981271347623b5319932796690c2d301668ac3a1965974ac9f5c3b8a22cea5", size = 2470801, upload-time = "2025-10-07T21:57:50.384Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/38/7ba7e7b57ccf2b04b63796c097c35b32339b2cb6e4d851d9dbb84426dc99/pymongo-4.15.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:482ca9b775747562ce1589df10c97a0e62a604ce5addf933e5819dd967c5e23c", size = 811331, upload-time = "2025-10-07T21:55:59.15Z" }, - { url = "https://files.pythonhosted.org/packages/11/36/4bd2aa400a64935b59d68d1c35c168bf61613f1f2bb824757079b2415cda/pymongo-4.15.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7eb497519f42ac89c30919a51f80e68a070cfc2f3b0543cac74833cd45a6b9c", size = 811673, upload-time = "2025-10-07T21:56:00.712Z" }, - { url = "https://files.pythonhosted.org/packages/37/fb/03c3bd14e6eb5236b360cff8598677c4b7b9557eed3021d9b3f6e82de51d/pymongo-4.15.3-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4a0a054e9937ec8fdb465835509b176f6b032851c8648f6a5d1b19932d0eacd6", size = 1185479, upload-time = "2025-10-07T21:56:02.297Z" }, - { url = "https://files.pythonhosted.org/packages/6d/27/b5f21d9a556e31d083bb17d0c026244a604a96f7bdb277fd48dee99415ee/pymongo-4.15.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49fd6e158cf75771b2685a8a221a40ab96010ae34dd116abd06371dc6c38ab60", size = 1203867, upload-time = "2025-10-07T21:56:03.621Z" }, - { url = "https://files.pythonhosted.org/packages/ba/09/ffe1a114d7a39f6746c27a6f5a717b1dc5ea763cb0458a9a679142f623aa/pymongo-4.15.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82a490f1ade4ec6a72068e3676b04c126e3043e69b38ec474a87c6444cf79098", size = 1242537, upload-time = "2025-10-07T21:56:04.973Z" }, - { url = "https://files.pythonhosted.org/packages/af/60/b7968e855284bb67d366dfb50b6a9df4f69676fbbae51f3e647d2dcb12eb/pymongo-4.15.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:982107c667921e896292f4be09c057e2f1a40c645c9bfc724af5dd5fb8398094", size = 1232832, upload-time = "2025-10-07T21:56:06.287Z" }, - { url = "https://files.pythonhosted.org/packages/23/47/763945c63690d5c1a54d1d2ace352ba150b9e49a5cfdf44fb237e092e604/pymongo-4.15.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:45aebbd369ca79b7c46eaea5b04d2e4afca4eda117b68965a07a9da05d774e4d", size = 1200177, upload-time = "2025-10-07T21:56:07.671Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c2/1ace9cf4b88addceb5077e5490238a9e20dc9fef75ae4de146f57f408a06/pymongo-4.15.3-cp310-cp310-win32.whl", hash = "sha256:90ad56bd1d769d2f44af74f0fd0c276512361644a3c636350447994412cbc9a1", size = 798320, upload-time = "2025-10-07T21:56:09.917Z" }, - { url = "https://files.pythonhosted.org/packages/1c/b7/86563ec80fc41f644c813a3625d8b5672fd1d2b52da53727eca766dfc162/pymongo-4.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:8bd6dd736f5d07a825caf52c38916d5452edc0fac7aee43ec67aba6f61c2dbb7", size = 808150, upload-time = "2025-10-07T21:56:11.562Z" }, - { url = "https://files.pythonhosted.org/packages/d5/b3/f136483c3d13224ad0b80ac2b7c8f7adb735a296b5e8c94cfc2415b77d70/pymongo-4.15.3-cp310-cp310-win_arm64.whl", hash = "sha256:300eaf83ad053e51966be1839324341b08eaf880d3dc63ada7942d5912e09c49", size = 800930, upload-time = "2025-10-07T21:56:12.917Z" }, - { url = "https://files.pythonhosted.org/packages/73/04/3dbc426c5868961d8308f19750243f8472f587f5f8a5029ce6953ba74b82/pymongo-4.15.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39a13d8f7141294404ce46dfbabb2f2d17e9b1192456651ae831fa351f86fbeb", size = 865889, upload-time = "2025-10-07T21:56:14.165Z" }, - { url = "https://files.pythonhosted.org/packages/8c/39/7f7652f53dd0eb0c4c3420a175183da757e9c53f9a2bf3ebc589758a1b9e/pymongo-4.15.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:17d13458baf4a6a9f2e787d95adf8ec50d412accb9926a044bd1c41029c323b2", size = 866230, upload-time = "2025-10-07T21:56:15.587Z" }, - { url = "https://files.pythonhosted.org/packages/6a/0b/84e119e6bab7b19cf4fa1ebb9b4c29bf6c0e76521ed8221b44e3f94a3a37/pymongo-4.15.3-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fe4bcb8acfb288e238190397d4a699aeb4adb70e8545a6f4e44f99d4e8096ab1", size = 1429788, upload-time = "2025-10-07T21:56:17.362Z" }, - { url = "https://files.pythonhosted.org/packages/30/39/9905fcb99903de6ac8483114d1c85efe56bc5df735857bdfcc372cf8a3ec/pymongo-4.15.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d09d895c7f08bcbed4d2e96a00e52e9e545ae5a37b32d2dc10099b205a21fc6d", size = 1456758, upload-time = "2025-10-07T21:56:18.841Z" }, - { url = "https://files.pythonhosted.org/packages/08/58/3c3ac32b8d6ebb654083d53f58e4621cd4c7f306b3b85acef667b80acf08/pymongo-4.15.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21c0a95a4db72562fd0805e2f76496bf432ba2e27a5651f4b9c670466260c258", size = 1514666, upload-time = "2025-10-07T21:56:20.488Z" }, - { url = "https://files.pythonhosted.org/packages/19/e2/52f41de224218dc787b7e1187a1ca1a51946dcb979ee553ec917745ccd8d/pymongo-4.15.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89e45d7fa987f4e246cdf43ff001e3f911f73eb19ba9dabc2a6d80df5c97883b", size = 1500703, upload-time = "2025-10-07T21:56:21.874Z" }, - { url = "https://files.pythonhosted.org/packages/34/0d/a5271073339ba6fc8a5f4e3a62baaa5dd8bf35246c37b512317e2a22848e/pymongo-4.15.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1246a82fa6dd73ac2c63aa7e463752d5d1ca91e0c7a23396b78f21273befd3a7", size = 1452013, upload-time = "2025-10-07T21:56:23.526Z" }, - { url = "https://files.pythonhosted.org/packages/a0/3b/f39b721ca0db9f0820e12eeffec84eb87b7502abb13a685226c5434f9618/pymongo-4.15.3-cp311-cp311-win32.whl", hash = "sha256:9483521c03f6017336f54445652ead3145154e8d3ea06418e52cea57fee43292", size = 844461, upload-time = "2025-10-07T21:56:24.867Z" }, - { url = "https://files.pythonhosted.org/packages/12/72/e58b9df862edbf238a1d71fa32749a6eaf30a3f60289602681351c29093a/pymongo-4.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:c57dad9f289d72af1d7c47a444c4d9fa401f951cedbbcc54c7dd0c2107d6d786", size = 859200, upload-time = "2025-10-07T21:56:26.393Z" }, - { url = "https://files.pythonhosted.org/packages/81/8f/64c15df5e87de759412c3b962950561202c9b39e5cc604061e056043e163/pymongo-4.15.3-cp311-cp311-win_arm64.whl", hash = "sha256:2fd3b99520f2bb013960ac29dece1b43f2f1b6d94351ca33ba1b1211ecf79a09", size = 848372, upload-time = "2025-10-07T21:56:27.994Z" }, - { url = "https://files.pythonhosted.org/packages/5b/92/7491a2046b41bfd3641da0a23529c88e27eac67c681de3cd9fbef4113d38/pymongo-4.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bd0497c564b0ae34fb816464ffc09986dd9ca29e2772a0f7af989e472fecc2ad", size = 920953, upload-time = "2025-10-07T21:56:29.737Z" }, - { url = "https://files.pythonhosted.org/packages/ce/0c/98864cbfa8fbc954ae7480c91a35f0dc4e3339dab0c55f669e4dbeac808f/pymongo-4.15.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:292fd5a3f045751a823a54cdea75809b2216a62cc5f74a1a96b337db613d46a8", size = 920690, upload-time = "2025-10-07T21:56:31.094Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a6/7dc8043a10a1c30153be2d6847ab37911b169d53a6b05d21871b35b3de82/pymongo-4.15.3-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:959ef69c5e687b6b749fbf2140c7062abdb4804df013ae0507caabf30cba6875", size = 1690357, upload-time = "2025-10-07T21:56:32.466Z" }, - { url = "https://files.pythonhosted.org/packages/0b/96/3d85da60094d2022217f2849e1b61a79af9d51ed8d05455d7413d68ab88e/pymongo-4.15.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de3bc878c3be54ae41c2cabc9e9407549ed4fec41f4e279c04e840dddd7c630c", size = 1726102, upload-time = "2025-10-07T21:56:33.952Z" }, - { url = "https://files.pythonhosted.org/packages/ac/fd/dfd6ddee0330171f2f52f7e5344c02d25d2dd8dfa95ce0e5e413579f52fd/pymongo-4.15.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07bcc36d11252f24fe671e7e64044d39a13d997b0502c6401161f28cc144f584", size = 1800630, upload-time = "2025-10-07T21:56:35.632Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3b/e19a5f2de227ff720bc76c41d166d508e6fbe1096ba1ad18ade43b790b5e/pymongo-4.15.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b63bac343b79bd209e830aac1f5d9d552ff415f23a924d3e51abbe3041265436", size = 1785478, upload-time = "2025-10-07T21:56:37.39Z" }, - { url = "https://files.pythonhosted.org/packages/75/d2/927c9b1383c6708fc50c3700ecb1c2876e67dde95ad5fb1d29d04e8ac083/pymongo-4.15.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b33d59bf6fa1ca1d7d96d4fccff51e41312358194190d53ef70a84c070f5287e", size = 1718548, upload-time = "2025-10-07T21:56:38.754Z" }, - { url = "https://files.pythonhosted.org/packages/fe/10/a63592d1445f894b18d04865c2d4c235e2261f3d63f31f45ba4fe0486ec4/pymongo-4.15.3-cp312-cp312-win32.whl", hash = "sha256:b3a0ec660d61efb91c16a5962ec937011fe3572c4338216831f102e53d294e5c", size = 891301, upload-time = "2025-10-07T21:56:40.043Z" }, - { url = "https://files.pythonhosted.org/packages/be/ba/a8fdc43044408ed769c83108fa569aa52ee87968bdbf1e2ea142b109c268/pymongo-4.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:f6b0513e5765fdde39f36e6a29a36c67071122b5efa748940ae51075beb5e4bc", size = 910928, upload-time = "2025-10-07T21:56:41.401Z" }, - { url = "https://files.pythonhosted.org/packages/b4/61/d53c17fdfaa9149864ab1fa84436ae218b72c969f00e4c124e017e461ce6/pymongo-4.15.3-cp312-cp312-win_arm64.whl", hash = "sha256:c4fdd8e6eab8ff77c1c8041792b5f760d48508623cd10b50d5639e73f1eec049", size = 896347, upload-time = "2025-10-07T21:56:43.271Z" }, - { url = "https://files.pythonhosted.org/packages/46/a4/e1ce9d408a1c1bcb1554ff61251b108e16cefd7db91b33faa2afc92294de/pymongo-4.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a47a3218f7900f65bf0f36fcd1f2485af4945757360e7e143525db9d715d2010", size = 975329, upload-time = "2025-10-07T21:56:44.674Z" }, - { url = "https://files.pythonhosted.org/packages/74/3c/6796f653d22be43cc0b13c07dbed84133eebbc334ebed4426459b7250163/pymongo-4.15.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09440e78dff397b2f34a624f445ac8eb44c9756a2688b85b3bf344d351d198e1", size = 975129, upload-time = "2025-10-07T21:56:46.104Z" }, - { url = "https://files.pythonhosted.org/packages/88/33/22453dbfe11031e89c9cbdfde6405c03960daaf5da1b4dfdd458891846b5/pymongo-4.15.3-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:97f9babdb98c31676f97d468f7fe2dc49b8a66fb6900effddc4904c1450196c8", size = 1950979, upload-time = "2025-10-07T21:56:47.877Z" }, - { url = "https://files.pythonhosted.org/packages/ba/07/094598e403112e2410a3376fb7845c69e2ec2dfc5ab5cc00b29dc2d26559/pymongo-4.15.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71413cd8f091ae25b1fec3af7c2e531cf9bdb88ce4079470e64835f6a664282a", size = 1995271, upload-time = "2025-10-07T21:56:49.396Z" }, - { url = "https://files.pythonhosted.org/packages/47/9a/29e44f3dee68defc56e50ed7c9d3802ebf967ab81fefb175d8d729c0f276/pymongo-4.15.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:76a8d4de8dceb69f6e06736198ff6f7e1149515ef946f192ff2594d2cc98fc53", size = 2086587, upload-time = "2025-10-07T21:56:50.896Z" }, - { url = "https://files.pythonhosted.org/packages/ff/d5/e9ff16aa57f671349134475b904fd431e7b86e152b01a949aef4f254b2d5/pymongo-4.15.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:77353978be9fc9e5fe56369682efed0aac5f92a2a1570704d62b62a3c9e1a24f", size = 2070201, upload-time = "2025-10-07T21:56:52.425Z" }, - { url = "https://files.pythonhosted.org/packages/d6/a3/820772c0b2bbb671f253cfb0bede4cf694a38fb38134f3993d491e23ec11/pymongo-4.15.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9897a837677e3814873d0572f7e5d53c23ce18e274f3b5b87f05fb6eea22615b", size = 1985260, upload-time = "2025-10-07T21:56:54.56Z" }, - { url = "https://files.pythonhosted.org/packages/6e/7b/365ac821aefad7e8d36a4bc472a94429449aade1ccb7805d9ca754df5081/pymongo-4.15.3-cp313-cp313-win32.whl", hash = "sha256:d66da207ccb0d68c5792eaaac984a0d9c6c8ec609c6bcfa11193a35200dc5992", size = 938122, upload-time = "2025-10-07T21:56:55.993Z" }, - { url = "https://files.pythonhosted.org/packages/80/f3/5ca27e1765fa698c677771a1c0e042ef193e207c15f5d32a21fa5b13d8c3/pymongo-4.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:52f40c4b8c00bc53d4e357fe0de13d031c4cddb5d201e1a027db437e8d2887f8", size = 962610, upload-time = "2025-10-07T21:56:57.397Z" }, - { url = "https://files.pythonhosted.org/packages/48/7c/42f0b6997324023e94939f8f32b9a8dd928499f4b5d7b4412905368686b5/pymongo-4.15.3-cp313-cp313-win_arm64.whl", hash = "sha256:fb384623ece34db78d445dd578a52d28b74e8319f4d9535fbaff79d0eae82b3d", size = 944300, upload-time = "2025-10-07T21:56:58.969Z" }, - { url = "https://files.pythonhosted.org/packages/e7/a3/d8aaf9c243ce1319bd2498004a9acccfcfb35a3ef9851abb856993d95255/pymongo-4.15.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:dcff15b9157c16bc796765d4d3d151df669322acfb0357e4c3ccd056153f0ff4", size = 1029873, upload-time = "2025-10-07T21:57:00.759Z" }, - { url = "https://files.pythonhosted.org/packages/64/10/91fd7791425ed3b56cbece6c23a36fb2696706a695655d8ea829e5e23c3a/pymongo-4.15.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1f681722c9f27e86c49c2e8a838e61b6ecf2285945fd1798bd01458134257834", size = 1029611, upload-time = "2025-10-07T21:57:02.488Z" }, - { url = "https://files.pythonhosted.org/packages/bb/9c/d9cf8d8a181f96877bca7bdec3e6ce135879d5e3d78694ea465833c53a3f/pymongo-4.15.3-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2c96dde79bdccd167b930a709875b0cd4321ac32641a490aebfa10bdcd0aa99b", size = 2211827, upload-time = "2025-10-07T21:57:03.907Z" }, - { url = "https://files.pythonhosted.org/packages/c2/40/12703964305216c155284100124222eaa955300a07d426c6e0ba3c9cbade/pymongo-4.15.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2d4ca446348d850ac4a5c3dc603485640ae2e7805dbb90765c3ba7d79129b37", size = 2264654, upload-time = "2025-10-07T21:57:05.41Z" }, - { url = "https://files.pythonhosted.org/packages/0f/70/bf3c18b5d0cae0b9714158b210b07b5891a875eb1c503271cfe045942fd3/pymongo-4.15.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0fd3de3a12ff0a8113a3f64cedb01f87397ab8eaaffa88d7f18ca66cd39385", size = 2371830, upload-time = "2025-10-07T21:57:06.9Z" }, - { url = "https://files.pythonhosted.org/packages/21/6d/2dfaed2ae66304ab842d56ed9a1bd2706ca0ecf97975b328a5eeceb2a4c0/pymongo-4.15.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e84dec392cf5f72d365e0aac73f627b0a3170193ebb038c3f7e7df11b7983ee7", size = 2351878, upload-time = "2025-10-07T21:57:08.92Z" }, - { url = "https://files.pythonhosted.org/packages/17/ed/fe46ff9adfa6dc11ad2e0694503adfc98f40583cfcc6db4dbaf582f0e357/pymongo-4.15.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d4b01a48369ea6d5bc83fea535f56279f806aa3e4991189f0477696dd736289", size = 2251356, upload-time = "2025-10-07T21:57:10.51Z" }, - { url = "https://files.pythonhosted.org/packages/12/c4/2e1a10b1e9bca9c106f2dc1b89d4ad70c63d387c194b3a1bfcca552b5a3f/pymongo-4.15.3-cp314-cp314-win32.whl", hash = "sha256:3561fa96c3123275ec5ccf919e595547e100c412ec0894e954aa0da93ecfdb9e", size = 992878, upload-time = "2025-10-07T21:57:12.119Z" }, - { url = "https://files.pythonhosted.org/packages/98/b5/14aa417a44ea86d4c31de83b26f6e6793f736cd60e7e7fda289ce5184bdf/pymongo-4.15.3-cp314-cp314-win_amd64.whl", hash = "sha256:9df2db6bd91b07400879b6ec89827004c0c2b55fc606bb62db93cafb7677c340", size = 1021209, upload-time = "2025-10-07T21:57:13.686Z" }, - { url = "https://files.pythonhosted.org/packages/94/9f/1097c6824fa50a4ffb11ba5194d2a9ef68d5509dd342e32ddb697d2efe4e/pymongo-4.15.3-cp314-cp314-win_arm64.whl", hash = "sha256:ff99864085d2c7f4bb672c7167680ceb7d273e9a93c1a8074c986a36dbb71cc6", size = 1000618, upload-time = "2025-10-07T21:57:15.212Z" }, - { url = "https://files.pythonhosted.org/packages/ad/31/37c76607a4f793f4491611741fa7a7c4238b956f48c4a9505cea0b5cf7ef/pymongo-4.15.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ffe217d2502f3fba4e2b0dc015ce3b34f157b66dfe96835aa64432e909dd0d95", size = 1086576, upload-time = "2025-10-07T21:57:16.742Z" }, - { url = "https://files.pythonhosted.org/packages/92/b2/6d17d279cdd293eeeb0c9d5baeb4f8cdebb45354fd81cfcef2d1c69303ab/pymongo-4.15.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:390c4954c774eda280898e73aea36482bf20cba3ecb958dbb86d6a68b9ecdd68", size = 1086656, upload-time = "2025-10-07T21:57:18.774Z" }, - { url = "https://files.pythonhosted.org/packages/55/fd/c5da8619beca207d7e6231f24ed269cb537c5311dad59fd9f2ef7d43204a/pymongo-4.15.3-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7dd2a49f088890ca08930bbf96121443b48e26b02b84ba0a3e1ae2bf2c5a9b48", size = 2531646, upload-time = "2025-10-07T21:57:20.63Z" }, - { url = "https://files.pythonhosted.org/packages/93/8f/66a7e12b874f41eb205f352b3a719e5a964b5ba103996f6ac45e80560111/pymongo-4.15.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f6feb678f26171f2a6b2cbb340949889154c7067972bd4cc129b62161474f08", size = 2603799, upload-time = "2025-10-07T21:57:22.591Z" }, - { url = "https://files.pythonhosted.org/packages/10/98/baf0d1f8016087500899cc4ae14e591f29b016c643e99ab332fcafe6f7bc/pymongo-4.15.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:446417a34ff6c2411ce3809e17ce9a67269c9f1cb4966b01e49e0c590cc3c6b3", size = 2725238, upload-time = "2025-10-07T21:57:24.091Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a2/112d8d3882d6e842f501e166fbe08dfc2bc9a35f8773cbcaa804f7991043/pymongo-4.15.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cfa4a0a0f024a0336640e1201994e780a17bda5e6a7c0b4d23841eb9152e868b", size = 2704837, upload-time = "2025-10-07T21:57:25.626Z" }, - { url = "https://files.pythonhosted.org/packages/38/fe/043a9aac7b3fba5b8e216f48359bd18fdbe46a4d93b081786f773b25e997/pymongo-4.15.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b03db2fe37c950aff94b29ded5c349b23729bccd90a0a5907bbf807d8c77298", size = 2582294, upload-time = "2025-10-07T21:57:27.221Z" }, - { url = "https://files.pythonhosted.org/packages/5b/fe/7a6a6b331d9f2024ab171028ab53d5d9026959b1d713fe170be591a4d9a8/pymongo-4.15.3-cp314-cp314t-win32.whl", hash = "sha256:e7cde58ef6470c0da922b65e885fb1ffe04deef81e526bd5dea429290fa358ca", size = 1043993, upload-time = "2025-10-07T21:57:28.727Z" }, - { url = "https://files.pythonhosted.org/packages/70/c8/bc64321711e19bd48ea3371f0082f10295c433833245d73e7606d3b9afbe/pymongo-4.15.3-cp314-cp314t-win_amd64.whl", hash = "sha256:fae552767d8e5153ed498f1bca92d905d0d46311d831eefb0f06de38f7695c95", size = 1078481, upload-time = "2025-10-07T21:57:30.372Z" }, - { url = "https://files.pythonhosted.org/packages/39/31/2bb2003bb978eb25dfef7b5f98e1c2d4a86e973e63b367cc508a9308d31c/pymongo-4.15.3-cp314-cp314t-win_arm64.whl", hash = "sha256:47ffb068e16ae5e43580d5c4e3b9437f05414ea80c32a1e5cac44a835859c259", size = 1051179, upload-time = "2025-10-07T21:57:31.829Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/74/81/6d66e62a5d1c5323dca79e9fb34ac8211df76f6c16625f9499a37b796314/pymongo-4.15.4.tar.gz", hash = "sha256:6ba7cdf46f03f406f77969a8081cfb659af16c0eee26b79a0a14e25f6c00827b", size = 2471218, upload-time = "2025-11-11T20:52:37.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/9e/5d6bd240a8d32e088078b37dcfa1579028c51d91168c0e992827ec1e87e6/pymongo-4.15.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:84c7c7624a1298295487d0dfd8dbec75d14db44c017b5087c7fe7d6996a96e3d", size = 811325, upload-time = "2025-11-11T20:50:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/1f/aa/1d707a836c436af60faa2db6f2706f9e74b5056d0bd77deb243c023b5e7b/pymongo-4.15.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:71a5ab372ebe4e05453bae86a008f6db98b5702df551219fb2f137c394d71c3a", size = 811668, upload-time = "2025-11-11T20:50:24.171Z" }, + { url = "https://files.pythonhosted.org/packages/e8/17/8c50a695a7029d582da50875085465f01bf83e5146fb7dc3f671168aedb7/pymongo-4.15.4-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eee407bf1058a8f0d5b203028997b42ea6fc80a996537cc2886f89573bc0770f", size = 1185476, upload-time = "2025-11-11T20:50:25.635Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e8/5449663ec341fb83c3e4d51011f65b61de8427620679510cca57386c9446/pymongo-4.15.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e286f5b9c13963bcaf9b9241846d388ac5022225a9e11c5364393a8cc3eb49", size = 1203857, upload-time = "2025-11-11T20:50:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ff/98768d4294f271175aedbad209d748ac769a3f35bee35f8c82b57b03ea4a/pymongo-4.15.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:67c3b84a2a0e1794b2fbfe22dc36711a03c6bc147d9d2e0f8072fabed7a65092", size = 1242538, upload-time = "2025-11-11T20:50:29.322Z" }, + { url = "https://files.pythonhosted.org/packages/18/91/57b4a08b81686e0148a93ecd0149d747a31be82aafa0708143d642662893/pymongo-4.15.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:94e50149fb9d982c234d0efa9c0eec4a04db7e82a412d3dae2c4f03a9926360e", size = 1232831, upload-time = "2025-11-11T20:50:31.326Z" }, + { url = "https://files.pythonhosted.org/packages/ff/17/52434425cde25e6d0743a6d8af8a8b88ffbd05cce595993facf09a5d0559/pymongo-4.15.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1903c0966969cf3e7b30922956bd82eb09e6a3f3d7431a727d12f20104f66d3", size = 1200182, upload-time = "2025-11-11T20:50:33.339Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7f/d5c975dcbfd339f3cd3eae2055fe6d96fb546508e1954fe263c0304e0317/pymongo-4.15.4-cp310-cp310-win32.whl", hash = "sha256:20ffcd883b6e187ef878558d0ebf9f09cc46807b6520022592522d3cdd21022d", size = 798325, upload-time = "2025-11-11T20:50:35.214Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1f/78b2d3d7b35284c5da80342ce2b7e4087901ff8fb030eccaa654b5d3d061/pymongo-4.15.4-cp310-cp310-win_amd64.whl", hash = "sha256:68ea93e7d19d3aa3182a6e41ba68288b9b234a3b0a70b368feb95fff3f94413f", size = 808144, upload-time = "2025-11-11T20:50:36.621Z" }, + { url = "https://files.pythonhosted.org/packages/30/d2/95505fb5a699180a215553f622702464bc47000e5e782cc846098dcdfc37/pymongo-4.15.4-cp310-cp310-win_arm64.whl", hash = "sha256:abfe72630190c0dc8f2222b02af7c4e5f72809d06b2ccb3f3ca83f6a7b60e302", size = 800933, upload-time = "2025-11-11T20:50:38.573Z" }, + { url = "https://files.pythonhosted.org/packages/71/a4/b1a724352ab47a8925f30931a6aa6f905dcf473d8404156ef608ec325fbd/pymongo-4.15.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b2967bda6ccac75aefad26c4ef295f5054181d69928bb9d1159227d6771e8887", size = 865881, upload-time = "2025-11-11T20:50:40.275Z" }, + { url = "https://files.pythonhosted.org/packages/09/d4/6f4db5b64b0b71f0cbe608a80aea8b2580b5e1db4da1f9a70ae5531e9f1d/pymongo-4.15.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7df1fad859c61bdbe0e2a0dec8f5893729d99b4407b88568e0e542d25f383f57", size = 866225, upload-time = "2025-11-11T20:50:41.842Z" }, + { url = "https://files.pythonhosted.org/packages/0f/44/9d96fa635b838348109f904f558aa6675fdfb0a9265060050d7a92afbf97/pymongo-4.15.4-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:990c4898787e706d0ab59141cf5085c981d89c3f86443cd6597939d9f25dd71d", size = 1429778, upload-time = "2025-11-11T20:50:43.801Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e6/eac0b3ca4ea1cd437983f1409cb6260e606cce11ea3cb6f5ccd8629fa5c2/pymongo-4.15.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad7ff0347e8306fc62f146bdad0635d9eec1d26e246c97c14dd1a189d3480e3f", size = 1456739, upload-time = "2025-11-11T20:50:45.479Z" }, + { url = "https://files.pythonhosted.org/packages/73/7e/b7adba0c8dfc2dced7632c61425a70048bddf953b07bf6232a4ea7f0fb7e/pymongo-4.15.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dd8c78c59fd7308239ef9bcafb7cd82f08cbc9466d1cfda22f9025c83468bf6d", size = 1514659, upload-time = "2025-11-11T20:50:47.517Z" }, + { url = "https://files.pythonhosted.org/packages/20/8b/cdc129f1bee5595018c52ff81baaec818301e705ee39cf00d9d5f68a3d0d/pymongo-4.15.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:44d95677aa23fe479bb531b393a4fad0210f808af52e4ab2b79c0b540c828957", size = 1500700, upload-time = "2025-11-11T20:50:49.183Z" }, + { url = "https://files.pythonhosted.org/packages/1f/02/e706a63f00542531a4c723258ae3da3439925de02215710a18813fbe1db4/pymongo-4.15.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ab985e61376ae5a04f162fb6bdddaffc7beec883ffbd9d84ea86a71be794d74", size = 1452011, upload-time = "2025-11-11T20:50:51.568Z" }, + { url = "https://files.pythonhosted.org/packages/37/36/6b78b105e8e1174ebda592ad31f02cb98ee9bd8bb2eeb621f54e2c714d03/pymongo-4.15.4-cp311-cp311-win32.whl", hash = "sha256:2f811e93dbcba0c488518ceae7873a40a64b6ad273622a18923ef2442eaab55c", size = 844471, upload-time = "2025-11-11T20:50:53.362Z" }, + { url = "https://files.pythonhosted.org/packages/a5/0d/3d009eed6ae045ee4f62877878070a07405af5e368d60a4a35efd177c25b/pymongo-4.15.4-cp311-cp311-win_amd64.whl", hash = "sha256:53bfcd8c11086a2457777cb4b1a6588d9dd6af77aeab47e04f2af02e3a077e59", size = 859189, upload-time = "2025-11-11T20:50:55.198Z" }, + { url = "https://files.pythonhosted.org/packages/d5/40/d5713b1d5e0b10402446632bab6a88918cd13e5fe1fa26beac177eb37dac/pymongo-4.15.4-cp311-cp311-win_arm64.whl", hash = "sha256:2096964b2b93607ed80a62ac6664396a826b7fe34e2b1eed3f20784681a17827", size = 848369, upload-time = "2025-11-11T20:50:57.164Z" }, + { url = "https://files.pythonhosted.org/packages/75/bb/09176c965d994352efd1407c9139799218f3fe1d18382dff34ef64e0bd22/pymongo-4.15.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4ab4eef031e722a8027c338c3d71704a8c85c17c64625d61c6effdf8a893b971", size = 920943, upload-time = "2025-11-11T20:50:59.056Z" }, + { url = "https://files.pythonhosted.org/packages/94/97/d212bd8d9106acecf6948cc0a0ed640f58d8afaed427481b9e79db08f45c/pymongo-4.15.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e12551e28007a341d15ebca5a024ef487edf304d612fba5efa1fd6b4d9a95a9", size = 920687, upload-time = "2025-11-11T20:51:00.683Z" }, + { url = "https://files.pythonhosted.org/packages/ff/81/7be727d6172fd80d8dd1c6fedb78675936396d2f2067fab270e443e04621/pymongo-4.15.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1d21998fb9ccb3ea6d59a9f9971591b9efbcfbbe46350f7f8badef9b107707f3", size = 1690340, upload-time = "2025-11-11T20:51:02.392Z" }, + { url = "https://files.pythonhosted.org/packages/42/5a/91bf00e9d30d18b3e8ef3fa222964ba1e073d82c5f38dae027e63d36bcfd/pymongo-4.15.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f83e8895d42eb51d259694affa9607c4d56e1c784928ccbbac568dc20df86a8", size = 1726082, upload-time = "2025-11-11T20:51:04.353Z" }, + { url = "https://files.pythonhosted.org/packages/ff/08/b7d8e765efa64cddf1844e8b889454542c765f8d119c87a4904f45addc07/pymongo-4.15.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bd8126a507afa8ce4b96976c8e28402d091c40b7d98e3b5987a371af059d9e7", size = 1800624, upload-time = "2025-11-11T20:51:06.222Z" }, + { url = "https://files.pythonhosted.org/packages/35/b0/40ec073ccc2cf95e8743315e6c92a81f37698d2e618c83ec7d9c3b647bd0/pymongo-4.15.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e799e2cba7fcad5ab29f678784f90b1792fcb6393d571ecbe4c47d2888af30f3", size = 1785469, upload-time = "2025-11-11T20:51:07.893Z" }, + { url = "https://files.pythonhosted.org/packages/82/da/b1a27064404d5081f5391c3c81e4a6904acccb4766598e3aa14399d36feb/pymongo-4.15.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:563e793ad87633e50ad43a8cd2c740fbb17fca4a4637185996575ddbe99960b8", size = 1718540, upload-time = "2025-11-11T20:51:09.574Z" }, + { url = "https://files.pythonhosted.org/packages/e7/8c/bee6159b4e434dc0413b399af2bd3795ef7427b2c2fe1b304df250c0a3d8/pymongo-4.15.4-cp312-cp312-win32.whl", hash = "sha256:39bb3c12c772241778f4d7bf74885782c8d68b309d3c69891fe39c729334adbd", size = 891308, upload-time = "2025-11-11T20:51:11.67Z" }, + { url = "https://files.pythonhosted.org/packages/cf/cb/cb70455fe2eadf4f6ccd27fe215e342b242e8b53780aeafb96cd1c3bf506/pymongo-4.15.4-cp312-cp312-win_amd64.whl", hash = "sha256:6f43326f36bc540b04f5a7f1aa8be40b112d7fc9f6e785ae3797cd72a804ffdd", size = 910911, upload-time = "2025-11-11T20:51:13.283Z" }, + { url = "https://files.pythonhosted.org/packages/41/81/20486a697474b7de25faee91d9c478eb410ae78cb4e50b15000184944a48/pymongo-4.15.4-cp312-cp312-win_arm64.whl", hash = "sha256:263cfa2731a4bbafdce2cf06cd511eba8957bd601b3cad9b4723f2543d42c730", size = 896347, upload-time = "2025-11-11T20:51:15.981Z" }, + { url = "https://files.pythonhosted.org/packages/51/10/09551492e484f7055194d91c071c827fc65261156e4daced35e67e97b893/pymongo-4.15.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6ff080f23a12c943346e2bba76cf19c3d14fb3625956792aa22b69767bfb36de", size = 975326, upload-time = "2025-11-11T20:51:17.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/6e/8f153a6d7eaec9b334975000e16bfd11ec4050e8729d3e2ee67d7022f526/pymongo-4.15.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c4690e01d03773f7af21b1a8428029bd534c9fe467c6b594c591d8b992c0a975", size = 975132, upload-time = "2025-11-11T20:51:19.58Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7d/037498c1354fae1ce2fc7738c981a7447a5fee021c22e76083540cc1f9d6/pymongo-4.15.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:78bfe3917d0606b30a91b02ad954c588007f82e2abb2575ac2665259b051a753", size = 1950964, upload-time = "2025-11-11T20:51:21.262Z" }, + { url = "https://files.pythonhosted.org/packages/ef/96/7c6b14956ef2ab99600d93b43429387394df6a99f5293cd0371c59a77a02/pymongo-4.15.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f53c83c3fd80fdb412ce4177d4f59b70b9bb1add6106877da044cf21e996316b", size = 1995249, upload-time = "2025-11-11T20:51:23.248Z" }, + { url = "https://files.pythonhosted.org/packages/2a/16/0e0495b38dd64efbfd6f2eb47535895c8df4a78e384aee78190fe2ecfa84/pymongo-4.15.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e41d6650c1cd77a8e7556ad65133455f819f8c8cdce3e9cf4bbf14252b7d805", size = 2086580, upload-time = "2025-11-11T20:51:25.294Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c0/692545232a17d5772d15c7e50d54415bdd9b88018e2228607c96766af961/pymongo-4.15.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b60fd8125f52efffd697490b6ccebc6e09d44069ad9c8795df0a684a9a8f4b3c", size = 2070189, upload-time = "2025-11-11T20:51:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/6f/9f/aae8eb4650d9a62f26baca4f4da2a0f5cd1aabcd4229dabc43cd71e09ea2/pymongo-4.15.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d1a1a0406acd000377f34ae91cdb501fa73601a2d071e4a661e0c862e1b166e", size = 1985254, upload-time = "2025-11-11T20:51:29.136Z" }, + { url = "https://files.pythonhosted.org/packages/b1/cd/50f49788caa317c7b00ccf0869805cb2b3046c2510f960cb07e8d3a74f73/pymongo-4.15.4-cp313-cp313-win32.whl", hash = "sha256:9c5710ed5f2af95315db0ee8ae02e9ff1e85e7b068c507d980bc24fe9d025257", size = 938134, upload-time = "2025-11-11T20:51:31.254Z" }, + { url = "https://files.pythonhosted.org/packages/10/ad/6e96ccb3b7ab8be2e22b1c50b98aed0cae19253174bca6807fc8fd1ce34c/pymongo-4.15.4-cp313-cp313-win_amd64.whl", hash = "sha256:61b0863c7f9b460314db79b7f8541d3b490b453ece49afd56b611b214fc4b3b1", size = 962595, upload-time = "2025-11-11T20:51:33.118Z" }, + { url = "https://files.pythonhosted.org/packages/22/23/9b9255e432df4bc276ecb9bb6e81c3376d8ee2b19de02d3751bb5c4a6fb1/pymongo-4.15.4-cp313-cp313-win_arm64.whl", hash = "sha256:0255af7d5c23c5e8cb4d9bb12906b142acebab0472117e1d5e3a8e6e689781cb", size = 944298, upload-time = "2025-11-11T20:51:35.13Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e6/f315ea84656adcd18d5b5e8b362b47c36bf606843098688cc0809b28c8a8/pymongo-4.15.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:539f9fa5bb04a09fc2965cdcae3fc91d1c6a1f4f1965b34df377bc7119e3d7cd", size = 1029994, upload-time = "2025-11-11T20:51:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0c/0c364db72cd80a503829885643478dd144a8bf05e1e853c89648a06ad34b/pymongo-4.15.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:68354a77cf78424d27216b1cb7c9b0f67da16aae855045279ba8d73bb61f5ad0", size = 1029615, upload-time = "2025-11-11T20:51:38.551Z" }, + { url = "https://files.pythonhosted.org/packages/50/71/6f37eea22ffa5b136c1ca0a21ba390c273b582d800bc979961fbd46c9bcc/pymongo-4.15.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a9a90d556c2ef1572d2aef525ef19477a82d659d117eb3a51fa99e617d07dc44", size = 2211805, upload-time = "2025-11-11T20:51:40.657Z" }, + { url = "https://files.pythonhosted.org/packages/24/09/3a538cb82766ce89559c4ca0d5694f782485080db6a8f628784dc7debba8/pymongo-4.15.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1aac57614fb86a3fa707af3537c30eda5e7fd1be712c1f723296292ac057afe", size = 2264618, upload-time = "2025-11-11T20:51:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/51/6b/66b4fe2d3c566ed655d95b1d8947dfea05642b05a285a3081d6cebc4f5da/pymongo-4.15.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6c21b49c5e021d9ce02cac33525c722d4c6887f7cde19a5a9154f66cb845e84", size = 2371810, upload-time = "2025-11-11T20:51:44.372Z" }, + { url = "https://files.pythonhosted.org/packages/92/2b/3989960c7de983c5cc05b2d43b26fa560fe9de433ee60b83259d6ee2cde3/pymongo-4.15.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e93828768470026099119295c68ed0dbc0a50022558be5e334f6dbda054f1d32", size = 2351848, upload-time = "2025-11-11T20:51:46.548Z" }, + { url = "https://files.pythonhosted.org/packages/31/93/ee9f8a42eed6ecb8dda52e586a470bf88007a298b0f1a2c4ea1ff352af8e/pymongo-4.15.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11840e9eb5a650ac190f2a3473631073daddbabdbb2779b6709dfddd3ba3b872", size = 2251338, upload-time = "2025-11-11T20:51:48.335Z" }, + { url = "https://files.pythonhosted.org/packages/a0/36/c6609f632bcaffcdf9f7e67cb888402a1df049a7c3ff2f56067a0b451a59/pymongo-4.15.4-cp314-cp314-win32.whl", hash = "sha256:f0907b46df97b01911bf2e10ddbb23c2303629e482d81372031fd7f4313b9013", size = 992893, upload-time = "2025-11-11T20:51:50.775Z" }, + { url = "https://files.pythonhosted.org/packages/f0/23/4ec0f7c9bf3397b6cafaf714f5bfe0a9944e7af088daa01d258eec031118/pymongo-4.15.4-cp314-cp314-win_amd64.whl", hash = "sha256:111d7f65ccbde908546cb36d14e22f12a73a4de236fd056f41ed515d1365f134", size = 1021204, upload-time = "2025-11-11T20:51:52.691Z" }, + { url = "https://files.pythonhosted.org/packages/2b/71/3813d15fa5ce6fb5fb40775bedc95a1970790f5aba968d92b014a796aab6/pymongo-4.15.4-cp314-cp314-win_arm64.whl", hash = "sha256:c689a5d057ef013612b5aa58e6bf52f7fdb186e22039f1a3719985b5d0399932", size = 1000608, upload-time = "2025-11-11T20:51:54.442Z" }, + { url = "https://files.pythonhosted.org/packages/12/e7/10f3bc034fcec374dc46462b369205527478199a803169cb10e9e4b48c68/pymongo-4.15.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:cdfa57760745387cde93615a48f622bf1eeae8ae28103a8a5100b9389eec22f9", size = 1086725, upload-time = "2025-11-11T20:51:57.266Z" }, + { url = "https://files.pythonhosted.org/packages/40/ee/b59cad7d46598d48708bd2a6559ea8b9cbb6fb9665d617b5a52b58de81b3/pymongo-4.15.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4fd6ba610e5a54090c4055a15f38d19ad8bf11e6bbc5a173e945c755a16db455", size = 1086660, upload-time = "2025-11-11T20:51:59.114Z" }, + { url = "https://files.pythonhosted.org/packages/0a/84/58efbde2b52a577f9162bb9b97605b6669354bb171bc241a0dc2639536d7/pymongo-4.15.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3c7945b8a5563aa3951db26ba534372fba4c781473f5d55ce6340b7523cb0f", size = 2531617, upload-time = "2025-11-11T20:52:01.006Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cd/7bd739d04b67c99f00c942465b8ab7659dc2c1ad80108b5f4f74eecdf9f3/pymongo-4.15.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41e98a31e79d74e9d78bc1638b71c3a10a910eae7d3318e2ae8587c760931451", size = 2603756, upload-time = "2025-11-11T20:52:03.029Z" }, + { url = "https://files.pythonhosted.org/packages/4a/39/5a3b01f7e5fd464656421246516723c02067e85bbfb52d30da7d79b8336f/pymongo-4.15.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d18d89073b5e752391c237d2ee86ceec1e02a4ad764b3029f24419eedd12723e", size = 2725205, upload-time = "2025-11-11T20:52:04.968Z" }, + { url = "https://files.pythonhosted.org/packages/c7/a8/b06231d5ea48d0fcc47bf6c2cebfd8dbea3eda1a1d7bf786443cb9ef5b94/pymongo-4.15.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edbff27a56a80b8fe5c0319200c44e63b1349bf20db27d9734ddcf23c0d72b35", size = 2704793, upload-time = "2025-11-11T20:52:07.164Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/c0ea0da1185d3be4e73923ab3b74f14f424b40f787c710690c83004f147a/pymongo-4.15.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f1d75f5b51304176631c12e5bf47eed021446669e5f99379b76fd2bd3929c1b4", size = 2582263, upload-time = "2025-11-11T20:52:09.016Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f7/29ce41f9e55b1dd912bed39b76e9326e23ff6c097c4a8de88b2c5bcd54e5/pymongo-4.15.4-cp314-cp314t-win32.whl", hash = "sha256:e1bf4e0689cc48e0cfa6aef17f107c298d8898de0c6e782ea5c98450ae93a62f", size = 1044009, upload-time = "2025-11-11T20:52:11.138Z" }, + { url = "https://files.pythonhosted.org/packages/01/71/3fade727cc4c7ac77fe19c4e3a6bbfb66d7f46796108ba106f236c64492f/pymongo-4.15.4-cp314-cp314t-win_amd64.whl", hash = "sha256:3fc347ea5eda6c3a7177c3a9e4e9b4e570a444a351effda4a898c2d352a1ccd1", size = 1078479, upload-time = "2025-11-11T20:52:13.324Z" }, + { url = "https://files.pythonhosted.org/packages/60/0f/d450350f103db4bb856cb1ee60c8b1fa68d5ac50c846896d74deba3e9950/pymongo-4.15.4-cp314-cp314t-win_arm64.whl", hash = "sha256:2d921b84c681c5385a6f7ba2b5740cb583544205a00877aad04b5b12ab86ad26", size = 1051155, upload-time = "2025-11-11T20:52:15.185Z" }, ] [[package]] @@ -752,14 +760,15 @@ wheels = [ [[package]] name = "yscope-spider-py" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lark" }, { name = "mariadb" }, + { name = "msgpack" }, { name = "msgpack-types" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/20/fc1170d5109639a466cb776bb8b78478d6ee6d61ccfb2021224e1349485a/yscope_spider_py-0.1.0.tar.gz", hash = "sha256:e7da1a928aacf717d9c97ed5a19da521e25288fb48ceb04b632de1574693296f", size = 23760, upload-time = "2025-09-21T21:19:27.178Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/4ff0569fc95f6210a4bff258bf070f09b39a6eb60e6772d3dad3f7eb9279/yscope_spider_py-0.2.0.tar.gz", hash = "sha256:43c819ad4e0a135e3ae63ca55d849b6b3015bcf9b982c337d5cf10ea07b4e947", size = 25554, upload-time = "2025-11-14T04:30:33.533Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/2d/5945cd02f57b2b54d6e09f54fb3b9f0c33441ba02cbfdfc97390c1182286/yscope_spider_py-0.1.0-py3-none-any.whl", hash = "sha256:a095de44ed957b3e365e0ee0ccb0622d5a5d10b2a2fec5d4a239cd5530e017ae", size = 33394, upload-time = "2025-09-21T21:19:26.122Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ac/9eb7fa22ac02641faf57532e44326c471840ebf125270cb4bfad0968f534/yscope_spider_py-0.2.0-py3-none-any.whl", hash = "sha256:a5d1e1c2182e81321cc3482bcca5b46998922c6d5ccfc5bbd1cc6326388435d0", size = 34481, upload-time = "2025-11-14T04:30:32.463Z" }, ] From 2f75a068cfddd7e5ebfcb22d6e2644e5931c6abd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 21:28:53 -0500 Subject: [PATCH 346/408] Fix merge --- .../clp-py-utils/clp_py_utils/initialize-spider-db.py | 8 ++++---- components/clp-py-utils/clp_py_utils/sql_adapter.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 2f570aec53..cc0e14ced0 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -7,9 +7,9 @@ from contextlib import closing from pydantic import ValidationError -from sql_adapter import SQL_Adapter +from sql_adapter import SqlAdapter -from clp_py_utils.clp_config import CLPConfig +from clp_py_utils.clp_config import ClpConfig from clp_py_utils.core import read_yaml_config_file # Setup logging @@ -227,7 +227,7 @@ def main(argv): config_path = pathlib.Path(parsed_args.config) try: - clp_config = CLPConfig.model_validate(read_yaml_config_file(config_path)) + clp_config = ClpConfig.model_validate(read_yaml_config_file(config_path)) clp_config.database.load_credentials_from_env() if clp_config.spider_db is None: return 0 @@ -245,7 +245,7 @@ def main(argv): return -1 try: - sql_adapter = SQL_Adapter(clp_config.database) + sql_adapter = SqlAdapter(clp_config.database) with closing(sql_adapter.create_root_mariadb_connection()) as db_conn, closing( db_conn.cursor() ) as db_cursor: diff --git a/components/clp-py-utils/clp_py_utils/sql_adapter.py b/components/clp-py-utils/clp_py_utils/sql_adapter.py index b422f50e09..3da95e27bc 100644 --- a/components/clp-py-utils/clp_py_utils/sql_adapter.py +++ b/components/clp-py-utils/clp_py_utils/sql_adapter.py @@ -59,7 +59,7 @@ def alive(self): class SqlAdapter: - def __init__(self, database_config: Database): + def __init__(self, database_config: Database, spider_database_config: Optional[SpiderDb] = None): self.database_config = database_config self._spider_database_config = spider_database_config @@ -85,7 +85,7 @@ def create_mysql_connection( def create_mariadb_connection( self, disable_localhost_socket_connection: bool = False - ) -> mariadb.connection: + ) -> mariadb.Connection: try: connection = mariadb.connect( **self.database_config.get_mysql_connection_params( From 999f8348d26fb2f37402203d562ec6fe11964554 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 21:31:40 -0500 Subject: [PATCH 347/408] Update lock file --- components/clp-mcp-server/uv.lock | 25 ++++++++++++++++------- integration-tests/uv.lock | 34 +++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/components/clp-mcp-server/uv.lock b/components/clp-mcp-server/uv.lock index 0e1cacb39c..c6404384f9 100644 --- a/components/clp-mcp-server/uv.lock +++ b/components/clp-mcp-server/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" [[package]] @@ -381,7 +381,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "boto3", specifier = ">=1.40.55" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "python-levenshtein", specifier = ">=0.27.1" }, @@ -914,12 +914,23 @@ wheels = [ [[package]] name = "mariadb" -version = "1.0.11" +version = "1.1.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/0a/ce5724f852f2937c6955bcea09659aa2d85d487df1c9de6711344b71527d/mariadb-1.0.11.zip", hash = "sha256:76916c892bc936c5b0f36e25a1411f651a7b7ce978992ae3a8de7e283efdacbf", size = 85926, upload-time = "2022-04-12T19:33:17.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/4a/6a6b7ad6a7b8156b41d0a6d849debe804f9d8696288ed2c6c31a9654357c/mariadb-1.0.11-cp310-cp310-win32.whl", hash = "sha256:4a583a80059d11f1895a2c93b1b7110948e87f0256da3e3222939a2530f0518e", size = 159743, upload-time = "2022-04-12T19:33:04.712Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a7/10740ceec1ce9d57f4bc3614e55efe2e72ae284e8c8d32eacabfbd7ad6cc/mariadb-1.0.11-cp310-cp310-win_amd64.whl", hash = "sha256:63c5c7cf99335e5c961e1d65a323576c9cb834e1a6a8084a6a8b4ffd85ca6213", size = 177316, upload-time = "2022-04-12T19:33:06.95Z" }, +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/ba/cedef19833be88e07bfff11964441cda8a998f1628dd3b2fa3e7751d36e0/mariadb-1.1.14.tar.gz", hash = "sha256:e6d702a53eccf20922e47f2f45cfb5c7a0c2c6c0a46e4ee2d8a80d0ff4a52f34", size = 111715, upload-time = "2025-10-07T06:45:48.017Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/bf/5e1a3a5be297c3a679e08f5359165491508fbfb64faf854dc1d626cea9c0/mariadb-1.1.14-cp310-cp310-win32.whl", hash = "sha256:4c7f33578da163a1b79929aae241f5f981d7b9d5a94d89e589aad7ec58e313ea", size = 185064, upload-time = "2025-10-07T06:45:24.858Z" }, + { url = "https://files.pythonhosted.org/packages/31/30/3a61991c13cb8257f5db64aca12bafaa3d811d407e1fae019139fd17c99b/mariadb-1.1.14-cp310-cp310-win_amd64.whl", hash = "sha256:3f6fdc4ded5e0500a6a29bf0c8bf1be94189dcef5a8d5e0e154a4b3456f86bcc", size = 202020, upload-time = "2025-10-07T06:45:27.785Z" }, + { url = "https://files.pythonhosted.org/packages/56/aa/a7b3c66b2792e8319ec9157d63851ff2e0b26496a05044e22b50a012a05e/mariadb-1.1.14-cp311-cp311-win32.whl", hash = "sha256:932a95016b7e9b8d78893aa5ee608e74199e3c6dd607dbe5e4da2010a4f67b88", size = 185061, upload-time = "2025-10-07T06:45:29.964Z" }, + { url = "https://files.pythonhosted.org/packages/54/04/ea2374867756b4082764484bc8b82e1798d94f171bcc914e08c60d640f8f/mariadb-1.1.14-cp311-cp311-win_amd64.whl", hash = "sha256:55ddbe5272c292cbcb2968d87681b5d2b327e65646a015e324b8eeb804d14531", size = 202016, upload-time = "2025-10-07T06:45:32.151Z" }, + { url = "https://files.pythonhosted.org/packages/00/04/659a8d30513700b5921ec96bddc07f550016c045fcbeb199d8cd18476ecc/mariadb-1.1.14-cp312-cp312-win32.whl", hash = "sha256:98d552a8bb599eceaa88f65002ad00bd88aeed160592c273a7e5c1d79ab733dd", size = 185266, upload-time = "2025-10-07T06:45:34.164Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a9/8f210291bc5fc044e20497454f40d35b3bab326e2cab6fccdc38121cb2c1/mariadb-1.1.14-cp312-cp312-win_amd64.whl", hash = "sha256:685a1ad2a24fd0aae1c4416fe0ac794adc84ab9209c8d0c57078f770d39731db", size = 202112, upload-time = "2025-10-07T06:45:35.824Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/51130048bcce038bb859978250515f1aad90e9c4d273630a704e0a8b1ae1/mariadb-1.1.14-cp313-cp313-win32.whl", hash = "sha256:3d2c795cde606f4e12c0d73282b062433f414cae035675b0d81f2d65c9b79ac5", size = 185221, upload-time = "2025-10-07T06:45:37.848Z" }, + { url = "https://files.pythonhosted.org/packages/af/23/e952a7e442913abd8079cd27b80b69474895c93b3727fad41c7642a80c62/mariadb-1.1.14-cp313-cp313-win_amd64.whl", hash = "sha256:7fd603c5cf23c47ef0d28fdc2b4b79919ee7f75d00ed070d3cd1054dcf816aeb", size = 202121, upload-time = "2025-10-07T06:45:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f2/7059f83543a4264b98777d7cc8aa203e1ca6a13a461f730d1c97f29628d4/mariadb-1.1.14-cp314-cp314-win32.whl", hash = "sha256:1a50b4612c0dd5b69690cebb34cef552a7f64dcadeb5aa91d70cd99bf01bc5b3", size = 190620, upload-time = "2025-10-07T06:45:41.349Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7c/7e094b0b396d742494f6346f2ffa9709e429970b0461aca50526f5f02f12/mariadb-1.1.14-cp314-cp314-win_amd64.whl", hash = "sha256:6659725837e48fa6af05e20128fb525029f706f1921d5dbf639a25b2f80b9f93", size = 206263, upload-time = "2025-10-07T06:45:43.227Z" }, ] [[package]] diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index c59fad1aa2..f309f0cfc7 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" [[package]] @@ -543,7 +543,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "boto3", specifier = ">=1.40.55" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "python-levenshtein", specifier = ">=0.27.1" }, @@ -977,14 +977,14 @@ dependencies = [ requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, { name = "celery", extras = ["redis"], specifier = ">=5.5.3" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "msgpack", specifier = ">=1.1.2" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pika", specifier = ">=1.3.2" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "pymongo", specifier = ">=4.15.3" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "yscope-spider-py", specifier = "==0.1.0" }, + { name = "yscope-spider-py", specifier = "==0.2.0" }, ] [[package]] @@ -1171,12 +1171,23 @@ wheels = [ [[package]] name = "mariadb" -version = "1.0.11" +version = "1.1.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/0a/ce5724f852f2937c6955bcea09659aa2d85d487df1c9de6711344b71527d/mariadb-1.0.11.zip", hash = "sha256:76916c892bc936c5b0f36e25a1411f651a7b7ce978992ae3a8de7e283efdacbf", size = 85926, upload-time = "2022-04-12T19:33:17.988Z" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/ba/cedef19833be88e07bfff11964441cda8a998f1628dd3b2fa3e7751d36e0/mariadb-1.1.14.tar.gz", hash = "sha256:e6d702a53eccf20922e47f2f45cfb5c7a0c2c6c0a46e4ee2d8a80d0ff4a52f34", size = 111715, upload-time = "2025-10-07T06:45:48.017Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/4a/6a6b7ad6a7b8156b41d0a6d849debe804f9d8696288ed2c6c31a9654357c/mariadb-1.0.11-cp310-cp310-win32.whl", hash = "sha256:4a583a80059d11f1895a2c93b1b7110948e87f0256da3e3222939a2530f0518e", size = 159743, upload-time = "2022-04-12T19:33:04.712Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a7/10740ceec1ce9d57f4bc3614e55efe2e72ae284e8c8d32eacabfbd7ad6cc/mariadb-1.0.11-cp310-cp310-win_amd64.whl", hash = "sha256:63c5c7cf99335e5c961e1d65a323576c9cb834e1a6a8084a6a8b4ffd85ca6213", size = 177316, upload-time = "2022-04-12T19:33:06.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/bf/5e1a3a5be297c3a679e08f5359165491508fbfb64faf854dc1d626cea9c0/mariadb-1.1.14-cp310-cp310-win32.whl", hash = "sha256:4c7f33578da163a1b79929aae241f5f981d7b9d5a94d89e589aad7ec58e313ea", size = 185064, upload-time = "2025-10-07T06:45:24.858Z" }, + { url = "https://files.pythonhosted.org/packages/31/30/3a61991c13cb8257f5db64aca12bafaa3d811d407e1fae019139fd17c99b/mariadb-1.1.14-cp310-cp310-win_amd64.whl", hash = "sha256:3f6fdc4ded5e0500a6a29bf0c8bf1be94189dcef5a8d5e0e154a4b3456f86bcc", size = 202020, upload-time = "2025-10-07T06:45:27.785Z" }, + { url = "https://files.pythonhosted.org/packages/56/aa/a7b3c66b2792e8319ec9157d63851ff2e0b26496a05044e22b50a012a05e/mariadb-1.1.14-cp311-cp311-win32.whl", hash = "sha256:932a95016b7e9b8d78893aa5ee608e74199e3c6dd607dbe5e4da2010a4f67b88", size = 185061, upload-time = "2025-10-07T06:45:29.964Z" }, + { url = "https://files.pythonhosted.org/packages/54/04/ea2374867756b4082764484bc8b82e1798d94f171bcc914e08c60d640f8f/mariadb-1.1.14-cp311-cp311-win_amd64.whl", hash = "sha256:55ddbe5272c292cbcb2968d87681b5d2b327e65646a015e324b8eeb804d14531", size = 202016, upload-time = "2025-10-07T06:45:32.151Z" }, + { url = "https://files.pythonhosted.org/packages/00/04/659a8d30513700b5921ec96bddc07f550016c045fcbeb199d8cd18476ecc/mariadb-1.1.14-cp312-cp312-win32.whl", hash = "sha256:98d552a8bb599eceaa88f65002ad00bd88aeed160592c273a7e5c1d79ab733dd", size = 185266, upload-time = "2025-10-07T06:45:34.164Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a9/8f210291bc5fc044e20497454f40d35b3bab326e2cab6fccdc38121cb2c1/mariadb-1.1.14-cp312-cp312-win_amd64.whl", hash = "sha256:685a1ad2a24fd0aae1c4416fe0ac794adc84ab9209c8d0c57078f770d39731db", size = 202112, upload-time = "2025-10-07T06:45:35.824Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/51130048bcce038bb859978250515f1aad90e9c4d273630a704e0a8b1ae1/mariadb-1.1.14-cp313-cp313-win32.whl", hash = "sha256:3d2c795cde606f4e12c0d73282b062433f414cae035675b0d81f2d65c9b79ac5", size = 185221, upload-time = "2025-10-07T06:45:37.848Z" }, + { url = "https://files.pythonhosted.org/packages/af/23/e952a7e442913abd8079cd27b80b69474895c93b3727fad41c7642a80c62/mariadb-1.1.14-cp313-cp313-win_amd64.whl", hash = "sha256:7fd603c5cf23c47ef0d28fdc2b4b79919ee7f75d00ed070d3cd1054dcf816aeb", size = 202121, upload-time = "2025-10-07T06:45:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f2/7059f83543a4264b98777d7cc8aa203e1ca6a13a461f730d1c97f29628d4/mariadb-1.1.14-cp314-cp314-win32.whl", hash = "sha256:1a50b4612c0dd5b69690cebb34cef552a7f64dcadeb5aa91d70cd99bf01bc5b3", size = 190620, upload-time = "2025-10-07T06:45:41.349Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7c/7e094b0b396d742494f6346f2ffa9709e429970b0461aca50526f5f02f12/mariadb-1.1.14-cp314-cp314-win_amd64.whl", hash = "sha256:6659725837e48fa6af05e20128fb525029f706f1921d5dbf639a25b2f80b9f93", size = 206263, upload-time = "2025-10-07T06:45:43.227Z" }, ] [[package]] @@ -2608,16 +2619,17 @@ wheels = [ [[package]] name = "yscope-spider-py" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lark" }, { name = "mariadb" }, + { name = "msgpack" }, { name = "msgpack-types" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/20/fc1170d5109639a466cb776bb8b78478d6ee6d61ccfb2021224e1349485a/yscope_spider_py-0.1.0.tar.gz", hash = "sha256:e7da1a928aacf717d9c97ed5a19da521e25288fb48ceb04b632de1574693296f", size = 23760, upload-time = "2025-09-21T21:19:27.178Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/4ff0569fc95f6210a4bff258bf070f09b39a6eb60e6772d3dad3f7eb9279/yscope_spider_py-0.2.0.tar.gz", hash = "sha256:43c819ad4e0a135e3ae63ca55d849b6b3015bcf9b982c337d5cf10ea07b4e947", size = 25554, upload-time = "2025-11-14T04:30:33.533Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/2d/5945cd02f57b2b54d6e09f54fb3b9f0c33441ba02cbfdfc97390c1182286/yscope_spider_py-0.1.0-py3-none-any.whl", hash = "sha256:a095de44ed957b3e365e0ee0ccb0622d5a5d10b2a2fec5d4a239cd5530e017ae", size = 33394, upload-time = "2025-09-21T21:19:26.122Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ac/9eb7fa22ac02641faf57532e44326c471840ebf125270cb4bfad0968f534/yscope_spider_py-0.2.0-py3-none-any.whl", hash = "sha256:a5d1e1c2182e81321cc3482bcca5b46998922c6d5ccfc5bbd1cc6326388435d0", size = 34481, upload-time = "2025-11-14T04:30:32.463Z" }, ] [[package]] From 375636b64e002a9b6f940cce3ebf86b21ad7fe90 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 21:35:53 -0500 Subject: [PATCH 348/408] Fix lint --- components/clp-py-utils/clp_py_utils/sql_adapter.py | 4 +++- .../scheduler/compress/task_manager/spider_task_manager.py | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/sql_adapter.py b/components/clp-py-utils/clp_py_utils/sql_adapter.py index 3da95e27bc..9586948a9c 100644 --- a/components/clp-py-utils/clp_py_utils/sql_adapter.py +++ b/components/clp-py-utils/clp_py_utils/sql_adapter.py @@ -59,7 +59,9 @@ def alive(self): class SqlAdapter: - def __init__(self, database_config: Database, spider_database_config: Optional[SpiderDb] = None): + def __init__( + self, database_config: Database, spider_database_config: Optional[SpiderDb] = None + ): self.database_config = database_config self._spider_database_config = spider_database_config diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index f112858e35..be85abb96d 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -23,9 +23,7 @@ def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None if job_results is None: return None if not isinstance(job_results, tuple): - return [ - CompressionTaskResult.model_validate_json(job_results.decode("utf-8")) - ] + return [CompressionTaskResult.model_validate_json(job_results.decode("utf-8"))] return [ CompressionTaskResult.model_validate_json(task_result.decode("utf-8")) for task_result in job_results From ecf685d730780d2c8d2e47ea16dadf0de3a74cb7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 22:50:46 -0500 Subject: [PATCH 349/408] Remove docker package in doc --- docs/src/dev-docs/building-package.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index 810511c2c8..1fe965fe07 100755 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -88,20 +88,6 @@ To clean up all build artifacts, run: task clean ``` -## Building a Docker image - -To build a Docker image containing the CLP package, run: - -```shell -task docker-images:package -``` - -This will create a Docker image named `clp-package:dev`. - -The package includes a `docker-compose.yaml` file that can be used to deploy CLP using Docker Compose. -If you want to manually deploy with Docker Compose instead of using the package scripts, see the -[Docker Compose design][docker-compose-design] for more information. - [Docker]: https://docs.docker.com/engine/install/ [docker-compose]: https://docs.docker.com/compose/ [design-deployment-orchestration]: design-deployment-orchestration.md From dc808fd28e3bc5d93885a3905cf28f13806d3db8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 23:15:07 -0500 Subject: [PATCH 350/408] Remove used function --- .../executor/compress/spider_compress.py | 1 - .../job_orchestration/utils/__init__.py | 0 .../job_orchestration/utils/spider_utils.py | 23 ------------------- 3 files changed, 24 deletions(-) delete mode 100644 components/job-orchestration/job_orchestration/utils/__init__.py delete mode 100644 components/job-orchestration/job_orchestration/utils/spider_utils.py diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index beabc59b41..72c431fdc0 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -4,7 +4,6 @@ from spider_py import Int8, Int64, TaskContext from job_orchestration.executor.compress.compression_task import compression_entry_point -from job_orchestration.utils.spider_utils import int8_list_to_utf8_str, utf8_str_to_int8_list # Setup logging logger = get_logger("spider_compression") diff --git a/components/job-orchestration/job_orchestration/utils/__init__.py b/components/job-orchestration/job_orchestration/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/components/job-orchestration/job_orchestration/utils/spider_utils.py b/components/job-orchestration/job_orchestration/utils/spider_utils.py deleted file mode 100644 index 2a6395f811..0000000000 --- a/components/job-orchestration/job_orchestration/utils/spider_utils.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -import spider_py - - -def int8_list_to_utf8_str(byte_list: list[spider_py.Int8]) -> str: - """ - Converts a list of `spider_py.Int8` values to a UTF-8 encoded string. - - :param byte_list: - :return: Decoded UTF-8 string constructed from the input byte list. - """ - return bytes(int(byte) for byte in byte_list).decode("utf-8") - - -def utf8_str_to_int8_list(utf8_str: str) -> list[spider_py.Int8]: - """ - Converts a UTF-8 encoded string to a list of `spider_py.Int8` values. - - :param utf8_str: - :return: A list of `spider_py.Int8` values representing the input string. - """ - return [spider_py.Int8(byte) for byte in utf8_str.encode("utf-8")] From b34acf332558153fb19f821ee69029915cd4a568 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 23:16:44 -0500 Subject: [PATCH 351/408] Fix docstring --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 8c3fa9eac8..3f3ac3ccb7 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -275,7 +275,7 @@ def _set_up_env_for_spider_db(self) -> EnvVarsDict: def _set_up_env_for_spider_scheduler(self) -> EnvVarsDict: """ - Sets up environment variables for the Spider database component. + Sets up environment variables for the Spider scheduler component. :return: Dictionary of environment variables necessary to launch the component. """ From 2e820be70c47d6d43b84a9042741d3d24a4c2122 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 23:20:30 -0500 Subject: [PATCH 352/408] Fix valdiator --- components/clp-py-utils/clp_py_utils/clp_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4b475b7a3c..cb5d499b84 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -269,6 +269,7 @@ class SpiderDb(Database): def validate_type(cls, value): if value != "mariadb": raise ValueError(f"Spider only support MariaDB storage.") + return value def get_url(self): self.ensure_credentials_loaded() From 9bf4c342ac1a6eafe9a5ea4743f77c5dd99db709 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 14 Nov 2025 23:47:59 -0500 Subject: [PATCH 353/408] Fix clp config spider db check --- .../clp-py-utils/clp_py_utils/initialize-spider-db.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index cc0e14ced0..1838e6c230 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -230,7 +230,8 @@ def main(argv): clp_config = ClpConfig.model_validate(read_yaml_config_file(config_path)) clp_config.database.load_credentials_from_env() if clp_config.spider_db is None: - return 0 + logger.error("Spider database configuration not found in CLP configuration.") + return -1 clp_config.spider_db.load_credentials_from_env() except (ValidationError, ValueError) as err: logger.error(err) @@ -240,9 +241,6 @@ def main(argv): return -1 spider_db_config = clp_config.spider_db - if not spider_db_config: - logger.error("Spider database configuration not found in CLP configuration.") - return -1 try: sql_adapter = SqlAdapter(clp_config.database) From e054086a8dc16babe7f88731bd8d422425d8ef48 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 15 Nov 2025 00:06:35 -0500 Subject: [PATCH 354/408] Fix exception catching --- components/clp-py-utils/clp_py_utils/initialize-spider-db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 1838e6c230..6725aadf50 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -236,7 +236,7 @@ def main(argv): except (ValidationError, ValueError) as err: logger.error(err) return -1 - except: + except Exception: logger.exception("Failed to load CLP configuration.") return -1 From 75412d752c03441b74d412a59821626f0412a9f0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 15 Nov 2025 00:09:37 -0500 Subject: [PATCH 355/408] Remove name validation --- components/clp-py-utils/clp_py_utils/initialize-spider-db.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 6725aadf50..3f37d24ac7 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -260,9 +260,6 @@ def main(argv): if not _validate_name(clp_user): logger.error(f"Invalid CLP database user name: {clp_user}") return -1 - if not _validate_name(db_password): - logger.error(f"Invalid database user password") - return -1 db_cursor.execute(f"""CREATE DATABASE IF NOT EXISTS `{db_name}`""") if db_password is not None: @@ -277,7 +274,7 @@ def main(argv): db_cursor.execute(f"""USE `{db_name}`""") for table_creator in table_creators: db_cursor.execute(table_creator) - except: + except Exception: logger.exception("Failed to setup Spider database.") return -1 From 75cab4214ca9ea655649bc3f15fc9d9d4d050694 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 15 Nov 2025 00:23:25 -0500 Subject: [PATCH 356/408] Add error checking for start spider worker script --- .../clp_py_utils/start-spider-worker.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index 893527ca7f..d64d08ef54 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -28,18 +28,32 @@ def main() -> None: clp_home = os.getenv("CLP_HOME", "/opt/clp") spider_worker_path = pathlib.Path(clp_home) / "bin" / "spider_worker" + if not spider_worker_path.exists(): + print(f"Error: spider_worker not found at {spider_worker_path}") + exit(1) # Start multiple spider workers processes = [] - for _ in range(concurrency): - process = subprocess.Popen( - [spider_worker_path, "--storage_url", storage_url, "--host", host] - ) - processes.append(process) - + try: + for _ in range(concurrency): + process = subprocess.Popen( + [spider_worker_path, "--storage_url", storage_url, "--host", host] + ) + processes.append(process) + except OSError as e: + print(f"Failed to start spider worker: {e}") + for process in processes: + process.terminate() + exit(1) + + failed = False for process in processes: exit_code = process.wait() - print(f"Spider worker exited with code {exit_code}") + if exit_code != 0: + print(f"Spider worker exited with code {exit_code}") + failed = True + if failed: + exit(1) if __name__ == "__main__": From d618074d6fa4243f73548a9485810fc1bde3ccb4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 15 Nov 2025 00:25:21 -0500 Subject: [PATCH 357/408] Fix fstring --- .../scheduler/compress/compression_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index ecf5610d7f..d249bfb013 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -528,7 +528,7 @@ def main(argv): task_manager = SpiderTaskManager(clp_config.spider_db.get_url()) else: logger.error( - f"Unsupported compression scheduler type:" f" {clp_config.compression_scheduler.type}" + f"Unsupported compression scheduler type: {clp_config.compression_scheduler.type}" ) return -1 From 616dd470d86bf01306f831f5c0a1528dad194b2b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 15 Nov 2025 00:55:15 -0500 Subject: [PATCH 358/408] Fix clp config --- components/clp-py-utils/clp_py_utils/clp_config.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index cb5d499b84..70a882204b 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -857,10 +857,12 @@ def dump_to_primitive_dict(self): "database", "queue", "redis", + "spider_db" } d = self.model_dump(exclude=custom_serialized_fields) for key in custom_serialized_fields: - d[key] = getattr(self, key).dump_to_primitive_dict() + value = getattr(self, key) + d[key] = None if value is None else value.dump_to_primitive_dict() return d @@ -913,8 +915,10 @@ def transform_for_container(self): self.stream_output.storage.transform_for_container() self.database.transform_for_container() - self.queue.transform_for_container() - self.redis.transform_for_container() + if self.queue is not None: + self.queue.transform_for_container() + if self.redis is not None: + self.redis.transform_for_container() self.results_cache.transform_for_container() self.query_scheduler.transform_for_container() self.reducer.transform_for_container() From 4845e92cc2d81c197ababef8e2ecb5d35d17075d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 15 Nov 2025 01:09:43 -0500 Subject: [PATCH 359/408] Fix lint --- components/clp-py-utils/clp_py_utils/clp_config.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 70a882204b..ef51b9eadd 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -853,12 +853,7 @@ def get_deployment_type(self) -> DeploymentType: return DeploymentType.FULL def dump_to_primitive_dict(self): - custom_serialized_fields = { - "database", - "queue", - "redis", - "spider_db" - } + custom_serialized_fields = {"database", "queue", "redis", "spider_db"} d = self.model_dump(exclude=custom_serialized_fields) for key in custom_serialized_fields: value = getattr(self, key) From e7cb62fc4d8577cd095bf50a9e2c7d5fc7f66e43 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 18 Nov 2025 17:52:02 -0500 Subject: [PATCH 360/408] Apply suggestions from code review Co-authored-by: Junhao Liao --- tools/deployment/package/docker-compose-spider-base.yaml | 2 +- tools/deployment/package/docker-compose-spider.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package/docker-compose-spider-base.yaml b/tools/deployment/package/docker-compose-spider-base.yaml index 98c11ba1ea..7676f28b66 100644 --- a/tools/deployment/package/docker-compose-spider-base.yaml +++ b/tools/deployment/package/docker-compose-spider-base.yaml @@ -23,7 +23,7 @@ services: extends: file: "docker-compose-all.yaml" service: "compression-scheduler" - # No need to depend on Redis and RabbitMQ + # Override: Spider does NOT require `queue` or `redis`. depends_on: db-table-creator: condition: "service_completed_successfully" diff --git a/tools/deployment/package/docker-compose-spider.yaml b/tools/deployment/package/docker-compose-spider.yaml index f2d5b68bfa..74b4713905 100644 --- a/tools/deployment/package/docker-compose-spider.yaml +++ b/tools/deployment/package/docker-compose-spider.yaml @@ -24,7 +24,7 @@ services: extends: file: "docker-compose-all.yaml" service: "compression-scheduler" - # No need to depend on Redis and RabbitMQ + # Override: Spider does NOT require `queue` or `redis`. depends_on: db-table-creator: condition: "service_completed_successfully" From bebb2e93e78627a2d448d3126935dca7a890400c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:29:50 -0500 Subject: [PATCH 361/408] Fix logger name --- components/clp-py-utils/clp_py_utils/initialize-spider-db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 3f37d24ac7..b3f882cc85 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -14,7 +14,7 @@ # Setup logging # Create logger -logger = logging.getLogger(__file__) +logger = logging.getLogger("initialize-spider-db") logger.setLevel(logging.INFO) # Setup console logging logging_console_handler = logging.StreamHandler() From 5b3b0cf14c115d74910bd521f59daee5bc9acf70 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:31:14 -0500 Subject: [PATCH 362/408] Fix deleted row --- tools/deployment/package/docker-compose-all.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index 3dfd1d8571..a3593fdd4e 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -376,6 +376,7 @@ services: garbage-collector: <<: *service_defaults hostname: "garbage_collector" + stop_grace_period: "10s" deploy: # Value must be either 0 or 1. Set to 0 to disable the garbage collector. replicas: "${CLP_GARBAGE_COLLECTOR_ENABLED:-1}" From c0db5208fce4d369f5252e4bbfec2b70f3179286 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:39:28 -0500 Subject: [PATCH 363/408] Use different credential for spider-db --- .../clp-py-utils/clp_py_utils/clp_config.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index ef51b9eadd..44310c6d46 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Annotated, Any, ClassVar, Literal, Optional, Union +from typing import Annotated, Any, ClassVar, Literal, Optional, Union, override from pydantic import ( BaseModel, @@ -29,6 +29,7 @@ DB_COMPONENT_NAME = "database" QUEUE_COMPONENT_NAME = "queue" REDIS_COMPONENT_NAME = "redis" +SPIDER_DB_COMPONENT_NAME = "spider_db" SPIDER_SCHEDULER_COMPONENT_NAME = "spider_scheduler" REDUCER_COMPONENT_NAME = "reducer" RESULTS_CACHE_COMPONENT_NAME = "results_cache" @@ -275,6 +276,19 @@ def get_url(self): self.ensure_credentials_loaded() return f"jdbc:mariadb://{self.host}:{self.port}/{self.name}?user={self.username}&password={self.password}" + @override + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): + config = read_yaml_config_file(credentials_file_path) + if config is None: + raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") + try: + self.username = get_config_value(config, f"{SPIDER_DB_COMPONENT_NAME}.user") + self.password = get_config_value(config, f"{SPIDER_DB_COMPONENT_NAME}.password") + except KeyError as ex: + raise ValueError( + f"Credentials file '{credentials_file_path}' does not contain key '{ex}'." + ) + class SpiderScheduler(BaseModel): host: str = "localhost" From e2a68895bfce28ea0c9f8e0c51e5ef1d53179d38 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:41:47 -0500 Subject: [PATCH 364/408] Print error when password is not available --- components/clp-py-utils/clp_py_utils/initialize-spider-db.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index b3f882cc85..a1c2d5ba3a 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -263,9 +263,8 @@ def main(argv): db_cursor.execute(f"""CREATE DATABASE IF NOT EXISTS `{db_name}`""") if db_password is not None: - db_cursor.execute( - f"""CREATE USER IF NOT EXISTS '{db_user}'@'%' IDENTIFIED BY '{db_password}'""" - ) + logger.error(f"Password must be set for Spider database user.") + return -1 else: db_cursor.execute(f"""CREATE USER IF NOT EXISTS '{db_user}'@'%' IDENTIFIED BY ''""") db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_user}'@'%'""") From 9ff80fa6f1b8682a40a566580c916e11d2239831 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:43:39 -0500 Subject: [PATCH 365/408] Rename variable for clarity --- .../clp_py_utils/initialize-spider-db.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index a1c2d5ba3a..17046ea398 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -247,30 +247,30 @@ def main(argv): with closing(sql_adapter.create_root_mariadb_connection()) as db_conn, closing( db_conn.cursor() ) as db_cursor: - db_name = spider_db_config.name - db_user = spider_db_config.username - db_password = spider_db_config.password - clp_user = clp_config.database.username - if not _validate_name(db_name): - logger.error(f"Invalid database name: {db_name}") + clp_db_user = clp_config.database.username + spider_db_name = spider_db_config.name + spider_db_user = spider_db_config.username + spider_db_password = spider_db_config.password + if not _validate_name(spider_db_name): + logger.error(f"Invalid database name: {spider_db_name}") return -1 - if not _validate_name(db_user): - logger.error(f"Invalid database user name: {db_user}") + if not _validate_name(spider_db_user): + logger.error(f"Invalid database user name: {spider_db_user}") return -1 - if not _validate_name(clp_user): - logger.error(f"Invalid CLP database user name: {clp_user}") + if not _validate_name(clp_db_user): + logger.error(f"Invalid CLP database user name: {clp_db_user}") return -1 - db_cursor.execute(f"""CREATE DATABASE IF NOT EXISTS `{db_name}`""") - if db_password is not None: + db_cursor.execute(f"""CREATE DATABASE IF NOT EXISTS `{spider_db_name}`""") + if spider_db_password is not None: logger.error(f"Password must be set for Spider database user.") return -1 else: - db_cursor.execute(f"""CREATE USER IF NOT EXISTS '{db_user}'@'%' IDENTIFIED BY ''""") - db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_user}'@'%'""") - db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{clp_user}'@'%'""") + db_cursor.execute(f"""CREATE USER IF NOT EXISTS '{spider_db_user}'@'%' IDENTIFIED BY ''""") + db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{spider_db_name}`.* TO '{spider_db_user}'@'%'""") + db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{spider_db_name}`.* TO '{clp_db_user}'@'%'""") - db_cursor.execute(f"""USE `{db_name}`""") + db_cursor.execute(f"""USE `{spider_db_name}`""") for table_creator in table_creators: db_cursor.execute(table_creator) except Exception: From 2bc5a4f11614af2ee201a6db46cb2c4fc177ed6c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:45:22 -0500 Subject: [PATCH 366/408] Check for redis and queue --- .../clp_package_utils/scripts/start_clp.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index fd3762bec1..3ca3effd6f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -61,8 +61,10 @@ def main(argv): ) validate_and_load_db_credentials_file(clp_config, clp_home, True) - validate_and_load_queue_credentials_file(clp_config, clp_home, True) - validate_and_load_redis_credentials_file(clp_config, clp_home, True) + if clp_config.queue is not None: + validate_and_load_queue_credentials_file(clp_config, clp_home, True) + if clp_config.redis is not None: + validate_and_load_redis_credentials_file(clp_config, clp_home, True) if clp_config.compression_scheduler.type == OrchestrationType.spider: validate_and_load_spider_db_credentials_file(clp_config, clp_home, True) clp_config.validate_logs_input_config(True) From 4ce8b5322885270cae82c7f344d9b6cb0298f89e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:47:06 -0500 Subject: [PATCH 367/408] Add check for redis and queue --- .../clp-package-utils/clp_package_utils/controller.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 3f3ac3ccb7..b50c2ed0d3 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -806,8 +806,10 @@ def set_up_env(self) -> None: # Component-specific config env_vars |= self._set_up_env_for_database() - env_vars |= self._set_up_env_for_queue() - env_vars |= self._set_up_env_for_redis() + if self._clp_config.redis is not None: + env_vars |= self._set_up_env_for_queue() + if self._clp_config.queue is not None: + env_vars |= self._set_up_env_for_redis() if self._clp_config.compression_scheduler.type == OrchestrationType.spider: env_vars |= self._set_up_env_for_spider_db() env_vars |= self._set_up_env_for_spider_scheduler() From 4aab3f1714a3d60dffcf8ad4c8a706fe10ddcab6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:51:41 -0500 Subject: [PATCH 368/408] Add transform for container for spider scheduler --- components/clp-py-utils/clp_py_utils/clp_config.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 44310c6d46..9888bffa86 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -291,8 +291,14 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): class SpiderScheduler(BaseModel): + DEFAULT_PORT: ClassVar[int] = 6000 + host: str = "localhost" - port: Port = 6000 + port: Port = DEFAULT_PORT + + def transform_for_container(self): + self.host = SPIDER_SCHEDULER_COMPONENT_NAME + self.port = self.DEFAULT_PORT class CompressionScheduler(BaseModel): @@ -928,6 +934,8 @@ def transform_for_container(self): self.queue.transform_for_container() if self.redis is not None: self.redis.transform_for_container() + if self.spider_scheduler is not None: + self.spider_scheduler.transform_for_container() self.results_cache.transform_for_container() self.query_scheduler.transform_for_container() self.reducer.transform_for_container() From fbfd166630d6be9c70120e2d59320339f9a6af5b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:54:37 -0500 Subject: [PATCH 369/408] Add domain name --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 9888bffa86..34b63f2c79 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -293,7 +293,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): class SpiderScheduler(BaseModel): DEFAULT_PORT: ClassVar[int] = 6000 - host: str = "localhost" + host: DomainStr = "localhost" port: Port = DEFAULT_PORT def transform_for_container(self): From 36c2a1e8a07c3f2a0f040beee5523d207c8fd759 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:55:16 -0500 Subject: [PATCH 370/408] Use enum --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 34b63f2c79..8357552bff 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -268,7 +268,7 @@ class SpiderDb(Database): @field_validator("type") @classmethod def validate_type(cls, value): - if value != "mariadb": + if value != DatabaseEngine.MARIADB: raise ValueError(f"Spider only support MariaDB storage.") return value From cc89521198ec36c6fec98c193313e4e9687550db Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:57:15 -0500 Subject: [PATCH 371/408] Fix error message --- components/clp-py-utils/clp_py_utils/clp_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 8357552bff..7e60eabb72 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -897,10 +897,10 @@ def validate_spider_config(self): if orchestration_type != OrchestrationType.spider: return self if self.spider_db is None: - raise ValueError("SpiderDb config must be set when using spider orchestration.") + raise ValueError("`spider_db` must be configured when using Spider orchestration.") if self.spider_scheduler is None: - raise ValueError("SpiderScheduler config must be set when using spider orchestration.") - if self.database.type != "mariadb": + raise ValueError("`spider_db` must be configured when using Spider orchestration.") + if self.database.type != DatabaseEngine.MARIADB: raise ValueError("Database type must be 'mariadb' when using spider orchestration.") return self From a4d069f5dadfbe713fdb36178f2f0d393ee41e1d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 15:59:43 -0500 Subject: [PATCH 372/408] Fix error message --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 7e60eabb72..c90f9610db 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -269,7 +269,7 @@ class SpiderDb(Database): @classmethod def validate_type(cls, value): if value != DatabaseEngine.MARIADB: - raise ValueError(f"Spider only support MariaDB storage.") + raise ValueError(f"Spider only supports MariaDB for the metadata database.") return value def get_url(self): From 62c338cf6d66f660698262f6fee9e3fdd4e954d2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 16:01:31 -0500 Subject: [PATCH 373/408] Apply suggestions from code review Co-authored-by: Junhao Liao --- components/clp-py-utils/clp_py_utils/clp_config.py | 6 +++--- components/clp-py-utils/clp_py_utils/sql_adapter.py | 2 +- tools/deployment/package/docker-compose-all.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index ef51b9eadd..1492cbcdb7 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -881,7 +881,7 @@ def validate_spider_config(self): if self.spider_scheduler is None: raise ValueError("SpiderScheduler config must be set when using spider orchestration.") if self.database.type != "mariadb": - raise ValueError("Database type must be 'mariadb' when using spider orchestration.") + raise ValueError(f"Spider only supports MariaDB for the metadata database.") return self @model_validator(mode="after") @@ -890,9 +890,9 @@ def validate_celery_config(self): if orchestration_type != OrchestrationType.celery: return self if self.queue is None: - raise ValueError("Queue config must be set when using celery orchestration.") + raise ValueError("`queue` must be configured when using Celery orchestration.") if self.redis is None: - raise ValueError("Redis config must be set when using celery orchestration.") + raise ValueError("`redis` must be configured when using Celery orchestration.") return self def transform_for_container(self): diff --git a/components/clp-py-utils/clp_py_utils/sql_adapter.py b/components/clp-py-utils/clp_py_utils/sql_adapter.py index 9586948a9c..1ddc9f903e 100644 --- a/components/clp-py-utils/clp_py_utils/sql_adapter.py +++ b/components/clp-py-utils/clp_py_utils/sql_adapter.py @@ -108,7 +108,7 @@ def create_connection(self, disable_localhost_socket_connection: bool = False): else: raise NotImplementedError - def create_root_mariadb_connection(self, disable_localhost_socket_connection: bool = False): + def create_root_mariadb_connection(self, disable_localhost_socket_connection: bool = False) -> mariadb.Connection: if "mariadb" != self.database_config.type: raise NotImplementedError params = self.database_config.get_mysql_connection_params( diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index 3dfd1d8571..e046479bae 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -324,7 +324,7 @@ services: "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", "--storage-url", "${SPIDER_DB_URL}", # NOTE: Leave host to spider scheduler's host. This only affects task placement. - "--host", "${SPIDER_SCHEDULER_HOST}", + "--host", "spider-scheduler", ] webui: From e786e756efe9c3fa48c982dcd33f038aefa446f6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 16:06:12 -0500 Subject: [PATCH 374/408] Check concurrency argument --- components/clp-py-utils/clp_py_utils/start-spider-worker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index d64d08ef54..a48ce05921 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -23,6 +23,9 @@ def main() -> None: # Parse arguments args = parse_args() concurrency = args.concurrency + if concurrency < 1: + print("Error: Concurrency must be at least 1.") + exit(1) storage_url = args.storage_url host = args.host From 6312bfb74bdc1eccfc2fd1e6facc586fed7c3da2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 16:07:56 -0500 Subject: [PATCH 375/408] Use logger instead of print --- .../clp_py_utils/start-spider-worker.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index a48ce05921..237346d8b1 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -1,8 +1,18 @@ import argparse +import logging import os import pathlib import subprocess +# Setup logging +# Create logger +logger = logging.getLogger("start-spider-worker") +logger.setLevel(logging.INFO) +# Setup console logging +logging_console_handler = logging.StreamHandler() +logging_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") +logging_console_handler.setFormatter(logging_formatter) +logger.addHandler(logging_console_handler) def parse_args() -> argparse.Namespace: """ @@ -24,7 +34,7 @@ def main() -> None: args = parse_args() concurrency = args.concurrency if concurrency < 1: - print("Error: Concurrency must be at least 1.") + logger.error("Concurrency must be at least 1.") exit(1) storage_url = args.storage_url host = args.host @@ -32,7 +42,7 @@ def main() -> None: clp_home = os.getenv("CLP_HOME", "/opt/clp") spider_worker_path = pathlib.Path(clp_home) / "bin" / "spider_worker" if not spider_worker_path.exists(): - print(f"Error: spider_worker not found at {spider_worker_path}") + logger.error(f"spider_worker not found at {spider_worker_path}") exit(1) # Start multiple spider workers @@ -44,7 +54,7 @@ def main() -> None: ) processes.append(process) except OSError as e: - print(f"Failed to start spider worker: {e}") + logger.error(f"Failed to start spider worker: {e}") for process in processes: process.terminate() exit(1) @@ -53,7 +63,7 @@ def main() -> None: for process in processes: exit_code = process.wait() if exit_code != 0: - print(f"Spider worker exited with code {exit_code}") + logger.error(f"Spider worker exited with code {exit_code}") failed = True if failed: exit(1) From 99f16627ae222a362b021ba1d9986dc59bbeb221 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 16:08:28 -0500 Subject: [PATCH 376/408] Remove unnecessary fix --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 9640996810..a941844d47 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -901,7 +901,7 @@ def validate_spider_config(self): if self.spider_scheduler is None: raise ValueError("`spider_db` must be configured when using Spider orchestration.") if self.database.type != DatabaseEngine.MARIADB: - raise ValueError(f"Spider only supports MariaDB for the metadata database.") + raise ValueError("Spider only supports MariaDB for the metadata database.") return self @model_validator(mode="after") From ee6e5a4a9975ffdb0cb4f9bb598d271c927e7106 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 17:10:26 -0500 Subject: [PATCH 377/408] Skip creating spider table if not configured --- .../clp-py-utils/clp_py_utils/create-db-tables.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/create-db-tables.py b/components/clp-py-utils/clp_py_utils/create-db-tables.py index 03c268b441..021fa18566 100644 --- a/components/clp-py-utils/clp_py_utils/create-db-tables.py +++ b/components/clp-py-utils/clp_py_utils/create-db-tables.py @@ -4,7 +4,8 @@ import subprocess import sys -from clp_py_utils.clp_config import StorageEngine +from clp_py_utils.clp_config import StorageEngine, ClpConfig +from clp_py_utils.core import read_yaml_config_file # Setup logging # Create logger @@ -51,6 +52,15 @@ def main(argv): # fmt: on subprocess.run(cmd, check=True) + try: + clp_config = ClpConfig.model_validate(read_yaml_config_file(pathlib.Path(config_file_path))) + clp_config.database.load_credentials_from_env() + if clp_config.spider_db is None: + logger.info("No spider database configured, skipping spider DB initialization.") + return 0 + except Exception as e: + logger.error(f"Failed to load CLP configuration: {e}") + return 1 # fmt: off cmd = [ "python3", str(script_dir / "initialize-spider-db.py"), From f1781ac29cff8e5547647915d53419a906128b1d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 17:21:39 -0500 Subject: [PATCH 378/408] Use internal network address --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index b50c2ed0d3..9b8431bc0c 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -262,7 +262,7 @@ def _set_up_env_for_spider_db(self) -> EnvVarsDict: # Database env_vars |= { - "SPIDER_DB_URL": self._clp_config.spider_db.get_url(), + "SPIDER_DB_URL": self._clp_config.spider_db.get_container_url(), } # Credentials diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index a941844d47..dff5d5738b 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -276,6 +276,10 @@ def get_url(self): self.ensure_credentials_loaded() return f"jdbc:mariadb://{self.host}:{self.port}/{self.name}?user={self.username}&password={self.password}" + def get_container_url(self): + self.ensure_credentials_loaded() + return f"jdbc:mariadb://{DB_COMPONENT_NAME}:{self.DEFAULT_PORT}/{self.name}?user={self.username}&password={self.password}" + @override def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) From 0b652d714449cedb5e035db04ea4910eb9f50f51 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 17:39:58 -0500 Subject: [PATCH 379/408] Remove override --- components/clp-py-utils/clp_py_utils/clp_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 2dacfd9bb2..0e89527095 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -1,7 +1,7 @@ import os import pathlib from enum import auto -from typing import Annotated, Any, ClassVar, Literal, override +from typing import Annotated, Any, ClassVar, Literal from pydantic import ( BaseModel, @@ -280,7 +280,6 @@ def get_container_url(self): self.ensure_credentials_loaded() return f"jdbc:mariadb://{DB_COMPONENT_NAME}:{self.DEFAULT_PORT}/{self.name}?user={self.username}&password={self.password}" - @override def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: From 46a031b0ca20a65f67c0552e2001dd696329afed Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 18:02:55 -0500 Subject: [PATCH 380/408] Fix username --- components/clp-py-utils/clp_py_utils/clp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 0e89527095..c257ffd636 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -285,7 +285,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{SPIDER_DB_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{SPIDER_DB_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{SPIDER_DB_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( From bf72451ff7a1021822453e960838a267aaa31448 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 18:03:26 -0500 Subject: [PATCH 381/408] Fix spider db password --- components/clp-py-utils/clp_py_utils/initialize-spider-db.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 17046ea398..3ed3440e54 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -262,11 +262,10 @@ def main(argv): return -1 db_cursor.execute(f"""CREATE DATABASE IF NOT EXISTS `{spider_db_name}`""") - if spider_db_password is not None: + if spider_db_password is None: logger.error(f"Password must be set for Spider database user.") return -1 - else: - db_cursor.execute(f"""CREATE USER IF NOT EXISTS '{spider_db_user}'@'%' IDENTIFIED BY ''""") + db_cursor.execute(f"""CREATE USER IF NOT EXISTS '{spider_db_user}'@'%' IDENTIFIED BY '{spider_db_password}'""") db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{spider_db_name}`.* TO '{spider_db_user}'@'%'""") db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{spider_db_name}`.* TO '{clp_db_user}'@'%'""") From 0e9c1bb5aad3054e0bf7a579a4eb9926907f4148 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 20:12:35 -0500 Subject: [PATCH 382/408] Bug fix --- .../scheduler/compress/task_manager/spider_task_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py index 03365df003..65cfbb654b 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py @@ -9,7 +9,6 @@ from job_orchestration.executor.compress.spider_compress import compress from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager from job_orchestration.scheduler.task_result import CompressionTaskResult -from job_orchestration.utils.spider_utils import int8_list_to_utf8_str, utf8_str_to_int8_list class SpiderTaskManager(TaskManager): From 3c08754590d4f05bb25e7d3506123e0e8f0abd95 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 22:43:33 -0500 Subject: [PATCH 383/408] Pass user and password using env --- .../clp_package_utils/controller.py | 17 +++++++++-------- .../clp-py-utils/clp_py_utils/clp_config.py | 9 +++++++++ .../clp_py_utils/create-db-tables.py | 2 +- .../scheduler/compress/compression_scheduler.py | 2 +- .../deployment/package/docker-compose-all.yaml | 4 +++- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0ab95f6e11..71927d8277 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -33,7 +33,8 @@ SPIDER_SCHEDULER_COMPONENT_NAME, StorageEngine, StorageType, - WEBUI_COMPONENT_NAME, + WEBUI_COMPONENT_NAME, SPIDER_DB_USER_ENV_VAR_NAME, CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_REDIS_PASS_ENV_VAR_NAME, SPIDER_DB_PASS_ENV_VAR_NAME, CLP_QUEUE_USER_ENV_VAR_NAME, CLP_QUEUE_PASS_ENV_VAR_NAME, ) from clp_py_utils.clp_metadata_db_utils import ( get_archives_table_name, @@ -141,8 +142,8 @@ def _set_up_env_for_database(self) -> EnvVarsDict: # Credentials env_vars |= { - "CLP_DB_PASS": self._clp_config.database.password, - "CLP_DB_USER": self._clp_config.database.username, + CLP_DB_PASS_ENV_VAR_NAME: self._clp_config.database.password, + CLP_DB_USER_ENV_VAR_NAME: self._clp_config.database.username, } # Paths @@ -187,8 +188,8 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: # Credentials env_vars |= { - "CLP_QUEUE_PASS": self._clp_config.queue.password, - "CLP_QUEUE_USER": self._clp_config.queue.username, + CLP_QUEUE_PASS_ENV_VAR_NAME: self._clp_config.queue.password, + CLP_QUEUE_USER_ENV_VAR_NAME: self._clp_config.queue.username, } # Paths @@ -237,7 +238,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: # Credentials env_vars |= { - "CLP_REDIS_PASS": self._clp_config.redis.password, + CLP_REDIS_PASS_ENV_VAR_NAME: self._clp_config.redis.password, } # Paths @@ -267,8 +268,8 @@ def _set_up_env_for_spider_db(self) -> EnvVarsDict: # Credentials env_vars |= { - "SPIDER_DB_USER": self._clp_config.spider_db.username, - "SPIDER_DB_PASS": self._clp_config.spider_db.password, + SPIDER_DB_PASS_ENV_VAR_NAME: self._clp_config.spider_db.password, + SPIDER_DB_USER_ENV_VAR_NAME: self._clp_config.spider_db.username, } return env_vars diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c257ffd636..4808f05e36 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -76,6 +76,8 @@ CLP_QUEUE_USER_ENV_VAR_NAME = "CLP_QUEUE_USER" CLP_QUEUE_PASS_ENV_VAR_NAME = "CLP_QUEUE_PASS" CLP_REDIS_PASS_ENV_VAR_NAME = "CLP_REDIS_PASS" +SPIDER_DB_USER_ENV_VAR_NAME = "SPIDER_DB_USER" +SPIDER_DB_PASS_ENV_VAR_NAME = "SPIDER_DB_PASS" # Serializer StrEnumSerializer = PlainSerializer(serialize_str_enum) @@ -280,6 +282,13 @@ def get_container_url(self): self.ensure_credentials_loaded() return f"jdbc:mariadb://{DB_COMPONENT_NAME}:{self.DEFAULT_PORT}/{self.name}?user={self.username}&password={self.password}" + def load_credentials_from_env(self): + """ + :raise ValueError: if any expected environment variable is not set. + """ + self.username = _get_env_var(SPIDER_DB_USER_ENV_VAR_NAME) + self.password = _get_env_var(SPIDER_DB_PASS_ENV_VAR_NAME) + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: diff --git a/components/clp-py-utils/clp_py_utils/create-db-tables.py b/components/clp-py-utils/clp_py_utils/create-db-tables.py index 021fa18566..eee0c8b0a5 100644 --- a/components/clp-py-utils/clp_py_utils/create-db-tables.py +++ b/components/clp-py-utils/clp_py_utils/create-db-tables.py @@ -56,7 +56,7 @@ def main(argv): clp_config = ClpConfig.model_validate(read_yaml_config_file(pathlib.Path(config_file_path))) clp_config.database.load_credentials_from_env() if clp_config.spider_db is None: - logger.info("No spider database configured, skipping spider DB initialization.") + logger.info("No spider database configured. Skipping Spider database initialization.") return 0 except Exception as e: logger.error(f"Failed to load CLP configuration: {e}") diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 1d6e4842bd..ebe80e82ba 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -523,7 +523,7 @@ def main(argv): if clp_config.compression_scheduler.type == OrchestrationType.celery: task_manager = CeleryTaskManager() elif clp_config.compression_scheduler.type == OrchestrationType.spider: - task_manager = SpiderTaskManager(clp_config.spider_db.get_url()) + task_manager = SpiderTaskManager(clp_config.spider_db.get_container_url()) else: logger.error( f"Unsupported compression scheduler type: {clp_config.compression_scheduler.type}" diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index a02e59389a..c2a04067c5 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -84,6 +84,8 @@ services: CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" + SPIDER_DB_PASS: "${SPIDER_DB_PASS:?Please set a value.}" + SPIDER_DB_USER: "${SPIDER_DB_USER:?Please set a value.}" volumes: - *volume_clp_config_readonly depends_on: @@ -215,7 +217,7 @@ services: ports: - host_ip: "${SPIDER_SCHEDULER_HOST}" published: "${SPIDER_SCHEDULER_PORT}" - target: "${SPIDER_SCHEDULER_PORT}" + target: 6000 command: [ "/opt/clp/bin/spider_scheduler", "--host", "${SPIDER_SCHEDULER_HOST}", From 8eb7e235e3e4f8f08ceebfbe01ea06c6fab022d7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 23:00:46 -0500 Subject: [PATCH 384/408] Fix env --- .../scheduler/compress/compression_scheduler.py | 1 + tools/deployment/package/docker-compose-spider-base.yaml | 3 +++ tools/deployment/package/docker-compose-spider.yaml | 3 +++ 3 files changed, 7 insertions(+) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index ebe80e82ba..0e6db37aad 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -523,6 +523,7 @@ def main(argv): if clp_config.compression_scheduler.type == OrchestrationType.celery: task_manager = CeleryTaskManager() elif clp_config.compression_scheduler.type == OrchestrationType.spider: + clp_config.spider_db.load_credentials_from_env() task_manager = SpiderTaskManager(clp_config.spider_db.get_container_url()) else: logger.error( diff --git a/tools/deployment/package/docker-compose-spider-base.yaml b/tools/deployment/package/docker-compose-spider-base.yaml index 7676f28b66..3223f20d81 100644 --- a/tools/deployment/package/docker-compose-spider-base.yaml +++ b/tools/deployment/package/docker-compose-spider-base.yaml @@ -27,6 +27,9 @@ services: depends_on: db-table-creator: condition: "service_completed_successfully" + environment: + SPIDER_DB_PASS: "${SPIDER_DB_PASS:?Please set a value.}" + SPIDER_DB_USER: "${SPIDER_DB_USER:?Please set a value.}" compression-worker: extends: diff --git a/tools/deployment/package/docker-compose-spider.yaml b/tools/deployment/package/docker-compose-spider.yaml index 74b4713905..66d1924cb9 100644 --- a/tools/deployment/package/docker-compose-spider.yaml +++ b/tools/deployment/package/docker-compose-spider.yaml @@ -28,6 +28,9 @@ services: depends_on: db-table-creator: condition: "service_completed_successfully" + environment: + SPIDER_DB_PASS: "${SPIDER_DB_PASS:?Please set a value.}" + SPIDER_DB_USER: "${SPIDER_DB_USER:?Please set a value.}" compression-worker: extends: From d22b423375e896bbc630a03e1cdefbcf912afc30 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 23:31:00 -0500 Subject: [PATCH 385/408] Fix error path return --- .../job_orchestration/executor/compress/compression_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index ff10e13b48..73f28bfab2 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -618,7 +618,7 @@ def compression_entry_point( status=CompressionTaskStatus.FAILED, duration=0, error_message=error_msg, - ) + ).model_dump() clp_io_config = ClpIoConfig.model_validate_json(clp_io_config_json) paths_to_compress = PathsToCompress.model_validate_json(paths_to_compress_json) From 07e44a50a91cc27432e365755b86ba2b1938ea44 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 23:52:37 -0500 Subject: [PATCH 386/408] Fix yml name --- tools/deployment/package/docker-compose-all.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index c2a04067c5..522546e21c 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -299,7 +299,7 @@ services: <<: *service_defaults hostname: "compression_worker" environment: - CLP_CONFIG_PATH: "/etc/clp-config.yml" + CLP_CONFIG_PATH: "/etc/clp-config.yaml" CLP_HOME: "/opt/clp" CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/compression_worker" From 7379b6191d3681a86315a568e52b1c60be73c0ae Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Nov 2025 23:59:25 -0500 Subject: [PATCH 387/408] lint --- .../clp_package_utils/controller.py | 13 +- components/clp-package-utils/uv.lock | 536 +++++++++--------- .../clp-py-utils/clp_py_utils/clp_config.py | 3 +- .../clp_py_utils/create-db-tables.py | 2 +- .../clp_py_utils/initialize-spider-db.py | 21 +- .../clp-py-utils/clp_py_utils/sql_adapter.py | 9 +- .../clp_py_utils/start-spider-worker.py | 1 + components/clp-py-utils/uv.lock | 32 +- .../executor/compress/spider_compress.py | 2 +- components/job-orchestration/uv.lock | 2 +- 10 files changed, 319 insertions(+), 302 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 71927d8277..54bbe4da56 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -13,6 +13,11 @@ from clp_py_utils.clp_config import ( API_SERVER_COMPONENT_NAME, AwsAuthType, + CLP_DB_PASS_ENV_VAR_NAME, + CLP_DB_USER_ENV_VAR_NAME, + CLP_QUEUE_PASS_ENV_VAR_NAME, + CLP_QUEUE_USER_ENV_VAR_NAME, + CLP_REDIS_PASS_ENV_VAR_NAME, ClpConfig, COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, @@ -30,11 +35,12 @@ REDIS_COMPONENT_NAME, REDUCER_COMPONENT_NAME, RESULTS_CACHE_COMPONENT_NAME, + SPIDER_DB_PASS_ENV_VAR_NAME, + SPIDER_DB_USER_ENV_VAR_NAME, SPIDER_SCHEDULER_COMPONENT_NAME, StorageEngine, StorageType, - WEBUI_COMPONENT_NAME, SPIDER_DB_USER_ENV_VAR_NAME, CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, - CLP_REDIS_PASS_ENV_VAR_NAME, SPIDER_DB_PASS_ENV_VAR_NAME, CLP_QUEUE_USER_ENV_VAR_NAME, CLP_QUEUE_PASS_ENV_VAR_NAME, + WEBUI_COMPONENT_NAME, ) from clp_py_utils.clp_metadata_db_utils import ( get_archives_table_name, @@ -907,8 +913,7 @@ def _get_docker_file_name(self) -> str: if deployment_type == DeploymentType.BASE: if compression_scheduler_type == OrchestrationType.spider: return "docker-compose-spider-base.yaml" - else: - return "docker-compose.base.yaml" + return "docker-compose.base.yaml" if compression_scheduler_type == OrchestrationType.spider: return "docker-compose-spider.yaml" return "docker-compose.yaml" diff --git a/components/clp-package-utils/uv.lock b/components/clp-package-utils/uv.lock index c75465ab74..9eadd6fa7d 100644 --- a/components/clp-package-utils/uv.lock +++ b/components/clp-package-utils/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" [[package]] @@ -43,100 +43,88 @@ wheels = [ [[package]] name = "boto3" -version = "1.40.74" +version = "1.41.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a2/37/0db5fc46548b347255310893f1a47971a1d8eb0dbc46dfb5ace8a1e7d45e/boto3-1.40.74.tar.gz", hash = "sha256:484e46bf394b03a7c31b34f90945ebe1390cb1e2ac61980d128a9079beac87d4", size = 111592, upload-time = "2025-11-14T20:29:10.991Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/4f/92744c97f42e214948b9c8eff86e7e72c7ca8be788867a8aea80dc192052/boto3-1.41.0.tar.gz", hash = "sha256:73bf7f63152406404c0359c013a692e884b98a3b297160058a38f00ef19e375b", size = 111589, upload-time = "2025-11-19T20:29:10.72Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/08/c52751748762901c0ca3c3019e3aa950010217f0fdf9940ebe68e6bb2f5a/boto3-1.40.74-py3-none-any.whl", hash = "sha256:41fc8844b37ae27b24bcabf8369769df246cc12c09453988d0696ad06d6aa9ef", size = 139360, upload-time = "2025-11-14T20:29:09.477Z" }, + { url = "https://files.pythonhosted.org/packages/32/35/5f4b70f20188614a485b26e80369b9fa260a06fb0ae328153d7fc647619f/boto3-1.41.0-py3-none-any.whl", hash = "sha256:d5c454bb23655b052073c8dc6703dda5360825b72b1691822ae7709050b96390", size = 139340, upload-time = "2025-11-19T20:29:09.03Z" }, ] [[package]] name = "botocore" -version = "1.40.74" +version = "1.41.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/dc/0412505f05286f282a75bb0c650e525ddcfaf3f6f1a05cd8e99d32a2db06/botocore-1.40.74.tar.gz", hash = "sha256:57de0b9ffeada06015b3c7e5186c77d0692b210d9e5efa294f3214df97e2f8ee", size = 14452479, upload-time = "2025-11-14T20:29:00.949Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/8d/af94a3a0a5dc3ff255fdbd9a4bdf8e41beb33ea61ebab92e3d8e017f9ee4/botocore-1.41.0.tar.gz", hash = "sha256:555afbf86a644bfa4ebd7bd98d717b53b792e6bbb2c49f2b308fb06964cf1655", size = 14580059, upload-time = "2025-11-19T20:28:59.605Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/a2/306dec16e3c84f3ca7aaead0084358c1c7fbe6501f6160844cbc93bc871e/botocore-1.40.74-py3-none-any.whl", hash = "sha256:f39f5763e35e75f0bd91212b7b36120b1536203e8003cd952ef527db79702b15", size = 14117911, upload-time = "2025-11-14T20:28:58.153Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5c/65591ff3d30e790921635602bf53f60b89dd1f39a2cc0dad980b70dd569c/botocore-1.41.0-py3-none-any.whl", hash = "sha256:a5018d6268eee358dfc5d86e596c3062b4e225690acaf946f54c00063b804bf8", size = 14243471, upload-time = "2025-11-19T20:28:56.965Z" }, ] [[package]] name = "brotli" -version = "1.1.0" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270, upload-time = "2023-09-07T14:05:41.643Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/16/c92ca344d646e71a43b8bb353f0a6490d7f6e06210f8554c8f874e454285/brotli-1.2.0.tar.gz", hash = "sha256:e310f77e41941c13340a95976fe66a8a95b01e783d430eeaf7a2f87e0a57dd0a", size = 7388632, upload-time = "2025-11-05T18:39:42.86Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/3a/dbf4fb970c1019a57b5e492e1e0eae745d32e59ba4d6161ab5422b08eefe/Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752", size = 873045, upload-time = "2023-09-07T14:03:16.894Z" }, - { url = "https://files.pythonhosted.org/packages/dd/11/afc14026ea7f44bd6eb9316d800d439d092c8d508752055ce8d03086079a/Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9", size = 446218, upload-time = "2023-09-07T14:03:18.917Z" }, - { url = "https://files.pythonhosted.org/packages/36/83/7545a6e7729db43cb36c4287ae388d6885c85a86dd251768a47015dfde32/Brotli-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3", size = 2903872, upload-time = "2023-09-07T14:03:20.398Z" }, - { url = "https://files.pythonhosted.org/packages/32/23/35331c4d9391fcc0f29fd9bec2c76e4b4eeab769afbc4b11dd2e1098fb13/Brotli-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d", size = 2941254, upload-time = "2023-09-07T14:03:21.914Z" }, - { url = "https://files.pythonhosted.org/packages/3b/24/1671acb450c902edb64bd765d73603797c6c7280a9ada85a195f6b78c6e5/Brotli-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e", size = 2857293, upload-time = "2023-09-07T14:03:24Z" }, - { url = "https://files.pythonhosted.org/packages/d5/00/40f760cc27007912b327fe15bf6bfd8eaecbe451687f72a8abc587d503b3/Brotli-1.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da", size = 3002385, upload-time = "2023-09-07T14:03:26.248Z" }, - { url = "https://files.pythonhosted.org/packages/b8/cb/8aaa83f7a4caa131757668c0fb0c4b6384b09ffa77f2fba9570d87ab587d/Brotli-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80", size = 2911104, upload-time = "2023-09-07T14:03:27.849Z" }, - { url = "https://files.pythonhosted.org/packages/bc/c4/65456561d89d3c49f46b7fbeb8fe6e449f13bdc8ea7791832c5d476b2faf/Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d", size = 2809981, upload-time = "2023-09-07T14:03:29.92Z" }, - { url = "https://files.pythonhosted.org/packages/05/1b/cf49528437bae28abce5f6e059f0d0be6fecdcc1d3e33e7c54b3ca498425/Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0", size = 2935297, upload-time = "2023-09-07T14:03:32.035Z" }, - { url = "https://files.pythonhosted.org/packages/81/ff/190d4af610680bf0c5a09eb5d1eac6e99c7c8e216440f9c7cfd42b7adab5/Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e", size = 2930735, upload-time = "2023-09-07T14:03:33.801Z" }, - { url = "https://files.pythonhosted.org/packages/80/7d/f1abbc0c98f6e09abd3cad63ec34af17abc4c44f308a7a539010f79aae7a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c", size = 2933107, upload-time = "2024-10-18T12:32:09.016Z" }, - { url = "https://files.pythonhosted.org/packages/34/ce/5a5020ba48f2b5a4ad1c0522d095ad5847a0be508e7d7569c8630ce25062/Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1", size = 2845400, upload-time = "2024-10-18T12:32:11.134Z" }, - { url = "https://files.pythonhosted.org/packages/44/89/fa2c4355ab1eecf3994e5a0a7f5492c6ff81dfcb5f9ba7859bd534bb5c1a/Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2", size = 3031985, upload-time = "2024-10-18T12:32:12.813Z" }, - { url = "https://files.pythonhosted.org/packages/af/a4/79196b4a1674143d19dca400866b1a4d1a089040df7b93b88ebae81f3447/Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec", size = 2927099, upload-time = "2024-10-18T12:32:14.733Z" }, - { url = "https://files.pythonhosted.org/packages/e9/54/1c0278556a097f9651e657b873ab08f01b9a9ae4cac128ceb66427d7cd20/Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2", size = 333172, upload-time = "2023-09-07T14:03:35.212Z" }, - { url = "https://files.pythonhosted.org/packages/f7/65/b785722e941193fd8b571afd9edbec2a9b838ddec4375d8af33a50b8dab9/Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128", size = 357255, upload-time = "2023-09-07T14:03:36.447Z" }, - { url = "https://files.pythonhosted.org/packages/96/12/ad41e7fadd5db55459c4c401842b47f7fee51068f86dd2894dd0dcfc2d2a/Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc", size = 873068, upload-time = "2023-09-07T14:03:37.779Z" }, - { url = "https://files.pythonhosted.org/packages/95/4e/5afab7b2b4b61a84e9c75b17814198ce515343a44e2ed4488fac314cd0a9/Brotli-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6", size = 446244, upload-time = "2023-09-07T14:03:39.223Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e6/f305eb61fb9a8580c525478a4a34c5ae1a9bcb12c3aee619114940bc513d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd", size = 2906500, upload-time = "2023-09-07T14:03:40.858Z" }, - { url = "https://files.pythonhosted.org/packages/3e/4f/af6846cfbc1550a3024e5d3775ede1e00474c40882c7bf5b37a43ca35e91/Brotli-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf", size = 2943950, upload-time = "2023-09-07T14:03:42.896Z" }, - { url = "https://files.pythonhosted.org/packages/b3/e7/ca2993c7682d8629b62630ebf0d1f3bb3d579e667ce8e7ca03a0a0576a2d/Brotli-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61", size = 2918527, upload-time = "2023-09-07T14:03:44.552Z" }, - { url = "https://files.pythonhosted.org/packages/b3/96/da98e7bedc4c51104d29cc61e5f449a502dd3dbc211944546a4cc65500d3/Brotli-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327", size = 2845489, upload-time = "2023-09-07T14:03:46.594Z" }, - { url = "https://files.pythonhosted.org/packages/e8/ef/ccbc16947d6ce943a7f57e1a40596c75859eeb6d279c6994eddd69615265/Brotli-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd", size = 2914080, upload-time = "2023-09-07T14:03:48.204Z" }, - { url = "https://files.pythonhosted.org/packages/80/d6/0bd38d758d1afa62a5524172f0b18626bb2392d717ff94806f741fcd5ee9/Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9", size = 2813051, upload-time = "2023-09-07T14:03:50.348Z" }, - { url = "https://files.pythonhosted.org/packages/14/56/48859dd5d129d7519e001f06dcfbb6e2cf6db92b2702c0c2ce7d97e086c1/Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265", size = 2938172, upload-time = "2023-09-07T14:03:52.395Z" }, - { url = "https://files.pythonhosted.org/packages/3d/77/a236d5f8cd9e9f4348da5acc75ab032ab1ab2c03cc8f430d24eea2672888/Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8", size = 2933023, upload-time = "2023-09-07T14:03:53.96Z" }, - { url = "https://files.pythonhosted.org/packages/f1/87/3b283efc0f5cb35f7f84c0c240b1e1a1003a5e47141a4881bf87c86d0ce2/Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f", size = 2935871, upload-time = "2024-10-18T12:32:16.688Z" }, - { url = "https://files.pythonhosted.org/packages/f3/eb/2be4cc3e2141dc1a43ad4ca1875a72088229de38c68e842746b342667b2a/Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757", size = 2847784, upload-time = "2024-10-18T12:32:18.459Z" }, - { url = "https://files.pythonhosted.org/packages/66/13/b58ddebfd35edde572ccefe6890cf7c493f0c319aad2a5badee134b4d8ec/Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0", size = 3034905, upload-time = "2024-10-18T12:32:20.192Z" }, - { url = "https://files.pythonhosted.org/packages/84/9c/bc96b6c7db824998a49ed3b38e441a2cae9234da6fa11f6ed17e8cf4f147/Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b", size = 2929467, upload-time = "2024-10-18T12:32:21.774Z" }, - { url = "https://files.pythonhosted.org/packages/e7/71/8f161dee223c7ff7fea9d44893fba953ce97cf2c3c33f78ba260a91bcff5/Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50", size = 333169, upload-time = "2023-09-07T14:03:55.404Z" }, - { url = "https://files.pythonhosted.org/packages/02/8a/fece0ee1057643cb2a5bbf59682de13f1725f8482b2c057d4e799d7ade75/Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1", size = 357253, upload-time = "2023-09-07T14:03:56.643Z" }, - { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693, upload-time = "2024-10-18T12:32:23.824Z" }, - { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489, upload-time = "2024-10-18T12:32:25.641Z" }, - { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081, upload-time = "2023-09-07T14:03:57.967Z" }, - { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244, upload-time = "2023-09-07T14:03:59.319Z" }, - { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505, upload-time = "2023-09-07T14:04:01.327Z" }, - { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152, upload-time = "2023-09-07T14:04:03.033Z" }, - { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252, upload-time = "2023-09-07T14:04:04.675Z" }, - { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955, upload-time = "2023-09-07T14:04:06.585Z" }, - { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304, upload-time = "2023-09-07T14:04:08.668Z" }, - { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452, upload-time = "2023-09-07T14:04:10.736Z" }, - { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751, upload-time = "2023-09-07T14:04:12.875Z" }, - { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757, upload-time = "2023-09-07T14:04:14.551Z" }, - { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146, upload-time = "2024-10-18T12:32:27.257Z" }, - { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055, upload-time = "2024-10-18T12:32:29.376Z" }, - { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102, upload-time = "2024-10-18T12:32:31.371Z" }, - { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029, upload-time = "2024-10-18T12:32:33.293Z" }, - { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276, upload-time = "2023-09-07T14:04:16.49Z" }, - { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255, upload-time = "2023-09-07T14:04:17.83Z" }, - { url = "https://files.pythonhosted.org/packages/0a/9f/fb37bb8ffc52a8da37b1c03c459a8cd55df7a57bdccd8831d500e994a0ca/Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5", size = 815681, upload-time = "2024-10-18T12:32:34.942Z" }, - { url = "https://files.pythonhosted.org/packages/06/b3/dbd332a988586fefb0aa49c779f59f47cae76855c2d00f450364bb574cac/Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8", size = 422475, upload-time = "2024-10-18T12:32:36.485Z" }, - { url = "https://files.pythonhosted.org/packages/bb/80/6aaddc2f63dbcf2d93c2d204e49c11a9ec93a8c7c63261e2b4bd35198283/Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f", size = 2906173, upload-time = "2024-10-18T12:32:37.978Z" }, - { url = "https://files.pythonhosted.org/packages/ea/1d/e6ca79c96ff5b641df6097d299347507d39a9604bde8915e76bf026d6c77/Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648", size = 2943803, upload-time = "2024-10-18T12:32:39.606Z" }, - { url = "https://files.pythonhosted.org/packages/ac/a3/d98d2472e0130b7dd3acdbb7f390d478123dbf62b7d32bda5c830a96116d/Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0", size = 2918946, upload-time = "2024-10-18T12:32:41.679Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a5/c69e6d272aee3e1423ed005d8915a7eaa0384c7de503da987f2d224d0721/Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089", size = 2845707, upload-time = "2024-10-18T12:32:43.478Z" }, - { url = "https://files.pythonhosted.org/packages/58/9f/4149d38b52725afa39067350696c09526de0125ebfbaab5acc5af28b42ea/Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368", size = 2936231, upload-time = "2024-10-18T12:32:45.224Z" }, - { url = "https://files.pythonhosted.org/packages/5a/5a/145de884285611838a16bebfdb060c231c52b8f84dfbe52b852a15780386/Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c", size = 2848157, upload-time = "2024-10-18T12:32:46.894Z" }, - { url = "https://files.pythonhosted.org/packages/50/ae/408b6bfb8525dadebd3b3dd5b19d631da4f7d46420321db44cd99dcf2f2c/Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284", size = 3035122, upload-time = "2024-10-18T12:32:48.844Z" }, - { url = "https://files.pythonhosted.org/packages/af/85/a94e5cfaa0ca449d8f91c3d6f78313ebf919a0dbd55a100c711c6e9655bc/Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7", size = 2930206, upload-time = "2024-10-18T12:32:51.198Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f0/a61d9262cd01351df22e57ad7c34f66794709acab13f34be2675f45bf89d/Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0", size = 333804, upload-time = "2024-10-18T12:32:52.661Z" }, - { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517, upload-time = "2024-10-18T12:32:54.066Z" }, + { url = "https://files.pythonhosted.org/packages/64/10/a090475284fc4a71aed40a96f32e44a7fe5bda39687353dd977720b211b6/brotli-1.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b90b767916ac44e93a8e28ce6adf8d551e43affb512f2377c732d486ac6514e", size = 863089, upload-time = "2025-11-05T18:38:01.181Z" }, + { url = "https://files.pythonhosted.org/packages/03/41/17416630e46c07ac21e378c3464815dd2e120b441e641bc516ac32cc51d2/brotli-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6be67c19e0b0c56365c6a76e393b932fb0e78b3b56b711d180dd7013cb1fd984", size = 445442, upload-time = "2025-11-05T18:38:02.434Z" }, + { url = "https://files.pythonhosted.org/packages/24/31/90cc06584deb5d4fcafc0985e37741fc6b9717926a78674bbb3ce018957e/brotli-1.2.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0bbd5b5ccd157ae7913750476d48099aaf507a79841c0d04a9db4415b14842de", size = 1532658, upload-time = "2025-11-05T18:38:03.588Z" }, + { url = "https://files.pythonhosted.org/packages/62/17/33bf0c83bcbc96756dfd712201d87342732fad70bb3472c27e833a44a4f9/brotli-1.2.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3f3c908bcc404c90c77d5a073e55271a0a498f4e0756e48127c35d91cf155947", size = 1631241, upload-time = "2025-11-05T18:38:04.582Z" }, + { url = "https://files.pythonhosted.org/packages/48/10/f47854a1917b62efe29bc98ac18e5d4f71df03f629184575b862ef2e743b/brotli-1.2.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1b557b29782a643420e08d75aea889462a4a8796e9a6cf5621ab05a3f7da8ef2", size = 1424307, upload-time = "2025-11-05T18:38:05.587Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b7/f88eb461719259c17483484ea8456925ee057897f8e64487d76e24e5e38d/brotli-1.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81da1b229b1889f25adadc929aeb9dbc4e922bd18561b65b08dd9343cfccca84", size = 1488208, upload-time = "2025-11-05T18:38:06.613Z" }, + { url = "https://files.pythonhosted.org/packages/26/59/41bbcb983a0c48b0b8004203e74706c6b6e99a04f3c7ca6f4f41f364db50/brotli-1.2.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ff09cd8c5eec3b9d02d2408db41be150d8891c5566addce57513bf546e3d6c6d", size = 1597574, upload-time = "2025-11-05T18:38:07.838Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e6/8c89c3bdabbe802febb4c5c6ca224a395e97913b5df0dff11b54f23c1788/brotli-1.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a1778532b978d2536e79c05dac2d8cd857f6c55cd0c95ace5b03740824e0e2f1", size = 1492109, upload-time = "2025-11-05T18:38:08.816Z" }, + { url = "https://files.pythonhosted.org/packages/ed/9a/4b19d4310b2dbd545c0c33f176b0528fa68c3cd0754e34b2f2bcf56548ae/brotli-1.2.0-cp310-cp310-win32.whl", hash = "sha256:b232029d100d393ae3c603c8ffd7e3fe6f798c5e28ddca5feabb8e8fdb732997", size = 334461, upload-time = "2025-11-05T18:38:10.729Z" }, + { url = "https://files.pythonhosted.org/packages/ac/39/70981d9f47705e3c2b95c0847dfa3e7a37aa3b7c6030aedc4873081ed005/brotli-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:ef87b8ab2704da227e83a246356a2b179ef826f550f794b2c52cddb4efbd0196", size = 369035, upload-time = "2025-11-05T18:38:11.827Z" }, + { url = "https://files.pythonhosted.org/packages/7a/ef/f285668811a9e1ddb47a18cb0b437d5fc2760d537a2fe8a57875ad6f8448/brotli-1.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:15b33fe93cedc4caaff8a0bd1eb7e3dab1c61bb22a0bf5bdfdfd97cd7da79744", size = 863110, upload-time = "2025-11-05T18:38:12.978Z" }, + { url = "https://files.pythonhosted.org/packages/50/62/a3b77593587010c789a9d6eaa527c79e0848b7b860402cc64bc0bc28a86c/brotli-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:898be2be399c221d2671d29eed26b6b2713a02c2119168ed914e7d00ceadb56f", size = 445438, upload-time = "2025-11-05T18:38:14.208Z" }, + { url = "https://files.pythonhosted.org/packages/cd/e1/7fadd47f40ce5549dc44493877db40292277db373da5053aff181656e16e/brotli-1.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350c8348f0e76fff0a0fd6c26755d2653863279d086d3aa2c290a6a7251135dd", size = 1534420, upload-time = "2025-11-05T18:38:15.111Z" }, + { url = "https://files.pythonhosted.org/packages/12/8b/1ed2f64054a5a008a4ccd2f271dbba7a5fb1a3067a99f5ceadedd4c1d5a7/brotli-1.2.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e1ad3fda65ae0d93fec742a128d72e145c9c7a99ee2fcd667785d99eb25a7fe", size = 1632619, upload-time = "2025-11-05T18:38:16.094Z" }, + { url = "https://files.pythonhosted.org/packages/89/5a/7071a621eb2d052d64efd5da2ef55ecdac7c3b0c6e4f9d519e9c66d987ef/brotli-1.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:40d918bce2b427a0c4ba189df7a006ac0c7277c180aee4617d99e9ccaaf59e6a", size = 1426014, upload-time = "2025-11-05T18:38:17.177Z" }, + { url = "https://files.pythonhosted.org/packages/26/6d/0971a8ea435af5156acaaccec1a505f981c9c80227633851f2810abd252a/brotli-1.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2a7f1d03727130fc875448b65b127a9ec5d06d19d0148e7554384229706f9d1b", size = 1489661, upload-time = "2025-11-05T18:38:18.41Z" }, + { url = "https://files.pythonhosted.org/packages/f3/75/c1baca8b4ec6c96a03ef8230fab2a785e35297632f402ebb1e78a1e39116/brotli-1.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9c79f57faa25d97900bfb119480806d783fba83cd09ee0b33c17623935b05fa3", size = 1599150, upload-time = "2025-11-05T18:38:19.792Z" }, + { url = "https://files.pythonhosted.org/packages/0d/1a/23fcfee1c324fd48a63d7ebf4bac3a4115bdb1b00e600f80f727d850b1ae/brotli-1.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:844a8ceb8483fefafc412f85c14f2aae2fb69567bf2a0de53cdb88b73e7c43ae", size = 1493505, upload-time = "2025-11-05T18:38:20.913Z" }, + { url = "https://files.pythonhosted.org/packages/36/e5/12904bbd36afeef53d45a84881a4810ae8810ad7e328a971ebbfd760a0b3/brotli-1.2.0-cp311-cp311-win32.whl", hash = "sha256:aa47441fa3026543513139cb8926a92a8e305ee9c71a6209ef7a97d91640ea03", size = 334451, upload-time = "2025-11-05T18:38:21.94Z" }, + { url = "https://files.pythonhosted.org/packages/02/8b/ecb5761b989629a4758c394b9301607a5880de61ee2ee5fe104b87149ebc/brotli-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:022426c9e99fd65d9475dce5c195526f04bb8be8907607e27e747893f6ee3e24", size = 369035, upload-time = "2025-11-05T18:38:22.941Z" }, + { url = "https://files.pythonhosted.org/packages/11/ee/b0a11ab2315c69bb9b45a2aaed022499c9c24a205c3a49c3513b541a7967/brotli-1.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:35d382625778834a7f3061b15423919aa03e4f5da34ac8e02c074e4b75ab4f84", size = 861543, upload-time = "2025-11-05T18:38:24.183Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2f/29c1459513cd35828e25531ebfcbf3e92a5e49f560b1777a9af7203eb46e/brotli-1.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7a61c06b334bd99bc5ae84f1eeb36bfe01400264b3c352f968c6e30a10f9d08b", size = 444288, upload-time = "2025-11-05T18:38:25.139Z" }, + { url = "https://files.pythonhosted.org/packages/3d/6f/feba03130d5fceadfa3a1bb102cb14650798c848b1df2a808356f939bb16/brotli-1.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acec55bb7c90f1dfc476126f9711a8e81c9af7fb617409a9ee2953115343f08d", size = 1528071, upload-time = "2025-11-05T18:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/2b/38/f3abb554eee089bd15471057ba85f47e53a44a462cfce265d9bf7088eb09/brotli-1.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:260d3692396e1895c5034f204f0db022c056f9e2ac841593a4cf9426e2a3faca", size = 1626913, upload-time = "2025-11-05T18:38:27.284Z" }, + { url = "https://files.pythonhosted.org/packages/03/a7/03aa61fbc3c5cbf99b44d158665f9b0dd3d8059be16c460208d9e385c837/brotli-1.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:072e7624b1fc4d601036ab3f4f27942ef772887e876beff0301d261210bca97f", size = 1419762, upload-time = "2025-11-05T18:38:28.295Z" }, + { url = "https://files.pythonhosted.org/packages/21/1b/0374a89ee27d152a5069c356c96b93afd1b94eae83f1e004b57eb6ce2f10/brotli-1.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adedc4a67e15327dfdd04884873c6d5a01d3e3b6f61406f99b1ed4865a2f6d28", size = 1484494, upload-time = "2025-11-05T18:38:29.29Z" }, + { url = "https://files.pythonhosted.org/packages/cf/57/69d4fe84a67aef4f524dcd075c6eee868d7850e85bf01d778a857d8dbe0a/brotli-1.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7a47ce5c2288702e09dc22a44d0ee6152f2c7eda97b3c8482d826a1f3cfc7da7", size = 1593302, upload-time = "2025-11-05T18:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/d5/3b/39e13ce78a8e9a621c5df3aeb5fd181fcc8caba8c48a194cd629771f6828/brotli-1.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:af43b8711a8264bb4e7d6d9a6d004c3a2019c04c01127a868709ec29962b6036", size = 1487913, upload-time = "2025-11-05T18:38:31.618Z" }, + { url = "https://files.pythonhosted.org/packages/62/28/4d00cb9bd76a6357a66fcd54b4b6d70288385584063f4b07884c1e7286ac/brotli-1.2.0-cp312-cp312-win32.whl", hash = "sha256:e99befa0b48f3cd293dafeacdd0d191804d105d279e0b387a32054c1180f3161", size = 334362, upload-time = "2025-11-05T18:38:32.939Z" }, + { url = "https://files.pythonhosted.org/packages/1c/4e/bc1dcac9498859d5e353c9b153627a3752868a9d5f05ce8dedd81a2354ab/brotli-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:b35c13ce241abdd44cb8ca70683f20c0c079728a36a996297adb5334adfc1c44", size = 369115, upload-time = "2025-11-05T18:38:33.765Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d4/4ad5432ac98c73096159d9ce7ffeb82d151c2ac84adcc6168e476bb54674/brotli-1.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9e5825ba2c9998375530504578fd4d5d1059d09621a02065d1b6bfc41a8e05ab", size = 861523, upload-time = "2025-11-05T18:38:34.67Z" }, + { url = "https://files.pythonhosted.org/packages/91/9f/9cc5bd03ee68a85dc4bc89114f7067c056a3c14b3d95f171918c088bf88d/brotli-1.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0cf8c3b8ba93d496b2fae778039e2f5ecc7cff99df84df337ca31d8f2252896c", size = 444289, upload-time = "2025-11-05T18:38:35.6Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b6/fe84227c56a865d16a6614e2c4722864b380cb14b13f3e6bef441e73a85a/brotli-1.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8565e3cdc1808b1a34714b553b262c5de5fbda202285782173ec137fd13709f", size = 1528076, upload-time = "2025-11-05T18:38:36.639Z" }, + { url = "https://files.pythonhosted.org/packages/55/de/de4ae0aaca06c790371cf6e7ee93a024f6b4bb0568727da8c3de112e726c/brotli-1.2.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:26e8d3ecb0ee458a9804f47f21b74845cc823fd1bb19f02272be70774f56e2a6", size = 1626880, upload-time = "2025-11-05T18:38:37.623Z" }, + { url = "https://files.pythonhosted.org/packages/5f/16/a1b22cbea436642e071adcaf8d4b350a2ad02f5e0ad0da879a1be16188a0/brotli-1.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67a91c5187e1eec76a61625c77a6c8c785650f5b576ca732bd33ef58b0dff49c", size = 1419737, upload-time = "2025-11-05T18:38:38.729Z" }, + { url = "https://files.pythonhosted.org/packages/46/63/c968a97cbb3bdbf7f974ef5a6ab467a2879b82afbc5ffb65b8acbb744f95/brotli-1.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4ecdb3b6dc36e6d6e14d3a1bdc6c1057c8cbf80db04031d566eb6080ce283a48", size = 1484440, upload-time = "2025-11-05T18:38:39.916Z" }, + { url = "https://files.pythonhosted.org/packages/06/9d/102c67ea5c9fc171f423e8399e585dabea29b5bc79b05572891e70013cdd/brotli-1.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3e1b35d56856f3ed326b140d3c6d9db91740f22e14b06e840fe4bb1923439a18", size = 1593313, upload-time = "2025-11-05T18:38:41.24Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/9526d14fa6b87bc827ba1755a8440e214ff90de03095cacd78a64abe2b7d/brotli-1.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54a50a9dad16b32136b2241ddea9e4df159b41247b2ce6aac0b3276a66a8f1e5", size = 1487945, upload-time = "2025-11-05T18:38:42.277Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e8/3fe1ffed70cbef83c5236166acaed7bb9c766509b157854c80e2f766b38c/brotli-1.2.0-cp313-cp313-win32.whl", hash = "sha256:1b1d6a4efedd53671c793be6dd760fcf2107da3a52331ad9ea429edf0902f27a", size = 334368, upload-time = "2025-11-05T18:38:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/ff/91/e739587be970a113b37b821eae8097aac5a48e5f0eca438c22e4c7dd8648/brotli-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:b63daa43d82f0cdabf98dee215b375b4058cce72871fd07934f179885aad16e8", size = 369116, upload-time = "2025-11-05T18:38:44.609Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/298c2ddf786bb7347a1cd71d63a347a79e5712a7c0cba9e3c3458ebd976f/brotli-1.2.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6c12dad5cd04530323e723787ff762bac749a7b256a5bece32b2243dd5c27b21", size = 863080, upload-time = "2025-11-05T18:38:45.503Z" }, + { url = "https://files.pythonhosted.org/packages/84/0c/aac98e286ba66868b2b3b50338ffbd85a35c7122e9531a73a37a29763d38/brotli-1.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3219bd9e69868e57183316ee19c84e03e8f8b5a1d1f2667e1aa8c2f91cb061ac", size = 445453, upload-time = "2025-11-05T18:38:46.433Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f1/0ca1f3f99ae300372635ab3fe2f7a79fa335fee3d874fa7f9e68575e0e62/brotli-1.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:963a08f3bebd8b75ac57661045402da15991468a621f014be54e50f53a58d19e", size = 1528168, upload-time = "2025-11-05T18:38:47.371Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a6/2ebfc8f766d46df8d3e65b880a2e220732395e6d7dc312c1e1244b0f074a/brotli-1.2.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9322b9f8656782414b37e6af884146869d46ab85158201d82bab9abbcb971dc7", size = 1627098, upload-time = "2025-11-05T18:38:48.385Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2f/0976d5b097ff8a22163b10617f76b2557f15f0f39d6a0fe1f02b1a53e92b/brotli-1.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cf9cba6f5b78a2071ec6fb1e7bd39acf35071d90a81231d67e92d637776a6a63", size = 1419861, upload-time = "2025-11-05T18:38:49.372Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/d76df7176a2ce7616ff94c1fb72d307c9a30d2189fe877f3dd99af00ea5a/brotli-1.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7547369c4392b47d30a3467fe8c3330b4f2e0f7730e45e3103d7d636678a808b", size = 1484594, upload-time = "2025-11-05T18:38:50.655Z" }, + { url = "https://files.pythonhosted.org/packages/d3/93/14cf0b1216f43df5609f5b272050b0abd219e0b54ea80b47cef9867b45e7/brotli-1.2.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:fc1530af5c3c275b8524f2e24841cbe2599d74462455e9bae5109e9ff42e9361", size = 1593455, upload-time = "2025-11-05T18:38:51.624Z" }, + { url = "https://files.pythonhosted.org/packages/b3/73/3183c9e41ca755713bdf2cc1d0810df742c09484e2e1ddd693bee53877c1/brotli-1.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d2d085ded05278d1c7f65560aae97b3160aeb2ea2c0b3e26204856beccb60888", size = 1488164, upload-time = "2025-11-05T18:38:53.079Z" }, + { url = "https://files.pythonhosted.org/packages/64/6a/0c78d8f3a582859236482fd9fa86a65a60328a00983006bcf6d83b7b2253/brotli-1.2.0-cp314-cp314-win32.whl", hash = "sha256:832c115a020e463c2f67664560449a7bea26b0c1fdd690352addad6d0a08714d", size = 339280, upload-time = "2025-11-05T18:38:54.02Z" }, + { url = "https://files.pythonhosted.org/packages/f5/10/56978295c14794b2c12007b07f3e41ba26acda9257457d7085b0bb3bb90c/brotli-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e7c0af964e0b4e3412a0ebf341ea26ec767fa0b4cf81abb5e897c9338b5ad6a3", size = 375639, upload-time = "2025-11-05T18:38:55.67Z" }, ] [[package]] @@ -274,7 +262,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "boto3", specifier = ">=1.40.55" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "python-levenshtein", specifier = ">=0.27.1" }, @@ -403,14 +391,14 @@ dependencies = [ requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, { name = "celery", extras = ["redis"], specifier = ">=5.5.3" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "msgpack", specifier = ">=1.1.2" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pika", specifier = ">=1.3.2" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "pymongo", specifier = ">=4.15.3" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "yscope-spider-py", specifier = "==0.1.0" }, + { name = "yscope-spider-py", specifier = "==0.2.0" }, ] [package.metadata.requires-dev] @@ -547,12 +535,23 @@ wheels = [ [[package]] name = "mariadb" -version = "1.0.11" +version = "1.1.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/0a/ce5724f852f2937c6955bcea09659aa2d85d487df1c9de6711344b71527d/mariadb-1.0.11.zip", hash = "sha256:76916c892bc936c5b0f36e25a1411f651a7b7ce978992ae3a8de7e283efdacbf", size = 85926, upload-time = "2022-04-12T19:33:17.988Z" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/ba/cedef19833be88e07bfff11964441cda8a998f1628dd3b2fa3e7751d36e0/mariadb-1.1.14.tar.gz", hash = "sha256:e6d702a53eccf20922e47f2f45cfb5c7a0c2c6c0a46e4ee2d8a80d0ff4a52f34", size = 111715, upload-time = "2025-10-07T06:45:48.017Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/4a/6a6b7ad6a7b8156b41d0a6d849debe804f9d8696288ed2c6c31a9654357c/mariadb-1.0.11-cp310-cp310-win32.whl", hash = "sha256:4a583a80059d11f1895a2c93b1b7110948e87f0256da3e3222939a2530f0518e", size = 159743, upload-time = "2022-04-12T19:33:04.712Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a7/10740ceec1ce9d57f4bc3614e55efe2e72ae284e8c8d32eacabfbd7ad6cc/mariadb-1.0.11-cp310-cp310-win_amd64.whl", hash = "sha256:63c5c7cf99335e5c961e1d65a323576c9cb834e1a6a8084a6a8b4ffd85ca6213", size = 177316, upload-time = "2022-04-12T19:33:06.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/bf/5e1a3a5be297c3a679e08f5359165491508fbfb64faf854dc1d626cea9c0/mariadb-1.1.14-cp310-cp310-win32.whl", hash = "sha256:4c7f33578da163a1b79929aae241f5f981d7b9d5a94d89e589aad7ec58e313ea", size = 185064, upload-time = "2025-10-07T06:45:24.858Z" }, + { url = "https://files.pythonhosted.org/packages/31/30/3a61991c13cb8257f5db64aca12bafaa3d811d407e1fae019139fd17c99b/mariadb-1.1.14-cp310-cp310-win_amd64.whl", hash = "sha256:3f6fdc4ded5e0500a6a29bf0c8bf1be94189dcef5a8d5e0e154a4b3456f86bcc", size = 202020, upload-time = "2025-10-07T06:45:27.785Z" }, + { url = "https://files.pythonhosted.org/packages/56/aa/a7b3c66b2792e8319ec9157d63851ff2e0b26496a05044e22b50a012a05e/mariadb-1.1.14-cp311-cp311-win32.whl", hash = "sha256:932a95016b7e9b8d78893aa5ee608e74199e3c6dd607dbe5e4da2010a4f67b88", size = 185061, upload-time = "2025-10-07T06:45:29.964Z" }, + { url = "https://files.pythonhosted.org/packages/54/04/ea2374867756b4082764484bc8b82e1798d94f171bcc914e08c60d640f8f/mariadb-1.1.14-cp311-cp311-win_amd64.whl", hash = "sha256:55ddbe5272c292cbcb2968d87681b5d2b327e65646a015e324b8eeb804d14531", size = 202016, upload-time = "2025-10-07T06:45:32.151Z" }, + { url = "https://files.pythonhosted.org/packages/00/04/659a8d30513700b5921ec96bddc07f550016c045fcbeb199d8cd18476ecc/mariadb-1.1.14-cp312-cp312-win32.whl", hash = "sha256:98d552a8bb599eceaa88f65002ad00bd88aeed160592c273a7e5c1d79ab733dd", size = 185266, upload-time = "2025-10-07T06:45:34.164Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a9/8f210291bc5fc044e20497454f40d35b3bab326e2cab6fccdc38121cb2c1/mariadb-1.1.14-cp312-cp312-win_amd64.whl", hash = "sha256:685a1ad2a24fd0aae1c4416fe0ac794adc84ab9209c8d0c57078f770d39731db", size = 202112, upload-time = "2025-10-07T06:45:35.824Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/51130048bcce038bb859978250515f1aad90e9c4d273630a704e0a8b1ae1/mariadb-1.1.14-cp313-cp313-win32.whl", hash = "sha256:3d2c795cde606f4e12c0d73282b062433f414cae035675b0d81f2d65c9b79ac5", size = 185221, upload-time = "2025-10-07T06:45:37.848Z" }, + { url = "https://files.pythonhosted.org/packages/af/23/e952a7e442913abd8079cd27b80b69474895c93b3727fad41c7642a80c62/mariadb-1.1.14-cp313-cp313-win_amd64.whl", hash = "sha256:7fd603c5cf23c47ef0d28fdc2b4b79919ee7f75d00ed070d3cd1054dcf816aeb", size = 202121, upload-time = "2025-10-07T06:45:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f2/7059f83543a4264b98777d7cc8aa203e1ca6a13a461f730d1c97f29628d4/mariadb-1.1.14-cp314-cp314-win32.whl", hash = "sha256:1a50b4612c0dd5b69690cebb34cef552a7f64dcadeb5aa91d70cd99bf01bc5b3", size = 190620, upload-time = "2025-10-07T06:45:41.349Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7c/7e094b0b396d742494f6346f2ffa9709e429970b0461aca50526f5f02f12/mariadb-1.1.14-cp314-cp314-win_amd64.whl", hash = "sha256:6659725837e48fa6af05e20128fb525029f706f1921d5dbf639a25b2f80b9f93", size = 206263, upload-time = "2025-10-07T06:45:43.227Z" }, ] [[package]] @@ -758,23 +757,33 @@ wheels = [ [[package]] name = "psutil" -version = "7.1.0" +version = "7.1.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b3/31/4723d756b59344b643542936e37a31d1d3204bcdc42a7daa8ee9eb06fb50/psutil-7.1.0.tar.gz", hash = "sha256:655708b3c069387c8b77b072fc429a57d0e214221d01c0a772df7dfedcb3bcd2", size = 497660, upload-time = "2025-09-17T20:14:52.902Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/88/bdd0a41e5857d5d703287598cbf08dad90aed56774ea52ae071bae9071b6/psutil-7.1.3.tar.gz", hash = "sha256:6c86281738d77335af7aec228328e944b30930899ea760ecf33a4dba66be5e74", size = 489059, upload-time = "2025-11-02T12:25:54.619Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/62/ce4051019ee20ce0ed74432dd73a5bb087a6704284a470bb8adff69a0932/psutil-7.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76168cef4397494250e9f4e73eb3752b146de1dd950040b29186d0cce1d5ca13", size = 245242, upload-time = "2025-09-17T20:14:56.126Z" }, - { url = "https://files.pythonhosted.org/packages/38/61/f76959fba841bf5b61123fbf4b650886dc4094c6858008b5bf73d9057216/psutil-7.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:5d007560c8c372efdff9e4579c2846d71de737e4605f611437255e81efcca2c5", size = 246682, upload-time = "2025-09-17T20:14:58.25Z" }, - { url = "https://files.pythonhosted.org/packages/88/7a/37c99d2e77ec30d63398ffa6a660450b8a62517cabe44b3e9bae97696e8d/psutil-7.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22e4454970b32472ce7deaa45d045b34d3648ce478e26a04c7e858a0a6e75ff3", size = 287994, upload-time = "2025-09-17T20:14:59.901Z" }, - { url = "https://files.pythonhosted.org/packages/9d/de/04c8c61232f7244aa0a4b9a9fbd63a89d5aeaf94b2fc9d1d16e2faa5cbb0/psutil-7.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c70e113920d51e89f212dd7be06219a9b88014e63a4cec69b684c327bc474e3", size = 291163, upload-time = "2025-09-17T20:15:01.481Z" }, - { url = "https://files.pythonhosted.org/packages/f4/58/c4f976234bf6d4737bc8c02a81192f045c307b72cf39c9e5c5a2d78927f6/psutil-7.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d4a113425c037300de3ac8b331637293da9be9713855c4fc9d2d97436d7259d", size = 293625, upload-time = "2025-09-17T20:15:04.492Z" }, - { url = "https://files.pythonhosted.org/packages/79/87/157c8e7959ec39ced1b11cc93c730c4fb7f9d408569a6c59dbd92ceb35db/psutil-7.1.0-cp37-abi3-win32.whl", hash = "sha256:09ad740870c8d219ed8daae0ad3b726d3bf9a028a198e7f3080f6a1888b99bca", size = 244812, upload-time = "2025-09-17T20:15:07.462Z" }, - { url = "https://files.pythonhosted.org/packages/bf/e9/b44c4f697276a7a95b8e94d0e320a7bf7f3318521b23de69035540b39838/psutil-7.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:57f5e987c36d3146c0dd2528cd42151cf96cd359b9d67cfff836995cc5df9a3d", size = 247965, upload-time = "2025-09-17T20:15:09.673Z" }, - { url = "https://files.pythonhosted.org/packages/26/65/1070a6e3c036f39142c2820c4b52e9243246fcfc3f96239ac84472ba361e/psutil-7.1.0-cp37-abi3-win_arm64.whl", hash = "sha256:6937cb68133e7c97b6cc9649a570c9a18ba0efebed46d8c5dae4c07fa1b67a07", size = 244971, upload-time = "2025-09-17T20:15:12.262Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/0c49e776b8734fef56ec9c5c57f923922f2cf0497d62e0f419465f28f3d0/psutil-7.1.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0005da714eee687b4b8decd3d6cc7c6db36215c9e74e5ad2264b90c3df7d92dc", size = 239751, upload-time = "2025-11-02T12:25:58.161Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8d/b31e39c769e70780f007969815195a55c81a63efebdd4dbe9e7a113adb2f/psutil-7.1.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:19644c85dcb987e35eeeaefdc3915d059dac7bd1167cdcdbf27e0ce2df0c08c0", size = 240368, upload-time = "2025-11-02T12:26:00.491Z" }, + { url = "https://files.pythonhosted.org/packages/62/61/23fd4acc3c9eebbf6b6c78bcd89e5d020cfde4acf0a9233e9d4e3fa698b4/psutil-7.1.3-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95ef04cf2e5ba0ab9eaafc4a11eaae91b44f4ef5541acd2ee91d9108d00d59a7", size = 287134, upload-time = "2025-11-02T12:26:02.613Z" }, + { url = "https://files.pythonhosted.org/packages/30/1c/f921a009ea9ceb51aa355cb0cc118f68d354db36eae18174bab63affb3e6/psutil-7.1.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1068c303be3a72f8e18e412c5b2a8f6d31750fb152f9cb106b54090296c9d251", size = 289904, upload-time = "2025-11-02T12:26:05.207Z" }, + { url = "https://files.pythonhosted.org/packages/a6/82/62d68066e13e46a5116df187d319d1724b3f437ddd0f958756fc052677f4/psutil-7.1.3-cp313-cp313t-win_amd64.whl", hash = "sha256:18349c5c24b06ac5612c0428ec2a0331c26443d259e2a0144a9b24b4395b58fa", size = 249642, upload-time = "2025-11-02T12:26:07.447Z" }, + { url = "https://files.pythonhosted.org/packages/df/ad/c1cd5fe965c14a0392112f68362cfceb5230819dbb5b1888950d18a11d9f/psutil-7.1.3-cp313-cp313t-win_arm64.whl", hash = "sha256:c525ffa774fe4496282fb0b1187725793de3e7c6b29e41562733cae9ada151ee", size = 245518, upload-time = "2025-11-02T12:26:09.719Z" }, + { url = "https://files.pythonhosted.org/packages/2e/bb/6670bded3e3236eb4287c7bcdc167e9fae6e1e9286e437f7111caed2f909/psutil-7.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b403da1df4d6d43973dc004d19cee3b848e998ae3154cc8097d139b77156c353", size = 239843, upload-time = "2025-11-02T12:26:11.968Z" }, + { url = "https://files.pythonhosted.org/packages/b8/66/853d50e75a38c9a7370ddbeefabdd3d3116b9c31ef94dc92c6729bc36bec/psutil-7.1.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad81425efc5e75da3f39b3e636293360ad8d0b49bed7df824c79764fb4ba9b8b", size = 240369, upload-time = "2025-11-02T12:26:14.358Z" }, + { url = "https://files.pythonhosted.org/packages/41/bd/313aba97cb5bfb26916dc29cf0646cbe4dd6a89ca69e8c6edce654876d39/psutil-7.1.3-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f33a3702e167783a9213db10ad29650ebf383946e91bc77f28a5eb083496bc9", size = 288210, upload-time = "2025-11-02T12:26:16.699Z" }, + { url = "https://files.pythonhosted.org/packages/c2/fa/76e3c06e760927a0cfb5705eb38164254de34e9bd86db656d4dbaa228b04/psutil-7.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fac9cd332c67f4422504297889da5ab7e05fd11e3c4392140f7370f4208ded1f", size = 291182, upload-time = "2025-11-02T12:26:18.848Z" }, + { url = "https://files.pythonhosted.org/packages/0f/1d/5774a91607035ee5078b8fd747686ebec28a962f178712de100d00b78a32/psutil-7.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:3792983e23b69843aea49c8f5b8f115572c5ab64c153bada5270086a2123c7e7", size = 250466, upload-time = "2025-11-02T12:26:21.183Z" }, + { url = "https://files.pythonhosted.org/packages/00/ca/e426584bacb43a5cb1ac91fae1937f478cd8fbe5e4ff96574e698a2c77cd/psutil-7.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:31d77fcedb7529f27bb3a0472bea9334349f9a04160e8e6e5020f22c59893264", size = 245756, upload-time = "2025-11-02T12:26:23.148Z" }, + { url = "https://files.pythonhosted.org/packages/ef/94/46b9154a800253e7ecff5aaacdf8ebf43db99de4a2dfa18575b02548654e/psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2bdbcd0e58ca14996a42adf3621a6244f1bb2e2e528886959c72cf1e326677ab", size = 238359, upload-time = "2025-11-02T12:26:25.284Z" }, + { url = "https://files.pythonhosted.org/packages/68/3a/9f93cff5c025029a36d9a92fef47220ab4692ee7f2be0fba9f92813d0cb8/psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:bc31fa00f1fbc3c3802141eede66f3a2d51d89716a194bf2cd6fc68310a19880", size = 239171, upload-time = "2025-11-02T12:26:27.23Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b1/5f49af514f76431ba4eea935b8ad3725cdeb397e9245ab919dbc1d1dc20f/psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb428f9f05c1225a558f53e30ccbad9930b11c3fc206836242de1091d3e7dd3", size = 263261, upload-time = "2025-11-02T12:26:29.48Z" }, + { url = "https://files.pythonhosted.org/packages/e0/95/992c8816a74016eb095e73585d747e0a8ea21a061ed3689474fabb29a395/psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56d974e02ca2c8eb4812c3f76c30e28836fffc311d55d979f1465c1feeb2b68b", size = 264635, upload-time = "2025-11-02T12:26:31.74Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/c3ed1a622b6ae2fd3c945a366e64eb35247a31e4db16cf5095e269e8eb3c/psutil-7.1.3-cp37-abi3-win_amd64.whl", hash = "sha256:f39c2c19fe824b47484b96f9692932248a54c43799a84282cfe58d05a6449efd", size = 247633, upload-time = "2025-11-02T12:26:33.887Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ad/33b2ccec09bf96c2b2ef3f9a6f66baac8253d7565d8839e024a6b905d45d/psutil-7.1.3-cp37-abi3-win_arm64.whl", hash = "sha256:bd0d69cee829226a761e92f28140bec9a5ee9d5b4fb4b0cc589068dbfff559b1", size = 244608, upload-time = "2025-11-02T12:26:36.136Z" }, ] [[package]] name = "pydantic" -version = "2.12.3" +version = "2.12.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -782,194 +791,198 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/1e/4f0a3233767010308f2fd6bd0814597e3f63f1dc98304a9112b8759df4ff/pydantic-2.12.3.tar.gz", hash = "sha256:1da1c82b0fc140bb0103bc1441ffe062154c8d38491189751ee00fd8ca65ce74", size = 819383, upload-time = "2025-10-17T15:04:21.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/6b/83661fa77dcefa195ad5f8cd9af3d1a7450fd57cc883ad04d65446ac2029/pydantic-2.12.3-py3-none-any.whl", hash = "sha256:6986454a854bc3bc6e5443e1369e06a3a456af9d339eda45510f517d9ea5c6bf", size = 462431, upload-time = "2025-10-17T15:04:19.346Z" }, + { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" }, ] [[package]] name = "pydantic-core" -version = "2.41.4" +version = "2.41.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/18/d0944e8eaaa3efd0a91b0f1fc537d3be55ad35091b6a87638211ba691964/pydantic_core-2.41.4.tar.gz", hash = "sha256:70e47929a9d4a1905a67e4b687d5946026390568a8e952b92824118063cee4d5", size = 457557, upload-time = "2025-10-14T10:23:47.909Z" } +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/3d/9b8ca77b0f76fcdbf8bc6b72474e264283f461284ca84ac3fde570c6c49a/pydantic_core-2.41.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2442d9a4d38f3411f22eb9dd0912b7cbf4b7d5b6c92c4173b75d3e1ccd84e36e", size = 2111197, upload-time = "2025-10-14T10:19:43.303Z" }, - { url = "https://files.pythonhosted.org/packages/59/92/b7b0fe6ed4781642232755cb7e56a86e2041e1292f16d9ae410a0ccee5ac/pydantic_core-2.41.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:30a9876226dda131a741afeab2702e2d127209bde3c65a2b8133f428bc5d006b", size = 1917909, upload-time = "2025-10-14T10:19:45.194Z" }, - { url = "https://files.pythonhosted.org/packages/52/8c/3eb872009274ffa4fb6a9585114e161aa1a0915af2896e2d441642929fe4/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d55bbac04711e2980645af68b97d445cdbcce70e5216de444a6c4b6943ebcccd", size = 1969905, upload-time = "2025-10-14T10:19:46.567Z" }, - { url = "https://files.pythonhosted.org/packages/f4/21/35adf4a753bcfaea22d925214a0c5b880792e3244731b3f3e6fec0d124f7/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e1d778fb7849a42d0ee5927ab0f7453bf9f85eef8887a546ec87db5ddb178945", size = 2051938, upload-time = "2025-10-14T10:19:48.237Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d0/cdf7d126825e36d6e3f1eccf257da8954452934ede275a8f390eac775e89/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b65077a4693a98b90ec5ad8f203ad65802a1b9b6d4a7e48066925a7e1606706", size = 2250710, upload-time = "2025-10-14T10:19:49.619Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1c/af1e6fd5ea596327308f9c8d1654e1285cc3d8de0d584a3c9d7705bf8a7c/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62637c769dee16eddb7686bf421be48dfc2fae93832c25e25bc7242e698361ba", size = 2367445, upload-time = "2025-10-14T10:19:51.269Z" }, - { url = "https://files.pythonhosted.org/packages/d3/81/8cece29a6ef1b3a92f956ea6da6250d5b2d2e7e4d513dd3b4f0c7a83dfea/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfe3aa529c8f501babf6e502936b9e8d4698502b2cfab41e17a028d91b1ac7b", size = 2072875, upload-time = "2025-10-14T10:19:52.671Z" }, - { url = "https://files.pythonhosted.org/packages/e3/37/a6a579f5fc2cd4d5521284a0ab6a426cc6463a7b3897aeb95b12f1ba607b/pydantic_core-2.41.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca2322da745bf2eeb581fc9ea3bbb31147702163ccbcbf12a3bb630e4bf05e1d", size = 2191329, upload-time = "2025-10-14T10:19:54.214Z" }, - { url = "https://files.pythonhosted.org/packages/ae/03/505020dc5c54ec75ecba9f41119fd1e48f9e41e4629942494c4a8734ded1/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e8cd3577c796be7231dcf80badcf2e0835a46665eaafd8ace124d886bab4d700", size = 2151658, upload-time = "2025-10-14T10:19:55.843Z" }, - { url = "https://files.pythonhosted.org/packages/cb/5d/2c0d09fb53aa03bbd2a214d89ebfa6304be7df9ed86ee3dc7770257f41ee/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:1cae8851e174c83633f0833e90636832857297900133705ee158cf79d40f03e6", size = 2316777, upload-time = "2025-10-14T10:19:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/ea/4b/c2c9c8f5e1f9c864b57d08539d9d3db160e00491c9f5ee90e1bfd905e644/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a26d950449aae348afe1ac8be5525a00ae4235309b729ad4d3399623125b43c9", size = 2320705, upload-time = "2025-10-14T10:19:59.016Z" }, - { url = "https://files.pythonhosted.org/packages/28/c3/a74c1c37f49c0a02c89c7340fafc0ba816b29bd495d1a31ce1bdeacc6085/pydantic_core-2.41.4-cp310-cp310-win32.whl", hash = "sha256:0cf2a1f599efe57fa0051312774280ee0f650e11152325e41dfd3018ef2c1b57", size = 1975464, upload-time = "2025-10-14T10:20:00.581Z" }, - { url = "https://files.pythonhosted.org/packages/d6/23/5dd5c1324ba80303368f7569e2e2e1a721c7d9eb16acb7eb7b7f85cb1be2/pydantic_core-2.41.4-cp310-cp310-win_amd64.whl", hash = "sha256:a8c2e340d7e454dc3340d3d2e8f23558ebe78c98aa8f68851b04dcb7bc37abdc", size = 2024497, upload-time = "2025-10-14T10:20:03.018Z" }, - { url = "https://files.pythonhosted.org/packages/62/4c/f6cbfa1e8efacd00b846764e8484fe173d25b8dab881e277a619177f3384/pydantic_core-2.41.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:28ff11666443a1a8cf2a044d6a545ebffa8382b5f7973f22c36109205e65dc80", size = 2109062, upload-time = "2025-10-14T10:20:04.486Z" }, - { url = "https://files.pythonhosted.org/packages/21/f8/40b72d3868896bfcd410e1bd7e516e762d326201c48e5b4a06446f6cf9e8/pydantic_core-2.41.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61760c3925d4633290292bad462e0f737b840508b4f722247d8729684f6539ae", size = 1916301, upload-time = "2025-10-14T10:20:06.857Z" }, - { url = "https://files.pythonhosted.org/packages/94/4d/d203dce8bee7faeca791671c88519969d98d3b4e8f225da5b96dad226fc8/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eae547b7315d055b0de2ec3965643b0ab82ad0106a7ffd29615ee9f266a02827", size = 1968728, upload-time = "2025-10-14T10:20:08.353Z" }, - { url = "https://files.pythonhosted.org/packages/65/f5/6a66187775df87c24d526985b3a5d78d861580ca466fbd9d4d0e792fcf6c/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef9ee5471edd58d1fcce1c80ffc8783a650e3e3a193fe90d52e43bb4d87bff1f", size = 2050238, upload-time = "2025-10-14T10:20:09.766Z" }, - { url = "https://files.pythonhosted.org/packages/5e/b9/78336345de97298cf53236b2f271912ce11f32c1e59de25a374ce12f9cce/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15dd504af121caaf2c95cb90c0ebf71603c53de98305621b94da0f967e572def", size = 2249424, upload-time = "2025-10-14T10:20:11.732Z" }, - { url = "https://files.pythonhosted.org/packages/99/bb/a4584888b70ee594c3d374a71af5075a68654d6c780369df269118af7402/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a926768ea49a8af4d36abd6a8968b8790f7f76dd7cbd5a4c180db2b4ac9a3a2", size = 2366047, upload-time = "2025-10-14T10:20:13.647Z" }, - { url = "https://files.pythonhosted.org/packages/5f/8d/17fc5de9d6418e4d2ae8c675f905cdafdc59d3bf3bf9c946b7ab796a992a/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916b9b7d134bff5440098a4deb80e4cb623e68974a87883299de9124126c2a8", size = 2071163, upload-time = "2025-10-14T10:20:15.307Z" }, - { url = "https://files.pythonhosted.org/packages/54/e7/03d2c5c0b8ed37a4617430db68ec5e7dbba66358b629cd69e11b4d564367/pydantic_core-2.41.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cf90535979089df02e6f17ffd076f07237efa55b7343d98760bde8743c4b265", size = 2190585, upload-time = "2025-10-14T10:20:17.3Z" }, - { url = "https://files.pythonhosted.org/packages/be/fc/15d1c9fe5ad9266a5897d9b932b7f53d7e5cfc800573917a2c5d6eea56ec/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7533c76fa647fade2d7ec75ac5cc079ab3f34879626dae5689b27790a6cf5a5c", size = 2150109, upload-time = "2025-10-14T10:20:19.143Z" }, - { url = "https://files.pythonhosted.org/packages/26/ef/e735dd008808226c83ba56972566138665b71477ad580fa5a21f0851df48/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:37e516bca9264cbf29612539801ca3cd5d1be465f940417b002905e6ed79d38a", size = 2315078, upload-time = "2025-10-14T10:20:20.742Z" }, - { url = "https://files.pythonhosted.org/packages/90/00/806efdcf35ff2ac0f938362350cd9827b8afb116cc814b6b75cf23738c7c/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0c19cb355224037c83642429b8ce261ae108e1c5fbf5c028bac63c77b0f8646e", size = 2318737, upload-time = "2025-10-14T10:20:22.306Z" }, - { url = "https://files.pythonhosted.org/packages/41/7e/6ac90673fe6cb36621a2283552897838c020db343fa86e513d3f563b196f/pydantic_core-2.41.4-cp311-cp311-win32.whl", hash = "sha256:09c2a60e55b357284b5f31f5ab275ba9f7f70b7525e18a132ec1f9160b4f1f03", size = 1974160, upload-time = "2025-10-14T10:20:23.817Z" }, - { url = "https://files.pythonhosted.org/packages/e0/9d/7c5e24ee585c1f8b6356e1d11d40ab807ffde44d2db3b7dfd6d20b09720e/pydantic_core-2.41.4-cp311-cp311-win_amd64.whl", hash = "sha256:711156b6afb5cb1cb7c14a2cc2c4a8b4c717b69046f13c6b332d8a0a8f41ca3e", size = 2021883, upload-time = "2025-10-14T10:20:25.48Z" }, - { url = "https://files.pythonhosted.org/packages/33/90/5c172357460fc28b2871eb4a0fb3843b136b429c6fa827e4b588877bf115/pydantic_core-2.41.4-cp311-cp311-win_arm64.whl", hash = "sha256:6cb9cf7e761f4f8a8589a45e49ed3c0d92d1d696a45a6feaee8c904b26efc2db", size = 1968026, upload-time = "2025-10-14T10:20:27.039Z" }, - { url = "https://files.pythonhosted.org/packages/e9/81/d3b3e95929c4369d30b2a66a91db63c8ed0a98381ae55a45da2cd1cc1288/pydantic_core-2.41.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ab06d77e053d660a6faaf04894446df7b0a7e7aba70c2797465a0a1af00fc887", size = 2099043, upload-time = "2025-10-14T10:20:28.561Z" }, - { url = "https://files.pythonhosted.org/packages/58/da/46fdac49e6717e3a94fc9201403e08d9d61aa7a770fab6190b8740749047/pydantic_core-2.41.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c53ff33e603a9c1179a9364b0a24694f183717b2e0da2b5ad43c316c956901b2", size = 1910699, upload-time = "2025-10-14T10:20:30.217Z" }, - { url = "https://files.pythonhosted.org/packages/1e/63/4d948f1b9dd8e991a5a98b77dd66c74641f5f2e5225fee37994b2e07d391/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:304c54176af2c143bd181d82e77c15c41cbacea8872a2225dd37e6544dce9999", size = 1952121, upload-time = "2025-10-14T10:20:32.246Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a7/e5fc60a6f781fc634ecaa9ecc3c20171d238794cef69ae0af79ac11b89d7/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:025ba34a4cf4fb32f917d5d188ab5e702223d3ba603be4d8aca2f82bede432a4", size = 2041590, upload-time = "2025-10-14T10:20:34.332Z" }, - { url = "https://files.pythonhosted.org/packages/70/69/dce747b1d21d59e85af433428978a1893c6f8a7068fa2bb4a927fba7a5ff/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9f5f30c402ed58f90c70e12eff65547d3ab74685ffe8283c719e6bead8ef53f", size = 2219869, upload-time = "2025-10-14T10:20:35.965Z" }, - { url = "https://files.pythonhosted.org/packages/83/6a/c070e30e295403bf29c4df1cb781317b6a9bac7cd07b8d3acc94d501a63c/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd96e5d15385d301733113bcaa324c8bcf111275b7675a9c6e88bfb19fc05e3b", size = 2345169, upload-time = "2025-10-14T10:20:37.627Z" }, - { url = "https://files.pythonhosted.org/packages/f0/83/06d001f8043c336baea7fd202a9ac7ad71f87e1c55d8112c50b745c40324/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98f348cbb44fae6e9653c1055db7e29de67ea6a9ca03a5fa2c2e11a47cff0e47", size = 2070165, upload-time = "2025-10-14T10:20:39.246Z" }, - { url = "https://files.pythonhosted.org/packages/14/0a/e567c2883588dd12bcbc110232d892cf385356f7c8a9910311ac997ab715/pydantic_core-2.41.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec22626a2d14620a83ca583c6f5a4080fa3155282718b6055c2ea48d3ef35970", size = 2189067, upload-time = "2025-10-14T10:20:41.015Z" }, - { url = "https://files.pythonhosted.org/packages/f4/1d/3d9fca34273ba03c9b1c5289f7618bc4bd09c3ad2289b5420481aa051a99/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a95d4590b1f1a43bf33ca6d647b990a88f4a3824a8c4572c708f0b45a5290ed", size = 2132997, upload-time = "2025-10-14T10:20:43.106Z" }, - { url = "https://files.pythonhosted.org/packages/52/70/d702ef7a6cd41a8afc61f3554922b3ed8d19dd54c3bd4bdbfe332e610827/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:f9672ab4d398e1b602feadcffcdd3af44d5f5e6ddc15bc7d15d376d47e8e19f8", size = 2307187, upload-time = "2025-10-14T10:20:44.849Z" }, - { url = "https://files.pythonhosted.org/packages/68/4c/c06be6e27545d08b802127914156f38d10ca287a9e8489342793de8aae3c/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:84d8854db5f55fead3b579f04bda9a36461dab0730c5d570e1526483e7bb8431", size = 2305204, upload-time = "2025-10-14T10:20:46.781Z" }, - { url = "https://files.pythonhosted.org/packages/b0/e5/35ae4919bcd9f18603419e23c5eaf32750224a89d41a8df1a3704b69f77e/pydantic_core-2.41.4-cp312-cp312-win32.whl", hash = "sha256:9be1c01adb2ecc4e464392c36d17f97e9110fbbc906bcbe1c943b5b87a74aabd", size = 1972536, upload-time = "2025-10-14T10:20:48.39Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c2/49c5bb6d2a49eb2ee3647a93e3dae7080c6409a8a7558b075027644e879c/pydantic_core-2.41.4-cp312-cp312-win_amd64.whl", hash = "sha256:d682cf1d22bab22a5be08539dca3d1593488a99998f9f412137bc323179067ff", size = 2031132, upload-time = "2025-10-14T10:20:50.421Z" }, - { url = "https://files.pythonhosted.org/packages/06/23/936343dbcba6eec93f73e95eb346810fc732f71ba27967b287b66f7b7097/pydantic_core-2.41.4-cp312-cp312-win_arm64.whl", hash = "sha256:833eebfd75a26d17470b58768c1834dfc90141b7afc6eb0429c21fc5a21dcfb8", size = 1969483, upload-time = "2025-10-14T10:20:52.35Z" }, - { url = "https://files.pythonhosted.org/packages/13/d0/c20adabd181a029a970738dfe23710b52a31f1258f591874fcdec7359845/pydantic_core-2.41.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:85e050ad9e5f6fe1004eec65c914332e52f429bc0ae12d6fa2092407a462c746", size = 2105688, upload-time = "2025-10-14T10:20:54.448Z" }, - { url = "https://files.pythonhosted.org/packages/00/b6/0ce5c03cec5ae94cca220dfecddc453c077d71363b98a4bbdb3c0b22c783/pydantic_core-2.41.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7393f1d64792763a48924ba31d1e44c2cfbc05e3b1c2c9abb4ceeadd912cced", size = 1910807, upload-time = "2025-10-14T10:20:56.115Z" }, - { url = "https://files.pythonhosted.org/packages/68/3e/800d3d02c8beb0b5c069c870cbb83799d085debf43499c897bb4b4aaff0d/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94dab0940b0d1fb28bcab847adf887c66a27a40291eedf0b473be58761c9799a", size = 1956669, upload-time = "2025-10-14T10:20:57.874Z" }, - { url = "https://files.pythonhosted.org/packages/60/a4/24271cc71a17f64589be49ab8bd0751f6a0a03046c690df60989f2f95c2c/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:de7c42f897e689ee6f9e93c4bec72b99ae3b32a2ade1c7e4798e690ff5246e02", size = 2051629, upload-time = "2025-10-14T10:21:00.006Z" }, - { url = "https://files.pythonhosted.org/packages/68/de/45af3ca2f175d91b96bfb62e1f2d2f1f9f3b14a734afe0bfeff079f78181/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:664b3199193262277b8b3cd1e754fb07f2c6023289c815a1e1e8fb415cb247b1", size = 2224049, upload-time = "2025-10-14T10:21:01.801Z" }, - { url = "https://files.pythonhosted.org/packages/af/8f/ae4e1ff84672bf869d0a77af24fd78387850e9497753c432875066b5d622/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95b253b88f7d308b1c0b417c4624f44553ba4762816f94e6986819b9c273fb2", size = 2342409, upload-time = "2025-10-14T10:21:03.556Z" }, - { url = "https://files.pythonhosted.org/packages/18/62/273dd70b0026a085c7b74b000394e1ef95719ea579c76ea2f0cc8893736d/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1351f5bbdbbabc689727cb91649a00cb9ee7203e0a6e54e9f5ba9e22e384b84", size = 2069635, upload-time = "2025-10-14T10:21:05.385Z" }, - { url = "https://files.pythonhosted.org/packages/30/03/cf485fff699b4cdaea469bc481719d3e49f023241b4abb656f8d422189fc/pydantic_core-2.41.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1affa4798520b148d7182da0615d648e752de4ab1a9566b7471bc803d88a062d", size = 2194284, upload-time = "2025-10-14T10:21:07.122Z" }, - { url = "https://files.pythonhosted.org/packages/f9/7e/c8e713db32405dfd97211f2fc0a15d6bf8adb7640f3d18544c1f39526619/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7b74e18052fea4aa8dea2fb7dbc23d15439695da6cbe6cfc1b694af1115df09d", size = 2137566, upload-time = "2025-10-14T10:21:08.981Z" }, - { url = "https://files.pythonhosted.org/packages/04/f7/db71fd4cdccc8b75990f79ccafbbd66757e19f6d5ee724a6252414483fb4/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:285b643d75c0e30abda9dc1077395624f314a37e3c09ca402d4015ef5979f1a2", size = 2316809, upload-time = "2025-10-14T10:21:10.805Z" }, - { url = "https://files.pythonhosted.org/packages/76/63/a54973ddb945f1bca56742b48b144d85c9fc22f819ddeb9f861c249d5464/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f52679ff4218d713b3b33f88c89ccbf3a5c2c12ba665fb80ccc4192b4608dbab", size = 2311119, upload-time = "2025-10-14T10:21:12.583Z" }, - { url = "https://files.pythonhosted.org/packages/f8/03/5d12891e93c19218af74843a27e32b94922195ded2386f7b55382f904d2f/pydantic_core-2.41.4-cp313-cp313-win32.whl", hash = "sha256:ecde6dedd6fff127c273c76821bb754d793be1024bc33314a120f83a3c69460c", size = 1981398, upload-time = "2025-10-14T10:21:14.584Z" }, - { url = "https://files.pythonhosted.org/packages/be/d8/fd0de71f39db91135b7a26996160de71c073d8635edfce8b3c3681be0d6d/pydantic_core-2.41.4-cp313-cp313-win_amd64.whl", hash = "sha256:d081a1f3800f05409ed868ebb2d74ac39dd0c1ff6c035b5162356d76030736d4", size = 2030735, upload-time = "2025-10-14T10:21:16.432Z" }, - { url = "https://files.pythonhosted.org/packages/72/86/c99921c1cf6650023c08bfab6fe2d7057a5142628ef7ccfa9921f2dda1d5/pydantic_core-2.41.4-cp313-cp313-win_arm64.whl", hash = "sha256:f8e49c9c364a7edcbe2a310f12733aad95b022495ef2a8d653f645e5d20c1564", size = 1973209, upload-time = "2025-10-14T10:21:18.213Z" }, - { url = "https://files.pythonhosted.org/packages/36/0d/b5706cacb70a8414396efdda3d72ae0542e050b591119e458e2490baf035/pydantic_core-2.41.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ed97fd56a561f5eb5706cebe94f1ad7c13b84d98312a05546f2ad036bafe87f4", size = 1877324, upload-time = "2025-10-14T10:21:20.363Z" }, - { url = "https://files.pythonhosted.org/packages/de/2d/cba1fa02cfdea72dfb3a9babb067c83b9dff0bbcb198368e000a6b756ea7/pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a870c307bf1ee91fc58a9a61338ff780d01bfae45922624816878dce784095d2", size = 1884515, upload-time = "2025-10-14T10:21:22.339Z" }, - { url = "https://files.pythonhosted.org/packages/07/ea/3df927c4384ed9b503c9cc2d076cf983b4f2adb0c754578dfb1245c51e46/pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d25e97bc1f5f8f7985bdc2335ef9e73843bb561eb1fa6831fdfc295c1c2061cf", size = 2042819, upload-time = "2025-10-14T10:21:26.683Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ee/df8e871f07074250270a3b1b82aad4cd0026b588acd5d7d3eb2fcb1471a3/pydantic_core-2.41.4-cp313-cp313t-win_amd64.whl", hash = "sha256:d405d14bea042f166512add3091c1af40437c2e7f86988f3915fabd27b1e9cd2", size = 1995866, upload-time = "2025-10-14T10:21:28.951Z" }, - { url = "https://files.pythonhosted.org/packages/fc/de/b20f4ab954d6d399499c33ec4fafc46d9551e11dc1858fb7f5dca0748ceb/pydantic_core-2.41.4-cp313-cp313t-win_arm64.whl", hash = "sha256:19f3684868309db5263a11bace3c45d93f6f24afa2ffe75a647583df22a2ff89", size = 1970034, upload-time = "2025-10-14T10:21:30.869Z" }, - { url = "https://files.pythonhosted.org/packages/54/28/d3325da57d413b9819365546eb9a6e8b7cbd9373d9380efd5f74326143e6/pydantic_core-2.41.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e9205d97ed08a82ebb9a307e92914bb30e18cdf6f6b12ca4bedadb1588a0bfe1", size = 2102022, upload-time = "2025-10-14T10:21:32.809Z" }, - { url = "https://files.pythonhosted.org/packages/9e/24/b58a1bc0d834bf1acc4361e61233ee217169a42efbdc15a60296e13ce438/pydantic_core-2.41.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:82df1f432b37d832709fbcc0e24394bba04a01b6ecf1ee87578145c19cde12ac", size = 1905495, upload-time = "2025-10-14T10:21:34.812Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a4/71f759cc41b7043e8ecdaab81b985a9b6cad7cec077e0b92cff8b71ecf6b/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3b4cc4539e055cfa39a3763c939f9d409eb40e85813257dcd761985a108554", size = 1956131, upload-time = "2025-10-14T10:21:36.924Z" }, - { url = "https://files.pythonhosted.org/packages/b0/64/1e79ac7aa51f1eec7c4cda8cbe456d5d09f05fdd68b32776d72168d54275/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b1eb1754fce47c63d2ff57fdb88c351a6c0150995890088b33767a10218eaa4e", size = 2052236, upload-time = "2025-10-14T10:21:38.927Z" }, - { url = "https://files.pythonhosted.org/packages/e9/e3/a3ffc363bd4287b80f1d43dc1c28ba64831f8dfc237d6fec8f2661138d48/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6ab5ab30ef325b443f379ddb575a34969c333004fca5a1daa0133a6ffaad616", size = 2223573, upload-time = "2025-10-14T10:21:41.574Z" }, - { url = "https://files.pythonhosted.org/packages/28/27/78814089b4d2e684a9088ede3790763c64693c3d1408ddc0a248bc789126/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:31a41030b1d9ca497634092b46481b937ff9397a86f9f51bd41c4767b6fc04af", size = 2342467, upload-time = "2025-10-14T10:21:44.018Z" }, - { url = "https://files.pythonhosted.org/packages/92/97/4de0e2a1159cb85ad737e03306717637842c88c7fd6d97973172fb183149/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a44ac1738591472c3d020f61c6df1e4015180d6262ebd39bf2aeb52571b60f12", size = 2063754, upload-time = "2025-10-14T10:21:46.466Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/8cb90ce4b9efcf7ae78130afeb99fd1c86125ccdf9906ef64b9d42f37c25/pydantic_core-2.41.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d72f2b5e6e82ab8f94ea7d0d42f83c487dc159c5240d8f83beae684472864e2d", size = 2196754, upload-time = "2025-10-14T10:21:48.486Z" }, - { url = "https://files.pythonhosted.org/packages/34/3b/ccdc77af9cd5082723574a1cc1bcae7a6acacc829d7c0a06201f7886a109/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c4d1e854aaf044487d31143f541f7aafe7b482ae72a022c664b2de2e466ed0ad", size = 2137115, upload-time = "2025-10-14T10:21:50.63Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ba/e7c7a02651a8f7c52dc2cff2b64a30c313e3b57c7d93703cecea76c09b71/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b568af94267729d76e6ee5ececda4e283d07bbb28e8148bb17adad93d025d25a", size = 2317400, upload-time = "2025-10-14T10:21:52.959Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ba/6c533a4ee8aec6b812c643c49bb3bd88d3f01e3cebe451bb85512d37f00f/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6d55fb8b1e8929b341cc313a81a26e0d48aa3b519c1dbaadec3a6a2b4fcad025", size = 2312070, upload-time = "2025-10-14T10:21:55.419Z" }, - { url = "https://files.pythonhosted.org/packages/22/ae/f10524fcc0ab8d7f96cf9a74c880243576fd3e72bd8ce4f81e43d22bcab7/pydantic_core-2.41.4-cp314-cp314-win32.whl", hash = "sha256:5b66584e549e2e32a1398df11da2e0a7eff45d5c2d9db9d5667c5e6ac764d77e", size = 1982277, upload-time = "2025-10-14T10:21:57.474Z" }, - { url = "https://files.pythonhosted.org/packages/b4/dc/e5aa27aea1ad4638f0c3fb41132f7eb583bd7420ee63204e2d4333a3bbf9/pydantic_core-2.41.4-cp314-cp314-win_amd64.whl", hash = "sha256:557a0aab88664cc552285316809cab897716a372afaf8efdbef756f8b890e894", size = 2024608, upload-time = "2025-10-14T10:21:59.557Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/51d89cc2612bd147198e120a13f150afbf0bcb4615cddb049ab10b81b79e/pydantic_core-2.41.4-cp314-cp314-win_arm64.whl", hash = "sha256:3f1ea6f48a045745d0d9f325989d8abd3f1eaf47dd00485912d1a3a63c623a8d", size = 1967614, upload-time = "2025-10-14T10:22:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c2/472f2e31b95eff099961fa050c376ab7156a81da194f9edb9f710f68787b/pydantic_core-2.41.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6c1fe4c5404c448b13188dd8bd2ebc2bdd7e6727fa61ff481bcc2cca894018da", size = 1876904, upload-time = "2025-10-14T10:22:04.062Z" }, - { url = "https://files.pythonhosted.org/packages/4a/07/ea8eeb91173807ecdae4f4a5f4b150a520085b35454350fc219ba79e66a3/pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:523e7da4d43b113bf8e7b49fa4ec0c35bf4fe66b2230bfc5c13cc498f12c6c3e", size = 1882538, upload-time = "2025-10-14T10:22:06.39Z" }, - { url = "https://files.pythonhosted.org/packages/1e/29/b53a9ca6cd366bfc928823679c6a76c7a4c69f8201c0ba7903ad18ebae2f/pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5729225de81fb65b70fdb1907fcf08c75d498f4a6f15af005aabb1fdadc19dfa", size = 2041183, upload-time = "2025-10-14T10:22:08.812Z" }, - { url = "https://files.pythonhosted.org/packages/c7/3d/f8c1a371ceebcaf94d6dd2d77c6cf4b1c078e13a5837aee83f760b4f7cfd/pydantic_core-2.41.4-cp314-cp314t-win_amd64.whl", hash = "sha256:de2cfbb09e88f0f795fd90cf955858fc2c691df65b1f21f0aa00b99f3fbc661d", size = 1993542, upload-time = "2025-10-14T10:22:11.332Z" }, - { url = "https://files.pythonhosted.org/packages/8a/ac/9fc61b4f9d079482a290afe8d206b8f490e9fd32d4fc03ed4fc698214e01/pydantic_core-2.41.4-cp314-cp314t-win_arm64.whl", hash = "sha256:d34f950ae05a83e0ede899c595f312ca976023ea1db100cd5aa188f7005e3ab0", size = 1973897, upload-time = "2025-10-14T10:22:13.444Z" }, - { url = "https://files.pythonhosted.org/packages/b0/12/5ba58daa7f453454464f92b3ca7b9d7c657d8641c48e370c3ebc9a82dd78/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:a1b2cfec3879afb742a7b0bcfa53e4f22ba96571c9e54d6a3afe1052d17d843b", size = 2122139, upload-time = "2025-10-14T10:22:47.288Z" }, - { url = "https://files.pythonhosted.org/packages/21/fb/6860126a77725c3108baecd10fd3d75fec25191d6381b6eb2ac660228eac/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:d175600d975b7c244af6eb9c9041f10059f20b8bbffec9e33fdd5ee3f67cdc42", size = 1936674, upload-time = "2025-10-14T10:22:49.555Z" }, - { url = "https://files.pythonhosted.org/packages/de/be/57dcaa3ed595d81f8757e2b44a38240ac5d37628bce25fb20d02c7018776/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f184d657fa4947ae5ec9c47bd7e917730fa1cbb78195037e32dcbab50aca5ee", size = 1956398, upload-time = "2025-10-14T10:22:52.19Z" }, - { url = "https://files.pythonhosted.org/packages/2f/1d/679a344fadb9695f1a6a294d739fbd21d71fa023286daeea8c0ed49e7c2b/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed810568aeffed3edc78910af32af911c835cc39ebbfacd1f0ab5dd53028e5c", size = 2138674, upload-time = "2025-10-14T10:22:54.499Z" }, - { url = "https://files.pythonhosted.org/packages/c4/48/ae937e5a831b7c0dc646b2ef788c27cd003894882415300ed21927c21efa/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:4f5d640aeebb438517150fdeec097739614421900e4a08db4a3ef38898798537", size = 2112087, upload-time = "2025-10-14T10:22:56.818Z" }, - { url = "https://files.pythonhosted.org/packages/5e/db/6db8073e3d32dae017da7e0d16a9ecb897d0a4d92e00634916e486097961/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:4a9ab037b71927babc6d9e7fc01aea9e66dc2a4a34dff06ef0724a4049629f94", size = 1920387, upload-time = "2025-10-14T10:22:59.342Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c1/dd3542d072fcc336030d66834872f0328727e3b8de289c662faa04aa270e/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4dab9484ec605c3016df9ad4fd4f9a390bc5d816a3b10c6550f8424bb80b18c", size = 1951495, upload-time = "2025-10-14T10:23:02.089Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c6/db8d13a1f8ab3f1eb08c88bd00fd62d44311e3456d1e85c0e59e0a0376e7/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8a5028425820731d8c6c098ab642d7b8b999758e24acae03ed38a66eca8335", size = 2139008, upload-time = "2025-10-14T10:23:04.539Z" }, - { url = "https://files.pythonhosted.org/packages/5d/d4/912e976a2dd0b49f31c98a060ca90b353f3b73ee3ea2fd0030412f6ac5ec/pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e5ab4fc177dd41536b3c32b2ea11380dd3d4619a385860621478ac2d25ceb00", size = 2106739, upload-time = "2025-10-14T10:23:06.934Z" }, - { url = "https://files.pythonhosted.org/packages/71/f0/66ec5a626c81eba326072d6ee2b127f8c139543f1bf609b4842978d37833/pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d88d0054d3fa11ce936184896bed3c1c5441d6fa483b498fac6a5d0dd6f64a9", size = 1932549, upload-time = "2025-10-14T10:23:09.24Z" }, - { url = "https://files.pythonhosted.org/packages/c4/af/625626278ca801ea0a658c2dcf290dc9f21bb383098e99e7c6a029fccfc0/pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b2a054a8725f05b4b6503357e0ac1c4e8234ad3b0c2ac130d6ffc66f0e170e2", size = 2135093, upload-time = "2025-10-14T10:23:11.626Z" }, - { url = "https://files.pythonhosted.org/packages/20/f6/2fba049f54e0f4975fef66be654c597a1d005320fa141863699180c7697d/pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0d9db5a161c99375a0c68c058e227bee1d89303300802601d76a3d01f74e258", size = 2187971, upload-time = "2025-10-14T10:23:14.437Z" }, - { url = "https://files.pythonhosted.org/packages/0e/80/65ab839a2dfcd3b949202f9d920c34f9de5a537c3646662bdf2f7d999680/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6273ea2c8ffdac7b7fda2653c49682db815aebf4a89243a6feccf5e36c18c347", size = 2147939, upload-time = "2025-10-14T10:23:16.831Z" }, - { url = "https://files.pythonhosted.org/packages/44/58/627565d3d182ce6dfda18b8e1c841eede3629d59c9d7cbc1e12a03aeb328/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:4c973add636efc61de22530b2ef83a65f39b6d6f656df97f678720e20de26caa", size = 2311400, upload-time = "2025-10-14T10:23:19.234Z" }, - { url = "https://files.pythonhosted.org/packages/24/06/8a84711162ad5a5f19a88cead37cca81b4b1f294f46260ef7334ae4f24d3/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b69d1973354758007f46cf2d44a4f3d0933f10b6dc9bf15cf1356e037f6f731a", size = 2316840, upload-time = "2025-10-14T10:23:21.738Z" }, - { url = "https://files.pythonhosted.org/packages/aa/8b/b7bb512a4682a2f7fbfae152a755d37351743900226d29bd953aaf870eaa/pydantic_core-2.41.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3619320641fd212aaf5997b6ca505e97540b7e16418f4a241f44cdf108ffb50d", size = 2149135, upload-time = "2025-10-14T10:23:24.379Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7d/138e902ed6399b866f7cfe4435d22445e16fff888a1c00560d9dc79a780f/pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:491535d45cd7ad7e4a2af4a5169b0d07bebf1adfd164b0368da8aa41e19907a5", size = 2104721, upload-time = "2025-10-14T10:23:26.906Z" }, - { url = "https://files.pythonhosted.org/packages/47/13/0525623cf94627f7b53b4c2034c81edc8491cbfc7c28d5447fa318791479/pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:54d86c0cada6aba4ec4c047d0e348cbad7063b87ae0f005d9f8c9ad04d4a92a2", size = 1931608, upload-time = "2025-10-14T10:23:29.306Z" }, - { url = "https://files.pythonhosted.org/packages/d6/f9/744bc98137d6ef0a233f808bfc9b18cf94624bf30836a18d3b05d08bf418/pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eca1124aced216b2500dc2609eade086d718e8249cb9696660ab447d50a758bd", size = 2132986, upload-time = "2025-10-14T10:23:32.057Z" }, - { url = "https://files.pythonhosted.org/packages/17/c8/629e88920171173f6049386cc71f893dff03209a9ef32b4d2f7e7c264bcf/pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c9024169becccf0cb470ada03ee578d7348c119a0d42af3dcf9eda96e3a247c", size = 2187516, upload-time = "2025-10-14T10:23:34.871Z" }, - { url = "https://files.pythonhosted.org/packages/2e/0f/4f2734688d98488782218ca61bcc118329bf5de05bb7fe3adc7dd79b0b86/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:26895a4268ae5a2849269f4991cdc97236e4b9c010e51137becf25182daac405", size = 2146146, upload-time = "2025-10-14T10:23:37.342Z" }, - { url = "https://files.pythonhosted.org/packages/ed/f2/ab385dbd94a052c62224b99cf99002eee99dbec40e10006c78575aead256/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:ca4df25762cf71308c446e33c9b1fdca2923a3f13de616e2a949f38bf21ff5a8", size = 2311296, upload-time = "2025-10-14T10:23:40.145Z" }, - { url = "https://files.pythonhosted.org/packages/fc/8e/e4f12afe1beeb9823bba5375f8f258df0cc61b056b0195fb1cf9f62a1a58/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5a28fcedd762349519276c36634e71853b4541079cab4acaaac60c4421827308", size = 2315386, upload-time = "2025-10-14T10:23:42.624Z" }, - { url = "https://files.pythonhosted.org/packages/48/f7/925f65d930802e3ea2eb4d5afa4cb8730c8dc0d2cb89a59dc4ed2fcb2d74/pydantic_core-2.41.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c173ddcd86afd2535e2b695217e82191580663a1d1928239f877f5a1649ef39f", size = 2147775, upload-time = "2025-10-14T10:23:45.406Z" }, + { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" }, + { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" }, + { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" }, + { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" }, + { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" }, + { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" }, + { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" }, + { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" }, + { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" }, + { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" }, + { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" }, + { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" }, + { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] [[package]] name = "pymongo" -version = "4.15.3" +version = "4.15.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dnspython" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/7b/a709c85dc716eb85b69f71a4bb375cf1e72758a7e872103f27551243319c/pymongo-4.15.3.tar.gz", hash = "sha256:7a981271347623b5319932796690c2d301668ac3a1965974ac9f5c3b8a22cea5", size = 2470801, upload-time = "2025-10-07T21:57:50.384Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/81/6d66e62a5d1c5323dca79e9fb34ac8211df76f6c16625f9499a37b796314/pymongo-4.15.4.tar.gz", hash = "sha256:6ba7cdf46f03f406f77969a8081cfb659af16c0eee26b79a0a14e25f6c00827b", size = 2471218, upload-time = "2025-11-11T20:52:37.31Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/38/7ba7e7b57ccf2b04b63796c097c35b32339b2cb6e4d851d9dbb84426dc99/pymongo-4.15.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:482ca9b775747562ce1589df10c97a0e62a604ce5addf933e5819dd967c5e23c", size = 811331, upload-time = "2025-10-07T21:55:59.15Z" }, - { url = "https://files.pythonhosted.org/packages/11/36/4bd2aa400a64935b59d68d1c35c168bf61613f1f2bb824757079b2415cda/pymongo-4.15.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7eb497519f42ac89c30919a51f80e68a070cfc2f3b0543cac74833cd45a6b9c", size = 811673, upload-time = "2025-10-07T21:56:00.712Z" }, - { url = "https://files.pythonhosted.org/packages/37/fb/03c3bd14e6eb5236b360cff8598677c4b7b9557eed3021d9b3f6e82de51d/pymongo-4.15.3-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4a0a054e9937ec8fdb465835509b176f6b032851c8648f6a5d1b19932d0eacd6", size = 1185479, upload-time = "2025-10-07T21:56:02.297Z" }, - { url = "https://files.pythonhosted.org/packages/6d/27/b5f21d9a556e31d083bb17d0c026244a604a96f7bdb277fd48dee99415ee/pymongo-4.15.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49fd6e158cf75771b2685a8a221a40ab96010ae34dd116abd06371dc6c38ab60", size = 1203867, upload-time = "2025-10-07T21:56:03.621Z" }, - { url = "https://files.pythonhosted.org/packages/ba/09/ffe1a114d7a39f6746c27a6f5a717b1dc5ea763cb0458a9a679142f623aa/pymongo-4.15.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82a490f1ade4ec6a72068e3676b04c126e3043e69b38ec474a87c6444cf79098", size = 1242537, upload-time = "2025-10-07T21:56:04.973Z" }, - { url = "https://files.pythonhosted.org/packages/af/60/b7968e855284bb67d366dfb50b6a9df4f69676fbbae51f3e647d2dcb12eb/pymongo-4.15.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:982107c667921e896292f4be09c057e2f1a40c645c9bfc724af5dd5fb8398094", size = 1232832, upload-time = "2025-10-07T21:56:06.287Z" }, - { url = "https://files.pythonhosted.org/packages/23/47/763945c63690d5c1a54d1d2ace352ba150b9e49a5cfdf44fb237e092e604/pymongo-4.15.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:45aebbd369ca79b7c46eaea5b04d2e4afca4eda117b68965a07a9da05d774e4d", size = 1200177, upload-time = "2025-10-07T21:56:07.671Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c2/1ace9cf4b88addceb5077e5490238a9e20dc9fef75ae4de146f57f408a06/pymongo-4.15.3-cp310-cp310-win32.whl", hash = "sha256:90ad56bd1d769d2f44af74f0fd0c276512361644a3c636350447994412cbc9a1", size = 798320, upload-time = "2025-10-07T21:56:09.917Z" }, - { url = "https://files.pythonhosted.org/packages/1c/b7/86563ec80fc41f644c813a3625d8b5672fd1d2b52da53727eca766dfc162/pymongo-4.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:8bd6dd736f5d07a825caf52c38916d5452edc0fac7aee43ec67aba6f61c2dbb7", size = 808150, upload-time = "2025-10-07T21:56:11.562Z" }, - { url = "https://files.pythonhosted.org/packages/d5/b3/f136483c3d13224ad0b80ac2b7c8f7adb735a296b5e8c94cfc2415b77d70/pymongo-4.15.3-cp310-cp310-win_arm64.whl", hash = "sha256:300eaf83ad053e51966be1839324341b08eaf880d3dc63ada7942d5912e09c49", size = 800930, upload-time = "2025-10-07T21:56:12.917Z" }, - { url = "https://files.pythonhosted.org/packages/73/04/3dbc426c5868961d8308f19750243f8472f587f5f8a5029ce6953ba74b82/pymongo-4.15.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39a13d8f7141294404ce46dfbabb2f2d17e9b1192456651ae831fa351f86fbeb", size = 865889, upload-time = "2025-10-07T21:56:14.165Z" }, - { url = "https://files.pythonhosted.org/packages/8c/39/7f7652f53dd0eb0c4c3420a175183da757e9c53f9a2bf3ebc589758a1b9e/pymongo-4.15.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:17d13458baf4a6a9f2e787d95adf8ec50d412accb9926a044bd1c41029c323b2", size = 866230, upload-time = "2025-10-07T21:56:15.587Z" }, - { url = "https://files.pythonhosted.org/packages/6a/0b/84e119e6bab7b19cf4fa1ebb9b4c29bf6c0e76521ed8221b44e3f94a3a37/pymongo-4.15.3-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fe4bcb8acfb288e238190397d4a699aeb4adb70e8545a6f4e44f99d4e8096ab1", size = 1429788, upload-time = "2025-10-07T21:56:17.362Z" }, - { url = "https://files.pythonhosted.org/packages/30/39/9905fcb99903de6ac8483114d1c85efe56bc5df735857bdfcc372cf8a3ec/pymongo-4.15.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d09d895c7f08bcbed4d2e96a00e52e9e545ae5a37b32d2dc10099b205a21fc6d", size = 1456758, upload-time = "2025-10-07T21:56:18.841Z" }, - { url = "https://files.pythonhosted.org/packages/08/58/3c3ac32b8d6ebb654083d53f58e4621cd4c7f306b3b85acef667b80acf08/pymongo-4.15.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21c0a95a4db72562fd0805e2f76496bf432ba2e27a5651f4b9c670466260c258", size = 1514666, upload-time = "2025-10-07T21:56:20.488Z" }, - { url = "https://files.pythonhosted.org/packages/19/e2/52f41de224218dc787b7e1187a1ca1a51946dcb979ee553ec917745ccd8d/pymongo-4.15.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89e45d7fa987f4e246cdf43ff001e3f911f73eb19ba9dabc2a6d80df5c97883b", size = 1500703, upload-time = "2025-10-07T21:56:21.874Z" }, - { url = "https://files.pythonhosted.org/packages/34/0d/a5271073339ba6fc8a5f4e3a62baaa5dd8bf35246c37b512317e2a22848e/pymongo-4.15.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1246a82fa6dd73ac2c63aa7e463752d5d1ca91e0c7a23396b78f21273befd3a7", size = 1452013, upload-time = "2025-10-07T21:56:23.526Z" }, - { url = "https://files.pythonhosted.org/packages/a0/3b/f39b721ca0db9f0820e12eeffec84eb87b7502abb13a685226c5434f9618/pymongo-4.15.3-cp311-cp311-win32.whl", hash = "sha256:9483521c03f6017336f54445652ead3145154e8d3ea06418e52cea57fee43292", size = 844461, upload-time = "2025-10-07T21:56:24.867Z" }, - { url = "https://files.pythonhosted.org/packages/12/72/e58b9df862edbf238a1d71fa32749a6eaf30a3f60289602681351c29093a/pymongo-4.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:c57dad9f289d72af1d7c47a444c4d9fa401f951cedbbcc54c7dd0c2107d6d786", size = 859200, upload-time = "2025-10-07T21:56:26.393Z" }, - { url = "https://files.pythonhosted.org/packages/81/8f/64c15df5e87de759412c3b962950561202c9b39e5cc604061e056043e163/pymongo-4.15.3-cp311-cp311-win_arm64.whl", hash = "sha256:2fd3b99520f2bb013960ac29dece1b43f2f1b6d94351ca33ba1b1211ecf79a09", size = 848372, upload-time = "2025-10-07T21:56:27.994Z" }, - { url = "https://files.pythonhosted.org/packages/5b/92/7491a2046b41bfd3641da0a23529c88e27eac67c681de3cd9fbef4113d38/pymongo-4.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bd0497c564b0ae34fb816464ffc09986dd9ca29e2772a0f7af989e472fecc2ad", size = 920953, upload-time = "2025-10-07T21:56:29.737Z" }, - { url = "https://files.pythonhosted.org/packages/ce/0c/98864cbfa8fbc954ae7480c91a35f0dc4e3339dab0c55f669e4dbeac808f/pymongo-4.15.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:292fd5a3f045751a823a54cdea75809b2216a62cc5f74a1a96b337db613d46a8", size = 920690, upload-time = "2025-10-07T21:56:31.094Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a6/7dc8043a10a1c30153be2d6847ab37911b169d53a6b05d21871b35b3de82/pymongo-4.15.3-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:959ef69c5e687b6b749fbf2140c7062abdb4804df013ae0507caabf30cba6875", size = 1690357, upload-time = "2025-10-07T21:56:32.466Z" }, - { url = "https://files.pythonhosted.org/packages/0b/96/3d85da60094d2022217f2849e1b61a79af9d51ed8d05455d7413d68ab88e/pymongo-4.15.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de3bc878c3be54ae41c2cabc9e9407549ed4fec41f4e279c04e840dddd7c630c", size = 1726102, upload-time = "2025-10-07T21:56:33.952Z" }, - { url = "https://files.pythonhosted.org/packages/ac/fd/dfd6ddee0330171f2f52f7e5344c02d25d2dd8dfa95ce0e5e413579f52fd/pymongo-4.15.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07bcc36d11252f24fe671e7e64044d39a13d997b0502c6401161f28cc144f584", size = 1800630, upload-time = "2025-10-07T21:56:35.632Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3b/e19a5f2de227ff720bc76c41d166d508e6fbe1096ba1ad18ade43b790b5e/pymongo-4.15.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b63bac343b79bd209e830aac1f5d9d552ff415f23a924d3e51abbe3041265436", size = 1785478, upload-time = "2025-10-07T21:56:37.39Z" }, - { url = "https://files.pythonhosted.org/packages/75/d2/927c9b1383c6708fc50c3700ecb1c2876e67dde95ad5fb1d29d04e8ac083/pymongo-4.15.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b33d59bf6fa1ca1d7d96d4fccff51e41312358194190d53ef70a84c070f5287e", size = 1718548, upload-time = "2025-10-07T21:56:38.754Z" }, - { url = "https://files.pythonhosted.org/packages/fe/10/a63592d1445f894b18d04865c2d4c235e2261f3d63f31f45ba4fe0486ec4/pymongo-4.15.3-cp312-cp312-win32.whl", hash = "sha256:b3a0ec660d61efb91c16a5962ec937011fe3572c4338216831f102e53d294e5c", size = 891301, upload-time = "2025-10-07T21:56:40.043Z" }, - { url = "https://files.pythonhosted.org/packages/be/ba/a8fdc43044408ed769c83108fa569aa52ee87968bdbf1e2ea142b109c268/pymongo-4.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:f6b0513e5765fdde39f36e6a29a36c67071122b5efa748940ae51075beb5e4bc", size = 910928, upload-time = "2025-10-07T21:56:41.401Z" }, - { url = "https://files.pythonhosted.org/packages/b4/61/d53c17fdfaa9149864ab1fa84436ae218b72c969f00e4c124e017e461ce6/pymongo-4.15.3-cp312-cp312-win_arm64.whl", hash = "sha256:c4fdd8e6eab8ff77c1c8041792b5f760d48508623cd10b50d5639e73f1eec049", size = 896347, upload-time = "2025-10-07T21:56:43.271Z" }, - { url = "https://files.pythonhosted.org/packages/46/a4/e1ce9d408a1c1bcb1554ff61251b108e16cefd7db91b33faa2afc92294de/pymongo-4.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a47a3218f7900f65bf0f36fcd1f2485af4945757360e7e143525db9d715d2010", size = 975329, upload-time = "2025-10-07T21:56:44.674Z" }, - { url = "https://files.pythonhosted.org/packages/74/3c/6796f653d22be43cc0b13c07dbed84133eebbc334ebed4426459b7250163/pymongo-4.15.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09440e78dff397b2f34a624f445ac8eb44c9756a2688b85b3bf344d351d198e1", size = 975129, upload-time = "2025-10-07T21:56:46.104Z" }, - { url = "https://files.pythonhosted.org/packages/88/33/22453dbfe11031e89c9cbdfde6405c03960daaf5da1b4dfdd458891846b5/pymongo-4.15.3-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:97f9babdb98c31676f97d468f7fe2dc49b8a66fb6900effddc4904c1450196c8", size = 1950979, upload-time = "2025-10-07T21:56:47.877Z" }, - { url = "https://files.pythonhosted.org/packages/ba/07/094598e403112e2410a3376fb7845c69e2ec2dfc5ab5cc00b29dc2d26559/pymongo-4.15.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71413cd8f091ae25b1fec3af7c2e531cf9bdb88ce4079470e64835f6a664282a", size = 1995271, upload-time = "2025-10-07T21:56:49.396Z" }, - { url = "https://files.pythonhosted.org/packages/47/9a/29e44f3dee68defc56e50ed7c9d3802ebf967ab81fefb175d8d729c0f276/pymongo-4.15.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:76a8d4de8dceb69f6e06736198ff6f7e1149515ef946f192ff2594d2cc98fc53", size = 2086587, upload-time = "2025-10-07T21:56:50.896Z" }, - { url = "https://files.pythonhosted.org/packages/ff/d5/e9ff16aa57f671349134475b904fd431e7b86e152b01a949aef4f254b2d5/pymongo-4.15.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:77353978be9fc9e5fe56369682efed0aac5f92a2a1570704d62b62a3c9e1a24f", size = 2070201, upload-time = "2025-10-07T21:56:52.425Z" }, - { url = "https://files.pythonhosted.org/packages/d6/a3/820772c0b2bbb671f253cfb0bede4cf694a38fb38134f3993d491e23ec11/pymongo-4.15.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9897a837677e3814873d0572f7e5d53c23ce18e274f3b5b87f05fb6eea22615b", size = 1985260, upload-time = "2025-10-07T21:56:54.56Z" }, - { url = "https://files.pythonhosted.org/packages/6e/7b/365ac821aefad7e8d36a4bc472a94429449aade1ccb7805d9ca754df5081/pymongo-4.15.3-cp313-cp313-win32.whl", hash = "sha256:d66da207ccb0d68c5792eaaac984a0d9c6c8ec609c6bcfa11193a35200dc5992", size = 938122, upload-time = "2025-10-07T21:56:55.993Z" }, - { url = "https://files.pythonhosted.org/packages/80/f3/5ca27e1765fa698c677771a1c0e042ef193e207c15f5d32a21fa5b13d8c3/pymongo-4.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:52f40c4b8c00bc53d4e357fe0de13d031c4cddb5d201e1a027db437e8d2887f8", size = 962610, upload-time = "2025-10-07T21:56:57.397Z" }, - { url = "https://files.pythonhosted.org/packages/48/7c/42f0b6997324023e94939f8f32b9a8dd928499f4b5d7b4412905368686b5/pymongo-4.15.3-cp313-cp313-win_arm64.whl", hash = "sha256:fb384623ece34db78d445dd578a52d28b74e8319f4d9535fbaff79d0eae82b3d", size = 944300, upload-time = "2025-10-07T21:56:58.969Z" }, - { url = "https://files.pythonhosted.org/packages/e7/a3/d8aaf9c243ce1319bd2498004a9acccfcfb35a3ef9851abb856993d95255/pymongo-4.15.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:dcff15b9157c16bc796765d4d3d151df669322acfb0357e4c3ccd056153f0ff4", size = 1029873, upload-time = "2025-10-07T21:57:00.759Z" }, - { url = "https://files.pythonhosted.org/packages/64/10/91fd7791425ed3b56cbece6c23a36fb2696706a695655d8ea829e5e23c3a/pymongo-4.15.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1f681722c9f27e86c49c2e8a838e61b6ecf2285945fd1798bd01458134257834", size = 1029611, upload-time = "2025-10-07T21:57:02.488Z" }, - { url = "https://files.pythonhosted.org/packages/bb/9c/d9cf8d8a181f96877bca7bdec3e6ce135879d5e3d78694ea465833c53a3f/pymongo-4.15.3-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2c96dde79bdccd167b930a709875b0cd4321ac32641a490aebfa10bdcd0aa99b", size = 2211827, upload-time = "2025-10-07T21:57:03.907Z" }, - { url = "https://files.pythonhosted.org/packages/c2/40/12703964305216c155284100124222eaa955300a07d426c6e0ba3c9cbade/pymongo-4.15.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2d4ca446348d850ac4a5c3dc603485640ae2e7805dbb90765c3ba7d79129b37", size = 2264654, upload-time = "2025-10-07T21:57:05.41Z" }, - { url = "https://files.pythonhosted.org/packages/0f/70/bf3c18b5d0cae0b9714158b210b07b5891a875eb1c503271cfe045942fd3/pymongo-4.15.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0fd3de3a12ff0a8113a3f64cedb01f87397ab8eaaffa88d7f18ca66cd39385", size = 2371830, upload-time = "2025-10-07T21:57:06.9Z" }, - { url = "https://files.pythonhosted.org/packages/21/6d/2dfaed2ae66304ab842d56ed9a1bd2706ca0ecf97975b328a5eeceb2a4c0/pymongo-4.15.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e84dec392cf5f72d365e0aac73f627b0a3170193ebb038c3f7e7df11b7983ee7", size = 2351878, upload-time = "2025-10-07T21:57:08.92Z" }, - { url = "https://files.pythonhosted.org/packages/17/ed/fe46ff9adfa6dc11ad2e0694503adfc98f40583cfcc6db4dbaf582f0e357/pymongo-4.15.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d4b01a48369ea6d5bc83fea535f56279f806aa3e4991189f0477696dd736289", size = 2251356, upload-time = "2025-10-07T21:57:10.51Z" }, - { url = "https://files.pythonhosted.org/packages/12/c4/2e1a10b1e9bca9c106f2dc1b89d4ad70c63d387c194b3a1bfcca552b5a3f/pymongo-4.15.3-cp314-cp314-win32.whl", hash = "sha256:3561fa96c3123275ec5ccf919e595547e100c412ec0894e954aa0da93ecfdb9e", size = 992878, upload-time = "2025-10-07T21:57:12.119Z" }, - { url = "https://files.pythonhosted.org/packages/98/b5/14aa417a44ea86d4c31de83b26f6e6793f736cd60e7e7fda289ce5184bdf/pymongo-4.15.3-cp314-cp314-win_amd64.whl", hash = "sha256:9df2db6bd91b07400879b6ec89827004c0c2b55fc606bb62db93cafb7677c340", size = 1021209, upload-time = "2025-10-07T21:57:13.686Z" }, - { url = "https://files.pythonhosted.org/packages/94/9f/1097c6824fa50a4ffb11ba5194d2a9ef68d5509dd342e32ddb697d2efe4e/pymongo-4.15.3-cp314-cp314-win_arm64.whl", hash = "sha256:ff99864085d2c7f4bb672c7167680ceb7d273e9a93c1a8074c986a36dbb71cc6", size = 1000618, upload-time = "2025-10-07T21:57:15.212Z" }, - { url = "https://files.pythonhosted.org/packages/ad/31/37c76607a4f793f4491611741fa7a7c4238b956f48c4a9505cea0b5cf7ef/pymongo-4.15.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ffe217d2502f3fba4e2b0dc015ce3b34f157b66dfe96835aa64432e909dd0d95", size = 1086576, upload-time = "2025-10-07T21:57:16.742Z" }, - { url = "https://files.pythonhosted.org/packages/92/b2/6d17d279cdd293eeeb0c9d5baeb4f8cdebb45354fd81cfcef2d1c69303ab/pymongo-4.15.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:390c4954c774eda280898e73aea36482bf20cba3ecb958dbb86d6a68b9ecdd68", size = 1086656, upload-time = "2025-10-07T21:57:18.774Z" }, - { url = "https://files.pythonhosted.org/packages/55/fd/c5da8619beca207d7e6231f24ed269cb537c5311dad59fd9f2ef7d43204a/pymongo-4.15.3-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7dd2a49f088890ca08930bbf96121443b48e26b02b84ba0a3e1ae2bf2c5a9b48", size = 2531646, upload-time = "2025-10-07T21:57:20.63Z" }, - { url = "https://files.pythonhosted.org/packages/93/8f/66a7e12b874f41eb205f352b3a719e5a964b5ba103996f6ac45e80560111/pymongo-4.15.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f6feb678f26171f2a6b2cbb340949889154c7067972bd4cc129b62161474f08", size = 2603799, upload-time = "2025-10-07T21:57:22.591Z" }, - { url = "https://files.pythonhosted.org/packages/10/98/baf0d1f8016087500899cc4ae14e591f29b016c643e99ab332fcafe6f7bc/pymongo-4.15.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:446417a34ff6c2411ce3809e17ce9a67269c9f1cb4966b01e49e0c590cc3c6b3", size = 2725238, upload-time = "2025-10-07T21:57:24.091Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a2/112d8d3882d6e842f501e166fbe08dfc2bc9a35f8773cbcaa804f7991043/pymongo-4.15.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cfa4a0a0f024a0336640e1201994e780a17bda5e6a7c0b4d23841eb9152e868b", size = 2704837, upload-time = "2025-10-07T21:57:25.626Z" }, - { url = "https://files.pythonhosted.org/packages/38/fe/043a9aac7b3fba5b8e216f48359bd18fdbe46a4d93b081786f773b25e997/pymongo-4.15.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b03db2fe37c950aff94b29ded5c349b23729bccd90a0a5907bbf807d8c77298", size = 2582294, upload-time = "2025-10-07T21:57:27.221Z" }, - { url = "https://files.pythonhosted.org/packages/5b/fe/7a6a6b331d9f2024ab171028ab53d5d9026959b1d713fe170be591a4d9a8/pymongo-4.15.3-cp314-cp314t-win32.whl", hash = "sha256:e7cde58ef6470c0da922b65e885fb1ffe04deef81e526bd5dea429290fa358ca", size = 1043993, upload-time = "2025-10-07T21:57:28.727Z" }, - { url = "https://files.pythonhosted.org/packages/70/c8/bc64321711e19bd48ea3371f0082f10295c433833245d73e7606d3b9afbe/pymongo-4.15.3-cp314-cp314t-win_amd64.whl", hash = "sha256:fae552767d8e5153ed498f1bca92d905d0d46311d831eefb0f06de38f7695c95", size = 1078481, upload-time = "2025-10-07T21:57:30.372Z" }, - { url = "https://files.pythonhosted.org/packages/39/31/2bb2003bb978eb25dfef7b5f98e1c2d4a86e973e63b367cc508a9308d31c/pymongo-4.15.3-cp314-cp314t-win_arm64.whl", hash = "sha256:47ffb068e16ae5e43580d5c4e3b9437f05414ea80c32a1e5cac44a835859c259", size = 1051179, upload-time = "2025-10-07T21:57:31.829Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9e/5d6bd240a8d32e088078b37dcfa1579028c51d91168c0e992827ec1e87e6/pymongo-4.15.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:84c7c7624a1298295487d0dfd8dbec75d14db44c017b5087c7fe7d6996a96e3d", size = 811325, upload-time = "2025-11-11T20:50:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/1f/aa/1d707a836c436af60faa2db6f2706f9e74b5056d0bd77deb243c023b5e7b/pymongo-4.15.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:71a5ab372ebe4e05453bae86a008f6db98b5702df551219fb2f137c394d71c3a", size = 811668, upload-time = "2025-11-11T20:50:24.171Z" }, + { url = "https://files.pythonhosted.org/packages/e8/17/8c50a695a7029d582da50875085465f01bf83e5146fb7dc3f671168aedb7/pymongo-4.15.4-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eee407bf1058a8f0d5b203028997b42ea6fc80a996537cc2886f89573bc0770f", size = 1185476, upload-time = "2025-11-11T20:50:25.635Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e8/5449663ec341fb83c3e4d51011f65b61de8427620679510cca57386c9446/pymongo-4.15.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e286f5b9c13963bcaf9b9241846d388ac5022225a9e11c5364393a8cc3eb49", size = 1203857, upload-time = "2025-11-11T20:50:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ff/98768d4294f271175aedbad209d748ac769a3f35bee35f8c82b57b03ea4a/pymongo-4.15.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:67c3b84a2a0e1794b2fbfe22dc36711a03c6bc147d9d2e0f8072fabed7a65092", size = 1242538, upload-time = "2025-11-11T20:50:29.322Z" }, + { url = "https://files.pythonhosted.org/packages/18/91/57b4a08b81686e0148a93ecd0149d747a31be82aafa0708143d642662893/pymongo-4.15.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:94e50149fb9d982c234d0efa9c0eec4a04db7e82a412d3dae2c4f03a9926360e", size = 1232831, upload-time = "2025-11-11T20:50:31.326Z" }, + { url = "https://files.pythonhosted.org/packages/ff/17/52434425cde25e6d0743a6d8af8a8b88ffbd05cce595993facf09a5d0559/pymongo-4.15.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1903c0966969cf3e7b30922956bd82eb09e6a3f3d7431a727d12f20104f66d3", size = 1200182, upload-time = "2025-11-11T20:50:33.339Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7f/d5c975dcbfd339f3cd3eae2055fe6d96fb546508e1954fe263c0304e0317/pymongo-4.15.4-cp310-cp310-win32.whl", hash = "sha256:20ffcd883b6e187ef878558d0ebf9f09cc46807b6520022592522d3cdd21022d", size = 798325, upload-time = "2025-11-11T20:50:35.214Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1f/78b2d3d7b35284c5da80342ce2b7e4087901ff8fb030eccaa654b5d3d061/pymongo-4.15.4-cp310-cp310-win_amd64.whl", hash = "sha256:68ea93e7d19d3aa3182a6e41ba68288b9b234a3b0a70b368feb95fff3f94413f", size = 808144, upload-time = "2025-11-11T20:50:36.621Z" }, + { url = "https://files.pythonhosted.org/packages/30/d2/95505fb5a699180a215553f622702464bc47000e5e782cc846098dcdfc37/pymongo-4.15.4-cp310-cp310-win_arm64.whl", hash = "sha256:abfe72630190c0dc8f2222b02af7c4e5f72809d06b2ccb3f3ca83f6a7b60e302", size = 800933, upload-time = "2025-11-11T20:50:38.573Z" }, + { url = "https://files.pythonhosted.org/packages/71/a4/b1a724352ab47a8925f30931a6aa6f905dcf473d8404156ef608ec325fbd/pymongo-4.15.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b2967bda6ccac75aefad26c4ef295f5054181d69928bb9d1159227d6771e8887", size = 865881, upload-time = "2025-11-11T20:50:40.275Z" }, + { url = "https://files.pythonhosted.org/packages/09/d4/6f4db5b64b0b71f0cbe608a80aea8b2580b5e1db4da1f9a70ae5531e9f1d/pymongo-4.15.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7df1fad859c61bdbe0e2a0dec8f5893729d99b4407b88568e0e542d25f383f57", size = 866225, upload-time = "2025-11-11T20:50:41.842Z" }, + { url = "https://files.pythonhosted.org/packages/0f/44/9d96fa635b838348109f904f558aa6675fdfb0a9265060050d7a92afbf97/pymongo-4.15.4-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:990c4898787e706d0ab59141cf5085c981d89c3f86443cd6597939d9f25dd71d", size = 1429778, upload-time = "2025-11-11T20:50:43.801Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e6/eac0b3ca4ea1cd437983f1409cb6260e606cce11ea3cb6f5ccd8629fa5c2/pymongo-4.15.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad7ff0347e8306fc62f146bdad0635d9eec1d26e246c97c14dd1a189d3480e3f", size = 1456739, upload-time = "2025-11-11T20:50:45.479Z" }, + { url = "https://files.pythonhosted.org/packages/73/7e/b7adba0c8dfc2dced7632c61425a70048bddf953b07bf6232a4ea7f0fb7e/pymongo-4.15.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dd8c78c59fd7308239ef9bcafb7cd82f08cbc9466d1cfda22f9025c83468bf6d", size = 1514659, upload-time = "2025-11-11T20:50:47.517Z" }, + { url = "https://files.pythonhosted.org/packages/20/8b/cdc129f1bee5595018c52ff81baaec818301e705ee39cf00d9d5f68a3d0d/pymongo-4.15.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:44d95677aa23fe479bb531b393a4fad0210f808af52e4ab2b79c0b540c828957", size = 1500700, upload-time = "2025-11-11T20:50:49.183Z" }, + { url = "https://files.pythonhosted.org/packages/1f/02/e706a63f00542531a4c723258ae3da3439925de02215710a18813fbe1db4/pymongo-4.15.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ab985e61376ae5a04f162fb6bdddaffc7beec883ffbd9d84ea86a71be794d74", size = 1452011, upload-time = "2025-11-11T20:50:51.568Z" }, + { url = "https://files.pythonhosted.org/packages/37/36/6b78b105e8e1174ebda592ad31f02cb98ee9bd8bb2eeb621f54e2c714d03/pymongo-4.15.4-cp311-cp311-win32.whl", hash = "sha256:2f811e93dbcba0c488518ceae7873a40a64b6ad273622a18923ef2442eaab55c", size = 844471, upload-time = "2025-11-11T20:50:53.362Z" }, + { url = "https://files.pythonhosted.org/packages/a5/0d/3d009eed6ae045ee4f62877878070a07405af5e368d60a4a35efd177c25b/pymongo-4.15.4-cp311-cp311-win_amd64.whl", hash = "sha256:53bfcd8c11086a2457777cb4b1a6588d9dd6af77aeab47e04f2af02e3a077e59", size = 859189, upload-time = "2025-11-11T20:50:55.198Z" }, + { url = "https://files.pythonhosted.org/packages/d5/40/d5713b1d5e0b10402446632bab6a88918cd13e5fe1fa26beac177eb37dac/pymongo-4.15.4-cp311-cp311-win_arm64.whl", hash = "sha256:2096964b2b93607ed80a62ac6664396a826b7fe34e2b1eed3f20784681a17827", size = 848369, upload-time = "2025-11-11T20:50:57.164Z" }, + { url = "https://files.pythonhosted.org/packages/75/bb/09176c965d994352efd1407c9139799218f3fe1d18382dff34ef64e0bd22/pymongo-4.15.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4ab4eef031e722a8027c338c3d71704a8c85c17c64625d61c6effdf8a893b971", size = 920943, upload-time = "2025-11-11T20:50:59.056Z" }, + { url = "https://files.pythonhosted.org/packages/94/97/d212bd8d9106acecf6948cc0a0ed640f58d8afaed427481b9e79db08f45c/pymongo-4.15.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e12551e28007a341d15ebca5a024ef487edf304d612fba5efa1fd6b4d9a95a9", size = 920687, upload-time = "2025-11-11T20:51:00.683Z" }, + { url = "https://files.pythonhosted.org/packages/ff/81/7be727d6172fd80d8dd1c6fedb78675936396d2f2067fab270e443e04621/pymongo-4.15.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1d21998fb9ccb3ea6d59a9f9971591b9efbcfbbe46350f7f8badef9b107707f3", size = 1690340, upload-time = "2025-11-11T20:51:02.392Z" }, + { url = "https://files.pythonhosted.org/packages/42/5a/91bf00e9d30d18b3e8ef3fa222964ba1e073d82c5f38dae027e63d36bcfd/pymongo-4.15.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f83e8895d42eb51d259694affa9607c4d56e1c784928ccbbac568dc20df86a8", size = 1726082, upload-time = "2025-11-11T20:51:04.353Z" }, + { url = "https://files.pythonhosted.org/packages/ff/08/b7d8e765efa64cddf1844e8b889454542c765f8d119c87a4904f45addc07/pymongo-4.15.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bd8126a507afa8ce4b96976c8e28402d091c40b7d98e3b5987a371af059d9e7", size = 1800624, upload-time = "2025-11-11T20:51:06.222Z" }, + { url = "https://files.pythonhosted.org/packages/35/b0/40ec073ccc2cf95e8743315e6c92a81f37698d2e618c83ec7d9c3b647bd0/pymongo-4.15.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e799e2cba7fcad5ab29f678784f90b1792fcb6393d571ecbe4c47d2888af30f3", size = 1785469, upload-time = "2025-11-11T20:51:07.893Z" }, + { url = "https://files.pythonhosted.org/packages/82/da/b1a27064404d5081f5391c3c81e4a6904acccb4766598e3aa14399d36feb/pymongo-4.15.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:563e793ad87633e50ad43a8cd2c740fbb17fca4a4637185996575ddbe99960b8", size = 1718540, upload-time = "2025-11-11T20:51:09.574Z" }, + { url = "https://files.pythonhosted.org/packages/e7/8c/bee6159b4e434dc0413b399af2bd3795ef7427b2c2fe1b304df250c0a3d8/pymongo-4.15.4-cp312-cp312-win32.whl", hash = "sha256:39bb3c12c772241778f4d7bf74885782c8d68b309d3c69891fe39c729334adbd", size = 891308, upload-time = "2025-11-11T20:51:11.67Z" }, + { url = "https://files.pythonhosted.org/packages/cf/cb/cb70455fe2eadf4f6ccd27fe215e342b242e8b53780aeafb96cd1c3bf506/pymongo-4.15.4-cp312-cp312-win_amd64.whl", hash = "sha256:6f43326f36bc540b04f5a7f1aa8be40b112d7fc9f6e785ae3797cd72a804ffdd", size = 910911, upload-time = "2025-11-11T20:51:13.283Z" }, + { url = "https://files.pythonhosted.org/packages/41/81/20486a697474b7de25faee91d9c478eb410ae78cb4e50b15000184944a48/pymongo-4.15.4-cp312-cp312-win_arm64.whl", hash = "sha256:263cfa2731a4bbafdce2cf06cd511eba8957bd601b3cad9b4723f2543d42c730", size = 896347, upload-time = "2025-11-11T20:51:15.981Z" }, + { url = "https://files.pythonhosted.org/packages/51/10/09551492e484f7055194d91c071c827fc65261156e4daced35e67e97b893/pymongo-4.15.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6ff080f23a12c943346e2bba76cf19c3d14fb3625956792aa22b69767bfb36de", size = 975326, upload-time = "2025-11-11T20:51:17.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/6e/8f153a6d7eaec9b334975000e16bfd11ec4050e8729d3e2ee67d7022f526/pymongo-4.15.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c4690e01d03773f7af21b1a8428029bd534c9fe467c6b594c591d8b992c0a975", size = 975132, upload-time = "2025-11-11T20:51:19.58Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7d/037498c1354fae1ce2fc7738c981a7447a5fee021c22e76083540cc1f9d6/pymongo-4.15.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:78bfe3917d0606b30a91b02ad954c588007f82e2abb2575ac2665259b051a753", size = 1950964, upload-time = "2025-11-11T20:51:21.262Z" }, + { url = "https://files.pythonhosted.org/packages/ef/96/7c6b14956ef2ab99600d93b43429387394df6a99f5293cd0371c59a77a02/pymongo-4.15.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f53c83c3fd80fdb412ce4177d4f59b70b9bb1add6106877da044cf21e996316b", size = 1995249, upload-time = "2025-11-11T20:51:23.248Z" }, + { url = "https://files.pythonhosted.org/packages/2a/16/0e0495b38dd64efbfd6f2eb47535895c8df4a78e384aee78190fe2ecfa84/pymongo-4.15.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e41d6650c1cd77a8e7556ad65133455f819f8c8cdce3e9cf4bbf14252b7d805", size = 2086580, upload-time = "2025-11-11T20:51:25.294Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c0/692545232a17d5772d15c7e50d54415bdd9b88018e2228607c96766af961/pymongo-4.15.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b60fd8125f52efffd697490b6ccebc6e09d44069ad9c8795df0a684a9a8f4b3c", size = 2070189, upload-time = "2025-11-11T20:51:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/6f/9f/aae8eb4650d9a62f26baca4f4da2a0f5cd1aabcd4229dabc43cd71e09ea2/pymongo-4.15.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d1a1a0406acd000377f34ae91cdb501fa73601a2d071e4a661e0c862e1b166e", size = 1985254, upload-time = "2025-11-11T20:51:29.136Z" }, + { url = "https://files.pythonhosted.org/packages/b1/cd/50f49788caa317c7b00ccf0869805cb2b3046c2510f960cb07e8d3a74f73/pymongo-4.15.4-cp313-cp313-win32.whl", hash = "sha256:9c5710ed5f2af95315db0ee8ae02e9ff1e85e7b068c507d980bc24fe9d025257", size = 938134, upload-time = "2025-11-11T20:51:31.254Z" }, + { url = "https://files.pythonhosted.org/packages/10/ad/6e96ccb3b7ab8be2e22b1c50b98aed0cae19253174bca6807fc8fd1ce34c/pymongo-4.15.4-cp313-cp313-win_amd64.whl", hash = "sha256:61b0863c7f9b460314db79b7f8541d3b490b453ece49afd56b611b214fc4b3b1", size = 962595, upload-time = "2025-11-11T20:51:33.118Z" }, + { url = "https://files.pythonhosted.org/packages/22/23/9b9255e432df4bc276ecb9bb6e81c3376d8ee2b19de02d3751bb5c4a6fb1/pymongo-4.15.4-cp313-cp313-win_arm64.whl", hash = "sha256:0255af7d5c23c5e8cb4d9bb12906b142acebab0472117e1d5e3a8e6e689781cb", size = 944298, upload-time = "2025-11-11T20:51:35.13Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e6/f315ea84656adcd18d5b5e8b362b47c36bf606843098688cc0809b28c8a8/pymongo-4.15.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:539f9fa5bb04a09fc2965cdcae3fc91d1c6a1f4f1965b34df377bc7119e3d7cd", size = 1029994, upload-time = "2025-11-11T20:51:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0c/0c364db72cd80a503829885643478dd144a8bf05e1e853c89648a06ad34b/pymongo-4.15.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:68354a77cf78424d27216b1cb7c9b0f67da16aae855045279ba8d73bb61f5ad0", size = 1029615, upload-time = "2025-11-11T20:51:38.551Z" }, + { url = "https://files.pythonhosted.org/packages/50/71/6f37eea22ffa5b136c1ca0a21ba390c273b582d800bc979961fbd46c9bcc/pymongo-4.15.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a9a90d556c2ef1572d2aef525ef19477a82d659d117eb3a51fa99e617d07dc44", size = 2211805, upload-time = "2025-11-11T20:51:40.657Z" }, + { url = "https://files.pythonhosted.org/packages/24/09/3a538cb82766ce89559c4ca0d5694f782485080db6a8f628784dc7debba8/pymongo-4.15.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1aac57614fb86a3fa707af3537c30eda5e7fd1be712c1f723296292ac057afe", size = 2264618, upload-time = "2025-11-11T20:51:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/51/6b/66b4fe2d3c566ed655d95b1d8947dfea05642b05a285a3081d6cebc4f5da/pymongo-4.15.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6c21b49c5e021d9ce02cac33525c722d4c6887f7cde19a5a9154f66cb845e84", size = 2371810, upload-time = "2025-11-11T20:51:44.372Z" }, + { url = "https://files.pythonhosted.org/packages/92/2b/3989960c7de983c5cc05b2d43b26fa560fe9de433ee60b83259d6ee2cde3/pymongo-4.15.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e93828768470026099119295c68ed0dbc0a50022558be5e334f6dbda054f1d32", size = 2351848, upload-time = "2025-11-11T20:51:46.548Z" }, + { url = "https://files.pythonhosted.org/packages/31/93/ee9f8a42eed6ecb8dda52e586a470bf88007a298b0f1a2c4ea1ff352af8e/pymongo-4.15.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11840e9eb5a650ac190f2a3473631073daddbabdbb2779b6709dfddd3ba3b872", size = 2251338, upload-time = "2025-11-11T20:51:48.335Z" }, + { url = "https://files.pythonhosted.org/packages/a0/36/c6609f632bcaffcdf9f7e67cb888402a1df049a7c3ff2f56067a0b451a59/pymongo-4.15.4-cp314-cp314-win32.whl", hash = "sha256:f0907b46df97b01911bf2e10ddbb23c2303629e482d81372031fd7f4313b9013", size = 992893, upload-time = "2025-11-11T20:51:50.775Z" }, + { url = "https://files.pythonhosted.org/packages/f0/23/4ec0f7c9bf3397b6cafaf714f5bfe0a9944e7af088daa01d258eec031118/pymongo-4.15.4-cp314-cp314-win_amd64.whl", hash = "sha256:111d7f65ccbde908546cb36d14e22f12a73a4de236fd056f41ed515d1365f134", size = 1021204, upload-time = "2025-11-11T20:51:52.691Z" }, + { url = "https://files.pythonhosted.org/packages/2b/71/3813d15fa5ce6fb5fb40775bedc95a1970790f5aba968d92b014a796aab6/pymongo-4.15.4-cp314-cp314-win_arm64.whl", hash = "sha256:c689a5d057ef013612b5aa58e6bf52f7fdb186e22039f1a3719985b5d0399932", size = 1000608, upload-time = "2025-11-11T20:51:54.442Z" }, + { url = "https://files.pythonhosted.org/packages/12/e7/10f3bc034fcec374dc46462b369205527478199a803169cb10e9e4b48c68/pymongo-4.15.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:cdfa57760745387cde93615a48f622bf1eeae8ae28103a8a5100b9389eec22f9", size = 1086725, upload-time = "2025-11-11T20:51:57.266Z" }, + { url = "https://files.pythonhosted.org/packages/40/ee/b59cad7d46598d48708bd2a6559ea8b9cbb6fb9665d617b5a52b58de81b3/pymongo-4.15.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4fd6ba610e5a54090c4055a15f38d19ad8bf11e6bbc5a173e945c755a16db455", size = 1086660, upload-time = "2025-11-11T20:51:59.114Z" }, + { url = "https://files.pythonhosted.org/packages/0a/84/58efbde2b52a577f9162bb9b97605b6669354bb171bc241a0dc2639536d7/pymongo-4.15.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3c7945b8a5563aa3951db26ba534372fba4c781473f5d55ce6340b7523cb0f", size = 2531617, upload-time = "2025-11-11T20:52:01.006Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cd/7bd739d04b67c99f00c942465b8ab7659dc2c1ad80108b5f4f74eecdf9f3/pymongo-4.15.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41e98a31e79d74e9d78bc1638b71c3a10a910eae7d3318e2ae8587c760931451", size = 2603756, upload-time = "2025-11-11T20:52:03.029Z" }, + { url = "https://files.pythonhosted.org/packages/4a/39/5a3b01f7e5fd464656421246516723c02067e85bbfb52d30da7d79b8336f/pymongo-4.15.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d18d89073b5e752391c237d2ee86ceec1e02a4ad764b3029f24419eedd12723e", size = 2725205, upload-time = "2025-11-11T20:52:04.968Z" }, + { url = "https://files.pythonhosted.org/packages/c7/a8/b06231d5ea48d0fcc47bf6c2cebfd8dbea3eda1a1d7bf786443cb9ef5b94/pymongo-4.15.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edbff27a56a80b8fe5c0319200c44e63b1349bf20db27d9734ddcf23c0d72b35", size = 2704793, upload-time = "2025-11-11T20:52:07.164Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/c0ea0da1185d3be4e73923ab3b74f14f424b40f787c710690c83004f147a/pymongo-4.15.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f1d75f5b51304176631c12e5bf47eed021446669e5f99379b76fd2bd3929c1b4", size = 2582263, upload-time = "2025-11-11T20:52:09.016Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f7/29ce41f9e55b1dd912bed39b76e9326e23ff6c097c4a8de88b2c5bcd54e5/pymongo-4.15.4-cp314-cp314t-win32.whl", hash = "sha256:e1bf4e0689cc48e0cfa6aef17f107c298d8898de0c6e782ea5c98450ae93a62f", size = 1044009, upload-time = "2025-11-11T20:52:11.138Z" }, + { url = "https://files.pythonhosted.org/packages/01/71/3fade727cc4c7ac77fe19c4e3a6bbfb66d7f46796108ba106f236c64492f/pymongo-4.15.4-cp314-cp314t-win_amd64.whl", hash = "sha256:3fc347ea5eda6c3a7177c3a9e4e9b4e570a444a351effda4a898c2d352a1ccd1", size = 1078479, upload-time = "2025-11-11T20:52:13.324Z" }, + { url = "https://files.pythonhosted.org/packages/60/0f/d450350f103db4bb856cb1ee60c8b1fa68d5ac50c846896d74deba3e9950/pymongo-4.15.4-cp314-cp314t-win_arm64.whl", hash = "sha256:2d921b84c681c5385a6f7ba2b5740cb583544205a00877aad04b5b12ab86ad26", size = 1051155, upload-time = "2025-11-11T20:52:15.185Z" }, ] [[package]] @@ -1380,14 +1393,15 @@ wheels = [ [[package]] name = "yscope-spider-py" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lark" }, { name = "mariadb" }, + { name = "msgpack" }, { name = "msgpack-types" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/20/fc1170d5109639a466cb776bb8b78478d6ee6d61ccfb2021224e1349485a/yscope_spider_py-0.1.0.tar.gz", hash = "sha256:e7da1a928aacf717d9c97ed5a19da521e25288fb48ceb04b632de1574693296f", size = 23760, upload-time = "2025-09-21T21:19:27.178Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/4ff0569fc95f6210a4bff258bf070f09b39a6eb60e6772d3dad3f7eb9279/yscope_spider_py-0.2.0.tar.gz", hash = "sha256:43c819ad4e0a135e3ae63ca55d849b6b3015bcf9b982c337d5cf10ea07b4e947", size = 25554, upload-time = "2025-11-14T04:30:33.533Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/2d/5945cd02f57b2b54d6e09f54fb3b9f0c33441ba02cbfdfc97390c1182286/yscope_spider_py-0.1.0-py3-none-any.whl", hash = "sha256:a095de44ed957b3e365e0ee0ccb0622d5a5d10b2a2fec5d4a239cd5530e017ae", size = 33394, upload-time = "2025-09-21T21:19:26.122Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ac/9eb7fa22ac02641faf57532e44326c471840ebf125270cb4bfad0968f534/yscope_spider_py-0.2.0-py3-none-any.whl", hash = "sha256:a5d1e1c2182e81321cc3482bcca5b46998922c6d5ccfc5bbd1cc6326388435d0", size = 34481, upload-time = "2025-11-14T04:30:32.463Z" }, ] diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4808f05e36..03774b33dd 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -264,14 +264,13 @@ def transform_for_container(self): class SpiderDb(Database): - name: str = "spider-db" @field_validator("type") @classmethod def validate_type(cls, value): if value != DatabaseEngine.MARIADB: - raise ValueError(f"Spider only supports MariaDB for the metadata database.") + raise ValueError("Spider only supports MariaDB for the metadata database.") return value def get_url(self): diff --git a/components/clp-py-utils/clp_py_utils/create-db-tables.py b/components/clp-py-utils/clp_py_utils/create-db-tables.py index eee0c8b0a5..17ed4cb809 100644 --- a/components/clp-py-utils/clp_py_utils/create-db-tables.py +++ b/components/clp-py-utils/clp_py_utils/create-db-tables.py @@ -4,7 +4,7 @@ import subprocess import sys -from clp_py_utils.clp_config import StorageEngine, ClpConfig +from clp_py_utils.clp_config import ClpConfig, StorageEngine from clp_py_utils.core import read_yaml_config_file # Setup logging diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 3ed3440e54..1273739722 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -244,9 +244,10 @@ def main(argv): try: sql_adapter = SqlAdapter(clp_config.database) - with closing(sql_adapter.create_root_mariadb_connection()) as db_conn, closing( - db_conn.cursor() - ) as db_cursor: + with ( + closing(sql_adapter.create_root_mariadb_connection()) as db_conn, + closing(db_conn.cursor()) as db_cursor, + ): clp_db_user = clp_config.database.username spider_db_name = spider_db_config.name spider_db_user = spider_db_config.username @@ -263,11 +264,17 @@ def main(argv): db_cursor.execute(f"""CREATE DATABASE IF NOT EXISTS `{spider_db_name}`""") if spider_db_password is None: - logger.error(f"Password must be set for Spider database user.") + logger.error("Password must be set for Spider database user.") return -1 - db_cursor.execute(f"""CREATE USER IF NOT EXISTS '{spider_db_user}'@'%' IDENTIFIED BY '{spider_db_password}'""") - db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{spider_db_name}`.* TO '{spider_db_user}'@'%'""") - db_cursor.execute(f"""GRANT ALL PRIVILEGES ON `{spider_db_name}`.* TO '{clp_db_user}'@'%'""") + db_cursor.execute( + f"""CREATE USER IF NOT EXISTS '{spider_db_user}'@'%' IDENTIFIED BY '{spider_db_password}'""" + ) + db_cursor.execute( + f"""GRANT ALL PRIVILEGES ON `{spider_db_name}`.* TO '{spider_db_user}'@'%'""" + ) + db_cursor.execute( + f"""GRANT ALL PRIVILEGES ON `{spider_db_name}`.* TO '{clp_db_user}'@'%'""" + ) db_cursor.execute(f"""USE `{spider_db_name}`""") for table_creator in table_creators: diff --git a/components/clp-py-utils/clp_py_utils/sql_adapter.py b/components/clp-py-utils/clp_py_utils/sql_adapter.py index 41f00f9a71..728d69b565 100644 --- a/components/clp-py-utils/clp_py_utils/sql_adapter.py +++ b/components/clp-py-utils/clp_py_utils/sql_adapter.py @@ -1,7 +1,6 @@ import contextlib import logging import time -from typing import Optional import mariadb import mysql.connector @@ -59,9 +58,7 @@ def alive(self): class SqlAdapter: - def __init__( - self, database_config: Database, spider_database_config: Optional[SpiderDb] = None - ): + def __init__(self, database_config: Database, spider_database_config: SpiderDb | None = None): self.database_config = database_config self._spider_database_config = spider_database_config @@ -109,7 +106,9 @@ def create_connection(self, disable_localhost_socket_connection: bool = False): return self.create_mariadb_connection(disable_localhost_socket_connection) raise NotImplementedError - def create_root_mariadb_connection(self, disable_localhost_socket_connection: bool = False) -> mariadb.Connection: + def create_root_mariadb_connection( + self, disable_localhost_socket_connection: bool = False + ) -> mariadb.Connection: if "mariadb" != self.database_config.type: raise NotImplementedError params = self.database_config.get_mysql_connection_params( diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index 237346d8b1..b3c98c1eb8 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -14,6 +14,7 @@ logging_console_handler.setFormatter(logging_formatter) logger.addHandler(logging_console_handler) + def parse_args() -> argparse.Namespace: """ Parses command line arguments. diff --git a/components/clp-py-utils/uv.lock b/components/clp-py-utils/uv.lock index 7701447606..3fda80e8c4 100644 --- a/components/clp-py-utils/uv.lock +++ b/components/clp-py-utils/uv.lock @@ -43,30 +43,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.40.74" +version = "1.41.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a2/37/0db5fc46548b347255310893f1a47971a1d8eb0dbc46dfb5ace8a1e7d45e/boto3-1.40.74.tar.gz", hash = "sha256:484e46bf394b03a7c31b34f90945ebe1390cb1e2ac61980d128a9079beac87d4", size = 111592, upload-time = "2025-11-14T20:29:10.991Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/4f/92744c97f42e214948b9c8eff86e7e72c7ca8be788867a8aea80dc192052/boto3-1.41.0.tar.gz", hash = "sha256:73bf7f63152406404c0359c013a692e884b98a3b297160058a38f00ef19e375b", size = 111589, upload-time = "2025-11-19T20:29:10.72Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/08/c52751748762901c0ca3c3019e3aa950010217f0fdf9940ebe68e6bb2f5a/boto3-1.40.74-py3-none-any.whl", hash = "sha256:41fc8844b37ae27b24bcabf8369769df246cc12c09453988d0696ad06d6aa9ef", size = 139360, upload-time = "2025-11-14T20:29:09.477Z" }, + { url = "https://files.pythonhosted.org/packages/32/35/5f4b70f20188614a485b26e80369b9fa260a06fb0ae328153d7fc647619f/boto3-1.41.0-py3-none-any.whl", hash = "sha256:d5c454bb23655b052073c8dc6703dda5360825b72b1691822ae7709050b96390", size = 139340, upload-time = "2025-11-19T20:29:09.03Z" }, ] [[package]] name = "botocore" -version = "1.40.74" +version = "1.41.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/dc/0412505f05286f282a75bb0c650e525ddcfaf3f6f1a05cd8e99d32a2db06/botocore-1.40.74.tar.gz", hash = "sha256:57de0b9ffeada06015b3c7e5186c77d0692b210d9e5efa294f3214df97e2f8ee", size = 14452479, upload-time = "2025-11-14T20:29:00.949Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/8d/af94a3a0a5dc3ff255fdbd9a4bdf8e41beb33ea61ebab92e3d8e017f9ee4/botocore-1.41.0.tar.gz", hash = "sha256:555afbf86a644bfa4ebd7bd98d717b53b792e6bbb2c49f2b308fb06964cf1655", size = 14580059, upload-time = "2025-11-19T20:28:59.605Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/a2/306dec16e3c84f3ca7aaead0084358c1c7fbe6501f6160844cbc93bc871e/botocore-1.40.74-py3-none-any.whl", hash = "sha256:f39f5763e35e75f0bd91212b7b36120b1536203e8003cd952ef527db79702b15", size = 14117911, upload-time = "2025-11-14T20:28:58.153Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5c/65591ff3d30e790921635602bf53f60b89dd1f39a2cc0dad980b70dd569c/botocore-1.41.0-py3-none-any.whl", hash = "sha256:a5018d6268eee358dfc5d86e596c3062b4e225690acaf946f54c00063b804bf8", size = 14243471, upload-time = "2025-11-19T20:28:56.965Z" }, ] [[package]] @@ -391,14 +391,14 @@ dependencies = [ requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, { name = "celery", extras = ["redis"], specifier = ">=5.5.3" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "msgpack", specifier = ">=1.1.2" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pika", specifier = ">=1.3.2" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "pymongo", specifier = ">=4.15.3" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "yscope-spider-py", specifier = "==0.1.0" }, + { name = "yscope-spider-py", specifier = "==0.2.0" }, ] [package.metadata.requires-dev] @@ -725,15 +725,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, -] - [[package]] name = "pathspec" version = "0.12.1" @@ -1402,14 +1393,15 @@ wheels = [ [[package]] name = "yscope-spider-py" -version = "0.1.0" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lark" }, { name = "mariadb" }, + { name = "msgpack" }, { name = "msgpack-types" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/20/fc1170d5109639a466cb776bb8b78478d6ee6d61ccfb2021224e1349485a/yscope_spider_py-0.1.0.tar.gz", hash = "sha256:e7da1a928aacf717d9c97ed5a19da521e25288fb48ceb04b632de1574693296f", size = 23760, upload-time = "2025-09-21T21:19:27.178Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/4ff0569fc95f6210a4bff258bf070f09b39a6eb60e6772d3dad3f7eb9279/yscope_spider_py-0.2.0.tar.gz", hash = "sha256:43c819ad4e0a135e3ae63ca55d849b6b3015bcf9b982c337d5cf10ea07b4e947", size = 25554, upload-time = "2025-11-14T04:30:33.533Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/2d/5945cd02f57b2b54d6e09f54fb3b9f0c33441ba02cbfdfc97390c1182286/yscope_spider_py-0.1.0-py3-none-any.whl", hash = "sha256:a095de44ed957b3e365e0ee0ccb0622d5a5d10b2a2fec5d4a239cd5530e017ae", size = 33394, upload-time = "2025-09-21T21:19:26.122Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ac/9eb7fa22ac02641faf57532e44326c471840ebf125270cb4bfad0968f534/yscope_spider_py-0.2.0-py3-none-any.whl", hash = "sha256:a5d1e1c2182e81321cc3482bcca5b46998922c6d5ccfc5bbd1cc6326388435d0", size = 34481, upload-time = "2025-11-14T04:30:32.463Z" }, ] diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py index 72c431fdc0..5615e7456a 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py @@ -1,7 +1,7 @@ import json from clp_py_utils.clp_logging import get_logger -from spider_py import Int8, Int64, TaskContext +from spider_py import Int64, TaskContext from job_orchestration.executor.compress.compression_task import compression_entry_point diff --git a/components/job-orchestration/uv.lock b/components/job-orchestration/uv.lock index 9e20270593..5f771201c1 100644 --- a/components/job-orchestration/uv.lock +++ b/components/job-orchestration/uv.lock @@ -252,7 +252,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "boto3", specifier = ">=1.40.55" }, - { name = "mariadb", specifier = ">=1.0.11,<1.1.dev0" }, + { name = "mariadb", specifier = ">=1.1.14" }, { name = "mysql-connector-python", specifier = ">=9.4.0" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "python-levenshtein", specifier = ">=0.27.1" }, From 9f82f9699dd1349f85e371d356798496c8dd5d89 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 00:06:28 -0500 Subject: [PATCH 388/408] Fix --- .../clp-package-utils/clp_package_utils/controller.py | 8 ++++---- components/clp-package-utils/clp_package_utils/general.py | 3 ++- components/clp-py-utils/clp_py_utils/clp_config.py | 4 +++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 54bbe4da56..dcd3232e84 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -40,7 +40,7 @@ SPIDER_SCHEDULER_COMPONENT_NAME, StorageEngine, StorageType, - WEBUI_COMPONENT_NAME, + WEBUI_COMPONENT_NAME, SPIDER_DB_COMPONENT_NAME, ) from clp_py_utils.clp_metadata_db_utils import ( get_archives_table_name, @@ -262,7 +262,7 @@ def _set_up_env_for_spider_db(self) -> EnvVarsDict: :return: Dictionary of environment variables necessary to launch the component. """ - component_name = "spider_db" + component_name = SPIDER_DB_COMPONENT_NAME logger.info(f"Setting up environment for {component_name}...") env_vars = EnvVarsDict() @@ -813,9 +813,9 @@ def set_up_env(self) -> None: # Component-specific config env_vars |= self._set_up_env_for_database() - if self._clp_config.redis is not None: - env_vars |= self._set_up_env_for_queue() if self._clp_config.queue is not None: + env_vars |= self._set_up_env_for_queue() + if self._clp_config.redis is not None: env_vars |= self._set_up_env_for_redis() if self._clp_config.compression_scheduler.type == OrchestrationType.spider: env_vars |= self._set_up_env_for_spider_db() diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 268ab1f80a..0bd4bd928c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -27,7 +27,7 @@ RESULTS_CACHE_COMPONENT_NAME, StorageType, WEBUI_COMPONENT_NAME, - WorkerConfig, + WorkerConfig, SPIDER_DB_COMPONENT_NAME, ) from clp_py_utils.clp_metadata_db_utils import ( MYSQL_TABLE_NAME_MAX_LEN, @@ -454,6 +454,7 @@ def generate_credentials_file(credentials_file_path: pathlib.Path): DB_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, QUEUE_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, REDIS_COMPONENT_NAME: {"password": secrets.token_urlsafe(16)}, + SPIDER_DB_COMPONENT_NAME: {"username": "spider-user", "password": secrets.token_urlsafe(8)}, } with open(credentials_file_path, "w") as f: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 03774b33dd..56d53d415e 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -904,7 +904,7 @@ def validate_spider_config(self): if self.spider_db is None: raise ValueError("`spider_db` must be configured when using Spider orchestration.") if self.spider_scheduler is None: - raise ValueError("`spider_db` must be configured when using Spider orchestration.") + raise ValueError("`spider_scheduler` must be configured when using Spider orchestration.") if self.database.type != DatabaseEngine.MARIADB: raise ValueError("Spider only supports MariaDB for the metadata database.") return self @@ -939,6 +939,8 @@ def transform_for_container(self): self.queue.transform_for_container() if self.redis is not None: self.redis.transform_for_container() + if self.spider_db is not None: + self.spider_db.transform_for_container() if self.spider_scheduler is not None: self.spider_scheduler.transform_for_container() self.results_cache.transform_for_container() From 05021117d8502490fe789d3c699842cb7d9535d3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 00:07:23 -0500 Subject: [PATCH 389/408] Fix lint --- components/clp-package-utils/clp_package_utils/controller.py | 3 ++- components/clp-package-utils/clp_package_utils/general.py | 3 ++- components/clp-py-utils/clp_py_utils/clp_config.py | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index dcd3232e84..3f36c95429 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -35,12 +35,13 @@ REDIS_COMPONENT_NAME, REDUCER_COMPONENT_NAME, RESULTS_CACHE_COMPONENT_NAME, + SPIDER_DB_COMPONENT_NAME, SPIDER_DB_PASS_ENV_VAR_NAME, SPIDER_DB_USER_ENV_VAR_NAME, SPIDER_SCHEDULER_COMPONENT_NAME, StorageEngine, StorageType, - WEBUI_COMPONENT_NAME, SPIDER_DB_COMPONENT_NAME, + WEBUI_COMPONENT_NAME, ) from clp_py_utils.clp_metadata_db_utils import ( get_archives_table_name, diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 0bd4bd928c..8d37ec717a 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -25,9 +25,10 @@ REDIS_COMPONENT_NAME, REDUCER_COMPONENT_NAME, RESULTS_CACHE_COMPONENT_NAME, + SPIDER_DB_COMPONENT_NAME, StorageType, WEBUI_COMPONENT_NAME, - WorkerConfig, SPIDER_DB_COMPONENT_NAME, + WorkerConfig, ) from clp_py_utils.clp_metadata_db_utils import ( MYSQL_TABLE_NAME_MAX_LEN, diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 56d53d415e..5f1ade4686 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -904,7 +904,9 @@ def validate_spider_config(self): if self.spider_db is None: raise ValueError("`spider_db` must be configured when using Spider orchestration.") if self.spider_scheduler is None: - raise ValueError("`spider_scheduler` must be configured when using Spider orchestration.") + raise ValueError( + "`spider_scheduler` must be configured when using Spider orchestration." + ) if self.database.type != DatabaseEngine.MARIADB: raise ValueError("Spider only supports MariaDB for the metadata database.") return self From a52a2666b736d5cc0351ef8669144d2200019b24 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 14:04:28 -0500 Subject: [PATCH 390/408] Use relative path for sibling --- .../clp-py-utils/clp_py_utils/initialize-spider-db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index 1273739722..727c25c413 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -7,10 +7,10 @@ from contextlib import closing from pydantic import ValidationError -from sql_adapter import SqlAdapter -from clp_py_utils.clp_config import ClpConfig -from clp_py_utils.core import read_yaml_config_file +from .clp_config import ClpConfig +from .core import read_yaml_config_file +from .sql_adapter import SqlAdapter # Setup logging # Create logger From b7668e35d2e23840bca0b4814cbecb0fcde3726e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 14:09:40 -0500 Subject: [PATCH 391/408] Add spider config in config template --- components/package-template/src/etc/clp-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/components/package-template/src/etc/clp-config.yaml b/components/package-template/src/etc/clp-config.yaml index f87a7acb2a..464319b633 100644 --- a/components/package-template/src/etc/clp-config.yaml +++ b/components/package-template/src/etc/clp-config.yaml @@ -23,6 +23,15 @@ #compression_scheduler: # jobs_poll_delay: 0.1 # seconds # logging_level: "INFO" +# type: "celery" +# +#spider_db: +# host: "localhost" +# port: 3306 +# name: "spider-db" +# +#spider_scheduler: +# host: "localhost" # #query_scheduler: # host: "localhost" From ae8a6ae982aedad6b1e10a9cf30a69ddb066304b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 14:10:30 -0500 Subject: [PATCH 392/408] Add spider db in credential template --- .../package-template/src/etc/credentials.template.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/package-template/src/etc/credentials.template.yaml b/components/package-template/src/etc/credentials.template.yaml index eb2f9b73f7..dd123131d6 100644 --- a/components/package-template/src/etc/credentials.template.yaml +++ b/components/package-template/src/etc/credentials.template.yaml @@ -11,3 +11,8 @@ ## Redis credentials #redis: # password: "pass" +# +## Spider DB credentials +#spider_db: +# username: "spider-user" +# password: "pass" From a2c449deac6dbb28343309680a051a30396834cd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 14:38:06 -0500 Subject: [PATCH 393/408] Fix script --- components/clp-py-utils/clp_py_utils/create-db-tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/create-db-tables.py b/components/clp-py-utils/clp_py_utils/create-db-tables.py index 17ed4cb809..2cb31796c1 100644 --- a/components/clp-py-utils/clp_py_utils/create-db-tables.py +++ b/components/clp-py-utils/clp_py_utils/create-db-tables.py @@ -63,7 +63,7 @@ def main(argv): return 1 # fmt: off cmd = [ - "python3", str(script_dir / "initialize-spider-db.py"), + "python3", "-m", "clp_py_utils.initialize-spider-db", "--config", str(config_file_path), ] # fmt: on From 92a97440917026d0b1668b78915e2797b3d37e37 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 15:37:57 -0500 Subject: [PATCH 394/408] Use same volume --- tools/deployment/package/docker-compose-all.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index e6a6bfc70c..bf85b005d4 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -39,6 +39,10 @@ x-volume-definitions: type: "bind" source: "${CLP_LOGS_DIR_HOST:-./var/log}" target: "/var/log" + clp-tmp: &volume_clp_tmp + type: "bind" + source: "${CLP_TMP_DIR_HOST:-./var/tmp}" + target: "/var/tmp" services: database: @@ -275,13 +279,11 @@ services: volumes: - *volume_clp_config_readonly - *volume_clp_logs + - *volume_clp_tmp - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/opt/clp/.aws:ro" - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/staged-archives" - - type: "bind" - source: "${CLP_TMP_DIR_HOST:-./var/tmp}" - target: "/var/tmp" command: [ "python3", "-u", @@ -309,13 +311,11 @@ services: volumes: - *volume_clp_config_readonly - *volume_clp_logs + - *volume_clp_tmp - "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives" - "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/opt/clp/.aws:ro" - "${CLP_LOGS_INPUT_DIR_HOST:-empty}:${CLP_LOGS_INPUT_DIR_CONTAINER:-/mnt/logs}" - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/staged-archives" - - type: "bind" - source: "${CLP_TMP_DIR_HOST:-./var/tmp}" - target: "/var/tmp" depends_on: db-table-creator: condition: "service_completed_successfully" From 84f016d91167e0b8303ca112b4fa2158f8948543 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 15:39:58 -0500 Subject: [PATCH 395/408] Add missing template --- components/package-template/src/etc/clp-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/package-template/src/etc/clp-config.yaml b/components/package-template/src/etc/clp-config.yaml index 464319b633..b10bfe56c8 100644 --- a/components/package-template/src/etc/clp-config.yaml +++ b/components/package-template/src/etc/clp-config.yaml @@ -32,6 +32,7 @@ # #spider_scheduler: # host: "localhost" +# port: 6000 # #query_scheduler: # host: "localhost" From 2293d50cd2e45fb449564526da71203be46bdfd2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 15:59:54 -0500 Subject: [PATCH 396/408] Construct spider db url in docker --- .../clp-package-utils/clp_package_utils/controller.py | 4 +--- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ---- tools/deployment/package/docker-compose-all.yaml | 10 ++++++++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 3d269b4274..e89bbb981f 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -269,9 +269,7 @@ def _set_up_env_for_spider_db(self) -> EnvVarsDict: env_vars = EnvVarsDict() # Database - env_vars |= { - "SPIDER_DB_URL": self._clp_config.spider_db.get_container_url(), - } + env_vars |= {"SPIDER_DB_NAME": self._clp_config.spider_db.name} # Credentials env_vars |= { diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 5f1ade4686..b268496498 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -273,10 +273,6 @@ def validate_type(cls, value): raise ValueError("Spider only supports MariaDB for the metadata database.") return value - def get_url(self): - self.ensure_credentials_loaded() - return f"jdbc:mariadb://{self.host}:{self.port}/{self.name}?user={self.username}&password={self.password}" - def get_container_url(self): self.ensure_credentials_loaded() return f"jdbc:mariadb://{DB_COMPONENT_NAME}:{self.DEFAULT_PORT}/{self.name}?user={self.username}&password={self.password}" diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index bf85b005d4..48bcc4da7e 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -226,7 +226,10 @@ services: "/opt/clp/bin/spider_scheduler", "--host", "${SPIDER_SCHEDULER_HOST}", "--port", "${SPIDER_SCHEDULER_PORT}", - "--storage_url", "${SPIDER_DB_URL}" + "--storage_url", "jdbc:mariadb://database:${CLP_DB_PORT:-3306}/\ + ${SPIDER_DB_NAME:-spider-db}?\ + user=${SPIDER_DB_USER:?Please set a value.}\ + &password=${SPIDER_DB_PASS:?Please set a value.}", ] compression-scheduler: @@ -324,7 +327,10 @@ services: "-u", "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--storage-url", "${SPIDER_DB_URL}", + "--storage-url", "jdbc:mariadb://database:${CLP_DB_PORT:-3305}/\ + ${SPIDER_DB_NAME:-spider-db}?\ + user=${SPIDER_DB_USER:?Please set a value.}\ + &password=${SPIDER_DB_PASS:?Please set a value.}", # NOTE: Leave host to spider scheduler's host. This only affects task placement. "--host", "spider-scheduler", ] From 60ef08be5797a2a670ded7615e854ca97cdea482 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 20 Nov 2025 16:21:10 -0500 Subject: [PATCH 397/408] Move get ip into start worker script --- components/clp-py-utils/clp_py_utils/start-spider-worker.py | 4 ++-- tools/deployment/package/docker-compose-all.yaml | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index b3c98c1eb8..20a5629c5c 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -2,6 +2,7 @@ import logging import os import pathlib +import socket import subprocess # Setup logging @@ -25,7 +26,6 @@ def parse_args() -> argparse.Namespace: "--concurrency", type=int, default=1, help="Number of concurrent spider workers." ) parser.add_argument("--storage-url", type=str, required=True, help="Spider storage URL.") - parser.add_argument("--host", type=str, required=True, help="Worker host address.") return parser.parse_args() @@ -38,7 +38,7 @@ def main() -> None: logger.error("Concurrency must be at least 1.") exit(1) storage_url = args.storage_url - host = args.host + host = socket.gethostbyname(socket.gethostname()) clp_home = os.getenv("CLP_HOME", "/opt/clp") spider_worker_path = pathlib.Path(clp_home) / "bin" / "spider_worker" diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index 48bcc4da7e..bc3078f7c0 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -331,8 +331,6 @@ services: ${SPIDER_DB_NAME:-spider-db}?\ user=${SPIDER_DB_USER:?Please set a value.}\ &password=${SPIDER_DB_PASS:?Please set a value.}", - # NOTE: Leave host to spider scheduler's host. This only affects task placement. - "--host", "spider-scheduler", ] webui: From dcc834a1b456caba0cd9bd323cc5e6dcfcd97304 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 21 Nov 2025 13:39:06 -0500 Subject: [PATCH 398/408] Bug fix --- tools/deployment/package/docker-compose-all.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index bc3078f7c0..932f71ee3e 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -327,7 +327,7 @@ services: "-u", "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", - "--storage-url", "jdbc:mariadb://database:${CLP_DB_PORT:-3305}/\ + "--storage-url", "jdbc:mariadb://database:${CLP_DB_PORT:-3306}/\ ${SPIDER_DB_NAME:-spider-db}?\ user=${SPIDER_DB_USER:?Please set a value.}\ &password=${SPIDER_DB_PASS:?Please set a value.}", From 74a31494d233ab61d189158e88182b13a8c3b1c7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 21 Nov 2025 13:41:58 -0500 Subject: [PATCH 399/408] Add back optional host, improve error handling --- .../clp-py-utils/clp_py_utils/start-spider-worker.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index 20a5629c5c..01acd6217d 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -26,6 +26,7 @@ def parse_args() -> argparse.Namespace: "--concurrency", type=int, default=1, help="Number of concurrent spider workers." ) parser.add_argument("--storage-url", type=str, required=True, help="Spider storage URL.") + parser.add_argument("--host", type=str, required=False, default=None, help="The worker host.") return parser.parse_args() @@ -38,7 +39,13 @@ def main() -> None: logger.error("Concurrency must be at least 1.") exit(1) storage_url = args.storage_url - host = socket.gethostbyname(socket.gethostname()) + host = args.host + if host is None: + try: + host = socket.gethostbyname(socket.gethostname()) + except (socket.gaierror, socket.herror) as e: + logger.error(f"Failed to resolve hostname: {e}") + exit(1) clp_home = os.getenv("CLP_HOME", "/opt/clp") spider_worker_path = pathlib.Path(clp_home) / "bin" / "spider_worker" From 01c60a22aed79db4d435280b4d95468f7fa44a44 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 24 Nov 2025 20:09:26 -0500 Subject: [PATCH 400/408] Apply suggestions from code review Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- .../clp_py_utils/start-spider-worker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index 01acd6217d..e41d58fbfa 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -23,7 +23,7 @@ def parse_args() -> argparse.Namespace: """ parser = argparse.ArgumentParser() parser.add_argument( - "--concurrency", type=int, default=1, help="Number of concurrent spider workers." + "--concurrency", type=int, default=1, help="Number of concurrent Spider workers." ) parser.add_argument("--storage-url", type=str, required=True, help="Spider storage URL.") parser.add_argument("--host", type=str, required=False, default=None, help="The worker host.") @@ -67,14 +67,14 @@ def main() -> None: process.terminate() exit(1) - failed = False + exit_code = 0 for process in processes: - exit_code = process.wait() - if exit_code != 0: - logger.error(f"Spider worker exited with code {exit_code}") - failed = True - if failed: - exit(1) + worker_proc_exit_code = process.wait() + if worker_proc_exit_code != 0: + logger.error(f"Spider worker exited with code {worker_proc_exit_code}") + exit_code = 1 + + exit(exit_code) if __name__ == "__main__": From 3c502d2c4c662bb0552eaa12b453df67e32fe9b0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 24 Nov 2025 20:20:15 -0500 Subject: [PATCH 401/408] Apply suggestion from code review Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- .../scheduler/compress/compression_scheduler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index 0e6db37aad..4521d4bad5 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -520,6 +520,7 @@ def main(argv): logger.info(f"Starting {COMPRESSION_SCHEDULER_COMPONENT_NAME}") sql_adapter = SqlAdapter(clp_config.database) + task_manager: CeleryTaskManager | SpiderTaskManager if clp_config.compression_scheduler.type == OrchestrationType.celery: task_manager = CeleryTaskManager() elif clp_config.compression_scheduler.type == OrchestrationType.spider: From 3d0c8edde4b49632fc3239370a9b4ec0d3f255e8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 24 Nov 2025 20:29:09 -0500 Subject: [PATCH 402/408] Rename argument --- .../clp-py-utils/clp_py_utils/start-spider-worker.py | 10 +++++----- tools/deployment/package/docker-compose-all.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index e41d58fbfa..910c02f6f7 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -23,7 +23,7 @@ def parse_args() -> argparse.Namespace: """ parser = argparse.ArgumentParser() parser.add_argument( - "--concurrency", type=int, default=1, help="Number of concurrent Spider workers." + "--num-workers", type=int, default=1, help="Number of concurrent Spider workers." ) parser.add_argument("--storage-url", type=str, required=True, help="Spider storage URL.") parser.add_argument("--host", type=str, required=False, default=None, help="The worker host.") @@ -34,9 +34,9 @@ def main() -> None: """Main function to start multiple spider workers.""" # Parse arguments args = parse_args() - concurrency = args.concurrency - if concurrency < 1: - logger.error("Concurrency must be at least 1.") + num_workers = args.num_workers + if num_workers < 1: + logger.error("Number of concurrent workers must be at least 1.") exit(1) storage_url = args.storage_url host = args.host @@ -56,7 +56,7 @@ def main() -> None: # Start multiple spider workers processes = [] try: - for _ in range(concurrency): + for _ in range(num_workers): process = subprocess.Popen( [spider_worker_path, "--storage_url", storage_url, "--host", host] ) diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index 932f71ee3e..ecab66c65e 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -326,7 +326,7 @@ services: "python3", "-u", "/opt/clp/lib/python3/site-packages/clp_py_utils/start-spider-worker.py", - "--concurrency", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", + "--num-workers", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}", "--storage-url", "jdbc:mariadb://database:${CLP_DB_PORT:-3306}/\ ${SPIDER_DB_NAME:-spider-db}?\ user=${SPIDER_DB_USER:?Please set a value.}\ From c312c19720e695426d436c1ad910010518abdf5d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 24 Nov 2025 20:47:19 -0500 Subject: [PATCH 403/408] Fix name --- components/clp-py-utils/clp_py_utils/start-spider-worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/start-spider-worker.py b/components/clp-py-utils/clp_py_utils/start-spider-worker.py index 910c02f6f7..80adeb546b 100644 --- a/components/clp-py-utils/clp_py_utils/start-spider-worker.py +++ b/components/clp-py-utils/clp_py_utils/start-spider-worker.py @@ -31,7 +31,7 @@ def parse_args() -> argparse.Namespace: def main() -> None: - """Main function to start multiple spider workers.""" + """Main function to start multiple Spider workers.""" # Parse arguments args = parse_args() num_workers = args.num_workers @@ -62,7 +62,7 @@ def main() -> None: ) processes.append(process) except OSError as e: - logger.error(f"Failed to start spider worker: {e}") + logger.error(f"Failed to start Spider worker: {e}") for process in processes: process.terminate() exit(1) From 4a0e7763207e74ce12d83327eb8508114332c9df Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 28 Nov 2025 16:59:01 -0500 Subject: [PATCH 404/408] Bug fix --- components/clp-package-utils/clp_package_utils/controller.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 0fc0d2d74b..9568123ed4 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -18,6 +18,9 @@ CLP_DB_ROOT_PASS_ENV_VAR_NAME, CLP_DB_ROOT_USER_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + CLP_QUEUE_PASS_ENV_VAR_NAME, + CLP_QUEUE_USER_ENV_VAR_NAME, + CLP_REDIS_PASS_ENV_VAR_NAME, ClpConfig, ClpDbUserType, COMPRESSION_JOBS_TABLE_NAME, From c20f321407ba9702d24c4615e9fbc910788a27fc Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 28 Nov 2025 17:51:48 -0500 Subject: [PATCH 405/408] Fix spider db credential --- .../clp_package_utils/controller.py | 4 ++-- .../clp-py-utils/clp_py_utils/clp_config.py | 20 ++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 9568123ed4..988c99caac 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -343,8 +343,8 @@ def _set_up_env_for_spider_db(self) -> EnvVarsDict: # Credentials env_vars |= { - SPIDER_DB_PASS_ENV_VAR_NAME: self._clp_config.spider_db.password, - SPIDER_DB_USER_ENV_VAR_NAME: self._clp_config.spider_db.username, + SPIDER_DB_PASS_ENV_VAR_NAME: self._clp_config.spider_db.credentials.password, + SPIDER_DB_USER_ENV_VAR_NAME: self._clp_config.spider_db.credentials.username, } return env_vars diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index d070650cb0..aaa62a1f4d 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -352,6 +352,9 @@ def transform_for_container(self): class SpiderDb(Database): name: str = "spider-db" + # Override credentials for spider db user + credentials: DbUserCredentials = DbUserCredentials() + @field_validator("type") @classmethod def validate_type(cls, value): @@ -361,22 +364,29 @@ def validate_type(cls, value): def get_container_url(self): self.ensure_credentials_loaded() - return f"jdbc:mariadb://{DB_COMPONENT_NAME}:{self.DEFAULT_PORT}/{self.name}?user={self.username}&password={self.password}" + return ( + f"jdbc:mariadb://{DB_COMPONENT_NAME}:{self.DEFAULT_PORT}/" + f"{self.name}?user={self.credentials.username}&password={self.credentials.password}" + ) def load_credentials_from_env(self): """ :raise ValueError: if any expected environment variable is not set. """ - self.username = _get_env_var(SPIDER_DB_USER_ENV_VAR_NAME) - self.password = _get_env_var(SPIDER_DB_PASS_ENV_VAR_NAME) + self.credentials.username = _get_env_var(SPIDER_DB_USER_ENV_VAR_NAME) + self.credentials.password = _get_env_var(SPIDER_DB_PASS_ENV_VAR_NAME) def load_credentials_from_file(self, credentials_file_path: pathlib.Path): config = read_yaml_config_file(credentials_file_path) if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{SPIDER_DB_COMPONENT_NAME}.username") - self.password = get_config_value(config, f"{SPIDER_DB_COMPONENT_NAME}.password") + self.credentials.username = get_config_value( + config, f"{SPIDER_DB_COMPONENT_NAME}.username" + ) + self.credentials.password = get_config_value( + config, f"{SPIDER_DB_COMPONENT_NAME}.password" + ) except KeyError as ex: raise ValueError( f"Credentials file '{credentials_file_path}' does not contain key '{ex}'." From bd9a78de2406b0c01b9d56a60abe70a90291121b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 28 Nov 2025 17:53:10 -0500 Subject: [PATCH 406/408] Fix mariadb type hint --- components/clp-py-utils/clp_py_utils/sql_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/sql_adapter.py b/components/clp-py-utils/clp_py_utils/sql_adapter.py index 9d1076490b..2fe080d987 100644 --- a/components/clp-py-utils/clp_py_utils/sql_adapter.py +++ b/components/clp-py-utils/clp_py_utils/sql_adapter.py @@ -70,7 +70,7 @@ def create_connection( self, disable_localhost_socket_connection: bool = False, user_type: ClpDbUserType = ClpDbUserType.CLP, - ) -> mysql.connector.abstracts.MySQLConnectionAbstract | mariadb.connection: + ) -> mysql.connector.abstracts.MySQLConnectionAbstract | mariadb.Connection: """ Creates a connection to the database. @@ -151,7 +151,7 @@ def _create_mariadb_connection( self, disable_localhost_socket_connection: bool = False, user_type: ClpDbUserType = ClpDbUserType.CLP, - ) -> mariadb.connection: + ) -> mariadb.Connection: try: connection = mariadb.connect( **self.database_config.get_mysql_connection_params( From 515a3d349865d4b19c7d62856372aea5679f4a1b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 28 Nov 2025 19:31:43 -0500 Subject: [PATCH 407/408] Fix db password --- components/clp-py-utils/clp_py_utils/clp_config.py | 11 ++++++++++- .../clp-py-utils/clp_py_utils/initialize-spider-db.py | 8 +++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index aaa62a1f4d..51cd6eeea1 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -362,6 +362,15 @@ def validate_type(cls, value): raise ValueError("Spider only supports MariaDB for the metadata database.") return value + def ensure_credentials_loaded(self, **kwargs) -> None: + """ + Ensures that credentials are loaded. + + :raise ValueError: If credentials are not loaded. + """ + if self.credentials.username is None or self.credentials.password is None: + raise ValueError("Credentials for spider_db are not loaded.") + def get_container_url(self): self.ensure_credentials_loaded() return ( @@ -369,7 +378,7 @@ def get_container_url(self): f"{self.name}?user={self.credentials.username}&password={self.credentials.password}" ) - def load_credentials_from_env(self): + def load_credentials_from_env(self, **kwargs) -> None: """ :raise ValueError: if any expected environment variable is not set. """ diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index dbb15ed02f..c1304e9d5b 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -229,6 +229,8 @@ def main(argv): try: clp_config = ClpConfig.model_validate(read_yaml_config_file(config_path)) clp_config.database.load_credentials_from_env() + clp_config.database.load_credentials_from_config(user_type=ClpDbUserType.CLP) + clp_config.database.load_credentials_from_config(user_type=ClpDbUserType.ROOT) if clp_config.spider_db is None: logger.error("Spider database configuration not found in CLP configuration.") return -1 @@ -248,9 +250,9 @@ def main(argv): closing(sql_adapter.create_connection(user_type=ClpDbUserType.ROOT)) as db_conn, closing(db_conn.cursor()) as db_cursor, ): - clp_db_user = clp_config.database.username - spider_db_name = spider_db_config.name - spider_db_user = spider_db_config.username + clp_db_user = clp_config.database.credentials[ClpDbUserType.CLP].username + spider_db_name = spider_db_config.credentials.name + spider_db_user = spider_db_config.credentials.username spider_db_password = spider_db_config.password if not _validate_name(spider_db_name): logger.error(f"Invalid database name: {spider_db_name}") From 5135fe89045e9f3612915a439232b258eaf64653 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 28 Nov 2025 22:07:18 -0500 Subject: [PATCH 408/408] Bug fix --- .../clp-py-utils/clp_py_utils/initialize-spider-db.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py index c1304e9d5b..e3c5be2648 100644 --- a/components/clp-py-utils/clp_py_utils/initialize-spider-db.py +++ b/components/clp-py-utils/clp_py_utils/initialize-spider-db.py @@ -229,8 +229,8 @@ def main(argv): try: clp_config = ClpConfig.model_validate(read_yaml_config_file(config_path)) clp_config.database.load_credentials_from_env() - clp_config.database.load_credentials_from_config(user_type=ClpDbUserType.CLP) - clp_config.database.load_credentials_from_config(user_type=ClpDbUserType.ROOT) + clp_config.database.load_credentials_from_env(user_type=ClpDbUserType.CLP) + clp_config.database.load_credentials_from_env(user_type=ClpDbUserType.ROOT) if clp_config.spider_db is None: logger.error("Spider database configuration not found in CLP configuration.") return -1 @@ -251,9 +251,9 @@ def main(argv): closing(db_conn.cursor()) as db_cursor, ): clp_db_user = clp_config.database.credentials[ClpDbUserType.CLP].username - spider_db_name = spider_db_config.credentials.name + spider_db_name = spider_db_config.name spider_db_user = spider_db_config.credentials.username - spider_db_password = spider_db_config.password + spider_db_password = spider_db_config.credentials.password if not _validate_name(spider_db_name): logger.error(f"Invalid database name: {spider_db_name}") return -1