feat(integration-tests): Spin up and tear down the clp-text and clp-json packages.

integration-tests/.pytest.ini

@@ -3,6 +3,7 @@ addopts =
     --capture=no
     --code-highlight=yes
     --color=yes
+    -rA
     --strict-config
     --strict-markers
     --verbose
@@ -18,3 +19,4 @@ markers =
     clp: mark tests that use the CLP storage engine
     clp_s: mark tests that use the CLP-S storage engine
     core: mark tests that test the CLP core binaries
+    package: mark tests that run when the CLP package is active

integration-tests/pyproject.toml

@@ -27,6 +27,8 @@ dev = [
     "ruff>=0.11.12",
     "pytest>=8.4.1",
     "pytest-env>=1.1.5",
+    "PyYAML>=6.0",
+    "types-PyYAML>=6.0.12.20240808",
 ]
 
 [tool.mypy]

integration-tests/tests/conftest.py

@@ -3,4 +3,6 @@
 pytest_plugins = [
     "tests.fixtures.integration_test_config",
     "tests.fixtures.integration_test_logs",
+    "tests.fixtures.package_instance_fixtures",
+    "tests.fixtures.package_config_fixtures",
 ]

integration-tests/tests/fixtures/integration_test_config.py

@@ -7,7 +7,6 @@
 from tests.utils.config import (
     CoreConfig,
     IntegrationTestConfig,
-    PackageConfig,
 )
 from tests.utils.utils import get_env_var
 
@@ -20,14 +19,6 @@ def core_config() -> CoreConfig:
     )
 
 
-@pytest.fixture(scope="session")
-def package_config() -> PackageConfig:
-    """Fixture that provides a PackageConfig shared across tests."""
-    return PackageConfig(
-        clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).expanduser().resolve()
-    )
-
-
 @pytest.fixture(scope="session")
 def integration_test_config() -> IntegrationTestConfig:
     """Fixture that provides an IntegrationTestConfig shared across tests."""

integration-tests/tests/fixtures/package_config_fixtures.py

@@ -0,0 +1,70 @@
+"""Fixtures that create and remove temporary config files for CLP packages."""
+
+import contextlib
+import logging
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+
+from tests.utils.clp_mode_utils import CLP_MODE_CONFIGS, get_clp_config_from_mode
+from tests.utils.config import PackageConfig
+from tests.utils.utils import get_env_var
+
+logger = logging.getLogger(__name__)
+
+
+def _build_package_config_for_mode(mode_name: str) -> PackageConfig:
+    """
+    Constructs a PackageConfig for the specified CLP mode.
+
+    :param mode_name:
+    :return: A PackageConfig object configured for the given mode.
+    :raise KeyError: if the mode name is unknown.
+    """
+    if mode_name not in CLP_MODE_CONFIGS:
+        err_msg = f"Unknown CLP mode '{mode_name}'."
+        raise KeyError(err_msg)
+
+    clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")).expanduser().resolve()
+    test_root_dir = Path(get_env_var("CLP_BUILD_DIR")).expanduser().resolve() / "integration-tests"
+
+    return PackageConfig(
+        clp_package_dir=clp_package_dir,
+        test_root_dir=test_root_dir,
+        mode_name=mode_name,
+    )
+
+
+@pytest.fixture
+def clp_config(
+    request: pytest.FixtureRequest,
+) -> Iterator[PackageConfig]:
+    """
+    Creates and maintains a PackageConfig object for a specific CLP mode.
+
+    :param request:
+    :return: An iterator that yields the PackageConfig object for the specified mode.
+    """
+    mode_name: str = request.param
+    logger.info("Creating a temporary config file for the %s package...", mode_name)
+
+    package_config = _build_package_config_for_mode(mode_name)
+
+    # Create the temp config file.
+    clp_config_obj = get_clp_config_from_mode(mode_name)
+    temp_config_file_path = PackageConfig.write_temp_config_file(
+        clp_config=clp_config_obj,
+        temp_config_dir=package_config.temp_config_dir,
+        mode_name=mode_name,
+    )
+
+    logger.info("The temporary config file has been written for the %s package.", mode_name)
+
+    try:
+        yield package_config
+    finally:
+        logger.info("Removing the temporary config file...")
+        with contextlib.suppress(FileNotFoundError):
+            temp_config_file_path.unlink()
+        logger.info("The temporary config file has been removed.")

integration-tests/tests/fixtures/package_instance_fixtures.py

@@ -0,0 +1,56 @@
+"""Fixtures that start and stop CLP package instances for integration tests."""
+
+import logging
+import subprocess
+from collections.abc import Iterator
+
+import pytest
+
+from tests.utils.clp_mode_utils import CLP_MODE_CONFIGS
+from tests.utils.config import (
+    PackageConfig,
+    PackageInstance,
+)
+from tests.utils.package_utils import (
+    start_clp_package,
+    stop_clp_package,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def clp_package(
+    clp_config: PackageConfig,
+) -> Iterator[PackageInstance]:
+    """
+    Starts a CLP package instance for the given configuration and stops it during teardown.
+
+    :param clp_config:
+    :return: Iterator that yields the running package instance.
+    """
+    mode_name = clp_config.mode_name
+    logger.info("Starting up the %s package...", mode_name)
+
+    # Start the package.
+    start_clp_package(clp_config)
+
+    instance: PackageInstance | None = None
+    try:
+        required_components = CLP_MODE_CONFIGS[mode_name][1]
+        instance = PackageInstance(package_config=clp_config, component_list=required_components)
+        instance_id = instance.clp_instance_id
+        logger.info(
+            "An instance of the %s package was started successfully. Its instance ID is '%s'",
+            mode_name,
+            instance_id,
+        )
+        yield instance
+    finally:
+        logger.info("Now stopping the %s package...", mode_name)
+        if instance is not None:
+            stop_clp_package(instance)
+        else:
+            # This means setup failed after start; fall back to calling stop script directly
+            subprocess.run([str(clp_config.stop_script_path)], check=True)
+        logger.info("The %s package was stopped successfully.", mode_name)

integration-tests/tests/test_package_start.py

@@ -0,0 +1,65 @@
+"""Integration tests verifying that the CLP package can be started and stopped."""
+
+import logging
+
+import pytest
+
+from tests.utils.clp_mode_utils import CLP_MODE_CONFIGS
+from tests.utils.config import PackageInstance
+from tests.utils.package_utils import (
+    validate_package_running,
+    validate_running_mode_correct,
+)
+
+TEST_MODES = CLP_MODE_CONFIGS.keys()
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.package
+@pytest.mark.parametrize("clp_config", TEST_MODES, indirect=True)
+def test_clp_package(clp_package: PackageInstance) -> None:
+    """
+    Validate that all of the components of the CLP package start up successfully for the selected
+    mode of operation.
+
+    :param clp_package:
+    """
+    mode_name = clp_package.package_config.mode_name
+    instance_id = clp_package.clp_instance_id
+
+    # Ensure that all package components are running.
+    logger.info(
+        "Checking if all components of %s package with instance ID '%s' are running properly...",
+        mode_name,
+        instance_id,
+    )
+
+    running, fail_msg = validate_package_running(clp_package)
+    if not running:
+        assert fail_msg is not None
+        pytest.fail(fail_msg)
+
+    logger.info(
+        "All components of the %s package with instance ID '%s' are running properly.",
+        mode_name,
+        instance_id,
+    )
+
+    # Ensure that the package is running in the correct mode.
+    logger.info(
+        "Checking that the %s package with instance ID '%s' is running in the correct mode...",
+        mode_name,
+        instance_id,
+    )
+
+    correct, fail_msg = validate_running_mode_correct(clp_package)
+    if not correct:
+        assert fail_msg is not None
+        pytest.fail(fail_msg)
+
+    logger.info(
+        "The %s package with instance ID '%s' is running in the correct mode.",
+        mode_name,
+        instance_id,
+    )

integration-tests/tests/utils/clp_mode_utils.py

@@ -0,0 +1,115 @@
+"""Provides utilities related to the user-level configurations of CLP's operating modes."""
+
+from collections.abc import Callable
+from typing import Any
+
+from clp_py_utils.clp_config import (
+    ClpConfig,
+    COMPRESSION_SCHEDULER_COMPONENT_NAME,
+    COMPRESSION_WORKER_COMPONENT_NAME,
+    DB_COMPONENT_NAME,
+    GARBAGE_COLLECTOR_COMPONENT_NAME,
+    Package,
+    QUERY_SCHEDULER_COMPONENT_NAME,
+    QUERY_WORKER_COMPONENT_NAME,
+    QueryEngine,
+    QUEUE_COMPONENT_NAME,
+    REDIS_COMPONENT_NAME,
+    REDUCER_COMPONENT_NAME,
+    RESULTS_CACHE_COMPONENT_NAME,
+    StorageEngine,
+    WEBUI_COMPONENT_NAME,
+)
+
+
+# TODO: This will eventually be replaced by a formalized mapping between component and service.
+def _to_docker_compose_service_name(name: str) -> str:
+    """
+    Convert a component name to a Docker Compose service name.
+
+    :param name:
+    :return: Service name with underscores replaced by hyphens
+    """
+    return name.replace("_", "-")
+
+
+CLP_MODE_CONFIGS: dict[str, tuple[Callable[[], ClpConfig], list[str]]] = {
+    "clp-text": (
+        lambda: ClpConfig(
+            package=Package(
+                storage_engine=StorageEngine.CLP,
+                query_engine=QueryEngine.CLP,
+            ),
+        ),
+        [
+            _to_docker_compose_service_name(DB_COMPONENT_NAME),
+            _to_docker_compose_service_name(QUEUE_COMPONENT_NAME),
+            _to_docker_compose_service_name(REDIS_COMPONENT_NAME),
+            _to_docker_compose_service_name(REDUCER_COMPONENT_NAME),
+            _to_docker_compose_service_name(RESULTS_CACHE_COMPONENT_NAME),
+            _to_docker_compose_service_name(COMPRESSION_SCHEDULER_COMPONENT_NAME),
+            _to_docker_compose_service_name(QUERY_SCHEDULER_COMPONENT_NAME),
+            _to_docker_compose_service_name(COMPRESSION_WORKER_COMPONENT_NAME),
+            _to_docker_compose_service_name(QUERY_WORKER_COMPONENT_NAME),
+            _to_docker_compose_service_name(WEBUI_COMPONENT_NAME),
+            _to_docker_compose_service_name(GARBAGE_COLLECTOR_COMPONENT_NAME),
+        ],
+    ),
+    "clp-json": (
+        lambda: ClpConfig(
+            package=Package(
+                storage_engine=StorageEngine.CLP_S,
+                query_engine=QueryEngine.CLP_S,
+            ),
+        ),
+        [
+            _to_docker_compose_service_name(DB_COMPONENT_NAME),
+            _to_docker_compose_service_name(QUEUE_COMPONENT_NAME),
+            _to_docker_compose_service_name(REDIS_COMPONENT_NAME),
+            _to_docker_compose_service_name(REDUCER_COMPONENT_NAME),
+            _to_docker_compose_service_name(RESULTS_CACHE_COMPONENT_NAME),
+            _to_docker_compose_service_name(COMPRESSION_SCHEDULER_COMPONENT_NAME),
+            _to_docker_compose_service_name(QUERY_SCHEDULER_COMPONENT_NAME),
+            _to_docker_compose_service_name(COMPRESSION_WORKER_COMPONENT_NAME),
+            _to_docker_compose_service_name(QUERY_WORKER_COMPONENT_NAME),
+            _to_docker_compose_service_name(WEBUI_COMPONENT_NAME),
+            _to_docker_compose_service_name(GARBAGE_COLLECTOR_COMPONENT_NAME),
+        ],
+    ),
+}
+
+
+def compute_mode_signature(config: ClpConfig) -> tuple[Any, ...]:
+    """
+    Constructs a signature that captures the mode-defining aspects of a ClpConfig object.
+
+    :param config:
+    :return: Tuple that encodes fields used to determine an operating mode.
+    """
+    return (
+        config.logs_input.type,
+        config.package.storage_engine.value,
+        config.package.query_engine.value,
+        config.mcp_server is not None,
+        config.presto is not None,
+        config.archive_output.storage.type,
+        config.stream_output.storage.type,
+        config.aws_config_directory is not None,
+        config.get_deployment_type(),
+    )
+
+
+def get_clp_config_from_mode(mode_name: str) -> ClpConfig:
+    """
+    Return a ClpConfig object for the given mode name.
+
+    :param mode_name:
+    :return: ClpConfig object corresponding to the mode.
+    :raise ValueError: If the mode is not supported.
+    """
+    try:
+        config = CLP_MODE_CONFIGS[mode_name][0]
+    except KeyError as err:
+        err_msg = f"Unsupported mode: {mode_name}"
+        raise ValueError(err_msg) from err
+    return config()