feat(integration-tests): Run task tests:integration:package to run all package integration tests.

integration-tests/.pytest.ini

@@ -3,6 +3,7 @@ addopts =
     --capture=no
     --code-highlight=yes
     --color=yes
+    -rA
     --strict-config
     --strict-markers
     --verbose
@@ -18,3 +19,4 @@ markers =
     clp: mark tests that use the CLP storage engine
     clp_s: mark tests that use the CLP-S storage engine
     core: mark tests that test the CLP core binaries
+    package: mark tests that run when the CLP package is active

integration-tests/pyproject.toml

@@ -27,6 +27,8 @@ dev = [
     "ruff>=0.11.12",
     "pytest>=8.4.1",
     "pytest-env>=1.1.5",
+    "PyYAML>=6.0",
+    "types-PyYAML>=6.0.12.20240808",
 ]
 
 [tool.mypy]

integration-tests/tests/conftest.py

@@ -1,6 +1,9 @@
-"""Make the fixtures defined in `tests/fixtures/` globally available without imports."""
+"""Global pytest setup."""
 
+# Make the fixtures defined in `tests/fixtures/` globally available without imports.
 pytest_plugins = [
     "tests.fixtures.integration_test_logs",
     "tests.fixtures.path_configs",
+    "tests.fixtures.package_instance",
+    "tests.fixtures.package_config",
 ]

integration-tests/tests/fixtures/package_config.py

@@ -0,0 +1,41 @@
+"""Fixtures that create and remove temporary config files for CLP packages."""
+
+import contextlib
+from collections.abc import Iterator
+
+import pytest
+
+from tests.utils.clp_mode_utils import (
+    get_clp_config_from_mode,
+)
+from tests.utils.config import PackageConfig, PackagePathConfig
+
+
+@pytest.fixture
+def fixt_package_config(
+    fixt_package_path_config: PackagePathConfig,
+    request: pytest.FixtureRequest,
+) -> Iterator[PackageConfig]:
+    """
+    Creates and maintains a PackageConfig object for a specific CLP mode.
+
+    :param request:
+    :return: An iterator that yields the PackageConfig object for the specified mode.
+    """
+    mode_name: str = request.param
+
+    # Get the ClpConfig for this mode.
+    clp_config_obj = get_clp_config_from_mode(mode_name)
+
+    # Construct PackageConfig.
+    package_config = PackageConfig(
+        path_config=fixt_package_path_config,
+        mode_name=mode_name,
+        clp_config=clp_config_obj,
+    )
+
+    try:
+        yield package_config
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            package_config.temp_config_file_path.unlink()

integration-tests/tests/fixtures/package_instance.py

@@ -0,0 +1,41 @@
+"""Fixtures that start and stop CLP package instances for integration tests."""
+
+import subprocess
+from collections.abc import Iterator
+
+import pytest
+
+from tests.utils.config import (
+    PackageConfig,
+    PackageInstance,
+)
+from tests.utils.package_utils import (
+    start_clp_package,
+    stop_clp_package,
+)
+
+
+@pytest.fixture
+def fixt_package_instance(fixt_package_config: PackageConfig) -> Iterator[PackageInstance]:
+    """
+    Starts a CLP package instance for the given configuration and stops it during teardown.
+
+    :param fixt_package_config:
+    :param request:
+    :return: Iterator that yields the running package instance.
+    """
+    mode_name = fixt_package_config.mode_name
+    instance: PackageInstance | None = None
+
+    try:
+        start_clp_package(fixt_package_config)
+        instance = PackageInstance(package_config=fixt_package_config)
+        yield instance
+    except RuntimeError:
+        pytest.fail(f"Failed to start the {mode_name} package.")
+    finally:
+        if instance is not None:
+            stop_clp_package(instance)
+        else:
+            # This means setup failed after start; fall back to calling stop script directly
+            subprocess.run([str(fixt_package_config.path_config.stop_script_path)], check=False)

integration-tests/tests/fixtures/path_configs.py

@@ -25,6 +25,11 @@ def integration_test_path_config() -> IntegrationTestPathConfig:
 
 
 @pytest.fixture(scope="session")
-def package_path_config() -> PackagePathConfig:
-    """Provides paths for the clp-package directory and its contents."""
-    return PackagePathConfig(clp_package_dir=resolve_path_env_var("CLP_PACKAGE_DIR"))
+def fixt_package_path_config(
+    integration_test_path_config: IntegrationTestPathConfig,
+) -> PackagePathConfig:
+    """Fixture that provides a PackagePathConfig shared across tests."""
+    return PackagePathConfig(
+        clp_package_dir=resolve_path_env_var("CLP_PACKAGE_DIR"),
+        test_root_dir=integration_test_path_config.test_root_dir,
+    )

integration-tests/tests/test_package_start.py

@@ -0,0 +1,26 @@
+"""Integration tests verifying that the CLP package can be started and stopped."""
+
+import logging
+
+import pytest
+
+from tests.utils.clp_mode_utils import CLP_MODE_CONFIGS
+from tests.utils.config import PackageInstance
+
+TEST_MODES = CLP_MODE_CONFIGS.keys()
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.package
+@pytest.mark.parametrize("fixt_package_config", TEST_MODES, indirect=True)
+def test_clp_package(fixt_package_instance: PackageInstance) -> None:
+    """
+    Validate that the CLP package starts up successfully for the selected mode of operation.
+
+    :param fixt_package_instance:
+    """
+    # TODO: write code that properly validates that the package is running. This is a placeholder.
+    mode_name = fixt_package_instance.package_config.mode_name
+    message = f"The '{mode_name}' package has been spun up successfully."
+    logger.info(message)

integration-tests/tests/utils/clp_mode_utils.py

@@ -0,0 +1,41 @@
+"""Provides utilities related to the user-level configurations of CLP's operating modes."""
+
+from collections.abc import Callable
+
+from clp_py_utils.clp_config import (
+    ClpConfig,
+    Package,
+    QueryEngine,
+    StorageEngine,
+)
+
+CLP_MODE_CONFIGS: dict[str, Callable[[], ClpConfig]] = {
+    "clp-text": lambda: ClpConfig(
+        package=Package(
+            storage_engine=StorageEngine.CLP,
+            query_engine=QueryEngine.CLP,
+        ),
+    ),
+    "clp-json": lambda: ClpConfig(
+        package=Package(
+            storage_engine=StorageEngine.CLP_S,
+            query_engine=QueryEngine.CLP_S,
+        ),
+    ),
+}
+
+
+def get_clp_config_from_mode(mode_name: str) -> ClpConfig:
+    """
+    Return a ClpConfig object for the given mode name.
+
+    :param mode_name:
+    :return: ClpConfig object corresponding to the mode.
+    :raise ValueError: If the mode is not supported.
+    """
+    try:
+        config = CLP_MODE_CONFIGS[mode_name]
+    except KeyError as err:
+        err_msg = f"Unsupported mode: {mode_name}"
+        raise ValueError(err_msg) from err
+    return config()

integration-tests/tests/utils/config.py

@@ -4,12 +4,19 @@
 
 from dataclasses import dataclass, field, InitVar
 from pathlib import Path
+from typing import TYPE_CHECKING
+
+import yaml
 
 from tests.utils.utils import (
     unlink,
     validate_dir_exists,
+    validate_file_exists,
 )
 
+if TYPE_CHECKING:
+    from clp_py_utils.clp_config import ClpConfig
+
 
 @dataclass(frozen=True)
 class ClpCorePathConfig:
@@ -61,8 +68,15 @@ class PackagePathConfig:
     #: Root directory containing all CLP package contents.
     clp_package_dir: Path
 
-    def __post_init__(self) -> None:
-        """Validates that the CLP package directory exists and contains all required directories."""
+    #: Root directory for package tests output.
+    test_root_dir: InitVar[Path]
+
+    #: Directory to store any cached package config files.
+    temp_config_dir: Path = field(init=False, repr=True)
+
+    def __post_init__(self, test_root_dir: Path) -> None:
+        """Validates init values and initializes attributes."""
+        # Validate that the CLP package directory exists and contains required directories.
         clp_package_dir = self.clp_package_dir
         validate_dir_exists(clp_package_dir)
 
@@ -75,6 +89,70 @@ def __post_init__(self) -> None:
             )
             raise RuntimeError(err_msg)
 
+        # Initialize cache directory for package tests.
+        validate_dir_exists(test_root_dir)
+        object.__setattr__(self, "temp_config_dir", test_root_dir / "temp_config_files")
+
+        # Create directories if they do not already exist.
+        self.temp_config_dir.mkdir(parents=True, exist_ok=True)
+
+    @property
+    def start_script_path(self) -> Path:
+        """:return: The absolute path to the package start script."""
+        return self.clp_package_dir / "sbin" / "start-clp.sh"
+
+    @property
+    def stop_script_path(self) -> Path:
+        """:return: The absolute path to the package stop script."""
+        return self.clp_package_dir / "sbin" / "stop-clp.sh"
+
+
+@dataclass(frozen=True)
+class PackageConfig:
+    """Metadata for a specific configuration of the CLP package."""
+
+    #: Path configuration for this package.
+    path_config: PackagePathConfig
+
+    #: Name of the mode of operation represented in this config.
+    mode_name: str
+
+    #: The ClpConfig instance that describes this package configuration.
+    clp_config: ClpConfig
+
+    def __post_init__(self) -> None:
+        """Write the temporary config file for this package."""
+        self._write_temp_config_file()
+
+    @property
+    def temp_config_file_path(self) -> Path:
+        """:return: The absolute path to the temporary configuration file for the package."""
+        return self.path_config.temp_config_dir / f"clp-config-{self.mode_name}.yaml"
+
+    def _write_temp_config_file(self) -> None:
+        """Writes the temporary config file for this package."""
+        temp_config_file_path = self.temp_config_file_path
+
+        payload = self.clp_config.dump_to_primitive_dict()  # type: ignore[no-untyped-call]
+
+        tmp_path = temp_config_file_path.with_suffix(temp_config_file_path.suffix + ".tmp")
+        with tmp_path.open("w", encoding="utf-8") as f:
+            yaml.safe_dump(payload, f, sort_keys=False)
+        tmp_path.replace(temp_config_file_path)
+
+
+@dataclass(frozen=True)
+class PackageInstance:
+    """Metadata for a running instance of the CLP package."""
+
+    #: Config describing this package instance.
+    package_config: PackageConfig
+
+    def __post_init__(self) -> None:
+        """Validates init values and initializes attributes."""
+        # Validate that the temp config file exists.
+        validate_file_exists(self.package_config.temp_config_file_path)
+
 
 @dataclass(frozen=True)
 class IntegrationTestPathConfig:

integration-tests/tests/utils/package_utils.py

@@ -0,0 +1,53 @@
+"""Provides utility functions related to the CLP package used across `integration-tests`."""
+
+import subprocess
+
+from tests.utils.config import (
+    PackageConfig,
+    PackageInstance,
+)
+
+
+def start_clp_package(package_config: PackageConfig) -> None:
+    """
+    Starts an instance of the CLP package.
+
+    :param package_config:
+    :raise RuntimeError: If the package fails to start.
+    """
+    path_config = package_config.path_config
+    start_script_path = path_config.start_script_path
+    temp_config_file_path = package_config.temp_config_file_path
+    try:
+        # fmt: off
+        start_cmd = [
+            str(start_script_path),
+            "--config", str(temp_config_file_path),
+        ]
+        # fmt: on
+        subprocess.run(start_cmd, check=True)
+    except Exception as err:
+        err_msg = f"Failed to start an instance of the {package_config.mode_name} package."
+        raise RuntimeError(err_msg) from err
+
+
+def stop_clp_package(instance: PackageInstance) -> None:
+    """
+    Stops an instance of the CLP package.
+
+    :param instance:
+    :raise RuntimeError: If the package fails to stop.
+    """
+    package_config = instance.package_config
+    path_config = package_config.path_config
+    stop_script_path = path_config.stop_script_path
+    try:
+        # fmt: off
+        stop_cmd = [
+            str(stop_script_path)
+        ]
+        # fmt: on
+        subprocess.run(stop_cmd, check=True)
+    except Exception as err:
+        err_msg = f"Failed to stop an instance of the {package_config.mode_name} package."
+        raise RuntimeError(err_msg) from err

integration-tests/tests/utils/utils.py

@@ -96,6 +96,16 @@ def validate_dir_exists(dir_path: Path) -> None:
         raise ValueError(err_msg)
 
 
+def validate_file_exists(file_path: Path) -> None:
+    """
+    :param file_path:
+    :raise ValueError: if the path does not exist or is not a file.
+    """
+    if not file_path.is_file():
+        err_msg = f"Path does not exist or is not a file: {file_path}"
+        raise ValueError(err_msg)
+
+
 def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]:
     """
     Normalize a JSON file to a stable, deterministically ordered form for comparison.