Merge branch 'main' into feature branch

Author: quinntaylormitchell

integration-tests/tests/conftest.py

@@ -4,8 +4,8 @@
 
 # Make the fixtures defined in `tests/fixtures/` globally available without imports.
 pytest_plugins = [
-    "tests.fixtures.integration_test_config",
     "tests.fixtures.integration_test_logs",
+    "tests.fixtures.path_configs",
     "tests.fixtures.package_instance",
     "tests.fixtures.package_config",
 ]

integration-tests/tests/fixtures/integration_test_config.py

@@ -1,4 +1,4 @@
-"""Define test logs fixtures."""
+"""Session-scoped test log fixtures shared across integration tests."""
 
 import logging
 import shutil
@@ -7,8 +7,8 @@
 import pytest
 
 from tests.utils.config import (
-    IntegrationTestConfig,
     IntegrationTestLogs,
+    IntegrationTestPathConfig,
 )
 from tests.utils.utils import unlink
 
@@ -18,12 +18,12 @@
 @pytest.fixture(scope="session")
 def hive_24hr(
     request: pytest.FixtureRequest,
-    integration_test_config: IntegrationTestConfig,
+    integration_test_path_config: IntegrationTestPathConfig,
 ) -> IntegrationTestLogs:
-    """Fixture that provides `hive_24hr` test logs shared across tests."""
+    """Provides shared `hive_24hr` test logs."""
     return _download_and_extract_dataset(
         request=request,
-        integration_test_config=integration_test_config,
+        integration_test_path_config=integration_test_path_config,
         name="hive-24hr",
         tarball_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1",
     )
@@ -32,27 +32,27 @@ def hive_24hr(
 @pytest.fixture(scope="session")
 def postgresql(
     request: pytest.FixtureRequest,
-    integration_test_config: IntegrationTestConfig,
+    integration_test_path_config: IntegrationTestPathConfig,
 ) -> IntegrationTestLogs:
-    """Fixture that provides `postgresql` test logs shared across tests."""
+    """Provides shared `postgresql` test logs."""
     return _download_and_extract_dataset(
         request=request,
-        integration_test_config=integration_test_config,
+        integration_test_path_config=integration_test_path_config,
         name="postgresql",
         tarball_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1",
     )
 
 
 def _download_and_extract_dataset(
     request: pytest.FixtureRequest,
-    integration_test_config: IntegrationTestConfig,
+    integration_test_path_config: IntegrationTestPathConfig,
     name: str,
     tarball_url: str,
 ) -> IntegrationTestLogs:
     integration_test_logs = IntegrationTestLogs(
         name=name,
         tarball_url=tarball_url,
-        integration_test_config=integration_test_config,
+        integration_test_path_config=integration_test_path_config,
     )
     if request.config.cache.get(name, False):
         logger.info("Test logs `%s` are up-to-date. Skipping download.", name)

integration-tests/tests/fixtures/integration_test_logs.py

@@ -0,0 +1,30 @@
+"""Session-scoped path configuration fixtures shared across integration tests."""
+
+import pytest
+
+from tests.utils.config import (
+    ClpCorePathConfig,
+    IntegrationTestPathConfig,
+    PackagePathConfig,
+)
+from tests.utils.utils import resolve_path_env_var
+
+
+@pytest.fixture(scope="session")
+def clp_core_path_config() -> ClpCorePathConfig:
+    """Provides paths for the CLP core binaries."""
+    return ClpCorePathConfig(clp_core_bins_dir=resolve_path_env_var("CLP_CORE_BINS_DIR"))
+
+
+@pytest.fixture(scope="session")
+def integration_test_path_config() -> IntegrationTestPathConfig:
+    """Provides paths for the integration-test directory and its contents."""
+    return IntegrationTestPathConfig(
+        test_root_dir=resolve_path_env_var("CLP_BUILD_DIR") / "integration-tests"
+    )
+
+
+@pytest.fixture(scope="session")
+def package_path_config() -> PackagePathConfig:
+    """Provides paths for the clp-package directory and its contents."""
+    return PackagePathConfig(clp_package_dir=resolve_path_env_var("CLP_PACKAGE_DIR"))

integration-tests/tests/fixtures/path_configs.py

@@ -7,10 +7,10 @@
 
 from tests.utils.asserting_utils import run_and_assert
 from tests.utils.config import (
-    CompressionTestConfig,
-    CoreConfig,
-    IntegrationTestConfig,
+    ClpCorePathConfig,
+    CompressionTestPathConfig,
     IntegrationTestLogs,
+    IntegrationTestPathConfig,
 )
 from tests.utils.utils import (
     is_dir_tree_content_equal,
@@ -38,28 +38,28 @@
 @text_datasets
 def test_clp_identity_transform(
     request: pytest.FixtureRequest,
-    core_config: CoreConfig,
-    integration_test_config: IntegrationTestConfig,
+    clp_core_path_config: ClpCorePathConfig,
+    integration_test_path_config: IntegrationTestPathConfig,
     test_logs_fixture: str,
 ) -> None:
     """
     Validate that compression and decompression by the core binary `clp` run successfully and are
     lossless.
 
     :param request:
-    :param core_config:
-    :param integration_test_config:
+    :param clp_core_path_config:
+    :param integration_test_path_config:
     :param test_logs_fixture:
     """
     integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture)
-    test_paths = CompressionTestConfig(
+    test_paths = CompressionTestPathConfig(
         test_name=f"clp-{integration_test_logs.name}",
         logs_source_dir=integration_test_logs.extraction_dir,
-        integration_test_config=integration_test_config,
+        integration_test_path_config=integration_test_path_config,
     )
     test_paths.clear_test_outputs()
 
-    bin_path = str(core_config.clp_binary_path)
+    bin_path = str(clp_core_path_config.clp_binary_path)
     src_path = str(test_paths.logs_source_dir)
     compression_path = str(test_paths.compression_dir)
     decompression_path = str(test_paths.decompression_dir)
@@ -92,41 +92,41 @@ def test_clp_identity_transform(
 @json_datasets
 def test_clp_s_identity_transform(
     request: pytest.FixtureRequest,
-    core_config: CoreConfig,
-    integration_test_config: IntegrationTestConfig,
+    clp_core_path_config: ClpCorePathConfig,
+    integration_test_path_config: IntegrationTestPathConfig,
     test_logs_fixture: str,
 ) -> None:
     """
     Validate that compression and decompression by the core binary `clp-s` run successfully and are
     lossless.
 
     :param request:
-    :param core_config:
-    :param integration_test_config:
+    :param clp_core_path_config:
+    :param integration_test_path_config:
     :param test_logs_fixture:
     """
     integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture)
     test_logs_name = integration_test_logs.name
 
-    test_paths = CompressionTestConfig(
+    test_paths = CompressionTestPathConfig(
         test_name=f"clp-s-{test_logs_name}",
         logs_source_dir=integration_test_logs.extraction_dir,
-        integration_test_config=integration_test_config,
+        integration_test_path_config=integration_test_path_config,
     )
-    _clp_s_compress_and_decompress(core_config, test_paths)
+    _clp_s_compress_and_decompress(clp_core_path_config, test_paths)
 
     # Recompress the decompressed output that's consolidated into a single json file, and decompress
     # it again to verify consistency. The compression input of the second iteration points to the
     # decompression output of the first.
     # TODO: Remove this check once we can directly compare decompressed logs (which would preserve
     #       the directory structure and row/key order) with the original downloaded logs.
     # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations
-    consolidated_json_test_paths = CompressionTestConfig(
+    consolidated_json_test_paths = CompressionTestPathConfig(
         test_name=f"clp-s-{test_logs_name}-consolidated-json",
         logs_source_dir=test_paths.decompression_dir,
-        integration_test_config=integration_test_config,
+        integration_test_path_config=integration_test_path_config,
     )
-    _clp_s_compress_and_decompress(core_config, consolidated_json_test_paths)
+    _clp_s_compress_and_decompress(clp_core_path_config, consolidated_json_test_paths)
 
     _consolidated_json_file_name = "original"
     input_path = consolidated_json_test_paths.logs_source_dir / _consolidated_json_file_name
@@ -140,10 +140,11 @@ def test_clp_s_identity_transform(
 
 
 def _clp_s_compress_and_decompress(
-    core_config: CoreConfig, test_paths: CompressionTestConfig
+    clp_core_path_config: ClpCorePathConfig,
+    test_paths: CompressionTestPathConfig,
 ) -> None:
     test_paths.clear_test_outputs()
-    bin_path = str(core_config.clp_s_binary_path)
+    bin_path = str(clp_core_path_config.clp_s_binary_path)
     src_path = str(test_paths.logs_source_dir)
     compression_path = str(test_paths.compression_dir)
     decompression_path = str(test_paths.decompression_dir)

integration-tests/tests/test_identity_transformation.py

@@ -21,10 +21,10 @@
 
 
 @dataclass(frozen=True)
-class CoreConfig:
-    """The configuration for the clp core binaries subject to testing."""
+class ClpCorePathConfig:
+    """Path configuration for the CLP core binaries."""
 
-    #:
+    #: Root directory containing all CLP core binaries.
     clp_core_bins_dir: Path
 
     def __post_init__(self) -> None:
@@ -35,7 +35,6 @@ def __post_init__(self) -> None:
         clp_core_bins_dir = self.clp_core_bins_dir
         validate_dir_exists(clp_core_bins_dir)
 
-        # Check for required CLP core binaries
         required_binaries = [
             "clg",
             "clo",
@@ -51,7 +50,7 @@ def __post_init__(self) -> None:
                 f"CLP core binaries at {clp_core_bins_dir} are incomplete."
                 f" Missing binaries: {', '.join(missing_binaries)}"
             )
-            raise ValueError(err_msg)
+            raise RuntimeError(err_msg)
 
     @property
     def clp_binary_path(self) -> Path:
@@ -86,15 +85,14 @@ def __post_init__(self, test_root_dir: Path) -> None:
         clp_package_dir = self.clp_package_dir
         validate_dir_exists(clp_package_dir)
 
-        # Check for required package script directories
         required_dirs = ["etc", "sbin"]
         missing_dirs = [d for d in required_dirs if not (clp_package_dir / d).is_dir()]
         if len(missing_dirs) > 0:
             err_msg = (
                 f"CLP package at {clp_package_dir} is incomplete."
                 f" Missing directories: {', '.join(missing_dirs)}"
             )
-            raise ValueError(err_msg)
+            raise RuntimeError(err_msg)
 
         # Initialize cache directory for package tests.
         validate_dir_exists(test_root_dir)
@@ -218,15 +216,18 @@ def _get_clp_instance_id(clp_instance_id_file_path: Path) -> str:
 
 
 @dataclass(frozen=True)
-class IntegrationTestConfig:
-    """The general configuration for integration tests."""
+class IntegrationTestPathConfig:
+    """Path configuration for CLP integration tests."""
 
-    #: Root directory for integration tests output.
+    #: Default directory for integration test output.
     test_root_dir: Path
-    logs_download_dir_init: InitVar[Path | None] = None
+
     #: Directory to store the downloaded logs.
     logs_download_dir: Path = field(init=False, repr=True)
 
+    #: Optional initialization value used to set `logs_download_dir`.
+    logs_download_dir_init: InitVar[Path | None] = None
+
     def __post_init__(self, logs_download_dir_init: Path | None) -> None:
         """Initialize and create required directories for integration tests."""
         if logs_download_dir_init is not None:
@@ -246,19 +247,19 @@ class IntegrationTestLogs:
     name: str
     #:
     tarball_url: str
-    integration_test_config: InitVar[IntegrationTestConfig]
+    integration_test_path_config: InitVar[IntegrationTestPathConfig]
     #:
     tarball_path: Path = field(init=False, repr=True)
     #:
     extraction_dir: Path = field(init=False, repr=True)
 
-    def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None:
+    def __post_init__(self, integration_test_path_config: IntegrationTestPathConfig) -> None:
         """Initialize and set tarball and extraction paths for integration test logs."""
         name = self.name.strip()
         if 0 == len(name):
             err_msg = "`name` cannot be empty."
             raise ValueError(err_msg)
-        logs_download_dir = integration_test_config.logs_download_dir
+        logs_download_dir = integration_test_path_config.logs_download_dir
         validate_dir_exists(logs_download_dir)
 
         object.__setattr__(self, "name", name)
@@ -267,26 +268,26 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None:
 
 
 @dataclass(frozen=True)
-class CompressionTestConfig:
-    """Compression test configuration providing per-test metadata for artifacts and directories."""
+class CompressionTestPathConfig:
+    """Per-test path configuration for compression workflow artifacts."""
 
     #:
     test_name: str
     #: Directory containing the original (uncompressed) log files used by this test.
     logs_source_dir: Path
-    integration_test_config: InitVar[IntegrationTestConfig]
+    integration_test_path_config: InitVar[IntegrationTestPathConfig]
     #: Path to store compressed archives generated by the test.
     compression_dir: Path = field(init=False, repr=True)
     #: Path to store decompressed logs generated by the test.
     decompression_dir: Path = field(init=False, repr=True)
 
-    def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None:
+    def __post_init__(self, integration_test_path_config: IntegrationTestPathConfig) -> None:
         """Initialize and set required directory paths for compression tests."""
         test_name = self.test_name.strip()
         if 0 == len(test_name):
             err_msg = "`test_name` cannot be empty."
             raise ValueError(err_msg)
-        test_root_dir = integration_test_config.test_root_dir
+        test_root_dir = integration_test_path_config.test_root_dir
         validate_dir_exists(test_root_dir)
 
         object.__setattr__(self, "test_name", test_name)