Add validation + compatibility tests for DeepLabCut-live

.github/workflows/testing-ci.yml

@@ -78,3 +78,40 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./.coverage.py312.xml
           fail_ci_if_error: false
+
+  dlclive-compat:
+    name: DLCLive Compatibility • ${{ matrix.label }} • py${{ matrix.python }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.11']
+        include:
+          - label: pypi-1.1
+            dlclive_spec: deeplabcut-live==1.1
+          - label: github-main
+            dlclive_spec: git+https://github.com/DeepLabCut/DeepLabCut-live.git@main
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python }}
+          cache: 'pip'
+
+      - name: Install package + test dependencies
+        run: |
+          python -m pip install -U pip wheel
+          python -m pip install -e .[test]
+
+      - name: Install matrix DLCLive build
+        run: |
+          python -m pip install --upgrade --force-reinstall "${{ matrix.dlclive_spec }}"
+          python -m pip show deeplabcut-live
+
+      - name: Run DLCLive compatibility tests
+        run: |
+          python -m pytest -m dlclive_compat tests/compat/test_dlclive_package_compat.py -q

dlclivegui/services/dlc_processor.py

@@ -15,7 +15,7 @@
 import numpy as np
 from PySide6.QtCore import QObject, Signal
 
-from dlclivegui.config import DLCProcessorSettings
+from dlclivegui.config import DLCProcessorSettings, ModelType
 from dlclivegui.processors.processor_utils import instantiate_from_scan
 from dlclivegui.temp import Engine  # type: ignore # TODO use main package enum when released
 
@@ -37,6 +37,63 @@
 class PoseResult:
     pose: np.ndarray | None
     timestamp: float
+    packet: "PosePacket | None" = None
+
+
+@dataclass(slots=True, frozen=True)
+class PoseSource:
+    backend: str    # e.g. "DLCLive"
+    model_type: ModelType | None = None
+
+
+@dataclass(slots=True, frozen=True)
+class PosePacket:
+    schema_version: int = 0
+    keypoints: np.ndarray | None = None
+    keypoint_names: list[str] | None = None
+    individual_ids: list[str] | None = None
+    source: PoseSource = PoseSource(backend="DLCLive")
+    raw: Any | None = None
+
+
+def validate_pose_array(pose: Any, *, source_backend: str = "DLCLive") -> np.ndarray:
+    """
+    Validate pose output shape and dtype.
+
+    Accepted runner output shapes:
+    - (K, 3): single-animal
+    - (N, K, 3): multi-animal
+    """
+    try:
+        arr = np.asarray(pose)
+    except Exception as exc:
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format: could not convert to array ({exc})"
+        ) from exc
+
+    if arr.ndim not in (2, 3):
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format: expected a 2D or 3D array, got ndim={arr.ndim}, shape={arr.shape!r}"
+        )
+
+    if arr.shape[-1] != 3:
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format: expected last dimension size 3 (x, y, likelihood), got shape={arr.shape!r}"
+        )
+
+    if arr.ndim == 2 and arr.shape[0] <= 0:
+        raise ValueError(f"{source_backend} returned an invalid pose output format: expected at least one keypoint")
+    if arr.ndim == 3 and (arr.shape[0] <= 0 or arr.shape[1] <= 0):
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format: expected at least one individual and one keypoint, got shape={arr.shape!r}"
+        )
+
+    if not np.issubdtype(arr.dtype, np.number):
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format: expected numeric values, got dtype={arr.dtype}"
+        )
+
+    return arr
 
 
 @dataclass
@@ -269,8 +326,17 @@
         # Time GPU inference (and processor overhead when present)
         with self._timed_processor() as proc_holder:
             inference_start = time.perf_counter()
-            pose = self._dlc.get_pose(frame, frame_time=timestamp)
+            raw_pose: Any = self._dlc.get_pose(frame, frame_time=timestamp)
             inference_time = time.perf_counter() - inference_start
+        pose_arr: np.ndarray = validate_pose_array(raw_pose, source_backend="DLCLive")
+        pose_packet = PosePacket(
+            schema_version=0,
+            keypoints=pose_arr,
+            keypoint_names=None,
+            individual_ids=None,
+            source=PoseSource(backend="DLCLive", model_type=self._settings.model_type),
+            raw=raw_pose,
+        )
 
         processor_overhead = 0.0
         gpu_inference_time = inference_time
@@ -280,7 +346,7 @@
 
         # Emit pose (measure signal overhead)
         signal_start = time.perf_counter()
-        self.pose_ready.emit(PoseResult(pose=pose, timestamp=timestamp))
+        self.pose_ready.emit(PoseResult(pose=pose_packet.keypoints, timestamp=timestamp, packet=pose_packet))
         signal_time = time.perf_counter() - signal_start
 
         end_ts = time.perf_counter()

pyproject.toml

@@ -115,6 +115,7 @@ markers = [
   "unit: Unit tests for individual components",
   "integration: Integration tests for component interaction",
   "functional: Functional tests for end-to-end workflows",
+  "dlclive_compat: Package/API compatibility tests against supported dlclive versions",
   "hardware: Tests that require specific hardware, notable camera backends",
   # "slow: Tests that take a long time to run",
   "gui: Tests that require GUI interaction",

tests/compat/test_dlclive_package_compat.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import importlib.metadata
+import inspect
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+
+def _get_signature_params(callable_obj) -> tuple[set[str], bool]:
+    """
+    Return allowed keyword names for callable, allowing for **kwargs.
+
+    Example:
+    >>> params, accepts_var_kw = _get_signature_params(lambda x, y, **kwargs: None, {"x", "y"})
+    >>> params == {"x", "y"}
+    True
+    >>> accepts_var_kw
+    True
+    """
+    sig = inspect.signature(callable_obj)
+    params = sig.parameters
+    accepts_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
+    return params, accepts_var_kw
+
+
+@pytest.mark.dlclive_compat
+def test_dlclive_package_is_importable():
+    from dlclive import DLCLive  # noqa: PLC0415
+
+    assert DLCLive is not None
+    # Helpful for CI logs to confirm matrix install result.
+    _ = importlib.metadata.version("deeplabcut-live")
+
+
+@pytest.mark.dlclive_compat
+def test_dlclive_constructor_accepts_gui_expected_kwargs():
+    """
+    GUI passes these kwargs when constructing DLCLive.
+    This test catches upstream API changes that would break initialization.
+    """
+    from dlclive import DLCLive  # noqa: PLC0415
+
+    expected = {
+        "model_path",
+        "model_type",
+        "processor",
+        "dynamic",
+        "resize",
+        "precision",
+        "single_animal",
+        "device",
+    }
+    params, accepts_var_kw = _get_signature_params(DLCLive.__init__)
+    missing = {name for name in expected if name not in params}
+    assert not missing, f"DLCLive.__init__ is missing expected kwargs called by GUI: {sorted(missing)}"
+    assert accepts_var_kw, "DLCLive.__init__ should accept **kwargs" # captures current behavior
+
+
+@pytest.mark.dlclive_compat
+def test_dlclive_methods_match_gui_usage():
+    """
+    GUI expects:
+    - init_inference(frame)
+    - get_pose(frame, frame_time=<float>)
+    """
+    from dlclive import DLCLive  # noqa: PLC0415
+
+    assert hasattr(DLCLive, "init_inference"), "DLCLive must provide init_inference(frame)"
+    assert hasattr(DLCLive, "get_pose"), "DLCLive must provide get_pose(frame, frame_time=...)"
+
+    init_params, _ = _get_signature_params(DLCLive.init_inference)
+    init_missing = {name for name in {"frame"} if name not in init_params}
+    assert not init_missing, f"DLCLive.init_inference signature mismatch, missing: {sorted(init_missing)}"
+
+    get_pose_params, _ = _get_signature_params(DLCLive.get_pose)
+    get_pose_missing = {name for name in {"frame", "frame_time"} if name not in get_pose_params}
+    assert not get_pose_missing, f"DLCLive.get_pose signature mismatch, missing: {sorted(get_pose_missing)}"
+
+
+@pytest.mark.dlclive_compat
+def test_dlclive_minimal_inference_smoke():
+    """
+    Real runtime smoke test (init + pose call) using a tiny exported model.
+
+    Opt-in via env vars:
+    - DLCLIVE_TEST_MODEL_PATH: absolute/relative path to exported model folder/file
+    - DLCLIVE_TEST_MODEL_TYPE: optional model type (default: pytorch)
+    """
+    model_path_env = os.getenv("DLCLIVE_TEST_MODEL_PATH", "").strip()
+    if not model_path_env:
+        pytest.skip("Set DLCLIVE_TEST_MODEL_PATH to run real DLCLive inference smoke test.")
+
+    model_path = Path(model_path_env).expanduser()
+    if not model_path.exists():
+        pytest.skip(f"DLCLIVE_TEST_MODEL_PATH does not exist: {model_path}")
+
+    model_type = os.getenv("DLCLIVE_TEST_MODEL_TYPE", "pytorch").strip() or "pytorch"
+
+    from dlclive import DLCLive  # noqa: PLC0415
+    from dlclivegui.services.dlc_processor import validate_pose_array  # noqa: PLC0415
+
+    dlc = DLCLive(
+        model_path=str(model_path),
+        model_type=model_type,
+        dynamic=[False, 0.5, 10],
+        resize=1.0,
+        precision="FP32",
+        single_animal=True,
+    )
+
+    frame = np.zeros((64, 64, 3), dtype=np.uint8)
+    dlc.init_inference(frame)
+    pose = dlc.get_pose(frame, frame_time=0.0)
+    pose_arr = validate_pose_array(pose, source_backend="DLCLive.get_pose")
+
+    assert pose_arr.ndim in (2, 3)
+    assert pose_arr.shape[-1] == 3
+    assert np.isfinite(pose_arr).all()

tests/services/test_pose_contract.py

@@ -0,0 +1,40 @@
+import numpy as np
+import pytest
+
+from dlclivegui.services.dlc_processor import validate_pose_array
+
+
+@pytest.mark.unit
+def test_validate_pose_array_keeps_single_animal_shape():
+    pose = np.ones((5, 3), dtype=np.float64)
+    out = validate_pose_array(pose)
+    assert out.shape == (5, 3)
+    assert out.dtype == np.float64
+
+
+@pytest.mark.unit
+def test_validate_pose_array_accepts_multi_animal():
+    pose = np.ones((2, 5, 3), dtype=np.float32)
+    out = validate_pose_array(pose)
+    assert out.shape == (2, 5, 3)
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    "bad_pose,expected",
+    [
+        (np.ones((5, 2), dtype=np.float32), "last dimension size 3"),
+        (np.ones((2, 5, 4), dtype=np.float32), "last dimension size 3"),
+        (np.ones((3,), dtype=np.float32), "expected a 2D or 3D array"),
+    ],
+)
+def test_validate_pose_array_rejects_invalid_shapes(bad_pose, expected):
+    with pytest.raises(ValueError, match=expected):
+        validate_pose_array(bad_pose)
+
+
+@pytest.mark.unit
+def test_validate_pose_array_rejects_non_numeric():
+    pose = np.array([[["x", "y", "p"]]], dtype=object)
+    with pytest.raises(ValueError, match="expected numeric values"):
+        validate_pose_array(pose)

tox.ini

@@ -11,7 +11,6 @@ description = Unit + smoke tests (exclude hardware) with coverage
 package = wheel
 extras = test
 
-
 # Helpful defaults for headless CI runs (Qt/OpenCV):
 setenv =
     PYTHONWARNINGS = default
@@ -21,9 +20,8 @@ setenv =
     OPENCV_VIDEOIO_PRIORITY_MSMF = 0
     COVERAGE_FILE = {toxinidir}/.coverage.{envname}
 
-# Keep behavior aligned with your GitHub Actions job:
 commands =
-    pytest -m "not hardware" --maxfail=1 --disable-warnings \
+    pytest -m "not hardware and not dlclive_compat" --maxfail=1 --disable-warnings \
         --cov={envsitepackagesdir}/dlclivegui \
         --cov-report=xml:{toxinidir}/.coverage.{envname}.xml \
         --cov-report=term-missing \