diff --git a/src/services/ray/README.md b/src/services/ray/README.md new file mode 100644 index 00000000..ff008254 --- /dev/null +++ b/src/services/ray/README.md @@ -0,0 +1,6 @@ +#running pytest +cd ndif_root + +pytest -q src/services/ray/tests/unit/deployments/controller/cluster/test_deployment.py + +pytest -q src/services/ray/tests/unit/deployments/controller/cluster/test_deployment.py::test_cache_logs_and_returns_none_on_failure \ No newline at end of file diff --git a/src/services/ray/pyproject.toml b/src/services/ray/pyproject.toml new file mode 100644 index 00000000..5a80e3b6 --- /dev/null +++ b/src/services/ray/pyproject.toml @@ -0,0 +1,47 @@ +# +[build-system] +requires = ["setuptools>=68", "wheel"] + +[project] +name = "ndif-ray-tests" +version = "0.0.1" + +[tool.coverage.run] +source = [ + "src/services/ray/src/ndif_ray/deployments", + "src/services/ray/src/ndif_ray/distributed", + "src/services/ray/src/ndif_ray/nn" +] +branch = true + +[tool.pytest.ini_options] +minversion = "8.2" +#addopts = "-ra -q" +addopts = "-ra -q --ignore=src/services/api --ignore=src/services/queue --ignore=src/services/base --ignore=src/common --ignore=src/ndif" + +# ✅ pytest will look only here for test files +testpaths = ["tests"] + +# ✅ pytest will treat only your internal Ray tree as the import root +pythonpath = ["src/services/ray/src"] + +# ✅ Ignore every other NDIF directory so tests there are never even collected +norecursedirs = [ + "src/common", + "src/ndif", + "src/services/api", + "src/services/base", + "src/services/queue", + "src/services/*/tests", + "node_modules", + ".venv", + "__pycache__" +] + +markers = [ + "raymock: uses mocked external Ray APIs (fast)", + "integration: requires real Ray runtime (slow)" +] +filterwarnings = [ + "ignore::DeprecationWarning" +] diff --git a/src/services/ray/src/ray/__init__.py b/src/services/ray/src/ndif_ray/__init__.py similarity index 100% rename from src/services/ray/src/ray/__init__.py rename to src/services/ray/src/ndif_ray/__init__.py diff --git a/src/services/ray/src/ray/config/ray_config.yml b/src/services/ray/src/ndif_ray/config/ray_config.yml similarity index 100% rename from src/services/ray/src/ray/config/ray_config.yml rename to src/services/ray/src/ndif_ray/config/ray_config.yml diff --git a/src/services/ray/src/ray/deployments/__init__.py b/src/services/ray/src/ndif_ray/deployments/__init__.py similarity index 100% rename from src/services/ray/src/ray/deployments/__init__.py rename to src/services/ray/src/ndif_ray/deployments/__init__.py diff --git a/src/services/ray/src/ray/deployments/controller/__init__.py b/src/services/ray/src/ndif_ray/deployments/controller/__init__.py similarity index 100% rename from src/services/ray/src/ray/deployments/controller/__init__.py rename to src/services/ray/src/ndif_ray/deployments/controller/__init__.py diff --git a/src/services/ray/src/ray/deployments/controller/cluster/__init__.py b/src/services/ray/src/ndif_ray/deployments/controller/cluster/__init__.py similarity index 100% rename from src/services/ray/src/ray/deployments/controller/cluster/__init__.py rename to src/services/ray/src/ndif_ray/deployments/controller/cluster/__init__.py diff --git a/src/services/ray/src/ray/deployments/controller/cluster/cluster.py b/src/services/ray/src/ndif_ray/deployments/controller/cluster/cluster.py similarity index 99% rename from src/services/ray/src/ray/deployments/controller/cluster/cluster.py rename to src/services/ray/src/ndif_ray/deployments/controller/cluster/cluster.py index 63e1ced2..d2c860c6 100755 --- a/src/services/ray/src/ray/deployments/controller/cluster/cluster.py +++ b/src/services/ray/src/ndif_ray/deployments/controller/cluster/cluster.py @@ -9,7 +9,8 @@ from ray._raylet import GcsClientOptions from ray.util.state import list_nodes -from .....types import MODEL_KEY +#from .....types import MODEL_KEY +from ndif_shared.types import MODEL_KEY from .evaluator import ModelEvaluator from .node import CandidateLevel, Node, Resources diff --git a/src/services/ray/src/ray/deployments/controller/cluster/deployment.py b/src/services/ray/src/ndif_ray/deployments/controller/cluster/deployment.py similarity index 96% rename from src/services/ray/src/ray/deployments/controller/cluster/deployment.py rename to src/services/ray/src/ndif_ray/deployments/controller/cluster/deployment.py index df168afb..080407f5 100755 --- a/src/services/ray/src/ray/deployments/controller/cluster/deployment.py +++ b/src/services/ray/src/ndif_ray/deployments/controller/cluster/deployment.py @@ -5,8 +5,8 @@ from typing import Any, Dict import ray -from .....types import MODEL_KEY - +#from .....types import MODEL_KEY +from ndif_shared.types import MODEL_KEY logger = logging.getLogger("ndif") class DeploymentLevel(Enum): diff --git a/src/services/ray/src/ray/deployments/controller/cluster/evaluator.py b/src/services/ray/src/ndif_ray/deployments/controller/cluster/evaluator.py similarity index 97% rename from src/services/ray/src/ray/deployments/controller/cluster/evaluator.py rename to src/services/ray/src/ndif_ray/deployments/controller/cluster/evaluator.py index 00ea94f4..e6283258 100755 --- a/src/services/ray/src/ray/deployments/controller/cluster/evaluator.py +++ b/src/services/ray/src/ndif_ray/deployments/controller/cluster/evaluator.py @@ -5,7 +5,8 @@ from nnsight.modeling.mixins import RemoteableMixin -from .....types import MODEL_KEY +#from .....types import MODEL_KEY +from ndif_shared.types import MODEL_KEY logger = logging.getLogger("ndif") diff --git a/src/services/ray/src/ray/deployments/controller/cluster/node.py b/src/services/ray/src/ndif_ray/deployments/controller/cluster/node.py similarity index 98% rename from src/services/ray/src/ray/deployments/controller/cluster/node.py rename to src/services/ray/src/ndif_ray/deployments/controller/cluster/node.py index aea01f9d..079256ae 100755 --- a/src/services/ray/src/ray/deployments/controller/cluster/node.py +++ b/src/services/ray/src/ndif_ray/deployments/controller/cluster/node.py @@ -6,7 +6,8 @@ import ray -from .....types import MODEL_KEY, NODE_ID +#from .....types import MODEL_KEY, NODE_ID +from ndif_shared.types import MODEL_KEY, NODE_ID from .deployment import Deployment, DeploymentLevel logger = logging.getLogger("ndif") diff --git a/src/services/ray/src/ray/deployments/controller/controller.py b/src/services/ray/src/ndif_ray/deployments/controller/controller.py similarity index 98% rename from src/services/ray/src/ray/deployments/controller/controller.py rename to src/services/ray/src/ndif_ray/deployments/controller/controller.py index e3c947cd..4b19ab33 100755 --- a/src/services/ray/src/ray/deployments/controller/controller.py +++ b/src/services/ray/src/ndif_ray/deployments/controller/controller.py @@ -15,7 +15,8 @@ ServeDeploySchema, ServeInstanceDetails, ) -from ....types import MODEL_KEY, RAY_APP_NAME, NODE_ID +#from ....types import MODEL_KEY, RAY_APP_NAME, NODE_ID +from ndif_shared.types import MODEL_KEY, RAY_APP_NAME, NODE_ID from ....logging.logger import set_logger from ....providers.objectstore import ObjectStoreProvider from ....providers.socketio import SioProvider diff --git a/src/services/ray/src/ray/deployments/controller/gcal/__init__.py b/src/services/ray/src/ndif_ray/deployments/controller/gcal/__init__.py similarity index 100% rename from src/services/ray/src/ray/deployments/controller/gcal/__init__.py rename to src/services/ray/src/ndif_ray/deployments/controller/gcal/__init__.py diff --git a/src/services/ray/src/ray/deployments/controller/gcal/controller.py b/src/services/ray/src/ndif_ray/deployments/controller/gcal/controller.py similarity index 96% rename from src/services/ray/src/ray/deployments/controller/gcal/controller.py rename to src/services/ray/src/ndif_ray/deployments/controller/gcal/controller.py index a3f43c14..a1149b7d 100755 --- a/src/services/ray/src/ray/deployments/controller/gcal/controller.py +++ b/src/services/ray/src/ndif_ray/deployments/controller/gcal/controller.py @@ -2,7 +2,8 @@ import time from ray import ray, serve -from .....types import MODEL_KEY, RAY_APP_NAME +#from .....types import MODEL_KEY, RAY_APP_NAME +from ndif_shared.types import MODEL_KEY, RAY_APP_NAME from ..controller import ControllerDeploymentArgs, _ControllerDeployment from .scheduler import SchedulingActor diff --git a/src/services/ray/src/ray/deployments/controller/gcal/scheduler.py b/src/services/ray/src/ndif_ray/deployments/controller/gcal/scheduler.py similarity index 100% rename from src/services/ray/src/ray/deployments/controller/gcal/scheduler.py rename to src/services/ray/src/ndif_ray/deployments/controller/gcal/scheduler.py diff --git a/src/services/ray/src/ray/deployments/modeling/__init__.py b/src/services/ray/src/ndif_ray/deployments/modeling/__init__.py similarity index 100% rename from src/services/ray/src/ray/deployments/modeling/__init__.py rename to src/services/ray/src/ndif_ray/deployments/modeling/__init__.py diff --git a/src/services/ray/src/ray/deployments/modeling/base.py b/src/services/ray/src/ndif_ray/deployments/modeling/base.py similarity index 100% rename from src/services/ray/src/ray/deployments/modeling/base.py rename to src/services/ray/src/ndif_ray/deployments/modeling/base.py diff --git a/src/services/ray/src/ray/deployments/modeling/model.py b/src/services/ray/src/ndif_ray/deployments/modeling/model.py similarity index 100% rename from src/services/ray/src/ray/deployments/modeling/model.py rename to src/services/ray/src/ndif_ray/deployments/modeling/model.py diff --git a/src/services/ray/src/ray/deployments/modeling/util.py b/src/services/ray/src/ndif_ray/deployments/modeling/util.py similarity index 100% rename from src/services/ray/src/ray/deployments/modeling/util.py rename to src/services/ray/src/ndif_ray/deployments/modeling/util.py diff --git a/src/services/ray/src/ray/distributed/__init__.py b/src/services/ray/src/ndif_ray/distributed/__init__.py similarity index 100% rename from src/services/ray/src/ray/distributed/__init__.py rename to src/services/ray/src/ndif_ray/distributed/__init__.py diff --git a/src/services/ray/src/ray/distributed/parallel_dims.py b/src/services/ray/src/ndif_ray/distributed/parallel_dims.py similarity index 100% rename from src/services/ray/src/ray/distributed/parallel_dims.py rename to src/services/ray/src/ndif_ray/distributed/parallel_dims.py diff --git a/src/services/ray/src/ray/distributed/tensor_parallelism/__init__.py b/src/services/ray/src/ndif_ray/distributed/tensor_parallelism/__init__.py similarity index 100% rename from src/services/ray/src/ray/distributed/tensor_parallelism/__init__.py rename to src/services/ray/src/ndif_ray/distributed/tensor_parallelism/__init__.py diff --git a/src/services/ray/src/ray/distributed/tensor_parallelism/plans/__init__.py b/src/services/ray/src/ndif_ray/distributed/tensor_parallelism/plans/__init__.py similarity index 100% rename from src/services/ray/src/ray/distributed/tensor_parallelism/plans/__init__.py rename to src/services/ray/src/ndif_ray/distributed/tensor_parallelism/plans/__init__.py diff --git a/src/services/ray/src/ray/distributed/tensor_parallelism/plans/llama.py b/src/services/ray/src/ndif_ray/distributed/tensor_parallelism/plans/llama.py similarity index 100% rename from src/services/ray/src/ray/distributed/tensor_parallelism/plans/llama.py rename to src/services/ray/src/ndif_ray/distributed/tensor_parallelism/plans/llama.py diff --git a/src/services/ray/src/ray/distributed/tensor_parallelism/test.py b/src/services/ray/src/ndif_ray/distributed/tensor_parallelism/test.py similarity index 100% rename from src/services/ray/src/ray/distributed/tensor_parallelism/test.py rename to src/services/ray/src/ndif_ray/distributed/tensor_parallelism/test.py diff --git a/src/services/ray/src/ray/distributed/util.py b/src/services/ray/src/ndif_ray/distributed/util.py similarity index 100% rename from src/services/ray/src/ray/distributed/util.py rename to src/services/ray/src/ndif_ray/distributed/util.py diff --git a/src/services/ray/src/ray/nn/__init__.py b/src/services/ray/src/ndif_ray/nn/__init__.py similarity index 100% rename from src/services/ray/src/ray/nn/__init__.py rename to src/services/ray/src/ndif_ray/nn/__init__.py diff --git a/src/services/ray/src/ray/nn/backend.py b/src/services/ray/src/ndif_ray/nn/backend.py similarity index 100% rename from src/services/ray/src/ray/nn/backend.py rename to src/services/ray/src/ndif_ray/nn/backend.py diff --git a/src/services/ray/src/ray/nn/ops.py b/src/services/ray/src/ndif_ray/nn/ops.py similarity index 100% rename from src/services/ray/src/ray/nn/ops.py rename to src/services/ray/src/ndif_ray/nn/ops.py diff --git a/src/services/ray/src/ray/nn/sandbox.py b/src/services/ray/src/ndif_ray/nn/sandbox.py similarity index 100% rename from src/services/ray/src/ray/nn/sandbox.py rename to src/services/ray/src/ndif_ray/nn/sandbox.py diff --git a/src/services/ray/src/ray/nn/security/__init__.py b/src/services/ray/src/ndif_ray/nn/security/__init__.py similarity index 100% rename from src/services/ray/src/ray/nn/security/__init__.py rename to src/services/ray/src/ndif_ray/nn/security/__init__.py diff --git a/src/services/ray/src/ray/nn/security/protected_environment.py b/src/services/ray/src/ndif_ray/nn/security/protected_environment.py similarity index 100% rename from src/services/ray/src/ray/nn/security/protected_environment.py rename to src/services/ray/src/ndif_ray/nn/security/protected_environment.py diff --git a/src/services/ray/src/ray/nn/security/protected_objects.py b/src/services/ray/src/ndif_ray/nn/security/protected_objects.py similarity index 100% rename from src/services/ray/src/ray/nn/security/protected_objects.py rename to src/services/ray/src/ndif_ray/nn/security/protected_objects.py diff --git a/src/services/ray/src/ray/resources.py b/src/services/ray/src/ndif_ray/resources.py similarity index 100% rename from src/services/ray/src/ray/resources.py rename to src/services/ray/src/ndif_ray/resources.py diff --git a/src/services/ray/src/ndif_shared/__init__.py b/src/services/ray/src/ndif_shared/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/services/ray/src/ndif_shared/types.py b/src/services/ray/src/ndif_shared/types.py new file mode 120000 index 00000000..eab5f910 --- /dev/null +++ b/src/services/ray/src/ndif_shared/types.py @@ -0,0 +1 @@ +../../../../common/types.py \ No newline at end of file diff --git a/src/services/ray/tests/conftest.py b/src/services/ray/tests/conftest.py new file mode 100644 index 00000000..f5cc3fb3 --- /dev/null +++ b/src/services/ray/tests/conftest.py @@ -0,0 +1,177 @@ +# ndif/src/services/ray/tests/conftest.py +import sys +import importlib +from pathlib import Path +import types +import pytest + +# 1) Put the service src on sys.path so ndif_ray is importable +SERVICE_SRC = Path(__file__).resolve().parents[1] / "src" # .../services/ray/src +sys.path.insert(0, str(SERVICE_SRC)) + +# 2) Force stdlib 'logging' (avoid shadowing by ./src/logging) +stdlib_logging = importlib.import_module("logging") +sys.modules["logging"] = stdlib_logging + +# 3) Stub 'logging_loki' so your internal logger can import it +if "logging_loki" not in sys.modules: + logging_loki = types.ModuleType("logging_loki") + class LokiHandler(stdlib_logging.Handler): + def __init__(self, *a, **k): super().__init__() + def emit(self, record): pass + logging_loki.LokiHandler = LokiHandler + sys.modules["logging_loki"] = logging_loki + +''' +# 4) Robust Ray stub: make 'ray' a *package* and add subpackages you need +def _ensure_pkg(dotted: str) -> types.ModuleType: + """ + Ensure a dotted module path exists in sys.modules as a *package* at each level + (i.e., has __path__), so that child imports work (e.g., ray._private.state). + """ + parts = dotted.split(".") + cur_name = "" + parent = None + for p in parts: + cur_name = f"{cur_name+'.' if cur_name else ''}{p}" + mod = sys.modules.get(cur_name) + if mod is None: + mod = types.ModuleType(cur_name) + mod.__path__ = [] # mark as package + sys.modules[cur_name] = mod + if parent: + setattr(parent, p, mod) + parent = mod + return sys.modules[dotted] + +# Create 'ray' as a package and provide APIs used by your code under test +ray_pkg = _ensure_pkg("ray") + +# Minimal functions used in deployment.py +def _default_get_actor(*a, **k): + raise RuntimeError("actor not found (stubbed)") +if not hasattr(ray_pkg, "get_actor"): + ray_pkg.get_actor = _default_get_actor +if not hasattr(ray_pkg, "kill"): + ray_pkg.kill = lambda *a, **k: None + +# Subpackages some of your modules import +rp = _ensure_pkg("ray._private") +rp_state = _ensure_pkg("ray._private.state") + +# Provide the exact attribute your code imports: from ray._private import services +if not hasattr(rp, "services"): + rp.services = types.SimpleNamespace( + get_node_ip_address=lambda *a, **k: "127.0.0.1" + ) + +# 👉 Provide GlobalState in ray._private.state +if not hasattr(rp_state, "GlobalState"): + class GlobalState: + def __init__(self, *a, **k): pass + # Ray versions differ; expose both names just in case + def initialize_global_state(self, *a, **k): pass + def _initialize_global_state(self, *a, **k): pass + def disconnect(self, *a, **k): pass + # Common query helpers some code calls; return safe defaults + def node_table(self, *a, **k): return [] + def job_table(self, *a, **k): return [] + def cluster_resources(self, *a, **k): return {} + def available_resources(self, *a, **k): return {} + rp_state.GlobalState = GlobalState + +# If anything references the dashboard Serve SDK, stub that too +sdk = _ensure_pkg("ray.dashboard.modules.serve.sdk") +if not hasattr(sdk, "ServeSubmissionClient"): + class _FakeClient: + def __init__(self, *a, **k): ... + def deploy_app(self, *a, **k): return {"ok": True} + sdk.ServeSubmissionClient = _FakeClient + +#end of ray stub +''' + +# --- Robust Ray stub for unit tests ------------------------------------------ +import sys, types + +def _ensure_pkg(dotted: str) -> types.ModuleType: + """Ensure a dotted path exists in sys.modules as a *package* at each level.""" + parts = dotted.split(".") + cur = None + path = [] + for seg in parts: + path.append(seg) + name = ".".join(path) + mod = sys.modules.get(name) + if mod is None: + mod = types.ModuleType(name) + mod.__path__ = [] # make it a package so children can import + sys.modules[name] = mod + if cur is not None: + setattr(cur, seg, mod) + cur = mod + return cur + +# 1) Top-level package 'ray' +ray_pkg = _ensure_pkg("ray") + +# minimal functions you actually use in deployment.py +if not hasattr(ray_pkg, "get_actor"): + def _default_get_actor(*a, **k): # default: simulate "not found" + raise RuntimeError("actor not found (stub)") + ray_pkg.get_actor = _default_get_actor +if not hasattr(ray_pkg, "kill"): + ray_pkg.kill = lambda *a, **k: None + +# 2) Private internals your code imports +# 2a) ray._private and ray._private.services +rp = _ensure_pkg("ray._private") +if not hasattr(rp, "services"): + rp.services = types.SimpleNamespace( + get_node_ip_address=lambda *a, **k: "127.0.0.1", + ) + +# 2b) ray._private.state.GlobalState +rp_state = _ensure_pkg("ray._private.state") +if not hasattr(rp_state, "GlobalState"): + class GlobalState: + def __init__(self, *a, **k): ... + def initialize_global_state(self, *a, **k): ... + def _initialize_global_state(self, *a, **k): ... + def disconnect(self, *a, **k): ... + def node_table(self, *a, **k): return [] + def job_table(self, *a, **k): return [] + def cluster_resources(self, *a, **k): return {} + def available_resources(self, *a, **k): return {} + rp_state.GlobalState = GlobalState + +# 2c) ray._raylet.GcsClientOptions +raylet = _ensure_pkg("ray._raylet") +if not hasattr(raylet, "GcsClientOptions"): + class GcsClientOptions: + def __init__(self, *a, **k): ... + raylet.GcsClientOptions = GcsClientOptions + +# 3) Public helpers your code imports +# 3a) ray.util.state.list_nodes +util_state = _ensure_pkg("ray.util.state") +if not hasattr(util_state, "list_nodes"): + def list_nodes(*a, **k): return [] + util_state.list_nodes = list_nodes + +# 3b) from ray import serve (provide an empty module-like object) +serve_mod = _ensure_pkg("ray.serve") +# if your unit tests later need specific attributes (e.g., serve.deployment), +# add minimal dummies here, e.g.: +# if not hasattr(serve_mod, "deployment"): +# def deployment(func=None, *a, **k): +# def wrapper(f): return f +# return wrapper if func is None else func +# serve_mod.deployment = deployment + + +# Utility fixture you already had +@pytest.fixture +def freeze_time(monkeypatch): + import time + monkeypatch.setattr(time, "time", lambda: 1_000.0, raising=True) diff --git a/src/services/ray/tests/unit/deployments/controller/cluster/test_deployment.py b/src/services/ray/tests/unit/deployments/controller/cluster/test_deployment.py new file mode 100644 index 00000000..3abb80c1 --- /dev/null +++ b/src/services/ray/tests/unit/deployments/controller/cluster/test_deployment.py @@ -0,0 +1,158 @@ +import types +from datetime import datetime, timezone + +import pytest + + +# ---- Helpers --------------------------------------------------------------- + +@pytest.fixture(autouse=True) +def patch_model_key(monkeypatch): + """ + The module calls MODEL_KEY(model_key) in __init__. + Patch it to a simple coercion so we don't depend on the real ndif types. + """ + # Import the module *once* so we can patch its symbols + import ndif_ray.deployments.controller.cluster.deployment as depmod + monkeypatch.setattr(depmod, "MODEL_KEY", lambda x: f"MK:{x}", raising=True) + return depmod + + +@pytest.fixture +def fixed_time(monkeypatch): + """Freeze time.time() so deployed timestamps are stable.""" + import time as _time + monkeypatch.setattr(_time, "time", lambda: 1_000.0, raising=True) + return 1_000.0 + + +@pytest.fixture +def module(monkeypatch, patch_model_key): + """Expose the imported deployment module (already has MODEL_KEY patched).""" + return patch_model_key + + +@pytest.fixture +def fake_logger(mocker, module): + lg = mocker.Mock() + module.logger = lg # replace module-level logger + return lg + + +@pytest.fixture +def fake_ray(mocker, module): + """Replace the ray module inside deployment.py with a stubbed namespace.""" + fake = types.SimpleNamespace() + fake.get_actor = mocker.Mock() + fake.kill = mocker.Mock() + module.ray = fake + return fake + + +# ---- Tests ----------------------------------------------------------------- + +def test_deployment_level_values(module): + # Ensure enum wiring is correct and values are serialized as expected + assert module.DeploymentLevel.HOT.value == "hot" + assert module.DeploymentLevel.WARM.value == "warm" + assert module.DeploymentLevel.COLD.value == "cold" + + +def test_init_and_get_state_casts_model_key_and_sets_fields(module, fixed_time): + d = module.Deployment( + model_key="my/model@v1", + deployment_level=module.DeploymentLevel.HOT, + gpus_required=2, + size_bytes=123456, + dedicated=True, + cached=False, + ) + + st = d.get_state() + # MODEL_KEY was patched to prefix with MK: + assert st["model_key"] == "MK:my/model@v1" + assert st["deployment_level"] == "hot" + assert st["gpus_required"] == 2 + assert st["size_bytes"] == 123456 + assert st["dedicated"] is True + assert st["cached"] is False + # deployed should be the frozen time + assert st["deployed"] == pytest.approx(1_000.0, rel=0, abs=0) + + +def test_end_time_uses_minimum_seconds(module, fixed_time): + d = module.Deployment( + model_key="k", + deployment_level=module.DeploymentLevel.WARM, + gpus_required=0, + size_bytes=0, + ) + dt = d.end_time(60) + assert dt == datetime.fromtimestamp(1_060.0, tz=timezone.utc) + + +def test_delete_kills_actor_on_success(module, fake_ray, fake_logger): + # Arrange: get_actor returns a fake actor object + actor = object() + fake_ray.get_actor.return_value = actor + + d = module.Deployment("k", module.DeploymentLevel.COLD, 0, 0) + d.delete() + + fake_ray.get_actor.assert_called_once_with("ModelActor:MK:k") + fake_ray.kill.assert_called_once_with(actor) + fake_logger.exception.assert_not_called() + + +def test_delete_logs_on_failure(module, fake_ray, fake_logger): + # Arrange: get_actor raises (e.g., actor not found) + fake_ray.get_actor.side_effect = RuntimeError("not found") + + d = module.Deployment("k", module.DeploymentLevel.COLD, 0, 0) + d.delete() + + fake_ray.kill.assert_not_called() + fake_logger.exception.assert_called_once() + # Optional: check message contains model key + msg = fake_logger.exception.call_args[0][0] + #assert "ModelActor" in msg and "MK:k" in msg + assert "Error removing actor" in msg + assert "MK:k" in msg + # don't assert "ModelActor" here + + +def test_cache_returns_object_ref_on_success(module, fake_ray, fake_logger, mocker): + # Fake Ray-style actor method proxy + class _ActorMethod: + def remote(self): + return "OBJECT_REF" + + class FakeHandle: + def __init__(self): + # Ray-style: attribute that has a .remote() method + self.to_cache = _ActorMethod() + + fake_ray.get_actor.return_value = FakeHandle() + + d = module.Deployment("k", module.DeploymentLevel.HOT, 1, 1) + ref = d.cache() + + fake_ray.get_actor.assert_called_once_with("ModelActor:MK:k") + assert ref == "OBJECT_REF" + fake_logger.exception.assert_not_called() + + + +def test_cache_logs_and_returns_none_on_failure(module, fake_ray, fake_logger): + fake_ray.get_actor.side_effect = RuntimeError("boom") + + d = module.Deployment("k", module.DeploymentLevel.HOT, 1, 1) + out = d.cache() + + assert out is None + fake_logger.exception.assert_called_once() + + # 👇 Add these lines to verify message content + msg = fake_logger.exception.call_args[0][0] + assert "Error adding actor" in msg + assert "MK:k" in msg \ No newline at end of file