diff --git a/giskard/ml_worker/websocket/__init__.py b/giskard/ml_worker/websocket/__init__.py index 58e2e5715a..7bdfb271d1 100644 --- a/giskard/ml_worker/websocket/__init__.py +++ b/giskard/ml_worker/websocket/__init__.py @@ -1,9 +1,9 @@ from enum import Enum -from typing import Dict, List, Optional import pydantic from packaging import version from pydantic import Field +from typing import Dict, List, Optional from giskard.core.validation import ConfiguredBaseModel @@ -323,6 +323,7 @@ class TestSuite(WorkerReply): class TestSuiteParam(ConfiguredBaseModel): + projectKey: str tests: Optional[List[SuiteTestArgument]] = None globalArguments: Optional[List[FuncArgument]] = None diff --git a/giskard/ml_worker/websocket/listener.py b/giskard/ml_worker/websocket/listener.py index 1be5c6d261..20f03b1dd2 100644 --- a/giskard/ml_worker/websocket/listener.py +++ b/giskard/ml_worker/websocket/listener.py @@ -10,12 +10,12 @@ from copy import copy from dataclasses import dataclass from pathlib import Path -from typing import Any, Callable, Dict, Optional, Union import numpy as np import pandas as pd import pkg_resources import psutil +from typing import Any, Callable, Dict, Optional, Union import giskard from giskard.client.giskard_client import GiskardClient @@ -562,7 +562,7 @@ def run_test_suite( identifier_single_test_results.append( websocket.IdentifierSingleTestResult( id=identifier, - result=map_result_to_single_test_result_ws(result, datasets), + result=map_result_to_single_test_result_ws(result, datasets, client, params.projectKey), arguments=function_argument_to_ws(arguments), ) ) diff --git a/tests/communications/fixtures/with_alias.json b/tests/communications/fixtures/with_alias.json index 2acfef1380..c7515e91df 100644 --- a/tests/communications/fixtures/with_alias.json +++ b/tests/communications/fixtures/with_alias.json @@ -2866,10 +2866,12 @@ ], "TestSuiteParam": [ { + "projectKey": "my_project", "tests": [], "globalArguments": [] }, { + "projectKey": "awesome_project", "tests": [ { "id": 2201, @@ -2942,6 +2944,7 @@ "globalArguments": [] }, { + "projectKey": "amazing_project", "tests": [ { "id": 5848, diff --git a/tests/communications/test_dto_serialization.py b/tests/communications/test_dto_serialization.py index 10df85046f..e0bae869fb 100644 --- a/tests/communications/test_dto_serialization.py +++ b/tests/communications/test_dto_serialization.py @@ -1,12 +1,12 @@ import json from pathlib import Path -from typing import Dict, List, Optional, Type import pydantic import pytest from packaging import version from polyfactory.factories.pydantic_factory import ModelFactory from pydantic import BaseModel +from typing import Dict, List, Optional, Type import giskard import giskard.ml_worker.websocket @@ -80,7 +80,7 @@ "TestFunctionArgument": ["name", "type", "optional", "default", "argOrder"], "TestMessage": ["type", "text"], "TestSuite": ["is_error", "is_pass", "logs"], - "TestSuiteParam": [], + "TestSuiteParam": ["projectKey"], "WeightsPerFeature": [], "WorkerReply": [], } diff --git a/tests/communications/test_websocket_actor_tests.py b/tests/communications/test_websocket_actor_tests.py index 4652d56b94..e9a1eccb53 100644 --- a/tests/communications/test_websocket_actor_tests.py +++ b/tests/communications/test_websocket_actor_tests.py @@ -1,5 +1,6 @@ import uuid +import pandas as pd import pytest from giskard import test @@ -21,6 +22,11 @@ def my_simple_test_successful(): return GiskardTestResult(passed=True) +@test +def my_simple_test__legacy_debug(): + return GiskardTestResult(passed=True, output_df=Dataset(pd.DataFrame({"test": [1]}))) + + @test def my_simple_test_error(): raise ValueError("Actively raise an error in the test.") @@ -299,6 +305,7 @@ def test_websocket_actor_run_ad_hoc_test_legacy_debug_no_name(enron_data: Datase def test_websocket_actor_run_test_suite(): with utils.MockedClient(mock_all=False) as (client, mr): params = websocket.TestSuiteParam( + projectKey=str(uuid.uuid4()), tests=[ websocket.SuiteTestArgument( id=0, @@ -339,6 +346,7 @@ def test_websocket_actor_run_test_suite(): def test_websocket_actor_run_test_suite_raise_error(): with utils.MockedClient(mock_all=False) as (client, mr): params = websocket.TestSuiteParam( + projectKey=str(uuid.uuid4()), tests=[ websocket.SuiteTestArgument( id=0, @@ -372,6 +380,7 @@ def my_test_return(value: int = MY_TEST_DEFAULT_VALUE): def test_websocket_actor_run_test_suite_with_global_arguments(): with utils.MockedClient(mock_all=False) as (client, mr): params = websocket.TestSuiteParam( + projectKey=str(uuid.uuid4()), tests=[ websocket.SuiteTestArgument( id=0, @@ -406,6 +415,7 @@ def test_websocket_actor_run_test_suite_with_global_arguments(): def test_websocket_actor_run_test_suite_with_test_input(): with utils.MockedClient(mock_all=False) as (client, mr): params = websocket.TestSuiteParam( + projectKey=str(uuid.uuid4()), tests=[ websocket.SuiteTestArgument( id=0, @@ -437,9 +447,41 @@ def test_websocket_actor_run_test_suite_with_test_input(): ) +def test_websocket_actor_run_test_suite_with_legacy_debug(): + with utils.MockedClient(mock_all=False) as (client, mr): + params = websocket.TestSuiteParam( + projectKey=str(uuid.uuid4()), + tests=[ + websocket.SuiteTestArgument( + id=0, + testUuid=my_simple_test__legacy_debug.meta.uuid, + arguments=[ + websocket.FuncArgument(name="value", int=MY_TEST_INPUT_VALUE, none=False), + ], + ), + ], + globalArguments=[ + websocket.FuncArgument(name="value", int=MY_TEST_GLOBAL_VALUE, none=False), + ], + ) + utils.register_uri_for_artifact_meta_info(mr, my_simple_test__legacy_debug, None) + utils.register_uri_for_any_dataset_artifact_info_upload(mr, True) + + reply = listener.run_test_suite(client, params) + + assert isinstance(reply, websocket.TestSuite) + assert not reply.is_error, reply.logs + assert reply.is_pass + + assert 1 == len(reply.results) + assert 0 == reply.results[0].id + assert reply.results[0].result.passed + + def test_websocket_actor_run_test_suite_with_kwargs(): with utils.MockedClient(mock_all=False) as (client, mr): params = websocket.TestSuiteParam( + projectKey=str(uuid.uuid4()), tests=[ websocket.SuiteTestArgument( id=0,