[GSK-2032] Fix debug master

giskard/core/core.py

@@ -169,26 +169,35 @@ def __init__(
             self.tags = self.populate_tags(tags)
 
             parameters = self.extract_parameters(callable_obj)
+            for param in parameters:
+                param.default = serialize_parameter(param.default)
 
-            self.args = {
-                parameter.name: FunctionArgument(
-                    name=parameter.name,
-                    type=extract_optional(parameter.annotation).__qualname__,
-                    optional=parameter.default != inspect.Parameter.empty,
-                    default=serialize_parameter(parameter.default),
-                    argOrder=idx,
-                )
-                for idx, parameter in enumerate(parameters.values())
-                if name != "self"
-            }
+            self.args = {param.name: param for param in parameters}
 
-    def extract_parameters(self, callable_obj):
+    def extract_parameters(self, callable_obj) -> List[FunctionArgument]:
         if inspect.isclass(callable_obj):
             parameters = list(inspect.signature(callable_obj.__init__).parameters.values())[1:]
         else:
             parameters = list(inspect.signature(callable_obj).parameters.values())
 
-        return parameters
+        return [
+            FunctionArgument(
+                name=parameter.name,
+                type=self.extract_parameter_type_name(parameter),
+                optional=parameter.default != inspect.Parameter.empty,
+                default=parameter.default,
+                argOrder=idx,
+            )
+            for idx, parameter in enumerate(parameters)
+        ]
+
+    @staticmethod
+    def extract_parameter_type_name(parameter):
+        return (
+            extract_optional(parameter.annotation).__qualname__
+            if hasattr(extract_optional(parameter.annotation), "__qualname__")
+            else None
+        )
 
     @staticmethod
     def extract_module_doc(func_doc):
@@ -293,10 +302,8 @@ def __init__(
         super().__init__(callable_obj, name, tags, version, type)
         self.debug_description = debug_description
 
-    def extract_parameters(self, callable_obj):
-        parameters = unknown_annotations_to_kwargs(CallableMeta.extract_parameters(self, callable_obj))
-
-        return {p.name: p for p in parameters}
+    def extract_parameters(self, callable_obj) -> List[FunctionArgument]:
+        return unknown_annotations_to_kwargs(CallableMeta.extract_parameters(self, callable_obj))
 
     def to_json(self):
         json = super().to_json()
@@ -346,10 +353,8 @@ def __init__(
         else:
             self.column_type = None
 
-    def extract_parameters(self, callable_obj):
-        parameters = unknown_annotations_to_kwargs(CallableMeta.extract_parameters(self, callable_obj)[1:])
-
-        return {p.name: p for p in parameters}
+    def extract_parameters(self, callable_obj) -> List[FunctionArgument]:
+        return unknown_annotations_to_kwargs(CallableMeta.extract_parameters(self, callable_obj)[1:])
 
     def to_json(self):
         json = super().to_json()
@@ -373,25 +378,37 @@ def init_from_json(self, json: Dict[str, Any]):
 SMT = TypeVar("SMT", bound=SavableMeta)
 
 
-def unknown_annotations_to_kwargs(parameters: List[inspect.Parameter]) -> List[inspect.Parameter]:
+def unknown_annotations_to_kwargs(parameters: List[FunctionArgument]) -> List[FunctionArgument]:
     from giskard.models.base import BaseModel
     from giskard.datasets.base import Dataset
     from giskard.ml_worker.testing.registry.slicing_function import SlicingFunction
     from giskard.ml_worker.testing.registry.transformation_function import TransformationFunction
 
     allowed_types = [str, bool, int, float, BaseModel, Dataset, SlicingFunction, TransformationFunction]
-    allowed_types = allowed_types + list(map(lambda x: Optional[x], allowed_types))
+    allowed_types = list(map(lambda x: x.__qualname__, allowed_types))
+
+    kwargs = [param for param in parameters if not any([param.type == allowed_type for allowed_type in allowed_types])]
 
-    has_kwargs = any(
-        [param for param in parameters if not any([param.annotation == allowed_type for allowed_type in allowed_types])]
-    )
+    parameters = [param for param in parameters if any([param.type == allowed_type for allowed_type in allowed_types])]
 
-    parameters = [
-        param for param in parameters if any([param.annotation == allowed_type for allowed_type in allowed_types])
-    ]
+    for idx, parameter in enumerate(parameters):
+        parameter.argOrder = idx
 
-    if has_kwargs:
-        parameters.append(inspect.Parameter(name="kwargs", kind=4, annotation=Kwargs))
+    if any(kwargs) > 0:
+        kwargs_with_default = [param for param in kwargs if param.default != inspect.Parameter.empty]
+        default_value = (
+            dict({param.name: param.default for param in kwargs_with_default}) if any(kwargs_with_default) else None
+        )
+
+        parameters.append(
+            FunctionArgument(
+                name="kwargs",
+                type="Kwargs",
+                default=default_value,
+                optional=len(kwargs_with_default) == len(kwargs),
+                argOrder=len(parameters),
+            )
+        )
 
     return parameters
 

giskard/ml_worker/testing/utils.py

@@ -1,13 +1,14 @@
+from typing import Optional
+
+import numbers
 from enum import Enum
 from functools import wraps
-import numbers
-from typing import Optional
 
-from giskard.datasets.base import Dataset
 from giskard.core.core import SupportedModelTypes
+from giskard.datasets.base import Dataset
 
 
-class Direction(Enum):
+class Direction(int, Enum):
     Invariant = 0
     Increasing = 1
     Decreasing = -1

giskard/ml_worker/websocket/listener.py

@@ -1,4 +1,4 @@
-from typing import Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, Optional, Union
 
 import json
 import logging
@@ -10,6 +10,7 @@
 import time
 import traceback
 from concurrent.futures import CancelledError, Future
+from copy import copy
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -581,6 +582,8 @@ def run_ad_hoc_test(
     test: GiskardTest = GiskardTest.download(params.testUuid, client, None)
 
     arguments = parse_function_arguments(client, params.arguments)
+    if "kwargs" in arguments:
+        arguments.update(**arguments.pop("kwargs"))
 
     arguments["debug"] = params.debug if params.debug else None
     debug_info = extract_debug_info(params.arguments) if params.debug else None
@@ -623,12 +626,17 @@ def run_test_suite(
 
         suite = Suite()
         for t in tests:
-            suite.add_test(t["test"].get_builder()(**t["arguments"]), t["id"])
+            test_args = t["arguments"]
+            if "kwargs" in test_args:
+                test_args: Dict[str, Any] = copy(test_args)
+                test_args.update(**test_args.pop("kwargs"))
+            suite.add_test(t["test"].get_builder()(**test_args), t["id"])
 
         suite_result = suite.run(**global_arguments)
 
         identifier_single_test_results = []
-        for identifier, result, arguments in suite_result.results:
+        for t, (identifier, result, _) in zip(tests, suite_result.results):
+            arguments = t["arguments"]
             identifier_single_test_results.append(
                 websocket.IdentifierSingleTestResult(
                     id=identifier,

giskard/ml_worker/websocket/utils.py

@@ -1,9 +1,10 @@
+from typing import Any, Dict, List, Optional
+
 import logging
 import os
 import posixpath
 import shutil
 from pathlib import Path
-from typing import Any, Dict, List, Optional
 
 from mlflow.store.artifact.artifact_repo import verify_artifact_path
 
@@ -193,7 +194,7 @@ def parse_function_arguments(client: Optional[GiskardClient], request_arguments:
         elif arg.kwargs is not None:
             kwargs = dict()
             exec(arg.kwargs, {"kwargs": kwargs})
-            arguments.update(kwargs)
+            arguments[arg.name] = kwargs
         else:
             raise IllegalArgumentError("Unknown argument type")
     return arguments

giskard/testing/tests/performance.py

@@ -1,7 +1,8 @@
 """Performance tests"""
-import inspect
 from typing import Optional
 
+import inspect
+
 import numpy as np
 import pandas as pd
 from sklearn.metrics import (
@@ -22,7 +23,10 @@
 from giskard.ml_worker.testing.utils import Direction, check_slice_not_empty
 from giskard.models.base import BaseModel
 from giskard.models.utils import np_type_to_native
-from giskard.testing.tests.debug_slicing_functions import incorrect_rows_slicing_fn, nlargest_abs_err_rows_slicing_fn
+from giskard.testing.tests.debug_slicing_functions import (
+    incorrect_rows_slicing_fn,
+    nlargest_abs_err_rows_slicing_fn,
+)
 
 from . import debug_description_prefix, debug_prefix
 

giskard/utils/artifacts.py

@@ -1,6 +1,7 @@
 import inspect
 import uuid
-from typing import Any, Optional, Union
+from enum import Enum
+from typing import Any, Optional, Union, Dict
 
 try:
     from types import NoneType
@@ -18,13 +19,23 @@ def _serialize_artifact(artifact, artifact_uuid: Optional[Union[str, uuid.UUID]]
     return str(artifact_uuid)
 
 
+def repr_parameter(value: Any) -> str:
+    if isinstance(value, Enum):
+        return repr(value.value)
+
+    return repr(value)
+
+
 def serialize_parameter(default_value: Any) -> PRIMITIVES:
     if default_value == inspect.Parameter.empty:
         return None
 
     if isinstance(default_value, PRIMITIVES.__args__):
         return default_value
 
+    if isinstance(default_value, Dict):
+        return "\n".join(f"kwargs[{repr(key)}] = {repr_parameter(value)}" for key, value in default_value.items())
+
     from ..ml_worker.core.savable import Artifact
 
     if isinstance(default_value, Artifact):

tests/communications/test_listener_utils.py

@@ -2,7 +2,11 @@
 
 from giskard.ml_worker import websocket
 from giskard.ml_worker.exceptions.IllegalArgumentError import IllegalArgumentError
-from giskard.ml_worker.websocket.listener import extract_debug_info, function_argument_to_ws, parse_function_arguments
+from giskard.ml_worker.websocket.listener import (
+    extract_debug_info,
+    function_argument_to_ws,
+    parse_function_arguments,
+)
 
 TEST_PROJECT_KEY = "123"
 TEST_MODEL_ID = "231"
@@ -30,7 +34,6 @@ def test_extract_debug_info():
 
 
 def test_function_argument_to_ws():
-
     # Domain classes creation should be tested somewhere else, do not test them.
     #   "dataset": Dataset,
     #   "model": BaseModel,
@@ -76,4 +79,4 @@ def test_parse_function_arguments():
     assert "int" in kwargs.keys() and kwargs["int"] == TEST_FUNC_ARGUMENT_INT
     assert "str" in kwargs.keys() and kwargs["str"] == TEST_FUNC_ARGUMENT_STR
     assert "bool" in kwargs.keys() and kwargs["bool"] == TEST_FUNC_ARGUMENT_BOOL
-    assert "bool1" in kwargs.keys() and kwargs["bool1"] == TEST_FUNC_ARGUMENT_BOOL
+    assert "kwargs" in kwargs.keys() and kwargs["kwargs"]["bool1"] == TEST_FUNC_ARGUMENT_BOOL