From 185ec0ca8609f120f40795a8430b08ae8095821f Mon Sep 17 00:00:00 2001 From: princyiakov Date: Wed, 31 Aug 2022 10:27:38 +0200 Subject: [PATCH 01/19] GSK-254 Custom Error added for AUC --- giskard-ml-worker/ml_worker/testing/performance_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/giskard-ml-worker/ml_worker/testing/performance_tests.py b/giskard-ml-worker/ml_worker/testing/performance_tests.py index 7d91d0173b..4b6e117717 100644 --- a/giskard-ml-worker/ml_worker/testing/performance_tests.py +++ b/giskard-ml-worker/ml_worker/testing/performance_tests.py @@ -44,8 +44,12 @@ def test_auc(self, actual_slice: GiskardDataset, model: GiskardModel, threshold= metric = roc_auc_score(actual_slice.df[actual_slice.target], model.run_predict(actual_slice).raw_prediction) else: - metric = roc_auc_score(actual_slice.df[actual_slice.target], - model.run_predict(actual_slice).all_predictions, multi_class='ovr') + try: + metric = roc_auc_score(actual_slice.df[actual_slice.target], + model.run_predict(actual_slice).all_predictions, multi_class='ovr') + except ValueError: + raise ValueError("Unable to perform AUC test : Dataset does not include all the classes declared " + "in classification_labels") return self.save_results( SingleTestResult( From 7554673b3b39b3bae4bd2c5e383eb5bd5db1a697 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Wed, 31 Aug 2022 10:28:31 +0200 Subject: [PATCH 02/19] GSK-254 Exception and Non Exception scenario added in unit test for small dataset for AUC test --- giskard-ml-worker/test/test_performance.py | 36 +++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/giskard-ml-worker/test/test_performance.py b/giskard-ml-worker/test/test_performance.py index 867a58669b..6c441e8e1c 100644 --- a/giskard-ml-worker/test/test_performance.py +++ b/giskard-ml-worker/test/test_performance.py @@ -37,6 +37,40 @@ def test_auc(data, model, threshold, expected_metric, actual_slices_size, reques assert results.passed +@pytest.mark.parametrize('data,model,threshold,expected_metric,actual_slices_size', + [('enron_data', 'enron_model', 0.5, 1, 7)]) +def test_auc_with_unique_target_NoException(data, model, threshold, expected_metric, actual_slices_size, request): + tests = GiskardTestFunctions() + data = request.getfixturevalue(data) + results = tests.performance.test_auc( + actual_slice=data.slice(lambda df: df.drop_duplicates('Target')), + model=request.getfixturevalue(model), + threshold=threshold + ) + + assert results.actual_slices_size[0] == actual_slices_size + assert round(results.metric, 2) == expected_metric + assert results.passed + + +@pytest.mark.parametrize('data,model,threshold,expected_metric,actual_slices_size', + [('enron_data', 'enron_model', 0.5, 1, 7)]) +def test_auc_with_unique_target_RaiseException(data, model, threshold, expected_metric, actual_slices_size, request): + + with pytest.raises(Exception): + tests = GiskardTestFunctions() + data = request.getfixturevalue(data) + results = tests.performance.test_auc( + actual_slice=data.slice(lambda df: df.drop_duplicates('Target').head()), + model=request.getfixturevalue(model), + threshold=threshold + ) + + assert results.actual_slices_size[0] == actual_slices_size + assert round(results.metric, 2) == expected_metric + assert results.passed + + @pytest.mark.parametrize('data,model,threshold,expected_metric,actual_slices_size', [('german_credit_data', 'german_credit_model', 0.5, 0.81, 1000), ('enron_data', 'enron_model', 0.5, 0.72, 50)]) @@ -264,4 +298,4 @@ def test_diff_reference_actual_rmse(data, model, threshold, expected_metric, req ) assert round(result.metric, 2) == expected_metric assert type(result.output_df) is bytes - assert result.passed \ No newline at end of file + assert result.passed From d4e1b4e6ddfca65cd3d6fbe8fae684b147317c81 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Wed, 31 Aug 2022 11:00:20 +0200 Subject: [PATCH 03/19] GSK-254 removed spaces --- giskard-ml-worker/ml_worker/testing/performance_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giskard-ml-worker/ml_worker/testing/performance_tests.py b/giskard-ml-worker/ml_worker/testing/performance_tests.py index 4b6e117717..97c40c8670 100644 --- a/giskard-ml-worker/ml_worker/testing/performance_tests.py +++ b/giskard-ml-worker/ml_worker/testing/performance_tests.py @@ -48,7 +48,7 @@ def test_auc(self, actual_slice: GiskardDataset, model: GiskardModel, threshold= metric = roc_auc_score(actual_slice.df[actual_slice.target], model.run_predict(actual_slice).all_predictions, multi_class='ovr') except ValueError: - raise ValueError("Unable to perform AUC test : Dataset does not include all the classes declared " + raise ValueError("Unable to perform AUC test: Dataset does not include all the classes declared " "in classification_labels") return self.save_results( From e63774a01f716deb812ecd5a6f0af9e814cf494c Mon Sep 17 00:00:00 2001 From: princyiakov Date: Wed, 31 Aug 2022 11:02:09 +0200 Subject: [PATCH 04/19] GSK-254 removed asserts for unit test raising exception --- giskard-ml-worker/test/test_performance.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/giskard-ml-worker/test/test_performance.py b/giskard-ml-worker/test/test_performance.py index 6c441e8e1c..ba01d1fae8 100644 --- a/giskard-ml-worker/test/test_performance.py +++ b/giskard-ml-worker/test/test_performance.py @@ -60,16 +60,12 @@ def test_auc_with_unique_target_RaiseException(data, model, threshold, expected_ with pytest.raises(Exception): tests = GiskardTestFunctions() data = request.getfixturevalue(data) - results = tests.performance.test_auc( + tests.performance.test_auc( actual_slice=data.slice(lambda df: df.drop_duplicates('Target').head()), model=request.getfixturevalue(model), threshold=threshold ) - assert results.actual_slices_size[0] == actual_slices_size - assert round(results.metric, 2) == expected_metric - assert results.passed - @pytest.mark.parametrize('data,model,threshold,expected_metric,actual_slices_size', [('german_credit_data', 'german_credit_model', 0.5, 0.81, 1000), From 9b8a59b1f78c89adc22d5ef185a701c143cd88f4 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Wed, 31 Aug 2022 11:04:11 +0200 Subject: [PATCH 05/19] GSK-254 code cleaning --- giskard-ml-worker/test/test_performance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/giskard-ml-worker/test/test_performance.py b/giskard-ml-worker/test/test_performance.py index ba01d1fae8..7c4911674a 100644 --- a/giskard-ml-worker/test/test_performance.py +++ b/giskard-ml-worker/test/test_performance.py @@ -39,7 +39,7 @@ def test_auc(data, model, threshold, expected_metric, actual_slices_size, reques @pytest.mark.parametrize('data,model,threshold,expected_metric,actual_slices_size', [('enron_data', 'enron_model', 0.5, 1, 7)]) -def test_auc_with_unique_target_NoException(data, model, threshold, expected_metric, actual_slices_size, request): +def test_auc_with_unique_target_no_exception(data, model, threshold, expected_metric, actual_slices_size, request): tests = GiskardTestFunctions() data = request.getfixturevalue(data) results = tests.performance.test_auc( @@ -55,8 +55,7 @@ def test_auc_with_unique_target_NoException(data, model, threshold, expected_met @pytest.mark.parametrize('data,model,threshold,expected_metric,actual_slices_size', [('enron_data', 'enron_model', 0.5, 1, 7)]) -def test_auc_with_unique_target_RaiseException(data, model, threshold, expected_metric, actual_slices_size, request): - +def test_auc_with_unique_target_raise_exception(data, model, threshold, expected_metric, actual_slices_size, request): with pytest.raises(Exception): tests = GiskardTestFunctions() data = request.getfixturevalue(data) From 500f4a28eebdcf5dedc6b027c8a9762497ecef1b Mon Sep 17 00:00:00 2001 From: princyiakov Date: Thu, 1 Sep 2022 16:47:46 +0200 Subject: [PATCH 06/19] GSK-254 set logging to info --- .../src/main/resources/config/application-dev.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/giskard-server/src/main/resources/config/application-dev.yml b/giskard-server/src/main/resources/config/application-dev.yml index 03555045e4..feecbcf847 100644 --- a/giskard-server/src/main/resources/config/application-dev.yml +++ b/giskard-server/src/main/resources/config/application-dev.yml @@ -15,10 +15,10 @@ logging: level: - ROOT: DEBUG - tech.jhipster: DEBUG - org.hibernate.SQL: DEBUG - ai.giskard: DEBUG + ROOT: INFO + tech.jhipster: INFO + org.hibernate.SQL: INFO + ai.giskard: INFO spring: devtools: From c4fafcd57509d9bc1619fec33dcf0af286ba4ae4 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Thu, 1 Sep 2022 16:48:07 +0200 Subject: [PATCH 07/19] GSK-254 improved logging for java --- .../src/main/java/ai/giskard/service/TestService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/giskard-server/src/main/java/ai/giskard/service/TestService.java b/giskard-server/src/main/java/ai/giskard/service/TestService.java index bf88d32b9b..3046ba4a69 100644 --- a/giskard-server/src/main/java/ai/giskard/service/TestService.java +++ b/giskard-server/src/main/java/ai/giskard/service/TestService.java @@ -58,6 +58,8 @@ public Optional saveTest(TestDTO dto) { public TestExecutionResultDTO runTest(Long testId) throws IOException { TestExecutionResultDTO res = new TestExecutionResultDTO(testId); Test test = testRepository.findById(testId).orElseThrow(() -> new EntityNotFoundException(Entity.TEST, testId)); + logger.info("Running test {}({}) for suite {}, project {}", + test.getName(), test.getId(), test.getTestSuite().getId(), test.getTestSuite().getProject().getKey()); TestExecution testExecution = new TestExecution(test); testExecutionRepository.save(testExecution); From 5d0fb98deb57815e002a57eb1679609ec386fbb9 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Thu, 1 Sep 2022 16:48:53 +0200 Subject: [PATCH 08/19] GSK-254 improved logging for deserialization --- .../ml_worker/utils/grpc_mapper.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/giskard-ml-worker/ml_worker/utils/grpc_mapper.py b/giskard-ml-worker/ml_worker/utils/grpc_mapper.py index 68f6dcd0aa..2782b573a2 100644 --- a/giskard-ml-worker/ml_worker/utils/grpc_mapper.py +++ b/giskard-ml-worker/ml_worker/utils/grpc_mapper.py @@ -1,4 +1,6 @@ from io import BytesIO +import logging +import time import cloudpickle import pandas as pd @@ -11,17 +13,26 @@ def deserialize_model(serialized_model: SerializedGiskardModel) -> GiskardModel: - return GiskardModel( + logging.info("Initiating model deserialization") + start_time = time.time() + + deserialized_model = GiskardModel( cloudpickle.load(ZstdDecompressor().stream_reader(serialized_model.serialized_prediction_function)), model_type=serialized_model.model_type, classification_threshold=serialized_model.threshold.value if serialized_model.HasField('threshold') else None, feature_names=list(serialized_model.feature_names), classification_labels=list(serialized_model.classification_labels) ) + logging.info(f"Successfully deserialized model in {(time.time() - start_time)}s") + + return deserialized_model def deserialize_dataset(serialized_dataset: SerializedGiskardDataset) -> GiskardDataset: - return GiskardDataset( + logging.info("Initiating dataset deserialization") + start_time = time.time() + + deserialized_dataset = GiskardDataset( df=pd.read_csv( BytesIO(decompress(serialized_dataset.serialized_df)), keep_default_na=False, @@ -31,3 +42,6 @@ def deserialize_dataset(serialized_dataset: SerializedGiskardDataset) -> Giskard feature_types=dict(serialized_dataset.feature_types), column_types=dict(serialized_dataset.column_types) ) + logging.info(f"Successfully deserialized dataset in {(time.time() - start_time)}s") + + return deserialized_dataset From 507dfa78f2bf409eddf23aa8f80c41e239f59cb2 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Thu, 1 Sep 2022 16:49:34 +0200 Subject: [PATCH 09/19] GSK-254 improved logging for test execution --- giskard-ml-worker/ml_worker/server/ml_worker_service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/giskard-ml-worker/ml_worker/server/ml_worker_service.py b/giskard-ml-worker/ml_worker/server/ml_worker_service.py index 5c62e83c79..e10af0dafc 100644 --- a/giskard-ml-worker/ml_worker/server/ml_worker_service.py +++ b/giskard-ml-worker/ml_worker/server/ml_worker_service.py @@ -1,5 +1,7 @@ import logging import re +import time + import grpc import numpy as np @@ -25,7 +27,6 @@ def __init__(self) -> None: def runTest(self, request: RunTestRequest, context: grpc.ServicerContext) -> TestResultMessage: from ml_worker.testing.functions import GiskardTestFunctions - model = deserialize_model(request.model) tests = GiskardTestFunctions() @@ -38,7 +39,9 @@ def runTest(self, request: RunTestRequest, context: grpc.ServicerContext) -> Tes if request.actual_ds.serialized_df: _globals['actual_ds'] = deserialize_dataset(request.actual_ds) try: + start_time = time.time() exec(request.code, _globals) + logging.info(f"Successfully executed the {tests.tests_results[0].name} test in {(time.time() - start_time)}s") except NameError as e: missing_name = re.findall(r"name '(\w+)' is not defined", str(e))[0] if missing_name == 'reference_ds': From 443cc1f88b25dd58d3162bb21f712802cb9426e3 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Fri, 2 Sep 2022 11:56:41 +0200 Subject: [PATCH 10/19] GSK-254 improved logging for deserialization of dataset and model --- giskard-ml-worker/ml_worker/utils/grpc_mapper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/giskard-ml-worker/ml_worker/utils/grpc_mapper.py b/giskard-ml-worker/ml_worker/utils/grpc_mapper.py index 2782b573a2..0818a8259b 100644 --- a/giskard-ml-worker/ml_worker/utils/grpc_mapper.py +++ b/giskard-ml-worker/ml_worker/utils/grpc_mapper.py @@ -10,10 +10,10 @@ from ml_worker.core.giskard_dataset import GiskardDataset from ml_worker.core.model import GiskardModel from generated.ml_worker_pb2 import SerializedGiskardModel, SerializedGiskardDataset +import tensorflow_text def deserialize_model(serialized_model: SerializedGiskardModel) -> GiskardModel: - logging.info("Initiating model deserialization") start_time = time.time() deserialized_model = GiskardModel( @@ -29,7 +29,6 @@ def deserialize_model(serialized_model: SerializedGiskardModel) -> GiskardModel: def deserialize_dataset(serialized_dataset: SerializedGiskardDataset) -> GiskardDataset: - logging.info("Initiating dataset deserialization") start_time = time.time() deserialized_dataset = GiskardDataset( From 87be88572e8d27263be4037eca6f5fc52edb0768 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Fri, 2 Sep 2022 11:57:17 +0200 Subject: [PATCH 11/19] GSK-254 removed tensorflow_text --- giskard-ml-worker/ml_worker/utils/grpc_mapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/giskard-ml-worker/ml_worker/utils/grpc_mapper.py b/giskard-ml-worker/ml_worker/utils/grpc_mapper.py index 0818a8259b..d28b1e6ede 100644 --- a/giskard-ml-worker/ml_worker/utils/grpc_mapper.py +++ b/giskard-ml-worker/ml_worker/utils/grpc_mapper.py @@ -10,7 +10,6 @@ from ml_worker.core.giskard_dataset import GiskardDataset from ml_worker.core.model import GiskardModel from generated.ml_worker_pb2 import SerializedGiskardModel, SerializedGiskardDataset -import tensorflow_text def deserialize_model(serialized_model: SerializedGiskardModel) -> GiskardModel: From 6866c597e2f99f17f1eb605a25acc5f14f3e39c4 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Fri, 2 Sep 2022 11:59:06 +0200 Subject: [PATCH 12/19] GSK-254 improved logging for metamorphic tests --- giskard-ml-worker/ml_worker/testing/metamorphic_tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/giskard-ml-worker/ml_worker/testing/metamorphic_tests.py b/giskard-ml-worker/ml_worker/testing/metamorphic_tests.py index a8c0c658b8..bb1557b58d 100644 --- a/giskard-ml-worker/ml_worker/testing/metamorphic_tests.py +++ b/giskard-ml-worker/ml_worker/testing/metamorphic_tests.py @@ -1,5 +1,6 @@ import pandas as pd - +import time +import logging from generated.ml_worker_pb2 import SingleTestResult, TestMessage, TestMessageType from ml_worker.core.giskard_dataset import GiskardDataset @@ -41,6 +42,8 @@ def _perturb_and_predict(ds: GiskardDataset, model: GiskardModel, perturbation_d return results_df, len(modified_rows) def _compare_prediction(self, results_df, prediction_task, output_sensitivity=None, flag=None): + start_time = time.time() + if flag == 'Invariance': if prediction_task == 'classification': passed_idx = results_df.loc[ @@ -60,6 +63,7 @@ def _compare_prediction(self, results_df, prediction_task, output_sensitivity=No results_df['prediction'] > results_df['perturbed_prediction']].index.values failed_idx = results_df.loc[~results_df.index.isin(passed_idx)].index.values + logging.info(f"Compared and predicted the data in {(time.time() - start_time)}s") return passed_idx, failed_idx def _test_metamorphic(self, From 3a2f889c270ad88760e8e03a01ea949dc8dbf7e2 Mon Sep 17 00:00:00 2001 From: princyiakov Date: Fri, 2 Sep 2022 11:59:34 +0200 Subject: [PATCH 13/19] GSK-254 improved logging for metamorphic tests --- giskard-ml-worker/ml_worker/testing/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/giskard-ml-worker/ml_worker/testing/utils.py b/giskard-ml-worker/ml_worker/testing/utils.py index b521c6c66a..e3356dc070 100644 --- a/giskard-ml-worker/ml_worker/testing/utils.py +++ b/giskard-ml-worker/ml_worker/testing/utils.py @@ -1,4 +1,6 @@ import pandas as pd +import time +import logging from generated.ml_worker_pb2 import SingleTestResult @@ -26,6 +28,7 @@ def ge_result_to_test_result(result, passed=True) -> SingleTestResult: def apply_perturbation_inplace(df: pd.DataFrame, perturbation_dict): + start_time = time.time() modified_rows = [] i = 0 for idx, r in df.iterrows(): @@ -38,4 +41,5 @@ def apply_perturbation_inplace(df: pd.DataFrame, perturbation_dict): modified_rows.append(i) df.loc[idx, pert_col] = new_value i += 1 + logging.info(f"Perturbed data in {(time.time() - start_time)}s") return modified_rows From 546b42e78965bbf48ed36cd28c157a3233d440dc Mon Sep 17 00:00:00 2001 From: princyiakov Date: Fri, 2 Sep 2022 12:00:28 +0200 Subject: [PATCH 14/19] GSK-254 improved logging for model.py --- giskard-ml-worker/ml_worker/core/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/giskard-ml-worker/ml_worker/core/model.py b/giskard-ml-worker/ml_worker/core/model.py index 91ff8b9025..af3aa52796 100644 --- a/giskard-ml-worker/ml_worker/core/model.py +++ b/giskard-ml-worker/ml_worker/core/model.py @@ -1,5 +1,6 @@ import logging from typing import List, Any, Optional, Callable, Iterable, Union +import time import numpy import pandas as pd @@ -31,6 +32,7 @@ def __init__(self, self.classification_labels = classification_labels def run_predict(self, dataset: GiskardDataset): + start_time = time.time() df = self.prepare_dataframe(dataset) raw_prediction = self.prediction_function(df) if self.model_type == "regression": @@ -62,6 +64,7 @@ def run_predict(self, dataset: GiskardDataset): raise ValueError( f"Prediction task is not supported: {self.model_type}" ) + logging.info(f"Predicted dataset with {dataset.df.shape} shape in {(time.time() - start_time)}s") return result def prepare_dataframe(self, dataset): From d8235243287440403868bfc64bff055cbef42fa7 Mon Sep 17 00:00:00 2001 From: andreybavt Date: Fri, 2 Sep 2022 16:01:17 +0200 Subject: [PATCH 15/19] GSK-254 AUC test throws error for multiclass model with small dataset --- .../ml_worker/testing/performance_tests.py | 12 ++++++------ giskard-ml-worker/test/test_performance.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/giskard-ml-worker/ml_worker/testing/performance_tests.py b/giskard-ml-worker/ml_worker/testing/performance_tests.py index 97c40c8670..c82584accb 100644 --- a/giskard-ml-worker/ml_worker/testing/performance_tests.py +++ b/giskard-ml-worker/ml_worker/testing/performance_tests.py @@ -44,12 +44,12 @@ def test_auc(self, actual_slice: GiskardDataset, model: GiskardModel, threshold= metric = roc_auc_score(actual_slice.df[actual_slice.target], model.run_predict(actual_slice).raw_prediction) else: - try: - metric = roc_auc_score(actual_slice.df[actual_slice.target], - model.run_predict(actual_slice).all_predictions, multi_class='ovr') - except ValueError: - raise ValueError("Unable to perform AUC test: Dataset does not include all the classes declared " - "in classification_labels") + predictions = model.run_predict(actual_slice).all_predictions + non_declared_categories = set(predictions.columns) - set(actual_slice.df[actual_slice.target].unique()) + assert not len(non_declared_categories), \ + f"Predicted classes don't exist in the dataset \"{actual_slice.target}\" column: {non_declared_categories}" + + metric = roc_auc_score(actual_slice.df[actual_slice.target], predictions, multi_class='ovr') return self.save_results( SingleTestResult( diff --git a/giskard-ml-worker/test/test_performance.py b/giskard-ml-worker/test/test_performance.py index 7c4911674a..a9836f1f59 100644 --- a/giskard-ml-worker/test/test_performance.py +++ b/giskard-ml-worker/test/test_performance.py @@ -56,7 +56,7 @@ def test_auc_with_unique_target_no_exception(data, model, threshold, expected_me @pytest.mark.parametrize('data,model,threshold,expected_metric,actual_slices_size', [('enron_data', 'enron_model', 0.5, 1, 7)]) def test_auc_with_unique_target_raise_exception(data, model, threshold, expected_metric, actual_slices_size, request): - with pytest.raises(Exception): + with pytest.raises(AssertionError) as e: tests = GiskardTestFunctions() data = request.getfixturevalue(data) tests.performance.test_auc( @@ -64,6 +64,7 @@ def test_auc_with_unique_target_raise_exception(data, model, threshold, expected model=request.getfixturevalue(model), threshold=threshold ) + assert "Predicted classes don't exist in the dataset" in str(e.value) @pytest.mark.parametrize('data,model,threshold,expected_metric,actual_slices_size', From a86d439f79a703efc7fcbe3bf281dbcca2cbf3ef Mon Sep 17 00:00:00 2001 From: andreybavt Date: Fri, 2 Sep 2022 17:51:22 +0200 Subject: [PATCH 16/19] GSK-254 AUC test throws error for multiclass model with small dataset improved timing --- giskard-ml-worker/logging_config.ini | 9 ++- giskard-ml-worker/ml_worker/core/model.py | 6 +- .../ml_worker/server/ml_worker_service.py | 5 +- .../ml_worker/testing/metamorphic_tests.py | 5 +- giskard-ml-worker/ml_worker/testing/utils.py | 4 +- .../ml_worker/utils/grpc_mapper.py | 15 ++-- giskard-ml-worker/ml_worker/utils/logging.py | 69 ++++++++++++++++++- giskard-ml-worker/test/fixtures/regression.py | 4 +- giskard-ml-worker/test/test_logging.py | 23 +++++++ giskard-ml-worker/test/test_utils.py | 15 ---- 10 files changed, 118 insertions(+), 37 deletions(-) create mode 100644 giskard-ml-worker/test/test_logging.py delete mode 100644 giskard-ml-worker/test/test_utils.py diff --git a/giskard-ml-worker/logging_config.ini b/giskard-ml-worker/logging_config.ini index fe50024807..9396a1b973 100644 --- a/giskard-ml-worker/logging_config.ini +++ b/giskard-ml-worker/logging_config.ini @@ -1,5 +1,5 @@ [loggers] -keys = root +keys = root,timer [handlers] keys = stream_handler @@ -11,6 +11,13 @@ keys = formatter level = DEBUG handlers = stream_handler +[logger_timer] +level = DEBUG +qualname=timer +handlers = stream_handler +propagate=0 + + [handler_stream_handler] class = StreamHandler level = DEBUG diff --git a/giskard-ml-worker/ml_worker/core/model.py b/giskard-ml-worker/ml_worker/core/model.py index af3aa52796..9ce98dc2be 100644 --- a/giskard-ml-worker/ml_worker/core/model.py +++ b/giskard-ml-worker/ml_worker/core/model.py @@ -1,4 +1,5 @@ import logging +from timeit import timeit from typing import List, Any, Optional, Callable, Iterable, Union import time @@ -8,6 +9,7 @@ from pydantic import BaseModel from ml_worker.core.giskard_dataset import GiskardDataset +from ml_worker.utils.logging import timer, Timer class ModelPredictionResults(BaseModel): @@ -32,7 +34,7 @@ def __init__(self, self.classification_labels = classification_labels def run_predict(self, dataset: GiskardDataset): - start_time = time.time() + timer = Timer() df = self.prepare_dataframe(dataset) raw_prediction = self.prediction_function(df) if self.model_type == "regression": @@ -64,7 +66,7 @@ def run_predict(self, dataset: GiskardDataset): raise ValueError( f"Prediction task is not supported: {self.model_type}" ) - logging.info(f"Predicted dataset with {dataset.df.shape} shape in {(time.time() - start_time)}s") + timer.stop(f"Predicted dataset with shape {dataset.df.shape}") return result def prepare_dataframe(self, dataset): diff --git a/giskard-ml-worker/ml_worker/server/ml_worker_service.py b/giskard-ml-worker/ml_worker/server/ml_worker_service.py index e10af0dafc..4b629d79e8 100644 --- a/giskard-ml-worker/ml_worker/server/ml_worker_service.py +++ b/giskard-ml-worker/ml_worker/server/ml_worker_service.py @@ -17,6 +17,7 @@ from ml_worker.exceptions.IllegalArgumentError import IllegalArgumentError from ml_worker.exceptions.giskard_exception import GiskardException from ml_worker.utils.grpc_mapper import deserialize_model, deserialize_dataset +from ml_worker.utils.logging import Timer logger = logging.getLogger() @@ -39,9 +40,9 @@ def runTest(self, request: RunTestRequest, context: grpc.ServicerContext) -> Tes if request.actual_ds.serialized_df: _globals['actual_ds'] = deserialize_dataset(request.actual_ds) try: - start_time = time.time() + timer = Timer() exec(request.code, _globals) - logging.info(f"Successfully executed the {tests.tests_results[0].name} test in {(time.time() - start_time)}s") + timer.stop(f"Test {tests.tests_results[0].name}") except NameError as e: missing_name = re.findall(r"name '(\w+)' is not defined", str(e))[0] if missing_name == 'reference_ds': diff --git a/giskard-ml-worker/ml_worker/testing/metamorphic_tests.py b/giskard-ml-worker/ml_worker/testing/metamorphic_tests.py index bb1557b58d..dd3e5b95ef 100644 --- a/giskard-ml-worker/ml_worker/testing/metamorphic_tests.py +++ b/giskard-ml-worker/ml_worker/testing/metamorphic_tests.py @@ -7,6 +7,7 @@ from ml_worker.core.model import GiskardModel from ml_worker.testing.abstract_test_collection import AbstractTestCollection from ml_worker.testing.utils import apply_perturbation_inplace +from ml_worker.utils.logging import timer class MetamorphicTests(AbstractTestCollection): @@ -41,9 +42,8 @@ def _perturb_and_predict(ds: GiskardDataset, model: GiskardModel, perturbation_d results_df["perturbed_prediction"] = results_df["prediction"] return results_df, len(modified_rows) + @timer("Compare and predict the data") def _compare_prediction(self, results_df, prediction_task, output_sensitivity=None, flag=None): - start_time = time.time() - if flag == 'Invariance': if prediction_task == 'classification': passed_idx = results_df.loc[ @@ -63,7 +63,6 @@ def _compare_prediction(self, results_df, prediction_task, output_sensitivity=No results_df['prediction'] > results_df['perturbed_prediction']].index.values failed_idx = results_df.loc[~results_df.index.isin(passed_idx)].index.values - logging.info(f"Compared and predicted the data in {(time.time() - start_time)}s") return passed_idx, failed_idx def _test_metamorphic(self, diff --git a/giskard-ml-worker/ml_worker/testing/utils.py b/giskard-ml-worker/ml_worker/testing/utils.py index e3356dc070..69e52d678f 100644 --- a/giskard-ml-worker/ml_worker/testing/utils.py +++ b/giskard-ml-worker/ml_worker/testing/utils.py @@ -3,6 +3,7 @@ import logging from generated.ml_worker_pb2 import SingleTestResult +from ml_worker.utils.logging import timer def ge_result_to_test_result(result, passed=True) -> SingleTestResult: @@ -27,8 +28,8 @@ def ge_result_to_test_result(result, passed=True) -> SingleTestResult: ) +@timer("Perturb data") def apply_perturbation_inplace(df: pd.DataFrame, perturbation_dict): - start_time = time.time() modified_rows = [] i = 0 for idx, r in df.iterrows(): @@ -41,5 +42,4 @@ def apply_perturbation_inplace(df: pd.DataFrame, perturbation_dict): modified_rows.append(i) df.loc[idx, pert_col] = new_value i += 1 - logging.info(f"Perturbed data in {(time.time() - start_time)}s") return modified_rows diff --git a/giskard-ml-worker/ml_worker/utils/grpc_mapper.py b/giskard-ml-worker/ml_worker/utils/grpc_mapper.py index d28b1e6ede..e8081a73fe 100644 --- a/giskard-ml-worker/ml_worker/utils/grpc_mapper.py +++ b/giskard-ml-worker/ml_worker/utils/grpc_mapper.py @@ -1,20 +1,20 @@ -from io import BytesIO import logging -import time +from io import BytesIO import cloudpickle import pandas as pd +import time from zstandard import ZstdDecompressor from zstandard import decompress +from generated.ml_worker_pb2 import SerializedGiskardModel, SerializedGiskardDataset from ml_worker.core.giskard_dataset import GiskardDataset from ml_worker.core.model import GiskardModel -from generated.ml_worker_pb2 import SerializedGiskardModel, SerializedGiskardDataset +from ml_worker.utils.logging import timer +@timer() def deserialize_model(serialized_model: SerializedGiskardModel) -> GiskardModel: - start_time = time.time() - deserialized_model = GiskardModel( cloudpickle.load(ZstdDecompressor().stream_reader(serialized_model.serialized_prediction_function)), model_type=serialized_model.model_type, @@ -22,14 +22,12 @@ def deserialize_model(serialized_model: SerializedGiskardModel) -> GiskardModel: feature_names=list(serialized_model.feature_names), classification_labels=list(serialized_model.classification_labels) ) - logging.info(f"Successfully deserialized model in {(time.time() - start_time)}s") return deserialized_model +@timer() def deserialize_dataset(serialized_dataset: SerializedGiskardDataset) -> GiskardDataset: - start_time = time.time() - deserialized_dataset = GiskardDataset( df=pd.read_csv( BytesIO(decompress(serialized_dataset.serialized_df)), @@ -40,6 +38,5 @@ def deserialize_dataset(serialized_dataset: SerializedGiskardDataset) -> Giskard feature_types=dict(serialized_dataset.feature_types), column_types=dict(serialized_dataset.column_types) ) - logging.info(f"Successfully deserialized dataset in {(time.time() - start_time)}s") return deserialized_dataset diff --git a/giskard-ml-worker/ml_worker/utils/logging.py b/giskard-ml-worker/ml_worker/utils/logging.py index abadea3bb5..e82dc861ef 100644 --- a/giskard-ml-worker/ml_worker/utils/logging.py +++ b/giskard-ml-worker/ml_worker/utils/logging.py @@ -1,6 +1,11 @@ +import logging import os -import sys +from datetime import timedelta +from functools import wraps from logging.config import fileConfig +from timeit import default_timer + +import sys def resolve(filename): @@ -21,3 +26,65 @@ def load_logging_config(): print(f"Failed to load logging config from {config_path}") else: fileConfig(resolve('logging_config.ini')) + + +timer_logger = logging.getLogger("timer") + + +class Timer: + message_template: str + + def __init__(self, message=None, start=True, level=logging.INFO) -> None: + if start: + self.start() + else: + self.__start_time = None + self.duration = None + self.message_template = message + self.message = None + self.level = level + + def start(self): + self.__start_time = default_timer() + + def stop(self, message=None, log=True): + if message: + self.message_template = message + if not self.__start_time: + timer_logger.error("Timer was not started") + return + + self.duration = timedelta(seconds=default_timer() - self.__start_time) + if log: + self.message = self.create_message(self.duration) + timer_logger.log(self.level, self.message) + return self.duration + + def __enter__(self): + if not self.__start_time: + self.start() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + + def create_message(self, duration): + if self.message_template: + return self.message_template.strip() + f" executed in {duration}" + else: + return f"Executed in {duration}" + + def prepare_message_template(self): + return self.message_template + + +def timer(message=None): + def timing_decorator(f): + @wraps(timing_decorator) + def wrap(*args, **kw): + with Timer(message if message else f'{f.__name__}'): + result = f(*args, **kw) + return result + + return wrap + + return timing_decorator diff --git a/giskard-ml-worker/test/fixtures/regression.py b/giskard-ml-worker/test/fixtures/regression.py index c390da43c6..f3accf03a5 100644 --- a/giskard-ml-worker/test/fixtures/regression.py +++ b/giskard-ml-worker/test/fixtures/regression.py @@ -6,7 +6,7 @@ from ml_worker.core.giskard_dataset import GiskardDataset from ml_worker.core.model import GiskardModel -from test.test_utils import timing +from ml_worker.utils.logging import timer @pytest.fixture() @@ -32,7 +32,7 @@ def diabetes_dataset_with_target(): @pytest.fixture() -@timing +@timer def linear_regression_diabetes() -> GiskardModel: diabetes = datasets.load_diabetes() diff --git a/giskard-ml-worker/test/test_logging.py b/giskard-ml-worker/test/test_logging.py new file mode 100644 index 0000000000..e7158ce66e --- /dev/null +++ b/giskard-ml-worker/test/test_logging.py @@ -0,0 +1,23 @@ +import time + +from ml_worker.utils.logging import Timer, timer + + +def test_timer(): + timer = Timer() + time.sleep(0.1) + duration = timer.stop() + assert duration.microseconds / 10 ** 6 >= 0.1 + + +@timer() +def wait(): + time.sleep(0.1) + + +def test_with_timer(): + wait() + # timer = Timer("Sleeping for {}") + # with timer: + # time.sleep(0.1) + # assert timer.message is not None diff --git a/giskard-ml-worker/test/test_utils.py b/giskard-ml-worker/test/test_utils.py deleted file mode 100644 index 2ae1f1d622..0000000000 --- a/giskard-ml-worker/test/test_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging -from functools import wraps -from time import time - - -def timing(f): - @wraps(f) - def wrap(*args, **kw): - ts = time() - result = f(*args, **kw) - te = time() - logging.info(f'func:{f.__name__} args:[{args},{kw}] took: {te - ts:.2f}s') - return result - - return wrap From 46c7b577f9b89c1472094979496370725073d0ea Mon Sep 17 00:00:00 2001 From: andreybavt Date: Fri, 2 Sep 2022 17:52:28 +0200 Subject: [PATCH 17/19] cleanup --- giskard-ml-worker/test/test_logging.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/giskard-ml-worker/test/test_logging.py b/giskard-ml-worker/test/test_logging.py index e7158ce66e..33ff4fd222 100644 --- a/giskard-ml-worker/test/test_logging.py +++ b/giskard-ml-worker/test/test_logging.py @@ -17,7 +17,3 @@ def wait(): def test_with_timer(): wait() - # timer = Timer("Sleeping for {}") - # with timer: - # time.sleep(0.1) - # assert timer.message is not None From 313b890f644fe36288c1d2fd022b93f9b2ad262d Mon Sep 17 00:00:00 2001 From: andreybavt Date: Fri, 2 Sep 2022 20:32:02 +0200 Subject: [PATCH 18/19] fix tests --- giskard-ml-worker/ml_worker/utils/logging.py | 8 +- .../enron_multilabel_classification.py | 6 +- .../test/fixtures/german_credit_scoring.py | 77 +++++++++---------- giskard-ml-worker/test/fixtures/regression.py | 52 ++++++------- giskard-ml-worker/test/test_data_drift.py | 10 ++- giskard-ml-worker/test/test_logging.py | 11 +-- 6 files changed, 78 insertions(+), 86 deletions(-) diff --git a/giskard-ml-worker/ml_worker/utils/logging.py b/giskard-ml-worker/ml_worker/utils/logging.py index e82dc861ef..97038d382b 100644 --- a/giskard-ml-worker/ml_worker/utils/logging.py +++ b/giskard-ml-worker/ml_worker/utils/logging.py @@ -4,6 +4,7 @@ from functools import wraps from logging.config import fileConfig from timeit import default_timer +import inspect import sys @@ -78,11 +79,12 @@ def prepare_message_template(self): def timer(message=None): - def timing_decorator(f): + @wraps(timer) + def timing_decorator(fn): @wraps(timing_decorator) def wrap(*args, **kw): - with Timer(message if message else f'{f.__name__}'): - result = f(*args, **kw) + with Timer(message if message else f'{fn.__module__}.{fn.__qualname__}'): + result = fn(*args, **kw) return result return wrap diff --git a/giskard-ml-worker/test/fixtures/enron_multilabel_classification.py b/giskard-ml-worker/test/fixtures/enron_multilabel_classification.py index 0cdc5323be..3f237bdbee 100644 --- a/giskard-ml-worker/test/fixtures/enron_multilabel_classification.py +++ b/giskard-ml-worker/test/fixtures/enron_multilabel_classification.py @@ -15,6 +15,7 @@ from ml_worker.core.giskard_dataset import GiskardDataset from ml_worker.core.model import GiskardModel +from ml_worker.utils.logging import timer, Timer from test import path input_types = { @@ -51,7 +52,7 @@ def enron_test_data(enron_data): @pytest.fixture() def enron_model(enron_data) -> GiskardModel: - start = time.time() + timer = Timer() columns_to_scale = [key for key in input_types.keys() if input_types[key] == "numeric"] @@ -87,9 +88,8 @@ def enron_model(enron_data) -> GiskardModel: stratify=Y) clf.fit(X_train, Y_train) - train_time = time.time() - start model_score = clf.score(X_test, Y_test) - logging.info(f"Trained model with score: {model_score} in {round(train_time * 1000)} ms") + timer.stop(f"Trained model with score: {model_score}") return GiskardModel( prediction_function=clf.predict_proba, diff --git a/giskard-ml-worker/test/fixtures/german_credit_scoring.py b/giskard-ml-worker/test/fixtures/german_credit_scoring.py index 9ffc393190..519eccb33e 100644 --- a/giskard-ml-worker/test/fixtures/german_credit_scoring.py +++ b/giskard-ml-worker/test/fixtures/german_credit_scoring.py @@ -13,6 +13,7 @@ from ml_worker.core.giskard_dataset import GiskardDataset from ml_worker.core.model import GiskardModel +from ml_worker.utils.logging import timer, Timer from test import path input_types = {'account_check_status': "category", @@ -50,6 +51,41 @@ def german_credit_data() -> GiskardDataset: ) +@pytest.fixture() +def german_credit_catboost(german_credit_data) -> GiskardModel: + from catboost import CatBoostClassifier + from sklearn import model_selection + + timer = Timer() + feature_types = {i: input_types[i] for i in input_types if i != 'default'} + + columns_to_encode = [key for key in feature_types.keys() if feature_types[key] == "category"] + + credit = german_credit_data.df + + Y = credit['default'] + X = credit.drop(columns="default") + X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, + test_size=0.20, + random_state=30, + stratify=Y) + cb = CatBoostClassifier(iterations=2, + learning_rate=1, + depth=2) + cb.fit(X_train, Y_train, columns_to_encode) + + model_score = cb.score(X_test, Y_test) + timer.stop(f"Trained model with score: {model_score}") + + return GiskardModel( + prediction_function=cb.predict_proba, + model_type='classification', + feature_names=list(input_types), + classification_threshold=0.5, + classification_labels=cb.classes_ + ) + + @pytest.fixture() def german_credit_test_data(german_credit_data): return GiskardDataset( @@ -61,7 +97,7 @@ def german_credit_test_data(german_credit_data): @pytest.fixture() def german_credit_model(german_credit_data) -> GiskardModel: - start = time.time() + timer = Timer() columns_to_scale = [key for key in input_types.keys() if input_types[key] == "numeric"] @@ -91,9 +127,8 @@ def german_credit_model(german_credit_data) -> GiskardModel: stratify=Y) clf.fit(X_train, Y_train) - train_time = time.time() - start model_score = clf.score(X_test, Y_test) - logging.info(f"Trained model with score: {model_score} in {round(train_time * 1000)} ms") + timer.stop(f"Trained model with score: {model_score}") return GiskardModel( prediction_function=clf.predict_proba, @@ -102,39 +137,3 @@ def german_credit_model(german_credit_data) -> GiskardModel: classification_threshold=0.5, classification_labels=clf.classes_ ) - - -@pytest.fixture() -def german_credit_catboost(german_credit_data) -> GiskardModel: - from catboost import CatBoostClassifier - from sklearn import model_selection - - start = time.time() - feature_types = {i: input_types[i] for i in input_types if i != 'default'} - - columns_to_encode = [key for key in feature_types.keys() if feature_types[key] == "category"] - - credit = german_credit_data.df - - Y = credit['default'] - X = credit.drop(columns="default") - X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, - test_size=0.20, - random_state=30, - stratify=Y) - cb = CatBoostClassifier(iterations=2, - learning_rate=1, - depth=2) - cb.fit(X_train, Y_train, columns_to_encode) - - train_time = time.time() - start - model_score = cb.score(X_test, Y_test) - logging.info(f"Trained model with score: {model_score} in {round(train_time * 1000)} ms") - - return GiskardModel( - prediction_function=cb.predict_proba, - model_type='classification', - feature_names=list(input_types), - classification_threshold=0.5, - classification_labels=cb.classes_ - ) diff --git a/giskard-ml-worker/test/fixtures/regression.py b/giskard-ml-worker/test/fixtures/regression.py index f3accf03a5..a035e5c6eb 100644 --- a/giskard-ml-worker/test/fixtures/regression.py +++ b/giskard-ml-worker/test/fixtures/regression.py @@ -1,39 +1,15 @@ -import logging - import pytest from sklearn import datasets, linear_model from sklearn.metrics import mean_squared_error from ml_worker.core.giskard_dataset import GiskardDataset from ml_worker.core.model import GiskardModel -from ml_worker.utils.logging import timer - - -@pytest.fixture() -def diabetes_dataset(): - diabetes = datasets.load_diabetes() - return GiskardDataset( - df=datasets.load_diabetes(as_frame=True)['data'], - feature_types={feature: 'numeric' for feature in diabetes['feature_names']}, - target='target' - ) - - -@pytest.fixture() -def diabetes_dataset_with_target(): - loaded = datasets.load_diabetes(as_frame=True) - data = loaded['data'] - data['target'] = loaded['target'] - return GiskardDataset( - df=data, - feature_types={feature: 'numeric' for feature in list(data.columns)}, - target='target' - ) +from ml_worker.utils.logging import Timer @pytest.fixture() -@timer def linear_regression_diabetes() -> GiskardModel: + timer = Timer() diabetes = datasets.load_diabetes() diabetes_x = diabetes['data'] @@ -56,10 +32,32 @@ def linear_regression_diabetes() -> GiskardModel: # Make predictions using the testing set diabetes_y_pred = regressor.predict(diabetes_x_test) - logging.info(f"Model MSE: {mean_squared_error(diabetes_y_test, diabetes_y_pred)}") + timer.stop(f"Model MSE: {mean_squared_error(diabetes_y_test, diabetes_y_pred)}") return GiskardModel( prediction_function=regressor.predict, model_type='regression', feature_names=diabetes['feature_names'], ) + + +@pytest.fixture() +def diabetes_dataset(): + diabetes = datasets.load_diabetes() + return GiskardDataset( + df=datasets.load_diabetes(as_frame=True)['data'], + feature_types={feature: 'numeric' for feature in diabetes['feature_names']}, + target='target' + ) + + +@pytest.fixture() +def diabetes_dataset_with_target(): + loaded = datasets.load_diabetes(as_frame=True) + data = loaded['data'] + data['target'] = loaded['target'] + return GiskardDataset( + df=data, + feature_types={feature: 'numeric' for feature in list(data.columns)}, + target='target' + ) diff --git a/giskard-ml-worker/test/test_data_drift.py b/giskard-ml-worker/test/test_data_drift.py index 3d82e64571..8f0c06677c 100644 --- a/giskard-ml-worker/test/test_data_drift.py +++ b/giskard-ml-worker/test/test_data_drift.py @@ -135,16 +135,18 @@ def test_drift_data_earth_movers_distance(data, threshold, expected_metric, colu assert results.passed -@pytest.mark.parametrize('data,model,threshold,expected_metric', - [('german_credit_data', 'german_credit_model', 1, 0.02), - ('enron_data', 'enron_model', 2, 1.36)]) +@pytest.mark.parametrize('data,model,threshold,expected_metric', [ + ('german_credit_data', 'german_credit_model', 1, 0.02), + ('enron_data', 'enron_model', 2, 1.36) +]) def test_drift_prediction_psi(data, model, threshold, expected_metric, request): tests = GiskardTestFunctions() data = request.getfixturevalue(data) + model = request.getfixturevalue(model) results = tests.drift.test_drift_prediction_psi( reference_slice=data.slice(lambda df: df.head(len(df) // 2)), actual_slice=data.slice(lambda df: df.tail(len(df) // 2)), - model=request.getfixturevalue(model), + model=model, threshold=threshold) assert round(results.metric, 2) == expected_metric diff --git a/giskard-ml-worker/test/test_logging.py b/giskard-ml-worker/test/test_logging.py index 33ff4fd222..808db676c7 100644 --- a/giskard-ml-worker/test/test_logging.py +++ b/giskard-ml-worker/test/test_logging.py @@ -1,6 +1,6 @@ import time -from ml_worker.utils.logging import Timer, timer +from ml_worker.utils.logging import Timer def test_timer(): @@ -8,12 +8,3 @@ def test_timer(): time.sleep(0.1) duration = timer.stop() assert duration.microseconds / 10 ** 6 >= 0.1 - - -@timer() -def wait(): - time.sleep(0.1) - - -def test_with_timer(): - wait() From ae5d7295bbc4f50362bbfbbf069d4fbd8eab8748 Mon Sep 17 00:00:00 2001 From: andreybavt Date: Fri, 2 Sep 2022 20:32:24 +0200 Subject: [PATCH 19/19] fix tests --- giskard-ml-worker/ml_worker/utils/logging.py | 1 - 1 file changed, 1 deletion(-) diff --git a/giskard-ml-worker/ml_worker/utils/logging.py b/giskard-ml-worker/ml_worker/utils/logging.py index 97038d382b..f61d3acaa3 100644 --- a/giskard-ml-worker/ml_worker/utils/logging.py +++ b/giskard-ml-worker/ml_worker/utils/logging.py @@ -4,7 +4,6 @@ from functools import wraps from logging.config import fileConfig from timeit import default_timer -import inspect import sys