Merge branch 'task/fix-scan-metrics' of github.com:Giskard-AI/giskard into task/fix-scan-metrics

Author: mattbit

python-client/giskard/core/model_validation.py

@@ -72,7 +72,7 @@ def _do_validate_model(model: BaseModel, validate_ds: Optional[Dataset] = None):
         else:  # Classification with target = None
             validate_model_execution(model, validate_ds)
 
-        if model.meta.model_type == SupportedModelTypes.CLASSIFICATION:
+        if model.meta.model_type == SupportedModelTypes.CLASSIFICATION and validate_ds.target is not None:
             validate_order_classifcation_labels(model, validate_ds)
 
 

python-client/giskard/scanner/calibration/underconfidence_detector.py

@@ -14,6 +14,8 @@
 
 @detector(name="underconfidence", tags=["underconfidence", "classification"])
 class UnderconfidenceDetector(LossBasedDetector):
+    _needs_target = False
+
     def __init__(self, threshold=0.1, p_threshold=0.95, method="tree"):
         self.threshold = threshold
         self.p_threshold = p_threshold

python-client/giskard/scanner/common/examples.py

@@ -22,7 +22,9 @@ def get_examples_dataframe(self, n=3, with_prediction: Union[int, bool] = 1):
         examples = dataset.df.copy()
 
         # Keep only interesting columns
-        cols_to_show = issue.features + [issue.dataset.target]
+        cols_to_show = issue.features
+        if issue.dataset.target is not None:
+            cols_to_show += [issue.dataset.target]
         examples = examples.loc[:, cols_to_show]
 
         # If metadata slice, add the metadata column
@@ -54,7 +56,10 @@ def get_examples_dataframe(self, n=3, with_prediction: Union[int, bool] = 1):
             else:
                 pred_examples = model_pred.prediction
 
-            examples[f"Predicted `{issue.dataset.target}`"] = pred_examples
+            predicted_label = "Predicted"
+            if issue.dataset.target is not None:
+                predicted_label += f" `{issue.dataset.target}`"
+            examples[predicted_label] = pred_examples
 
         n = min(len(examples), n)
         if n > 0:

python-client/giskard/scanner/common/loss_based_detector.py

@@ -20,7 +20,13 @@ class LossBasedDetector(Detector):
     MAX_DATASET_SIZE = 10_000_000
     LOSS_COLUMN_NAME = "__gsk__loss"
 
+    _needs_target = True
+
     def run(self, model: BaseModel, dataset: Dataset):
+        if self._needs_target and dataset.target is None:
+            logger.info(f"{self.__class__.__name__}: Skipping detection because the dataset has no target column.")
+            return []
+
         logger.info(f"{self.__class__.__name__}: Running")
 
         # Check if we have enough data to run the scan

python-client/giskard/scanner/correlation/spurious_correlation_detector.py

@@ -36,12 +36,14 @@ def run(self, model: BaseModel, dataset: Dataset):
 
         # Prepare dataset for slicing
         df = dataset.df.copy()
-        df[dataset.target] = pd.Categorical(df[dataset.target])
-        wdata = Dataset(df, column_types=dataset.column_types)
+        if dataset.target is not None:
+            df.drop(columns=dataset.target, inplace=True)
+        df["__gsk__target"] = pd.Categorical(ds_predictions)
+        wdata = Dataset(df, target="__gsk__target", column_types=dataset.column_types)
         wdata.load_metadata_from_instance(dataset.column_meta)
 
         # Find slices
-        sliced_cols = SliceFinder("tree").run(wdata, features, target=dataset.target)
+        sliced_cols = SliceFinder("tree").run(wdata, features, target=wdata.target)
 
         measure_fn, measure_name = self._get_measure_fn()
         issues = []

python-client/tests/scan/test_scanner.py

@@ -78,6 +78,16 @@ def test_scanner_raises_exception_if_no_detectors_available(german_credit_data,
         scanner.analyze(german_credit_model, german_credit_data)
 
 
+def test_scanner_works_if_dataset_has_no_target(titanic_model, titanic_dataset):
+    scanner = Scanner()
+    no_target_dataset = Dataset(titanic_dataset.df, target=None)
+    result = scanner.analyze(titanic_model, no_target_dataset, raise_exceptions=True)
+
+    assert isinstance(result, ScanResult)
+    assert result.has_issues()
+    assert result.to_html()
+
+
 def test_scan_raises_exception_if_no_dataset_provided(german_credit_model):
     scanner = Scanner()
     with pytest.raises(ValueError) as info: