Skip to content

[GSK-1812] Fix wrong descriptions of performance issues #1431

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 28, 2023
Merged

[GSK-1812] Fix wrong descriptions of performance issues #1431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions python-client/giskard/scanner/performance/performance_bias_detector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,7 @@ def _detect_for_metric(

if is_issue:
level = IssueLevel.MAJOR if abs(relative_delta) > 2 * self.threshold else IssueLevel.MEDIUM
desc = (
"For records in your dataset where {slicing_fn}, the recall is 96.8% lower than the global recall."
)
desc = "For records in your dataset where {slicing_fn}, the {metric} is {abs_deviation_perc}% {comparison_op} than the global {metric}."

issue = Issue(
model=model,
Expand All @@ -229,6 +227,9 @@ def _detect_for_metric(
"slice_metric": slice_metric,
"reference_metric": ref_metric,
"deviation": f"{relative_delta*100:+.2f}% than global",
"deviation_perc": round(relative_delta * 100, 2),
"abs_deviation_perc": round(abs(relative_delta) * 100, 2),
"comparison_op": "lower" if metric.greater_is_better else "greater",
"slice_size": len(sliced_dataset),
"threshold": self.threshold,
"p_value": p_value,
Expand Down
13 changes: 12 additions & 1 deletion python-client/tests/scan/test_performance_bias_detector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import logging
from unittest import mock

import numpy as np
import pandas as pd
import pytest
from unittest import mock

from giskard import Dataset
from giskard.scanner.issues import Issue
Expand Down Expand Up @@ -58,6 +59,16 @@ def test_performance_bias_detector_with_tabular(german_credit_model, german_cred
assert len(issues) > 0
assert all([isinstance(issue, Issue) for issue in issues])

# Check that descriptions are coherent
assert (
issues[0].description
== 'For records in your dataset where `purpose` == "business", the Precision is 5.33% lower than the global Precision.'
)
for issue in issues:
assert issue.meta["metric"] in issue.description
assert f'{issue.meta["abs_deviation_perc"]}%' in issue.description
assert str(issue.slicing_fn) in issue.description


@pytest.mark.slow
def test_performance_bias_detector_with_text_features(enron_model, enron_data):
Expand Down