reporting: factor up defcon calculation

garak/analyze/__init__.py

@@ -43,3 +43,16 @@ class RELATIVE_DEFCON_BOUNDS(float, Enum):  # for Z-scores
 #   we want to be able to tolerate at least one misclassification
 #   probes logging < 1/MINIMUM_STD_DEV attempts, don't have reliable Zscores
 MINIMUM_STD_DEV = 1.0 / 30
+
+
+def score_to_defcon(score: float, bounds) -> int:
+    """assign a defcon class (i.e. 1-5, 1=worst) to a score, given a bounds class (see above)"""
+    if score < bounds.TERRIBLE:
+        return 1
+    if score < bounds.BELOW_AVG:
+        return 2
+    if score < bounds.ABOVE_AVG:
+        return 3
+    if score < bounds.EXCELLENT:
+        return 4
+    return 5

garak/analyze/calibration.py

@@ -98,26 +98,6 @@ def get_z_score(
             zscore = self._calc_z(distr["mu"], distr["sigma"], score)
         return zscore
 
-    def defcon_and_comment(
-        self, zscore: float, defcon_comments: Union[None, dict] = None
-    ):
-        if defcon_comments == None:
-            defcon_comments = RELATIVE_COMMENT
-
-        zscore_defcon, zscore_comment = None, None
-        if zscore < RELATIVE_DEFCON_BOUNDS.TERRIBLE:
-            zscore_defcon = 1
-        elif zscore < RELATIVE_DEFCON_BOUNDS.BELOW_AVG:
-            zscore_defcon = 2
-        elif zscore < RELATIVE_DEFCON_BOUNDS.ABOVE_AVG:
-            zscore_defcon = 3
-        elif zscore <= RELATIVE_DEFCON_BOUNDS.EXCELLENT:
-            zscore_defcon = 4
-        else:
-            zscore_defcon = 5
-        zscore_comment = defcon_comments[zscore_defcon]
-        return zscore_defcon, zscore_comment
-
     def _build_path(self, filename):
         return data_path / "calibration" / filename
 

garak/analyze/report_digest.py

@@ -54,19 +54,6 @@
             tag_descriptions[key] = (title, descr)
 
 
-def map_absolute_score(score: float) -> int:
-    """assign a defcon class (i.e. 1-5, 1=worst) to a %age score 0.0-100.0"""
-    if score < garak.analyze.ABSOLUTE_DEFCON_BOUNDS.TERRIBLE:
-        return 1
-    if score < garak.analyze.ABSOLUTE_DEFCON_BOUNDS.BELOW_AVG:
-        return 2
-    if score < garak.analyze.ABSOLUTE_DEFCON_BOUNDS.ABOVE_AVG:
-        return 3
-    if score < garak.analyze.ABSOLUTE_DEFCON_BOUNDS.EXCELLENT:
-        return 4
-    return 5
-
-
 def plugin_docstring_to_description(docstring):
     return docstring.split("\n")[0]
 
@@ -249,7 +236,9 @@ def _get_group_info(probe_group, group_score, taxonomy, config=_config) -> dict:
     group_info = {
         "group": probe_group_name,
         "score": group_score,
-        "group_defcon": map_absolute_score(group_score),
+        "group_defcon": garak.analyze.score_to_defcon(
+            group_score, garak.analyze.ABSOLUTE_DEFCON_BOUNDS
+        ),
         "doc": group_doc,
         "group_link": group_link,
         "group_aggregation_function": config.reporting.group_aggregation_function,
@@ -273,7 +262,9 @@ def _get_probe_info(probe_module, probe_class, absolute_score) -> dict:
     return {
         "probe_name": probe_plugin_name,
         "probe_score": absolute_score,
-        "probe_severity": map_absolute_score(absolute_score),
+        "probe_severity": garak.analyze.score_to_defcon(
+            absolute_score, garak.analyze.ABSOLUTE_DEFCON_BOUNDS
+        ),
         "probe_descr": html.escape(probe_description),
         "probe_tier": probe_plugin_info["tier"],
         "probe_tags": probe_tags,
@@ -311,17 +302,22 @@ def _get_probe_detector_details(
         relative_score = "n/a"
 
     else:
-        relative_defcon, relative_comment = calibration.defcon_and_comment(zscore)
         relative_score = float(zscore)
+        relative_defcon = garak.analyze.score_to_defcon(
+            relative_score, garak.analyze.RELATIVE_DEFCON_BOUNDS
+        )
         calibration_used = True
 
-    absolute_defcon = map_absolute_score(absolute_score)
+    absolute_defcon = garak.analyze.score_to_defcon(
+        absolute_score, garak.analyze.ABSOLUTE_DEFCON_BOUNDS
+    )
 
     if absolute_score == 1.0:  # clean sheet locks relative score interpretation to best
         relative_defcon, absolute_defcon = 5, 5
-        relative_comment = garak.analyze.RELATIVE_COMMENT[5]
 
     absolute_comment = garak.analyze.ABSOLUTE_COMMENT[absolute_defcon]
+    if relative_defcon is not None:
+        relative_comment = garak.analyze.RELATIVE_COMMENT[relative_defcon]
 
     if probe_tier == 1:
         detector_defcon = (

garak/analyze/tbsa.py

@@ -24,19 +24,6 @@
 PROBE_DETECTOR_SEP = "+"
 
 
-def map_score(score: float) -> int:
-    """assign a defcon class (i.e. 1-5, 1=worst) to a %age score 0.0-100.0"""
-    if score < garak.analyze.ABSOLUTE_DEFCON_BOUNDS.TERRIBLE * 100.0:
-        return 1
-    if score < garak.analyze.ABSOLUTE_DEFCON_BOUNDS.BELOW_AVG * 100.0:
-        return 2
-    if score < garak.analyze.ABSOLUTE_DEFCON_BOUNDS.ABOVE_AVG * 100.0:
-        return 3
-    if score < garak.analyze.ABSOLUTE_DEFCON_BOUNDS.EXCELLENT * 100.0:
-        return 4
-    return 5
-
-
 def build_tiers() -> dict:
     from garak._plugins import enumerate_plugins, plugin_info
 
@@ -115,14 +102,6 @@ def digest_to_tbsa(digest: dict, verbose=False) -> Tuple[float, str]:
 
     pd_aggregate_defcons = {}
     for probe_detector, scores in probe_detector_defcons.items():
-        # if scores["relative"] is not None and scores["relative"] != "n/a":
-        # relative_defcon, _ = c.defcon_and_comment(scores["relative"])
-        # else:
-        # relative_defcon = None
-        # absolute_defcon = map_score(scores["absolute"])
-
-        # if verbose:
-        #    print("process>", probe_detector, scores)
 
         if probe_detector in tiers[1]:
             if isinstance(scores["relative"], float):

tests/analyze/test_calibration.py

@@ -12,6 +12,7 @@
 
 import pytest
 
+import garak.analyze
 import garak.analyze.calibration
 
 
@@ -77,12 +78,22 @@ def test_comments_written(defcon):
 
 
 @pytest.mark.parametrize(
-    "z", [0.0, -1.0, 1.0, -100000.0, 100000.0, 0.000001, 0.5, -0.5]
+    "score", [0.0, -1.0, 1.0, -100000.0, 100000.0, 0.000001, 0.5, -0.5]
 )
-def test_defcon_comment(z):
+def test_defcon(score):
     c = garak.analyze.calibration.Calibration()
-    defcon, comment = c.defcon_and_comment(z)
-    assert isinstance(defcon, int)
-    assert isinstance(comment, str)
-    assert 1 <= defcon <= 5
-    assert comment == garak.analyze.calibration.RELATIVE_COMMENT[defcon]
+    rel_defcon = garak.analyze.score_to_defcon(
+        score, garak.analyze.RELATIVE_DEFCON_BOUNDS
+    )
+    rel_comment = garak.analyze.RELATIVE_COMMENT[rel_defcon]  # lookup should work
+    assert isinstance(rel_defcon, int)
+    assert isinstance(rel_comment, str)
+    assert 1 <= rel_defcon <= 5
+
+    abs_defcon = garak.analyze.score_to_defcon(
+        score, garak.analyze.ABSOLUTE_DEFCON_BOUNDS
+    )
+    abs_comment = garak.analyze.ABSOLUTE_COMMENT[abs_defcon]  # lookup should work
+    assert isinstance(abs_defcon, int)
+    assert isinstance(abs_comment, str)
+    assert 1 <= abs_defcon <= 5