improve prompt translation support for Conversation related types (#1441)

Author: jmartin-tech

garak/probes/base.py

@@ -72,24 +72,29 @@ def __init__(self, config_root=_config):
         """
         self._load_config(config_root)
         self.probename = str(self.__class__).split("'")[1]
-        
+
         # Handle deprecated recommended_detector migration
         if (
             self.primary_detector is None
             and self.recommended_detector != ["always.Fail"]
             and len(self.recommended_detector) > 0
         ):
             from garak import command
+
             command.deprecation_notice(
                 f"recommended_detector in probe {self.probename}",
                 "0.9.0.6",
                 logging=logging,
             )
             self.primary_detector = self.recommended_detector[0]
             if len(self.recommended_detector) > 1:
-                existing_extended = list(self.extended_detectors) if self.extended_detectors else []
-                self.extended_detectors = existing_extended + list(self.recommended_detector[1:])
-        
+                existing_extended = (
+                    list(self.extended_detectors) if self.extended_detectors else []
+                )
+                self.extended_detectors = existing_extended + list(
+                    self.recommended_detector[1:]
+                )
+
         if hasattr(_config.system, "verbose") and _config.system.verbose > 0:
             print(
                 f"loading {Style.BRIGHT}{Fore.LIGHTYELLOW_EX}probe: {Style.RESET_ALL}{self.probename}"
@@ -362,11 +367,9 @@ def probe(self, generator) -> Iterable[garak.attempt.Attempt]:
 
         # build list of attempts
         attempts_todo: Iterable[garak.attempt.Attempt] = []
-        prompts = list(
+        prompts = copy.deepcopy(
             self.prompts
-        )  # will this still make a copy if prompts are `Message` objects?
-        lang = self.lang
-        # account for visual jailbreak until Turn/Conversation is supported
+        )  # make a copy to avoid mutating source list
         preparation_bar = tqdm.tqdm(
             total=len(prompts),
             leave=False,
@@ -387,35 +390,47 @@ def probe(self, generator) -> Iterable[garak.attempt.Attempt]:
             for prompt in prompts:
                 if isinstance(prompt, garak.attempt.Message):
                     prompt.text = self.langprovider.get_text(
-                        prompt.text, notify_callback=preparation_bar.update
-                    )
+                        [prompt.text], notify_callback=preparation_bar.update
+                    )[0]
                     prompt.lang = self.langprovider.target_lang
                 if isinstance(prompt, garak.attempt.Conversation):
                     for turn in prompt.turns:
                         msg = turn.content
                         msg.text = self.langprovider.get_text(
-                            msg.text, notify_callback=preparation_bar.update
-                        )
+                            [msg.text], notify_callback=preparation_bar.update
+                        )[0]
                         msg.lang = self.langprovider.target_lang
         lang = self.langprovider.target_lang
         preparation_bar.close()
         for seq, prompt in enumerate(prompts):
-            notes = (
-                {
-                    "pre_translation_prompt": garak.attempt.Conversation(
-                        [
-                            garak.attempt.Turn(
-                                "user",
-                                garak.attempt.Message(
-                                    self.prompts[seq], lang=self.lang
-                                ),
-                            )
-                        ]
-                    )
-                }
-                if lang != self.lang
-                else None
-            )
+            notes = None
+            if lang != self.lang:
+                pre_translation_prompt = copy.deepcopy(self.prompts[seq])
+                if isinstance(pre_translation_prompt, str):
+                    notes = {
+                        "pre_translation_prompt": garak.attempt.Conversation(
+                            [
+                                garak.attempt.Turn(
+                                    "user",
+                                    garak.attempt.Message(
+                                        pre_translation_prompt, lang=self.lang
+                                    ),
+                                )
+                            ]
+                        )
+                    }
+                elif isinstance(pre_translation_prompt, garak.attempt.Message):
+                    pre_translation_prompt.lang = self.lang
+                    notes = {
+                        "pre_translation_prompt": garak.attempt.Conversation(
+                            [pre_translation_prompt]
+                        )
+                    }
+                elif isinstance(pre_translation_prompt, garak.attempt.Message):
+                    for turn in pre_translation_prompt.turns:
+                        turn.context.lang = self.lang
+                    notes = {"pre_translation_prompt": pre_translation_prompt}
+
             attempts_todo.append(self._mint_attempt(prompt, seq, notes, lang))
 
         # buff hook

tests/langservice/probes/test_probes_base.py

@@ -240,6 +240,8 @@ def test_multi_modal_probe_translation(classname, mocker):
         expected_provision_calls += len(probe_instance.attempt_descrs) * 2
 
     assert prompt_mock.call_count == expected_provision_calls
+    for prompt in probe_instance.prompts:
+        assert isinstance(prompt.text, str)
 
 
 @pytest.mark.parametrize("classname", PROBES)