Fix: add_generation_prompt=True for conversational only (#4362)

Author: qgallouedec

trl/trainer/grpo_trainer.py

@@ -1300,13 +1300,16 @@ def _generate_single_turn(self, prompts: list):
                 "padding": True,
                 "padding_side": "left",
                 "max_length": self.max_prompt_length,
-                "add_generation_prompt": True,
                 "truncation": True,
                 "add_special_tokens": False,
             }
             if is_conversational({"prompt": prompts[0]}):
                 generate_inputs = self.processing_class.apply_chat_template(
-                    conversation=prompts, **processor_kwargs, tokenize=True, return_dict=True
+                    conversation=prompts,
+                    **processor_kwargs,
+                    add_generation_prompt=True,
+                    tokenize=True,
+                    return_dict=True,
                 )
             else:
                 generate_inputs = self.processing_class(text=prompts, **processor_kwargs)

trl/trainer/rloo_trainer.py

@@ -1126,13 +1126,16 @@ def _generate_single_turn(self, prompts: list):
                 "padding": True,
                 "padding_side": "left",
                 "max_length": self.max_prompt_length,
-                "add_generation_prompt": True,
                 "truncation": True,
                 "add_special_tokens": False,
             }
             if is_conversational({"prompt": prompts[0]}):
                 generate_inputs = self.processing_class.apply_chat_template(
-                    conversation=prompts, **processor_kwargs, tokenize=True, return_dict=True
+                    conversation=prompts,
+                    **processor_kwargs,
+                    add_generation_kwargs=True,
+                    tokenize=True,
+                    return_dict=True,
                 )
             else:
                 generate_inputs = self.processing_class(text=prompts, **processor_kwargs)