[recipe] fix: fix issue when running split ppo (verl-project#2745)

Author: sunyi0505

examples/split_placement/config/ppo_trainer_split.yaml

@@ -2,7 +2,7 @@
 
 hydra:
   searchpath:
-    - file://verl/trainer/config
+    - file://../../verl/trainer/config
 
 defaults:
   - ppo_trainer

examples/split_placement/split_monkey_patch.py

@@ -169,6 +169,7 @@ def fit(self):
                         lam=self.config.algorithm.lam,
                         num_repeat=self.config.actor_rollout_ref.rollout.n,
                         norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo,
+                        config=self.config.algorithm,
                     )
 
                 # implement critic warmup

verl/workers/critic/dp_critic.py

@@ -160,7 +160,11 @@ def compute_values(self, data: DataProto) -> torch.Tensor:
         micro_batch_size = data.meta_info["micro_batch_size"]
         use_dynamic_bsz = data.meta_info["use_dynamic_bsz"]
         has_multi_modal_inputs = "multi_modal_inputs" in data.non_tensor_batch.keys()
-        select_keys = ["responses", "input_ids", "response_mask", "attention_mask", "position_ids"]
+        select_keys = (
+            ["responses", "input_ids", "response_mask", "attention_mask", "position_ids"]
+            if "response_mask" in data.batch
+            else ["responses", "input_ids", "attention_mask", "position_ids"]
+        )
         non_tensor_select_keys = ["multi_modal_inputs"] if has_multi_modal_inputs else []
 
         data = data.select(batch_keys=select_keys, non_tensor_batch_keys=non_tensor_select_keys)
@@ -182,8 +186,9 @@ def compute_values(self, data: DataProto) -> torch.Tensor:
         if use_dynamic_bsz:
             values = restore_dynamic_batch(values, batch_idx_list)
 
-        response_mask = data.batch["response_mask"]
-        values = values * response_mask  # Only action tokens have values
+        if "response_mask" in data.batch:
+            response_mask = data.batch["response_mask"]
+            values = values * response_mask  # Only action tokens have values
         return values
 
     @GPUMemoryLogger(role="dp critic", logger=logger)