fix(rollout_corr): prevent silent failure when rollout_rs enabled without threshold

Fixes critical bug where setting rollout_rs="token" but rollout_rs_threshold=None
would silently skip rejection sampling instead of raising an error.

Changes:
- Raise ValueError if rollout_rs is set but rollout_rs_threshold is explicitly None
- Update docs to clarify rollout_rs_threshold is required when rollout_rs is enabled
- Default value (2.0) still works when parameter is omitted entirely

Author: szrlee

docs/advance/rollout_corr.md

@@ -61,7 +61,7 @@ algorithm:
     rollout_is: token                      # IS weights: "token", "sequence", or null
     rollout_is_threshold: 2.0              # Upper threshold for IS weights
     rollout_rs: null                       # Rejection sampling: "token", "sequence", "geometric", or null
-    rollout_rs_threshold: null             # RS upper threshold (uses rollout_is_threshold if null)
+    rollout_rs_threshold: null             # RS upper threshold (required if rollout_rs is enabled)
     rollout_rs_threshold_lower: null       # RS lower threshold (auto-reciprocal if null)
     rollout_token_veto_threshold: null     # Per-token veto threshold (null = disabled)
 

verl/trainer/ppo/mismatch_helper.py

@@ -550,8 +550,8 @@ def compute_rollout_correction_and_rejection_mask(
             default 2.0.
         rollout_rs: Rejection sampling aggregation level (see compute_rollout_rejection_mask for options).
             Set to None to disable rejection sampling.
-        rollout_rs_threshold: Upper threshold for rejection sampling (used if rollout_rs is set),
-            default 2.0.
+        rollout_rs_threshold: Upper threshold for rejection sampling. Required if rollout_rs is enabled.
+            Default 2.0.
         rollout_rs_threshold_lower: Lower threshold for rejection sampling (used if rollout_rs is set).
             Defaults to 1/rollout_rs_threshold if None.
         rollout_token_veto_threshold: Minimum allowed token-level IS weight. Sequences containing
@@ -599,7 +599,12 @@ def compute_rollout_correction_and_rejection_mask(
 
     # Step 3: Compute rejection mask (if enabled)
     modified_response_mask: torch.Tensor = response_mask.clone()
-    if rollout_rs is not None and rollout_rs_threshold is not None:
+    if rollout_rs is not None:
+        if rollout_rs_threshold is None:
+            raise ValueError(
+                "rollout_rs_threshold must be explicitly provided when rollout_rs is enabled. "
+                "Set rollout_rs_threshold to the desired threshold value."
+            )
         modified_response_mask, rs_metrics = compute_rollout_rejection_mask(
             log_ratio=log_ratio,
             response_mask=response_mask,