fix image-text mixed data

hiyouga · hiyouga · commit ab88ddc8bffb · 2025-06-17T19:59:49.000Z
diff --git a/verl/models/monkey_patch.py b/verl/models/monkey_patch.py
@@ -15,24 +15,49 @@
 
 from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
 
+from ..utils.py_functional import is_transformers_version_greater_than
 from .transformers.flash_attention_utils import flash_attention_forward
-from .transformers.qwen2_vl import qwen2_vl_attn_forward
+from .transformers.qwen2_vl import (
+    qwen2_vl_attn_forward,
+    qwen2_vl_base_forward_new,
+    qwen2_vl_forward_new,
+    qwen2_vl_forward_old,
+)
 
 
 def apply_ulysses_patch(model_type: str) -> None:
     if model_type in ("llama", "gemma", "gemma2", "mistral", "qwen2", "qwen3", "qwen3_moe"):
         ALL_ATTENTION_FUNCTIONS["flash_attention_2"] = flash_attention_forward
     elif model_type in ("qwen2_vl", "qwen2_5_vl"):
-        try:
+        if is_transformers_version_greater_than("4.53.0"):
+            from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLAttention
+            from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLAttention
+
+            Qwen2VLAttention.forward = qwen2_vl_attn_forward
+            Qwen2_5_VLAttention.forward = qwen2_vl_attn_forward
+        else:
             from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLFlashAttention2
             from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLFlashAttention2
-        except ImportError:  # transformers >= 4.52.4
+
+            Qwen2VLFlashAttention2.forward = qwen2_vl_attn_forward
+            Qwen2_5_VLFlashAttention2.forward = qwen2_vl_attn_forward
+
+        if is_transformers_version_greater_than("4.52.0"):
             from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
-                Qwen2_5_VLAttention as Qwen2_5_VLFlashAttention2,
+                Qwen2_5_VLForConditionalGeneration,
+                Qwen2_5_VLModel,
             )
-            from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLAttention as Qwen2VLFlashAttention2
+            from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLForConditionalGeneration, Qwen2VLModel
+
+            Qwen2VLModel.forward = qwen2_vl_base_forward_new
+            Qwen2_5_VLModel.forward = qwen2_vl_base_forward_new
+            Qwen2VLForConditionalGeneration.forward = qwen2_vl_forward_new
+            Qwen2_5_VLForConditionalGeneration.forward = qwen2_vl_forward_new
+        else:
+            from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLForConditionalGeneration
+            from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLForConditionalGeneration
 
-        Qwen2VLFlashAttention2.forward = qwen2_vl_attn_forward
-        Qwen2_5_VLFlashAttention2.forward = qwen2_vl_attn_forward
+            Qwen2VLForConditionalGeneration.forward = qwen2_vl_forward_old
+            Qwen2_5_VLForConditionalGeneration.forward = qwen2_vl_forward_old
     else:
         raise NotImplementedError(f"Model architecture {model_type} is not supported yet.")
diff --git a/verl/models/transformers/qwen2_vl.py b/verl/models/transformers/qwen2_vl.py
@@ -19,18 +19,29 @@
 
 import torch
 
+from ...utils.py_functional import is_transformers_version_greater_than
 from .flash_attention_utils import flash_attention_forward
 
 
-try:
+if is_transformers_version_greater_than("4.52.0"):
     from transformers.models.qwen2_vl.modeling_qwen2_vl import (
         Qwen2VLAttention,
+        Qwen2VLCausalLMOutputWithPast,
+        Qwen2VLForConditionalGeneration,
+        Qwen2VLModel,
+        Qwen2VLModelOutputWithPast,
         apply_multimodal_rotary_pos_emb,
         repeat_kv,
     )
     from transformers.models.qwen2_vl.processing_qwen2_vl import Qwen2VLProcessor
-except ImportError:
-    pass
+else:
+    from transformers.models.qwen2_vl.modeling_qwen2_vl import (
+        Qwen2VLAttention,
+        Qwen2VLCausalLMOutputWithPast,
+        Qwen2VLForConditionalGeneration,
+        apply_multimodal_rotary_pos_emb,
+        repeat_kv,
+    )
 
 
 def get_rope_index(
@@ -183,3 +194,184 @@ def qwen2_vl_attn_forward(
     attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
     attn_output = self.o_proj(attn_output)
     return attn_output, None, None
+
+
+def _get_input_embeds(
+    model: "Qwen2VLModel",
+    input_ids: torch.LongTensor,
+    attention_mask: Optional[torch.Tensor] = None,
+    pixel_values: Optional[torch.FloatTensor] = None,
+    pixel_values_videos: Optional[torch.FloatTensor] = None,
+    image_grid_thw: Optional[torch.LongTensor] = None,
+    video_grid_thw: Optional[torch.LongTensor] = None,
+):
+    inputs_embeds = model.get_input_embeddings()(input_ids)
+    if pixel_values is not None:
+        pixel_values = pixel_values.type(model.visual.dtype)
+        image_embeds = model.visual(pixel_values, grid_thw=image_grid_thw)
+        n_image_tokens = (input_ids == model.config.image_token_id).sum().item()
+        n_image_features = image_embeds.shape[0]
+        if n_image_tokens != n_image_features:
+            raise ValueError(
+                f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}"
+            )
+
+        mask = input_ids == model.config.image_token_id
+        mask_unsqueezed = mask.unsqueeze(-1)
+        mask_expanded = mask_unsqueezed.expand_as(inputs_embeds)
+        image_mask = mask_expanded.to(inputs_embeds.device)
+
+        image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+        inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
+
+    if pixel_values_videos is not None:
+        pixel_values_videos = pixel_values_videos.type(model.visual.dtype)
+        video_embeds = model.visual(pixel_values_videos, grid_thw=video_grid_thw)
+        n_video_tokens = (input_ids == model.config.video_token_id).sum().item()
+        n_video_features = video_embeds.shape[0]
+        if n_video_tokens != n_video_features:
+            raise ValueError(
+                f"Video features and video tokens do not match: tokens: {n_video_tokens}, features {n_video_features}"
+            )
+
+        mask = input_ids == model.config.video_token_id
+        mask_unsqueezed = mask.unsqueeze(-1)
+        mask_expanded = mask_unsqueezed.expand_as(inputs_embeds)
+        video_mask = mask_expanded.to(inputs_embeds.device)
+
+        video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+        inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
+
+    if pixel_values is None and pixel_values_videos is None:
+        pixel_values = torch.zeros((16, 1176), dtype=inputs_embeds.dtype, device=inputs_embeds.device)
+        image_grid_thw = torch.tensor([[1, 4, 4]], dtype=torch.long, device=inputs_embeds.device)
+        image_embeds = model.visual(pixel_values, grid_thw=image_grid_thw)
+        inputs_embeds += 0.0 * image_embeds.mean()
+
+    if attention_mask is not None:
+        attention_mask = attention_mask.to(inputs_embeds.device)
+
+    return inputs_embeds, attention_mask
+
+
+def qwen2_vl_forward_old(
+    self: "Qwen2VLForConditionalGeneration",
+    input_ids: torch.LongTensor,
+    attention_mask: Optional[torch.Tensor] = None,
+    position_ids: Optional[torch.LongTensor] = None,
+    labels: Optional[torch.LongTensor] = None,
+    pixel_values: Optional[torch.FloatTensor] = None,
+    pixel_values_videos: Optional[torch.FloatTensor] = None,
+    image_grid_thw: Optional[torch.LongTensor] = None,
+    video_grid_thw: Optional[torch.LongTensor] = None,
+    **kwargs,
+) -> "Qwen2VLCausalLMOutputWithPast":
+    inputs_embeds, attention_mask = _get_input_embeds(
+        self, input_ids, attention_mask, pixel_values, pixel_values_videos, image_grid_thw, video_grid_thw
+    )
+    outputs = self.model(
+        input_ids=None,
+        pixel_values=pixel_values,
+        pixel_values_videos=pixel_values_videos,
+        image_grid_thw=image_grid_thw,
+        video_grid_thw=video_grid_thw,
+        position_ids=position_ids,
+        attention_mask=attention_mask,
+        past_key_values=None,
+        inputs_embeds=inputs_embeds,
+        use_cache=False,
+        output_attentions=False,
+        output_hidden_states=False,
+        return_dict=True,
+        cache_position=None,
+    )
+    hidden_states = outputs[0]
+    logits = self.lm_head(hidden_states)
+
+    return Qwen2VLCausalLMOutputWithPast(
+        loss=None,
+        logits=logits,
+        past_key_values=None,
+        hidden_states=None,
+        attentions=None,
+        rope_deltas=None,
+    )
+
+
+def qwen2_vl_base_forward_new(
+    self: "Qwen2VLModel",
+    input_ids: torch.LongTensor,
+    attention_mask: Optional[torch.Tensor] = None,
+    position_ids: Optional[torch.LongTensor] = None,
+    labels: Optional[torch.LongTensor] = None,
+    pixel_values: Optional[torch.FloatTensor] = None,
+    pixel_values_videos: Optional[torch.FloatTensor] = None,
+    image_grid_thw: Optional[torch.LongTensor] = None,
+    video_grid_thw: Optional[torch.LongTensor] = None,
+    **kwargs,
+):
+    inputs_embeds, attention_mask = _get_input_embeds(
+        self, input_ids, attention_mask, pixel_values, pixel_values_videos, image_grid_thw, video_grid_thw
+    )
+    outputs = self.language_model(
+        input_ids=None,
+        position_ids=position_ids,
+        attention_mask=attention_mask,
+        past_key_values=None,
+        inputs_embeds=inputs_embeds,
+        use_cache=False,
+        output_attentions=False,
+        output_hidden_states=False,
+        return_dict=True,
+        cache_position=None,
+    )
+
+    output = Qwen2VLModelOutputWithPast(
+        last_hidden_state=outputs.last_hidden_state,
+        past_key_values=outputs.past_key_values,
+        hidden_states=outputs.hidden_states,
+        attentions=outputs.attentions,
+        rope_deltas=None,
+    )
+    return output
+
+
+def qwen2_vl_forward_new(
+    self: "Qwen2VLForConditionalGeneration",
+    input_ids: torch.LongTensor,
+    attention_mask: Optional[torch.Tensor] = None,
+    position_ids: Optional[torch.LongTensor] = None,
+    labels: Optional[torch.LongTensor] = None,
+    pixel_values: Optional[torch.FloatTensor] = None,
+    pixel_values_videos: Optional[torch.FloatTensor] = None,
+    image_grid_thw: Optional[torch.LongTensor] = None,
+    video_grid_thw: Optional[torch.LongTensor] = None,
+    **kwargs,
+) -> "Qwen2VLCausalLMOutputWithPast":
+    outputs = self.model(
+        input_ids=input_ids,
+        pixel_values=pixel_values,
+        pixel_values_videos=pixel_values_videos,
+        image_grid_thw=image_grid_thw,
+        video_grid_thw=video_grid_thw,
+        position_ids=position_ids,
+        attention_mask=attention_mask,
+        past_key_values=None,
+        inputs_embeds=None,
+        use_cache=False,
+        output_attentions=False,
+        output_hidden_states=False,
+        return_dict=True,
+        cache_position=None,
+    )
+    hidden_states = outputs[0]
+    logits = self.lm_head(hidden_states)
+
+    return Qwen2VLCausalLMOutputWithPast(
+        loss=None,
+        logits=logits,
+        past_key_values=None,
+        hidden_states=None,
+        attentions=None,
+        rope_deltas=None,
+    )
diff --git a/verl/utils/dataset.py b/verl/utils/dataset.py
@@ -156,10 +156,20 @@ def _build_messages(self, example: Dict[str, Any]) -> List[Dict[str, Any]]:
 
     def _filter_overlong_prompts(self, example: Dict[str, Any]) -> bool:
         messages = self._build_messages(example)
-        processing_class = self.processor if self.processor is not None else self.tokenizer
-        return (
-            len(processing_class.apply_chat_template(messages, add_generation_prompt=True)) <= self.max_prompt_length
-        )
+        if self.image_key in example:
+            prompt = self.processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
+            images = example[self.image_key] or []
+            if self.image_dir is not None and len(images) != 0 and isinstance(images[0], str):  # image paths
+                images = [os.path.join(self.image_dir, image) for image in images]
+
+            resized_images = [
+                process_image(image, min_pixels=self.min_pixels, max_pixels=self.max_pixels) for image in images
+            ] or None
+            model_inputs = self.processor(resized_images, [prompt], add_special_tokens=False, return_tensors="pt")
+            return model_inputs["input_ids"].size(-1) <= self.max_prompt_length
+        else:
+            input_ids = self.tokenizer.apply_chat_template(messages, add_generation_prompt=True)
+            return len(input_ids) <= self.max_prompt_length
 
     def __len__(self):
         return len(self.dataset)
@@ -176,7 +186,7 @@ def __getitem__(self, index):
 
             resized_images = [
                 process_image(image, min_pixels=self.min_pixels, max_pixels=self.max_pixels) for image in images
-            ]
+            ] or None
             model_inputs = self.processor(resized_images, [prompt], add_special_tokens=False, return_tensors="pt")
             input_ids = model_inputs.pop("input_ids")[0]
             attention_mask = model_inputs.pop("attention_mask")[0]
diff --git a/verl/utils/py_functional.py b/verl/utils/py_functional.py
@@ -15,6 +15,7 @@
 Contain small python utility functions
 """
 
+import importlib.metadata
 import importlib.util
 import re
 from contextlib import contextmanager
@@ -24,6 +25,7 @@
 import numpy as np
 import yaml
 from codetiming import Timer
+from packaging import version
 from yaml import Dumper
 
 
@@ -53,6 +55,18 @@ def is_package_available(name: str) -> bool:
     return importlib.util.find_spec(name) is not None
 
 
+def get_package_version(name: str) -> "version.Version":
+    try:
+        return version.parse(importlib.metadata.version(name))
+    except Exception:
+        return version.parse("0.0.0")
+
+
+@lru_cache
+def is_transformers_version_greater_than(content: str):
+    return get_package_version("transformers") >= version.parse(content)
+
+
 def union_two_dict(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> Dict[str, Any]:
     """Union two dict. Will throw an error if there is an item not the same object with the same key."""
     for key in dict2.keys():
diff --git a/verl/utils/tokenizer.py b/verl/utils/tokenizer.py
@@ -44,7 +44,7 @@ def get_processor(model_path: str, override_chat_template: Optional[str] = None,
         processor.chat_template = override_chat_template
 
     # Avoid load tokenizer, see:
-    # https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/auto/processing_auto.py#L344
+    # https://github.com/huggingface/transformers/blob/v4.52.4/src/transformers/models/auto/processing_auto.py#L386
     if processor is not None and "Processor" not in processor.__class__.__name__:
         processor = None
 
diff --git a/verl/workers/actor/dp_actor.py b/verl/workers/actor/dp_actor.py
@@ -71,12 +71,17 @@ def _forward_micro_batch(self, micro_batch: Dict[str, torch.Tensor], temperature
         if position_ids.dim() == 3:  # qwen2vl mrope
             position_ids = position_ids.transpose(0, 1)  # (bsz, 3, seqlen) -> (3, bsz, seqlen)
 
-        multi_modal_inputs = {}
+        multi_modal_inputs = defaultdict(list)
         if "multi_modal_inputs" in micro_batch:
-            for key in micro_batch["multi_modal_inputs"][0].keys():
-                multi_modal_inputs[key] = torch.cat(
-                    [inputs[key] for inputs in micro_batch["multi_modal_inputs"]], dim=0
-                )
+            for input_dict in micro_batch["multi_modal_inputs"]:
+                for key, value in input_dict.items():
+                    multi_modal_inputs[key].append(value)
+
+            for key, value in multi_modal_inputs.items():
+                if len(value) != 0:
+                    multi_modal_inputs[key] = torch.cat(value, dim=0)
+                else:
+                    multi_modal_inputs[key] = None
 
         if self.config.padding_free:
             input_ids_rmpad, indices, *_ = unpad_input(
diff --git a/verl/workers/critic/dp_critic.py b/verl/workers/critic/dp_critic.py
@@ -59,12 +59,17 @@ def _forward_micro_batch(self, micro_batch: Dict[str, torch.Tensor]) -> torch.Te
         if position_ids.dim() == 3:  # qwen2vl mrope
             position_ids = position_ids.transpose(0, 1)  # (bsz, 3, seqlen) -> (3, bsz, seqlen)
 
-        multi_modal_inputs = {}
+        multi_modal_inputs = defaultdict(list)
         if "multi_modal_inputs" in micro_batch:
-            for key in micro_batch["multi_modal_inputs"][0].keys():
-                multi_modal_inputs[key] = torch.cat(
-                    [inputs[key] for inputs in micro_batch["multi_modal_inputs"]], dim=0
-                )
+            for input_dict in micro_batch["multi_modal_inputs"]:
+                for key, value in input_dict.items():
+                    multi_modal_inputs[key].append(value)
+
+            for key, value in multi_modal_inputs.items():
+                if len(value) != 0:
+                    multi_modal_inputs[key] = torch.cat(value, dim=0)
+                else:
+                    multi_modal_inputs[key] = None
 
         if self.config.padding_free:
             input_ids_rmpad, indices, *_ = unpad_input(
diff --git a/verl/workers/fsdp_workers.py b/verl/workers/fsdp_workers.py
diff --git a/verl/workers/rollout/vllm_rollout_spmd.py b/verl/workers/rollout/vllm_rollout_spmd.py
