[Refactor] simplify multimodal data processing

python/sglang/srt/managers/schedule_batch.py

@@ -37,7 +37,7 @@
 import threading
 from enum import Enum, auto
 from http import HTTPStatus
-from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
 
 import numpy as np
 import torch
@@ -201,7 +201,7 @@ class MultimodalDataItem:
     For example, if there are 3 images and 1 audio inputs, there will be 2 MultimodalDataItem.
     One for images and one for audio.
 
-    We put the common fields first and the model-specific fields last.
+    We put the common fields first and the model-specific fields in model_specific_data.
     """
 
     modality: Modality
@@ -211,33 +211,22 @@ class MultimodalDataItem:
     # the raw features returned by processor, e.g. pixel_values or audio_features
     feature: Union[torch.Tensor, np.ndarray] = None
 
+    # Common fields used across multiple models
     image_sizes: Tuple[int, int] = None
 
-    audio_feature_lens: Optional[List[torch.Tensor]] = None
-    audio_offsets: Optional[List[Tuple[int, int]]] = None
     precomputed_features: Optional[Union[torch.Tensor, np.ndarray]] = None
 
-    # For qwen-vl
-    image_grid_thw: Union[torch.Tensor, np.ndarray] = None
-    second_per_grid_ts: Optional[List[torch.Tensor]] = None
-
-    # For deepseek-vl
-    image_emb_mask: Optional[torch.Tensor] = None
-    image_spatial_crop: Optional[torch.Tensor] = None
-
-    # For minicpmv
-    # [num_images, (n, w, h)]
-    tgt_size: Tuple[int, int] = None
-
-    # For mllama
-    aspect_ratio_id: Optional[List[torch.Tensor]] = None
-    aspect_ratio_mask: Optional[List[torch.Tensor]] = None
-
-    # For kimi-vl
-    image_grid_hws: Optional[List[torch.Tensor]] = None
-
-    # For gemma3n
-    input_features_mask: Optional[torch.Tensor] = None
+    # Model-specific data stored in a dictionary
+    # This should contains all the individual model-specific fields like:
+    # - image_grid_thw, second_per_grid_ts (qwen-vl)
+    # - image_emb_mask, image_spatial_crop (deepseek-vl)
+    # - tgt_size (minicpmv)
+    # - aspect_ratio_id, aspect_ratio_mask (mllama)
+    # - image_grid_hws (kimi-vl)
+    # - input_features_mask (gemma3n)
+    # - audio_feature_lens, audio_offsets (minicpmo)
+    # - ...
+    model_specific_data: Dict[str, Any] = dataclasses.field(default_factory=dict)
 
     @staticmethod
     def is_empty_list(l):
@@ -303,7 +292,7 @@ def from_dict(obj: dict):
     def merge(self, other):
         self.feature += other.feature
         self.image_sizes += other.image_sizes
-        self.image_offsets += other.image_offsets
+        self.offsets += other.offsets
         self.hash = hash((self.hash, other.hash))
         self.set_pad_value()
 

python/sglang/srt/models/mllama.py

@@ -836,19 +836,10 @@ def __init__(
             prefix="multi_modal_projector",
         )
         self.logits_processor = LogitsProcessor(config.text_config)
+        self.padding_pattern = MultiModalityDataPaddingPatternMultimodalTokens()
 
     def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
-        pixel_values = torch.cat([item.feature for item in mm_inputs.mm_items], dim=0)
-        pad_values = [item.pad_value for item in mm_inputs.mm_items]
-
-        num_concurrent_media, num_tiles = pixel_values.shape[1:3]
-        num_patches = self.vision_model.num_patches
-        image_len = num_concurrent_media * num_tiles * num_patches
-        mm_inputs.num_image_tokens = image_len
-
-        pad_ids = pad_values * ((image_len + len(pad_values)) // len(pad_values))
-
-        return pad_ids[:image_len] + input_ids
+        return self.padding_pattern.pad_input_tokens(input_ids, mm_inputs)
 
     def _batch_image_inputs(self, forward_batch: ForwardBatch):
         if forward_batch.forward_mode.is_decode() or all(forward_batch.encoder_cached):

python/sglang/srt/models/mllama4.py

@@ -81,6 +81,7 @@ def __init__(
         self.logits_processor = LogitsProcessor(
             config.text_config if hasattr(config, "text_config") else config
         )
+        self.padding_pattern = MultiModalityDataPaddingPatternMultimodalTokens()
 
     def _has_vision_weights(self, config) -> bool:
         """Check if the model has vision components by examining the checkpoint."""
@@ -135,8 +136,7 @@ def _check_vision_weights_in_index(self, index_file: str) -> bool:
             return False
 
     def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
-        pattern = MultiModalityDataPaddingPatternMultimodalTokens()
-        return pattern.pad_input_tokens(input_ids, mm_inputs)
+        return self.padding_pattern.pad_input_tokens(input_ids, mm_inputs)
 
     def get_image_feature(
         self,

python/sglang/srt/multimodal/processors/base_processor.py

@@ -529,9 +529,9 @@ def collect_mm_items_from_processor_output(
 
                 if attr_name in self.FEATURE_NAMES:
                     attr_name = "feature"
-
-                # Set attribute
-                setattr(items[modality], attr_name, value)
+                    setattr(items[modality], attr_name, value)
+                else:
+                    items[modality].model_specific_data[attr_name] = value
 
         return list(items.values())
 

python/sglang/srt/multimodal/processors/clip.py

@@ -1,9 +1,10 @@
 from typing import List, Union
 
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
 from sglang.srt.models.clip import CLIPModel
-from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor
-from sglang.srt.utils import load_image
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
 
 
 class ClipImageProcessor(BaseMultimodalProcessor):
@@ -15,19 +16,17 @@ def __init__(self, hf_config, server_args, _processor):
     async def process_mm_data_async(
         self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
     ):
-        if isinstance(input_text, list):
-            assert len(input_text) and isinstance(input_text[0], int)
-            input_text = self._processor.tokenizer.decode(input_text)
-
-        images = [load_image(image)[0] for image in image_data]
-
-        image_inputs = self.process_mm_data(input_text=input_text, images=images)
-        image_inputs["data_hashes"] = [hash(str(image_data))]
-        image_inputs["input_ids"] = image_inputs["input_ids"].tolist()[0]
-        image_inputs["mm_items"] = [
-            MultimodalDataItem(
-                feature=image_inputs["pixel_values"], modality=Modality.IMAGE
-            )
-        ]
-
-        return image_inputs
+        base_output = self.load_mm_data(
+            prompt=input_text,
+            multimodal_tokens=MultimodalSpecialTokens(
+                image_token=self._processor.tokenizer.image_token
+            ),
+            image_data=image_data,
+        )
+
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(base_output)
+
+        return {
+            "input_ids": input_ids.tolist(),
+            "mm_items": mm_items,
+        }

python/sglang/srt/multimodal/processors/deepseek_vl_v2.py

@@ -18,9 +18,6 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 from typing import List, Union
 
-import torch
-
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
 from sglang.srt.models.deepseek_vl2 import DeepseekVL2ForCausalLM
 from sglang.srt.multimodal.processors.base_processor import (
     BaseMultimodalProcessor,
@@ -39,7 +36,6 @@ async def process_mm_data_async(
         self,
         image_data: List[Union[str, bytes]],
         input_text,
-        request_obj,
         max_req_input_len,
         *args,
         **kwargs
@@ -50,34 +46,10 @@ async def process_mm_data_async(
             multimodal_tokens=MultimodalSpecialTokens(image_token=self.IMAGE_TOKEN),
             max_req_input_len=max_req_input_len,
         )
-        res = self.process_mm_data(
-            input_text=base_output.input_text,
-            images=base_output.images,
-            max_req_input_len=max_req_input_len,
-            conversations=base_output.input_text,
-        )
-        images_seq_mask = res["images_seq_mask"]
-        images_spatial_crop = res["images_spatial_crop"]
-        batched_images_spatial_crop = []
-        batched_images_spatial_crop.append(images_spatial_crop)
-        batched_images_spatial_crop = torch.stack(batched_images_spatial_crop, dim=0)
-
-        items = []
-        input_ids = res["input_ids"]
-        image_offsets = self.get_mm_items_offset(
-            input_ids=input_ids, mm_token_id=self._processor.image_token_id
-        )
-        item = MultimodalDataItem(
-            feature=res["images"],
-            offsets=image_offsets,
-            modality=Modality.IMAGE,
-            image_emb_mask=images_seq_mask,
-            image_spatial_crop=batched_images_spatial_crop,
-        )
-        items += [item]
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(base_output)
 
         return {
-            "mm_items": items,
+            "mm_items": mm_items,
             "input_ids": input_ids.tolist(),
             "im_token_id": self._processor.image_token_id,
         }

python/sglang/srt/multimodal/processors/janus_pro.py

@@ -33,26 +33,10 @@ async def process_mm_data_async(
             max_req_input_len=max_req_input_len,
         )
 
-        images = base_out.images
-        res = self.process_mm_data(
-            input_text=base_out.input_text,
-            prompt=base_out.input_text,
-            images=images,
-        )
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(base_out)
 
-        input_ids = res["input_ids"].flatten()
-        image_offsets = self.get_mm_items_offset(
-            input_ids=input_ids, mm_token_id=processor.image_id
-        )
         return {
-            "mm_items": [
-                MultimodalDataItem(
-                    feature=res["pixel_values"],
-                    image_emb_mask=res["images_emb_mask"],
-                    offsets=image_offsets,
-                    modality=Modality.IMAGE,
-                )
-            ],
+            "mm_items": mm_items,
             "input_ids": input_ids.tolist(),
             "im_start_id": processor.image_start_id,
             "im_end_id": processor.image_end_id,

python/sglang/srt/multimodal/processors/minicpm.py

@@ -116,7 +116,7 @@ async def process_mm_data_async(
             item = MultimodalDataItem(
                 feature=pixel_values,
                 offsets=image_offsets,
-                tgt_size=tgt_sizes_flat,
+                model_specific_data={"tgt_size": tgt_sizes_flat},
                 modality=Modality.IMAGE,
             )
             items += [item]
@@ -136,7 +136,7 @@ async def process_mm_data_async(
                 audio_offsets = None
             item = MultimodalDataItem(
                 feature=[res["audio_features"]],
-                audio_feature_lens=res["audio_feature_lens"],
+                model_specific_data={"audio_feature_lens": res["audio_feature_lens"]},
                 offsets=audio_offsets,
                 modality=Modality.AUDIO,
             )

python/sglang/srt/multimodal/processors/mlama.py

@@ -1,34 +1,33 @@
 from typing import List, Union
 
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
 from sglang.srt.models.mllama import MllamaForConditionalGeneration
-from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor
-from sglang.srt.utils import load_image
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
 
 
 class MllamaImageProcessor(BaseMultimodalProcessor):
     models = [MllamaForConditionalGeneration]
 
     def __init__(self, hf_config, server_args, _processor):
         super().__init__(hf_config, server_args, _processor)
+        self.IM_TOKEN = self._processor.image_token
+        self.IM_TOKEN_ID = self._processor.image_token_id
 
     async def process_mm_data_async(
         self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
     ):
-        if isinstance(input_text, list):
-            assert len(input_text) and isinstance(input_text[0], int)
-            input_text = self._processor.tokenizer.decode(input_text)
+        base_out = self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            multimodal_tokens=MultimodalSpecialTokens(image_token=self.IM_TOKEN),
+        )
 
-        images = [load_image(image)[0] for image in image_data]
-        image_inputs = self.process_mm_data(input_text=input_text, images=images)
-        image_inputs["input_ids"] = image_inputs["input_ids"].tolist()[0]
-        image_inputs["mm_items"] = [
-            MultimodalDataItem(
-                feature=image_inputs["pixel_values"],
-                aspect_ratio_id=image_inputs["aspect_ratio_ids"],
-                aspect_ratio_mask=image_inputs["aspect_ratio_mask"],
-                modality=Modality.IMAGE,
-            )
-        ]
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(base_out)
 
-        return image_inputs
+        return {
+            "mm_items": mm_items,
+            "input_ids": input_ids.tolist(),
+            "im_token_id": self.IM_TOKEN_ID,
+        }