Skip to content

[Bugfix] remove post_layernorm in siglip #8106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Sep 4, 2024
Merged

[Bugfix] remove post_layernorm in siglip #8106

Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
4a65e17
[Bugfix] remove post_layernorm in siglip
wnma3mz Sep 3, 2024
229bdee
add post_layernorm when num_hidden_layers_override is None for siglip
wnma3mz Sep 3, 2024
98b19bc
update num_hidden_layers_override equals config.num_hidden_layers
wnma3mz Sep 3, 2024
2ec4f3e
format code
wnma3mz Sep 3, 2024
b93318f
fix load_weights
wnma3mz Sep 3, 2024
908d316
Merge branch 'main' into main
wnma3mz Sep 3, 2024
459974b
format code
wnma3mz Sep 3, 2024
e06c4de
format code
wnma3mz Sep 3, 2024
4da0638
format code
wnma3mz Sep 3, 2024
544ab65
move need_post_layernorm to SiglipVisionTransformer
wnma3mz Sep 4, 2024
5ea1163
make need_post_layernorm as a property
wnma3mz Sep 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions vllm/model_executor/models/siglip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,19 +438,28 @@ def __init__(
config: SiglipVisionConfig,
quant_config: Optional[QuantizationConfig] = None,
num_hidden_layers_override: Optional[int] = None,
need_post_layernorm: bool = True,
):
super().__init__()
self.config = config
embed_dim = config.hidden_size

if (num_hidden_layers_override is None
or num_hidden_layers_override == config.num_hidden_layers):
self.need_post_layernorm = True
elif num_hidden_layers_override > config.num_hidden_layers:
raise ValueError(
"num_hidden_layers_override cannot be greater than "
"num_hidden_layers")
else:
self.need_post_layernorm = False

self.embeddings = SiglipVisionEmbeddings(config)
self.encoder = SiglipEncoder(
config,
quant_config=quant_config,
num_hidden_layers_override=num_hidden_layers_override,
)
if need_post_layernorm:
if self.need_post_layernorm:
self.post_layernorm = nn.LayerNorm(embed_dim,
eps=config.layer_norm_eps)
else:
Expand Down Expand Up @@ -492,16 +501,6 @@ def __init__(
num_hidden_layers_override: Optional[int] = None,
):
super().__init__()
if (num_hidden_layers_override is None
or num_hidden_layers_override == config.num_hidden_layers):
self.need_post_layernorm = True
elif num_hidden_layers_override > config.num_hidden_layers:
raise ValueError(
"num_hidden_layers_override cannot be greater than "
"num_hidden_layers")
else:
self.need_post_layernorm = False

num_heads = config.num_attention_heads
tp_size = get_tensor_model_parallel_world_size()
self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
Expand All @@ -510,7 +509,8 @@ def __init__(
config,
quant_config,
num_hidden_layers_override=num_hidden_layers_override,
need_post_layernorm=self.need_post_layernorm)
)
self.need_post_layernorm = self.vision_model.need_post_layernorm

def get_input_embeddings(self) -> nn.Module:
return self.vision_model.embeddings.patch_embedding
Expand Down