Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions mistralrs-core/src/pipeline/loaders/normal_loaders.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3004,8 +3004,8 @@ impl IsqModelLoader for Qwen3Loader {
Regex::new(r"layers\.(\d+)\.mlp\.down_proj\.(weight|bias)$")?,
])
}
fn immediate_isq_predicates_moqe(&self, config: &str) -> Result<Vec<Regex>> {
self.isq_layer_regexes_moqe(config)
fn immediate_isq_predicates(&self, config: &str) -> Result<Vec<Regex>> {
self.isq_layer_regexes(config)
}
}

Expand Down Expand Up @@ -3189,6 +3189,9 @@ impl IsqModelLoader for Qwen3MoELoader {
Regex::new(r"layers\.(\d+)\.mlp\.experts\.(\d+)\.down_proj\.(weight|bias)$")?,
])
}
fn immediate_isq_predicates(&self, config: &str) -> Result<Vec<Regex>> {
self.isq_layer_regexes(config)
}
fn immediate_isq_predicates_moqe(&self, config: &str) -> Result<Vec<Regex>> {
self.isq_layer_regexes_moqe(config)
}
Expand Down
4 changes: 4 additions & 0 deletions mistralrs-core/src/pipeline/normal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,10 @@ impl Loader for NormalLoader {
} else {
self.inner.immediate_isq_predicates(&config)?
};
info!("Applying ISQ to {in_situ_quant:?}");
if predicates.is_empty() {
warn!("No predicates for this model and ISQ setting deteced. ISQ will not be applied to any weights!");
}
mistralrs_quant::set_immediate_isq(in_situ_quant, predicates);
false
} else {
Expand Down
4 changes: 4 additions & 0 deletions mistralrs-core/src/pipeline/vision.rs
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,10 @@ impl Loader for VisionLoader {
&& self.config.write_uqff.is_none()
{
let predicates = self.inner.immediate_isq_predicates(&config)?;
info!("Applying ISQ to {in_situ_quant:?}");
if predicates.is_empty() {
warn!("No predicates for this model and ISQ setting deteced. ISQ will not be applied to any weights!");
}
mistralrs_quant::set_immediate_isq(in_situ_quant, predicates);
false
} else {
Expand Down
Loading