Add the conformer backbone (phi4mm audio)

.typos.toml

@@ -10,7 +10,9 @@ extend-ignore-identifiers-re = [
     "thr",
     "nd",
     "uneeded",
-    "tese"
+    "tese",
+    "seperable",
+    "Seperable",
 ]
 
 [files]

examples/python/custom_tool_call.py

@@ -8,6 +8,7 @@
     ToolChoice,
 )
 
+
 def local_search(query: str):
     results = []
     for root, _, files in os.walk("."):
@@ -35,6 +36,7 @@ def tool_cb(name: str, args: dict) -> str:
         return json.dumps(local_search(args.get("query", "")))
     return ""
 
+
 schema = json.dumps(
     {
         "type": "function",
@@ -51,7 +53,9 @@ def tool_cb(name: str, args: dict) -> str:
 )
 
 runner = Runner(
-    which=Which.Plain(model_id="NousResearch/Hermes-3-Llama-3.1-8B", arch=Architecture.Llama),
+    which=Which.Plain(
+        model_id="NousResearch/Hermes-3-Llama-3.1-8B", arch=Architecture.Llama
+    ),
     tool_callbacks={"local_search": tool_cb},
 )
 

mistralrs-core/Cargo.toml

@@ -94,6 +94,10 @@ ahash.workspace = true
 num-traits.workspace = true
 libc.workspace = true
 bm25.workspace = true
+rubato = "0.16.2"
+rustfft = "6.3.0"
+hound = "3.5.1"
+apodize = "1.0.0"
 
 [features]
 pyo3_macros = ["pyo3"]

mistralrs-core/src/engine/add_request.rs

@@ -122,10 +122,21 @@ impl Engine {
                 ref images,
                 messages: _,
                 enable_thinking: _,
+                audios: _,
             } => Some(images.clone()),
             _ => None,
         };
 
+        let audios = match request.messages {
+            RequestMessage::VisionChat {
+                images: _,
+                messages: _,
+                enable_thinking: _,
+                ref audios,
+            } => Some(audios.clone()),
+            _ => None,
+        };
+
         let matcher = Arc::new(handle_seq_error!(
             ToolCallingMatcher::new(request.tool_choice.unwrap_or(ToolChoice::Auto),),
             request.response
@@ -157,6 +168,7 @@ impl Engine {
             }
             | RequestMessage::VisionChat {
                 images: _,
+                audios: _,
                 messages,
                 enable_thinking,
             } => {
@@ -497,6 +509,7 @@ impl Engine {
                     None
                 },
                 images.clone(),
+                audios.clone(),
                 block_size,
                 Some(matcher.clone()),
                 image_generation_format,

mistralrs-core/src/layers.rs

@@ -7,7 +7,8 @@ use candle_core::{
     Context, DType, Device, IndexOp, Result, Tensor, D,
 };
 use candle_nn::{
-    Conv2d, Conv2dConfig, Embedding, GroupNorm, LayerNorm, LayerNormConfig, Linear, Module,
+    BatchNorm, BatchNormConfig, Conv1d, Conv1dConfig, Conv2d, Conv2dConfig, Embedding, GroupNorm,
+    LayerNorm, LayerNormConfig, Linear, Module,
 };
 use float8::F8E4M3;
 use half::{bf16, f16};
@@ -67,6 +68,34 @@ pub fn layer_norm<C: Into<LayerNormConfig>>(
     }
 }
 
+pub fn batch_norm<C: Into<BatchNormConfig>>(
+    num_features: usize,
+    config: C,
+    vb: ShardedVarBuilder,
+) -> Result<BatchNorm> {
+    let config = config.into();
+    if config.eps < 0. {
+        candle_core::bail!("batch-norm eps cannot be negative {}", config.eps)
+    }
+    let running_mean = vb.get(num_features, "running_mean")?;
+    let running_var = vb.get(num_features, "running_var")?;
+
+    if config.affine {
+        let weight = vb.get(num_features, "weight")?;
+        let bias = vb.get(num_features, "bias")?;
+        BatchNorm::new(
+            num_features,
+            running_mean,
+            running_var,
+            weight,
+            bias,
+            config.eps,
+        )
+    } else {
+        BatchNorm::new_no_bias(num_features, running_mean, running_var, config.eps)
+    }
+}
+
 pub fn group_norm(
     num_groups: usize,
     num_channels: usize,
@@ -117,6 +146,35 @@ pub fn conv2d_no_bias(
     Ok(Conv2d::new(ws, None, cfg))
 }
 
+pub fn conv1d(
+    in_channels: usize,
+    out_channels: usize,
+    kernel_size: usize,
+    cfg: Conv1dConfig,
+    vb: ShardedVarBuilder,
+) -> Result<Conv1d> {
+    let ws = vb.get(
+        (out_channels, in_channels / cfg.groups, kernel_size),
+        "weight",
+    )?;
+    let bs = vb.get(out_channels, "bias")?;
+    Ok(Conv1d::new(ws, Some(bs), cfg))
+}
+
+pub fn conv1d_no_bias(
+    in_channels: usize,
+    out_channels: usize,
+    kernel_size: usize,
+    cfg: Conv1dConfig,
+    vb: ShardedVarBuilder,
+) -> Result<Conv1d> {
+    let ws = vb.get(
+        (out_channels, in_channels / cfg.groups, kernel_size),
+        "weight",
+    )?;
+    Ok(Conv1d::new(ws, None, cfg))
+}
+
 pub fn linear(in_dim: usize, out_dim: usize, vb: ShardedVarBuilder) -> Result<Linear> {
     let ws = vb.get((out_dim, in_dim), "weight")?;
     let bs = vb.get(out_dim, "bias")?;

mistralrs-core/src/lib.rs

@@ -89,16 +89,18 @@ pub use pipeline::{
     DiffusionLoaderBuilder, DiffusionLoaderType, GGMLLoader, GGMLLoaderBuilder, GGMLSpecificConfig,
     GGUFLoader, GGUFLoaderBuilder, GGUFSpecificConfig, GemmaLoader, Idefics2Loader,
     IsqOrganization, LLaVALoader, LLaVANextLoader, LlamaLoader, Loader, LocalModelPaths,
-    LoraAdapterPaths, MistralLoader, MixtralLoader, ModelKind, ModelPaths, NormalLoader,
-    NormalLoaderBuilder, NormalLoaderType, NormalSpecificConfig, Phi2Loader, Phi3Loader,
-    Phi3VLoader, Qwen2Loader, SpeculativeConfig, SpeculativeLoader, SpeculativePipeline,
-    SpeechLoader, SpeechPipeline, Starcoder2Loader, TokenSource, VisionLoader, VisionLoaderBuilder,
-    VisionLoaderType, VisionPromptPrefixer, VisionSpecificConfig, UQFF_MULTI_FILE_DELIMITER,
+    LoraAdapterPaths, MistralLoader, MixtralLoader, ModelKind, ModelPaths,
+    MultimodalPromptPrefixer, NormalLoader, NormalLoaderBuilder, NormalLoaderType,
+    NormalSpecificConfig, Phi2Loader, Phi3Loader, Phi3VLoader, Qwen2Loader, SpeculativeConfig,
+    SpeculativeLoader, SpeculativePipeline, SpeechLoader, SpeechPipeline, Starcoder2Loader,
+    TokenSource, VisionLoader, VisionLoaderBuilder, VisionLoaderType, VisionSpecificConfig,
+    UQFF_MULTI_FILE_DELIMITER,
 };
 pub use request::{
-    ApproximateUserLocation, Constraint, DetokenizationRequest, ImageGenerationResponseFormat,
-    LlguidanceGrammar, MessageContent, NormalRequest, Request, RequestMessage, SearchContextSize,
-    TokenizationRequest, WebSearchOptions, WebSearchUserLocation,
+    ApproximateUserLocation, AudioInput, Constraint, DetokenizationRequest,
+    ImageGenerationResponseFormat, LlguidanceGrammar, MessageContent, NormalRequest, Request,
+    RequestMessage, SearchContextSize, TokenizationRequest, WebSearchOptions,
+    WebSearchUserLocation,
 };
 pub use response::*;
 pub use sampler::{

mistralrs-core/src/pipeline/amoe.rs

@@ -584,6 +584,7 @@ fn new_dummy_seq(
         None,
         None,
         images,
+        None,
         None, // TODO incorrect for PagedAttention
         None,
         None,