Better launch, tool handling

Author: EricLBuehler

docs/MCP.md

@@ -87,7 +87,107 @@ The response is a `CallToolResult` event whose `content` array contains a single
 
 Error cases are mapped to `CallToolError` with `is_error = true`.
 
-## 5. Limitations & future work
+## 5. Example clients
+
+### Rust
+
+```rust
+use anyhow::Result;
+use rust_mcp_sdk::{
+    mcp_client::client_runtime,
+    schema::{
+        CallToolRequestParams, ClientCapabilities, CreateMessageRequest,
+        Implementation, InitializeRequestParams, Message, LATEST_PROTOCOL_VERSION,
+    },
+    ClientSseTransport, ClientSseTransportOptions,
+};
+
+struct Handler;
+#[async_trait::async_trait]
+impl rust_mcp_sdk::mcp_client::ClientHandler for Handler {}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let transport = ClientSseTransport::new(
+        "http://localhost:4321/mcp/stream",
+        ClientSseTransportOptions::default(),
+    )?;
+
+    let details = InitializeRequestParams {
+        capabilities: ClientCapabilities::default(),
+        client_info: Implementation { name: "mcp-client".into(), version: "0.1".into() },
+        protocol_version: LATEST_PROTOCOL_VERSION.into(),
+    };
+
+    let client = client_runtime::create_client(details, transport, Handler);
+    client.clone().start().await?;
+
+    let req = CreateMessageRequest {
+        model: "mistralai/Mistral-7B-Instruct-v0.3".into(),
+        messages: vec![Message::user("Explain Rust ownership.")],
+        ..Default::default()
+    };
+
+    let result = client
+        .call_tool(CallToolRequestParams::new("chat", req.into()))
+        .await?;
+
+    println!("{}", result.content[0].as_text_content()?.text);
+    client.shut_down().await?;
+    Ok(())
+}
+```
+
+### Python
+
+```py
+import json
+import requests
+from sseclient import SSEClient
+
+payload = {
+    "kind": "callToolRequest",
+    "id": "123",
+    "params": {
+        "name": "chat",
+        "arguments": {
+            "model": "mistralai/Mistral-7B-Instruct-v0.3",
+            "messages": [
+                {"role": "user", "content": "Explain Rust ownership."}
+            ],
+        },
+    },
+}
+
+resp = requests.post(
+    "http://localhost:4321/mcp/stream",
+    headers={"Content-Type": "application/json"},
+    data=json.dumps(payload),
+    stream=True,
+)
+for event in SSEClient(resp):
+    print(event.data)
+```
+
+### HTTP
+
+```bash
+curl -N -X POST http://localhost:4321/mcp/stream \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "kind": "callToolRequest",
+    "id": "123",
+    "params": {
+      "name": "chat",
+      "arguments": {
+        "model": "mistralai/Mistral-7B-Instruct-v0.3",
+        "messages": [{"role": "user", "content": "Explain Rust ownership."}]
+      }
+    }
+  }'
+```
+
+## 6. Limitations & future work
 
 • Only synchronous, single-shot requests are supported right now.  
 • Streaming responses (`partialCallToolResult`) are not yet implemented.  

mistralrs-server/src/main.rs

@@ -2,6 +2,7 @@ use anyhow::Result;
 use clap::Parser;
 use mistralrs_core::{initialize_logging, ModelSelected, TokenSource};
 use rust_mcp_sdk::schema::LATEST_PROTOCOL_VERSION;
+use tokio::join;
 use tracing::info;
 
 use mistralrs_server_core::{
@@ -26,7 +27,7 @@ struct Args {
 
     /// Port to serve on.
     #[arg(short, long)]
-    port: Option<String>,
+    port: Option<u16>,
 
     /// Log all responses and requests to this file
     #[clap(long, short)]
@@ -191,45 +192,54 @@ async fn main() -> Result<()> {
         return Ok(());
     }
 
-    // Needs to be after the .build call as that is where the daemon waits.
-    let setting_server = if !args.interactive_mode {
-        let port = args.port.expect("Interactive mode was not specified, so expected port to be specified. Perhaps you forgot `-i` or `--port`?");
-        let ip = args
-            .serve_ip
-            .clone()
-            .unwrap_or_else(|| "0.0.0.0".to_string());
-
-        // Create listener early to validate address before model loading
-        let listener = tokio::net::TcpListener::bind(format!("{ip}:{port}")).await?;
-        Some((listener, ip, port))
-    } else {
-        None
-    };
+    if !args.interactive_mode && args.port.is_none() && args.mcp_port.is_none() {
+        anyhow::bail!("Interactive mode was not specified, so expected port to be specified. Perhaps you forgot `-i` or `--port` or `--mcp-port`?")
+    }
 
-    if let Some(port) = args.mcp_port {
+    let mcp_port = if let Some(port) = args.mcp_port {
         let host = args
             .serve_ip
             .clone()
             .unwrap_or_else(|| "0.0.0.0".to_string());
         info!("MCP server listening on http://{host}:{port}.");
         info!("MCP protocol version is {}.", LATEST_PROTOCOL_VERSION);
         let mcp_server = mcp_server::create_mcp_server(mistralrs.clone(), host, port);
+
         tokio::spawn(async move {
             if let Err(e) = mcp_server.start().await {
                 eprintln!("MCP server error: {e}");
             }
-        });
-    }
+        })
+    } else {
+        tokio::spawn(async {})
+    };
 
-    let app = MistralRsServerRouterBuilder::new()
-        .with_mistralrs(mistralrs)
-        .build()
-        .await?;
+    let oai_port = if let Some(port) = args.port {
+        let ip = args
+            .serve_ip
+            .clone()
+            .unwrap_or_else(|| "0.0.0.0".to_string());
+
+        // Create listener early to validate address before model loading
+        let listener = tokio::net::TcpListener::bind(format!("{ip}:{port}")).await?;
+
+        let app = MistralRsServerRouterBuilder::new()
+            .with_mistralrs(mistralrs)
+            .build()
+            .await?;
 
-    if let Some((listener, ip, port)) = setting_server {
         info!("OpenAI-compatible server listening on http://{ip}:{port}.");
-        axum::serve(listener, app).await?;
+
+        tokio::spawn(async move {
+            if let Err(e) = axum::serve(listener, app).await {
+                eprintln!("OpenAI server error: {e}");
+            }
+        })
+    } else {
+        tokio::spawn(async {})
     };
 
+    let (_, _) = join!(oai_port, mcp_port);
+
     Ok(())
 }