WIP

Author: matthewhaynesonline

mistralrs-server-core/src/chat_completion.rs

@@ -23,8 +23,9 @@ use either::Either;
 use indexmap::IndexMap;
 use itertools::Itertools;
 use mistralrs_core::{
-    ChatCompletionResponse, Constraint, DrySamplingParams, MistralRs, NormalRequest, Request,
-    RequestMessage, Response, SamplingParams, StopTokens as InternalStopTokens,
+    ChatCompletionChunkResponse, ChatCompletionResponse, Constraint, DrySamplingParams, MistralRs,
+    NormalRequest, Request, RequestMessage, Response, SamplingParams,
+    StopTokens as InternalStopTokens,
 };
 use serde::Serialize;
 
@@ -47,6 +48,7 @@ pub struct Streamer {
     rx: Receiver<Response>,
     done_state: DoneState,
     state: SharedMistralState,
+    chunks: Vec<ChatCompletionChunkResponse>,
 }
 
 impl futures::Stream for Streamer {
@@ -64,6 +66,7 @@ impl futures::Stream for Streamer {
                 return Poll::Ready(Some(Ok(Event::default().data("[DONE]"))));
             }
             DoneState::Done => {
+                dbg!(&self.chunks);
                 return Poll::Ready(None);
             }
             DoneState::Running => (),
@@ -474,10 +477,12 @@ pub async fn send_request(state: &SharedMistralState, request: Request) -> Resul
 }
 
 pub fn create_chat_streamer(rx: Receiver<Response>, state: SharedMistralState) -> Sse<Streamer> {
+    let chunks: Vec<_> = vec![];
     let streamer = Streamer {
         rx,
         done_state: DoneState::Running,
         state,
+        chunks,
     };
 
     let keep_alive_interval = get_keep_alive_interval();