fix: Add hack waiting for model loading

Author: hiro-v

extensions/inference-nitro-extension/src/index.ts

@@ -198,7 +198,6 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
     instance: JanInferenceNitroExtension
   ) {
     if (data.model.engine !== 'nitro') { return }
-
     const timestamp = Date.now();
     const message: ThreadMessage = {
       id: ulid(),

web/hooks/useSendChatMessage.ts

@@ -1,4 +1,4 @@
-import { useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 
 import {
   ChatCompletionMessage,
@@ -11,6 +11,7 @@ import {
   Thread,
   ThreadMessage,
   events,
+  Model,
 } from '@janhq/core'
 import { ConversationalExtension } from '@janhq/core'
 import { useAtom, useAtomValue, useSetAtom } from 'jotai'
@@ -48,6 +49,11 @@ export default function useSendChatMessage() {
   const { startModel } = useActiveModel()
   const [queuedMessage, setQueuedMessage] = useState(false)
 
+  const modelRef = useRef<Model | undefined>()
+  useEffect(() => {
+    modelRef.current = activeModel
+  }, [activeModel])
+
   const resendChatMessage = async () => {
     if (!activeThread) {
       console.error('No active thread')
@@ -78,19 +84,35 @@ export default function useSendChatMessage() {
       id: ulid(),
       messages: messages,
       threadId: activeThread.id,
-      model: activeThread.assistants[0].model??selectedModel,
+      model: activeThread.assistants[0].model ?? selectedModel,
     }
 
     const modelId = selectedModel?.id ?? activeThread.assistants[0].model.id
 
     if (activeModel?.id !== modelId) {
       setQueuedMessage(true)
-      await startModel(modelId)
+      startModel(modelId)
+      await WaitForModelStarting(modelId)
       setQueuedMessage(false)
     }
     events.emit(EventName.OnMessageSent, messageRequest)
   }
 
+  // TODO: Refactor @louis
+  const WaitForModelStarting = async (modelId: string) => {
+    return new Promise<void>((resolve) => {
+      setTimeout(async () => {
+        if (modelRef.current?.id !== modelId) {
+          console.log('waiting for model to start')
+          await WaitForModelStarting(modelId)
+          resolve()
+        } else {
+          resolve()
+        }
+      }, 200)
+    })
+  }
+
   const sendChatMessage = async () => {
     if (!currentPrompt || currentPrompt.trim().length === 0) {
       return
@@ -167,8 +189,10 @@ export default function useSendChatMessage() {
       id: msgId,
       threadId: activeThread.id,
       messages,
-      parameters: activeThread.assistants[0].model.parameters,
-      model: selectedModel??activeThread.assistants[0].model,
+      parameters:
+        selectedModel?.parameters ??
+        activeThread.assistants[0].model.parameters,
+      model: selectedModel ?? activeThread.assistants[0].model,
     }
     const timestamp = Date.now()
     const threadMessage: ThreadMessage = {
@@ -200,9 +224,11 @@ export default function useSendChatMessage() {
 
     if (activeModel?.id !== modelId) {
       setQueuedMessage(true)
-      await startModel(modelId)
+      startModel(modelId)
+      await WaitForModelStarting(modelId)
       setQueuedMessage(false)
     }
+    console.log('messageRequest', messageRequest)
     events.emit(EventName.OnMessageSent, messageRequest)
   }