fix: 4238 - fix default max_tokens set on remote models

extensions/inference-openai-extension/resources/models.json

@@ -99,10 +99,10 @@
     "format": "api",
     "settings": {},
     "parameters": {
+      "max_tokens": 32768,
       "temperature": 1,
       "top_p": 1,
       "stream": true,
-      "max_tokens": 32768,
       "frequency_penalty": 0,
       "presence_penalty": 0
     },
@@ -126,9 +126,9 @@
     "format": "api",
     "settings": {},
     "parameters": {
+      "max_tokens": 65536,
       "temperature": 1,
       "top_p": 1,
-      "max_tokens": 65536,
       "stream": true,
       "frequency_penalty": 0,
       "presence_penalty": 0

web/containers/ModelDropdown/index.tsx

@@ -99,7 +99,7 @@
   )
 
   const isModelSupportRagAndTools = useCallback((model: Model) => {
     return (
       model?.engine === InferenceEngine.openai ||
       isLocalEngine(model?.engine as InferenceEngine)
     )
@@ -110,7 +110,7 @@
       configuredModels
         .concat(
           downloadedModels.filter(
             (e) => !configuredModels.some((x) => x.id === e.id)
           )
         )
         .filter((e) =>
@@ -120,24 +120,24 @@
           if (searchFilter === 'local') {
             return isLocalEngine(e.engine)
           }
           if (searchFilter === 'remote') {
             return !isLocalEngine(e.engine)
           }
         })
         .sort((a, b) => a.name.localeCompare(b.name))
         .sort((a, b) => {
           const aInDownloadedModels = downloadedModels.some(
             (item) => item.id === a.id
           )
           const bInDownloadedModels = downloadedModels.some(
             (item) => item.id === b.id
           )
           if (aInDownloadedModels && !bInDownloadedModels) {
             return -1
           } else if (!aInDownloadedModels && bInDownloadedModels) {
             return 1
           } else {
             return 0
           }
         }),
     [configuredModels, searchText, searchFilter, downloadedModels]
@@ -145,7 +145,7 @@
 
   useEffect(() => {
     if (open && searchInputRef.current) {
       searchInputRef.current.focus()
     }
   }, [open])
 
@@ -162,9 +162,9 @@
 
   const onClickModelItem = useCallback(
     async (modelId: string) => {
       const model = downloadedModels.find((m) => m.id === modelId)
       setSelectedModel(model)
       setOpen(false)
 
       if (activeThread) {
         // Change assistand tools based on model support RAG
@@ -192,8 +192,12 @@
           model?.settings.ctx_len ?? 8192
         )
         const overriddenParameters = {
-          ctx_len: Math.min(8192, model?.settings.ctx_len ?? 8192),
-          max_tokens: defaultContextLength,
+          ctx_len: !isLocalEngine(model?.engine)
+            ? undefined
+            : defaultContextLength,
+          max_tokens: !isLocalEngine(model?.engine)
+            ? (model?.parameters.max_tokens ?? 8192)
+            : defaultContextLength,
         }
 
         const modelParams = {

web/hooks/useCreateNewThread.ts

@@ -17,6 +17,7 @@ import { fileUploadAtom } from '@/containers/Providers/Jotai'
 
 import { toaster } from '@/containers/Toast'
 
+import { isLocalEngine } from '@/utils/modelEngine'
 import { generateThreadId } from '@/utils/thread'
 
 import { useActiveModel } from './useActiveModel'
@@ -113,12 +114,14 @@ export const useCreateNewThread = () => {
     )
 
     const overriddenSettings = {
-      ctx_len: defaultContextLength,
+      ctx_len: !isLocalEngine(model?.engine) ? undefined : defaultContextLength,
     }
 
     // Use ctx length by default
     const overriddenParameters = {
-      max_tokens: defaultContextLength,
+      max_tokens: !isLocalEngine(model?.engine)
+        ? (model?.parameters.token_limit ?? 8192)
+        : defaultContextLength,
     }
 
     const createdAt = Date.now()

web/utils/modelEngine.ts

@@ -38,7 +38,9 @@ export const getLogoEngine = (engine: InferenceEngine) => {
  * @param engine
  * @returns
  */
-export const isLocalEngine = (engine: string) => {
+export const isLocalEngine = (engine?: string) => {
+  if (!engine) return false
+
   const engineObj = EngineManager.instance().get(engine)
   if (!engineObj) return false
   return (