Merge pull request #4486 from janhq/main

chore: sync from main into dev

Author: louis-jan

core/src/browser/extensions/engines/helpers/sse.ts

@@ -91,8 +91,12 @@ export function requestInference(
                   const toParse = cachedLines + line
                   if (!line.includes('data: [DONE]')) {
                     const data = JSON.parse(toParse.replace('data: ', ''))
-                    if ('error' in data) {
-                      subscriber.error(data.error)
+                    if (
+                      'error' in data ||
+                      'message' in data ||
+                      'detail' in data
+                    ) {
+                      subscriber.error(data.error ?? data)
                       subscriber.complete()
                       return
                     }

extensions/engine-management-extension/models/anthropic.json

@@ -8,7 +8,7 @@
     "inference_params": {
       "max_tokens": 4096,
       "temperature": 0.7,
-      "stream": false
+      "stream": true
     },
     "engine": "anthropic"
   },
@@ -21,7 +21,7 @@
     "inference_params": {
       "max_tokens": 8192,
       "temperature": 0.7,
-      "stream": false
+      "stream": true
     },
     "engine": "anthropic"
   },

extensions/engine-management-extension/models/martian.json

@@ -5,7 +5,7 @@
     "name": "Martian Model Router",
     "version": "1.0",
     "description": "Martian Model Router dynamically routes requests to the best LLM in real-time",
-    "parameters": {
+    "inference_params": {
       "max_tokens": 4096,
       "temperature": 0.7,
       "top_p": 0.95,

extensions/engine-management-extension/models/mistral.json

@@ -5,7 +5,7 @@
     "name": "Mistral Small",
     "version": "1.1",
     "description": "Mistral Small is the ideal choice for simple tasks (Classification, Customer Support, or Text Generation) at an affordable price.",
-    "parameters": {
+    "inference_params": {
       "max_tokens": 32000,
       "temperature": 0.7,
       "top_p": 0.95,
@@ -19,7 +19,7 @@
     "name": "Mistral Large",
     "version": "1.1",
     "description": "Mistral Large is ideal for complex tasks (Synthetic Text Generation, Code Generation, RAG, or Agents).",
-    "parameters": {
+    "inference_params": {
       "max_tokens": 32000,
       "temperature": 0.7,
       "top_p": 0.95,
@@ -33,7 +33,7 @@
     "name": "Mixtral 8x22B",
     "version": "1.1",
     "description": "Mixtral 8x22B is a high-performance, cost-effective model designed for complex tasks.",
-    "parameters": {
+    "inference_params": {
       "max_tokens": 32000,
       "temperature": 0.7,
       "top_p": 0.95,

extensions/engine-management-extension/models/nvidia.json

@@ -5,7 +5,7 @@
     "name": "Mistral 7B",
     "version": "1.1",
     "description": "Mistral 7B with NVIDIA",
-    "parameters": {
+    "inference_params": {
       "max_tokens": 1024,
       "temperature": 0.3,
       "top_p": 1,

extensions/engine-management-extension/models/openrouter.json

@@ -5,7 +5,7 @@
     "name": "OpenRouter",
     "version": "1.0",
     "description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
-    "parameters": {
+    "inference_params": {
       "max_tokens": 128000,
       "temperature": 0.7,
       "top_p": 0.95,

extensions/engine-management-extension/resources/anthropic.json

@@ -15,7 +15,7 @@
     },
     "transform_resp": {
       "chat_completions": {
-        "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.delta.text }}\" {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": \"{{ input_request.model }}\", \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"{{ input_request.role }}\", \"content\": \"{% if input_request.content and input_request.content.0.type == \"text\" %} \"{{input_request.content.0.text}}\" {% endif %}\", \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.stop_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.usage.input_tokens }}, \"completion_tokens\": {{ input_request.usage.output_tokens }}, \"total_tokens\": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 }, \"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}"
+        "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.delta.text }}\" {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": \"{{ input_request.model }}\", \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"{{ input_request.role }}\", \"content\": {% if input_request.content and input_request.content.0.type == \"text\" %} \"{{input_request.content.0.text}}\" {% else %} null {% endif %}, \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.stop_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.usage.input_tokens }}, \"completion_tokens\": {{ input_request.usage.output_tokens }}, \"total_tokens\": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 }, \"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}"
       }
     }
   }

extensions/engine-management-extension/resources/mistral.json

@@ -15,7 +15,7 @@
     },
     "transform_resp": {
       "chat_completions": {
-        "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
+        "template": "{{tojson(input_request)}}"
       }
     }
   }

extensions/inference-cortex-extension/bin/version.txt

@@ -1 +1 @@
-1.0.9-rc3
+1.0.9-rc4

web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx

@@ -1,29 +1,28 @@
 import { Tooltip, Button, Badge } from '@janhq/joi'
 
-import { useAtom, useAtomValue } from 'jotai'
+import { useAtom } from 'jotai'
 
 import { useActiveModel } from '@/hooks/useActiveModel'
 
+import { useGetEngines } from '@/hooks/useEngineManagement'
+
 import { toGibibytes } from '@/utils/converter'
 
 import { isLocalEngine } from '@/utils/modelEngine'
 
-import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom'
 import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom'
 
 const TableActiveModel = () => {
   const { activeModel, stateModel, stopModel } = useActiveModel()
-  const engines = useAtomValue(installedEnginesAtom)
+  const { engines } = useGetEngines()
 
   const [serverEnabled, setServerEnabled] = useAtom(serverEnabledAtom)
 
   return (
     <div className="w-1/2">
       <div className="overflow-hidden border-b border-[hsla(var(--app-border))]">
         <table className="w-full px-8">
-          {activeModel &&
-          engines &&
-          isLocalEngine(engines, activeModel.engine) ? (
+          {activeModel && isLocalEngine(engines, activeModel.engine) ? (
             <tbody>
               <tr>
                 <td