Allow use embeddings with local models

core/src/browser/extensions/engines/AIEngine.ts

@@ -255,7 +255,7 @@ export abstract class AIEngine extends BaseExtension {
   /**
    * Loads a model into memory
    */
-  abstract load(modelId: string, settings?: any): Promise<SessionInfo>
+  abstract load(modelId: string, settings?: any, isEmbedding?: boolean): Promise<SessionInfo>
 
   /**
    * Unloads a model from memory
@@ -299,4 +299,10 @@ export abstract class AIEngine extends BaseExtension {
    * @param modelId
    */
   abstract isToolSupported(modelId: string): Promise<boolean>
+
+  /**
+   * Check if a embedding is supported by the model
+   * @param modelId
+   */
+  abstract isEmbeddingsSupported(modelId: string): Promise<boolean>
 }

extensions-web/src/jan-provider-web/provider.ts

@@ -391,4 +391,11 @@ export default class JanProviderWeb extends AIEngine {
     console.log(`Checking tool support for Jan model ${modelId}: supported`)
     return true
   }
-}
+
+  async isEmbeddingsSupported(modelId: string): Promise<boolean> {
+    // Jan models need to check from wed for embeddings support
+    console.log(`Checking embeddings support for Jan model ${modelId}: not supported`);
+    return false;
+  }
+
+}

extensions/llamacpp-extension/src/index.ts

@@ -1668,7 +1668,7 @@ export default class llamacpp_extension extends AIEngine {
     if (cfg.no_kv_offload) args.push('--no-kv-offload')
     if (isEmbedding) {
       args.push('--embedding')
-      args.push('--pooling', 'mean')
+      //args.push('--pooling mean')
     } else {
       if (cfg.ctx_size > 0) args.push('--ctx-size', String(cfg.ctx_size))
       if (cfg.n_predict > 0) args.push('--n-predict', String(cfg.n_predict))
@@ -1683,7 +1683,6 @@ export default class llamacpp_extension extends AIEngine {
       }
       if (cfg.defrag_thold && cfg.defrag_thold != 0.1)
         args.push('--defrag-thold', String(cfg.defrag_thold))
-
       if (cfg.rope_scaling && cfg.rope_scaling != 'none')
         args.push('--rope-scaling', cfg.rope_scaling)
       if (cfg.rope_scale && cfg.rope_scale != 1)
@@ -2190,6 +2189,37 @@ export default class llamacpp_extension extends AIEngine {
       'tokenizer.chat_template'
     ]?.includes('tools')
   }
+
+  /**
+   * Check if a embeddings is supported by the model
+   * Currently read from GGUF metadata
+   * @param modelId
+   * @returns
+   */
+  async isEmbeddingsSupported(modelId: string): Promise<boolean> {
+    const janDataFolderPath = await getJanDataFolderPath()
+    const modelConfigPath = await joinPath([
+      this.providerPath,
+      'models',
+      modelId,
+      'model.yml',
+    ])
+    const modelConfig = await invoke<ModelConfig>('read_yaml', {
+      path: modelConfigPath,
+    })
+    // model option is required
+    // NOTE: model_path and mmproj_path can be either relative to Jan's data folder or absolute path
+    const modelPath = await joinPath([
+      janDataFolderPath,
+      modelConfig.model_path,
+    ])
+
+    const metadata = await readGgufMetadata(modelPath);
+    const hasPoolingTypeKey = Object.keys(metadata?.metadata).some(key => key.includes("pooling_type"));
+
+    return hasPoolingTypeKey;
+  }
+
   /**
    * Get total system memory including both VRAM and RAM
    */

src-tauri/src/core/server/proxy.rs

@@ -564,8 +564,10 @@ async fn proxy_request(
                 .unwrap());
         }
     };
+    // Redirect to OpenAI compatible endpoint /v1 on llama.cpp server
+    //https://github.com/ggml-org/llama.cpp/tree/master/tools/server
 
-    let upstream_url = format!("http://127.0.0.1:{port}{destination_path}");
+    let upstream_url = format!("http://127.0.0.1:{}/v1{}", port, destination_path);
 
     let mut outbound_req = client.request(method.clone(), &upstream_url);
 

web-app/src/containers/dialogs/EditModel.tsx

@@ -16,6 +16,9 @@ import {
   IconEye,
   IconTool,
   IconAlertTriangle,
+  // IconWorld,
+  // IconAtom,
+  IconCodeCircle2,
   IconLoader2,
   IconSparkles,
 } from '@tabler/icons-react'
@@ -288,6 +291,50 @@ export const DialogEditModel = ({
                 disabled={isLoading || !(capabilities.tools && capabilities.vision)}
               />
             </div>
+
+            <div className="flex items-center justify-between">
+              <div className="flex items-center space-x-2">
+                <IconCodeCircle2 className="size-4 text-main-view-fg/70" />
+                <span className="text-sm">
+                  {t('providers:editModel.embeddings')}
+                </span>
+              </div>
+              <Switch
+                id="embedding-capability"
+                checked={capabilities.embeddings}
+                onCheckedChange={(checked) =>
+                  handleCapabilityChange('embeddings', checked)
+                }
+              />
+            </div>
+
+            {/* <div className="flex items-center justify-between">
+              <div className="flex items-center space-x-2">
+                <IconWorld className="size-4 text-main-view-fg/70" />
+                <span className="text-sm">Web Search</span>
+              </div>
+              <Switch
+                id="web_search-capability"
+                checked={capabilities.web_search}
+                onCheckedChange={(checked) =>
+                  handleCapabilityChange('web_search', checked)
+                }
+              />
+            </div> */}
+
+            {/* <div className="flex items-center justify-between">
+              <div className="flex items-center space-x-2">
+                <IconAtom className="size-4 text-main-view-fg/70" />
+                <span className="text-sm">{t('reasoning')}</span>
+              </div>
+              <Switch
+                id="reasoning-capability"
+                checked={capabilities.reasoning}
+                onCheckedChange={(checked) =>
+                  handleCapabilityChange('reasoning', checked)
+                }
+              />
+            </div> */}
           </div>
         </div>
 

web-app/src/services/__tests__/models.test.ts

@@ -47,6 +47,7 @@ describe('DefaultModelsService', () => {
     isModelSupported: vi.fn(),
     isToolSupported: vi.fn(),
     checkMmprojExists: vi.fn(),
+    isEmbeddingsSupported: vi.fn(),
   }
 
   const mockEngineManager = {

web-app/src/services/models/default.ts

@@ -328,7 +328,15 @@
         )
       : undefined
 
-    return engine.load(model, settings).catch((error) => {
+      const selectedModel = provider.models.find((m: any) => m.id === model)
+      var isEmbeddings = false;
+      if (selectedModel) {
+        const modelCapabilities = selectedModel.capabilities || []
+        isEmbeddings = modelCapabilities.includes('embeddings')
+      }
+
+
+    return engine.load(model, settings, isEmbeddings).catch((error) => {
       console.error(
         `Failed to start model ${model} for provider ${provider.provider}:`,
         error
@@ -344,6 +352,13 @@
     return engine.isToolSupported(modelId)
   }
 
+  async isEmbeddingsSupported(modelId: string): Promise<boolean> {
+    const engine = this.getEngine()
+    if (!engine) return false
+
+    return engine.isEmbeddingsSupported(modelId)
+  }
+
   async checkMmprojExistsAndUpdateOffloadMMprojSetting(
     modelId: string,
     updateProvider?: (

web-app/src/services/models/types.ts

@@ -125,6 +125,7 @@ export interface ModelsService {
     model: string
   ): Promise<SessionInfo | undefined>
   isToolSupported(modelId: string): Promise<boolean>
+  isEmbeddingsSupported(modelId: string): Promise<boolean>
   checkMmprojExistsAndUpdateOffloadMMprojSetting(
     modelId: string,
     updateProvider?: (

web-app/src/services/providers/tauri.ts

@@ -78,7 +78,7 @@ export class TauriProvidersService extends DefaultProvidersService {
                 try {
                   const toolSupported = await value.isToolSupported(model.id)
                   if (toolSupported) {
-                    capabilities = [ModelCapabilities.TOOLS]
+                    capabilities.push(ModelCapabilities.TOOLS);
                   }
                 } catch (error) {
                   console.warn(
@@ -87,6 +87,18 @@ export class TauriProvidersService extends DefaultProvidersService {
                   )
                   // Continue without tool capabilities if check fails
                 }
+
+                // Try to check embeddings support, but don't let failures block the model
+                try {
+                  const embeddingsSupported = await value.isEmbeddingsSupported(model.id);
+                  if (embeddingsSupported) {
+                    capabilities.push(ModelCapabilities.EMBEDDINGS);
+                  }
+                } catch (error) {
+                  console.warn(`Failed to check embeddings support for model ${model.id}:`, error);
+                  // Continue without embeddings capabilities if check fails
+                }
+
               }
 
               return {