feat: Multiple inference engines for nitro and openai

.gitignore

@@ -17,7 +17,7 @@ package-lock.json
 core/lib/**
 
 # Nitro binary files
-extensions/inference-extension/nitro/*/nitro
-extensions/inference-extension/nitro/*/*.exe
-extensions/inference-extension/nitro/*/*.dll
-extensions/inference-extension/nitro/*/*.metal
+extensions/inference-nitro-extension/bin/*/nitro
+extensions/inference-nitro-extension/bin/*/*.exe
+extensions/inference-nitro-extension/bin/*/*.dll
+extensions/inference-nitro-extension/bin/*/*.metal

core/src/events.ts

@@ -8,6 +8,18 @@ export enum EventName {
   OnMessageResponse = "OnMessageResponse",
   /** The `OnMessageUpdate` event is emitted when a message is updated. */
   OnMessageUpdate = "OnMessageUpdate",
+  /** The `OnModelInit` event is emitted when a model inits. */
+  OnModelInit = "OnModelInit",
+  /** The `OnModelReady` event is emitted when a model ready. */
+  OnModelReady = "OnModelReady",
+  /** The `OnModelFail` event is emitted when a model fails loading. */
+  OnModelFail = "OnModelFail",
+  /** The `OnModelStop` event is emitted when a model start to stop. */
+  OnModelStop = "OnModelStop",
+  /** The `OnModelStopped` event is emitted when a model stopped ok. */
+  OnModelStopped = "OnModelStopped",
+  /** The `OnInferenceStopped` event is emitted when a inference is stopped. */
+  OnInferenceStopped = "OnInferenceStopped",
 }
 
 /**

core/src/extensions/inference.ts

@@ -5,26 +5,10 @@ import { BaseExtension } from "../extension";
  * Inference extension. Start, stop and inference models.
  */
 export abstract class InferenceExtension extends BaseExtension {
-  /**
-   * Initializes the model for the extension.
-   * @param modelId - The ID of the model to initialize.
-   */
-  abstract initModel(modelId: string, settings?: ModelSettingParams): Promise<void>;
-
-  /**
-   * Stops the model for the extension.
-   */
-  abstract stopModel(): Promise<void>;
-
-  /**
-   * Stops the streaming inference.
-   */
-  abstract stopInference(): Promise<void>;
-
   /**
    * Processes an inference request.
    * @param data - The data for the inference request.
    * @returns The result of the inference request.
    */
-  abstract inferenceRequest(data: MessageRequest): Promise<ThreadMessage>;
+  abstract inference(data: MessageRequest): Promise<ThreadMessage>;
 }

core/src/fs.ts

@@ -5,81 +5,82 @@
  * @returns {Promise<any>} A Promise that resolves when the file is written successfully.
  */
 const writeFile: (path: string, data: string) => Promise<any> = (path, data) =>
-  global.core.api?.writeFile(path, data);
+  global.core.api?.writeFile(path, data)
 
 /**
  * Checks whether the path is a directory.
  * @param path - The path to check.
  * @returns {boolean} A boolean indicating whether the path is a directory.
  */
-const isDirectory = (path: string): Promise<boolean> =>
-  global.core.api?.isDirectory(path);
+const isDirectory = (path: string): Promise<boolean> => global.core.api?.isDirectory(path)
 
 /**
  * Reads the contents of a file at the specified path.
  * @param {string} path - The path of the file to read.
  * @returns {Promise<any>} A Promise that resolves with the contents of the file.
  */
-const readFile: (path: string) => Promise<any> = (path) =>
-  global.core.api?.readFile(path);
+const readFile: (path: string) => Promise<any> = (path) => global.core.api?.readFile(path)
+/**
+ * Check whether the file exists
+ * @param {string} path
+ * @returns {boolean} A boolean indicating whether the path is a file.
+ */
+const exists = (path: string): Promise<boolean> => global.core.api?.exists(path)
 /**
  * List the directory files
  * @param {string} path - The path of the directory to list files.
  * @returns {Promise<any>} A Promise that resolves with the contents of the directory.
  */
-const listFiles: (path: string) => Promise<any> = (path) =>
-  global.core.api?.listFiles(path);
+const listFiles: (path: string) => Promise<any> = (path) => global.core.api?.listFiles(path)
 /**
  * Creates a directory at the specified path.
  * @param {string} path - The path of the directory to create.
  * @returns {Promise<any>} A Promise that resolves when the directory is created successfully.
  */
-const mkdir: (path: string) => Promise<any> = (path) =>
-  global.core.api?.mkdir(path);
+const mkdir: (path: string) => Promise<any> = (path) => global.core.api?.mkdir(path)
 
 /**
  * Removes a directory at the specified path.
  * @param {string} path - The path of the directory to remove.
  * @returns {Promise<any>} A Promise that resolves when the directory is removed successfully.
  */
-const rmdir: (path: string) => Promise<any> = (path) =>
-  global.core.api?.rmdir(path);
+const rmdir: (path: string) => Promise<any> = (path) => global.core.api?.rmdir(path)
 /**
  * Deletes a file from the local file system.
  * @param {string} path - The path of the file to delete.
  * @returns {Promise<any>} A Promise that resolves when the file is deleted.
  */
-const deleteFile: (path: string) => Promise<any> = (path) =>
-  global.core.api?.deleteFile(path);
+const deleteFile: (path: string) => Promise<any> = (path) => global.core.api?.deleteFile(path)
 
 /**
  * Appends data to a file at the specified path.
  * @param path path to the file
  * @param data data to append
  */
 const appendFile: (path: string, data: string) => Promise<any> = (path, data) =>
-  global.core.api?.appendFile(path, data);
+  global.core.api?.appendFile(path, data)
 
 const copyFile: (src: string, dest: string) => Promise<any> = (src, dest) =>
-  global.core.api?.copyFile(src, dest);
+  global.core.api?.copyFile(src, dest)
 
 /**
  * Reads a file line by line.
  * @param {string} path - The path of the file to read.
  * @returns {Promise<any>} A promise that resolves to the lines of the file.
  */
 const readLineByLine: (path: string) => Promise<any> = (path) =>
-  global.core.api?.readLineByLine(path);
+  global.core.api?.readLineByLine(path)
 
 export const fs = {
   isDirectory,
   writeFile,
   readFile,
+  exists,
   listFiles,
   mkdir,
   rmdir,
   deleteFile,
   appendFile,
   readLineByLine,
   copyFile,
-};
+}

core/src/types/index.ts

@@ -41,8 +41,8 @@ export type MessageRequest = {
   /** Messages for constructing a chat completion request **/
   messages?: ChatCompletionMessage[];
 
-  /** Runtime parameters for constructing a chat completion request **/
-  parameters?: ModelRuntimeParam;
+  /** Settings for constructing a chat completion request **/
+  model?: ModelInfo;
 };
 
 /**
@@ -153,7 +153,8 @@ export type ThreadAssistantInfo = {
 export type ModelInfo = {
   id: string;
   settings: ModelSettingParams;
-  parameters: ModelRuntimeParam;
+  parameters: ModelRuntimeParams;
+  engine?: InferenceEngine;
 };
 
 /**
@@ -166,6 +167,17 @@ export type ThreadState = {
   error?: Error;
   lastMessage?: string;
 };
+/**
+ * Represents the inference engine.
+ * @stored
+ */
+
+enum InferenceEngine {
+  nitro = "nitro",
+  openai = "openai",
+  nvidia_triton = "nvidia_triton",
+  hf_endpoint = "hf_endpoint",
+}
 
 /**
  * Model type defines the shape of a model object.
@@ -228,12 +240,16 @@ export interface Model {
   /**
    * The model runtime parameters.
    */
-  parameters: ModelRuntimeParam;
+  parameters: ModelRuntimeParams;
 
   /**
    * Metadata of the model.
    */
   metadata: ModelMetadata;
+  /**
+   * The model engine.
+   */
+  engine: InferenceEngine;
 }
 
 export type ModelMetadata = {
@@ -268,7 +284,7 @@ export type ModelSettingParams = {
 /**
  * The available model runtime parameters.
  */
-export type ModelRuntimeParam = {
+export type ModelRuntimeParams = {
   temperature?: number;
   token_limit?: number;
   top_k?: number;

docs/openapi/specs/models.yaml

@@ -289,7 +289,7 @@ components:
             engine:
               type: string
               description: "The engine used by the model."
-              example: "llamacpp"
+              enum: [nitro, openai, hf_inference]
             quantization:
               type: string
               description: "Quantization parameter of the model."

electron/handlers/fs.ts

@@ -50,6 +50,19 @@ export function handleFsIPCs() {
     })
   })
 
+  /**
+   * Checks whether a file exists in the user data directory.
+   * @param event - The event object.
+   * @param path - The path of the file to check.
+   * @returns A promise that resolves with a boolean indicating whether the file exists.
+   */
+  ipcMain.handle('exists', async (_event, path: string) => {
+    return new Promise((resolve, reject) => {
+      const fullPath = join(userSpacePath, path)
+      fs.existsSync(fullPath) ? resolve(true) : resolve(false)
+    })
+  })
+
   /**
    * Writes data to a file in the user data directory.
    * @param event - The event object.

electron/invokers/fs.ts

@@ -27,6 +27,12 @@ export function fsInvokers() {
      */
     readFile: (path: string) => ipcRenderer.invoke('readFile', path),
 
+    /**
+     * Reads a file at the specified path.
+     * @param {string} path - The path of the file to read.
+     */
+    exists: (path: string) => ipcRenderer.invoke('exists', path),
+
     /**
      * Writes data to a file at the specified path.
      * @param {string} path - The path of the file to write to.

extensions/inference-nitro-extension/download.bat

@@ -0,0 +1,3 @@
+@echo off
+set /p NITRO_VERSION=<./bin/version.txt
+.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda.tar.gz -e --strip 1 -o ./bin/win-cuda && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu

extensions/inference-nitro-extension/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@janhq/inference-nitro-extension",
+  "version": "1.0.0",
+  "description": "Inference Engine for Nitro Extension, powered by @janhq/nitro, bring a high-performance Llama model inference in pure C++.",
+  "main": "dist/index.js",
+  "module": "dist/module.js",
+  "author": "Jan <[email protected]>",
+  "license": "AGPL-3.0",
+  "scripts": {
+    "build": "tsc -b . && webpack --config webpack.config.js",
+    "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && chmod +x ./bin/linux-start.sh && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda.tar.gz -e --strip 1 -o ./bin/linux-cuda && chmod +x ./bin/linux-cuda/nitro && chmod +x ./bin/linux-start.sh",
+    "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro",
+    "downloadnitro:win32": "download.bat",
+    "downloadnitro": "run-script-os",
+    "build:publish:darwin": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../electron/pre-install",
+    "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../electron/pre-install",
+    "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" &&  npm pack && cpx *.tgz ../../electron/pre-install",
+    "build:publish": "run-script-os"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./main": "./dist/module.js"
+  },
+  "devDependencies": {
+    "cpx": "^1.5.0",
+    "rimraf": "^3.0.2",
+    "run-script-os": "^1.1.6",
+    "webpack": "^5.88.2",
+    "webpack-cli": "^5.1.4"
+  },
+  "dependencies": {
+    "@janhq/core": "file:../../core",
+    "download-cli": "^1.1.1",
+    "electron-log": "^5.0.1",
+    "fetch-retry": "^5.0.6",
+    "kill-port": "^2.0.1",
+    "path-browserify": "^1.0.1",
+    "rxjs": "^7.8.1",
+    "tcp-port-used": "^1.0.2",
+    "ts-loader": "^9.5.0",
+    "ulid": "^2.3.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "dist/*",
+    "package.json",
+    "README.md"
+  ],
+  "bundleDependencies": [
+    "tcp-port-used",
+    "kill-port",
+    "fetch-retry",
+    "electron-log"
+  ]
+}