Skip to content

Commit dea29ff

Browse files
vansangpfievsangjanai
authored and
louis-jan
committed
fix: Jan supports cortex with llama-server (#4904)
* fix: update logic to pull llama-cpp engine * chore: bump cortex version * fix: chat macos * chore: bump llama.cpp * chore: download windows * chore: download linux * chore: bump cortex to 1.0.13-rc6 * fix: stop cortex before exit * chore: bum llama.cpp engine to b5350 --------- Co-authored-by: vansangpfiev <[email protected]>
1 parent f3a808c commit dea29ff

File tree

12 files changed

+108
-99
lines changed

12 files changed

+108
-99
lines changed

‎extensions/engine-management-extension/rolldown.config.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ export default defineConfig([
1515
`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
1616
),
1717
PLATFORM: JSON.stringify(process.platform),
18-
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'),
18+
CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
1919
DEFAULT_REMOTE_ENGINES: JSON.stringify(engines),
2020
DEFAULT_REMOTE_MODELS: JSON.stringify(models),
2121
DEFAULT_REQUEST_PAYLOAD_TRANSFORM: JSON.stringify(
@@ -38,7 +38,7 @@ export default defineConfig([
3838
file: 'dist/node/index.cjs.js',
3939
},
4040
define: {
41-
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'),
41+
CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
4242
},
4343
},
4444
])

‎extensions/engine-management-extension/src/node/index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@ const symlinkEngines = async () => {
1616
appResourcePath(),
1717
'shared',
1818
'engines',
19-
'cortex.llamacpp'
19+
'llama.cpp'
2020
)
2121
const symlinkEnginePath = path.join(
2222
getJanDataFolderPath(),
2323
'engines',
24-
'cortex.llamacpp'
24+
'llama.cpp'
2525
)
2626
const variantFolders = await readdir(sourceEnginePath)
2727
const isStandalone = process.platform === 'linux'

‎extensions/engine-management-extension/src/utils.ts‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,23 +32,23 @@ const gpuRunMode = (settings?: GpuSetting): RunMode => {
3232
*/
3333
const os = (settings?: GpuSetting): string => {
3434
return PLATFORM === 'win32'
35-
? 'windows-amd64'
35+
? 'win'
3636
: PLATFORM === 'darwin'
3737
? settings?.cpu?.arch === 'arm64'
38-
? 'mac-arm64'
39-
: 'mac-amd64'
40-
: 'linux-amd64'
38+
? 'macos-arm64'
39+
: 'macos-x64'
40+
: 'linux'
4141
}
4242

4343
/**
44-
* The CUDA version that will be set - either '11-7' or '12-0'.
44+
* The CUDA version that will be set - either 'cu12.0' or 'cu11.7'.
4545
* @param settings
4646
* @returns
4747
*/
48-
const cudaVersion = (settings?: GpuSetting): '12-0' | '11-7' | undefined => {
48+
const cudaVersion = (settings?: GpuSetting): 'cu12.0' | 'cu11.7' | undefined => {
4949
return settings.gpus?.some((gpu) => gpu.version.includes('12'))
50-
? '12-0'
51-
: '11-7'
50+
? 'cu12.0'
51+
: 'cu11.7'
5252
}
5353

5454
/**
@@ -84,15 +84,17 @@ export const engineVariant = async (
8484
: 'noavx',
8585
runMode,
8686
cudaVersion(gpuSetting),
87+
'x64',
8788
]
8889
: // For cpu only we need to check all available supported instructions
8990
[
9091
(gpuSetting.cpu?.instructions ?? ['noavx']).find((e) =>
9192
instructionBinaryNames.includes(e.toLowerCase())
9293
) ?? 'noavx',
94+
'x64',
9395
]),
9496
].filter(Boolean)
95-
: [platform, 'vulkan']
97+
: [platform, 'vulkan', 'x64']
9698

9799
let engineVariantString = engineVariant.join('-')
98100

‎extensions/inference-cortex-extension/bin/version.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.0.12
1+
1.0.13-rc6

‎extensions/inference-cortex-extension/download.bat‎

Lines changed: 22 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,38 @@
22
set BIN_PATH=./bin
33
set SHARED_PATH=./../../electron/shared
44
set /p CORTEX_VERSION=<./bin/version.txt
5-
set ENGINE_VERSION=0.1.56
5+
set ENGINE_VERSION=b5350
66

7-
@REM Download cortex.llamacpp binaries
8-
set DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64
9-
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%
10-
set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan
7+
@REM Download llama.cpp binaries
8+
set DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
9+
set DOWNLOAD_GGML_URL=https://github.com/ggml-org/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
10+
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%
11+
set SUBFOLDERS=win-noavx-cuda-cu12.0-x64 win-noavx-cuda-cu11.7-x64 win-avx2-cuda-cu12.0-x64 win-avx2-cuda-cu11.7-x64 win-noavx-x64 win-avx-x64 win-avx2-x64 win-avx512-x64 win-vulkan-x64
1112

1213
call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
13-
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-12-0/v%ENGINE_VERSION%
14-
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-11-7.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-11-7/v%ENGINE_VERSION%
15-
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx-cuda-12-0/v%ENGINE_VERSION%
16-
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-11-7.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx-cuda-11-7/v%ENGINE_VERSION%
17-
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx/v%ENGINE_VERSION%
18-
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx/v%ENGINE_VERSION%
19-
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2/v%ENGINE_VERSION%
20-
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx512/v%ENGINE_VERSION%
21-
call .\node_modules\.bin\download %DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-vulkan/v%ENGINE_VERSION%
22-
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
23-
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
14+
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-cu12.0-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-cuda-cu12.0-x64/%ENGINE_VERSION%
15+
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-cu11.7-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-cuda-cu11.7-x64/%ENGINE_VERSION%
16+
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-cu12.0-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-cuda-cu12.0-x64/%ENGINE_VERSION%
17+
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-cu11.7-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-cuda-cu11.7-x64/%ENGINE_VERSION%
18+
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-noavx-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-x64/%ENGINE_VERSION%
19+
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx-x64/%ENGINE_VERSION%
20+
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx2-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-x64/%ENGINE_VERSION%
21+
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx512-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx512-x64/%ENGINE_VERSION%
22+
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-vulkan-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-vulkan-x64/%ENGINE_VERSION%
23+
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu12.0-x64.tar.gz -e --strip 1 -o %BIN_PATH%
24+
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu11.7-x64.tar.gz -e --strip 1 -o %BIN_PATH%
2425

2526
move %BIN_PATH%\cortex-server-beta.exe %BIN_PATH%\cortex-server.exe
2627
del %BIN_PATH%\cortex-beta.exe
2728
del %BIN_PATH%\cortex.exe
2829

29-
@REM Loop through each folder and move DLLs (excluding engine.dll)
30+
@REM Loop through each folder and move DLLs
3031
for %%F in (%SUBFOLDERS%) do (
31-
echo Processing folder: %SHARED_PATH%\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%
32+
echo Processing folder: %SHARED_PATH%\engines\llama.cpp\%%F\%ENGINE_VERSION%
3233

33-
@REM Move all .dll files except engine.dll
34-
for %%D in (%SHARED_PATH%\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%\*.dll) do (
35-
if /I not "%%~nxD"=="engine.dll" (
36-
move "%%D" "%BIN_PATH%"
37-
)
34+
@REM Move cu*.dll files
35+
for %%D in (%SHARED_PATH%\engines\llama.cpp\%%F\%ENGINE_VERSION%\cu*.dll) do (
36+
move "%%D" "%BIN_PATH%"
3837
)
3938
)
4039

‎extensions/inference-cortex-extension/download.sh‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22

33
# Read CORTEX_VERSION
44
CORTEX_VERSION=$(cat ./bin/version.txt)
5-
ENGINE_VERSION=0.1.56
5+
ENGINE_VERSION=b5350
66
CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download"
7-
ENGINE_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}"
8-
CUDA_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}"
7+
ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}/llama-${ENGINE_VERSION}-bin
8+
CUDA_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}
99
BIN_PATH=./bin
1010
SHARED_PATH="../../electron/shared"
1111
# Detect platform
@@ -20,17 +20,17 @@ if [ "$OS_TYPE" == "Linux" ]; then
2020
chmod +x "./bin/cortex-server"
2121

2222
# Download engines for Linux
23-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx/v${ENGINE_VERSION}" 1
24-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx/v${ENGINE_VERSION}" 1
25-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2/v${ENGINE_VERSION}" 1
26-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx512.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx512/v${ENGINE_VERSION}" 1
27-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-12-0.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-12-0/v${ENGINE_VERSION}" 1
28-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-11-7.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-11-7/v${ENGINE_VERSION}" 1
29-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-12-0.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-12-0/v${ENGINE_VERSION}" 1
30-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-11-7.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-11-7/v${ENGINE_VERSION}" 1
31-
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}" 1
32-
download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
33-
download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
23+
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-x64/${ENGINE_VERSION}" 1
24+
download "${ENGINE_DOWNLOAD_URL}-linux-avx-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx-x64/${ENGINE_VERSION}" 1
25+
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-x64/${ENGINE_VERSION}" 1
26+
download "${ENGINE_DOWNLOAD_URL}-linux-avx512-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx512-x64/${ENGINE_VERSION}" 1
27+
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu12.0-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu12.0-x64/${ENGINE_VERSION}" 1
28+
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu11.7-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu11.7-x64/${ENGINE_VERSION}" 1
29+
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu12.0-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu12.0-x64/${ENGINE_VERSION}" 1
30+
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu11.7-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu11.7-x64/${ENGINE_VERSION}" 1
31+
download "${ENGINE_DOWNLOAD_URL}-linux-vulkan-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-vulkan-x64/${ENGINE_VERSION}" 1
32+
download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu12.0-x64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
33+
download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu11.7-x64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
3434

3535
elif [ "$OS_TYPE" == "Darwin" ]; then
3636
# macOS downloads
@@ -41,8 +41,8 @@ elif [ "$OS_TYPE" == "Darwin" ]; then
4141
chmod +x "./bin/cortex-server"
4242

4343
# Download engines for macOS
44-
download "${ENGINE_DOWNLOAD_URL}-mac-arm64.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/mac-arm64/v${ENGINE_VERSION}"
45-
download "${ENGINE_DOWNLOAD_URL}-mac-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/mac-amd64/v${ENGINE_VERSION}"
44+
download "${ENGINE_DOWNLOAD_URL}-macos-arm64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/macos-arm64/${ENGINE_VERSION}"
45+
download "${ENGINE_DOWNLOAD_URL}-macos-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/macos-x64/${ENGINE_VERSION}"
4646

4747
else
4848
echo "Unsupported operating system: $OS_TYPE"

‎extensions/inference-cortex-extension/rolldown.config.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ export default defineConfig([
1919
CORTEX_SOCKET_URL: JSON.stringify(
2020
`ws://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
2121
),
22-
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'),
22+
CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
2323
},
2424
},
2525
{

‎src-tauri/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ rand = "0.8"
3232
tauri-plugin-http = { version = "2", features = ["unsafe-headers"] }
3333
tauri-plugin-store = "2"
3434
hyper = { version = "0.14", features = ["server"] }
35-
reqwest = { version = "0.11", features = ["json"] }
35+
reqwest = { version = "0.11", features = ["json", "blocking"] }
3636
tokio = { version = "1", features = ["full"] }
3737
rmcp = { git = "https://github.com/modelcontextprotocol/rust-sdk", branch = "main", features = [
3838
"client",

‎src-tauri/binaries/download.bat‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
@echo off
22

3-
set CORTEX_VERSION=1.0.13-rc1
4-
set ENGINE_VERSION=0.1.55
5-
set ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64
3+
set CORTEX_VERSION=1.0.13-rc6
4+
set ENGINE_VERSION=b5350
5+
set ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
6+
set ENGINE_DOWNLOAD_GGML_URL=https://github.com/ggml-org/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
67
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%
78
set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan
89
set BIN_PATH="./"
@@ -11,32 +12,30 @@ set DOWNLOAD_TOOL=..\..\extensions\inference-cortex-extension\node_modules\.bin\
1112
@REM Download cortex.llamacpp binaries
1213

1314
call %DOWNLOAD_TOOL% -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
14-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2-cuda-12-0/v%ENGINE_VERSION%
15-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-11-7.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2-cuda-11-7/v%ENGINE_VERSION%
16-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-12-0.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx-cuda-12-0/v%ENGINE_VERSION%
17-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-11-7.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx-cuda-11-7/v%ENGINE_VERSION%
18-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx/v%ENGINE_VERSION%
19-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx/v%ENGINE_VERSION%
20-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2/v%ENGINE_VERSION%
21-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx512/v%ENGINE_VERSION%
22-
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-vulkan/v%ENGINE_VERSION%
23-
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
24-
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
15+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-cu12.0-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-avx2-cuda-cu12.0-x64/%ENGINE_VERSION%
16+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-cu11.7-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-avx2-cuda-cu11.7-x64/%ENGINE_VERSION%
17+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-cu12.0-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-noavx-cuda-cu12.0-x64/%ENGINE_VERSION%
18+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-cu11.7-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-noavx-cuda-cu11.7-x64/%ENGINE_VERSION%
19+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-noavx-x64.zip -e --strip 1 -o./engines/llama.cpp/win-noavx-x64/%ENGINE_VERSION%
20+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx-x64/%ENGINE_VERSION%
21+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx2-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx2-x64/%ENGINE_VERSION%
22+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx512-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx512-x64/%ENGINE_VERSION%
23+
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-vulkan-x64.zip -e --strip 1 -o./engines/llama.cpp/win-vulkan-x64/%ENGINE_VERSION%
24+
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu12.0-x64.tar.gz -e --strip 1 -o %BIN_PATH%
25+
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu11.7-x64.tar.gz -e --strip 1 -o %BIN_PATH%
2526

2627
move %BIN_PATH%cortex-server-beta.exe %BIN_PATH%cortex-server.exe
2728
copy %BIN_PATH%cortex-server.exe %BIN_PATH%cortex-server-x86_64-pc-windows-msvc.exe
2829
del %BIN_PATH%cortex-beta.exe
2930
del %BIN_PATH%cortex.exe
3031

31-
@REM Loop through each folder and move DLLs (excluding engine.dll)
32+
@REM Loop through each folder and move DLLs
3233
for %%F in (%SUBFOLDERS%) do (
33-
echo Processing folder: .\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%
34+
echo Processing folder: .\engines\llama.cpp\%%F\%ENGINE_VERSION%
3435

35-
@REM Move all .dll files except engine.dll
36-
for %%D in (.\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%\*.dll) do (
37-
if /I not "%%~nxD"=="engine.dll" (
38-
move "%%D" "%BIN_PATH%"
39-
)
36+
@REM Move cu*.dll files
37+
for %%D in (.\engines\engines\llama.cpp\%%F\%ENGINE_VERSION%\cu*.dll) do (
38+
move "%%D" "%BIN_PATH%"
4039
)
4140
)
4241

0 commit comments

Comments
 (0)