Merge remote-tracking branch 'ggerganov/master'

* ggerganov/master: (72 commits)
  node : add win platform check for require path (ggml-org#3363)
  ci : update main-cuda.Dockerfile (ggml-org#3371)
  whisper : fixed crash in GPU device selection on multi-GPU systems (ggml-org#3372)
  wasm : change ggml model host to HF (ggml-org#3369)
  ruby : Add ruby binding for max_len (ggml-org#3365)
  stream.wasm : add language selection support (ggml-org#3354)
  whisper : reset conv scheduler when CoreML is used (ggml-org#3350)
  ggml : remove old kompute, cann (skip) (ggml-org#3349)
  talk-llama : sync llama.cpp
  sync : ggml
  vulkan : add fp16 support for the conv_2d kernel (llama/14872)
  vulkan: skip empty set_rows to avoid invalid API usage (llama/14860)
  HIP: Enable Matrix cores for MMQ Kernels, Enable stream-K for CDNA 3 (llama/14624)
  CANN: Implement GLU ops (llama/14884)
  musa: fix build warnings (unused variable) (llama/14869)
  ggml-cpu : disable GGML_NNPA by default due to instability (llama/14880)
  metal: SSM_SCAN performance (llama/14743)
  opencl: add fused `rms_norm_mul` (llama/14841)
  ggml : remove invalid portPos specifiers from dot files (llama/14838)
  rpc : check for null buffers in get/set/copy tensor endpoints (llama/14868)
  ...

Author: bygreencn

.devops/main-cuda.Dockerfile

@@ -1,6 +1,6 @@
 ARG UBUNTU_VERSION=22.04
 # This needs to generally match the container host's environment.
-ARG CUDA_VERSION=12.3.1
+ARG CUDA_VERSION=13.0.0
 # Target the CUDA build image
 ARG BASE_CUDA_DEV_CONTAINER=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
 # Target the CUDA runtime image
@@ -20,12 +20,12 @@ RUN apt-get update && \
     && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 # Ref: https://stackoverflow.com/a/53464012
-ENV CUDA_MAIN_VERSION=12.3
+ENV CUDA_MAIN_VERSION=13.0
 ENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 
 COPY .. .
 # Enable cuBLAS
-RUN make base.en CMAKE_ARGS="-DGGML_CUDA=1"
+RUN make base.en CMAKE_ARGS="-DGGML_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES='75;80;86;90'"
 
 RUN find /app/build -name "*.o" -delete && \
     find /app/build -name "*.a" -delete && \
@@ -34,7 +34,7 @@ RUN find /app/build -name "*.o" -delete && \
     rm -rf /app/build/_deps
 
 FROM ${BASE_CUDA_RUN_CONTAINER} AS runtime
-ENV CUDA_MAIN_VERSION=12.3
+ENV CUDA_MAIN_VERSION=13.0
 ENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 WORKDIR /app
 

.devops/main-musa.Dockerfile

@@ -1,10 +1,10 @@
 ARG UBUNTU_VERSION=22.04
 # This needs to generally match the container host's environment.
-ARG MUSA_VERSION=rc4.0.1
+ARG MUSA_VERSION=rc4.2.0
 # Target the MUSA build image
-ARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-mudnn-devel-ubuntu${UBUNTU_VERSION}
+ARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64
 # Target the MUSA runtime image
-ARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-mudnn-runtime-ubuntu${UBUNTU_VERSION}
+ARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64
 
 FROM ${BASE_MUSA_DEV_CONTAINER} AS build
 WORKDIR /app

.github/workflows/build.yml

@@ -6,6 +6,25 @@ on:
       - master
     tags:
       - 'v*'
+    paths: ['.github/workflows/build.yml',
+            '**/CMakeLists.txt',
+            '**/Makefile',
+            '**/*.mk',
+            '**/*.cmake',
+            '**/*.in',
+            '**/*.h',
+            '**/*.hpp',
+            '**/*.c',
+            '**/*.cpp',
+            '**/*.cu',
+            '**/*.cuh',
+            '**/*.cl',
+            '**/*.swift',
+            '**/*.m',
+            '**/*.mm',
+            '**/*.metal',
+            '**/*.comp',
+            '**/*.java']
   pull_request:
     types: [opened, synchronize, reopened]
   workflow_dispatch:

README.md

@@ -386,7 +386,7 @@ Run the inference examples as usual, for example:
 ## Moore Threads GPU support
 
 With Moore Threads cards the processing of the models is done efficiently on the GPU via muBLAS and custom MUSA kernels.
-First, make sure you have installed `MUSA SDK rc4.0.1`: https://developer.mthreads.com/sdk/download/musa?equipment=&os=&driverVersion=&version=4.0.1
+First, make sure you have installed `MUSA SDK rc4.2.0`: https://developer.mthreads.com/sdk/download/musa?equipment=&os=&driverVersion=&version=4.2.0
 
 Now build `whisper.cpp` with MUSA support:
 

bindings/go/Makefile

@@ -15,15 +15,16 @@ BUILD_DIR := build_go
 MODELS_DIR := models
 EXAMPLES_DIR := $(wildcard examples/*)
 INCLUDE_PATH := $(abspath ../../include):$(abspath ../../ggml/include)
-LIBRARY_PATH := $(abspath ../../${BUILD_DIR}/src:$(abspath ../../${BUILD_DIR}/ggml/src))
+LIBRARY_PATH := $(abspath ../../${BUILD_DIR}/src):$(abspath ../../${BUILD_DIR}/ggml/src)
 
 ifeq ($(GGML_CUDA),1)
 	LIBRARY_PATH := $(LIBRARY_PATH):$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib/
 	BUILD_FLAGS := -ldflags "-extldflags '-lcudart -lcuda -lcublas'"
 endif
 
 ifeq ($(UNAME_S),Darwin)
-	EXT_LDFLAGS := -framework Foundation -framework Metal -framework MetalKit
+	LIBRARY_PATH := $(LIBRARY_PATH):$(abspath ../../${BUILD_DIR}/ggml/src/ggml-blas):$(abspath ../../${BUILD_DIR}/ggml/src/ggml-metal)
+	EXT_LDFLAGS := -framework Foundation -framework Metal -framework MetalKit -lggml-metal -lggml-blas
 endif
 
 all: clean whisper examples

bindings/go/whisper.go

@@ -9,7 +9,9 @@ import (
 // CGO
 
 /*
-#cgo LDFLAGS: -lwhisper -lggml -lggml-base -lggml-cpu  -lm -lstdc++ -fopenmp
+#cgo LDFLAGS: -lwhisper -lggml -lggml-base -lggml-cpu -lm -lstdc++
+#cgo linux LDFLAGS: -fopenmp
+#cgo darwin LDFLAGS: -lggml-metal -lggml-blas
 #cgo darwin LDFLAGS: -framework Accelerate -framework Metal -framework Foundation -framework CoreGraphics
 #include <whisper.h>
 #include <stdlib.h>

bindings/ruby/ext/ruby_whisper_params.c

@@ -26,7 +26,7 @@
   rb_define_method(cParams, #param_name, ruby_whisper_params_get_ ## param_name, 0); \
   rb_define_method(cParams, #param_name "=", ruby_whisper_params_set_ ## param_name, 1);
 
-#define RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT 35
+#define RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT 36
 
 extern VALUE cParams;
 extern VALUE cVADParams;
@@ -49,6 +49,7 @@ static ID id_print_timestamps;
 static ID id_suppress_blank;
 static ID id_suppress_nst;
 static ID id_token_timestamps;
+static ID id_max_len;
 static ID id_split_on_word;
 static ID id_initial_prompt;
 static ID id_diarize;
@@ -514,6 +515,33 @@ ruby_whisper_params_set_token_timestamps(VALUE self, VALUE value)
 {
   BOOL_PARAMS_SETTER(self, token_timestamps, value)
 }
+
+/*
+ * max segment length in characters.
+ *
+ * call-seq:
+ *   max_len -> Integer
+ */
+static VALUE
+ruby_whisper_params_get_max_len(VALUE self)
+{
+  ruby_whisper_params *rwp;
+  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);
+  return INT2NUM(rwp->params.max_len);
+}
+/*
+ * call-seq:
+ *   max_len = length -> length
+ */
+static VALUE
+ruby_whisper_params_set_max_len(VALUE self, VALUE value)
+{
+  ruby_whisper_params *rwp;
+  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);
+  rwp->params.max_len = NUM2INT(value);
+  return value;
+}
+
 /*
  * If true, split on word rather than on token (when used with max_len).
  *
@@ -1137,6 +1165,7 @@ ruby_whisper_params_initialize(int argc, VALUE *argv, VALUE self)
       SET_PARAM_IF_SAME(suppress_blank)
       SET_PARAM_IF_SAME(suppress_nst)
       SET_PARAM_IF_SAME(token_timestamps)
+      SET_PARAM_IF_SAME(max_len)
       SET_PARAM_IF_SAME(split_on_word)
       SET_PARAM_IF_SAME(initial_prompt)
       SET_PARAM_IF_SAME(offset)
@@ -1271,30 +1300,31 @@ init_ruby_whisper_params(VALUE *mWhisper)
   DEFINE_PARAM(suppress_blank, 8)
   DEFINE_PARAM(suppress_nst, 9)
   DEFINE_PARAM(token_timestamps, 10)
-  DEFINE_PARAM(split_on_word, 11)
-  DEFINE_PARAM(initial_prompt, 12)
-  DEFINE_PARAM(diarize, 13)
-  DEFINE_PARAM(offset, 14)
-  DEFINE_PARAM(duration, 15)
-  DEFINE_PARAM(max_text_tokens, 16)
-  DEFINE_PARAM(temperature, 17)
-  DEFINE_PARAM(max_initial_ts, 18)
-  DEFINE_PARAM(length_penalty, 19)
-  DEFINE_PARAM(temperature_inc, 20)
-  DEFINE_PARAM(entropy_thold, 21)
-  DEFINE_PARAM(logprob_thold, 22)
-  DEFINE_PARAM(no_speech_thold, 23)
-  DEFINE_PARAM(new_segment_callback, 24)
-  DEFINE_PARAM(new_segment_callback_user_data, 25)
-  DEFINE_PARAM(progress_callback, 26)
-  DEFINE_PARAM(progress_callback_user_data, 27)
-  DEFINE_PARAM(encoder_begin_callback, 28)
-  DEFINE_PARAM(encoder_begin_callback_user_data, 29)
-  DEFINE_PARAM(abort_callback, 30)
-  DEFINE_PARAM(abort_callback_user_data, 31)
-  DEFINE_PARAM(vad, 32)
-  DEFINE_PARAM(vad_model_path, 33)
-  DEFINE_PARAM(vad_params, 34)
+  DEFINE_PARAM(max_len, 11)
+  DEFINE_PARAM(split_on_word, 12)
+  DEFINE_PARAM(initial_prompt, 13)
+  DEFINE_PARAM(diarize, 14)
+  DEFINE_PARAM(offset, 15)
+  DEFINE_PARAM(duration, 16)
+  DEFINE_PARAM(max_text_tokens, 17)
+  DEFINE_PARAM(temperature, 18)
+  DEFINE_PARAM(max_initial_ts, 19)
+  DEFINE_PARAM(length_penalty, 20)
+  DEFINE_PARAM(temperature_inc, 21)
+  DEFINE_PARAM(entropy_thold, 22)
+  DEFINE_PARAM(logprob_thold, 23)
+  DEFINE_PARAM(no_speech_thold, 24)
+  DEFINE_PARAM(new_segment_callback, 25)
+  DEFINE_PARAM(new_segment_callback_user_data, 26)
+  DEFINE_PARAM(progress_callback, 27)
+  DEFINE_PARAM(progress_callback_user_data, 28)
+  DEFINE_PARAM(encoder_begin_callback, 29)
+  DEFINE_PARAM(encoder_begin_callback_user_data, 30)
+  DEFINE_PARAM(abort_callback, 31)
+  DEFINE_PARAM(abort_callback_user_data, 32)
+  DEFINE_PARAM(vad, 33)
+  DEFINE_PARAM(vad_model_path, 34)
+  DEFINE_PARAM(vad_params, 35)
 
   rb_define_method(cParams, "on_new_segment", ruby_whisper_params_on_new_segment, 0);
   rb_define_method(cParams, "on_progress", ruby_whisper_params_on_progress, 0);

bindings/ruby/sig/whisper.rbs

@@ -135,6 +135,7 @@ module Whisper
       ?suppress_blank: boolish,
       ?suppress_nst: boolish,
       ?token_timestamps: boolish,
+      ?max_len: Integer,
       ?split_on_word: boolish,
       ?initial_prompt: string | nil,
       ?diarize: boolish,
@@ -222,6 +223,12 @@ module Whisper
     #
     def token_timestamps: () -> (true | false)
 
+    def max_len=: (Integer) -> Integer
+
+    # max segment length in characters.
+    #
+    def max_len: () -> Integer
+
     def split_on_word=: (boolish) -> boolish
 
     # If true, split on word rather than on token (when used with max_len).

bindings/ruby/test/test_params.rb

@@ -13,6 +13,7 @@ class TestParams < TestBase
     :suppress_blank,
     :suppress_nst,
     :token_timestamps,
+    :max_len,
     :split_on_word,
     :initial_prompt,
     :diarize,
@@ -139,6 +140,13 @@ def test_token_timestamps
     assert !@params.token_timestamps
   end
 
+  def test_max_len
+    @params.max_len = 42
+    assert_equal @params.max_len, 42
+    @params.max_len = 0
+    assert_equal @params.max_len, 0
+  end
+
   def test_split_on_word
     @params.split_on_word = true
     assert @params.split_on_word

build-xcframework.sh

@@ -15,6 +15,7 @@ GGML_METAL_EMBED_LIBRARY=ON
 GGML_BLAS_DEFAULT=ON
 GGML_METAL_USE_BF16=ON
 GGML_OPENMP=OFF
+BUILD_STATIC_XCFRAMEWORK=${BUILD_STATIC_XCFRAMEWORK:-OFF}
 
 COMMON_C_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
 COMMON_CXX_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
@@ -327,6 +328,15 @@ combine_static_libraries() {
         arch_flags+=" -arch $arch"
     done
 
+
+    if [[ "${BUILD_STATIC_XCFRAMEWORK}" == "ON" ]]; then
+        echo "Packaging static framework for ${platform}."
+        mkdir -p "$(dirname "${base_dir}/${output_lib}")"
+        cp "${temp_dir}/combined.a" "${base_dir}/${output_lib}"
+        rm -rf "${temp_dir}"
+        return
+    fi
+
     # Create dynamic library
     echo "Creating dynamic library for ${platform}."
     xcrun -sdk $sdk clang++ -dynamiclib \
@@ -529,6 +539,20 @@ combine_static_libraries "build-tvos-device" "Release-appletvos" "tvos" "false"
 
 # Create XCFramework with correct debug symbols paths
 echo "Creating XCFramework..."
+
+if [[ "${BUILD_STATIC_XCFRAMEWORK}" == "ON" ]]; then
+    xcodebuild -create-xcframework \
+        -framework $(pwd)/build-ios-sim/framework/whisper.framework \
+        -framework $(pwd)/build-ios-device/framework/whisper.framework \
+        -framework $(pwd)/build-macos/framework/whisper.framework \
+        -framework $(pwd)/build-visionos/framework/whisper.framework \
+        -framework $(pwd)/build-visionos-sim/framework/whisper.framework \
+        -framework $(pwd)/build-tvos-device/framework/whisper.framework \
+        -framework $(pwd)/build-tvos-sim/framework/whisper.framework \
+        -output $(pwd)/build-apple/whisper.xcframework
+    exit 0
+fi
+
 xcodebuild -create-xcframework \
     -framework $(pwd)/build-ios-sim/framework/whisper.framework \
     -debug-symbols $(pwd)/build-ios-sim/dSYMs/whisper.dSYM \