Skip to content

Commit 5fec9cb

Browse files
hipuddinghipudding
committed
[CANN] Add Ascend NPU backend
Ascend is a full-stack AI computing infrastructure for industry applications and services based on Huawei Ascend processors and software. CANN (Compute Architecture of Neural Networks), developped by Huawei, is a heterogeneous computing architecture for AI. This commit adding Ascend NPU as a new backend, which implements the following features: 1. Ascend NPU register; 2. Ascend NPU runtime(device memory, streams, events). 3. Part of GGML_OPs through aclnn library. 4. Introduce a new test file named test-backend-runtime, for testing runtime functionality.
1 parent 0642b22 commit 5fec9cb

File tree

18 files changed

+1935
-9
lines changed

18 files changed

+1935
-9
lines changed

‎CMakeLists.txt‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -769,6 +769,99 @@ if (LLAMA_CPU_HBM)
769769
target_link_libraries(ggml PUBLIC memkind)
770770
endif()
771771

772+
if (LLAMA_CANN)
773+
if ("cann${CANN_INSTALL_DIR}" STREQUAL "cann" AND DEFINED ENV{ASCEND_TOOLKIT_HOME})
774+
set(CANN_INSTALL_DIR $ENV{ASCEND_TOOLKIT_HOME})
775+
message(STATUS "CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}")
776+
endif()
777+
778+
if (CANN_INSTALL_DIR)
779+
# Only Support Linux.
780+
if (LLAMA_CANN)
781+
if (NOT UNIX)
782+
set(LLAMA_CANN OFF)
783+
message(WARNING "CANN: CANN toolkit supports unix but not ${CMAKE_SYSTEM_NAME}. Turning off LLAMA_CANN")
784+
endif()
785+
endif()
786+
787+
# Supported platforms: x86-64, arm64
788+
if (LLAMA_CANN)
789+
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
790+
elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
791+
else()
792+
set(LLAMA_CANN OFF)
793+
message(WARNING "CANN: CANN toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}. Turning off LLAMA_CANN")
794+
endif()
795+
endif()
796+
797+
# Set headers
798+
set(CANN_INCLUDE_DIRS "${CANN_INSTALL_DIR}/include" "${CANN_INSTALL_DIR}/include/aclnn")
799+
# Find libs
800+
set(CANN_LIBRARIES "")
801+
# TODO: optimize find libs.
802+
# * libascendcl.so
803+
if (LLAMA_CANN)
804+
set(lib_dir "${CANN_INSTALL_DIR}/acllib/lib64")
805+
find_library(found_lib_ascendcl NAMES ascendcl PATHS ${lib_dir} NO_DEFAULT_PATH)
806+
if (found_lib_ascendcl)
807+
set(lib_ascendcl ${found_lib_ascendcl})
808+
list(APPEND CANN_LIBRARIES ${lib_ascendcl})
809+
message(STATUS "CANN: libascendcl.so is found at ${lib_dir}")
810+
else()
811+
set(LLAMA_CANN OFF)
812+
message(WARNING "CANN: Missing libascendcl.so. Turning off LLAMA_CANN")
813+
endif()
814+
endif()
815+
816+
# * libnnopbase.so
817+
if (LLAMA_CANN)
818+
set(lib_dir "${CANN_INSTALL_DIR}/acllib/lib64")
819+
find_library(found_lib_nnopbase NAMES nnopbase PATHS ${lib_dir} NO_DEFAULT_PATH)
820+
if (found_lib_nnopbase)
821+
set(lib_nnopbase ${found_lib_nnopbase})
822+
list(APPEND CANN_LIBRARIES ${lib_nnopbase})
823+
message(STATUS "CANN: libnnopbase.so is found at ${lib_dir}")
824+
else()
825+
set(LLAMA_CANN OFF)
826+
message(WARNING "CANN: Missing libnnopbase.so. Turning off LLAMA_CANN")
827+
endif()
828+
endif()
829+
830+
# * libopapi.so
831+
if (LLAMA_CANN)
832+
set(lib_dir "${CANN_INSTALL_DIR}/lib64")
833+
find_library(found_lib_opapi NAMES opapi PATHS ${lib_dir} NO_DEFAULT_PATH)
834+
if (found_lib_opapi)
835+
set(lib_opapi ${found_lib_opapi})
836+
list(APPEND CANN_LIBRARIES ${lib_opapi})
837+
message(STATUS "CANN: libopapi.so is found at ${lib_dir}")
838+
else()
839+
set(LLAMA_CANN OFF)
840+
message(WARNING "CANN: Missing libopapi.so. Turning off LLAMA_CANN")
841+
endif()
842+
endif()
843+
844+
# Set headers and libs
845+
if (LLAMA_CANN)
846+
message(STATUS "CANN: CANN_INCLUDE_DIRS = ${CANN_INCLUDE_DIRS}")
847+
message(STATUS "CANN: CANN_LIBRARIES = ${CANN_LIBRARIES}")
848+
set(GGML_HEADERS_CANN ggml-cann.h)
849+
file(GLOB GGML_SOURCES_CUDA "ggml-cann/*.cpp")
850+
list(APPEND GGML_SOURCES_CANN "ggml-cann.cpp")
851+
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${CANN_LIBRARIES})
852+
set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CANN_INCLUDE_DIRS})
853+
add_compile_definitions(GGML_USE_CANN)
854+
endif()
855+
else()
856+
set(LLAMA_CANN OFF)
857+
message(WARNING "CANN: Can't find CANN_INSTALL_DIR, do you forget to source set_var.sh. Turning off LLAMA_CANN")
858+
endif()
859+
860+
if(NOT LLAMA_CANN)
861+
message(WARNING "CANN: LLAMA_CANN is turned OFF, see above for details.")
862+
endif()
863+
endif()
864+
772865
if (LLAMA_PERF)
773866
add_compile_definitions(GGML_PERF)
774867
endif()
@@ -1147,6 +1240,7 @@ add_library(ggml OBJECT
11471240
${GGML_SOURCES_KOMPUTE} ${GGML_HEADERS_KOMPUTE}
11481241
${GGML_SOURCES_VULKAN} ${GGML_HEADERS_VULKAN}
11491242
${GGML_SOURCES_ROCM} ${GGML_HEADERS_ROCM}
1243+
${GGML_SOURCES_CANN} ${GGML_HEADERS_CANN}
11501244
)
11511245

11521246
target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES})

‎common/common.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@
5656
#define GGML_USE_CUDA_SYCL_VULKAN
5757
#endif
5858

59+
#if (defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL)) || defined(GGML_USE_CANN)
60+
#define GGML_USE_CUDA_SYCL_CANN
61+
#endif
62+
5963
#if defined(LLAMA_USE_CURL)
6064
#ifdef __linux__
6165
#include <linux/limits.h>
@@ -861,9 +865,9 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
861865
return true;
862866
}
863867
params.main_gpu = std::stoi(argv[i]);
864-
#ifndef GGML_USE_CUDA_SYCL
865-
fprintf(stderr, "warning: llama.cpp was compiled without CUDA/SYCL. Setting the main GPU has no effect.\n");
866-
#endif // GGML_USE_CUDA_SYCL
868+
#ifndef GGML_USE_CUDA_SYCL_CANN
869+
fprintf(stderr, "warning: llama.cpp was compiled without CUDA/SYCL/CANN. Setting the main GPU has no effect.\n");
870+
#endif // GGML_USE_CUDA_SYCL_CANN
867871
return true;
868872
}
869873
if (arg == "--split-mode" || arg == "-sm") {

‎examples/llama-bench/llama-bench.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@
2323
#include "ggml-cuda.h"
2424
#include "ggml-sycl.h"
2525

26+
#ifdef GGML_USE_CANN
27+
#include "ggml-cann.h"
28+
#endif
29+
2630
// utils
2731
static uint64_t get_time_ns() {
2832
using clock = std::chrono::high_resolution_clock;
@@ -134,6 +138,17 @@ static std::string get_gpu_info() {
134138
id += "/";
135139
}
136140
}
141+
#endif
142+
#ifdef GGML_USE_CANN
143+
uint32_t count = ggml_backend_cann_get_device_count();
144+
for (uint32_t i = 0; i < count; i++) {
145+
char buf[128];
146+
ggml_backend_cann_get_device_description(i, buf, sizeof(buf));
147+
id += buf;
148+
if (i < count - 1) {
149+
id += "/";
150+
}
151+
}
137152
#endif
138153
// TODO: other backends
139154
return id;

‎examples/llava/clip.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@
1515
#include "ggml-metal.h"
1616
#endif
1717

18+
#ifdef GGML_USE_CANN
19+
#include "ggml-cann.h"
20+
#endif
21+
1822
#define STB_IMAGE_IMPLEMENTATION
1923
#include "stb_image.h"
2024

@@ -978,6 +982,11 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
978982
printf("%s: CLIP using Metal backend\n", __func__);
979983
#endif
980984

985+
#ifdef GGML_USE_CANN
986+
new_clip->backend = ggml_backend_cann_init(0);
987+
printf("%s: CLIP using CANN backend\n", __func__);
988+
#endif
989+
981990

982991
if (!new_clip->backend) {
983992
new_clip->backend = ggml_backend_cpu_init();

‎examples/server/server.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2539,17 +2539,17 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
25392539
}
25402540
}
25412541
#else
2542-
LOG_WARNING("llama.cpp was compiled without CUDA. It is not possible to set a tensor split.\n", {});
2542+
LOG_WARNING("llama.cpp was compiled without CUDA/SYCL. It is not possible to set a tensor split.\n", {});
25432543
#endif // GGML_USE_CUDA
25442544
} else if (arg == "--main-gpu" || arg == "-mg") {
25452545
if (++i >= argc) {
25462546
invalid_param = true;
25472547
break;
25482548
}
2549-
#if defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL)
2549+
#if defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL) || defined(GGML_USE_CANN)
25502550
params.main_gpu = std::stoi(argv[i]);
25512551
#else
2552-
LOG_WARNING("llama.cpp was compiled without CUDA. It is not possible to set a main GPU.", {});
2552+
LOG_WARNING("llama.cpp was compiled without cuBLAS/SYCL/CANN. It is not possible to set a main GPU.", {});
25532553
#endif
25542554
} else if (arg == "--lora") {
25552555
if (++i >= argc) {

‎ggml-backend.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,11 @@ GGML_CALL static void ggml_backend_registry_init(void) {
445445
extern GGML_CALL void ggml_backend_kompute_reg_devices(void);
446446
ggml_backend_kompute_reg_devices();
447447
#endif
448+
449+
#ifdef GGML_USE_CANN
450+
extern GGML_CALL int ggml_backend_cann_reg_devices(void);
451+
ggml_backend_cann_reg_devices();
452+
#endif
448453
}
449454

450455
GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {

0 commit comments

Comments
 (0)