Skip to content

Commit e03cae2

Browse files
wyanzhaomax-krasnyansky
wyanzhao
authored and
max-krasnyansky
committed
hexagon: add HTP kernel for GGML_OP_GATED_DELTA_NET
Implement the Gated Delta Net recurrence on HVX with: - 4-row fused kernels for PP (prompt processing) path - 8-row fused kernels for TG (token generation) path, reducing K/Q/gate vector reload overhead by 2x - Separate PP/TG thread functions for I-cache isolation - VTCM state scratchpad with DMA in/out for TG single-cycle access - Vectorized gate exp via hvx_exp_f32
1 parent b46812d commit e03cae2

6 files changed

Lines changed: 1045 additions & 27 deletions

File tree

‎ggml/src/ggml-hexagon/ggml-hexagon.cpp‎

Lines changed: 84 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2261,6 +2261,58 @@ static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_sess
22612261
return true;
22622262
}
22632263

2264+
static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
2265+
const struct ggml_tensor * q = op->src[0];
2266+
const struct ggml_tensor * k = op->src[1];
2267+
const struct ggml_tensor * v = op->src[2];
2268+
const struct ggml_tensor * g = op->src[3];
2269+
const struct ggml_tensor * beta = op->src[4];
2270+
const struct ggml_tensor * state = op->src[5];
2271+
const struct ggml_tensor * dst = op;
2272+
2273+
if (!q || !k || !v || !g || !beta || !state) {
2274+
return false;
2275+
}
2276+
2277+
if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 ||
2278+
g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 ||
2279+
dst->type != GGML_TYPE_F32) {
2280+
return false;
2281+
}
2282+
2283+
if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) ||
2284+
!ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) ||
2285+
!ggml_is_contiguous(dst)) {
2286+
return false;
2287+
}
2288+
2289+
const int64_t S_v = v->ne[0];
2290+
const int64_t H = v->ne[1];
2291+
const int64_t n_tokens = v->ne[2];
2292+
const int64_t n_seqs = v->ne[3];
2293+
2294+
if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) {
2295+
return false;
2296+
}
2297+
if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 ||
2298+
q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 ||
2299+
(n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {
2300+
return false;
2301+
}
2302+
if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {
2303+
return false;
2304+
}
2305+
if (ggml_nelements(state) != S_v * S_v * H * n_seqs) {
2306+
return false;
2307+
}
2308+
if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs) {
2309+
return false;
2310+
}
2311+
2312+
GGML_UNUSED(sess);
2313+
return true;
2314+
}
2315+
22642316
static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {
22652317
const struct ggml_tensor * src0 = dst->src[0];
22662318
const struct ggml_tensor * src1 = dst->src[1];
@@ -2777,33 +2829,34 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) {
27772829

27782830
static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
27792831
switch (t->op) {
2780-
case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;
2781-
case GGML_OP_MUL_MAT: return HTP_OP_MUL_MAT;
2782-
case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID;
2783-
case GGML_OP_MUL: return HTP_OP_MUL;
2784-
case GGML_OP_ADD: return HTP_OP_ADD;
2785-
case GGML_OP_ADD_ID: return HTP_OP_ADD_ID;
2786-
case GGML_OP_SUB: return HTP_OP_SUB;
2787-
case GGML_OP_DIV: return HTP_OP_DIV;
2788-
case GGML_OP_CPY: return HTP_OP_CPY;
2789-
case GGML_OP_CONT: return HTP_OP_CPY;
2790-
case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS;
2791-
case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS;
2792-
case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS;
2793-
case GGML_OP_ARGSORT: return HTP_OP_ARGSORT;
2794-
case GGML_OP_L2_NORM: return HTP_OP_L2_NORM;
2795-
case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM;
2796-
case GGML_OP_SCALE: return HTP_OP_SCALE;
2797-
case GGML_OP_SQR: return HTP_OP_SQR;
2798-
case GGML_OP_SQRT: return HTP_OP_SQRT;
2799-
case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;
2800-
case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;
2801-
case GGML_OP_ROPE: return HTP_OP_ROPE;
2802-
case GGML_OP_REPEAT: return HTP_OP_REPEAT;
2803-
case GGML_OP_CUMSUM: return HTP_OP_CUMSUM;
2804-
case GGML_OP_FILL: return HTP_OP_FILL;
2805-
case GGML_OP_DIAG: return HTP_OP_DIAG;
2806-
case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;
2832+
case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;
2833+
case GGML_OP_MUL_MAT: return HTP_OP_MUL_MAT;
2834+
case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID;
2835+
case GGML_OP_MUL: return HTP_OP_MUL;
2836+
case GGML_OP_ADD: return HTP_OP_ADD;
2837+
case GGML_OP_ADD_ID: return HTP_OP_ADD_ID;
2838+
case GGML_OP_SUB: return HTP_OP_SUB;
2839+
case GGML_OP_DIV: return HTP_OP_DIV;
2840+
case GGML_OP_CPY: return HTP_OP_CPY;
2841+
case GGML_OP_CONT: return HTP_OP_CPY;
2842+
case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS;
2843+
case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS;
2844+
case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS;
2845+
case GGML_OP_ARGSORT: return HTP_OP_ARGSORT;
2846+
case GGML_OP_L2_NORM: return HTP_OP_L2_NORM;
2847+
case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM;
2848+
case GGML_OP_SCALE: return HTP_OP_SCALE;
2849+
case GGML_OP_SQR: return HTP_OP_SQR;
2850+
case GGML_OP_SQRT: return HTP_OP_SQRT;
2851+
case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;
2852+
case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;
2853+
case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;
2854+
case GGML_OP_ROPE: return HTP_OP_ROPE;
2855+
case GGML_OP_REPEAT: return HTP_OP_REPEAT;
2856+
case GGML_OP_CUMSUM: return HTP_OP_CUMSUM;
2857+
case GGML_OP_FILL: return HTP_OP_FILL;
2858+
case GGML_OP_DIAG: return HTP_OP_DIAG;
2859+
case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;
28072860
case GGML_OP_UNARY:
28082861
switch (ggml_get_unary_op(t)) {
28092862
case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU;
@@ -3341,6 +3394,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
33413394
supp = ggml_hexagon_supported_ssm_conv(sess, op);
33423395
break;
33433396

3397+
case GGML_OP_GATED_DELTA_NET:
3398+
supp = ggml_hexagon_supported_gated_delta_net(sess, op);
3399+
break;
3400+
33443401
case GGML_OP_CUMSUM:
33453402
supp = ggml_hexagon_supported_cumsum(sess, op);
33463403
break;

‎ggml/src/ggml-hexagon/htp/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ add_library(${HTP_LIB} SHARED
3737
fill-ops.c
3838
diag-ops.c
3939
solve-tri-ops.c
40+
gated-delta-net-ops.c
4041
)
4142

4243
target_compile_definitions(${HTP_LIB} PRIVATE

0 commit comments

Comments
 (0)