diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index b510cb9571..0dcfad3afa 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -496,14 +496,29 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexe return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, ggml_op op) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, const ggml_tensor * op) { const char * name = nullptr; - switch (op) { - case GGML_OP_DSV4_HC_COMB: name = "kernel_dsv4_hc_comb_f32"; break; - case GGML_OP_DSV4_HC_PRE: name = "kernel_dsv4_hc_pre_f32"; break; - case GGML_OP_DSV4_HC_POST: name = "kernel_dsv4_hc_post_f32"; break; - default: GGML_ABORT("fatal error"); + switch (op->op) { + case GGML_OP_DSV4_HC_COMB: + name = "kernel_dsv4_hc_comb_f32"; + break; + case GGML_OP_DSV4_HC_PRE: + if (ggml_get_op_params_i32(op, 1) != 0) { + name = "kernel_dsv4_hc_pre_gated_f32"; + } else { + name = "kernel_dsv4_hc_pre_f32"; + } + break; + case GGML_OP_DSV4_HC_POST: + if (op->src[3]) { + name = "kernel_dsv4_hc_post_f32"; + } else { + name = "kernel_dsv4_hc_post_nocomb_f32"; + } + break; + default: + GGML_ABORT("fatal error"); } ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index f6243ffbd1..0514f9ef04 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -126,7 +126,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_ad struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, enum ggml_op op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tail); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index c734c8e133..952d1c0a64 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1802,8 +1802,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && op->src[0]->ne[1] == 4 && - op->src[1]->ne[0] == 4 && - op->src[1]->ne[2] == 1 && ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]); case GGML_OP_DSV4_HC_POST: @@ -1811,17 +1809,15 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->src[2]->type == GGML_TYPE_F32 && - op->src[3] != NULL && - op->src[3]->type == GGML_TYPE_F32 && + (op->src[3] == NULL || op->src[3]->type == GGML_TYPE_F32) && op->type == GGML_TYPE_F32 && op->src[1]->ne[1] == 4 && op->src[2]->ne[0] == 4 && - op->src[3]->ne[0] == 4 && - op->src[3]->ne[1] == 4 && + (op->src[3] == NULL || (op->src[3]->ne[0] == 4 && op->src[3]->ne[1] == 4)) && ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && ggml_is_contiguous_rows(op->src[2]) && - ggml_is_contiguous_rows(op->src[3]); + (op->src[3] == NULL || ggml_is_contiguous_rows(op->src[3])); case GGML_OP_SSM_SCAN: return has_simdgroup_reduction; case GGML_OP_SSM_CONV: diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 7a2c65aaa2..d84ca937b7 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -1283,8 +1283,10 @@ typedef struct { uint64_t nb_x2; uint64_t nb_w0; uint64_t nb_w1; + uint64_t nb_w2; uint64_t nb_d0; uint64_t nb_d1; + float scale; } ggml_metal_kargs_dsv4_hc_pre; typedef struct { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index cc1bebfaaa..77c399bdbc 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -1405,7 +1405,7 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_encoder_t enc = ctx->enc; - auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op->op); + auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op); ggml_metal_encoder_set_pipeline(enc, pipeline); @@ -1467,8 +1467,10 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { /*.nb_x2 =*/ x->nb[2], /*.nb_w0 =*/ weights->nb[0], /*.nb_w1 =*/ weights->nb[1], + /*.nb_w2 =*/ weights->nb[2], /*.nb_d0 =*/ op->nb[0], /*.nb_d1 =*/ op->nb[1], + /*.scale =*/ ggml_get_op_params_f32(op, 0), }; ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); @@ -1491,7 +1493,6 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(residual->type == GGML_TYPE_F32); GGML_ASSERT(post->type == GGML_TYPE_F32); - GGML_ASSERT(comb->type == GGML_TYPE_F32); GGML_ASSERT(op->type == GGML_TYPE_F32); GGML_ASSERT(residual->ne[1] == 4); @@ -1505,9 +1506,9 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { /*.nb_r2 =*/ residual->nb[2], /*.nb_p0 =*/ post->nb[0], /*.nb_p1 =*/ post->nb[1], - /*.nb_c0 =*/ comb->nb[0], - /*.nb_c1 =*/ comb->nb[1], - /*.nb_c2 =*/ comb->nb[2], + /*.nb_c0 =*/ comb ? comb->nb[0] : 0, + /*.nb_c1 =*/ comb ? comb->nb[1] : 0, + /*.nb_c2 =*/ comb ? comb->nb[2] : 0, /*.nb_d0 =*/ op->nb[0], /*.nb_d1 =*/ op->nb[1], /*.nb_d2 =*/ op->nb[2], @@ -1517,8 +1518,12 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(residual), 2); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(post), 3); - ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4); - ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + if (comb) { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + } else { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4); + } const int n_tiles = (args.n_embd + 31)/32; const int nsg = std::min(4, n_tiles); diff --git a/ggml/src/ggml-metal/kernels/misc.metal b/ggml/src/ggml-metal/kernels/misc.metal index 11104b4d8d..15a18e04ab 100644 --- a/ggml/src/ggml-metal/kernels/misc.metal +++ b/ggml/src/ggml-metal/kernels/misc.metal @@ -531,7 +531,73 @@ kernel void kernel_dsv4_hc_pre_f32( result = fma(*(device const float *) (xb + ih*args.nb_x1), w[ih], result); } - *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = result; + *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result; +} + +kernel void kernel_dsv4_hc_pre_gated_f32( + constant ggml_metal_kargs_dsv4_hc_pre & args, + device const char * x, + device const char * gate, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr ushort hc = 4; + + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + if (i0 >= args.n_embd) { + return; + } + + device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2; + device const char * gb = gate + i0*args.nb_w0 + it*args.nb_w2; + float result = 0.0f; + FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) { + const float g = 1.0f/(1.0f + exp(-*(device const float *) (gb + ih*args.nb_w1))); + result = fma(*(device const float *) (xb + ih*args.nb_x1), g, result); + } + + *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result; +} + +kernel void kernel_dsv4_hc_post_nocomb_f32( + constant ggml_metal_kargs_dsv4_hc_post & args, + device const char * x, + device const char * residual, + device const char * post, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr ushort hc = 4; + + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + float post_lane = 0.0f; + if (tiisg < hc) { + post_lane = *(device const float *) (post + tiisg*args.nb_p0 + it*args.nb_p1); + } + + float post_reg[hc]; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + post_reg[idst] = simd_shuffle(post_lane, idst); + } + + if (i0 >= args.n_embd) { + return; + } + + const float xv = *(device const float *) (x + i0*args.nb_x0 + it*args.nb_x1); + device const char * rb = residual + i0*args.nb_r0 + it*args.nb_r2; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + const float rv = *(device const float *) (rb + idst*args.nb_r1); + *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = xv*post_reg[idst] + rv; + } } kernel void kernel_dsv4_hc_post_f32(