mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-01 15:40:46 -05:00
7e1e28cae3
* mtmd : add Nemotron 3 Nano Omni support (parakeet) This commit adds support for the subsampling and encoder part of Nemotron Nemo 3 omni model. The Parakeet subsampling/encoder were taken from parakeet.cpp which is currently a pull request against whisper.cpp. I've tried to copy the code a close as possible to hopefully enable easy patching between the these two project later. Refs: https://github.com/ggml-org/whisper.cpp/pull/3735 * mtmd : generate rel pos tensor in graph instead of in conversion [no ci] This commit removes the generation of the relative positional tensor in the model conversion script and instead computes it in the encoder graph. This is only done for the window of positions required for the current audio sample. * mtmd : add clip_get_model to clip API [no ci] This commit adds a function to get access to the clip_model. It also removes the two functions clip_get_mel_filter_tensor, and clip_get_window_tensor(const struct clip_ctx * ctx) which can now use clip_get_model to access the model tensors that it needs. * mtmd : read mel_filters and window into hparams * mtmd : use set_input_f32 lambda [no ci] * mtmd : add better asserts for mel_filters and hann window [no ci] * mtmd : add missing size_t cast * mtmd : change type of pad to size_t * mtmd : zero initialize samples_padded * mtmd : remove unsued ctx member from parakeet preprocessor * mtmd : make log_mel_spectrogram_parakeet_worker_thread private static * mtmd : sync/update parakeeet impl with latest whisper.cpp This commit updates the parakeet code in mtmd to reflect the latest updates to parakeet.cpp in whisper.cpp. A follow up commit will address the currently hardcoded dw_pad and see if we can add n_conv_kernel as a model metadata field. * mtmd : add audio_conv_kernel_size to model conversion This commit updates the model conversion to read the conv_kernel_size field from the sound_config section of the models config.json file. It then uses this field instead of the hardcoded values in parakeet.cpp. * mtmd : cleanup [no ci] * conversion : call super().filter_tensors [no ci] * do not discard result of super filter_tensors * mtmd : use build_mm instead of ggml_mul_mat * mtmd : use build_ffn * mtmd : move and reuse get_vector lambda * mtmd : use build_inp_raw for parakeet * mtmd : throw exception in get_scalar instead of assert * mtmd : fix std::min call * mtmt : use .c_str in throw clause in get_vector * mtmd : check for F32 type and non-empty tensor in get_vector The get_vector lambda is used by get_scalar but also standalone to read in the mel_filters and the window data. Therefor we are not checking for 1D tensors but allowing multiple dimensions. We do have a check in get_scalar to verify the size of the vector. * mtmd : replace hardcoded 1101 for n_tokens_real * mtmd : assert subsampling_factor is 8 This commit adds an assert of the parakeet subsampling factor to check that it is 8. The motivation for this is that this model currently has three convolutions with a stride of 2. If the underlying model updates the subsampling factor these convolution operations will need to be updated and this will produce and error if this occurs. * mtmd : remove unused ggml_tensors attn_pos_w and mm_norm_w * mtmd : remove single thread path This commit removes the single thread path which was a left over from the original parakeet.cpp where n_threads is configurable. * fix some security issues --------- Co-authored-by: Sigbjørn Skjæret <1629204+CISC@users.noreply.github.com> Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
422 lines
19 KiB
C++
422 lines
19 KiB
C++
#include "models.h"
|
|
|
|
static constexpr int PARAKEET_LOCAL_ATTN_THRESHOLD = 8192;
|
|
static constexpr int PARAKEET_LOCAL_ATTN_WINDOW = 128;
|
|
|
|
// conv subsampling + conformer encoder
|
|
ggml_cgraph * clip_graph_parakeet::build() {
|
|
|
|
// Conv subsampling
|
|
ggml_tensor * inp = build_inp_raw(1);
|
|
inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
|
|
|
// [freq, time, channels, batch]
|
|
ggml_tensor * cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[0], inp, 2, 2, 1, 1, 1, 1);
|
|
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[0]);
|
|
cb(cur, "pre_conv_0", -1);
|
|
|
|
cur = ggml_relu(ctx0, cur);
|
|
cb(cur, "pre_conv_0_relu", -1);
|
|
|
|
// [freq, time, channels, batch]
|
|
cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[2], cur, 2, 2, 1, 1, 1, 1);
|
|
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[2]);
|
|
cb(cur, "pre_conv_2", -1);
|
|
|
|
// [freq, time, channels, batch]
|
|
cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[3], cur, 1, 1, 0, 0, 1, 1);
|
|
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[3]);
|
|
cb(cur, "pre_conv_3", -1);
|
|
|
|
cur = ggml_relu(ctx0, cur);
|
|
cb(cur, "pre_conv_3_relu", -1);
|
|
|
|
// [freq, time, channels, batch]
|
|
cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[5], cur, 2, 2, 1, 1, 1, 1);
|
|
cb(cur, "pre_conv_5_direct", -1);
|
|
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[5]);
|
|
cb(cur, "pre_conv_5", -1);
|
|
|
|
// [freq, time, channels, batch]
|
|
cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[6], cur, 1, 1, 0, 0, 1, 1);
|
|
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[6]);
|
|
cb(cur, "pre_conv_6", -1);
|
|
|
|
cur = ggml_relu(ctx0, cur);
|
|
cb(cur, "pre_conv_6_relu", -1);
|
|
|
|
// [freq, time, chan]
|
|
cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
|
|
// [freq, chan, time]
|
|
cur = ggml_cont(ctx0, cur);
|
|
|
|
const int n_freq = cur->ne[0];
|
|
const int n_chan = cur->ne[1];
|
|
const int n_frames = cur->ne[2];
|
|
|
|
// [freq, time, chan, batch] -> [(freq * chan), time]
|
|
cur = ggml_reshape_2d(ctx0, cur, n_freq * n_chan, n_frames);
|
|
|
|
cur = build_mm(model.pre_encode_out_w, cur);
|
|
cur = ggml_add(ctx0, cur, model.pre_encode_out_b);
|
|
|
|
ggml_set_name(cur, "pre_enc_out");
|
|
|
|
// Encoder
|
|
|
|
const auto & hparams = model.hparams;
|
|
const int n_layer = hparams.n_layer;
|
|
const int n_state = hparams.n_embd;
|
|
const float fc_factor = 0.5f;
|
|
|
|
const int n_time = cur->ne[1];
|
|
const bool local_attn = n_time > PARAKEET_LOCAL_ATTN_THRESHOLD;
|
|
const int att_left = local_attn ? PARAKEET_LOCAL_ATTN_WINDOW : n_time - 1;
|
|
const int att_right = local_attn ? PARAKEET_LOCAL_ATTN_WINDOW : n_time - 1;
|
|
const int window_size = local_attn ? att_left + att_right + 1 : 2 * n_time - 1;
|
|
const int d_half = n_state / 2;
|
|
const int mask_dim = local_attn ? window_size : n_time;
|
|
|
|
// mask [key, n_time]
|
|
struct ggml_tensor * attn_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, mask_dim, n_time);
|
|
ggml_set_name(attn_mask, "attn_mask");
|
|
ggml_set_input(attn_mask);
|
|
|
|
struct ggml_tensor * local_mask = nullptr;
|
|
if (local_attn) {
|
|
const int chunk = att_left + att_right;
|
|
local_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, chunk + window_size - 1, chunk);
|
|
ggml_set_name(local_mask, "local_mask");
|
|
ggml_set_input(local_mask);
|
|
}
|
|
|
|
struct ggml_tensor * pos_freqs = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, d_half);
|
|
ggml_set_name(pos_freqs, "pos_freqs");
|
|
ggml_set_input(pos_freqs);
|
|
|
|
struct ggml_tensor * rel_positions = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, window_size);
|
|
ggml_set_name(rel_positions, "rel_positions");
|
|
ggml_set_input(rel_positions);
|
|
|
|
struct ggml_tensor * freqs = ggml_repeat_4d(ctx0, pos_freqs, d_half, window_size, 1, 1);
|
|
struct ggml_tensor * theta = ggml_mul(ctx0, freqs, rel_positions);
|
|
|
|
struct ggml_tensor * sin = ggml_reshape_3d(ctx0, ggml_sin(ctx0, theta), 1, d_half, window_size);
|
|
struct ggml_tensor * cos = ggml_reshape_3d(ctx0, ggml_cos(ctx0, theta), 1, d_half, window_size);
|
|
struct ggml_tensor * pos_emb = ggml_reshape_2d(ctx0, ggml_cont(ctx0, ggml_concat(ctx0, sin, cos, 0)), n_state, window_size);
|
|
ggml_set_name(pos_emb, "pos_emb");
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
const auto & layer = model.layers[il];
|
|
// FFN1
|
|
{
|
|
struct ggml_tensor * residual = cur;
|
|
ggml_format_name(cur, "enc_%d_res", il);
|
|
|
|
// norm
|
|
cur = ggml_norm(ctx0, cur, hparams.eps);
|
|
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ff_norm_w), layer.ff_norm_b);
|
|
ggml_format_name(cur, "enc_%d_ffn_norm_1", il);
|
|
|
|
cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_SILU, il);
|
|
ggml_format_name(cur, "enc_%d_ffn_1", il);
|
|
|
|
cur = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, fc_factor));
|
|
ggml_format_name(cur, "enc_%d_res_ffn", il);
|
|
}
|
|
|
|
// self attention block using relative positional encoding from model.position_embedding.
|
|
{
|
|
// [feat, time_frames, 1, 1]
|
|
struct ggml_tensor * residual = cur;
|
|
|
|
cur = ggml_norm(ctx0, cur, hparams.eps);
|
|
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ln_1_w), layer.ln_1_b);
|
|
ggml_format_name(cur, "enc_%d_attn_norm", il);
|
|
|
|
const int n_head = hparams.n_head;
|
|
const int d_head = n_state / n_head;
|
|
|
|
// [feat, time_frames, 1, 1]
|
|
struct ggml_tensor * Q_cur = build_mm(layer.q_w, cur);
|
|
struct ggml_tensor * K_cur = build_mm(layer.k_w, cur);
|
|
struct ggml_tensor * V_cur = build_mm(layer.v_w, cur);
|
|
|
|
// [d_head, n_heads, n_time, 1]
|
|
Q_cur = ggml_reshape_3d(ctx0, Q_cur, d_head, n_head, n_time);
|
|
K_cur = ggml_reshape_3d(ctx0, K_cur, d_head, n_head, n_time);
|
|
V_cur = ggml_reshape_3d(ctx0, V_cur, d_head, n_head, n_time);
|
|
|
|
// [n_state, window_size]
|
|
struct ggml_tensor * pos = build_mm(layer.linear_pos_w, pos_emb);
|
|
// [feat, head, window_size, 1]
|
|
pos = ggml_reshape_3d(ctx0, pos, d_head, n_head, pos_emb->ne[1]);
|
|
// [feat, window_size, head, 1]
|
|
pos = ggml_cont(ctx0, ggml_permute(ctx0, pos, 0, 2, 1, 3));
|
|
ggml_format_name(pos, "enc_%d_attn_pos", il);
|
|
|
|
if (local_attn) {
|
|
const int chunk = att_left + att_right;
|
|
const int n_group = (n_time + chunk - 1) / chunk;
|
|
const int n_time_padded = n_group * chunk;
|
|
const int n_kv_chunk = chunk + window_size - 1;
|
|
const int n_kv_dense = n_kv_chunk * n_group;
|
|
const bool need_padding = n_time_padded > n_time;
|
|
|
|
Q_cur = ggml_cont(ctx0, ggml_permute(ctx0, Q_cur, 0, 2, 1, 3));
|
|
K_cur = ggml_cont(ctx0, ggml_permute(ctx0, K_cur, 0, 2, 1, 3));
|
|
V_cur = ggml_cont(ctx0, ggml_permute(ctx0, V_cur, 0, 2, 1, 3));
|
|
|
|
// content bias
|
|
struct ggml_tensor * bias_u = ggml_reshape_3d(ctx0, layer.pos_bias_u, d_head, 1, n_head);
|
|
struct ggml_tensor * Q_u = ggml_add(ctx0, Q_cur, bias_u);
|
|
|
|
// position bias
|
|
struct ggml_tensor * bias_v = ggml_reshape_3d(ctx0, layer.pos_bias_v, d_head, 1, n_head);
|
|
struct ggml_tensor * Q_v = ggml_add(ctx0, Q_cur, bias_v);
|
|
|
|
// right pad the time dimension
|
|
struct ggml_tensor * Q_u_padded = need_padding ?
|
|
ggml_pad_ext(ctx0, Q_u, 0, 0, 0, n_time_padded - n_time, 0, 0, 0, 0) : Q_u;
|
|
Q_u_padded = ggml_reshape_4d(ctx0, Q_u_padded, d_head, chunk, n_group, n_head);
|
|
|
|
// pad front and back for the first and last time frames
|
|
struct ggml_tensor * K_padded = ggml_pad_ext(ctx0, K_cur, 0, 0, att_left, att_right, 0, 0, 0, 0);
|
|
if (n_kv_dense > K_padded->ne[1]) {
|
|
K_padded = ggml_pad_ext(ctx0, K_padded, 0, 0, 0, n_kv_dense - K_padded->ne[1], 0, 0, 0, 0);
|
|
}
|
|
|
|
// sliding window view: each group spans n_kv_chunk keys but steps by chunk
|
|
struct ggml_tensor * K_chunk = ggml_view_4d(ctx0, K_padded,
|
|
d_head, n_kv_chunk, n_group, n_head,
|
|
K_padded->nb[1],
|
|
(size_t) chunk * K_padded->nb[1],
|
|
K_padded->nb[2],
|
|
0);
|
|
K_chunk = ggml_cont(ctx0, K_chunk);
|
|
|
|
struct ggml_tensor * content_scores = ggml_mul_mat(ctx0, K_chunk, Q_u_padded);
|
|
|
|
// trim the dense output down to window_size scores per query
|
|
content_scores = ggml_view_4d(ctx0, content_scores,
|
|
window_size, chunk, n_group, n_head,
|
|
(size_t) (chunk + window_size) * content_scores->nb[0],
|
|
content_scores->nb[2],
|
|
content_scores->nb[3],
|
|
0);
|
|
content_scores = ggml_cont(ctx0, content_scores);
|
|
|
|
// ungroup: [window_size, n_time_padded, n_head]
|
|
content_scores = ggml_reshape_3d(ctx0, content_scores, window_size, n_time_padded, n_head);
|
|
if (need_padding) {
|
|
content_scores = ggml_view_3d(ctx0, content_scores,
|
|
window_size, n_time, n_head,
|
|
content_scores->nb[1],
|
|
content_scores->nb[2],
|
|
0);
|
|
}
|
|
|
|
// Q_v: [d_head, time, head]
|
|
Q_v = ggml_cont(ctx0, ggml_permute(ctx0, Q_v, 0, 2, 1, 3));
|
|
struct ggml_tensor * rel_pos_scores = ggml_mul_mat(ctx0, pos, Q_v);
|
|
|
|
struct ggml_tensor * attn_scores = ggml_add(ctx0, content_scores, rel_pos_scores);
|
|
attn_scores = ggml_soft_max_ext(ctx0, attn_scores, attn_mask, 1.0f / std::sqrt(d_head), 0.0f);
|
|
ggml_format_name(attn_scores, "enc_%d_attn_probs", il);
|
|
|
|
// expand probs back to n_kv_chunk width for the V matmul
|
|
struct ggml_tensor * probs_padded = need_padding ?
|
|
ggml_pad_ext(ctx0, attn_scores, 0, 0, 0, n_time_padded - n_time, 0, 0, 0, 0) : attn_scores;
|
|
|
|
probs_padded = ggml_reshape_4d(ctx0, probs_padded, window_size, chunk, n_group, n_head);
|
|
probs_padded = ggml_pad_ext(ctx0, probs_padded, 0, chunk, 0, 0, 0, 0, 0, 0);
|
|
probs_padded = ggml_view_4d(ctx0, probs_padded,
|
|
n_kv_chunk, chunk, n_group, n_head,
|
|
(size_t) n_kv_chunk * probs_padded->nb[0],
|
|
probs_padded->nb[2],
|
|
probs_padded->nb[3],
|
|
0);
|
|
probs_padded = ggml_cont(ctx0, probs_padded);
|
|
probs_padded = ggml_mul(ctx0, probs_padded, local_mask);
|
|
|
|
struct ggml_tensor * V_padded = ggml_pad_ext(ctx0, V_cur, 0, 0, att_left, att_right, 0, 0, 0, 0);
|
|
if (n_kv_dense > V_padded->ne[1]) {
|
|
V_padded = ggml_pad_ext(ctx0, V_padded, 0, 0, 0, n_kv_dense - V_padded->ne[1], 0, 0, 0, 0);
|
|
}
|
|
V_padded = ggml_cont(ctx0, ggml_transpose(ctx0, V_padded));
|
|
|
|
struct ggml_tensor * V_chunk = ggml_view_4d(ctx0, V_padded,
|
|
n_kv_chunk, d_head, n_group, n_head,
|
|
V_padded->nb[1],
|
|
(size_t) chunk * V_padded->nb[0],
|
|
V_padded->nb[2],
|
|
0);
|
|
V_chunk = ggml_cont(ctx0, V_chunk);
|
|
|
|
cur = ggml_mul_mat(ctx0, V_chunk, probs_padded);
|
|
cur = ggml_reshape_3d(ctx0, cur, d_head, n_time_padded, n_head);
|
|
if (need_padding) {
|
|
cur = ggml_view_3d(ctx0, cur, d_head, n_time, n_head, cur->nb[1], cur->nb[2], 0);
|
|
}
|
|
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
|
|
cur = ggml_reshape_2d(ctx0, cur, n_state, n_time);
|
|
cur = build_mm(layer.o_w, cur);
|
|
} else {
|
|
// full attention
|
|
struct ggml_tensor * Q_u = ggml_add(ctx0, Q_cur, layer.pos_bias_u);
|
|
ggml_format_name(Q_u, "enc_%d_attn_q_u", il);
|
|
|
|
struct ggml_tensor * K_prep = ggml_permute(ctx0, K_cur, 0, 2, 1, 3);
|
|
struct ggml_tensor * Q_prep = ggml_permute(ctx0, Q_u, 0, 2, 1, 3);
|
|
struct ggml_tensor * content_scores = ggml_mul_mat(ctx0, K_prep, Q_prep);
|
|
ggml_format_name(content_scores, "enc_%d_attn_content_scores", il);
|
|
|
|
struct ggml_tensor * Q_v = ggml_add(ctx0, Q_cur, layer.pos_bias_v);
|
|
ggml_format_name(Q_v, "enc_%d_attn_q_v", il);
|
|
|
|
Q_v = ggml_permute(ctx0, Q_v, 0, 2, 1, 3);
|
|
Q_v = ggml_cont(ctx0, Q_v);
|
|
ggml_format_name(Q_v, "enc_%d_attn_q_v_perm", il);
|
|
|
|
struct ggml_tensor * rel_pos_scores = ggml_mul_mat(ctx0, pos, Q_v);
|
|
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos", il);
|
|
|
|
// Relative positional shift
|
|
{
|
|
const auto pos_window = rel_pos_scores->ne[0];
|
|
const auto n_frame = rel_pos_scores->ne[1];
|
|
const auto n_head = rel_pos_scores->ne[2];
|
|
|
|
rel_pos_scores = ggml_pad(ctx0, rel_pos_scores, 1, 0, 0, 0);
|
|
rel_pos_scores = ggml_roll(ctx0, rel_pos_scores, 1, 0, 0, 0);
|
|
|
|
rel_pos_scores = ggml_reshape_3d(ctx0, rel_pos_scores, n_frame, pos_window + 1, n_head);
|
|
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
|
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_reshaped", il);
|
|
|
|
int center = pos_window / 2;
|
|
size_t offset = rel_pos_scores->nb[0] * (center+1);
|
|
|
|
rel_pos_scores = ggml_view_3d(ctx0, rel_pos_scores,
|
|
n_frame, pos_window, n_head,
|
|
(pos_window) * 4,
|
|
rel_pos_scores->nb[2],
|
|
offset);
|
|
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
|
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_shifted", il);
|
|
|
|
rel_pos_scores = ggml_view_3d(ctx0, rel_pos_scores,
|
|
content_scores->ne[0],
|
|
content_scores->ne[1],
|
|
rel_pos_scores->ne[2],
|
|
rel_pos_scores->nb[1],
|
|
rel_pos_scores->nb[2],
|
|
0);
|
|
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
|
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_shifted_view", il);
|
|
}
|
|
|
|
struct ggml_tensor * attn_scores = ggml_add(ctx0, content_scores, rel_pos_scores);
|
|
ggml_format_name(attn_scores, "enc_%d_attn_scores", il);
|
|
attn_scores = ggml_scale(ctx0, attn_scores, 1.0f / std::sqrt(d_head));
|
|
attn_scores = ggml_add(ctx0, attn_scores, attn_mask);
|
|
ggml_format_name(attn_scores, "enc_%d_attn_scores_scaled", il);
|
|
|
|
struct ggml_tensor * probs = ggml_soft_max(ctx0, attn_scores);
|
|
ggml_format_name(probs, "enc_%d_attn_probs", il);
|
|
|
|
V_cur = ggml_cont(ctx0, ggml_permute(ctx0, V_cur, 1, 2, 0, 3));
|
|
ggml_format_name(V_cur, "enc_%d_attn_v_cur", il);
|
|
cur = ggml_mul_mat(ctx0, probs, V_cur);
|
|
ggml_format_name(cur, "enc_%d_attn_inp", il);
|
|
|
|
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3);
|
|
cur = ggml_cont_2d(ctx0, cur, n_state, n_time);
|
|
cur = build_mm(layer.o_w, cur);
|
|
}
|
|
ggml_format_name(cur, "enc_%d_attn_out", il);
|
|
|
|
cur = ggml_add(ctx0, residual, cur);
|
|
ggml_format_name(cur, "enc_%d_attn_res", il);
|
|
}
|
|
|
|
// Convolution
|
|
{
|
|
struct ggml_tensor * residual = cur;
|
|
ggml_format_name(cur, "enc_%d_residual_conv", il);
|
|
|
|
cur = ggml_norm(ctx0, cur, hparams.eps);
|
|
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.norm_conv_w), layer.norm_conv_b);
|
|
ggml_format_name(cur, "enc_%d_norm_conv", il);
|
|
|
|
// pointwise 1d convolution:
|
|
cur = build_mm(layer.conv_pw1_w, cur);
|
|
ggml_format_name(cur, "enc_%d_conv_pw1", il);
|
|
|
|
{
|
|
int64_t d = cur->ne[0] / 2;
|
|
struct ggml_tensor * signal = ggml_view_2d(ctx0, cur, d, cur->ne[1], cur->nb[1], 0);
|
|
struct ggml_tensor * gate = ggml_view_2d(ctx0, cur, d, cur->ne[1], cur->nb[1], d * cur->nb[0]);
|
|
|
|
cur = ggml_mul(ctx0, signal, ggml_sigmoid(ctx0, gate));
|
|
ggml_format_name(cur, "enc_%d_conv_glu", il);
|
|
}
|
|
|
|
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
|
|
|
// use ggml_ssm_conv for f32 precision
|
|
const int dw_pad = (hparams.audio_conv_kernel_size - 1) / 2;
|
|
cur = ggml_pad(ctx0, cur, dw_pad, 0, 0, 0);
|
|
cur = ggml_roll(ctx0, cur, dw_pad, 0, 0, 0);
|
|
cur = ggml_pad(ctx0, cur, dw_pad, 0, 0, 0);
|
|
ggml_format_name(cur, "enc_%d_conv_dw_pad", il);
|
|
|
|
cur = ggml_ssm_conv(ctx0, cur, layer.conv_dw_w);
|
|
ggml_format_name(cur, "enc_%d_conv_1d_dw", il);
|
|
|
|
cur = ggml_sub(ctx0, cur, layer.conv_norm_mean);
|
|
struct ggml_tensor * std = ggml_sqrt(ctx0, layer.conv_norm_var);
|
|
cur = ggml_div(ctx0, cur, std);
|
|
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.conv_norm_w), layer.conv_norm_b);
|
|
ggml_format_name(cur, "enc_%d_conv_bn", il);
|
|
|
|
cur = ggml_silu(ctx0, cur);
|
|
ggml_format_name(cur, "enc_%d_conv_silu", il);
|
|
|
|
cur = build_mm(layer.conv_pw2_w, cur);
|
|
ggml_format_name(cur, "enc_%d_conv_pw2", il);
|
|
|
|
cur = ggml_add(ctx0, residual, cur);
|
|
ggml_format_name(cur, "enc_%d_conv_res", il);
|
|
}
|
|
|
|
// FFN2
|
|
{
|
|
struct ggml_tensor * residual = cur;
|
|
cur = ggml_norm(ctx0, cur, hparams.eps);
|
|
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ff_norm_1_w), layer.ff_norm_1_b);
|
|
ggml_format_name(cur, "enc_%d_ffn_norm_2", il);
|
|
|
|
cur = build_ffn(cur, layer.ff_up_1_w, nullptr, nullptr, nullptr, layer.ff_down_1_w, nullptr, FFN_SILU, il);
|
|
cur = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, 0.5));
|
|
ggml_format_name(cur, "enc_%d_ffn_res", il);
|
|
}
|
|
|
|
cur = ggml_norm(ctx0, cur, hparams.eps);
|
|
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ln_2_w), layer.ln_2_b);
|
|
}
|
|
|
|
cb(cur, "encoder_out", -1);
|
|
|
|
cur = ggml_rms_norm(ctx0, cur, 1e-6);
|
|
cur = ggml_mul(ctx0, cur, model.mm_norm_pre_w);
|
|
cb(cur, "sound_projection.norm", -1);
|
|
|
|
cur = build_ffn(cur, model.mm_0_w, model.mm_0_b, nullptr, nullptr, model.mm_1_w, model.mm_1_b, FFN_RELU_SQR, -1);
|
|
cb(cur, "projected", -1);
|
|
|
|
ggml_build_forward_expand(gf, cur);
|
|
|
|
return gf;
|
|
}
|