From 4afa009405863a41b4e86e0bca141ac62f9ea194 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 17 Sep 2026 23:18:56 -0500 Subject: [PATCH] whisper : optional ANEForge encoder backend (Apple Neural Engine) (#3905) * whisper : optional ANEForge encoder backend (Apple Neural Engine) Runs the Whisper encoder directly on the Apple Neural Engine via ANEForge, an alternative to the CoreML encoder backend. About 2x faster than the CoreML encoder and faster than the Metal GPU encoder on M-series, at ~5x lower energy, with the same transcripts (cosine 0.999 vs the reference encoder). Mirrors the existing CoreML/OpenVINO seam: fills embd_enc from the external aneforge package via the same whisper_encode_external path, gated at runtime by the ANEFORGE_ENCODER env var. With the variable unset the build behaves exactly like stock whisper.cpp (the backend dlopens its dispatch dylib only when enabled, no new link-time dependency). Encoder only; the decoder is untouched. Adds src/aneforge/whisper-aneforge.{h,cpp}, one line to src/CMakeLists.txt, and ~30 lines to src/whisper.cpp (include, state field, encode branch, init, free). * aneforge: one statement per line, align dlsym block (review) * aneforge: guard the backend to Apple Silicon, stub elsewhere (fix non-Apple CI) whisper-aneforge.cpp uses and __fp16, which are unavailable on Windows (MSVC) and non-ARM Linux, so the unconditional build broke those CI jobs. Wrap the real implementation in #if defined(__APPLE__) && defined(__aarch64__) and provide no-op stubs elsewhere; the encoder is Apple-Neural-Engine-only anyway, and the library now links on every platform. * examples : include whisper-aneforge.cpp in android examples Refs: https://github.com/ggml-org/whisper.cpp/actions/runs/32758607709/job/97689705921?pr=3905#step:5:200 * ci : specify explicit packages (platform-tools) This commit updates the android jobs to specify platform-tools as an explicit package. The motivation for this to try to fix the Setup Android SDK step which is also trying to install an obolete tools package. Refs: https://github.com/ggml-org/whisper.cpp/actions/runs/35215042978/job/105181619536?pr=3905 --------- Co-authored-by: Daniel Bevenius --- .github/workflows/build-android.yml | 3 + .../app/src/main/jni/whisper/CMakeLists.txt | 1 + .../lib/src/main/jni/whisper/CMakeLists.txt | 1 + src/CMakeLists.txt | 2 + src/aneforge/whisper-aneforge.cpp | 170 ++++++++++++++++++ src/aneforge/whisper-aneforge.h | 25 +++ src/whisper.cpp | 57 ++++-- 7 files changed, 244 insertions(+), 15 deletions(-) create mode 100644 src/aneforge/whisper-aneforge.cpp create mode 100644 src/aneforge/whisper-aneforge.h diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml index 571c35872..3c0a78cc8 100644 --- a/.github/workflows/build-android.yml +++ b/.github/workflows/build-android.yml @@ -42,6 +42,8 @@ jobs: - name: Setup Android SDK uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 + with: + packages: platform-tools - name: Build run: | @@ -72,6 +74,7 @@ jobs: uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 with: cmdline-tools-version: 9.0 + packages: platform-tools - name: Build run: | diff --git a/examples/whisper.android.java/app/src/main/jni/whisper/CMakeLists.txt b/examples/whisper.android.java/app/src/main/jni/whisper/CMakeLists.txt index b22bb15be..6043da105 100644 --- a/examples/whisper.android.java/app/src/main/jni/whisper/CMakeLists.txt +++ b/examples/whisper.android.java/app/src/main/jni/whisper/CMakeLists.txt @@ -18,6 +18,7 @@ message(STATUS " Whisper version: ${WHISPER_VERSION}") set(SOURCE_FILES ${WHISPER_LIB_DIR}/src/whisper.cpp + ${WHISPER_LIB_DIR}/src/aneforge/whisper-aneforge.cpp ${CMAKE_SOURCE_DIR}/jni.c ) diff --git a/examples/whisper.android/lib/src/main/jni/whisper/CMakeLists.txt b/examples/whisper.android/lib/src/main/jni/whisper/CMakeLists.txt index a82d0d7b7..43a4485dd 100644 --- a/examples/whisper.android/lib/src/main/jni/whisper/CMakeLists.txt +++ b/examples/whisper.android/lib/src/main/jni/whisper/CMakeLists.txt @@ -22,6 +22,7 @@ option(GGML_HOME "whisper: Path to external GGML source" OFF) set( SOURCE_FILES ${WHISPER_LIB_DIR}/src/whisper.cpp + ${WHISPER_LIB_DIR}/src/aneforge/whisper-aneforge.cpp ${CMAKE_SOURCE_DIR}/jni.c ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b6eb16f24..c5f69771f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -189,6 +189,8 @@ add_library(whisper ../include/whisper.h whisper-arch.h whisper.cpp + aneforge/whisper-aneforge.h + aneforge/whisper-aneforge.cpp ) add_library(parakeet diff --git a/src/aneforge/whisper-aneforge.cpp b/src/aneforge/whisper-aneforge.cpp new file mode 100644 index 000000000..f9f7eb9ea --- /dev/null +++ b/src/aneforge/whisper-aneforge.cpp @@ -0,0 +1,170 @@ +// ANEForge encoder backend (see whisper-aneforge.h). dlopen's the ANEForge dispatch +// dylib (libane_e5rt_dispatch.dylib) so no extra link configuration is needed; the +// path comes from ANEFORGE_DYLIB. Compiles the persisted encoder MIL once at init +// (compile-with-cache; the on-device program is not cold-loadable across processes), +// then dispatches it per encode. +#include "whisper-aneforge.h" + +// ANEForge runs on the Apple Neural Engine (Apple Silicon) only. Build a no-op stub on every +// other platform so the whisper library still links; the encoder just stays unavailable there. +#if defined(__APPLE__) && defined(__aarch64__) + +#include +#include +#include +#include +#include +#include +#include + +typedef struct ane_e5rt_program ane_e5rt_program_t; +typedef ane_e5rt_program_t * (*compile_fn)(const char *, const char *, uint64_t, + const char * const *, const size_t *, size_t, const char * const *, const size_t *, size_t); +typedef int (*set_in_fn)(ane_e5rt_program_t *, const char *, const uint16_t *, size_t); +typedef int (*get_out_fn)(ane_e5rt_program_t *, const char *, uint16_t *, size_t); +typedef int (*exec_fn)(ane_e5rt_program_t *); +typedef void (*release_fn)(ane_e5rt_program_t *); + +struct whisper_aneforge_context { + void * dl = nullptr; + ane_e5rt_program_t * prog = nullptr; + set_in_fn set_input = nullptr; + get_out_fn get_output = nullptr; + exec_fn execute = nullptr; + release_fn release = nullptr; + std::string mel_port, pos_port, out_port; + size_t mel_n = 0, pos_n = 0, out_n = 0; + std::vector mel16; + std::vector out16; +}; + +// arm64 has a native IEEE binary16 type; these casts compile to NEON fcvt and +// auto-vectorize, unlike branchy scalar bit-twiddling. ANEForge's fp16 ports are +// IEEE binary16, which __fp16 matches bit-for-bit. +static inline void f32_to_f16(const float * src, uint16_t * dst, size_t n) { + __fp16 * d = (__fp16 *) dst; + for (size_t i = 0; i < n; i++) d[i] = (__fp16) src[i]; +} +static inline void f16_to_f32(const uint16_t * src, float * dst, size_t n) { + const __fp16 * s = (const __fp16 *) src; + for (size_t i = 0; i < n; i++) dst[i] = (float) s[i]; +} + +struct whisper_aneforge_context * whisper_aneforge_init(const char * bundle_dir) { + const char * dylib = getenv("ANEFORGE_DYLIB"); + if (!dylib) { + fprintf(stderr, "aneforge: set ANEFORGE_DYLIB to libane_e5rt_dispatch.dylib\n"); + return nullptr; + } + void * dl = dlopen(dylib, RTLD_NOW | RTLD_LOCAL); + if (!dl) { + fprintf(stderr, "aneforge: dlopen(%s) failed: %s\n", dylib, dlerror()); + return nullptr; + } + + auto ctx = new whisper_aneforge_context(); + ctx->dl = dl; + auto compile = (compile_fn) dlsym(dl, "ane_e5rt_program_compile"); + ctx->set_input = (set_in_fn) dlsym(dl, "ane_e5rt_program_set_input_fp16"); + ctx->get_output = (get_out_fn) dlsym(dl, "ane_e5rt_program_get_output_fp16"); + ctx->execute = (exec_fn) dlsym(dl, "ane_e5rt_program_execute"); + ctx->release = (release_fn) dlsym(dl, "ane_e5rt_program_release"); + if (!compile || !ctx->set_input || !ctx->get_output || !ctx->execute || !ctx->release) { + fprintf(stderr, "aneforge: missing dispatch symbols\n"); + whisper_aneforge_free(ctx); + return nullptr; + } + + // ports.txt: three lines "name nelems" for mel, pos, output (in that order). + std::string dir = bundle_dir; + FILE * pf = fopen((dir + "/ports.txt").c_str(), "r"); + if (!pf) { + fprintf(stderr, "aneforge: no ports.txt in %s\n", bundle_dir); + whisper_aneforge_free(ctx); + return nullptr; + } + char nm[256]; + size_t ne; + std::string * ports[3] = {&ctx->mel_port, &ctx->pos_port, &ctx->out_port}; + size_t * nes[3] = {&ctx->mel_n, &ctx->pos_n, &ctx->out_n}; + for (int i = 0; i < 3; i++) { + if (fscanf(pf, "%255s %zu", nm, &ne) != 2) { + fclose(pf); + whisper_aneforge_free(ctx); + return nullptr; + } + *ports[i] = nm; + *nes[i] = ne; + } + fclose(pf); + + std::string mil = dir + "/model.mil", cache = dir + "/cache"; + const char * in_names[2] = {ctx->mel_port.c_str(), ctx->pos_port.c_str()}; + size_t in_bytes[2] = {ctx->mel_n * 2, ctx->pos_n * 2}; + const char * out_name = ctx->out_port.c_str(); + size_t out_bytes = ctx->out_n * 2; + ctx->prog = compile(mil.c_str(), cache.c_str(), 0x4 /*ANE*/, in_names, in_bytes, 2, &out_name, &out_bytes, 1); + if (!ctx->prog) { + fprintf(stderr, "aneforge: compile failed\n"); + whisper_aneforge_free(ctx); + return nullptr; + } + + // The positional-embedding port is a constant; set it once from pos.f16. + std::vector pos(ctx->pos_n); + FILE * pp = fopen((dir + "/pos.f16").c_str(), "rb"); + if (!pp || fread(pos.data(), 2, ctx->pos_n, pp) != ctx->pos_n) { + fprintf(stderr, "aneforge: pos.f16 read failed\n"); + if (pp) { + fclose(pp); + } + whisper_aneforge_free(ctx); + return nullptr; + } + fclose(pp); + ctx->set_input(ctx->prog, ctx->pos_port.c_str(), pos.data(), ctx->pos_n); + + ctx->mel16.resize(ctx->mel_n); + ctx->out16.resize(ctx->out_n); + fprintf(stderr, "aneforge: encoder ready (mel=%s pos=%s out=%s)\n", + ctx->mel_port.c_str(), ctx->pos_port.c_str(), ctx->out_port.c_str()); + return ctx; +} + +void whisper_aneforge_encode(struct whisper_aneforge_context * ctx, + int64_t n_mel, int64_t n_len, const float * mel, float * out) { + size_t n = (size_t) n_mel * (size_t) n_len; + if (n != ctx->mel_n) { + fprintf(stderr, "aneforge: mel size %zu != %zu\n", n, ctx->mel_n); + return; + } + f32_to_f16(mel, ctx->mel16.data(), n); + ctx->set_input(ctx->prog, ctx->mel_port.c_str(), ctx->mel16.data(), ctx->mel_n); + ctx->execute(ctx->prog); + ctx->get_output(ctx->prog, ctx->out_port.c_str(), ctx->out16.data(), ctx->out_n); + // The encoder output is [S, d_model] row-major, which is whisper.cpp's embd_enc layout. + f16_to_f32(ctx->out16.data(), out, ctx->out_n); +} + +void whisper_aneforge_free(struct whisper_aneforge_context * ctx) { + if (!ctx) return; + if (ctx->prog && ctx->release) ctx->release(ctx->prog); + if (ctx->dl) dlclose(ctx->dl); + delete ctx; +} + +#else // not (Apple && arm64): the ANE is unavailable, so stub the entry points. + +struct whisper_aneforge_context {}; + +struct whisper_aneforge_context * whisper_aneforge_init(const char *) { + return nullptr; +} + +void whisper_aneforge_encode(struct whisper_aneforge_context *, int64_t, int64_t, const float *, float *) { +} + +void whisper_aneforge_free(struct whisper_aneforge_context *) { +} + +#endif diff --git a/src/aneforge/whisper-aneforge.h b/src/aneforge/whisper-aneforge.h new file mode 100644 index 000000000..7ecfe1499 --- /dev/null +++ b/src/aneforge/whisper-aneforge.h @@ -0,0 +1,25 @@ +// ANEForge encoder backend for whisper.cpp: runs the audio encoder directly on the +// Apple Neural Engine via ANEForge's e5rt dispatch shim, in place of the ggml/Metal or +// CoreML encoder. Enabled at runtime by setting ANEFORGE_ENCODER to a bundle directory +// (model.mil + weights.bin + ports.txt + pos.f16), produced by export_encoder.py. +#pragma once +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct whisper_aneforge_context; + +struct whisper_aneforge_context * whisper_aneforge_init(const char * bundle_dir); + +// mel: n_mel x n_len fp32 (channel-major, as whisper stores it). out: the encoder +// output, n_ctx x n_state fp32, written in the [n_state, n_ctx] ggml layout. +void whisper_aneforge_encode(struct whisper_aneforge_context * ctx, + int64_t n_mel, int64_t n_len, const float * mel, float * out); + +void whisper_aneforge_free(struct whisper_aneforge_context * ctx); + +#ifdef __cplusplus +} +#endif diff --git a/src/whisper.cpp b/src/whisper.cpp index 662f80644..4c73bbc86 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -10,6 +10,8 @@ #include "coreml/whisper-encoder.h" #endif +#include "aneforge/whisper-aneforge.h" + #ifdef WHISPER_USE_OPENVINO #include "openvino/whisper-openvino-encoder.h" #endif @@ -903,6 +905,8 @@ struct whisper_state { whisper_coreml_context * ctx_coreml = nullptr; #endif + whisper_aneforge_context * ctx_aneforge = nullptr; + #ifdef WHISPER_USE_OPENVINO whisper_openvino_context * ctx_openvino = nullptr; #endif @@ -1990,7 +1994,9 @@ static bool whisper_encode_external(const whisper_state & wstate) { const bool use_vitisai = wstate.ctx_vitisai != nullptr; #endif - return use_coreml || use_openvino || use_vitisai; + const bool use_aneforge = wstate.ctx_aneforge != nullptr; + + return use_coreml || use_openvino || use_vitisai || use_aneforge; } static bool whisper_cross_external(const whisper_state & wstate) { @@ -2443,26 +2449,30 @@ static bool whisper_encode_internal( } else { ggml_backend_sched_reset(sched); + if (wstate.ctx_aneforge != nullptr) { + whisper_aneforge_encode(wstate.ctx_aneforge, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data); + } else { #if defined(WHISPER_USE_COREML) - whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data); + whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data); #elif defined(WHISPER_USE_VITISAI) - if (whisper_vitisai_has_cross_proj(wstate.ctx_vitisai)) { - const auto & hp = wctx.model.hparams; - const int n_ctx = wstate.exp_n_audio_ctx > 0 - ? wstate.exp_n_audio_ctx : hp.n_audio_ctx; - if (!whisper_vitisai_encode_with_cross( - wstate.ctx_vitisai, mel, wstate.embd_enc, - wstate.kv_cross.k, wstate.kv_cross.v, - hp.n_text_layer, n_ctx, hp.n_text_state, - hp.n_text_head, wctx.params.flash_attn)) { + if (whisper_vitisai_has_cross_proj(wstate.ctx_vitisai)) { + const auto & hp = wctx.model.hparams; + const int n_ctx = wstate.exp_n_audio_ctx > 0 + ? wstate.exp_n_audio_ctx : hp.n_audio_ctx; + if (!whisper_vitisai_encode_with_cross( + wstate.ctx_vitisai, mel, wstate.embd_enc, + wstate.kv_cross.k, wstate.kv_cross.v, + hp.n_text_layer, n_ctx, hp.n_text_state, + hp.n_text_head, wctx.params.flash_attn)) { + return false; + } + } else if (!whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc)) { return false; } - } else if (!whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc)) { - return false; - } #elif defined(WHISPER_USE_OPENVINO) - whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc); + whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc); #endif + } } } @@ -3536,6 +3546,18 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { } #endif + if (const char * aneforge_dir = getenv("ANEFORGE_ENCODER")) { + WHISPER_LOG_INFO("%s: loading ANEForge encoder from '%s'\n", __func__, aneforge_dir); + WHISPER_LOG_INFO("%s: compiling for the ANE (one time) ...\n", __func__); + state->ctx_aneforge = whisper_aneforge_init(aneforge_dir); + if (!state->ctx_aneforge) { + WHISPER_LOG_ERROR("%s: failed to load ANEForge encoder from '%s'\n", __func__, aneforge_dir); + whisper_free_state(state); + return nullptr; + } + WHISPER_LOG_INFO("%s: ANEForge encoder loaded\n", __func__); + } + state->logits.reserve(ctx->vocab.n_vocab * ctx->model.hparams.n_text_ctx); state->batch = whisper_batch_init(ctx->model.hparams.n_text_ctx, WHISPER_MAX_DECODERS); @@ -3911,6 +3933,11 @@ void whisper_free_state(struct whisper_state * state) { } #endif + if (state->ctx_aneforge != nullptr) { + whisper_aneforge_free(state->ctx_aneforge); + state->ctx_aneforge = nullptr; + } + #ifdef WHISPER_USE_OPENVINO if (state->ctx_openvino != nullptr) { whisper_openvino_free(state->ctx_openvino);