mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-17 08:17:40 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7f106b5459 | ||
|
|
d92ea9bb12 | ||
|
|
8d1599fca1 | ||
|
|
054a1babf3 | ||
|
|
513ad7ae07 | ||
|
|
67cf8967eb | ||
|
|
4df3f2959c | ||
|
|
a49db739fb | ||
|
|
eada89c315 | ||
|
|
8f647f1fdb |
@@ -25,6 +25,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${CANN_BASE_IMAGE} AS build
|
||||
|
||||
@@ -15,6 +15,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
|
||||
@@ -23,6 +23,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_CUDA_DEV_CONTAINER} AS build
|
||||
|
||||
@@ -17,6 +17,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
|
||||
@@ -21,6 +21,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
|
||||
|
||||
+134
-37
@@ -22,12 +22,19 @@
|
||||
spirv-headers,
|
||||
nodejs,
|
||||
importNpmLock,
|
||||
syclToolkit ? null,
|
||||
level-zero,
|
||||
ocl-icd,
|
||||
intel-graphics-compiler,
|
||||
intel-compute-runtime,
|
||||
makeBinaryWrapper,
|
||||
useBlas ?
|
||||
builtins.all (x: !x) [
|
||||
useCuda
|
||||
useMetalKit
|
||||
useRocm
|
||||
useVulkan
|
||||
useSycl
|
||||
]
|
||||
&& blas.meta.available,
|
||||
useCuda ? config.cudaSupport,
|
||||
@@ -38,11 +45,28 @@
|
||||
rocmGpuTargets ? builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets,
|
||||
useVulkan ? false,
|
||||
useRpc ? false,
|
||||
useSycl ? false,
|
||||
# Rebuild, not a runtime switch. cmake default is off.
|
||||
syclF16 ? false,
|
||||
# null keeps JIT. AOT makes icpx run ocloc, which DPC++ does not ship.
|
||||
syclDeviceArch ? null,
|
||||
syclMaxParallelLinkJobs ? null,
|
||||
llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake
|
||||
|
||||
# It's necessary to consistently use backendStdenv when building with CUDA support,
|
||||
# otherwise we get libstdc++ errors downstream.
|
||||
effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
|
||||
# CUDA needs its backend stdenv or libstdc++ mismatches downstream.
|
||||
# SYCL needs icpx which is provided by the oneAPI stdenv
|
||||
effectiveStdenv ?
|
||||
if useCuda && useSycl then
|
||||
throw "llama-cpp: useCuda and useSycl are mutually exclusive"
|
||||
else if useCuda then
|
||||
cudaPackages.backendStdenv
|
||||
else if useSycl then
|
||||
if syclToolkit == null then
|
||||
throw "llama-cpp: useSycl requires syclToolkit, consider setting it to intel-oneapi-toolkit"
|
||||
else
|
||||
syclToolkit.stdenv
|
||||
else
|
||||
stdenv,
|
||||
enableStatic ? effectiveStdenv.hostPlatform.isStatic,
|
||||
precompileMetalShaders ? false,
|
||||
useWebUi ? true,
|
||||
@@ -65,7 +89,8 @@ let
|
||||
++ lib.optionals useMetalKit [ "MetalKit" ]
|
||||
++ lib.optionals useMpi [ "MPI" ]
|
||||
++ lib.optionals useRocm [ "ROCm" ]
|
||||
++ lib.optionals useVulkan [ "Vulkan" ];
|
||||
++ lib.optionals useVulkan [ "Vulkan" ]
|
||||
++ lib.optionals useSycl [ "SYCL" ];
|
||||
|
||||
pnameSuffix =
|
||||
strings.optionalString (suffices != [ ])
|
||||
@@ -108,6 +133,26 @@ let
|
||||
shaderc
|
||||
spirv-headers
|
||||
];
|
||||
|
||||
syclAot = useSycl && syclDeviceArch != null && syclDeviceArch != "";
|
||||
|
||||
# source ocloc binary from intel-compute-runtime
|
||||
oclocAot = runCommand "ocloc-aot" { nativeBuildInputs = [ makeBinaryWrapper ]; } ''
|
||||
mkdir -p "$out/bin"
|
||||
shopt -s nullglob
|
||||
exact=(${lib.getBin intel-compute-runtime}/bin/ocloc)
|
||||
vers=(${lib.getBin intel-compute-runtime}/bin/ocloc-*)
|
||||
if [ ''${#exact[@]} -eq 1 ]; then
|
||||
src=''${exact[0]}
|
||||
elif [ ''${#vers[@]} -eq 1 ]; then
|
||||
src=''${vers[0]}
|
||||
else
|
||||
echo "expected one ocloc binary from intel-compute-runtime" >&2
|
||||
exit 1
|
||||
fi
|
||||
makeWrapper "$src" "$out/bin/ocloc" \
|
||||
--prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ intel-graphics-compiler ]}
|
||||
'';
|
||||
in
|
||||
|
||||
effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
@@ -137,7 +182,9 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
webui = stdenvNoCC.mkDerivation {
|
||||
pname = "webui";
|
||||
version = llamaVersion;
|
||||
src = lib.cleanSource ../../ui/apps/chat;
|
||||
# the app compiles Llama UI from its source in ../../lib/ui, so the source is ui/ as a whole
|
||||
src = lib.cleanSource ../../ui;
|
||||
sourceRoot = "source/apps/chat";
|
||||
|
||||
nativeBuildInputs = [
|
||||
nodejs
|
||||
@@ -168,6 +215,13 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
# see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
|
||||
__noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
|
||||
|
||||
# Do not fortify under SYCL as it breaks compiled memcpy invocations
|
||||
# _FORTIFY_SOURCE rewrites memcpy -> __memcpy_chk
|
||||
hardeningDisable = optionals useSycl [
|
||||
"fortify"
|
||||
"fortify3"
|
||||
];
|
||||
|
||||
nativeBuildInputs =
|
||||
[
|
||||
cmake
|
||||
@@ -181,7 +235,12 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
autoAddDriverRunpath
|
||||
]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ]
|
||||
++ optionals useSycl [
|
||||
autoAddDriverRunpath
|
||||
makeBinaryWrapper
|
||||
]
|
||||
++ optionals syclAot [ oclocAot ];
|
||||
|
||||
buildInputs =
|
||||
optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
|
||||
@@ -190,39 +249,53 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
++ optionals useRocm rocmBuildInputs
|
||||
++ optionals useBlas [ blas ]
|
||||
++ optionals useVulkan vulkanBuildInputs
|
||||
++ optionals useSycl [
|
||||
level-zero
|
||||
ocl-icd
|
||||
intel-graphics-compiler
|
||||
]
|
||||
++ [ openssl ];
|
||||
|
||||
cmakeFlags =
|
||||
[
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
)
|
||||
cmakeFlags = [
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
)
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
];
|
||||
)
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
]
|
||||
++ optionals useSycl [
|
||||
(cmakeBool "GGML_SYCL" true)
|
||||
(cmakeBool "GGML_SYCL_F16" syclF16)
|
||||
]
|
||||
++ optionals syclAot [
|
||||
(cmakeFeature "GGML_SYCL_DEVICE_ARCH" syclDeviceArch)
|
||||
]
|
||||
++ optionals (syclAot && syclMaxParallelLinkJobs != null) [
|
||||
(cmakeFeature "GGML_SYCL_MAX_PARALLEL_LINK_JOBS" (toString syclMaxParallelLinkJobs))
|
||||
];
|
||||
|
||||
# Environment variables needed for ROCm
|
||||
env = optionalAttrs useRocm {
|
||||
@@ -237,11 +310,35 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
cp $src/include/llama.h $out/include/
|
||||
'';
|
||||
|
||||
# Fix SYCL binary calling missing important libraries
|
||||
# Also use Level Zero + sysman INSTEAD of OpenCL
|
||||
# due to OpenCL reporting free memory as total memory,
|
||||
# breaking `-fit` functionality
|
||||
postFixup = lib.optionalString useSycl ''
|
||||
driverLibs=${
|
||||
lib.makeLibraryPath [
|
||||
intel-graphics-compiler
|
||||
ocl-icd
|
||||
]
|
||||
}
|
||||
for b in "$out"/bin/*; do
|
||||
[ -f "$b" ] && [ -x "$b" ] || continue
|
||||
patchelf --print-interpreter "$b" >/dev/null 2>&1 || continue
|
||||
patchelf --add-needed libOpenCL.so.1 --add-needed libigc.so.2 \
|
||||
--add-rpath "$driverLibs" "$b"
|
||||
wrapProgram "$b" \
|
||||
--set-default ONEAPI_DEVICE_SELECTOR level_zero:gpu \
|
||||
--set-default ZES_ENABLE_SYSMAN 1
|
||||
done
|
||||
'';
|
||||
|
||||
meta = {
|
||||
# Configurations we don't want even the CI to evaluate. Results in the
|
||||
# "unsupported platform" messages. This is mostly a no-op, because
|
||||
# cudaPackages would've refused to evaluate anyway.
|
||||
badPlatforms = optionals useCuda lib.platforms.darwin;
|
||||
badPlatforms =
|
||||
optionals useCuda lib.platforms.darwin
|
||||
++ optionals useSycl (lib.subtractLists [ "x86_64-linux" ] lib.platforms.all);
|
||||
|
||||
# Configurations that are known to result in build failures. Can be
|
||||
# overridden by importing Nixpkgs with `allowBroken = true`.
|
||||
|
||||
@@ -34,6 +34,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
## Build Image
|
||||
|
||||
@@ -23,6 +23,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
### Build image
|
||||
|
||||
@@ -15,6 +15,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
|
||||
@@ -15,6 +15,9 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
|
||||
@@ -18,6 +18,7 @@ on:
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -26,6 +27,7 @@ on:
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ on:
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -22,6 +23,7 @@ on:
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -107,6 +109,33 @@ jobs:
|
||||
run: npm run test:unit
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
lib-checks:
|
||||
name: Library checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
needs: ui-build
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "jinja/runtime.h"
|
||||
#include "jinja/value.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
@@ -857,8 +858,35 @@ static value tts_norm_builtin_sub(const func_args & args) {
|
||||
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx) {
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value) {
|
||||
const auto it = std::lower_bound(dict.keys.begin(), dict.keys.end(), key);
|
||||
if (it == dict.keys.end() || *it != key) {
|
||||
return false;
|
||||
}
|
||||
const size_t idx = it - dict.keys.begin();
|
||||
if (idx >= dict.values.size()) {
|
||||
throw std::runtime_error("tts_dict: malformed dictionary, a key has no value");
|
||||
}
|
||||
value = dict.values[idx];
|
||||
return true;
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const common_tts_dicts> dicts) {
|
||||
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
|
||||
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
|
||||
ctx.set_val("tts_dict", mk_val<value_func>("tts_dict", [dicts](const func_args & args) -> value {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_vals<value_string, value_string>();
|
||||
const std::string name = args.get_pos(0)->as_string().str();
|
||||
const auto it = dicts ? dicts->find(name) : common_tts_dicts::const_iterator();
|
||||
if (!dicts || it == dicts->end()) {
|
||||
throw std::runtime_error("tts_dict: unknown dictionary '" + name + "'");
|
||||
}
|
||||
std::string doc;
|
||||
if (!common_tts_dict_find(it->second, args.get_pos(1)->as_string().str(), doc)) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return value_from_json(common_json::parse(doc), false);
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
+16
-3
@@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
@@ -11,9 +12,10 @@
|
||||
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
|
||||
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
|
||||
//
|
||||
// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// functions exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
|
||||
// tts_norm_sub(text, pattern, repl) -> string
|
||||
// tts_dict(name, key) -> value of key in a dictionary of the model, or none
|
||||
|
||||
namespace jinja {
|
||||
struct context;
|
||||
@@ -57,6 +59,17 @@ std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & te
|
||||
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
|
||||
|
||||
// add tts_norm_scan() and tts_norm_sub() to the context of a task template
|
||||
// a dictionary of the model, e.g. a pronunciation lexicon
|
||||
struct common_tts_dict {
|
||||
std::vector<std::string> keys; // sorted by their bytes
|
||||
std::vector<std::string> values; // the JSON document of each key
|
||||
};
|
||||
|
||||
using common_tts_dicts = std::map<std::string, common_tts_dict>;
|
||||
|
||||
// binary search of key in a dictionary; on success, value is set to the JSON document
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value);
|
||||
|
||||
// add tts_norm_scan(), tts_norm_sub() and tts_dict() to the context of a task template
|
||||
// they are not jinja global builtins on purpose: chat templates are not affected
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx);
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx, std::shared_ptr<const common_tts_dicts> dicts = nullptr);
|
||||
|
||||
+24
-4
@@ -318,6 +318,7 @@ struct common_chat_templates {
|
||||
std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
|
||||
std::unique_ptr<common_chat_template> template_tool_use;
|
||||
std::unique_ptr<common_chat_template> template_tts; // the prompt of a text to speak
|
||||
std::shared_ptr<common_tts_dicts> tts_dicts; // dictionaries of the model for tts_dict(), by name
|
||||
};
|
||||
|
||||
common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
|
||||
@@ -806,10 +807,10 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
|
||||
}
|
||||
|
||||
// renders a task template (e.g. "tts") with the given variables
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars, const std::shared_ptr<common_tts_dicts> & tts_dicts) {
|
||||
jinja::context ctx(tmpl.source());
|
||||
// text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
// text normalization helpers: tts_norm_scan(), tts_norm_sub() and tts_dict(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx, tts_dicts);
|
||||
jinja::global_from_json(ctx, vars, false);
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
@@ -893,7 +894,7 @@ std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templa
|
||||
{"messages", json::array({{{"role", "user"}, {"content", text}}})},
|
||||
{"language", language},
|
||||
{"voice", voice},
|
||||
});
|
||||
}, chat_templates->tts_dicts);
|
||||
}
|
||||
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
@@ -1024,6 +1025,25 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
|
||||
}
|
||||
}
|
||||
// dictionaries for tts_dict(), listed by name in tokenizer.tts_dicts
|
||||
tmpls->tts_dicts = std::make_shared<common_tts_dicts>();
|
||||
for (int32_t i = 0; i < llama_model_meta_arr_n(model, "tokenizer.tts_dicts"); i++) {
|
||||
const std::string name = llama_model_meta_arr_str(model, "tokenizer.tts_dicts", i);
|
||||
const std::string key = "tokenizer.tts_dict." + name + ".keys";
|
||||
const std::string val_key = "tokenizer.tts_dict." + name + ".values";
|
||||
const int32_t n = llama_model_meta_arr_n(model, key.c_str());
|
||||
if (n < 0 || llama_model_meta_arr_n(model, val_key.c_str()) != n) {
|
||||
LOG_ERR("%s: the tts dictionary '%s' has no value for each of its keys (ignoring it)\n", __func__, name.c_str());
|
||||
continue;
|
||||
}
|
||||
auto & dict = (*tmpls->tts_dicts)[name];
|
||||
dict.keys.reserve(n);
|
||||
dict.values.reserve(n);
|
||||
for (int32_t j = 0; j < n; j++) {
|
||||
dict.keys.emplace_back(llama_model_meta_arr_str(model, key.c_str(), j));
|
||||
dict.values.emplace_back(llama_model_meta_arr_str(model, val_key.c_str(), j));
|
||||
}
|
||||
}
|
||||
}
|
||||
return tmpls;
|
||||
}
|
||||
|
||||
@@ -191,7 +191,7 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
if (is_float) {
|
||||
return mk_val<value_float>(res);
|
||||
} else {
|
||||
return mk_val<value_int>(static_cast<int64_t>(res));
|
||||
return mk_val<value_int>(float_to_int(res));
|
||||
}
|
||||
} else if (op.value == "/") {
|
||||
JJ_DEBUG("Division operation: %f / %f", a, b);
|
||||
@@ -222,7 +222,7 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
if (is_float) {
|
||||
return mk_val<value_float>(rem);
|
||||
} else {
|
||||
return mk_val<value_int>(static_cast<int64_t>(rem));
|
||||
return mk_val<value_int>(float_to_int(rem));
|
||||
}
|
||||
} else if (op.value == "<") {
|
||||
JJ_DEBUG("Comparison operation: %f < %f is %d", a, b, a < b);
|
||||
|
||||
@@ -711,7 +711,7 @@ const func_builtins & value_float_t::get_builtins() const {
|
||||
}},
|
||||
{"int", [](const func_args & args) -> value {
|
||||
args.ensure_vals<value_float>();
|
||||
int64_t val = static_cast<int64_t>(args.get_pos(0)->as_float());
|
||||
int64_t val = float_to_int(args.get_pos(0)->as_float());
|
||||
return mk_val<value_int>(val);
|
||||
}},
|
||||
{"float", [](const func_args & args) -> value {
|
||||
@@ -1120,7 +1120,7 @@ static value array_sum(const func_args & args) {
|
||||
is_float = is_float || is_val<value_float>(v);
|
||||
sum += v->as_float();
|
||||
}
|
||||
return is_float ? value(mk_val<value_float>(sum)) : value(mk_val<value_int>(static_cast<int64_t>(sum)));
|
||||
return is_float ? value(mk_val<value_float>(sum)) : value(mk_val<value_int>(float_to_int(sum)));
|
||||
}
|
||||
|
||||
const func_builtins & value_array_t::get_builtins() const {
|
||||
@@ -1674,6 +1674,11 @@ bool value_compare(const value & a, const value & b, value_compare_op op) {
|
||||
return result;
|
||||
}
|
||||
|
||||
template<>
|
||||
value value_from_json(const common_json & json_val, bool mark_input) {
|
||||
return from_json(json_val, mark_input);
|
||||
}
|
||||
|
||||
template<>
|
||||
void global_from_json(context & ctx, const common_json & json_obj, bool mark_input) {
|
||||
// printf("global_from_json: %s\n" , json_obj.dump(2).c_str());
|
||||
|
||||
+20
-2
@@ -105,6 +105,10 @@ struct context; // forward declaration
|
||||
template<typename T_JSON>
|
||||
void global_from_json(context & ctx, const T_JSON & json_obj, bool mark_input);
|
||||
|
||||
// convert a single JSON value, same rules as global_from_json()
|
||||
template<typename T_JSON>
|
||||
value value_from_json(const T_JSON & json_val, bool mark_input);
|
||||
|
||||
//
|
||||
// base value type
|
||||
//
|
||||
@@ -233,6 +237,20 @@ struct value_equality {
|
||||
// primitive value types
|
||||
//
|
||||
|
||||
// double to int64, saturating and with nan as 0; a plain cast is undefined outside the int64 range
|
||||
inline int64_t float_to_int(double v) {
|
||||
if (std::isnan(v)) {
|
||||
return 0;
|
||||
}
|
||||
if (v >= 9223372036854775808.0) {
|
||||
return INT64_MAX;
|
||||
}
|
||||
if (v <= -9223372036854775808.0) {
|
||||
return INT64_MIN;
|
||||
}
|
||||
return static_cast<int64_t>(v);
|
||||
}
|
||||
|
||||
struct value_int_t : public value_t {
|
||||
static constexpr auto kind_id = value_kind::INTEGER;
|
||||
int64_t val_int;
|
||||
@@ -240,7 +258,7 @@ struct value_int_t : public value_t {
|
||||
value_int_t(int64_t v) : value_t(kind_id) {
|
||||
val_int = v;
|
||||
val_flt = static_cast<double>(v);
|
||||
if (static_cast<int64_t>(val_flt) != v) {
|
||||
if (float_to_int(val_flt) != v) {
|
||||
val_flt = v < 0 ? -INFINITY : INFINITY;
|
||||
}
|
||||
}
|
||||
@@ -276,7 +294,7 @@ struct value_float_t : public value_t {
|
||||
double val_flt;
|
||||
value_float_t(double v) : value_t(kind_id) {
|
||||
val_flt = v;
|
||||
val_int = std::isfinite(v) ? static_cast<int64_t>(v) : 0;
|
||||
val_int = std::isfinite(v) ? float_to_int(v) : 0;
|
||||
}
|
||||
virtual std::string type() const override { return "Float"; }
|
||||
virtual double as_float() const override { return val_flt; }
|
||||
|
||||
@@ -285,6 +285,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"PocketTTSModel": "pockettts",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
|
||||
@@ -363,6 +364,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
"Gemma4ForConditionalGeneration": "gemma",
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, gguf
|
||||
from .drama3 import fold_snake
|
||||
from .qwen import Qwen3Model
|
||||
|
||||
# MiniMax Music 3, diffusers layout: one checkpoint per component
|
||||
# text model: the global LLM (language_model/), samples the semantic code of each frame
|
||||
# mmproj: RVQ depth decoder, condition encoder, flow-matching DiT and DAC decoder (vocoder)
|
||||
|
||||
LM_DIR = "language_model"
|
||||
TOK_DIR = "tokenizer"
|
||||
MMPROJ_DIRS = ("rvq_depth_decoder", "condition_encoder", "transformer", "vocoder")
|
||||
|
||||
# token ids of the checkpoint, see encoders.py of the diffusers pipeline
|
||||
AUDIO_START = 151669
|
||||
AUDIO_END = 151670
|
||||
AUDIO_CODE_OFFSET = 151675
|
||||
N_SEMANTIC = 16384
|
||||
|
||||
|
||||
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
return ModelBase.load_hparams(dir_model / LM_DIR, is_mistral_format=False)
|
||||
|
||||
|
||||
def _index_dirs(model: ModelBase, dirs: Iterable[str], keep: Callable[[str], bool]) -> dict[str, Callable[[], Tensor]]:
|
||||
# the names get the component dir as prefix
|
||||
root = model.dir_model
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
try:
|
||||
for d in dirs:
|
||||
model.dir_model = root / d
|
||||
prefix = "model" if d == LM_DIR else "diffusion_pytorch_model"
|
||||
for name, gen in model.index_local_tensors(prefix).items():
|
||||
if keep(f"{d}.{name}"):
|
||||
tensors[f"{d}.{name}"] = gen
|
||||
finally:
|
||||
model.dir_model = root
|
||||
return tensors
|
||||
|
||||
|
||||
def _fold_wn(g: Tensor, v: Tensor) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = g.float(), v.float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxMusic3ForConditionalGeneration")
|
||||
class MiniMaxMusic3TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
tensors = _index_dirs(self, [LM_DIR], lambda _: True)
|
||||
return {name.removeprefix(LM_DIR + "."): gen for name, gen in tensors.items()}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
|
||||
self.gguf_writer.add_classifier_free_guidance(True)
|
||||
|
||||
def set_vocab(self):
|
||||
root = self.dir_model
|
||||
self.dir_model = root / TOK_DIR
|
||||
try:
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
finally:
|
||||
self.dir_model = root
|
||||
|
||||
# the semantic codes are not in the tokenizer, name them for debugging
|
||||
for i in range(N_SEMANTIC):
|
||||
tokens[AUDIO_CODE_OFFSET + i] = f"<|audio_code_{i}|>"
|
||||
toktypes[AUDIO_CODE_OFFSET + i] = gguf.TokenType.CONTROL
|
||||
|
||||
# the generation ends with <|audio_end|>, the chat template of the tokenizer is not used
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "MiniMaxAI-MiniMax-Music3-tts.jinja"
|
||||
with open(template_path, encoding="utf-8") as f:
|
||||
special_vocab.chat_template = [{"name": "tts", "template": f.read()}]
|
||||
special_vocab.special_token_ids["eos"] = AUDIO_END
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(tokpre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxMusic3ForConditionalGeneration")
|
||||
class MiniMaxMusic3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
hparams["text_config"] = _load_hparams(dir_model)
|
||||
for d in MMPROJ_DIRS:
|
||||
hparams[d] = ModelBase.load_hparams(dir_model / d, is_mistral_format=False)
|
||||
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
# the generic gen layers are the depth decoder
|
||||
return self.global_config["rvq_depth_decoder"]
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
# from the backbone, only the embeddings of the semantic codes
|
||||
return _index_dirs(self, (LM_DIR, *MMPROJ_DIRS),
|
||||
lambda name: not name.startswith(LM_DIR + ".") or name == f"{LM_DIR}.model.embed_tokens.weight")
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
return item
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.MINIMAX_MUSIC3_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(hp["num_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6)
|
||||
# the DiT, the condition encoder and the vocoder hparams come from the tensor shapes or are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# only the transformer layers follow --outtype, the convs and the small projections stay F32
|
||||
if n_dims >= 2 and new_name.startswith(("a.gen.code.blk.", "a.gen.flow.blk.")):
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
if new_name.startswith(("a.gen.code.head", "a.gen.code.embd", "a.gen.code.out_embd", "a.gen.code.proj_in")):
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
comp, key = name.split(".", 1)
|
||||
if comp == LM_DIR:
|
||||
yield ("a.gen.code.out_embd.weight", data_torch[AUDIO_CODE_OFFSET:AUDIO_CODE_OFFSET + N_SEMANTIC])
|
||||
elif comp == "rvq_depth_decoder":
|
||||
yield from self._depth(data_torch, key)
|
||||
elif comp == "condition_encoder":
|
||||
yield from self._cond(data_torch, key)
|
||||
elif comp == "transformer":
|
||||
yield from self._dit(data_torch, key)
|
||||
# the vocoder is folded in generate_extra_tensors()
|
||||
|
||||
def _depth(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.code"
|
||||
n_cb = self.hparams["num_codebooks"] - 1
|
||||
if key == "audio_embeddings.weight":
|
||||
# one table per residual codebook
|
||||
yield (f"{p}.embd.weight", t.reshape(n_cb, -1, t.shape[-1]))
|
||||
elif key.startswith("audio_heads."):
|
||||
if key == "audio_heads.0.weight":
|
||||
heads = [self._tensor(f"rvq_depth_decoder.audio_heads.{i}.weight") for i in range(n_cb)]
|
||||
yield (f"{p}.head.weight", torch.stack(heads))
|
||||
elif key == "projection.weight":
|
||||
yield (f"{p}.proj_in.weight", t)
|
||||
elif key == "pos_embedding.weight":
|
||||
yield (f"{p}.position_embd.weight", t)
|
||||
elif key == "norm.weight":
|
||||
yield (f"{p}.post_ln.weight", t)
|
||||
else:
|
||||
_, il, rest = key.split(".", 2)
|
||||
layer = {
|
||||
"input_layernorm.weight": "ln1.weight",
|
||||
"attn.to_q.weight": "attn_q.weight",
|
||||
"attn.to_k.weight": "attn_k.weight",
|
||||
"attn.to_v.weight": "attn_v.weight",
|
||||
"attn.to_out.weight": "attn_out.weight",
|
||||
"post_attention_layernorm.weight": "ln2.weight",
|
||||
"gate_proj.weight": "ffn_gate.weight",
|
||||
"up_proj.weight": "ffn_up.weight",
|
||||
"down_proj.weight": "ffn_down.weight",
|
||||
}
|
||||
yield (f"{p}.blk.{il}.{layer[rest]}", t)
|
||||
|
||||
def _cond(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.cond"
|
||||
if key == "layer_weight_logits":
|
||||
# softmax mix of the 8 hidden states of a frame, with the scale folded in
|
||||
scale = self._tensor("condition_encoder.layer_scale").float()
|
||||
yield (f"{p}.layer_w", torch.softmax(t.float(), dim=0) * scale)
|
||||
elif key == "proj.weight":
|
||||
yield (f"{p}.proj.weight", t)
|
||||
elif key == "proj.bias":
|
||||
yield (f"{p}.proj.bias", t)
|
||||
|
||||
def _dit(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.flow"
|
||||
if key.startswith("transformer_blocks."):
|
||||
_, il, rest = key.split(".", 2)
|
||||
q = f"{p}.blk.{il}"
|
||||
if rest.startswith("ff_in."):
|
||||
# value half first, then the silu gate half
|
||||
suffix = rest.removeprefix("ff_in.")
|
||||
up, gate = t.chunk(2, dim=0)
|
||||
yield (f"{q}.ffn_up.{suffix}", up)
|
||||
yield (f"{q}.ffn_gate.{suffix}", gate)
|
||||
return
|
||||
layer = {
|
||||
"norm1.weight": "ln1.weight",
|
||||
"norm1.bias": "ln1.bias",
|
||||
"attn.to_q.weight": "attn_q.weight",
|
||||
"attn.to_k.weight": "attn_k.weight",
|
||||
"attn.to_v.weight": "attn_v.weight",
|
||||
"attn.to_out.0.weight": "attn_out.weight",
|
||||
"norm2.weight": "ln2.weight",
|
||||
"norm2.bias": "ln2.bias",
|
||||
"ff_out.weight": "ffn_down.weight",
|
||||
"ff_out.bias": "ffn_down.bias",
|
||||
}
|
||||
yield (f"{q}.{layer[rest]}", t)
|
||||
return
|
||||
if key == "time_proj.weight":
|
||||
# random Fourier features, 2 * pi folded in
|
||||
yield (f"{p}.time_freq", t.float().flatten() * 2.0 * torch.pi)
|
||||
return
|
||||
out = {
|
||||
"time_embed.linear_1.weight": "time_in.weight",
|
||||
"time_embed.linear_1.bias": "time_in.bias",
|
||||
"time_embed.linear_2.weight": "time_out.weight",
|
||||
"time_embed.linear_2.bias": "time_out.bias",
|
||||
"preprocess_conv.weight": "pre.weight",
|
||||
"proj_in.weight": "proj_in.weight",
|
||||
"proj_out.weight": "proj_out.weight",
|
||||
"postprocess_conv.weight": "post.weight",
|
||||
}[key]
|
||||
yield (f"{p}.{out}", t.squeeze(-1) if key.endswith("conv.weight") else t)
|
||||
|
||||
def _tensor(self, name: str) -> Tensor:
|
||||
# read directly, without the dtype handling of prepare_tensors()
|
||||
return self.model_tensors[name]().float()
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# DAC decoder, same layout as the yue2 VAE (a.gen.vae.*), with the snake and the weight norm folded
|
||||
def get(key: str) -> Tensor:
|
||||
return self._tensor(f"vocoder.{key}")
|
||||
|
||||
def wn(key: str) -> Tensor:
|
||||
return _fold_wn(get(key + ".weight_g"), get(key + ".weight_v"))
|
||||
|
||||
def snake(dst: str, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
a, b = fold_snake(get(key + ".alpha"))
|
||||
yield (f"{dst}.alpha", a)
|
||||
yield (f"{dst}.beta", b)
|
||||
|
||||
p = "a.gen.vae"
|
||||
yield (f"{p}.in_proj.weight", get("dec_in_proj.weight"))
|
||||
yield (f"{p}.in_proj.bias", get("dec_in_proj.bias"))
|
||||
yield (f"{p}.conv_in.weight", wn("conv_in"))
|
||||
yield (f"{p}.conv_in.bias", get("conv_in.bias"))
|
||||
n_blk = len(self.global_config["vocoder"]["upsampling_ratios"])
|
||||
for b in range(n_blk):
|
||||
s = f"blocks.{b}"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.snake1")
|
||||
yield (f"{p}.blk.{b}.up.weight", wn(f"{s}.conv_t1"))
|
||||
yield (f"{p}.blk.{b}.up.bias", get(f"{s}.conv_t1.bias"))
|
||||
for r in range(3):
|
||||
rs = f"{s}.res_unit{r + 1}"
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.snake1")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.weight", wn(f"{rs}.conv1"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.bias", get(f"{rs}.conv1.bias"))
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.snake2")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.weight", wn(f"{rs}.conv2"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.bias", get(f"{rs}.conv2.bias"))
|
||||
yield from snake(f"{p}.post_snake", "snake_out")
|
||||
yield (f"{p}.conv_out.weight", wn("conv_out"))
|
||||
yield (f"{p}.conv_out.bias", get("conv_out.bias"))
|
||||
@@ -14,6 +14,7 @@ Supported models:
|
||||
- [ZONOS3](#zonos3)
|
||||
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
|
||||
- [YuE2](#yue2) (songs)
|
||||
- [MiniMax Music 3](#minimax-music-3) (songs)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
@@ -306,3 +307,48 @@ python convert_hf_to_gguf.py YuE2-3B --mmproj --outfile mmproj-yue2-3b.gguf
|
||||
```
|
||||
|
||||
The mmproj holds the NAR transformer layers (loaded with libllama as a Qwen3 text model, in the `--outtype` type), and the latent projections and the VAE decoder, always in F32.
|
||||
|
||||
## MiniMax Music 3
|
||||
|
||||
[MiniMaxAI/MiniMax-Music3](https://huggingface.co/MiniMaxAI/MiniMax-Music3) generates songs (vocals and accompaniment) from a music description and lyrics, as 44.1 kHz stereo. For each 40 ms frame, the backbone (a Qwen3 text model) samples the semantic code and the RVQ depth decoder the 7 other codes; the hidden states of both go to a flow-matching DiT, then a DAC decoder makes the audio. Both use the settings of the reference (the diffusers pipeline).
|
||||
|
||||
The prompt has the same format as YuE2, the description goes under `[Tags]`:
|
||||
|
||||
```
|
||||
[Tags]
|
||||
Genre: acoustic pop. BPM: 96. Key: C major. Vocals: soft female lead. Arrangement: fingerpicked guitar and soft piano.
|
||||
[Lyrics]
|
||||
[verse]
|
||||
Morning light filtering through the pine
|
||||
[chorus]
|
||||
Softly the world begins to breathe
|
||||
```
|
||||
|
||||
Available params:
|
||||
- The description and the lyrics are cleaned up like the reference: markdown is removed, a line that starts with structure tags (e.g. `[Verse]`) keeps only the tags
|
||||
- The generation stops at the end-of-audio token or at the limit of the reference (9000 frames, 6 minutes); `-n` sets a lower limit in frames (25 per second)
|
||||
- `--temp` changes the temperature of the codes (default 1.0); `--top-k` and `--top-p` are not used, the codes are sampled with top-k 50 like the reference
|
||||
- The codes are sampled with classifier-free guidance, scale 1.5 like the reference; `llama-server` takes `cfg_scale` to change it (1.0 disables it). The uncond prompt runs as a second sequence that shares the context, so a song needs about 2 x (prompt tokens + frames) positions: the default context (10240) holds about 3 minutes with a short description, use a larger `-c` (e.g. 20480) for longer songs
|
||||
- The flow matching runs 30 Euler steps like the reference; `llama-server` takes `flow_steps` to change it
|
||||
- The audio is made in windows of 200 frames (8 s) every 100 frames, like the reference: a window is processed during the generation once the frame after it exists, so with streaming the audio arrives in steps of about 4 s
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m minimax-music3.gguf \
|
||||
-mm mmproj-minimax-music3.gguf \
|
||||
-f song.txt \
|
||||
--output song.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The conversion reads the diffusers layout of the repository, the `qwen_7B` directory and the `.pth` files are not needed:
|
||||
|
||||
```sh
|
||||
hf download MiniMaxAI/MiniMax-Music3 --local-dir MiniMax-Music3 --exclude "qwen_7B/*" --exclude "*.pth"
|
||||
python convert_hf_to_gguf.py MiniMax-Music3 --outfile minimax-music3.gguf
|
||||
python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-music3.gguf
|
||||
```
|
||||
|
||||
The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32.
|
||||
|
||||
@@ -7,6 +7,8 @@ Jinja has no regex support, and exposing `std::regex` / Python `re` would not he
|
||||
- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments
|
||||
- `tts_norm_sub(text, pattern, repl)`: replace every match
|
||||
|
||||
A third function, `tts_dict(name, key)`, gives the template access to dictionaries stored in the model (e.g. a pronunciation lexicon).
|
||||
|
||||
Implementations:
|
||||
- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
|
||||
- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
|
||||
@@ -69,6 +71,22 @@ Replaces every match of `pattern`, with the same scanning rules as `tts_norm_sca
|
||||
|
||||
Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.
|
||||
|
||||
### `tts_dict(name, key)`
|
||||
|
||||
Looks up `key` in the dictionary `name` of the model and returns its value, or `none` if the key is missing. An unknown dictionary name is an error. A model can ship any number of dictionaries, for example a pronunciation lexicon per language:
|
||||
|
||||
```jinja
|
||||
{%- set e = tts_dict('lexicon_en', word) -%}
|
||||
{%- if e is none -%}{{ spell(word) }}
|
||||
{%- elif e is mapping -%}{{ e[tag] if tag in e else e.DEFAULT }}
|
||||
{%- else -%}{{ e }}
|
||||
{%- endif -%}
|
||||
```
|
||||
|
||||
The lookup is exact: no case folding and no normalization, do them in the template.
|
||||
|
||||
The GGUF lists the names of the dictionaries in `tokenizer.tts_dicts`. Each one is stored as two arrays of strings of the same length: `tokenizer.tts_dict.<name>.keys`, sorted by the UTF-8 bytes of the key, and `tokenizer.tts_dict.<name>.values`. A value is a JSON document (string, number, list, object, ...) and is returned decoded. `GGUFWriter.add_tts_dicts(dicts)` writes all of this.
|
||||
|
||||
## Pattern syntax
|
||||
|
||||
Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
|
||||
|
||||
@@ -164,6 +164,11 @@
|
||||
}
|
||||
// lib.optionalAttrs (system == "x86_64-linux") {
|
||||
rocm = config.legacyPackages.llamaPackagesRocm.llama-cpp;
|
||||
|
||||
sycl = config.packages.default.override {
|
||||
useSycl = true;
|
||||
syclToolkit = pkgs.intel-oneapi-toolkit;
|
||||
};
|
||||
};
|
||||
|
||||
# Packages exposed in `.#checks` will be built by the CI and by
|
||||
|
||||
@@ -639,6 +639,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
float * const __restrict__ KQ_rowsum,
|
||||
const int jt,
|
||||
const int kb0,
|
||||
const int kb0_next,
|
||||
const int k_VKQ_sup) {
|
||||
#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
@@ -1030,10 +1031,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
if (!last_iter) {
|
||||
if (ncols2 > 1 || mask_h) {
|
||||
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
|
||||
(mask_h, tile_mask, stride_mask, kb0_next*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
|
||||
}
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, indices);
|
||||
(K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, kb0_next*nbatch_fa, k_VKQ_sup, indices);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1108,7 +1109,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
scale, slope, logit_softcap, ne01, ne02,
|
||||
stride_K, stride_V, stride_mask,
|
||||
tile_Q, tile_K, tile_V, tile_mask,
|
||||
Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0);
|
||||
Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0, kb0_next);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
}
|
||||
@@ -1285,7 +1286,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
const int jt,
|
||||
const int zt_gqa,
|
||||
const int kb0_start,
|
||||
const int kb0_stop) {
|
||||
const int kb0_stop,
|
||||
const int kb0_max) {
|
||||
#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
//In this kernel Q, K, V are matrices while i, j, k are matrix indices.
|
||||
|
||||
@@ -1438,7 +1440,13 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
int kb0 = kb0_start;
|
||||
// With multi-stage loading the K/V block order is rotated, see flash_attn_ext_f16.
|
||||
// Then K/V blocks >= iter_k wrap around to 0 and the fully masked K/V blocks >= kb0_max are skipped.
|
||||
const int iter_k = (ne11 + nbatch_fa - 1) / nbatch_fa;
|
||||
const int nkb_pre = max(0, min(kb0_stop, kb0_max) - kb0_start);
|
||||
const int nkb_post = max(0, min(kb0_stop - iter_k, kb0_max));
|
||||
const int nkb = max(1, nkb_pre + nkb_post);
|
||||
int kb0 = nstages > 1 && nkb_pre == 0 && nkb_post > 0 ? 0 : kb0_start;
|
||||
|
||||
// Preload mask and K data for first iteration when using cp_async with multiple stages:
|
||||
if constexpr (nstages > 1) {
|
||||
@@ -1456,7 +1464,29 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
}
|
||||
|
||||
// kb0_start is always < kb0_stop so the last iter can be executed unconditionally.
|
||||
if constexpr (ncols2 == 1 || use_sparse) {
|
||||
if constexpr (nstages > 1) {
|
||||
constexpr bool oob_check = false;
|
||||
for (int ikb = 1; ikb < nkb; ++ikb) {
|
||||
const int kb0_next = kb0 + 1 == kb0_max ? 0 : kb0 + 1;
|
||||
constexpr bool last_iter = false;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
flash_attn_ext_f16_iter
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0_next, k_VKQ_sup);
|
||||
kb0 = kb0_next;
|
||||
}
|
||||
constexpr bool last_iter = true;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
flash_attn_ext_f16_iter
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
|
||||
} else if constexpr (ncols2 == 1 || use_sparse) {
|
||||
constexpr bool oob_check = true;
|
||||
for (; kb0 < kb0_stop-1; ++kb0) {
|
||||
constexpr bool last_iter = false;
|
||||
@@ -1466,7 +1496,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0 + 1, k_VKQ_sup);
|
||||
}
|
||||
constexpr bool last_iter = true;
|
||||
const int k_VKQ_sup = ne11 - kb0*nbatch_fa;
|
||||
@@ -1475,7 +1505,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
|
||||
} else {
|
||||
constexpr bool oob_check = false;
|
||||
for (; kb0 < kb0_stop-1; ++kb0) {
|
||||
@@ -1486,7 +1516,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0 + 1, k_VKQ_sup);
|
||||
}
|
||||
constexpr bool last_iter = true;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
@@ -1495,7 +1525,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
|
||||
}
|
||||
|
||||
// With multi-stage loading there is no __syncthreads at the end of the iter,
|
||||
@@ -1893,7 +1923,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dstk_fixup,
|
||||
scale, slope, logit_softcap, ne01, ne02, gqa_ratio,
|
||||
stride_Q1, stride_Q2, stride_K, stride_V, stride_mask,
|
||||
jt, kb0_start, kb0_stop);
|
||||
jt, kb0_start, kb0_stop, kb0_max);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
}
|
||||
@@ -2021,6 +2051,18 @@ static __global__ void flash_attn_ext_f16(
|
||||
int kb0_start = kbc % iter_k;
|
||||
int kb0_stop = min(iter_k, kb0_start + kbc_stop - kbc);
|
||||
|
||||
// Rotate the K/V block order: kbc uses K/V block (kbc - kbc_tile) % iter_k instead of kbc % iter_k, kbc_tile is the start of the CUDA block that started the tile.
|
||||
// Then concurrent CUDA blocks use the same K/V blocks and can share them in L2. Only with multi-stage loading, else they all wait for the same K/V data.
|
||||
constexpr bool rotate = ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, use_sparse) > 1;
|
||||
const int kbc0 = kbc;
|
||||
int kbc_tile = kbc;
|
||||
if (rotate && kb0_start != 0) {
|
||||
// Get the CUDA block that started the tile, inverse of the kbc formula above:
|
||||
const int64_t nkbc = int64_t(iter_k)*iter_j*iter_z_gqa*ne12*ne03;
|
||||
const int64_t b_first = ((kbc - kb0_start + 1)*int64_t(gridDim.x) - 1) / nkbc;
|
||||
kbc_tile = b_first*nkbc / gridDim.x;
|
||||
}
|
||||
|
||||
while (kbc < kbc_stop && kb0_stop == iter_k) {
|
||||
// z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index
|
||||
const int sequence = kbc /(iter_k*iter_j*iter_z_gqa*ne12);
|
||||
@@ -2043,26 +2085,30 @@ static __global__ void flash_attn_ext_f16(
|
||||
|
||||
const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
|
||||
|
||||
// With rotation kb0_stop is not cut, process_tile skips the fully masked K/V blocks >= kb0_max.
|
||||
if (use_sparse) {
|
||||
kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa);
|
||||
} else if (KV_max) {
|
||||
} else if (KV_max && !rotate) {
|
||||
kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);
|
||||
}
|
||||
const int kb0_max = rotate && KV_max ? KV_max[sequence*iter_j + jt] / nbatch_fa : iter_k;
|
||||
const int kb0_rot = rotate ? (kbc - kbc_tile) % iter_k : kb0_start;
|
||||
constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
|
||||
if (kb0_start == 0) {
|
||||
constexpr bool needs_fixup = false; // CUDA block is working on an entire tile.
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
|
||||
} else {
|
||||
constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile.
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
|
||||
}
|
||||
|
||||
kbc += iter_k;
|
||||
kbc -= kbc % iter_k;
|
||||
kbc_tile = kbc0;
|
||||
|
||||
kb0_start = 0;
|
||||
kb0_stop = min(iter_k, kbc_stop - kbc);
|
||||
@@ -2095,15 +2141,17 @@ static __global__ void flash_attn_ext_f16(
|
||||
|
||||
if (use_sparse) {
|
||||
kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa);
|
||||
} else if (KV_max) {
|
||||
} else if (KV_max && !rotate) {
|
||||
kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);
|
||||
}
|
||||
const int kb0_max = rotate && KV_max ? KV_max[sequence*iter_j + jt] / nbatch_fa : iter_k;
|
||||
const int kb0_rot = rotate ? (kbc - kbc_tile) % iter_k : kb0_start;
|
||||
|
||||
constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
|
||||
constexpr bool needs_fixup = false;
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
|
||||
#else
|
||||
GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
|
||||
max_bias, m0, m1, n_head_log2, logit_softcap,
|
||||
|
||||
@@ -1481,10 +1481,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
|
||||
|
||||
const dim3 block_dims(warp_size, nwarps, 1);
|
||||
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, /*fallback =*/ false, prec_src1, has_fusion>), nbytes_shared);
|
||||
if constexpr (!has_fusion) {
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, /*fallback =*/ true, prec_src1, false>), nbytes_shared);
|
||||
}
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, fallback, prec_src1, has_fusion>), nbytes_shared);
|
||||
|
||||
const int I_dst = has_fusion ? config.I/2 : config.I; // Rows of dst per tile, see mul_mat_q.
|
||||
const int nti = (args.nrows_x + I_dst - 1) / I_dst;
|
||||
|
||||
@@ -1330,7 +1330,9 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device
|
||||
static const bool disable_flip = getenv("GGML_VK_DISABLE_FA_FLIP") != nullptr;
|
||||
static const char * force_flip = getenv("GGML_VK_FORCE_FA_FLIP");
|
||||
const bool flip_arch = force_flip != nullptr || device->vendor_id == VK_VENDOR_ID_NVIDIA ||
|
||||
(device->vendor_id == VK_VENDOR_ID_AMD && (device->architecture == AMD_RDNA3 || device->architecture == AMD_RDNA4));
|
||||
(device->vendor_id == VK_VENDOR_ID_AMD && (device->architecture == AMD_RDNA3 || device->architecture == AMD_RDNA4) &&
|
||||
// AMD proprietary driver crashes compiling the flip shader
|
||||
device->driver_id != vk::DriverId::eAmdProprietary);
|
||||
if (allow_flip && !disable_flip && flip_arch && n_rows >= 64 &&
|
||||
(result.subgroup_size == 32 || result.subgroup_size == 64) &&
|
||||
hsk % 16 == 0 && hsv % 16 == 0 && hsk <= 256 && hsv <= 256 &&
|
||||
|
||||
@@ -415,6 +415,9 @@ class Keys:
|
||||
CHAT_TEMPLATE = "tokenizer.chat_template"
|
||||
CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"
|
||||
CHAT_TEMPLATES = "tokenizer.chat_templates"
|
||||
TTS_DICTS = "tokenizer.tts_dicts"
|
||||
TTS_DICT_KEYS_N = "tokenizer.tts_dict.{name}.keys"
|
||||
TTS_DICT_VALUES_N = "tokenizer.tts_dict.{name}.values"
|
||||
# Normalizer constants
|
||||
NORMALIZER_NFC = "tokenizer.ggml.normalizer.nfc"
|
||||
NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
|
||||
@@ -7288,6 +7291,7 @@ class VisionProjectorType:
|
||||
ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder
|
||||
ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder
|
||||
YUE2_GEN = "yue2_gen" # audio generation: flow-matching NAR + Oobleck VAE decoder
|
||||
MINIMAX_MUSIC3_GEN = "minimax_music3_gen" # audio generation: RVQ depth decoder + flow-matching DiT + DAC decoder
|
||||
FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
|
||||
FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
|
||||
HUNYUANVL = "hunyuanvl"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
@@ -1458,6 +1459,15 @@ class GGUFWriter:
|
||||
|
||||
self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value)
|
||||
|
||||
def add_tts_dicts(self, dicts: Mapping[str, Mapping[str, Any]]) -> None:
|
||||
# dictionaries for tts_dict(): keys sorted by their bytes, values as JSON
|
||||
self.add_array(Keys.Tokenizer.TTS_DICTS, list(dicts))
|
||||
for name, value in dicts.items():
|
||||
keys = sorted(value, key=lambda k: k.encode("utf-8"))
|
||||
self.add_array(Keys.Tokenizer.TTS_DICT_KEYS_N.format(name=name), keys)
|
||||
self.add_array(Keys.Tokenizer.TTS_DICT_VALUES_N.format(name=name),
|
||||
[json.dumps(value[k], ensure_ascii=False, separators=(",", ":")) for k in keys])
|
||||
|
||||
def add_suppress_tokens(self, tokens: Sequence[int]) -> None:
|
||||
self.add_array(Keys.Tokenizer.SUPPRESS_TOKENS, tokens)
|
||||
|
||||
|
||||
@@ -632,6 +632,17 @@ extern "C" {
|
||||
// Get metadata value as a string by index
|
||||
LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
|
||||
|
||||
// GGUF metadata arrays of strings
|
||||
// currently, only these keys are support:
|
||||
// - tokenizer.tts_dicts
|
||||
// - tokenizer.tts_dict.*
|
||||
|
||||
// Get the number of elements, or -1 if there is no such array
|
||||
LLAMA_API int32_t llama_model_meta_arr_n(const struct llama_model * model, const char * key);
|
||||
|
||||
// Get element i, owned by the model; nullptr if not found
|
||||
LLAMA_API const char * llama_model_meta_arr_str(const struct llama_model * model, const char * key, int32_t i);
|
||||
|
||||
// Get a string describing the model type
|
||||
LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
{#- MiniMax Music 3 prompt, same text format as YuE2: "[Tags]\n<caption>\n[Lyrics]\n<lyrics>" -#}
|
||||
{#- the caption and lyrics cleanup follows _clean_caption() and _normalize_lyrics() of the diffusers pipeline (encoders.py) -#}
|
||||
{%- set text = messages[0]['content'] -%}
|
||||
{%- set parts = text.split('\n[Lyrics]\n') -%}
|
||||
{%- if parts | length < 2 -%}
|
||||
{{- raise_exception('MiniMax Music 3 expects "[Tags]\\n<caption>\\n[Lyrics]\\n<lyrics>"') -}}
|
||||
{%- endif -%}
|
||||
{%- set caption = tts_norm_sub(parts[0], '^%[Tags%]\n', '') -%}
|
||||
{%- set lyrics = parts[1:] | join('\n[Lyrics]\n') -%}
|
||||
|
||||
{#- caption: <|key value|> -> "key is value", then strip the markdown -#}
|
||||
{%- set ns = namespace(text='', lines=[]) -%}
|
||||
{%- for s in tts_norm_scan(caption, '<%|([^|]*)%|>') -%}
|
||||
{%- if s.rule == 'match' -%}
|
||||
{%- set ns.text = ns.text ~ tts_norm_sub(s.groups[0].strip(), '^(%S+)%s+(.+)', '%1 is %2') -%}
|
||||
{%- else -%}
|
||||
{%- set ns.text = ns.text ~ s.text -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- for raw in ns.text.split('\n') -%}
|
||||
{%- set line = tts_norm_sub(raw, '^%s{0,3}#{1,6}%s+', '') -%}
|
||||
{%- set line = tts_norm_sub(line, '^%s*[*+%-]%s+', '') -%}
|
||||
{%- set line = tts_norm_sub(line, '^%s*%*%s+', '') -%}
|
||||
{%- set ln = namespace(v=line) -%}
|
||||
{%- for _ in range(4) -%}
|
||||
{%- set ln.v = tts_norm_sub(ln.v, '%*%*([^*]+)%*%*', '%1') -%}
|
||||
{%- endfor -%}
|
||||
{%- set line = tts_norm_sub(ln.v, '(?<!%*)%*([^*\n]+)%*(?!%*)', '%1').rstrip() -%}
|
||||
{%- set line = tts_norm_sub(line, '^%s*[%-*_]{3,}%s*$', '') -%}
|
||||
{%- set ns.lines = ns.lines + [line] -%}
|
||||
{%- endfor -%}
|
||||
{%- set caption = ns.lines | join('\n') -%}
|
||||
{%- set caption = caption.replace('• ', '').replace(' ', '') -%}
|
||||
{%- set caption = tts_norm_sub(caption, '\n{2,}', '\n') -%}
|
||||
|
||||
{#- lyrics: a line that starts with structure tags keeps only the tags, the tags are lowercase -#}
|
||||
{%- set ns.lines = [] -%}
|
||||
{%- for line in lyrics.split('\n') -%}
|
||||
{%- set seg = tts_norm_scan(line, '^[ \t]*((?:%[[^%]]+%][ \t]*)+)') -%}
|
||||
{%- if seg | length > 0 and seg[0].rule == 'match' -%}
|
||||
{%- set ns.lines = ns.lines + [seg[0].groups[0].strip()] -%}
|
||||
{%- else -%}
|
||||
{%- set ns.lines = ns.lines + [line] -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- set lyrics = (ns.lines | join('\n')).replace('] ', ']\n').replace(' [', '\n[').replace(' ^ ', '\n') -%}
|
||||
{%- set ns.text = '' -%}
|
||||
{%- for s in tts_norm_scan(lyrics, '%[([^%]]+)%]') -%}
|
||||
{%- set ns.text = ns.text ~ ('[' ~ (s.groups[0] | lower) ~ ']' if s.rule == 'match' else s.text) -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{{- '<|im_start|><|caption_start|>' ~ caption ~ '<|caption_end|><|lyrics_start|>[start]\n' ~ ns.text ~ '<|lyrics_end|><|im_end|><|audio_start|>' -}}
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reference implementation of TTS normalization patterns, the tts_norm_scan() and tts_norm_sub() template functions.
|
||||
"""Reference implementation of the template functions tts_norm_scan(), tts_norm_sub() (TTS normalization patterns) and tts_dict().
|
||||
|
||||
The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation
|
||||
(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py).
|
||||
@@ -8,12 +8,14 @@ Usage with jinja2:
|
||||
|
||||
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
|
||||
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
|
||||
env.globals["tts_dict"] = chat_tts_norm.make_tts_dict(dicts) # dicts: name -> {"keys", "values"} of tokenizer.tts_dict.<name>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from typing import Any, Callable
|
||||
|
||||
# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp)
|
||||
|
||||
@@ -554,3 +556,23 @@ def tts_norm_sub(text: str, pattern: str, repl: str) -> str:
|
||||
if gs >= 0:
|
||||
out.append(text[gs:ge])
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def make_tts_dict(dicts: dict[str, dict[str, list[str]]]) -> Callable[[str, str], Any]:
|
||||
# a dictionary is {"keys": [...], "values": [...]}, the values are JSON documents
|
||||
parsed: dict[str, dict[str, str]] = {}
|
||||
|
||||
def tts_dict(name: str, key: str) -> Any:
|
||||
if not isinstance(name, str) or not isinstance(key, str):
|
||||
raise TypeError("tts_dict: name and key must be strings")
|
||||
if name not in dicts:
|
||||
raise KeyError(f"tts_dict: unknown dictionary '{name}'")
|
||||
if name not in parsed:
|
||||
keys, values = dicts[name]["keys"], dicts[name]["values"]
|
||||
if len(keys) != len(values):
|
||||
raise ValueError("tts_dict: malformed dictionary, a key has no value")
|
||||
parsed[name] = dict(zip(keys, values))
|
||||
doc = parsed[name].get(key)
|
||||
return None if doc is None else json.loads(doc)
|
||||
|
||||
return tts_dict
|
||||
|
||||
+33
-7
@@ -19,7 +19,11 @@ set(LLAMA_UI_GZIP "" CACHE STRING "Apply gzip compress to assets to save ban
|
||||
|
||||
set(DIST_DIR "${UI_BINARY_DIR}/dist")
|
||||
set(SRC_DIST_DIR "${UI_SOURCE_DIR}/dist")
|
||||
set(WORK_DIR "${UI_BINARY_DIR}/ui-src")
|
||||
# The stage keeps the layout of ui/: the app in apps/chat, with the sources of
|
||||
# Llama UI that it compiles next to it in lib/ui (see ui/apps/chat/scripts/llama-ui.js).
|
||||
set(STAGE_DIR "${UI_BINARY_DIR}/ui-src")
|
||||
set(WORK_DIR "${STAGE_DIR}/apps/chat")
|
||||
set(UI_LIB_DIR "${UI_SOURCE_DIR}/../../lib/ui")
|
||||
set(STAMP_FILE "${UI_BINARY_DIR}/.ui-stamp")
|
||||
set(EMBED_STAMP "${UI_BINARY_DIR}/.ui-embed.sha256")
|
||||
set(UI_CPP "${UI_BINARY_DIR}/ui.cpp")
|
||||
@@ -72,11 +76,13 @@ function(ui_validate_assets files in_dir)
|
||||
set(found_index FALSE)
|
||||
set(found_manifest FALSE)
|
||||
set(found_sw FALSE)
|
||||
# the service worker: SvelteKit's service-worker.js, or Workbox's workbox-*.js in the
|
||||
# prebuilt archives of builds before SvelteKit 3
|
||||
set(found_service_worker FALSE)
|
||||
set(found_build_json FALSE)
|
||||
set(found_version_json FALSE)
|
||||
set(found_bundle_js FALSE)
|
||||
set(found_bundle_css FALSE)
|
||||
set(found_workbox_js FALSE)
|
||||
|
||||
foreach(f ${files})
|
||||
get_filename_component(base "${f}" NAME)
|
||||
@@ -86,6 +92,8 @@ function(ui_validate_assets files in_dir)
|
||||
set(found_manifest TRUE)
|
||||
elseif(base STREQUAL "sw.js")
|
||||
set(found_sw TRUE)
|
||||
elseif(base STREQUAL "service-worker.js" OR base MATCHES "^workbox.*\\.js$")
|
||||
set(found_service_worker TRUE)
|
||||
elseif(base STREQUAL "build.json")
|
||||
set(found_build_json TRUE)
|
||||
elseif(base STREQUAL "version.json")
|
||||
@@ -94,8 +102,6 @@ function(ui_validate_assets files in_dir)
|
||||
set(found_bundle_js TRUE)
|
||||
elseif(base MATCHES "^bundle.*\\.css$")
|
||||
set(found_bundle_css TRUE)
|
||||
elseif(base MATCHES "^workbox.*\\.js$")
|
||||
set(found_workbox_js TRUE)
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
@@ -109,6 +115,9 @@ function(ui_validate_assets files in_dir)
|
||||
if(NOT found_sw)
|
||||
list(APPEND missing "sw.js")
|
||||
endif()
|
||||
if(NOT found_service_worker)
|
||||
list(APPEND missing "service-worker.js (or workbox[hash].js in an older build)")
|
||||
endif()
|
||||
if(NOT found_build_json)
|
||||
list(APPEND missing "build.json")
|
||||
endif()
|
||||
@@ -121,9 +130,6 @@ function(ui_validate_assets files in_dir)
|
||||
if(NOT found_bundle_css)
|
||||
list(APPEND missing "bundle[hash].css")
|
||||
endif()
|
||||
if(NOT found_workbox_js)
|
||||
list(APPEND missing "workbox[hash].js")
|
||||
endif()
|
||||
|
||||
if(missing)
|
||||
set(listing "")
|
||||
@@ -314,6 +320,16 @@ function(npm_build_should_skip out_var)
|
||||
endfunction()
|
||||
|
||||
function(stage_sources)
|
||||
# everything but the app goes: the library, restaged below, and what a stage
|
||||
# from before this layout left at the top, its node_modules included
|
||||
if(EXISTS "${STAGE_DIR}")
|
||||
file(GLOB staged RELATIVE "${STAGE_DIR}" "${STAGE_DIR}/*")
|
||||
list(REMOVE_ITEM staged "apps")
|
||||
foreach(entry ${staged})
|
||||
file(REMOVE_RECURSE "${STAGE_DIR}/${entry}")
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
if(EXISTS "${WORK_DIR}")
|
||||
file(GLOB staged RELATIVE "${WORK_DIR}" "${WORK_DIR}/*")
|
||||
list(REMOVE_ITEM staged "node_modules")
|
||||
@@ -327,6 +343,16 @@ function(stage_sources)
|
||||
NO_SOURCE_PERMISSIONS
|
||||
PATTERN "node_modules" EXCLUDE
|
||||
)
|
||||
|
||||
# what the app reads of the library: its sources and the dependency list
|
||||
file(COPY "${UI_LIB_DIR}/package.json"
|
||||
DESTINATION "${STAGE_DIR}/lib/ui"
|
||||
NO_SOURCE_PERMISSIONS
|
||||
)
|
||||
file(COPY "${UI_LIB_DIR}/src/lib"
|
||||
DESTINATION "${STAGE_DIR}/lib/ui/src"
|
||||
NO_SOURCE_PERMISSIONS
|
||||
)
|
||||
endfunction()
|
||||
|
||||
function(npm_build out_var)
|
||||
|
||||
@@ -1471,6 +1471,20 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
|
||||
for (int i = 0; i < gguf_get_n_kv(ctx); i++) {
|
||||
gguf_type type = gguf_get_kv_type(ctx, i);
|
||||
if (type == GGUF_TYPE_ARRAY) {
|
||||
// only the string arrays listed here are kept, see llama_model_meta_arr_str()
|
||||
static const char * const kept_prefixes[] = { "tokenizer.tts_dict" };
|
||||
const std::string arr_name = gguf_get_key(ctx, i);
|
||||
const bool is_kept = std::any_of(std::begin(kept_prefixes), std::end(kept_prefixes), [&](const char * prefix) {
|
||||
return arr_name.rfind(prefix, 0) == 0;
|
||||
});
|
||||
if (is_kept && gguf_get_arr_type(ctx, i) == GGUF_TYPE_STRING) {
|
||||
auto & arr = gguf_kv_arr[arr_name];
|
||||
const size_t n = gguf_get_arr_n(ctx, i);
|
||||
arr.reserve(n);
|
||||
for (size_t j = 0; j < n; j++) {
|
||||
arr.emplace_back(gguf_get_arr_str(ctx, i, j));
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const char * name = gguf_get_key(ctx, i);
|
||||
@@ -3944,6 +3958,22 @@ int32_t llama_model_meta_val_str_by_index(const llama_model * model, int32_t i,
|
||||
return snprintf(buf, buf_size, "%s", it->second.c_str());
|
||||
}
|
||||
|
||||
int32_t llama_model_meta_arr_n(const llama_model * model, const char * key) {
|
||||
const auto it = model->gguf_kv_arr.find(key);
|
||||
if (it == model->gguf_kv_arr.end() || it->second.size() > (size_t) INT32_MAX) {
|
||||
return -1;
|
||||
}
|
||||
return (int32_t) it->second.size();
|
||||
}
|
||||
|
||||
const char * llama_model_meta_arr_str(const llama_model * model, const char * key, int32_t i) {
|
||||
const auto it = model->gguf_kv_arr.find(key);
|
||||
if (it == model->gguf_kv_arr.end() || i < 0 || (size_t) i >= it->second.size()) {
|
||||
return nullptr;
|
||||
}
|
||||
return it->second[i].c_str();
|
||||
}
|
||||
|
||||
int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size) {
|
||||
return snprintf(buf, buf_size, "%s", model->desc().c_str());
|
||||
}
|
||||
|
||||
@@ -956,6 +956,9 @@ struct llama_model {
|
||||
// gguf metadata
|
||||
std::unordered_map<std::string, std::string> gguf_kv;
|
||||
|
||||
// the gguf metadata string arrays kept after load, see load_hparams()
|
||||
std::unordered_map<std::string, std::vector<std::string>> gguf_kv_arr;
|
||||
|
||||
// list of devices used in this model
|
||||
std::vector<llama_device> devices;
|
||||
|
||||
|
||||
@@ -540,7 +540,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn_linear(
|
||||
cb(final_output, "final_output", il);
|
||||
|
||||
// Output projection
|
||||
cur = build_lora_mm(model.layers[il].ssm_out, final_output);
|
||||
cur = build_lora_mm(model.layers[il].ssm_out, final_output, model.layers[il].ssm_out_s);
|
||||
cb(cur, "linear_attn_out", il);
|
||||
|
||||
return cur;
|
||||
|
||||
+165
-2
@@ -45,12 +45,30 @@ static void test_caps(testing & t);
|
||||
static void test_string_parts(testing & t);
|
||||
static void test_fuzzing(testing & t);
|
||||
static void test_tts_norm_patterns(testing & t);
|
||||
static void test_tts_dict(testing & t);
|
||||
static void test_tts_template_zonos3(testing & t);
|
||||
static void test_tts_template_music3(testing & t);
|
||||
|
||||
static void test_shared_render(testing & t);
|
||||
|
||||
static bool g_python_mode = false;
|
||||
|
||||
// dictionaries given to tts_dict() in tests
|
||||
static std::shared_ptr<const common_tts_dicts> test_tts_dicts() {
|
||||
static const auto dicts = std::make_shared<const common_tts_dicts>(common_tts_dicts{
|
||||
{"lex", {
|
||||
{"", "Zed", "big", "hello", "list", "null", "read", "w1", "w2", "zoo", "\xc3\xa9t\xc3\xa9", "\xe6\x97\xa5"},
|
||||
{"\"empty\"", "\"z3d\"", "123456789012345678901234567890", "\"h@loU\"", "[1,2,null]", "null",
|
||||
"{\"DEFAULT\":\"rid\",\"VBD\":\"rEd\"}", "{\"a\":1.5,\"b\":-2}", "{\"a\":0.25}", "\"zu\"", "\"ete\"", "\"ni\""},
|
||||
}},
|
||||
{"other", {{"hello"}, {"\"bonjour\""}}},
|
||||
{"empty", {}},
|
||||
// values that must be rejected: nested deeper than the engine allows, and not JSON
|
||||
{"bad", {{"deep", "json"}, {std::string(200, '[') + std::string(200, ']'), "{not json"}}},
|
||||
});
|
||||
return dicts;
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
testing t(std::cout);
|
||||
t.verbose = true;
|
||||
@@ -85,7 +103,9 @@ int main(int argc, char *argv[]) {
|
||||
t.test("array methods", test_array_methods);
|
||||
t.test("object methods", test_object_methods);
|
||||
t.test("tts norm patterns", test_tts_norm_patterns);
|
||||
t.test("tts dict", test_tts_dict);
|
||||
t.test("tts template zonos3", test_tts_template_zonos3);
|
||||
t.test("tts template music3", test_tts_template_music3);
|
||||
if (!g_python_mode) {
|
||||
t.test("shared render", test_shared_render);
|
||||
t.test("hasher", test_hasher);
|
||||
@@ -3041,6 +3061,113 @@ static void test_tts_norm_patterns(testing & t) {
|
||||
}
|
||||
}
|
||||
|
||||
// tts_dict(name, key): dictionaries of the model, see docs/tts-norm-patterns.md
|
||||
// only available to task templates, like tts_norm_scan(); the dictionaries of test_tts_dicts() stand for the ones of a model
|
||||
static void test_tts_dict(testing & t) {
|
||||
test_template(t, "tts_dict is defined for task templates",
|
||||
"{{ tts_dict is defined }}",
|
||||
json::object(), "True", true);
|
||||
|
||||
test_template(t, "tts_dict returns the decoded JSON value",
|
||||
"{{ tts_dict('lex', 'hello') }}|{{ tts_dict('lex', 'read').VBD }}|{{ tts_dict('lex', 'read') is mapping }}|{{ tts_dict('lex', 'list')[1] + 1 }}",
|
||||
json::object(), "h@loU|rEd|True|3", true);
|
||||
|
||||
test_template(t, "tts_dict returns none for a missing key",
|
||||
"{{ tts_dict('lex', 'nope') is none }}|{{ tts_dict('lex', 'hell') is none }}|{{ tts_dict('lex', 'hello!') is none }}|{{ tts_dict('lex', 'Hello') is none }}",
|
||||
json::object(), "True|True|True|True", true);
|
||||
|
||||
test_template(t, "tts_dict first, last, empty and non-ASCII keys",
|
||||
"{{ tts_dict('lex', '') }}|{{ tts_dict('lex', 'Zed') }}|{{ tts_dict('lex', 'zoo') }}|{{ tts_dict('lex', '\xc3\xa9t\xc3\xa9') }}|{{ tts_dict('lex', '\xe6\x97\xa5') }}",
|
||||
json::object(), "empty|z3d|zu|ete|ni", true);
|
||||
|
||||
test_template(t, "tts_dict null value is none, like a missing key",
|
||||
"{{ tts_dict('lex', 'null') is none }}",
|
||||
json::object(), "True", true);
|
||||
|
||||
test_template(t, "tts_dict number outside the int64 range",
|
||||
"{{ tts_dict('lex', 'big') > 9000000000000000000 }}|{{ tts_dict('lex', 'big') is number }}",
|
||||
json::object(), "True|True", true);
|
||||
|
||||
test_template(t, "tts_dict selects the dictionary by name",
|
||||
"{{ tts_dict('other', 'hello') }}|{{ tts_dict('other', 'zoo') is none }}|{{ tts_dict('empty', 'hello') is none }}",
|
||||
json::object(), "bonjour|True|True", true);
|
||||
|
||||
test_template(t, "tts_dict perceptron-style score",
|
||||
"{%- set ws = [tts_dict('lex', 'w1'), tts_dict('lex', 'w2')] -%}"
|
||||
"{{ ws | map(attribute='a', default=0.0) | sum }}|{{ ws | map(attribute='b', default=0.0) | sum }}",
|
||||
json::object(), "1.75|-2.0", true);
|
||||
|
||||
if (!g_python_mode) {
|
||||
test_template_error(t, "error: unknown dictionary", "{{ tts_dict('nope', 'a') }}", "unknown dictionary 'nope'", true);
|
||||
test_template_error(t, "error: key must be a string", "{{ tts_dict('lex', 1) }}", "expected value of type", true);
|
||||
test_template_error(t, "error: missing argument", "{{ tts_dict('lex') }}", "expected between 2 and 2 arguments", true);
|
||||
test_template_error(t, "error: value is not JSON", "{{ tts_dict('bad', 'json') }}", "parse error", true);
|
||||
test_template_error(t, "error: value nested too deep", "{{ tts_dict('bad', 'deep') }}", "levels deep", true);
|
||||
|
||||
t.test("a key without a value is an error", [](testing & t) {
|
||||
std::string value;
|
||||
bool thrown = false;
|
||||
try {
|
||||
common_tts_dict_find(common_tts_dict{{"a", "m"}, {"1"}}, "m", value);
|
||||
} catch (const std::exception &) {
|
||||
thrown = true;
|
||||
}
|
||||
t.assert_true("throws", thrown);
|
||||
t.assert_true("the first key still has its value", common_tts_dict_find(common_tts_dict{{"a", "m"}, {"1"}}, "a", value) && value == "1");
|
||||
});
|
||||
|
||||
t.test("every key of a large dictionary is found", [](testing & t) {
|
||||
common_tts_dict dict;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
char key[16];
|
||||
snprintf(key, sizeof(key), "k%04d", i);
|
||||
dict.keys.push_back(key);
|
||||
dict.values.push_back(std::to_string(i));
|
||||
}
|
||||
bool ok = true;
|
||||
std::string value;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
ok = ok && common_tts_dict_find(dict, dict.keys[i], value) && value == std::to_string(i);
|
||||
}
|
||||
ok = ok && !common_tts_dict_find(dict, "k", value) && !common_tts_dict_find(dict, "k10000", value) && !common_tts_dict_find(dict, "a", value);
|
||||
t.assert_true("all found, none extra", ok);
|
||||
});
|
||||
|
||||
// the GGUF is not trusted: with keys in any order, a hit must be an entry of the dictionary
|
||||
t.test("random keys in any order", [](testing & t) {
|
||||
static const char raw[] = "ab\t\n\xff\"{";
|
||||
const std::string alphabet(raw, sizeof(raw) - 1);
|
||||
std::mt19937 rng(42);
|
||||
auto random_str = [&](size_t max_len) {
|
||||
std::string res(rng() % (max_len + 1), '\0');
|
||||
for (auto & c : res) {
|
||||
c = alphabet[rng() % alphabet.size()];
|
||||
}
|
||||
return res;
|
||||
};
|
||||
bool ok = true;
|
||||
for (int i = 0; i < 20000; i++) {
|
||||
common_tts_dict dict;
|
||||
const size_t n = rng() % 8;
|
||||
for (size_t j = 0; j < n; j++) {
|
||||
dict.keys.push_back(random_str(2));
|
||||
dict.values.push_back(std::to_string(j));
|
||||
}
|
||||
const std::string key = random_str(2);
|
||||
std::string value;
|
||||
if (common_tts_dict_find(dict, key, value)) {
|
||||
bool found = false;
|
||||
for (size_t j = 0; j < n; j++) {
|
||||
found = found || (dict.keys[j] == key && dict.values[j] == value);
|
||||
}
|
||||
ok = ok && found;
|
||||
}
|
||||
}
|
||||
t.assert_true("every hit is an entry of the dictionary", ok);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// the ZONOS3 "tts" template spells out numbers in en, fr, de, es, it, pt, ja and zh, other languages pass through
|
||||
static void test_tts_template_zonos3(testing & t) {
|
||||
std::string path = __FILE__;
|
||||
@@ -3107,6 +3234,37 @@ static void test_tts_template_zonos3(testing & t) {
|
||||
}
|
||||
}
|
||||
|
||||
// the MiniMax Music 3 "tts" template: caption and lyrics cleanup of the diffusers pipeline (encoders.py), then the special-token prompt
|
||||
static void test_tts_template_music3(testing & t) {
|
||||
std::string path = __FILE__;
|
||||
const size_t pos = path.find_last_of("/\\");
|
||||
path = (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../models/templates/MiniMaxAI-MiniMax-Music3-tts.jinja";
|
||||
std::ifstream fs(path, std::ios_base::binary);
|
||||
if (!t.assert_true("template file is readable: " + path, fs.is_open())) {
|
||||
return;
|
||||
}
|
||||
const std::string tmpl((std::istreambuf_iterator<char>(fs)), std::istreambuf_iterator<char>());
|
||||
|
||||
const std::vector<std::array<std::string, 2>> cases = {
|
||||
{ "[Tags]\nGenre: acoustic pop. BPM: 96.\nVocals: soft female lead.\n[Lyrics]\n[verse]\nMorning light\n[chorus]\nSoftly",
|
||||
"<|im_start|><|caption_start|>Genre: acoustic pop. BPM: 96.\nVocals: soft female lead.<|caption_end|><|lyrics_start|>[start]\n[verse]\nMorning light\n[chorus]\nSoftly<|lyrics_end|><|im_end|><|audio_start|>" },
|
||||
// markdown, <|key value|> tags, inline and lowercased section tags
|
||||
{ "[Tags]\n# Global Metadata\n- **bpm** is 92. *key* is E\n ### h3 title\n---\n\n\n\xe2\x80\xa2 dot item\n indented four\n<|genre Blues Rock|> and <|solo|>\n**a **b** c**\n"
|
||||
"[Lyrics]\n[Verse] I'm learning how\n [Pre-Chorus] [Bass Quartet] text dropped\nline one [Chorus] line two ^ next",
|
||||
"<|im_start|><|caption_start|>Global Metadata\nbpm is 92. key is E\nh3 title\ndot item\nindented four\ngenre is Blues Rock and solo\na b c<|caption_end|>"
|
||||
"<|lyrics_start|>[start]\n[verse]\n[pre-chorus]\n[bass quartet]\nline one\n[chorus]\nline two\nnext<|lyrics_end|><|im_end|><|audio_start|>" },
|
||||
};
|
||||
for (const auto & c : cases) {
|
||||
// same variables as common_chat_get_tts_prompt()
|
||||
const json vars = {
|
||||
{"messages", json::array({{{"role", "user"}, {"content", c[0]}}})},
|
||||
{"language", ""},
|
||||
{"voice", ""},
|
||||
};
|
||||
test_template(t, c[0], tmpl, vars, c[1], true);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) {
|
||||
t.test(name, [&tmpl, &expect, tts_norm](testing & t) {
|
||||
std::string error;
|
||||
@@ -3115,7 +3273,7 @@ static void test_template_error(testing & t, const std::string & name, const std
|
||||
jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl));
|
||||
jinja::context ctx(tmpl);
|
||||
if (tts_norm) {
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
common_chat_tts_norm_add_builtins(ctx, test_tts_dicts());
|
||||
}
|
||||
jinja::runtime runtime(ctx);
|
||||
runtime.gather_string_parts(runtime.execute(ast));
|
||||
@@ -3135,7 +3293,7 @@ static void test_template_cpp(testing & t, const std::string & name, const std::
|
||||
|
||||
jinja::context ctx(tmpl);
|
||||
if (tts_norm) {
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
common_chat_tts_norm_add_builtins(ctx, test_tts_dicts());
|
||||
}
|
||||
jinja::global_from_json(ctx, vars, true);
|
||||
|
||||
@@ -3196,6 +3354,7 @@ if merged_input.get("tts_norm_ref"):
|
||||
spec.loader.exec_module(chat_tts_norm)
|
||||
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
|
||||
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
|
||||
env.globals["tts_dict"] = chat_tts_norm.make_tts_dict(merged_input["tts_dicts"])
|
||||
|
||||
template = env.from_string(tmpl)
|
||||
result = template.render(**vars_json)
|
||||
@@ -3217,6 +3376,10 @@ static void test_template_py(testing & t, const std::string & name, const std::s
|
||||
merged["vars"] = vars;
|
||||
if (tts_norm) {
|
||||
merged["tts_norm_ref"] = tts_norm_ref_path();
|
||||
merged["tts_dicts"] = json::object();
|
||||
for (const auto & d : *test_tts_dicts()) {
|
||||
merged["tts_dicts"][d.first] = {{"keys", d.second.keys}, {"values", d.second.values}};
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
@@ -88,6 +88,8 @@ add_library(mtmd
|
||||
models/fish-s2-spkenc.cpp
|
||||
models/zonos3-gen.cpp
|
||||
models/yue2-gen.cpp
|
||||
models/snake-vae.cpp
|
||||
models/music3-gen.cpp
|
||||
models/zonos3-spkenc.cpp
|
||||
models/pockettts-seanet.cpp
|
||||
models/pockettts-spkenc.cpp
|
||||
|
||||
@@ -116,7 +116,7 @@ Some stages are a full LLM-sized transformer that attends to the keys/values of
|
||||
- In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams`
|
||||
- If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary
|
||||
- Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them
|
||||
- If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()` (see `docs/tts-norm-patterns.md`) and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`
|
||||
- If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()`, and `tts_dict()` for dictionaries stored in the GGUF such as a pronunciation lexicon (see `docs/tts-norm-patterns.md`), and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`
|
||||
4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
|
||||
- 10-20% changes is to add new backbone (text) model and conversion
|
||||
- 60% changes inside `mtmd-helper-gen.cpp`
|
||||
|
||||
@@ -637,15 +637,26 @@ struct mtmd_graph_zonos3_gen : mtmd_graph {
|
||||
|
||||
std::vector<c2w_state_slot> list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
|
||||
|
||||
// graphs that decode with a mtmd_snake_vae (yue2, minimax_music3), see snake-vae.cpp
|
||||
struct mtmd_graph_snake_vae : mtmd_graph {
|
||||
using mtmd_graph::mtmd_graph;
|
||||
|
||||
// x: [T, C]
|
||||
ggml_tensor * snake(ggml_tensor * x, const mtmd_snake_vae::snake & s) const;
|
||||
ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
|
||||
// x: [T, n_latent] -> [T * ratio, n_out]
|
||||
ggml_tensor * build_snake_vae(ggml_tensor * x, const mtmd_snake_vae & vae) const;
|
||||
};
|
||||
|
||||
//
|
||||
// YuE2 generation, the NAR transformer itself runs on a llama_context, see mtmd_gen_flow_yue2():
|
||||
// LATENT_IN = latent rows -> backbone embd rows
|
||||
// LATENT_OUT = backbone hidden rows -> velocity rows
|
||||
// GEN_WAV = Oobleck VAE decoder, latent rows -> interleaved stereo PCM
|
||||
//
|
||||
struct mtmd_graph_yue2_gen : mtmd_graph {
|
||||
struct mtmd_graph_yue2_gen : mtmd_graph_snake_vae {
|
||||
mtmd_graph_yue2_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_rows)
|
||||
: mtmd_graph(ctx, img), gen_process(gen_process), n_rows(n_rows) {}
|
||||
: mtmd_graph_snake_vae(ctx, img), gen_process(gen_process), n_rows(n_rows) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
static constexpr int n_latent = 64;
|
||||
@@ -659,9 +670,48 @@ struct mtmd_graph_yue2_gen : mtmd_graph {
|
||||
|
||||
mtmd_subctx_gen_process gen_process;
|
||||
int n_rows;
|
||||
};
|
||||
|
||||
ggml_tensor * snake(ggml_tensor * x, const mtmd_yue2::snake & s) const;
|
||||
ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
|
||||
//
|
||||
// MiniMax Music 3 generation, see mtmd_gen_wav_music3() for the flow-matching loop:
|
||||
// GEN_CODE = depth decoder of one frame: backbone hidden rows (cond, then uncond with guidance) + semantic code -> the 8 codes, the next backbone input and the condition row (mix of the 8 hidden states of the frame)
|
||||
// FLOW_COND = condition rows of a window -> DiT condition, resampled to the latent rate
|
||||
// FLOW_STEP = one DiT pass, cond and uncond rows as a batch of 2
|
||||
// GEN_WAV = DAC decoder, latent rows -> interleaved stereo PCM
|
||||
//
|
||||
struct mtmd_graph_music3_gen : mtmd_graph_snake_vae {
|
||||
mtmd_graph_music3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process,
|
||||
int n_rows, int n_out, float cfg_scale, float temp)
|
||||
: mtmd_graph_snake_vae(ctx, img), gen_process(gen_process), n_rows(n_rows), n_out(n_out), cfg_scale(cfg_scale), temp(temp) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
static constexpr int n_latent = 128; // two channels of 64
|
||||
static constexpr int n_channels = 2;
|
||||
static constexpr int vae_ratio = 512; // output samples per latent frame
|
||||
static constexpr int dit_d_head = 64;
|
||||
static constexpr int dit_n_rot = 32; // partial rotary
|
||||
static constexpr float dit_theta = 10000.0f;
|
||||
static constexpr float dit_eps = 1e-5f;
|
||||
static constexpr int top_k = 50;
|
||||
|
||||
// sampling and flow matching of the reference (encoders.py, denoise.py)
|
||||
static constexpr float depth_cfg = 1.5f;
|
||||
static constexpr float dit_cfg = 1.7f;
|
||||
static constexpr int flow_steps = 30;
|
||||
static constexpr int n_overlap = 172; // latent frames shared by neighboring windows
|
||||
|
||||
mtmd_subctx_gen_process gen_process;
|
||||
int n_rows; // GEN_CODE: backbone rows, FLOW_COND: frames, FLOW_STEP and GEN_WAV: latent frames
|
||||
int n_out; // FLOW_COND: latent frames
|
||||
float cfg_scale;
|
||||
float temp;
|
||||
|
||||
static int64_t latent_frames(int64_t n_frames);
|
||||
|
||||
ggml_cgraph * build_depth();
|
||||
ggml_cgraph * build_cond();
|
||||
ggml_cgraph * build_dit();
|
||||
ggml_cgraph * build_vocoder();
|
||||
};
|
||||
|
||||
struct mtmd_graph_soprano : mtmd_graph {
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <cmath>
|
||||
|
||||
// latent frames of a window of n_frames, see MiniMaxMusic3ConditionEncoder of the reference
|
||||
int64_t mtmd_graph_music3_gen::latent_frames(int64_t n_frames) {
|
||||
const double n = (double) n_frames * 44100.0 / 24000.0 * 960.0 / 512.0;
|
||||
return std::max<int64_t>(1, (int64_t) n);
|
||||
}
|
||||
|
||||
// one frame of the depth decoder (RVQDepthDecoder): row 0 is the backbone hidden, row p > 0 the embedding of code p - 1
|
||||
// each step reruns the 4 layers over the rows so far (at most 8, no KV cache), like the reference
|
||||
ggml_cgraph * mtmd_graph_music3_gen::build_depth() {
|
||||
const int64_t n_cb = model.gen_code_head_w->ne[2]; // residual codebooks
|
||||
const int64_t n_voc = model.gen_code_head_w->ne[1];
|
||||
const int64_t B = n_rows; // cond, then uncond with guidance
|
||||
|
||||
ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, B);
|
||||
ggml_set_name(inp_h, "inp_feats");
|
||||
ggml_set_input(inp_h);
|
||||
|
||||
ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
|
||||
ggml_set_name(code0, "inp_code0");
|
||||
ggml_set_input(code0);
|
||||
|
||||
ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_cb);
|
||||
ggml_set_name(inp_rand, "inp_rand");
|
||||
ggml_set_input(inp_rand);
|
||||
|
||||
mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, 1.0f);
|
||||
|
||||
// [n_embd, 1, B], the codes are the same in all rows
|
||||
auto project = [&](ggml_tensor * x) {
|
||||
ggml_tensor * y = build_mm(model.gen_code_proj_in_w, x);
|
||||
y = ggml_reshape_3d(ctx0, y, n_embd, 1, y->ne[1]);
|
||||
return y->ne[2] == B ? y : ggml_repeat_4d(ctx0, y, n_embd, 1, B, 1);
|
||||
};
|
||||
|
||||
ggml_tensor * e0 = ggml_get_rows(ctx0, model.gen_code_out_embd_w, code0);
|
||||
ggml_tensor * seq = ggml_concat(ctx0, project(inp_h), project(e0), 1);
|
||||
|
||||
ggml_tensor * out_codes = code0;
|
||||
ggml_tensor * hiddens = ggml_view_2d(ctx0, inp_h, n_embd, 1, inp_h->nb[1], 0); // cond row of each step
|
||||
ggml_tensor * embd_sum = e0;
|
||||
|
||||
for (int64_t g = 0; g < n_cb; g++) {
|
||||
const int64_t n_pos = g + 2;
|
||||
|
||||
ggml_tensor * ones = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos), 1.0f);
|
||||
build_vit_opts opts;
|
||||
opts.attn_mask = ggml_log(ctx0, ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG)); // 0 = keep, -inf = masked
|
||||
if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
|
||||
opts.attn_mask = ggml_cast(ctx0, opts.attn_mask, GGML_TYPE_F16);
|
||||
}
|
||||
ggml_tensor * pos = ggml_view_2d(ctx0, model.position_embeddings, n_embd, n_pos, model.position_embeddings->nb[1], 0);
|
||||
|
||||
ggml_tensor * out = build_vit(seq, n_pos, NORM_TYPE_RMS, FFN_SILU, pos, nullptr, opts); // [n_embd, n_pos, B]
|
||||
ggml_tensor * last = ggml_cont(ctx0, ggml_view_2d(ctx0, out, n_embd, B, out->nb[2], (n_pos - 1) * out->nb[1]));
|
||||
hiddens = ggml_concat(ctx0, hiddens, ggml_view_2d(ctx0, last, n_embd, 1, last->nb[1], 0), 1);
|
||||
|
||||
ggml_tensor * head = ggml_view_2d(ctx0, model.gen_code_head_w, n_embd, n_voc, model.gen_code_head_w->nb[1], g * model.gen_code_head_w->nb[2]);
|
||||
ggml_tensor * logits = ggml_mul_mat(ctx0, head, last); // [n_voc, B]
|
||||
if (B > 1) {
|
||||
ggml_tensor * cond = ggml_view_1d(ctx0, logits, n_voc, 0);
|
||||
ggml_tensor * uncond = ggml_view_1d(ctx0, logits, n_voc, logits->nb[1]);
|
||||
logits = ggml_add(ctx0, uncond, ggml_scale(ctx0, ggml_sub(ctx0, cond, uncond), cfg_scale));
|
||||
}
|
||||
|
||||
ggml_tensor * code;
|
||||
if (temp > 0.0f) {
|
||||
ggml_tensor * rnd = ggml_view_1d(ctx0, inp_rand, 1, g * inp_rand->nb[0]);
|
||||
code = cg.do_sampling(ggml_scale(ctx0, logits, 1.0f / temp), rnd);
|
||||
} else {
|
||||
code = ggml_argmax(ctx0, logits);
|
||||
}
|
||||
code = ggml_reshape_1d(ctx0, code, 1);
|
||||
out_codes = ggml_concat(ctx0, out_codes, code, 0);
|
||||
|
||||
ggml_tensor * table = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd, n_voc, model.gen_code_embd_w->nb[1], g * model.gen_code_embd_w->nb[2]);
|
||||
ggml_tensor * e = ggml_get_rows(ctx0, table, code);
|
||||
embd_sum = ggml_add(ctx0, embd_sum, e);
|
||||
if (g + 1 < n_cb) {
|
||||
seq = ggml_concat(ctx0, seq, project(e), 1);
|
||||
}
|
||||
}
|
||||
|
||||
ggml_set_name(out_codes, "out_codes");
|
||||
ggml_set_output(out_codes);
|
||||
ggml_build_forward_expand(gf, out_codes);
|
||||
|
||||
// condition row of the frame: the 8 hidden states mixed by the condition encoder weights
|
||||
ggml_tensor * cond_row = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, hiddens)), model.music3.cond_layer_w);
|
||||
ggml_set_name(cond_row, "out_feats");
|
||||
ggml_set_output(cond_row);
|
||||
ggml_build_forward_expand(gf, cond_row);
|
||||
|
||||
// backbone input of the next frame: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), must be the last node
|
||||
ggml_tensor * out_embd = ggml_scale(ctx0, embd_sum, 1.0f / sqrtf((float) (n_cb + 1)));
|
||||
cb(out_embd, "gen_audio_out", -1);
|
||||
ggml_build_forward_expand(gf, out_embd);
|
||||
return gf;
|
||||
}
|
||||
|
||||
// frame rows [n_embd, n_rows] -> conv k = 3 -> nearest resampling to n_out latent frames
|
||||
ggml_cgraph * mtmd_graph_music3_gen::build_cond() {
|
||||
const auto & m = model.music3;
|
||||
|
||||
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_rows);
|
||||
ggml_set_name(inp, "inp_feats");
|
||||
ggml_set_input(inp);
|
||||
|
||||
ggml_tensor * idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
|
||||
ggml_set_name(idx, "inp_idx");
|
||||
ggml_set_input(idx);
|
||||
|
||||
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_embd]
|
||||
x = conv1d(x, m.cond_w, m.cond_b, 1);
|
||||
x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [n_cond, T]
|
||||
x = ggml_get_rows(ctx0, x, idx);
|
||||
ggml_set_name(x, "out_feats");
|
||||
ggml_set_output(x);
|
||||
ggml_build_forward_expand(gf, x);
|
||||
return gf;
|
||||
}
|
||||
|
||||
// one velocity pass of the DiT over a window, the cond and uncond (zero condition) inputs as a batch of 2
|
||||
// the input rows are [latent, zeros, condition], the time embedding is prepended as one extra token
|
||||
ggml_cgraph * mtmd_graph_music3_gen::build_dit() {
|
||||
const auto & m = model.music3;
|
||||
const int64_t n_in = m.pre_w->ne[0];
|
||||
const int64_t n_dit = m.proj_in_w->ne[1];
|
||||
const int64_t n_head = n_dit / dit_d_head;
|
||||
const int64_t n_pos = n_rows + 1;
|
||||
const int64_t B = 2;
|
||||
|
||||
ggml_tensor * inp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_in, n_rows, B);
|
||||
ggml_set_name(inp, "inp_feats");
|
||||
ggml_set_input(inp);
|
||||
|
||||
ggml_tensor * inp_t = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
|
||||
ggml_set_name(inp_t, "inp_t");
|
||||
ggml_set_input(inp_t);
|
||||
|
||||
ggml_tensor * x = ggml_add(ctx0, inp, build_mm(m.pre_w, inp));
|
||||
x = build_mm(m.proj_in_w, x); // [n_dit, n_rows, B]
|
||||
|
||||
// time token: Fourier features -> linear -> silu -> linear
|
||||
ggml_tensor * ang = ggml_mul(ctx0, m.time_freq, inp_t);
|
||||
ggml_tensor * temb = ggml_concat(ctx0, ggml_cos(ctx0, ang), ggml_sin(ctx0, ang), 0);
|
||||
temb = ggml_silu(ctx0, ggml_add(ctx0, build_mm(m.time_in_w, temb), m.time_in_b));
|
||||
temb = ggml_add(ctx0, build_mm(m.time_out_w, temb), m.time_out_b);
|
||||
temb = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, temb, n_dit, 1, 1), n_dit, 1, B, 1);
|
||||
x = ggml_concat(ctx0, temb, x, 1); // [n_dit, n_pos, B]
|
||||
x = ggml_reshape_2d(ctx0, x, n_dit, n_pos * B);
|
||||
|
||||
ggml_tensor * pos = ggml_cast(ctx0, ggml_arange(ctx0, 0.0f, (float) n_pos, 1.0f), GGML_TYPE_I32);
|
||||
const float kq = 1.0f / sqrtf((float) dit_d_head);
|
||||
|
||||
for (size_t il = 0; il < m.dit.size(); il++) {
|
||||
const auto & layer = m.dit[il];
|
||||
|
||||
ggml_tensor * h = build_norm(x, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, dit_eps, (int) il);
|
||||
ggml_tensor * q = ggml_reshape_4d(ctx0, build_mm(layer.q_w, h), dit_d_head, n_head, n_pos, B);
|
||||
ggml_tensor * k = ggml_reshape_4d(ctx0, build_mm(layer.k_w, h), dit_d_head, n_head, n_pos, B);
|
||||
ggml_tensor * v = ggml_reshape_4d(ctx0, build_mm(layer.v_w, h), dit_d_head, n_head, n_pos, B);
|
||||
|
||||
// only the first dit_n_rot dims of each head rotate
|
||||
q = ggml_rope_ext(ctx0, q, pos, nullptr, dit_n_rot, GGML_ROPE_TYPE_NEOX, 0, dit_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
k = ggml_rope_ext(ctx0, k, pos, nullptr, dit_n_rot, GGML_ROPE_TYPE_NEOX, 0, dit_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
|
||||
x = ggml_add(ctx0, x, build_attn(layer.o_w, nullptr, q, k, v, nullptr, kq, (int) il));
|
||||
|
||||
h = build_norm(x, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, dit_eps, (int) il);
|
||||
h = build_ffn(h, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b, layer.ff_down_w, layer.ff_down_b, FFN_SILU, (int) il);
|
||||
x = ggml_add(ctx0, x, h);
|
||||
}
|
||||
|
||||
// drop the time token
|
||||
x = ggml_reshape_3d(ctx0, x, n_dit, n_pos, B);
|
||||
x = ggml_view_3d(ctx0, x, n_dit, n_rows, B, x->nb[1], x->nb[2], x->nb[1]);
|
||||
x = build_mm(m.proj_out_w, ggml_cont(ctx0, x));
|
||||
x = ggml_add(ctx0, x, build_mm(m.post_w, x)); // [n_latent, n_rows, B]
|
||||
|
||||
// classifier-free guidance
|
||||
ggml_tensor * cond = ggml_view_2d(ctx0, x, n_latent, n_rows, x->nb[1], 0);
|
||||
ggml_tensor * uncond = ggml_view_2d(ctx0, x, n_latent, n_rows, x->nb[1], x->nb[2]);
|
||||
x = ggml_add(ctx0, uncond, ggml_scale(ctx0, ggml_sub(ctx0, cond, uncond), cfg_scale));
|
||||
ggml_set_name(x, "out_feats");
|
||||
ggml_set_output(x);
|
||||
ggml_build_forward_expand(gf, x);
|
||||
return gf;
|
||||
}
|
||||
|
||||
// the two channels are decoded as two mono streams of n_latent / 2 channels, then tanh
|
||||
ggml_cgraph * mtmd_graph_music3_gen::build_vocoder() {
|
||||
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_rows);
|
||||
ggml_set_name(inp, "inp_feats");
|
||||
ggml_set_input(inp);
|
||||
|
||||
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_latent]
|
||||
const int64_t n_half = n_latent / n_channels;
|
||||
ggml_tensor * out = nullptr;
|
||||
for (int c = 0; c < n_channels; c++) {
|
||||
ggml_tensor * xc = ggml_cont(ctx0, ggml_view_2d(ctx0, x, n_rows, n_half, x->nb[1], c * n_half * x->nb[1]));
|
||||
ggml_tensor * y = ggml_tanh(ctx0, build_snake_vae(xc, model.snake_vae)); // [T * ratio, 1]
|
||||
out = out ? ggml_concat(ctx0, out, y, 1) : y;
|
||||
}
|
||||
|
||||
out = ggml_cont(ctx0, ggml_transpose(ctx0, out)); // interleaved
|
||||
ggml_set_name(out, "out_audio");
|
||||
ggml_set_output(out);
|
||||
ggml_build_forward_expand(gf, out);
|
||||
return gf;
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_music3_gen::build() {
|
||||
switch (gen_process) {
|
||||
case MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: return build_depth();
|
||||
case MTMD_SUBCTX_GEN_PROCESS_FLOW_COND: return build_cond();
|
||||
case MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP: return build_dit();
|
||||
case MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: return build_vocoder();
|
||||
default: GGML_ABORT("minimax_music3: unsupported gen process");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
#include "models.h"
|
||||
|
||||
// SnakeBeta, alpha and beta folded at conversion: x + beta * sin(alpha * x)^2
|
||||
// x: [T, C], alpha/beta: [C]
|
||||
ggml_tensor * mtmd_graph_snake_vae::snake(ggml_tensor * x, const mtmd_snake_vae::snake & s) const {
|
||||
ggml_tensor * a = ggml_reshape_2d(ctx0, s.alpha, 1, s.alpha->ne[0]);
|
||||
ggml_tensor * b = ggml_reshape_2d(ctx0, s.beta, 1, s.beta->ne[0]);
|
||||
|
||||
// expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them
|
||||
ggml_build_forward_expand(gf, a);
|
||||
ggml_build_forward_expand(gf, b);
|
||||
|
||||
ggml_tensor * y = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, a)));
|
||||
return ggml_add(ctx0, x, ggml_mul(ctx0, y, b));
|
||||
}
|
||||
|
||||
// same-length conv, stride 1; x: [T, IC], w: [K, IC, OC], returns [T, OC]
|
||||
// im2col in F32: the reference decoders run in FP32
|
||||
ggml_tensor * mtmd_graph_snake_vae::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
|
||||
const int pad = (int) (w->ne[0] - 1) / 2 * dilation;
|
||||
ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, pad, 0, dilation, 0, false, GGML_TYPE_F32); // [1, T, IC * K]
|
||||
ggml_tensor * y = ggml_mul_mat(ctx0,
|
||||
ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
|
||||
ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
ggml_tensor * mtmd_graph_snake_vae::build_snake_vae(ggml_tensor * x, const mtmd_snake_vae & vae) const {
|
||||
static constexpr int dilations[3] = { 1, 3, 9 };
|
||||
|
||||
if (vae.in_proj_w) {
|
||||
x = conv1d(x, vae.in_proj_w, vae.in_proj_b, 1);
|
||||
}
|
||||
x = conv1d(x, vae.in_w, vae.in_b, 1);
|
||||
for (size_t ib = 0; ib < vae.blocks.size(); ib++) {
|
||||
const auto & blk = vae.blocks[ib];
|
||||
const int stride = (int) (blk.up_w->ne[0] / 2);
|
||||
const int pad = (stride + 1) / 2; // ceil(stride / 2) of the reference
|
||||
x = snake(x, blk.act);
|
||||
{
|
||||
// ConvTranspose1d as GEMM + col2im, same as qwen3tts; up_w: [K, OC, IC]
|
||||
const int64_t K = blk.up_w->ne[0];
|
||||
const int64_t OC = blk.up_w->ne[1];
|
||||
ggml_tensor * w2 = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, blk.up_w, K * OC, blk.up_w->ne[2])));
|
||||
ggml_tensor * col = ggml_mul_mat(ctx0, w2, ggml_cont(ctx0, ggml_transpose(ctx0, x))); // [K * OC, T]
|
||||
x = ggml_col2im_1d(ctx0, col, stride, (int) OC, pad);
|
||||
}
|
||||
x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, blk.up_b, 1, blk.up_b->ne[0]));
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
const auto & ru = blk.res[ir];
|
||||
ggml_tensor * h = snake(x, ru.act1);
|
||||
h = conv1d(h, ru.conv1_w, ru.conv1_b, dilations[ir]);
|
||||
h = snake(h, ru.act2);
|
||||
h = conv1d(h, ru.conv2_w, ru.conv2_b, 1);
|
||||
x = ggml_add(ctx0, x, h);
|
||||
}
|
||||
cb(x, "vae_block", (int) ib);
|
||||
}
|
||||
x = snake(x, vae.post);
|
||||
return conv1d(x, vae.out_w, vae.out_b, 1);
|
||||
}
|
||||
@@ -1,33 +1,5 @@
|
||||
#include "models.h"
|
||||
|
||||
// SnakeBeta, alpha and beta folded at conversion: x + beta * sin(alpha * x)^2
|
||||
// x: [T, C], alpha/beta: [C]
|
||||
ggml_tensor * mtmd_graph_yue2_gen::snake(ggml_tensor * x, const mtmd_yue2::snake & s) const {
|
||||
ggml_tensor * a = ggml_reshape_2d(ctx0, s.alpha, 1, s.alpha->ne[0]);
|
||||
ggml_tensor * b = ggml_reshape_2d(ctx0, s.beta, 1, s.beta->ne[0]);
|
||||
|
||||
// expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them
|
||||
ggml_build_forward_expand(gf, a);
|
||||
ggml_build_forward_expand(gf, b);
|
||||
|
||||
ggml_tensor * y = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, a)));
|
||||
return ggml_add(ctx0, x, ggml_mul(ctx0, y, b));
|
||||
}
|
||||
|
||||
// same-length conv, stride 1; x: [T, IC], w: [K, IC, OC], returns [T, OC]
|
||||
// im2col in F32: the reference decoder runs in FP32
|
||||
ggml_tensor * mtmd_graph_yue2_gen::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
|
||||
const int pad = (int) (w->ne[0] - 1) / 2 * dilation;
|
||||
ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, pad, 0, dilation, 0, false, GGML_TYPE_F32); // [1, T, IC * K]
|
||||
ggml_tensor * y = ggml_mul_mat(ctx0,
|
||||
ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
|
||||
ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_yue2_gen::build() {
|
||||
const auto & y = model.yue2;
|
||||
|
||||
@@ -46,40 +18,13 @@ ggml_cgraph * mtmd_graph_yue2_gen::build() {
|
||||
}
|
||||
|
||||
GGML_ASSERT(gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV);
|
||||
static constexpr int dilations[3] = { 1, 3, 9 };
|
||||
|
||||
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_rows);
|
||||
ggml_set_name(inp, "inp_feats");
|
||||
ggml_set_input(inp);
|
||||
|
||||
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_latent]
|
||||
x = conv1d(x, y.vae_in_w, y.vae_in_b, 1);
|
||||
for (size_t ib = 0; ib < y.vae.size(); ib++) {
|
||||
const auto & blk = y.vae[ib];
|
||||
const int stride = (int) (blk.up_w->ne[0] / 2);
|
||||
const int pad = (stride + 1) / 2; // ceil(stride / 2) of the reference
|
||||
x = snake(x, blk.act);
|
||||
{
|
||||
// ConvTranspose1d as GEMM + col2im, same as qwen3tts; up_w: [K, OC, IC]
|
||||
const int64_t K = blk.up_w->ne[0];
|
||||
const int64_t OC = blk.up_w->ne[1];
|
||||
ggml_tensor * w2 = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, blk.up_w, K * OC, blk.up_w->ne[2])));
|
||||
ggml_tensor * col = ggml_mul_mat(ctx0, w2, ggml_cont(ctx0, ggml_transpose(ctx0, x))); // [K * OC, T]
|
||||
x = ggml_col2im_1d(ctx0, col, stride, (int) OC, pad);
|
||||
}
|
||||
x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, blk.up_b, 1, blk.up_b->ne[0]));
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
const auto & ru = blk.res[ir];
|
||||
ggml_tensor * h = snake(x, ru.act1);
|
||||
h = conv1d(h, ru.conv1_w, ru.conv1_b, dilations[ir]);
|
||||
h = snake(h, ru.act2);
|
||||
h = conv1d(h, ru.conv2_w, ru.conv2_b, 1);
|
||||
x = ggml_add(ctx0, x, h);
|
||||
}
|
||||
cb(x, "vae_block", (int) ib);
|
||||
}
|
||||
x = snake(x, y.vae_post);
|
||||
x = conv1d(x, y.vae_out_w, nullptr, 1); // [T, n_channels]
|
||||
x = build_snake_vae(x, model.snake_vae); // [T, n_channels]
|
||||
|
||||
x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // interleaved
|
||||
ggml_set_name(x, "out_audio");
|
||||
|
||||
@@ -1049,6 +1049,27 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
|
||||
}
|
||||
builder = std::make_unique<mtmd_graph_yue2_gen>(ctx, img, gen_process, n_rows);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
{
|
||||
using g = mtmd_graph_music3_gen;
|
||||
const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
|
||||
// row size of the input of each process; the reserve pass builds GEN_CODE with one row
|
||||
size_t n_col = (size_t) ctx->model.hparams.n_embd;
|
||||
if (gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP) {
|
||||
n_col = 2 * (size_t) ctx->model.music3.pre_w->ne[0];
|
||||
} else if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
|
||||
n_col = g::n_latent;
|
||||
}
|
||||
int n_rows = 1;
|
||||
if (params && params->feats) {
|
||||
GGML_ASSERT(!params->feats->empty() && params->feats->size() % n_col == 0);
|
||||
n_rows = (int) (params->feats->size() / n_col);
|
||||
}
|
||||
const int n_out = gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_COND ? (int) g::latent_frames(n_rows) : 0;
|
||||
const float cfg_scale = params ? params->cfg_scale : 1.0f;
|
||||
const float temp = params ? params->temp : 1.0f;
|
||||
builder = std::make_unique<mtmd_graph_music3_gen>(ctx, img, gen_process, n_rows, n_out, cfg_scale, temp);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DRAMA3_GEN:
|
||||
case PROJECTOR_TYPE_FISH_S2_GEN:
|
||||
{
|
||||
|
||||
@@ -3810,6 +3810,466 @@ private:
|
||||
bool wav_header_sent = false;
|
||||
};
|
||||
|
||||
// MiniMax Music 3: the backbone samples the semantic code of each frame with guidance, the depth decoder (mtmd) the other codes
|
||||
// the condition rows of the frames go to the flow matching and the vocoder (mtmd) in windows, like the reference (diffusers pipeline)
|
||||
class music3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
|
||||
public:
|
||||
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
|
||||
|
||||
bool supports_cfg() const override { return true; }
|
||||
bool supports_flow_steps() const override { return true; }
|
||||
|
||||
// step_gen() stops at the frame limit of the reference
|
||||
int32_t max_frames() const override { return 0; }
|
||||
|
||||
bool check_context() const override {
|
||||
if (llama_vocab_n_tokens(vocab) < tok_code0 + n_semantic) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the vocab of the backbone has no minimax_music3 audio codes\n");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void reset() override {
|
||||
seq_id = 0;
|
||||
seq_uncond = 0;
|
||||
use_cfg = false;
|
||||
cfg_scale = 1.0f;
|
||||
temp = 1.0f;
|
||||
flow_steps = 0;
|
||||
pos = 0;
|
||||
prompt.clear();
|
||||
prompt_uncond.clear();
|
||||
prompt_pos = 0;
|
||||
uprompt_pos = 0;
|
||||
logits_cond.clear();
|
||||
logits_uncond.clear();
|
||||
h_rows.clear();
|
||||
n_gen = 0;
|
||||
cond_rows.clear();
|
||||
n_rows_dropped = 0;
|
||||
n_win_done = 0;
|
||||
done = false;
|
||||
decoded = false;
|
||||
audio_pcm.clear();
|
||||
out_buf.clear();
|
||||
stream = false;
|
||||
pcm_sent = 0;
|
||||
wav_header_sent = false;
|
||||
gen_state.reset();
|
||||
}
|
||||
|
||||
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
|
||||
reset();
|
||||
seq_id = inp->seq_id;
|
||||
out_type = inp->out_type;
|
||||
stream = inp->stream;
|
||||
temp = inp->temp >= 0.0f ? inp->temp : 1.0f;
|
||||
flow_steps = inp->flow_steps;
|
||||
cfg_scale = inp->cfg_scale > 0.0f ? inp->cfg_scale : 1.5f;
|
||||
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
|
||||
if (!init_gen_state(inp->seed)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// the "tts" template renders the whole prompt, special tokens included
|
||||
const std::string text(inp->prompt, inp->prompt_len);
|
||||
prompt.resize(text.size() + 16);
|
||||
const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), prompt.data(), (int32_t) prompt.size(), false, true);
|
||||
if (n < 3 || prompt[0] != tok_im_start || prompt[n - 1] != tok_audio_start) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: minimax_music3 expects the prompt of its tts template\n");
|
||||
return 1;
|
||||
}
|
||||
prompt.resize((size_t) n);
|
||||
if (n > max_prompt) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the prompt has %d tokens, the maximum is %d\n", n, max_prompt);
|
||||
return 1;
|
||||
}
|
||||
|
||||
llama_memory_t mem = llama_get_memory(lctx);
|
||||
if (!llama_memory_seq_rm(mem, seq_id, 0, -1)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
|
||||
return 1;
|
||||
}
|
||||
use_cfg = cfg_scale != 1.0f;
|
||||
if (use_cfg) {
|
||||
// the uncond sequence lives half a seq space away from the cond one
|
||||
const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx);
|
||||
seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id);
|
||||
if (!llama_model_classifier_free_guidance(model) || seq_id >= n_seq_max / 2) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: cfg needs a paired sequence (n_seq_max = %d, seq_id = %d), reconvert the model or use cfg_scale = 1\n", n_seq_max, seq_id);
|
||||
return 1;
|
||||
}
|
||||
if (!llama_memory_seq_rm(mem, seq_uncond, 0, -1)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: cannot reset the cfg sequence\n");
|
||||
return 1;
|
||||
}
|
||||
// the uncond prompt keeps <|im_start|> and the last 2 tokens, the rest is <|audio_cfg|>
|
||||
prompt_uncond = prompt;
|
||||
std::fill(prompt_uncond.begin() + 1, prompt_uncond.end() - 2, tok_audio_cfg);
|
||||
}
|
||||
if (n_pos_needed((llama_pos) n + 1) > (int64_t) llama_n_ctx_seq(lctx)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the prompt does not fit in the context\n");
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t step_prompt(int32_t n_batch) override {
|
||||
GGML_ASSERT(n_batch > 0);
|
||||
const int32_t n_prompt = (int32_t) prompt.size();
|
||||
if (prompt_pos < n_prompt) {
|
||||
const int32_t n = std::min(n_batch, n_prompt - prompt_pos);
|
||||
llama_pos p = prompt_pos;
|
||||
if (!decode_tokens(prompt.data() + prompt_pos, n, seq_id, p)) {
|
||||
return -1;
|
||||
}
|
||||
prompt_pos += n;
|
||||
if (prompt_pos == n_prompt) {
|
||||
save_outputs(0, -1);
|
||||
}
|
||||
} else if (use_cfg && uprompt_pos < n_prompt) {
|
||||
const int32_t n = std::min(n_batch, n_prompt - uprompt_pos);
|
||||
llama_pos p = uprompt_pos;
|
||||
if (!decode_tokens(prompt_uncond.data() + uprompt_pos, n, seq_uncond, p)) {
|
||||
return -1;
|
||||
}
|
||||
uprompt_pos += n;
|
||||
if (uprompt_pos == n_prompt) {
|
||||
save_outputs(1, -1);
|
||||
}
|
||||
}
|
||||
pos = n_prompt;
|
||||
const int32_t n_left = (n_prompt - prompt_pos) + (use_cfg ? n_prompt - uprompt_pos : 0);
|
||||
return n_left;
|
||||
}
|
||||
|
||||
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
|
||||
GGML_UNUSED(sampled);
|
||||
GGML_UNUSED(h_state_in);
|
||||
*h_state_out = nullptr;
|
||||
*out_stop = false;
|
||||
|
||||
// the first frame only moves the backbone past <|audio_start|>, its condition row is dropped like the reference
|
||||
const size_t n_rows = n_frames();
|
||||
while (!done && n_frames() == n_rows) {
|
||||
if (!gen_frame()) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (n_frames() == n_rows) {
|
||||
*out_stop = true;
|
||||
return 0;
|
||||
}
|
||||
*h_state_out = h_rows.data();
|
||||
*out_stop = done;
|
||||
|
||||
// a window is final once a frame past it exists, see MiniMaxMusic3PrepareChunksStep of the reference
|
||||
while (n_frames() > window_start(n_win_done) + win_frames) {
|
||||
if (!run_window(false)) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
|
||||
*out_sample_rate = info.sample_rate;
|
||||
const uint16_t n_ch = (uint16_t) info.n_channels;
|
||||
|
||||
if (!stream) {
|
||||
if (flush() != 0) {
|
||||
return 1;
|
||||
}
|
||||
if (out_n_samples) {
|
||||
*out_n_samples = (int64_t) (audio_pcm.size() / n_ch);
|
||||
}
|
||||
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
||||
*out_data = (const char *) audio_pcm.data();
|
||||
*out_data_len = audio_pcm.size() * sizeof(float);
|
||||
return 0;
|
||||
}
|
||||
out_buf.clear();
|
||||
if (!write_wav16(out_buf, audio_pcm, info.sample_rate, n_ch)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
|
||||
return 1;
|
||||
}
|
||||
*out_data = out_buf.data();
|
||||
*out_data_len = out_buf.size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const size_t n_new = audio_pcm.size() - pcm_sent;
|
||||
if (out_n_samples) {
|
||||
*out_n_samples = (int64_t) (n_new / n_ch);
|
||||
}
|
||||
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
||||
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
|
||||
*out_data_len = n_new * sizeof(float);
|
||||
pcm_sent = audio_pcm.size();
|
||||
return 0;
|
||||
}
|
||||
out_buf.clear();
|
||||
if (!wav_header_sent) {
|
||||
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate, n_ch);
|
||||
wav_header_sent = true;
|
||||
}
|
||||
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
|
||||
pcm_sent = audio_pcm.size();
|
||||
*out_data = out_buf.data();
|
||||
*out_data_len = out_buf.size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t flush() override {
|
||||
if (decoded) {
|
||||
return 0;
|
||||
}
|
||||
if (n_frames() == 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: minimax_music3 stopped before the first frame\n");
|
||||
return 1;
|
||||
}
|
||||
done = true;
|
||||
// the windows start every win_hop frames before n_frames - win_hop, the last one ends at n_frames
|
||||
while (n_win_done == 0 || window_start(n_win_done) + win_hop < n_frames()) {
|
||||
const bool last = window_start(n_win_done + 1) + win_hop >= n_frames();
|
||||
if (!run_window(last)) {
|
||||
return 1;
|
||||
}
|
||||
if (last) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
decoded = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
// token ids of the checkpoint, see encoders.py of the reference
|
||||
static constexpr llama_token tok_im_start = 151644;
|
||||
static constexpr llama_token tok_audio_cfg = 151654;
|
||||
static constexpr llama_token tok_audio_start = 151669;
|
||||
static constexpr llama_token tok_audio_end = 151670;
|
||||
static constexpr llama_token tok_code0 = 151675;
|
||||
static constexpr int32_t n_semantic = 16384;
|
||||
static constexpr int32_t max_prompt = 5000;
|
||||
static constexpr size_t max_audio_frames = 9000;
|
||||
static constexpr int32_t top_k = 50;
|
||||
|
||||
// windows of the flow matching and their crop, see before_denoise.py and decoders.py of the reference
|
||||
static constexpr size_t win_frames = 200;
|
||||
static constexpr size_t win_hop = 100;
|
||||
static constexpr size_t crop_left = 86; // latent frames
|
||||
static constexpr size_t crop_right = 344 - 86; // latent frames
|
||||
static constexpr size_t hop_samples = 512; // samples per latent frame
|
||||
|
||||
// with guidance, both sequences share the unified KV cache
|
||||
int64_t n_pos_needed(llama_pos n) const {
|
||||
return use_cfg ? 2 * (int64_t) n : (int64_t) n;
|
||||
}
|
||||
|
||||
size_t n_frames() const {
|
||||
return n_rows_dropped + cond_rows.size() / (size_t) n_embd;
|
||||
}
|
||||
|
||||
static size_t window_start(size_t k) {
|
||||
return k * win_hop;
|
||||
}
|
||||
|
||||
bool decode_tokens(const llama_token * toks, int32_t n, llama_seq_id seq, llama_pos & p) {
|
||||
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
|
||||
if (!batch_set_tokens(batch.get(), toks, n, p, seq)) {
|
||||
return false;
|
||||
}
|
||||
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
|
||||
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
|
||||
return false;
|
||||
}
|
||||
p += n;
|
||||
return true;
|
||||
}
|
||||
|
||||
// the context outputs belong to the last decode of any sequence (server slots), keep ours right away
|
||||
// only the window that sample() reads: <|audio_end|> and the semantic codes
|
||||
void save_outputs(int row, int32_t idx) {
|
||||
auto & lg = row == 0 ? logits_cond : logits_uncond;
|
||||
const float * l = llama_get_logits_ith(lctx, idx);
|
||||
lg.assign(l + tok_audio_end, l + tok_code0 + n_semantic);
|
||||
const float * h = llama_get_embeddings_ith(lctx, idx);
|
||||
h_rows.resize((size_t) (use_cfg ? 2 : 1) * n_embd);
|
||||
std::copy(h, h + n_embd, h_rows.begin() + (size_t) row * n_embd);
|
||||
}
|
||||
|
||||
// feeds the next backbone input row to both sequences
|
||||
bool decode_embd(const float * embd) {
|
||||
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
|
||||
const int n_seq = use_cfg ? 2 : 1;
|
||||
const llama_seq_id seqs[2] = { seq_id, seq_uncond };
|
||||
for (int i = 0; i < n_seq; i++) {
|
||||
const llama_embd row = { embd, 1, (size_t) n_embd };
|
||||
const int32_t idx = llama_batch_ext_add_embd(batch.get(), seqs[i], row);
|
||||
if (idx < 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: failed to add a row to the batch (error %d)\n", idx);
|
||||
return false;
|
||||
}
|
||||
llama_batch_ext_set_pos(batch.get(), idx, &pos);
|
||||
llama_batch_ext_set_output_logits(batch.get(), idx, true);
|
||||
}
|
||||
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
|
||||
return false;
|
||||
}
|
||||
pos++;
|
||||
for (int i = 0; i < n_seq; i++) {
|
||||
save_outputs(i, i);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// semantic code or <|audio_end|>: guidance on the top-k of the cond logits, then top-k sampling, see SemanticGenerationStep
|
||||
llama_token sample() {
|
||||
const size_t n = logits_cond.size();
|
||||
std::vector<std::pair<float, size_t>> cand;
|
||||
cand.reserve(n);
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
const llama_token t = tok_audio_end + (llama_token) i;
|
||||
if (t == tok_audio_end || t >= tok_code0) {
|
||||
cand.emplace_back(logits_cond[i], i);
|
||||
}
|
||||
}
|
||||
const size_t k = std::min(cand.size(), (size_t) top_k);
|
||||
std::nth_element(cand.begin(), cand.begin() + (k - 1), cand.end(), std::greater<std::pair<float, size_t>>());
|
||||
const float thr = cand[k - 1].first;
|
||||
|
||||
std::vector<std::pair<float, size_t>> guided;
|
||||
for (const auto & c : cand) {
|
||||
if (c.first >= thr) {
|
||||
const float u = use_cfg ? logits_uncond[c.second] : c.first;
|
||||
guided.emplace_back(u + (c.first - u) * cfg_scale, c.second);
|
||||
}
|
||||
}
|
||||
std::sort(guided.begin(), guided.end(), std::greater<std::pair<float, size_t>>());
|
||||
if (temp <= 0.0f) {
|
||||
return tok_audio_end + (llama_token) guided[0].second;
|
||||
}
|
||||
// values below the k-th one are masked, ties stay
|
||||
const float gthr = guided[std::min(guided.size(), (size_t) top_k) - 1].first;
|
||||
std::vector<double> p;
|
||||
double sum = 0.0;
|
||||
for (const auto & g : guided) {
|
||||
if (g.first < gthr) {
|
||||
break;
|
||||
}
|
||||
p.push_back(std::exp((double) (g.first - guided[0].first) / temp));
|
||||
sum += p.back();
|
||||
}
|
||||
double u = std::uniform_real_distribution<double>(0.0, sum)(rng);
|
||||
for (size_t i = 0; i < p.size(); i++) {
|
||||
u -= p[i];
|
||||
if (u <= 0.0) {
|
||||
return tok_audio_end + (llama_token) guided[i].second;
|
||||
}
|
||||
}
|
||||
return tok_audio_end + (llama_token) guided[p.size() - 1].second;
|
||||
}
|
||||
|
||||
// one frame: semantic code, depth decoder, condition row, then the next backbone input
|
||||
bool gen_frame() {
|
||||
const llama_token tok = sample();
|
||||
if (tok == tok_audio_end) {
|
||||
done = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
|
||||
inp.code0 = tok - tok_code0;
|
||||
inp.feats = h_rows.data();
|
||||
inp.n_feats = h_rows.size();
|
||||
inp.cfg_scale = use_cfg ? cfg_scale : 1.0f;
|
||||
inp.temp = temp;
|
||||
mtmd_gen_out out{};
|
||||
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats != (size_t) n_embd) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: depth decoder failed\n");
|
||||
return false;
|
||||
}
|
||||
if (n_gen++ > 0) {
|
||||
cond_rows.insert(cond_rows.end(), out.feats, out.feats + out.n_feats);
|
||||
if (n_frames() >= max_audio_frames) {
|
||||
done = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (n_pos_needed(pos + 1) > (int64_t) llama_n_ctx_seq(lctx)) {
|
||||
LOG_WRN("mtmd_helper_gen_audio: the context is full, the song ends at %zu frames\n", n_frames());
|
||||
done = true;
|
||||
return true;
|
||||
}
|
||||
return decode_embd(out.embd);
|
||||
}
|
||||
|
||||
// flow matching and vocoder of one window, then the crop of the reference
|
||||
bool run_window(bool last) {
|
||||
const size_t s = window_start(n_win_done);
|
||||
const size_t e = std::min(s + win_frames, n_frames());
|
||||
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
||||
inp.feats = cond_rows.data() + (s - n_rows_dropped) * n_embd;
|
||||
inp.n_feats = (e - s) * n_embd;
|
||||
inp.n_steps = flow_steps;
|
||||
mtmd_gen_out out{};
|
||||
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: flow matching failed\n");
|
||||
return false;
|
||||
}
|
||||
const size_t n_ch = (size_t) info.n_channels;
|
||||
const size_t n = out.n_samples / n_ch;
|
||||
const size_t left = n_win_done == 0 ? 0 : crop_left * hop_samples;
|
||||
const size_t right = last ? 0 : crop_right * hop_samples;
|
||||
if (n < left + right) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: window is too short\n");
|
||||
return false;
|
||||
}
|
||||
audio_pcm.insert(audio_pcm.end(), out.audio + left * n_ch, out.audio + (n - right) * n_ch);
|
||||
n_win_done++;
|
||||
// the rows before the next window are not needed anymore
|
||||
const size_t n_drop = std::min(window_start(n_win_done), n_frames()) - n_rows_dropped;
|
||||
cond_rows.erase(cond_rows.begin(), cond_rows.begin() + n_drop * n_embd);
|
||||
n_rows_dropped += n_drop;
|
||||
return true;
|
||||
}
|
||||
|
||||
llama_seq_id seq_id = 0;
|
||||
llama_seq_id seq_uncond = 0;
|
||||
bool use_cfg = false;
|
||||
float cfg_scale = 1.0f;
|
||||
float temp = 1.0f;
|
||||
int32_t flow_steps = 0; // 0 = the model default
|
||||
llama_pos pos = 0;
|
||||
std::vector<llama_token> prompt;
|
||||
std::vector<llama_token> prompt_uncond;
|
||||
int32_t prompt_pos = 0;
|
||||
int32_t uprompt_pos = 0;
|
||||
std::vector<float> logits_cond; // see save_outputs()
|
||||
std::vector<float> logits_uncond;
|
||||
std::vector<float> h_rows; // backbone hidden of the cond, then uncond sequence
|
||||
size_t n_gen = 0; // frames sampled, the first one included
|
||||
std::vector<float> cond_rows; // one condition row per frame, from frame n_rows_dropped
|
||||
size_t n_rows_dropped = 0;
|
||||
size_t n_win_done = 0;
|
||||
bool done = false;
|
||||
bool decoded = false;
|
||||
std::mt19937 rng;
|
||||
|
||||
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
||||
bool stream = false;
|
||||
std::vector<float> audio_pcm; // interleaved
|
||||
std::vector<char> out_buf;
|
||||
size_t pcm_sent = 0;
|
||||
bool wav_header_sent = false;
|
||||
};
|
||||
|
||||
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
|
||||
switch (mtmd_gen_audio_get_info(mctx).type) {
|
||||
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
|
||||
@@ -3828,6 +4288,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new zonos3_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_YUE2:
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new yue2_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3:
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new music3_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_FISH_S2:
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new fish_s2_gen_audio_pipeline(lctx, mctx));
|
||||
default:
|
||||
|
||||
@@ -614,6 +614,7 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_ZONOS3_GEN,
|
||||
PROJECTOR_TYPE_ZONOS3_SPKENC,
|
||||
PROJECTOR_TYPE_YUE2_GEN,
|
||||
PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN,
|
||||
PROJECTOR_TYPE_FISH_S2_GEN,
|
||||
PROJECTOR_TYPE_FISH_S2_SPKENC,
|
||||
PROJECTOR_TYPE_MUSE_GLIMMER,
|
||||
@@ -709,6 +710,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
|
||||
{ PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"},
|
||||
{ PROJECTOR_TYPE_YUE2_GEN, "yue2_gen"},
|
||||
{ PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN, "minimax_music3_gen"},
|
||||
{ PROJECTOR_TYPE_FISH_S2_GEN, "fish_s2_gen"},
|
||||
{ PROJECTOR_TYPE_FISH_S2_SPKENC, "fish_s2_spkenc"},
|
||||
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
|
||||
|
||||
+118
-34
@@ -1097,6 +1097,10 @@ struct mtmd_model_loader {
|
||||
{
|
||||
hparams.audio_sample_rate = 48000;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
{
|
||||
hparams.audio_sample_rate = 44100;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DRAMA3_GEN:
|
||||
case PROJECTOR_TYPE_FISH_S2_GEN:
|
||||
{
|
||||
@@ -1759,6 +1763,44 @@ struct mtmd_model_loader {
|
||||
c2w.dac_post_conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias"));
|
||||
};
|
||||
|
||||
// a.gen.vae.*, see mtmd_snake_vae
|
||||
auto load_snake_vae = [&]() {
|
||||
auto & vae = model.snake_vae;
|
||||
const std::string p = "a.gen.vae";
|
||||
auto load_snake = [&](const std::string & q) {
|
||||
return mtmd_snake_vae::snake{ get_tensor(q + ".alpha"), get_tensor(q + ".beta") };
|
||||
};
|
||||
vae.in_proj_w = get_tensor(p + ".in_proj.weight", false);
|
||||
vae.in_proj_b = get_tensor(p + ".in_proj.bias", false);
|
||||
vae.in_w = get_tensor(p + ".conv_in.weight");
|
||||
vae.in_b = get_tensor(p + ".conv_in.bias");
|
||||
for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.up.weight", p.c_str(), b).c_str()) >= 0; b++) {
|
||||
const std::string q = p + ".blk." + std::to_string(b);
|
||||
mtmd_snake_vae::block blk;
|
||||
blk.act = load_snake(q + ".snake");
|
||||
blk.up_w = get_tensor(q + ".up.weight");
|
||||
blk.up_b = get_tensor(q + ".up.bias");
|
||||
for (int r = 0; r < 3; r++) {
|
||||
const std::string rq = q + ".res." + std::to_string(r);
|
||||
mtmd_snake_vae::res_unit ru;
|
||||
ru.act1 = load_snake(rq + ".snake1");
|
||||
ru.conv1_w = get_tensor(rq + ".conv1.weight");
|
||||
ru.conv1_b = get_tensor(rq + ".conv1.bias");
|
||||
ru.act2 = load_snake(rq + ".snake2");
|
||||
ru.conv2_w = get_tensor(rq + ".conv2.weight");
|
||||
ru.conv2_b = get_tensor(rq + ".conv2.bias");
|
||||
blk.res.push_back(ru);
|
||||
}
|
||||
vae.blocks.push_back(std::move(blk));
|
||||
}
|
||||
vae.post = load_snake(p + ".post_snake");
|
||||
vae.out_w = get_tensor(p + ".conv_out.weight");
|
||||
vae.out_b = get_tensor(p + ".conv_out.bias", false);
|
||||
if (vae.blocks.empty()) {
|
||||
throw std::runtime_error(mtmd_format("%s: no VAE decoder blocks\n", __func__));
|
||||
}
|
||||
};
|
||||
|
||||
switch (model.proj_type) {
|
||||
case PROJECTOR_TYPE_MLP:
|
||||
case PROJECTOR_TYPE_MLP_NORM:
|
||||
@@ -2926,44 +2968,83 @@ struct mtmd_model_loader {
|
||||
read_host("a.gen.nar.time.2.weight", y.time2_w, n_embd, n_embd);
|
||||
read_host("a.gen.nar.time.2.bias", y.time2_b, n_embd, 1);
|
||||
|
||||
const std::string p = "a.gen.vae";
|
||||
auto load_snake = [&](const std::string & q) {
|
||||
return mtmd_yue2::snake{ get_tensor(q + ".alpha"), get_tensor(q + ".beta") };
|
||||
};
|
||||
y.vae_in_w = get_tensor(p + ".conv_in.weight");
|
||||
y.vae_in_b = get_tensor(p + ".conv_in.bias");
|
||||
for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.up.weight", p.c_str(), b).c_str()) >= 0; b++) {
|
||||
const std::string q = p + ".blk." + std::to_string(b);
|
||||
mtmd_yue2::block blk;
|
||||
blk.act = load_snake(q + ".snake");
|
||||
blk.up_w = get_tensor(q + ".up.weight");
|
||||
blk.up_b = get_tensor(q + ".up.bias");
|
||||
for (int r = 0; r < 3; r++) {
|
||||
const std::string rq = q + ".res." + std::to_string(r);
|
||||
mtmd_yue2::res_unit ru;
|
||||
ru.act1 = load_snake(rq + ".snake1");
|
||||
ru.conv1_w = get_tensor(rq + ".conv1.weight");
|
||||
ru.conv1_b = get_tensor(rq + ".conv1.bias");
|
||||
ru.act2 = load_snake(rq + ".snake2");
|
||||
ru.conv2_w = get_tensor(rq + ".conv2.weight");
|
||||
ru.conv2_b = get_tensor(rq + ".conv2.bias");
|
||||
blk.res.push_back(ru);
|
||||
}
|
||||
y.vae.push_back(std::move(blk));
|
||||
}
|
||||
y.vae_post = load_snake(p + ".post_snake");
|
||||
y.vae_out_w = get_tensor(p + ".conv_out.weight");
|
||||
|
||||
// the tile cropping in mtmd_gen_wav_yue2() depends on the total stride
|
||||
int64_t ratio = 1;
|
||||
for (const auto & blk : y.vae) {
|
||||
ratio *= blk.up_w->ne[0] / 2;
|
||||
}
|
||||
if (ratio != mtmd_graph_yue2_gen::vae_ratio || y.vae_in_w->ne[1] != mtmd_graph_yue2_gen::n_latent ||
|
||||
y.vae_out_w->ne[2] != mtmd_graph_yue2_gen::n_channels) {
|
||||
load_snake_vae();
|
||||
const auto & vae = model.snake_vae;
|
||||
if (vae.ratio() != mtmd_graph_yue2_gen::vae_ratio || vae.n_latent() != mtmd_graph_yue2_gen::n_latent ||
|
||||
vae.out_w->ne[2] != mtmd_graph_yue2_gen::n_channels) {
|
||||
throw std::runtime_error(mtmd_format("%s: yue2: unexpected VAE decoder layout\n", __func__));
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
{
|
||||
// generic a.gen.code.* tensors (depth decoder layers, position_embd, post_ln) are loaded above
|
||||
using g = mtmd_graph_music3_gen;
|
||||
model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
|
||||
model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight"));
|
||||
model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight"));
|
||||
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
|
||||
// the depth decoder sees the backbone hidden, then one row per code (all but the last)
|
||||
const int64_t n_embd = hparams.n_embd;
|
||||
if (!model.post_ln_w || !model.position_embeddings || n_embd != hparams.projection_dim ||
|
||||
model.gen_code_proj_in_w->ne[0] != n_embd || model.gen_code_proj_in_w->ne[1] != n_embd ||
|
||||
model.gen_code_embd_w->ne[0] != n_embd || model.gen_code_head_w->ne[0] != n_embd ||
|
||||
model.gen_code_out_embd_w->ne[0] != n_embd || model.position_embeddings->ne[0] != n_embd ||
|
||||
model.gen_code_embd_w->ne[2] != model.gen_code_head_w->ne[2] ||
|
||||
model.gen_code_embd_w->ne[1] != model.gen_code_head_w->ne[1] ||
|
||||
model.position_embeddings->ne[1] < model.gen_code_head_w->ne[2] + 1) {
|
||||
throw std::runtime_error(mtmd_format("%s: minimax_music3: unexpected depth decoder layout\n", __func__));
|
||||
}
|
||||
|
||||
auto & m = model.music3;
|
||||
m.cond_layer_w = get_tensor("a.gen.cond.layer_w");
|
||||
m.cond_w = get_tensor("a.gen.cond.proj.weight");
|
||||
m.cond_b = get_tensor("a.gen.cond.proj.bias");
|
||||
m.time_freq = get_tensor("a.gen.flow.time_freq");
|
||||
m.time_in_w = get_tensor("a.gen.flow.time_in.weight");
|
||||
m.time_in_b = get_tensor("a.gen.flow.time_in.bias");
|
||||
m.time_out_w = get_tensor("a.gen.flow.time_out.weight");
|
||||
m.time_out_b = get_tensor("a.gen.flow.time_out.bias");
|
||||
m.pre_w = get_tensor("a.gen.flow.pre.weight");
|
||||
m.proj_in_w = get_tensor("a.gen.flow.proj_in.weight");
|
||||
m.proj_out_w = get_tensor("a.gen.flow.proj_out.weight");
|
||||
m.post_w = get_tensor("a.gen.flow.post.weight");
|
||||
for (int il = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format(TN_ATTN_Q, "a.gen.flow", il, "weight").c_str()) >= 0; il++) {
|
||||
const char * q = "a.gen.flow";
|
||||
mtmd_layer layer;
|
||||
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, q, il, "weight"));
|
||||
layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, q, il, "bias"));
|
||||
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, q, il, "weight"));
|
||||
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, q, il, "weight"));
|
||||
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, q, il, "weight"));
|
||||
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, q, il, "weight"));
|
||||
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, q, il, "weight"));
|
||||
layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, q, il, "bias"));
|
||||
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, q, il, "weight"));
|
||||
layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, q, il, "bias"));
|
||||
layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, q, il, "weight"));
|
||||
layer.ff_gate_b = get_tensor(mtmd_format(TN_FFN_GATE, q, il, "bias"));
|
||||
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, q, il, "weight"));
|
||||
layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, q, il, "bias"));
|
||||
m.dit.push_back(layer);
|
||||
}
|
||||
|
||||
load_snake_vae();
|
||||
const auto & vae = model.snake_vae;
|
||||
const int64_t n_dit_embd = m.proj_in_w->ne[1];
|
||||
const int64_t n_cond = m.cond_w->ne[2];
|
||||
if (m.dit.empty() || m.cond_layer_w->ne[0] != model.gen_code_head_w->ne[2] + 1 ||
|
||||
m.cond_w->ne[0] != 3 || m.cond_w->ne[1] != hparams.n_embd ||
|
||||
m.pre_w->ne[0] != 2 * g::n_latent + n_cond || m.pre_w->ne[1] != m.pre_w->ne[0] ||
|
||||
m.proj_in_w->ne[0] != m.pre_w->ne[0] || m.proj_out_w->ne[0] != n_dit_embd || m.proj_out_w->ne[1] != g::n_latent ||
|
||||
m.post_w->ne[0] != g::n_latent || m.post_w->ne[1] != g::n_latent || m.cond_b->ne[0] != n_cond ||
|
||||
m.time_freq->ne[0] * 2 != m.time_in_w->ne[0] || m.time_in_w->ne[1] != m.time_out_w->ne[0] ||
|
||||
m.time_in_b->ne[0] != m.time_in_w->ne[1] || m.time_out_w->ne[1] != n_dit_embd || m.time_out_b->ne[0] != n_dit_embd ||
|
||||
n_dit_embd % g::dit_d_head != 0 ||
|
||||
vae.ratio() != g::vae_ratio || vae.n_latent() * 2 != g::n_latent || vae.out_w->ne[2] != 1) {
|
||||
throw std::runtime_error(mtmd_format("%s: minimax_music3: unexpected DiT or vocoder layout\n", __func__));
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DRAMA3_GEN:
|
||||
{
|
||||
// generic a.gen.code.blk.* tensors are loaded by the loop above
|
||||
@@ -4610,6 +4691,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
|
||||
case PROJECTOR_TYPE_DRAMA3_GEN:
|
||||
case PROJECTOR_TYPE_ZONOS3_GEN:
|
||||
case PROJECTOR_TYPE_YUE2_GEN:
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
case PROJECTOR_TYPE_FISH_S2_GEN:
|
||||
{
|
||||
// one hidden-state vector fed back to the talker per call
|
||||
@@ -4811,6 +4893,8 @@ int mtmd_model::n_mmproj_embd() const {
|
||||
return gen_code_out_embd_w->ne[0];
|
||||
case PROJECTOR_TYPE_YUE2_GEN:
|
||||
return yue2.inp_w->ne[1];
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
return gen_code_out_embd_w->ne[0];
|
||||
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
|
||||
return spk_proj_w->ne[1];
|
||||
case PROJECTOR_TYPE_POCKETTTS_GEN:
|
||||
|
||||
+50
-10
@@ -943,12 +943,12 @@ struct mtmd_zonos3_wav {
|
||||
int64_t rs_kernel_len = 0;
|
||||
};
|
||||
|
||||
// yue2: projections of the flow-matching NAR (a.gen.nar.*) and the Oobleck VAE decoder (a.gen.vae.*)
|
||||
// the NAR transformer layers are loaded by libllama, see mtmd_yue2_nar_load()
|
||||
struct mtmd_yue2 {
|
||||
// snake-activated conv decoder of the DAC / Oobleck family (a.gen.vae.*): yue2 VAE, minimax_music3 vocoder
|
||||
struct mtmd_snake_vae {
|
||||
// x + beta * sin(alpha * x)^2, alpha and beta folded at conversion
|
||||
struct snake {
|
||||
ggml_tensor * alpha = nullptr; // exp(alpha)
|
||||
ggml_tensor * beta = nullptr; // 1 / (exp(beta) + eps)
|
||||
ggml_tensor * alpha = nullptr;
|
||||
ggml_tensor * beta = nullptr;
|
||||
};
|
||||
struct res_unit {
|
||||
snake act1;
|
||||
@@ -965,6 +965,31 @@ struct mtmd_yue2 {
|
||||
std::vector<res_unit> res;
|
||||
};
|
||||
|
||||
ggml_tensor * in_proj_w = nullptr; // optional k = 1 conv before conv_in
|
||||
ggml_tensor * in_proj_b = nullptr;
|
||||
ggml_tensor * in_w = nullptr;
|
||||
ggml_tensor * in_b = nullptr;
|
||||
std::vector<block> blocks;
|
||||
snake post;
|
||||
ggml_tensor * out_w = nullptr;
|
||||
ggml_tensor * out_b = nullptr; // optional
|
||||
|
||||
// output samples per latent frame
|
||||
int64_t ratio() const {
|
||||
int64_t r = 1;
|
||||
for (const auto & blk : blocks) {
|
||||
r *= blk.up_w->ne[0] / 2;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
int64_t n_latent() const {
|
||||
return (in_proj_w ? in_proj_w : in_w)->ne[1];
|
||||
}
|
||||
};
|
||||
|
||||
// yue2: projections of the flow-matching NAR (a.gen.nar.*), the VAE is mtmd_model::snake_vae
|
||||
// the NAR transformer layers are loaded by libllama, see mtmd_yue2_nar_load()
|
||||
struct mtmd_yue2 {
|
||||
ggml_tensor * inp_w = nullptr; // latent -> backbone embd
|
||||
ggml_tensor * inp_b = nullptr;
|
||||
ggml_tensor * out_w = nullptr; // backbone hidden -> velocity
|
||||
@@ -975,12 +1000,25 @@ struct mtmd_yue2 {
|
||||
std::vector<float> time0_b;
|
||||
std::vector<float> time2_w; // [n_embd, n_embd]
|
||||
std::vector<float> time2_b;
|
||||
};
|
||||
|
||||
ggml_tensor * vae_in_w = nullptr;
|
||||
ggml_tensor * vae_in_b = nullptr;
|
||||
std::vector<block> vae;
|
||||
snake vae_post;
|
||||
ggml_tensor * vae_out_w = nullptr;
|
||||
// minimax_music3: condition encoder (a.gen.cond.*) and flow-matching DiT (a.gen.flow.*)
|
||||
// the depth decoder is the generic a.gen.code.* layers, the vocoder is mtmd_model::snake_vae
|
||||
struct mtmd_music3 {
|
||||
ggml_tensor * cond_layer_w = nullptr; // mix of the hidden states of a frame, softmax and scale folded in
|
||||
ggml_tensor * cond_w = nullptr; // k = 3 conv
|
||||
ggml_tensor * cond_b = nullptr;
|
||||
|
||||
ggml_tensor * time_freq = nullptr; // random Fourier features, 2 * pi folded in
|
||||
ggml_tensor * time_in_w = nullptr;
|
||||
ggml_tensor * time_in_b = nullptr;
|
||||
ggml_tensor * time_out_w = nullptr;
|
||||
ggml_tensor * time_out_b = nullptr;
|
||||
ggml_tensor * pre_w = nullptr; // k = 1 conv over [latent, zeros, cond], residual
|
||||
ggml_tensor * proj_in_w = nullptr;
|
||||
ggml_tensor * proj_out_w = nullptr;
|
||||
ggml_tensor * post_w = nullptr; // k = 1 conv, residual
|
||||
std::vector<mtmd_layer> dit;
|
||||
};
|
||||
|
||||
struct mtmd_vocos {
|
||||
@@ -1267,6 +1305,8 @@ struct mtmd_model {
|
||||
std::vector<mtmd_drama3_layer> drama3_layers;
|
||||
mtmd_zonos3_wav zonos3;
|
||||
mtmd_yue2 yue2;
|
||||
mtmd_music3 music3;
|
||||
mtmd_snake_vae snake_vae; // yue2, minimax_music3
|
||||
|
||||
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
|
||||
mtmd_seanet seanet;
|
||||
|
||||
@@ -1510,6 +1510,40 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
{
|
||||
set_input_f32("inp_feats", *params->feats);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
{
|
||||
set_input_f32("inp_feats", *params->feats);
|
||||
if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) {
|
||||
if (params->code0 < 0 || (int64_t) params->code0 >= model.gen_code_out_embd_w->ne[1]) {
|
||||
LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) model.gen_code_out_embd_w->ne[1]);
|
||||
return false;
|
||||
}
|
||||
std::vector<int32_t> code0 = { params->code0 };
|
||||
set_input_i32("inp_code0", code0);
|
||||
// greedy (temp <= 0) has no random input
|
||||
if (ggml_graph_get_tensor(gf, "inp_rand") != nullptr) {
|
||||
std::uniform_real_distribution<float> dist(0.0f, 1.0f);
|
||||
std::vector<float> r(model.gen_code_head_w->ne[2]);
|
||||
for (auto & v : r) {
|
||||
v = dist(params->gen_state->rng);
|
||||
}
|
||||
set_input_f32("inp_rand", r);
|
||||
}
|
||||
} else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_COND) {
|
||||
// F.interpolate(mode="nearest") of the reference
|
||||
ggml_tensor * t = get_inp_tensor("inp_idx");
|
||||
const int64_t n_in = (int64_t) params->feats->size() / model.hparams.n_embd;
|
||||
const int64_t n_out = t->ne[0];
|
||||
const float scale = (float) n_in / (float) n_out;
|
||||
std::vector<int32_t> idx(n_out);
|
||||
for (int64_t i = 0; i < n_out; i++) {
|
||||
idx[i] = (int32_t) std::min<int64_t>((int64_t) floorf((float) i * scale), n_in - 1);
|
||||
}
|
||||
set_input_i32("inp_idx", idx);
|
||||
} else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP) {
|
||||
set_input_f32("inp_t", { params->flow_t });
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GEMMA4V:
|
||||
case PROJECTOR_TYPE_GEMMA4UV:
|
||||
{
|
||||
@@ -2633,8 +2667,9 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio);
|
||||
}
|
||||
|
||||
// drop the tail audio that comes from the code-0 rear padding, drama3, fish_s2 and yue2 have no padding
|
||||
if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN && model.proj_type != PROJECTOR_TYPE_YUE2_GEN) {
|
||||
// drop the tail audio that comes from the code-0 rear padding, drama3, fish_s2, yue2 and minimax_music3 have no padding
|
||||
if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN && model.proj_type != PROJECTOR_TYPE_YUE2_GEN &&
|
||||
model.proj_type != PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN) {
|
||||
const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
|
||||
const int64_t n_frames_w = hparams.wav_tfm_swa;
|
||||
const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
|
||||
|
||||
@@ -71,6 +71,8 @@ enum mtmd_subctx_gen_process {
|
||||
MTMD_SUBCTX_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens
|
||||
MTMD_SUBCTX_GEN_PROCESS_LATENT_IN, // internal: flow latent rows to backbone embd rows
|
||||
MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT, // internal: backbone hidden rows to flow velocity rows
|
||||
MTMD_SUBCTX_GEN_PROCESS_FLOW_COND, // internal: frame rows to flow condition rows
|
||||
MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP, // internal: one velocity pass of the flow transformer
|
||||
};
|
||||
|
||||
// persistent state of a causal audio encoder, for streaming input (one per stream)
|
||||
@@ -127,6 +129,8 @@ struct mtmd_subctx_encode_params {
|
||||
std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
|
||||
std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
|
||||
float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
|
||||
float cfg_scale = 1.0f; // classifier-free guidance, 1.0 = off
|
||||
float flow_t = 0.0f; // FLOW_STEP: flow-matching time, 0 = noise
|
||||
bool * out_is_eos = nullptr;
|
||||
|
||||
// TOKENIZE: speech embedding rows of the produced codes
|
||||
|
||||
@@ -2606,6 +2606,11 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
|
||||
info.sample_rate = 48000;
|
||||
info.n_channels = mtmd_graph_yue2_gen::n_channels;
|
||||
break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
info.type = MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3;
|
||||
info.sample_rate = 44100;
|
||||
info.n_channels = mtmd_graph_music3_gen::n_channels;
|
||||
break;
|
||||
case PROJECTOR_TYPE_FISH_S2_GEN:
|
||||
info.type = MTMD_GEN_AUDIO_TYPE_FISH_S2;
|
||||
info.sample_rate = 44100;
|
||||
@@ -3028,6 +3033,12 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
|
||||
case PROJECTOR_TYPE_SOPRANO:
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
||||
break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
// top-k 50 is fixed in the graph, like the reference
|
||||
inp.top_k = 50;
|
||||
inp.top_p = 1.0f;
|
||||
inp.temp = 1.0f;
|
||||
break;
|
||||
case PROJECTOR_TYPE_DRAMA3_GEN:
|
||||
// codebook heads, same as the reference (sglang-omni fishaudio_drama3)
|
||||
inp.top_k = 30;
|
||||
@@ -3076,6 +3087,10 @@ struct mtmd_gen_state {
|
||||
// zonos3: the iSTFT head runs on the host
|
||||
std::unique_ptr<mtmd_audio_zonos3_synth> zonos3_synth;
|
||||
|
||||
// minimax_music3: tail of the previous window, the next one starts from it, see mtmd_gen_wav_music3()
|
||||
std::vector<float> flow_prev_latent;
|
||||
std::vector<float> flow_prev_cond;
|
||||
|
||||
~mtmd_gen_state() {
|
||||
delete gen_state;
|
||||
}
|
||||
@@ -3116,6 +3131,8 @@ void mtmd_gen_state_reset(mtmd_gen_state * state) {
|
||||
state->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
|
||||
}
|
||||
state->zonos3_synth.reset();
|
||||
state->flow_prev_latent.clear();
|
||||
state->flow_prev_cond.clear();
|
||||
}
|
||||
|
||||
// zonos3 GEN_WAV: n_codes == 0 flushes and resets the state
|
||||
@@ -3446,6 +3463,155 @@ static int32_t mtmd_gen_wav_yue2(mtmd_context * ctx, mtmd_gen_state * state, con
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
// minimax_music3: one pass of an mtmd graph on rows of floats
|
||||
static bool music3_encode(mtmd_context * ctx, mtmd_gen_state * state, mtmd_subctx_encode_params & params, const std::vector<float> & in) {
|
||||
mtmd_image_f32 dummy;
|
||||
dummy.set_size({1, 1}, false, true);
|
||||
dummy.cpy_buf(std::vector<float>(1, 0.0f));
|
||||
mtmd_image_f32_batch batch;
|
||||
batch.is_audio = true;
|
||||
batch.entries.push_back(std::move(dummy));
|
||||
|
||||
params.imgs = &batch;
|
||||
params.n_threads = ctx->n_threads;
|
||||
params.feats = ∈
|
||||
params.gen_state = state->gen_state;
|
||||
return ctx->ctx_gen_a->encode(¶ms);
|
||||
}
|
||||
|
||||
// minimax_music3 GEN_CODE: the depth decoder of one frame, see _generate_depth_codes() of the reference (encoders.py)
|
||||
static int32_t mtmd_gen_code_music3(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
|
||||
using g = mtmd_graph_music3_gen;
|
||||
const size_t n_embd = (size_t) ctx->ctx_gen_a->model.n_mmproj_embd();
|
||||
if (!inp->feats || (inp->n_feats != n_embd && inp->n_feats != 2 * n_embd)) {
|
||||
LOG_ERR("%s: minimax_music3 needs 1 or 2 backbone hidden rows\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
const std::vector<float> rows(inp->feats, inp->feats + inp->n_feats);
|
||||
|
||||
std::vector<float> out_embd(n_embd);
|
||||
mtmd_subctx_encode_params params;
|
||||
params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
|
||||
params.out_embd = &out_embd;
|
||||
params.out_codes = &state->out_codes;
|
||||
params.out_feats = &state->out_feats;
|
||||
params.code0 = inp->code0;
|
||||
params.cfg_scale = inp->cfg_scale > 0.0f ? inp->cfg_scale : g::depth_cfg;
|
||||
params.temp = inp->temp;
|
||||
if (!music3_encode(ctx, state, params, rows)) {
|
||||
LOG_ERR("%s: encode failed (depth decoder)\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
state->out_embd = std::move(out_embd);
|
||||
|
||||
out->embd = state->out_embd.data();
|
||||
out->codes = state->out_codes.data();
|
||||
out->n_codes = state->out_codes.size();
|
||||
out->feats = state->out_feats.data();
|
||||
out->n_feats = state->out_feats.size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// minimax_music3 GEN_WAV: condition rows of one window -> flow matching -> vocoder, see denoise.py of the reference
|
||||
// a window starts from the tail of the previous one, kept in the state
|
||||
static int32_t mtmd_gen_wav_music3(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
|
||||
using g = mtmd_graph_music3_gen;
|
||||
const auto & m = ctx->ctx_gen_a->model.music3;
|
||||
const size_t n_embd = (size_t) ctx->ctx_gen_a->model.hparams.n_embd;
|
||||
const size_t n_cond = (size_t) m.cond_w->ne[2];
|
||||
const size_t n_in = (size_t) m.pre_w->ne[0]; // [latent, zeros, condition]
|
||||
const size_t n_lc = g::n_latent;
|
||||
if (!inp->feats || inp->n_feats == 0 || inp->n_feats % n_embd != 0) {
|
||||
LOG_ERR("%s: minimax_music3 needs whole condition rows\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
const std::vector<float> rows(inp->feats, inp->feats + inp->n_feats);
|
||||
|
||||
std::vector<float> cond;
|
||||
{
|
||||
mtmd_subctx_encode_params params;
|
||||
params.gen_process = MTMD_SUBCTX_GEN_PROCESS_FLOW_COND;
|
||||
params.out_feats = &cond;
|
||||
if (!music3_encode(ctx, state, params, rows)) {
|
||||
LOG_ERR("%s: encode failed (condition)\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
const size_t n_lat = cond.size() / n_cond;
|
||||
|
||||
// the overlap takes the condition of the previous window, its latents are blended toward the previous ones
|
||||
auto & prev_lat = state->flow_prev_latent;
|
||||
auto & prev_cond = state->flow_prev_cond;
|
||||
const size_t n_ov = std::min(prev_lat.size() / n_lc, n_lat);
|
||||
std::copy(prev_cond.begin(), prev_cond.begin() + n_ov * n_cond, cond.begin());
|
||||
|
||||
std::normal_distribution<float> dist(0.0f, 1.0f);
|
||||
std::vector<float> x(n_lat * n_lc);
|
||||
for (auto & v : x) {
|
||||
v = dist(state->gen_state->rng);
|
||||
}
|
||||
const std::vector<float> noise_ov(x.begin(), x.begin() + n_ov * n_lc);
|
||||
|
||||
// batch 0 is conditioned, batch 1 has a zero condition
|
||||
std::vector<float> dit_in(2 * n_lat * n_in, 0.0f);
|
||||
for (size_t i = 0; i < n_lat; i++) {
|
||||
std::copy(cond.begin() + i * n_cond, cond.begin() + (i + 1) * n_cond, dit_in.begin() + i * n_in + 2 * n_lc);
|
||||
}
|
||||
|
||||
// FlowMatchEulerDiscreteScheduler with invert_sigmas: t = 1 - linspace(1, 1 / n, n), then 1
|
||||
const int n_steps = inp->n_steps > 0 ? inp->n_steps : g::flow_steps;
|
||||
std::vector<float> ts(n_steps + 1, 1.0f);
|
||||
for (int i = 0; i < n_steps; i++) {
|
||||
const double sigma = n_steps > 1 ? 1.0 + i * (1.0 / n_steps - 1.0) / (n_steps - 1) : 1.0;
|
||||
ts[i] = 1.0f - (float) sigma;
|
||||
}
|
||||
|
||||
std::vector<float> vel;
|
||||
for (int i = 0; i < n_steps; i++) {
|
||||
const float t = ts[i];
|
||||
for (size_t j = 0; j < n_ov * n_lc; j++) {
|
||||
x[j] = (1.0f - (1.0f - 1e-6f) * t) * noise_ov[j] + t * prev_lat[j];
|
||||
}
|
||||
for (size_t b = 0; b < 2; b++) {
|
||||
for (size_t r = 0; r < n_lat; r++) {
|
||||
std::copy(x.begin() + r * n_lc, x.begin() + (r + 1) * n_lc, dit_in.begin() + (b * n_lat + r) * n_in);
|
||||
}
|
||||
}
|
||||
mtmd_subctx_encode_params params;
|
||||
params.gen_process = MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP;
|
||||
params.flow_t = t;
|
||||
params.cfg_scale = g::dit_cfg;
|
||||
params.out_feats = &vel;
|
||||
if (!music3_encode(ctx, state, params, dit_in) || vel.size() != x.size()) {
|
||||
LOG_ERR("%s: encode failed (flow step %d)\n", __func__, i);
|
||||
return 1;
|
||||
}
|
||||
const float dt = ts[i + 1] - ts[i];
|
||||
for (size_t j = 0; j < x.size(); j++) {
|
||||
x[j] += dt * vel[j];
|
||||
}
|
||||
}
|
||||
std::copy(prev_lat.begin(), prev_lat.begin() + n_ov * n_lc, x.begin());
|
||||
|
||||
// the next window starts from latent frames [L - 2 * n_overlap, L - n_overlap) of this one
|
||||
const size_t s0 = n_lat > 2 * (size_t) g::n_overlap ? n_lat - 2 * g::n_overlap : 0;
|
||||
const size_t s1 = n_lat > (size_t) g::n_overlap ? n_lat - g::n_overlap : 0;
|
||||
prev_lat.assign(x.begin() + s0 * n_lc, x.begin() + s1 * n_lc);
|
||||
prev_cond.assign(cond.begin() + s0 * n_cond, cond.begin() + s1 * n_cond);
|
||||
|
||||
mtmd_subctx_encode_params params;
|
||||
params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
|
||||
params.out_audio = &state->out_audio;
|
||||
if (!music3_encode(ctx, state, params, x)) {
|
||||
LOG_ERR("%s: encode failed (vocoder)\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
out->audio = state->out_audio.data();
|
||||
out->n_samples = state->out_audio.size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
|
||||
if (!state || state->sctx != ctx->ctx_gen_a.get()) {
|
||||
LOG_ERR("%s: invalid gen state\n", __func__);
|
||||
@@ -3476,6 +3642,10 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
|
||||
return inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE ? mtmd_gen_flow_yue2(ctx, state, inp, out)
|
||||
: mtmd_gen_wav_yue2(ctx, state, inp, out);
|
||||
}
|
||||
if (sctx->model.proj_type == PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN) {
|
||||
return inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE ? mtmd_gen_code_music3(ctx, state, inp, out)
|
||||
: mtmd_gen_wav_music3(ctx, state, inp, out);
|
||||
}
|
||||
|
||||
if (sctx->model.proj_type == PROJECTOR_TYPE_SOPRANO) {
|
||||
const size_t n_embd = (size_t) sctx->model.n_mmproj_embd();
|
||||
|
||||
+7
-1
@@ -474,6 +474,7 @@ enum mtmd_gen_audio_type {
|
||||
MTMD_GEN_AUDIO_TYPE_DRAMA3,
|
||||
MTMD_GEN_AUDIO_TYPE_ZONOS3,
|
||||
MTMD_GEN_AUDIO_TYPE_YUE2,
|
||||
MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3,
|
||||
MTMD_GEN_AUDIO_TYPE_FISH_S2,
|
||||
};
|
||||
|
||||
@@ -504,6 +505,7 @@ enum mtmd_gen_process_type {
|
||||
// for pocket-tts, this is mimi decoder
|
||||
// for lfm3a, this is the codec decoder + ISTFT, n_codes == 0 flushes the tail
|
||||
// for yue2, this is one VAE pass, the caller splits a long song into tiles
|
||||
// for minimax_music3, this is one window: flow matching + vocoder
|
||||
};
|
||||
|
||||
struct mtmd_gen_inp {
|
||||
@@ -533,8 +535,12 @@ struct mtmd_gen_inp {
|
||||
size_t backbone_state_size;
|
||||
int32_t backbone_n_past;
|
||||
|
||||
// flow-matching GEN_CODE: steps of the ODE solver, <= 0 = model default (yue2: 32)
|
||||
// flow matching: steps of the ODE solver, <= 0 = model default (yue2 GEN_CODE: 32, minimax_music3 GEN_WAV: 30)
|
||||
int32_t n_steps;
|
||||
|
||||
// minimax_music3 GEN_CODE: feats holds the backbone hidden rows, cond then uncond with guidance
|
||||
// classifier-free guidance scale of the residual codes, <= 0 = model default (1.5)
|
||||
float cfg_scale;
|
||||
};
|
||||
|
||||
struct mtmd_gen_out {
|
||||
|
||||
@@ -880,9 +880,9 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list
|
||||
|
||||
`loudness`: Target loudness in LUFS, `< 0` (default `0.0`, the model's own loudness). Only for models that support it, others reject a value other than `0.0` with a 400 error.
|
||||
|
||||
`cfg_scale`: Classifier-free guidance scale of the audio codes, `> 0`, `1.0` disables it (default `0.0`, the model's own). Only for models that support it (YuE2), others reject a value other than `0.0` with a 400 error.
|
||||
`cfg_scale`: Classifier-free guidance scale of the audio codes, `> 0`, `1.0` disables it (default `0.0`, the model's own). Only for models that support it (YuE2, MiniMax Music 3), others reject a value other than `0.0` with a 400 error.
|
||||
|
||||
`flow_steps`: Steps of the flow-matching ODE solver, `1` to `1024` (default `0`, the model's own). Fewer steps are faster and the audio is less accurate. Only for models that support it (YuE2), others reject a value other than `0` with a 400 error.
|
||||
`flow_steps`: Steps of the flow-matching ODE solver, `1` to `1024` (default `0`, the model's own). Fewer steps are faster and the audio is less accurate. Only for models that support it (YuE2, MiniMax Music 3), others reject a value other than `0` with a 400 error.
|
||||
|
||||
`n_predict`: Max number of audio frames to generate. Defaults to the limit of the model (`512` for most models); generation normally stops earlier once the model emits an end-of-speech token.
|
||||
|
||||
|
||||
@@ -564,9 +564,10 @@ bool server_http_context::init_listener(const common_params & params) {
|
||||
srv->Get(params.api_prefix + "/index.html", serve_asset_cached("index.html", true, cache_revalidate));
|
||||
|
||||
// All remaining assets registered directly from the embedded asset table.
|
||||
// PWA revalidation files (sw.js, manifest, version.json) use no-cache;
|
||||
// PWA revalidation files (service workers, manifest, version.json) use no-cache;
|
||||
// everything else is immutable.
|
||||
static const std::unordered_set<std::string> no_cache_names = {
|
||||
"service-worker.js",
|
||||
"sw.js",
|
||||
"manifest.webmanifest",
|
||||
"_app/version.json",
|
||||
@@ -588,9 +589,10 @@ bool server_http_context::init_listener(const common_params & params) {
|
||||
}
|
||||
|
||||
// a browser that used the built-in UI keeps its service worker, so it shows the old UI even after the UI is replaced or disabled
|
||||
// answer the worker's update check with a worker that removes itself; a sw.js in public_path is served first
|
||||
// answer the worker's update check with a worker that removes itself; a worker file in public_path is served first
|
||||
// sw.js is the worker path of older UI builds, service-worker.js the current one
|
||||
if (!params.ui || !params.public_path.empty()) {
|
||||
srv->Get(params.api_prefix + "/sw.js", [](const httplib::Request &, httplib::Response & res) {
|
||||
const auto serve_sw_remove = [](const httplib::Request &, httplib::Response & res) {
|
||||
static constexpr const char * sw_remove_js = R"(
|
||||
self.addEventListener('install', () => self.skipWaiting());
|
||||
self.addEventListener('activate', (e) => e.waitUntil((async () => {
|
||||
@@ -601,7 +603,9 @@ self.addEventListener('activate', (e) => e.waitUntil((async () => {
|
||||
)";
|
||||
res.set_header("Cache-Control", "no-cache");
|
||||
res.set_content(sw_remove_js, "application/javascript");
|
||||
});
|
||||
};
|
||||
srv->Get(params.api_prefix + "/sw.js", serve_sw_remove);
|
||||
srv->Get(params.api_prefix + "/service-worker.js", serve_sw_remove);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# ui
|
||||
|
||||
The UI of llama.cpp: the packages in `lib` and the apps built on them.
|
||||
|
||||
```text
|
||||
ui/
|
||||
├── lib/
|
||||
│ └── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components
|
||||
└── apps/
|
||||
└── chat/ Llama Chat, the chat app that llama-server serves
|
||||
```
|
||||
|
||||
- [`lib/ui`](lib/ui/README.md) draws the parts and handles interaction, with no business logic. It is built with
|
||||
`@sveltejs/package` and published to npm as `@ggml-org/llama-ui`. Apps in this repo skip the package: they compile
|
||||
its sources as part of their own build, so they hold no copy of a component, a token or a font.
|
||||
- [`apps/chat`](apps/chat/README.md) talks to llama-server, stores conversations and runs the agentic loop. CMake
|
||||
builds it into llama-server (see `scripts/ui-assets.cmake`).
|
||||
|
||||
Each folder is its own npm package with its own `package-lock.json`: run `npm ci` in the one you work on. Building,
|
||||
checking and testing `apps/chat` needs no install in `lib/ui`.
|
||||
@@ -1,5 +1,5 @@
|
||||
<script lang="ts">
|
||||
import * as Tooltip from '../../src/lib/components/ui/tooltip';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
|
||||
interface Props {
|
||||
children: any;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { Preview } from '@storybook/sveltekit';
|
||||
import '../src/app.css';
|
||||
import ModeWatcherDecorator from './decorators/ModeWatcherDecorator.svelte';
|
||||
import TooltipProviderDecorator from './decorators/TooltipProviderDecorator.svelte';
|
||||
import type { Preview } from '@storybook/sveltekit';
|
||||
|
||||
const preview: Preview = {
|
||||
parameters: {
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Instructions for ui/apps/chat
|
||||
|
||||
These add to the repository's [AGENTS.md](../../../AGENTS.md) for work in `ui/apps/chat`: **Llama Chat**, the chat client that llama-server bundles. **Llama UI**, the library, is `ui/lib/ui` (`@ggml-org/llama-ui`): Chat imports its tokens, fonts and base components from there (`@ggml-org/llama-ui/ui/<name>`) and keeps no copy of them. `DESIGN.md` lives here.
|
||||
|
||||
- **Design:** [DESIGN.md](DESIGN.md) holds the tokens, roles and rules, written for agents. Follow it for anything visual; when it and the CSS disagree, the CSS wins and DESIGN.md is updated.
|
||||
- **Components:** the `SKILL.md` next to a component folder (such as [src/lib/components/app/SKILL.md](src/lib/components/app/SKILL.md)) holds that folder's conventions.
|
||||
File diff suppressed because it is too large
Load Diff
+11
-11
@@ -209,7 +209,7 @@ ui/apps/chat/ → build → build/ui/apps/chat/dist/
|
||||
### SvelteKit Configuration
|
||||
|
||||
```javascript
|
||||
// svelte.config.js
|
||||
// vite.config.ts - options passed to sveltekit({ ... })
|
||||
adapter: adapter({
|
||||
pages: '../../../build/ui/apps/chat/dist', // Output directory
|
||||
assets: '../../../build/ui/apps/chat/dist', // Static assets
|
||||
@@ -339,7 +339,7 @@ flowchart TB
|
||||
|
||||
#### Components (`src/lib/components/`)
|
||||
|
||||
Components are organized in `app/` (application-specific) and `ui/` (shadcn-svelte primitives).
|
||||
Application components live in `app/`. The base components (the shadcn-svelte primitives: button, dialog, tooltip…), the design tokens, the fonts and the surfaces are Llama UI's, in [`ui/lib/ui`](../../lib/ui/README.md): the app imports them as `@ggml-org/llama-ui/ui/<name>` and compiles them from source as part of its own build, with no install or build of the library (see `scripts/llama-ui.js`).
|
||||
|
||||
**Chat Components** (`app/chat/`):
|
||||
|
||||
@@ -363,7 +363,7 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel
|
||||
| `DialogConfirmation` | Generic confirmation for destructive actions |
|
||||
| `DialogConversationRename` | Edit conversation title |
|
||||
|
||||
**Server/Model Components** (`app/server/`, `app/models/`):
|
||||
**Server/Model Components** (`app/servers/`, `app/models/`):
|
||||
|
||||
| Component | Responsibility |
|
||||
| ----------------------------------- | ------------------------------------------------------------------------------------------------------------- |
|
||||
@@ -688,14 +688,14 @@ llama-server serves the UI with `Cross-Origin-Embedder-Policy: require-corp` and
|
||||
|
||||
The sidebar stacks four surfaces, outermost first:
|
||||
|
||||
| Layer | Example | Corners |
|
||||
| ----- | ---------------------------------- | ---------------------------------- |
|
||||
| Page | the body background | none |
|
||||
| Panel | the sidebar | `rounded-2xl` |
|
||||
| Group | the sidebar's footer, inset in it | `rounded-(--surface-group-radius)` |
|
||||
| Row | Models, MCP or Settings in a group | `rounded-(--surface-row-radius)` |
|
||||
| Layer | Example | Corners |
|
||||
| ----- | ---------------------------------- | -------------------------------- |
|
||||
| Page | the body background | none |
|
||||
| Panel | the sidebar | `rounded-2xl` |
|
||||
| Group | the sidebar's footer, inset in it | `inset-group-surface` |
|
||||
| Row | Models, MCP or Settings in a group | `rounded-(--surface-row-radius)` |
|
||||
|
||||
A row keeps its corners concentric with its group's: its radius is the group's less the padding between their edges, so the curves run parallel instead of pinching at the corners. `app.css` holds the three tokens (`--surface-group-radius`, `--surface-group-padding` and the derived `--surface-row-radius`); change the group's radius or padding there and the rows follow. A group pads its rows with `p-(--surface-group-padding)`. `SidebarNavigationActions` rounds every row from the token, so the actions at the top of the sidebar (New chat, Search) match the footer's rows; a phone sizes its rows for a thumb instead. A selected row is raised and edged (`bg-background`, `shadow-sm`, `ring-1`), where a hovered one is only tinted, so the two never look alike. The group's hairline border is left out of the sum. `tests/client/nested-surfaces.svelte.test.ts` checks the rule on the rendered footer.
|
||||
A row keeps its corners concentric with its group's: its radius is the group's less the padding between their edges, so the curves run parallel instead of pinching at the corners. `app.css` holds the three tokens (`--surface-group-radius`, which follows the design system's `--inset-group-radius`, `--surface-group-padding` and the derived `--surface-row-radius`); change the group's radius or padding and the rows follow. A group pads its rows with `p-(--surface-group-padding)`. `SidebarNavigationActions` rounds every row from the token, so the actions at the top of the sidebar (New chat, Search) match the footer's rows; a phone sizes its rows for a thumb instead. A selected row takes `selected-row-surface`, the group's edge and shade around a tint, where a hovered one is only tinted, so the two never look alike. The group's hairline border is left out of the sum. `tests/client/nested-surfaces.svelte.test.ts` checks the rule on the rendered footer.
|
||||
|
||||
---
|
||||
|
||||
@@ -748,7 +748,7 @@ npm run check # TypeScript type checking
|
||||
ui/apps/chat/
|
||||
├── src/
|
||||
│ ├── lib/
|
||||
│ │ ├── components/ # UI components (app/, ui/)
|
||||
│ │ ├── components/ # Application components (base ones come from ui/lib/ui)
|
||||
│ │ ├── hooks/ # Svelte hooks
|
||||
│ │ ├── stores/ # State management
|
||||
│ │ ├── services/ # API and database services
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// For more info, see https://github.com/storybookjs/eslint-plugin-storybook#configuration-flat-config-format
|
||||
import svelteConfig from './svelte.config.js';
|
||||
import { includeIgnoreFile } from '@eslint/compat';
|
||||
import js from '@eslint/js';
|
||||
import { loadConfig } from '@sveltejs/load-config';
|
||||
import prettier from 'eslint-config-prettier';
|
||||
import perfectionist from 'eslint-plugin-perfectionist';
|
||||
import simpleImportSort from 'eslint-plugin-simple-import-sort';
|
||||
@@ -11,6 +11,7 @@ import globals from 'globals';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import ts from 'typescript-eslint';
|
||||
|
||||
const svelteConfig = (await loadConfig('./', { traverse: false }))?.config;
|
||||
const gitignorePath = fileURLToPath(new URL('./.gitignore', import.meta.url));
|
||||
// Require a blank line between sibling element-like nodes in a Svelte template
|
||||
// (elements, components, and the {#if} / {#each} / {#await} / {#snippet} /
|
||||
|
||||
Generated
+1952
-6547
File diff suppressed because it is too large
Load Diff
+39
-33
@@ -14,7 +14,7 @@
|
||||
"reset": "rm -rf .svelte-kit node_modules",
|
||||
"format": "eslint --fix . && prettier --write .",
|
||||
"format:files": "sh -c 'eslint --fix \"$@\" && prettier --write \"$@\"' sh",
|
||||
"lint": "prettier --check . && eslint .",
|
||||
"lint": "svelte-kit sync && prettier --check . && eslint .",
|
||||
"test": "npm run test:ui -- --run && npm run test:client -- --run && npm run test:unit -- --run && npm run test:e2e",
|
||||
"test:e2e": "playwright test",
|
||||
"test:e2e:pwa": "playwright test tests/e2e/pwa.e2e.ts",
|
||||
@@ -28,8 +28,8 @@
|
||||
"test:unit:pwa": "vitest --project=unit --run tests/unit/pwa.spec.ts",
|
||||
"test:pwa": "npm run test:unit:pwa && npm run test:e2e:pwa",
|
||||
"test:ui": "vitest --project=ui",
|
||||
"storybook": "storybook dev -p 6006",
|
||||
"build-storybook": "storybook build",
|
||||
"storybook": "svelte-kit sync && storybook dev -p 6006",
|
||||
"build-storybook": "svelte-kit sync && storybook build",
|
||||
"cleanup": "rm -rf .svelte-kit build node_modules test-results dist dev-dist debug-storybook.log static/pwa-*.png static/maskable-icon-*.png static/apple-touch-icon-*.png static/apple-splash-*.png static/favicon*.ico"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -39,28 +39,29 @@
|
||||
"@codemirror/language": "6.13.1",
|
||||
"@codemirror/state": "6.7.6",
|
||||
"@codemirror/view": "6.43.15",
|
||||
"@eslint/compat": "1.4.1",
|
||||
"@eslint/js": "9.39.2",
|
||||
"@eslint/compat": "2.1.1",
|
||||
"@eslint/js": "10.0.1",
|
||||
"@internationalized/date": "3.12.2",
|
||||
"@lezer/highlight": "1.2.5",
|
||||
"@lucide/svelte": "1.46.0",
|
||||
"@modelcontextprotocol/sdk": "1.32.1",
|
||||
"@playwright/test": "1.56.1",
|
||||
"@storybook/addon-a11y": "10.6.2",
|
||||
"@storybook/addon-docs": "10.6.2",
|
||||
"@storybook/addon-mcp": "10.6.2",
|
||||
"@storybook/addon-svelte-csf": "5.1.5",
|
||||
"@storybook/addon-vitest": "10.6.2",
|
||||
"@storybook/sveltekit": "10.6.2",
|
||||
"@sveltejs/adapter-static": "3.0.10",
|
||||
"@sveltejs/kit": "2.70.2",
|
||||
"@sveltejs/vite-plugin-svelte": "6.2.1",
|
||||
"@storybook/addon-a11y": "10.6.6",
|
||||
"@storybook/addon-docs": "10.6.6",
|
||||
"@storybook/addon-mcp": "10.6.6",
|
||||
"@storybook/addon-svelte-csf": "5.1.6",
|
||||
"@storybook/addon-vitest": "10.6.6",
|
||||
"@storybook/sveltekit": "10.6.6",
|
||||
"@sveltejs/adapter-static": "4.0.1",
|
||||
"@sveltejs/kit": "3.2.0",
|
||||
"@sveltejs/load-config": "0.2.3",
|
||||
"@sveltejs/vite-plugin-svelte": "7.3.1",
|
||||
"@tailwindcss/forms": "0.5.10",
|
||||
"@tailwindcss/typography": "0.5.20",
|
||||
"@tailwindcss/vite": "4.1.11",
|
||||
"@tailwindcss/vite": "4.3.5",
|
||||
"@types/mdast": "4.0.4",
|
||||
"@types/node": "24.13.0",
|
||||
"@vite-pwa/assets-generator": "1.0.4",
|
||||
"@vite-pwa/sveltekit": "1.1.0",
|
||||
"@vitest/browser": "4.1.12",
|
||||
"@vitest/browser-playwright": "4.1.12",
|
||||
"@vitest/coverage-v8": "4.1.12",
|
||||
@@ -68,18 +69,17 @@
|
||||
"clsx": "2.1.1",
|
||||
"dexie": "4.4.3",
|
||||
"dompurify": "3.4.17",
|
||||
"eslint": "9.39.4",
|
||||
"eslint": "10.14.0",
|
||||
"eslint-config-prettier": "10.1.8",
|
||||
"eslint-plugin-perfectionist": "^5.10.1",
|
||||
"eslint-plugin-simple-import-sort": "^14.0.0",
|
||||
"eslint-plugin-storybook": "10.6.2",
|
||||
"eslint-plugin-svelte": "3.19.0",
|
||||
"eslint-plugin-storybook": "10.6.6",
|
||||
"eslint-plugin-svelte": "3.24.1",
|
||||
"fflate": "0.8.3",
|
||||
"globals": "16.5.0",
|
||||
"highlight.js": "11.11.1",
|
||||
"http-server": "14.1.1",
|
||||
"llama-brand": "https://codeload.github.com/ggml-org/llama-brand/tar.gz/f05540643c1ca2a18d732ed6491c839686da9a3a",
|
||||
"mdast": "3.0.0",
|
||||
"@ggml-org/llama-brand": "https://codeload.github.com/ggml-org/llama-brand/tar.gz/f05540643c1ca2a18d732ed6491c839686da9a3a",
|
||||
"mdsvex": "0.12.7",
|
||||
"mermaid": "11.17.2",
|
||||
"mode-watcher": "1.1.0",
|
||||
@@ -98,32 +98,38 @@
|
||||
"remark-math": "6.0.0",
|
||||
"remark-rehype": "11.1.2",
|
||||
"sass": "1.100.0",
|
||||
"storybook": "10.6.2",
|
||||
"svelte": "5.56.1",
|
||||
"svelte-check": "4.6.0",
|
||||
"storybook": "10.6.6",
|
||||
"svelte": "5.57.3",
|
||||
"svelte-check": "4.7.8",
|
||||
"svelte-sonner": "1.1.1",
|
||||
"tailwind-merge": "3.6.0",
|
||||
"tailwind-variants": "3.3.1",
|
||||
"tailwindcss": "4.3.0",
|
||||
"tailwindcss": "4.3.5",
|
||||
"tw-animate-css": "1.4.0",
|
||||
"typescript": "5.9.3",
|
||||
"typescript-eslint": "8.60.1",
|
||||
"typescript": "6.0.3",
|
||||
"typescript-eslint": "8.72.1",
|
||||
"unified": "11.0.5",
|
||||
"unist-util-visit": "5.1.0",
|
||||
"uuid": "13.0.2",
|
||||
"vaul-svelte": "1.0.0-next.7",
|
||||
"vite": "7.3.7",
|
||||
"vite-plugin-devtools-json": "0.2.1",
|
||||
"vite": "8.3.6",
|
||||
"vite-plugin-devtools-json": "1.1.2",
|
||||
"vitest": "4.1.12",
|
||||
"vitest-browser-svelte": "2.1.1",
|
||||
"workbox-window": "7.4.1"
|
||||
"vitest-browser-svelte": "2.1.1"
|
||||
},
|
||||
"overrides": {
|
||||
"@storybook/svelte": "10.6.2",
|
||||
"@storybook/svelte": "10.6.6",
|
||||
"cookie": "1.1.1",
|
||||
"katex": "0.18.11",
|
||||
"http-server": {
|
||||
"html-encoding-sniffer": "6.0.0"
|
||||
},
|
||||
"katex": "0.19.0",
|
||||
"minimatch": "11.0.5",
|
||||
"postcss-selector-parser": "7.1.6",
|
||||
"sharp": "0.35.5",
|
||||
"valibot": "1.4.2"
|
||||
},
|
||||
"imports": {
|
||||
"#lib/*": "./src/lib/*"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,8 +10,8 @@ const require = createRequire(import.meta.url);
|
||||
|
||||
/** The web favicons llama-brand draws for the UI, optically centred, keyed by the file they become. */
|
||||
export const BRAND_FAVICONS = {
|
||||
[FAVICON_PATHS.SVG_DARK]: 'llama-brand/logo/svg/app/web/favicon-dark.svg',
|
||||
[FAVICON_PATHS.SVG_LIGHT]: 'llama-brand/logo/svg/app/web/favicon.svg'
|
||||
[FAVICON_PATHS.SVG_DARK]: '@ggml-org/llama-brand/logo/svg/app/web/favicon-dark.svg',
|
||||
[FAVICON_PATHS.SVG_LIGHT]: '@ggml-org/llama-brand/logo/svg/app/web/favicon.svg'
|
||||
} as const;
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
/**
|
||||
* Llama UI (@ggml-org/llama-ui, in ui/lib/ui) has no build step in the repo: this app compiles it
|
||||
* from source. vite.config.ts (through vite-plugin-llama-ui.ts) and tsconfig.json share what that
|
||||
* takes; tests/unit/llama-ui-tsconfig-paths.test.ts checks that tsconfig.json's paths match:
|
||||
*
|
||||
* - LLAMA_UI_EXPORTS maps the entry points of the package to their sources, as the exports of
|
||||
* its package.json map them to its build, so an import reads the same either way.
|
||||
* - LLAMA_UI_DEPENDENCIES lists what those sources import. They sit outside this folder, so
|
||||
* walking up from them would not find this app's node_modules, and would find a second copy
|
||||
* of Svelte if ui/lib/ui has its own install. Both configs resolve these names from this
|
||||
* app's node_modules instead.
|
||||
*/
|
||||
// with forward slashes on every platform, as Vite writes the paths of the modules it handles
|
||||
const root = fileURLToPath(new URL('../../../lib/ui/', import.meta.url)).replace(/\\/g, '/');
|
||||
const packageJson = JSON.parse(readFileSync(`${root}package.json`, 'utf8'));
|
||||
const source = `${root}src/lib`;
|
||||
/** [import specifier, source path]; `*` stands for one path segment or more. */
|
||||
const LLAMA_UI_EXPORTS = [
|
||||
['@ggml-org/llama-ui/ui/*', `${source}/components/ui/*/index.ts`],
|
||||
['@ggml-org/llama-ui/app/*', `${source}/components/app/*/index.ts`],
|
||||
['@ggml-org/llama-ui/models', `${source}/components/models/index.ts`],
|
||||
['@ggml-org/llama-ui/constants', `${source}/constants/index.ts`],
|
||||
['@ggml-org/llama-ui/enums', `${source}/enums/index.ts`],
|
||||
['@ggml-org/llama-ui/code', `${source}/utils/index.ts`],
|
||||
['@ggml-org/llama-ui/utils', `${source}/components/ui/utils.ts`],
|
||||
['@ggml-org/llama-ui/styles/*', `${source}/styles/*`],
|
||||
['@ggml-org/llama-ui/theme.css', `${source}/styles/theme.css`],
|
||||
['@ggml-org/llama-ui/tokens.css', `${source}/styles/tokens.css`],
|
||||
['@ggml-org/llama-ui/fonts.css', `${source}/styles/fonts.css`],
|
||||
['@ggml-org/llama-ui/surfaces.css', `${source}/styles/surfaces.css`],
|
||||
['@ggml-org/llama-ui', `${source}/index.ts`]
|
||||
];
|
||||
|
||||
/** The folder of the library, ending in a slash. */
|
||||
export const LLAMA_UI_ROOT = root;
|
||||
|
||||
/** The packages the library's sources import: its dependencies and peer dependencies. */
|
||||
export const LLAMA_UI_DEPENDENCIES = Object.keys({
|
||||
...packageJson.dependencies,
|
||||
...packageJson.peerDependencies
|
||||
}).sort();
|
||||
|
||||
/** The entry points as Vite aliases. */
|
||||
export const LLAMA_UI_ALIASES = LLAMA_UI_EXPORTS.map(([specifier, path]) => ({
|
||||
find: new RegExp(`^${specifier.replace(/[.]/g, '\\.').replace('*', '(.+)')}$`),
|
||||
replacement: path.replace('*', '$1')
|
||||
}));
|
||||
|
||||
/**
|
||||
* The entry points and the dependencies as TypeScript paths.
|
||||
* @param {string} nodeModules this app's node_modules, as the tsconfig that takes the paths sees it
|
||||
*/
|
||||
export function llamaUiPaths(nodeModules) {
|
||||
/** @type {Record<string, string[]>} */
|
||||
const paths = {};
|
||||
|
||||
for (const [specifier, path] of LLAMA_UI_EXPORTS) paths[specifier] = [path];
|
||||
|
||||
for (const name of LLAMA_UI_DEPENDENCIES) {
|
||||
paths[name] = [`${nodeModules}/${name}`];
|
||||
paths[`${name}/*`] = [`${nodeModules}/${name}/*`];
|
||||
}
|
||||
|
||||
// Svelte declares the subpaths with no file of their own (svelte/reactivity, svelte/attachments) in
|
||||
// types/index.d.ts, which only its package exports point at; without this fallback, a library source
|
||||
// that imports one would get its types from a second copy of Svelte in ui/lib/ui/node_modules
|
||||
paths['svelte/*'] = [`${nodeModules}/svelte/*`, `${nodeModules}/svelte/types/index.d.ts`];
|
||||
|
||||
// the icons sit behind lucide's package exports, which a path mapping does not read
|
||||
paths['@lucide/svelte/icons/*'] = [`${nodeModules}/@lucide/svelte/dist/icons/*`];
|
||||
|
||||
return paths;
|
||||
}
|
||||
@@ -3,8 +3,6 @@ import { existsSync, writeFileSync } from 'node:fs';
|
||||
import { resolve } from 'path';
|
||||
import type { Plugin } from 'vite';
|
||||
|
||||
let processed = false;
|
||||
|
||||
const OUTPUT_DIR = process.env.LLAMA_UI_OUT_DIR ?? BUILD_CONFIG.OUTPUT_DIR;
|
||||
|
||||
/**
|
||||
@@ -16,13 +14,10 @@ const OUTPUT_DIR = process.env.LLAMA_UI_OUT_DIR ?? BUILD_CONFIG.OUTPUT_DIR;
|
||||
export function buildInfoPlugin(): Plugin {
|
||||
return {
|
||||
apply: 'build',
|
||||
closeBundle() {
|
||||
setTimeout(() => {
|
||||
buildApp: {
|
||||
// post order runs after the SvelteKit adapter has written the output
|
||||
async handler() {
|
||||
try {
|
||||
if (processed) return;
|
||||
|
||||
processed = true;
|
||||
|
||||
const buildNumber = process.env.LLAMA_BUILD_NUMBER || 'b0000';
|
||||
const outDir = resolve(OUTPUT_DIR);
|
||||
const indexPath = resolve(outDir, 'index.html');
|
||||
@@ -36,7 +31,8 @@ export function buildInfoPlugin(): Plugin {
|
||||
} catch (error) {
|
||||
console.error('Failed to write build.json:', error);
|
||||
}
|
||||
}, 100);
|
||||
},
|
||||
order: 'post'
|
||||
},
|
||||
name: 'llamacpp:build-info'
|
||||
};
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { LLAMA_UI_ALIASES, LLAMA_UI_DEPENDENCIES, LLAMA_UI_ROOT } from './llama-ui.js';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { type Plugin, transformWithOxc } from 'vite';
|
||||
|
||||
const TYPESCRIPT_FILE = /\.ts$/;
|
||||
// what the library's sources need of a tsconfig: imports and exports stay as written
|
||||
const TSCONFIG = { compilerOptions: { verbatimModuleSyntax: true } };
|
||||
// the app's tsconfig.json, for every module and for the dependency scan alike
|
||||
const APP_TSCONFIG = fileURLToPath(new URL('../tsconfig.json', import.meta.url));
|
||||
const LIBRARY_FILE = new RegExp(`^${LLAMA_UI_ROOT.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`);
|
||||
|
||||
/**
|
||||
* Compiles Llama UI (@ggml-org/llama-ui, in ui/lib/ui) from its source as part of this app, with
|
||||
* no install or build of the library itself (see ./llama-ui.js for the why):
|
||||
*
|
||||
* - its entry points resolve to their sources,
|
||||
* - what those sources import resolves from this app's node_modules,
|
||||
* - the dev server may serve them,
|
||||
* - their TypeScript is stripped here, and the app's tsconfig.json applies to them. Vite would
|
||||
* read the tsconfig.json nearest to each file, and the library's extends one that only exists
|
||||
* once the library is installed.
|
||||
*/
|
||||
export function llamaUiPlugin(): Plugin {
|
||||
return {
|
||||
config() {
|
||||
return {
|
||||
// the scan looks for each file's nearest tsconfig on its own, see `tsconfig` below
|
||||
optimizeDeps: { rolldownOptions: { tsconfig: APP_TSCONFIG } },
|
||||
oxc: { exclude: [/\.js$/, LIBRARY_FILE] },
|
||||
resolve: { alias: LLAMA_UI_ALIASES, dedupe: LLAMA_UI_DEPENDENCIES },
|
||||
server: { fs: { allow: [LLAMA_UI_ROOT] } },
|
||||
// one tsconfig for every module, instead of the nearest one, which for the library's
|
||||
// sources extends a config only its own install has
|
||||
tsconfig: APP_TSCONFIG
|
||||
};
|
||||
},
|
||||
enforce: 'pre',
|
||||
name: 'llama-ui',
|
||||
async transform(code, id) {
|
||||
const file = id.split('?')[0];
|
||||
|
||||
if (!LIBRARY_FILE.test(file) || !TYPESCRIPT_FILE.test(file)) return;
|
||||
|
||||
const result = await transformWithOxc(code, file, {
|
||||
lang: 'ts',
|
||||
sourcemap: true,
|
||||
tsconfig: TSCONFIG
|
||||
});
|
||||
|
||||
return { code: result.code, map: result.map };
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { HEADERS } from '../src/lib/constants/headers.constants';
|
||||
import {
|
||||
PWA_FILE_PATHS,
|
||||
PWA_MANIFEST,
|
||||
PWA_MANIFEST_PLUGIN
|
||||
} from '../src/lib/constants/pwa.constants';
|
||||
import type { Plugin } from 'vite';
|
||||
|
||||
const MANIFEST_SOURCE = JSON.stringify(PWA_MANIFEST);
|
||||
// the build writes files relative to its output directory: the path without its leading slash
|
||||
const MANIFEST_FILE = PWA_FILE_PATHS.MANIFEST.slice(1);
|
||||
|
||||
/**
|
||||
* Serve manifest.webmanifest in dev and emit it next to the client build output.
|
||||
* The adapter copies the client output, so the file lands at the root of the build.
|
||||
*/
|
||||
export function pwaManifestPlugin(): Plugin {
|
||||
return {
|
||||
configureServer(server) {
|
||||
server.middlewares.use(PWA_FILE_PATHS.MANIFEST, (_req, res) => {
|
||||
res.setHeader(HEADERS.CONTENT_TYPE, PWA_MANIFEST_PLUGIN.CONTENT_TYPE);
|
||||
res.end(MANIFEST_SOURCE);
|
||||
});
|
||||
},
|
||||
generateBundle() {
|
||||
if (this.environment.name !== PWA_MANIFEST_PLUGIN.ENVIRONMENT) return;
|
||||
|
||||
this.emitFile({ fileName: MANIFEST_FILE, source: MANIFEST_SOURCE, type: 'asset' });
|
||||
},
|
||||
name: PWA_MANIFEST_PLUGIN.NAME
|
||||
};
|
||||
}
|
||||
@@ -3,8 +3,6 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { resolve } from 'path';
|
||||
import type { Plugin } from 'vite';
|
||||
|
||||
let processed = false;
|
||||
|
||||
const OUTPUT_DIR = process.env.LLAMA_UI_OUT_DIR ?? BUILD_CONFIG.OUTPUT_DIR;
|
||||
|
||||
function rewrite(path: string, pairs: [string, string][]): void {
|
||||
@@ -31,35 +29,25 @@ function rewrite(path: string, pairs: [string, string][]): void {
|
||||
* SvelteKit bakes root absolute /_app/ paths into the SPA fallback because paths.relative
|
||||
* does not apply to a depth agnostic fallback page. Rewriting to ./_app/ lets a plain
|
||||
* recursive copy of the output into /any/subdir/ resolve assets against the document URL.
|
||||
* Runs after adapter-static writes index.html and the PWA plugin writes sw.js, deferred the
|
||||
* same way as buildInfoPlugin so the emitted files exist.
|
||||
* Runs after adapter-static writes index.html.
|
||||
*/
|
||||
export function relativizeBasePlugin(): Plugin {
|
||||
return {
|
||||
apply: 'build',
|
||||
closeBundle() {
|
||||
setTimeout(() => {
|
||||
buildApp: {
|
||||
// post order runs after the SvelteKit adapter has written the output
|
||||
async handler() {
|
||||
try {
|
||||
if (processed) return;
|
||||
|
||||
processed = true;
|
||||
|
||||
const outDir = resolve(OUTPUT_DIR);
|
||||
|
||||
// index.html: modulepreload, stylesheet and bootstrap import reference "/_app/
|
||||
rewrite(resolve(outDir, 'index.html'), [['"/_app/', '"./_app/']]);
|
||||
|
||||
// sw.js: the only absolute entries are the navigate fallback precache key and handler
|
||||
rewrite(resolve(outDir, 'sw.js'), [
|
||||
['{url:"/"', '{url:"./"'],
|
||||
['createHandlerBoundToURL("/"', 'createHandlerBoundToURL("./"']
|
||||
]);
|
||||
|
||||
console.log('Relativized base refs in index.html and sw.js');
|
||||
console.log('Relativized base refs in index.html');
|
||||
} catch (error) {
|
||||
console.error('Failed to relativize base refs:', error);
|
||||
}
|
||||
}, 100);
|
||||
},
|
||||
order: 'post'
|
||||
},
|
||||
name: 'llamacpp:relativize-base'
|
||||
};
|
||||
|
||||
@@ -11,8 +11,6 @@ import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { resolve } from 'path';
|
||||
import type { Plugin } from 'vite';
|
||||
|
||||
let processed = false;
|
||||
|
||||
const OUTPUT_DIR = process.env.LLAMA_UI_OUT_DIR ?? BUILD_CONFIG.OUTPUT_DIR;
|
||||
|
||||
/**
|
||||
@@ -85,13 +83,10 @@ export function generateSplashScreenLinks(outDir: string): string[] {
|
||||
export function splashScreenPlugin(): Plugin {
|
||||
return {
|
||||
apply: 'build',
|
||||
closeBundle() {
|
||||
setTimeout(() => {
|
||||
buildApp: {
|
||||
// post order runs after the SvelteKit adapter has written the output
|
||||
async handler() {
|
||||
try {
|
||||
if (processed) return;
|
||||
|
||||
processed = true;
|
||||
|
||||
const outDir = resolve(OUTPUT_DIR);
|
||||
const indexPath = resolve(outDir, 'index.html');
|
||||
|
||||
@@ -123,7 +118,8 @@ export function splashScreenPlugin(): Plugin {
|
||||
} catch (error) {
|
||||
console.error('Failed to process build output:', error);
|
||||
}
|
||||
}, 100);
|
||||
},
|
||||
order: 'post'
|
||||
},
|
||||
name: 'llamacpp:splash-screen'
|
||||
};
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
set(UI_SOURCE_GLOBS
|
||||
src/*
|
||||
static/*
|
||||
# Llama UI, which the app compiles from source
|
||||
../../lib/ui/src/lib/*
|
||||
)
|
||||
|
||||
set(UI_SOURCE_FILES
|
||||
@@ -10,7 +12,9 @@ set(UI_SOURCE_FILES
|
||||
package-lock.json
|
||||
src/.gitignore
|
||||
vite.config.ts
|
||||
svelte.config.js
|
||||
tsconfig.json
|
||||
scripts/llama-ui.js
|
||||
scripts/vite-plugin-llama-ui.ts
|
||||
../../lib/ui/package.json
|
||||
scripts/vite-plugin-llama-cpp-build.ts
|
||||
)
|
||||
|
||||
+12
-150
@@ -5,39 +5,15 @@
|
||||
@plugin '@tailwindcss/forms';
|
||||
@plugin '@tailwindcss/typography';
|
||||
@import 'tw-animate-css';
|
||||
/* the brand's colours, which Llama UI's primitives build on */
|
||||
@import '@ggml-org/llama-brand/css/colors.css';
|
||||
/* Llama UI's theme (ui/lib/ui): fonts, tokens, surfaces, the shadcn names and the base styles
|
||||
its components read. By path rather than as @ggml-org/llama-ui/theme.css: prettier's Tailwind
|
||||
plugin reads this file too, and it knows no aliases */
|
||||
@import '../../../lib/ui/src/lib/styles/theme.css';
|
||||
@import './lib/styles/safe-area.css';
|
||||
|
||||
@custom-variant dark (&:is(.dark *));
|
||||
|
||||
/* shadcn components key their orientation styles off these, toggle-group among them */
|
||||
@custom-variant data-horizontal (&[data-orientation='horizontal']);
|
||||
@custom-variant data-vertical (&[data-orientation='vertical']);
|
||||
|
||||
/*
|
||||
* Tailwind v4 dropped the pointer cursor from buttons. Every control a user can
|
||||
* press gets it back, and anything disabled keeps the cursor its own class asks for.
|
||||
*/
|
||||
@layer base {
|
||||
button:not(:disabled),
|
||||
[role='button']:not([data-disabled]),
|
||||
[role='checkbox']:not([data-disabled]),
|
||||
[role='combobox']:not([data-disabled]),
|
||||
[role='menuitem']:not([data-disabled]),
|
||||
[role='menuitemcheckbox']:not([data-disabled]),
|
||||
[role='menuitemradio']:not([data-disabled]),
|
||||
[role='option']:not([data-disabled]),
|
||||
[role='radio']:not([data-disabled]),
|
||||
[role='switch']:not([data-disabled]),
|
||||
[role='tab']:not([data-disabled]),
|
||||
label[for],
|
||||
select,
|
||||
summary {
|
||||
cursor: pointer;
|
||||
}
|
||||
}
|
||||
|
||||
:root {
|
||||
--radius: 0.625rem;
|
||||
/* the page runs under the status bar and over the home indicator
|
||||
(viewport-fit=cover); what is pinned to a screen edge keeps clear of them
|
||||
with the pt-safe / pb-safe utilities (see lib/styles/safe-area.css) */
|
||||
@@ -49,39 +25,6 @@
|
||||
--page-top: calc(var(--corner-control-top) + 3.75rem);
|
||||
/* expanded desktop sidebar width, overridden once the user drags its edge */
|
||||
--sidebar-width: 18rem;
|
||||
--background: oklch(1 0 0);
|
||||
--foreground: oklch(0.145 0 0);
|
||||
--card: oklch(1 0 0);
|
||||
--card-foreground: oklch(0.145 0 0);
|
||||
--popover: oklch(1 0 0);
|
||||
--popover-foreground: oklch(0.145 0 0);
|
||||
--primary: oklch(0.205 0 0);
|
||||
--primary-foreground: oklch(0.985 0 0);
|
||||
--secondary: oklch(0.95 0 0);
|
||||
--secondary-foreground: oklch(0.205 0 0);
|
||||
--muted: oklch(0.97 0 0);
|
||||
--muted-foreground: oklch(0.556 0 0);
|
||||
--accent: oklch(0.95 0 0);
|
||||
--accent-foreground: oklch(0.205 0 0);
|
||||
--destructive: oklch(0.577 0.245 27.325);
|
||||
--border: oklch(0.875 0 0);
|
||||
--input: oklch(0.92 0 0);
|
||||
--ring: oklch(0.708 0 0);
|
||||
--chart-1: oklch(0.646 0.222 41.116);
|
||||
--chart-2: oklch(0.6 0.118 184.704);
|
||||
--chart-3: oklch(0.398 0.07 227.392);
|
||||
--chart-4: oklch(0.828 0.189 84.429);
|
||||
--chart-5: oklch(0.769 0.188 70.08);
|
||||
--sidebar: oklch(0.985 0 0);
|
||||
--sidebar-foreground: oklch(0.145 0 0);
|
||||
--sidebar-primary: oklch(0.205 0 0);
|
||||
--sidebar-primary-foreground: oklch(0.985 0 0);
|
||||
--sidebar-accent: oklch(0.97 0 0);
|
||||
--sidebar-accent-foreground: oklch(0.205 0 0);
|
||||
--sidebar-border: oklch(0.922 0 0);
|
||||
--sidebar-ring: oklch(0.708 0 0);
|
||||
--code-background: oklch(0.985 0 0);
|
||||
--code-foreground: oklch(0.145 0 0);
|
||||
/* phone sidebar width, which the page slides aside to reveal */
|
||||
--sidebar-reveal: min(80dvw, 20rem);
|
||||
/* the desktop sidebar's muted/60 over the page, made opaque so the slid page's corners can match it */
|
||||
@@ -96,9 +39,6 @@
|
||||
var(--sidebar-reveal-tint) calc(var(--sidebar-reveal-progress) * 100%),
|
||||
var(--background)
|
||||
);
|
||||
--font-mono:
|
||||
ui-monospace, SFMono-Regular, 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas,
|
||||
'Liberation Mono', Menlo, monospace;
|
||||
--layer-popover: 1000000;
|
||||
|
||||
--chat-form-area-height: 8rem;
|
||||
@@ -110,7 +50,7 @@
|
||||
group's: its radius is the group's less the padding between their edges. The sidebar's
|
||||
actions above the footer share the row radius, so every action row rounds alike. See "Nested
|
||||
surfaces" in the README; tests/client/nested-surfaces.svelte.test.ts holds the rule */
|
||||
--surface-group-radius: 1rem;
|
||||
--surface-group-radius: var(--inset-group-radius);
|
||||
--surface-group-padding: 0.25rem;
|
||||
--surface-row-radius: calc(var(--surface-group-radius) - var(--surface-group-padding));
|
||||
|
||||
@@ -150,86 +90,12 @@
|
||||
|
||||
.dark {
|
||||
--chat-form-sheet-fill: color-mix(in oklab, var(--muted) 75%, var(--background));
|
||||
--background: oklch(0.16 0 0);
|
||||
--foreground: oklch(0.985 0 0);
|
||||
--card: oklch(0.205 0 0);
|
||||
--card-foreground: oklch(0.985 0 0);
|
||||
--popover: oklch(0.205 0 0);
|
||||
--popover-foreground: oklch(0.985 0 0);
|
||||
--primary: oklch(0.922 0 0);
|
||||
--primary-foreground: oklch(0.205 0 0);
|
||||
--secondary: oklch(0.29 0 0);
|
||||
--secondary-foreground: oklch(0.985 0 0);
|
||||
--muted: oklch(0.269 0 0);
|
||||
--muted-foreground: oklch(0.708 0 0);
|
||||
--accent: oklch(0.269 0 0);
|
||||
--accent-foreground: oklch(0.985 0 0);
|
||||
--destructive: oklch(0.704 0.191 22.216);
|
||||
--border: oklch(1 0 0 / 30%);
|
||||
--input: oklch(1 0 0 / 30%);
|
||||
--ring: oklch(0.556 0 0);
|
||||
--chart-1: oklch(0.488 0.243 264.376);
|
||||
--chart-2: oklch(0.696 0.17 162.48);
|
||||
--chart-3: oklch(0.769 0.188 70.08);
|
||||
--chart-4: oklch(0.627 0.265 303.9);
|
||||
--chart-5: oklch(0.645 0.246 16.439);
|
||||
--sidebar: oklch(0.2 0 0);
|
||||
--sidebar-foreground: oklch(0.985 0 0);
|
||||
--sidebar-primary: oklch(0.488 0.243 264.376);
|
||||
--sidebar-primary-foreground: oklch(0.985 0 0);
|
||||
--sidebar-accent: oklch(0.269 0 0);
|
||||
--sidebar-accent-foreground: oklch(0.985 0 0);
|
||||
--sidebar-border: oklch(1 0 0 / 10%);
|
||||
--sidebar-ring: oklch(0.556 0 0);
|
||||
--code-background: oklch(0.225 0 0);
|
||||
--code-foreground: oklch(0.875 0 0);
|
||||
}
|
||||
|
||||
@theme inline {
|
||||
--radius-sm: calc(var(--radius) - 4px);
|
||||
--radius-md: calc(var(--radius) - 2px);
|
||||
--radius-lg: var(--radius);
|
||||
--radius-xl: calc(var(--radius) + 4px);
|
||||
--color-background: var(--background);
|
||||
--color-foreground: var(--foreground);
|
||||
--color-card: var(--card);
|
||||
--color-card-foreground: var(--card-foreground);
|
||||
--color-popover: var(--popover);
|
||||
--color-popover-foreground: var(--popover-foreground);
|
||||
--color-primary: var(--primary);
|
||||
--color-primary-foreground: var(--primary-foreground);
|
||||
--color-secondary: var(--secondary);
|
||||
--color-secondary-foreground: var(--secondary-foreground);
|
||||
--color-muted: var(--muted);
|
||||
--color-muted-foreground: var(--muted-foreground);
|
||||
--color-accent: var(--accent);
|
||||
--color-accent-foreground: var(--accent-foreground);
|
||||
--color-destructive: var(--destructive);
|
||||
--color-border: var(--border);
|
||||
--color-input: var(--input);
|
||||
--color-ring: var(--ring);
|
||||
--color-chart-1: var(--chart-1);
|
||||
--color-chart-2: var(--chart-2);
|
||||
--color-chart-3: var(--chart-3);
|
||||
--color-chart-4: var(--chart-4);
|
||||
--color-chart-5: var(--chart-5);
|
||||
--color-sidebar: var(--sidebar);
|
||||
--color-sidebar-foreground: var(--sidebar-foreground);
|
||||
--color-sidebar-primary: var(--sidebar-primary);
|
||||
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
|
||||
--color-sidebar-accent: var(--sidebar-accent);
|
||||
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
|
||||
--color-sidebar-border: var(--sidebar-border);
|
||||
--color-sidebar-ring: var(--sidebar-ring);
|
||||
}
|
||||
|
||||
@layer base {
|
||||
* {
|
||||
@apply border-border outline-ring/50;
|
||||
}
|
||||
|
||||
/* over the theme's page background: it blends into the sidebar tint as the phone sidebar opens */
|
||||
body {
|
||||
@apply bg-(--sidebar-reveal-bg) text-foreground antialiased;
|
||||
@apply bg-(--sidebar-reveal-bg);
|
||||
scrollbar-width: thin;
|
||||
scrollbar-gutter: stable;
|
||||
}
|
||||
@@ -292,7 +158,7 @@
|
||||
}
|
||||
|
||||
*:hover {
|
||||
scrollbar-color: color-mix(in oklch, var(--muted-foreground) 30%, transparent) transparent;
|
||||
scrollbar-color: color-mix(in oklch, var(--fg-muted) 30%, transparent) transparent;
|
||||
}
|
||||
|
||||
*::-webkit-scrollbar {
|
||||
@@ -311,15 +177,11 @@
|
||||
}
|
||||
|
||||
*:hover::-webkit-scrollbar-thumb {
|
||||
background: color-mix(in oklch, var(--muted-foreground) 30%, transparent);
|
||||
background: color-mix(in oklch, var(--fg-muted) 30%, transparent);
|
||||
}
|
||||
|
||||
*::-webkit-scrollbar-thumb:hover {
|
||||
background: color-mix(in oklch, var(--muted-foreground) 50%, transparent);
|
||||
}
|
||||
|
||||
:where(code, pre, kbd, samp) {
|
||||
font-family: var(--font-mono);
|
||||
background: color-mix(in oklch, var(--fg-muted) 50%, transparent);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Vendored
+2
-4
@@ -1,9 +1,7 @@
|
||||
// See https://svelte.dev/docs/kit/types#app.d.ts
|
||||
// for information about these interfaces
|
||||
|
||||
import 'vite-plugin-pwa/pwa-assets';
|
||||
import 'vite-plugin-pwa/svelte';
|
||||
import { ModelModality, ServerModelStatus, ServerRole } from '$lib/enums';
|
||||
import { ModelModality, ServerModelStatus, ServerRole } from '#lib/enums/index.ts';
|
||||
// Import chat types from dedicated module
|
||||
import type {
|
||||
// API types
|
||||
@@ -57,7 +55,7 @@ import type {
|
||||
SettingsConfigType,
|
||||
SettingsConfigValue,
|
||||
SettingsFieldConfig
|
||||
} from '$lib/types';
|
||||
} from '#lib/types/index.ts';
|
||||
|
||||
declare global {
|
||||
// namespace App {
|
||||
|
||||
@@ -1,18 +1,49 @@
|
||||
<script lang="ts">
|
||||
import ActionIcon from './ActionIcon.svelte';
|
||||
import { Copy } from '@lucide/svelte';
|
||||
import { ICON_CLASS_DEFAULT } from '$lib/constants';
|
||||
import { copyToClipboard } from '$lib/utils';
|
||||
import { ActionIcon } from '@ggml-org/llama-ui/app/actions';
|
||||
import { Check, Copy } from '@lucide/svelte';
|
||||
import { COPY_FEEDBACK_RESET_MS, ICON_CLASS_DEFAULT } from '#lib/constants/index.ts';
|
||||
import { copyToClipboard } from '#lib/utils/index.ts';
|
||||
|
||||
export let ariaLabel: string = 'Copy to clipboard';
|
||||
export let canCopy: boolean = true;
|
||||
export let text: string;
|
||||
interface Props {
|
||||
ariaLabel?: string;
|
||||
canCopy?: boolean;
|
||||
text: string;
|
||||
}
|
||||
|
||||
let { ariaLabel = 'Copy to clipboard', canCopy = true, text }: Props = $props();
|
||||
|
||||
// after a copy the icon turns into a green check for a moment, then back
|
||||
let isCopied = $state(false);
|
||||
// the swap pops in only after the first copy, never on the initial render
|
||||
let hasSwapped = $state(false);
|
||||
let resetTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
let iconClass = $derived(
|
||||
[
|
||||
ICON_CLASS_DEFAULT,
|
||||
isCopied && 'text-fg-success',
|
||||
hasSwapped && 'animate-in fade-in zoom-in-75 duration-150 motion-reduce:animate-none'
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
);
|
||||
|
||||
async function copy() {
|
||||
if (!canCopy || !(await copyToClipboard(text))) return;
|
||||
|
||||
clearTimeout(resetTimer);
|
||||
isCopied = true;
|
||||
hasSwapped = true;
|
||||
resetTimer = setTimeout(() => (isCopied = false), COPY_FEEDBACK_RESET_MS);
|
||||
}
|
||||
|
||||
$effect(() => () => clearTimeout(resetTimer));
|
||||
</script>
|
||||
|
||||
<ActionIcon
|
||||
disabled={!canCopy}
|
||||
icon={Copy}
|
||||
iconSize={ICON_CLASS_DEFAULT}
|
||||
onclick={() => canCopy && copyToClipboard(text)}
|
||||
icon={isCopied ? Check : Copy}
|
||||
iconSize={iconClass}
|
||||
onclick={copy}
|
||||
tooltip={ariaLabel}
|
||||
/>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
<script lang="ts">
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { deviceStore } from '$lib/stores';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { GLASS_KEY_CLASS } from '#lib/constants/index.ts';
|
||||
import { deviceStore } from '#lib/stores/index.ts';
|
||||
import type { Component, Snippet } from 'svelte';
|
||||
import type { HTMLButtonAttributes } from 'svelte/elements';
|
||||
|
||||
@@ -33,11 +34,7 @@
|
||||
{...props}
|
||||
{...rest}
|
||||
aria-label={label}
|
||||
class={[
|
||||
'inline-flex shrink-0 cursor-pointer items-center justify-center rounded-full border border-border/20 bg-muted/60 text-accent-foreground shadow-sm backdrop-blur-sm transition-colors hover:bg-muted',
|
||||
size === 'md' ? 'size-10' : 'size-9',
|
||||
className
|
||||
]}
|
||||
class={[GLASS_KEY_CLASS, size === 'md' ? 'size-10' : 'size-9', className]}
|
||||
type="button"
|
||||
>
|
||||
{#if Icon}
|
||||
@@ -48,7 +45,7 @@
|
||||
</button>
|
||||
{/snippet}
|
||||
|
||||
<!-- a round key over the page: the menu and context gauge keys at a phone's top corners, and the
|
||||
<!-- a round matte glass key: the menu and context gauge keys at a phone's top corners, and the
|
||||
key back to the bottom of a chat -->
|
||||
{#if tooltip && !deviceStore.isMobile}
|
||||
<Tooltip.Root>
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
*/
|
||||
|
||||
/** Styled icon button for action triggers with tooltip. */
|
||||
export { default as ActionIcon } from './ActionIcon.svelte';
|
||||
export { ActionIcon } from '@ggml-org/llama-ui/app/actions';
|
||||
|
||||
/** Copy-to-clipboard icon button with clipboard logic. */
|
||||
export { default as ActionIconCopyToClipboard } from './ActionIconCopyToClipboard.svelte';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { MODALITY_ICONS, MODALITY_LABELS } from '$lib/constants';
|
||||
import { ModelModality } from '$lib/enums';
|
||||
import { MODALITY_ICONS, MODALITY_LABELS } from '#lib/constants/index.ts';
|
||||
import { ModelModality } from '#lib/enums/index.ts';
|
||||
|
||||
interface Props {
|
||||
modalities: ModelModality[];
|
||||
@@ -22,7 +22,7 @@
|
||||
className
|
||||
]}
|
||||
>
|
||||
<ModalityIcon class="h-3 w-3" />
|
||||
<ModalityIcon class="size-icon-xs" />
|
||||
|
||||
{MODALITY_LABELS[modality]}
|
||||
</span>
|
||||
|
||||
+4
-4
@@ -4,10 +4,10 @@
|
||||
DialogChatAttachmentsPreview,
|
||||
DialogMcpResourcePreview,
|
||||
ScrollCarousel
|
||||
} from '$lib/components/app';
|
||||
import { ScrollCarouselVariant } from '$lib/enums';
|
||||
import type { DatabaseMessageExtraMcpResource } from '$lib/types';
|
||||
import { getAttachmentDisplayItems, isMcpPrompt, isMcpResource } from '$lib/utils';
|
||||
} from '#lib/components/app/index.ts';
|
||||
import { ScrollCarouselVariant } from '#lib/enums/index.ts';
|
||||
import type { DatabaseMessageExtraMcpResource } from '#lib/types/index.ts';
|
||||
import { getAttachmentDisplayItems, isMcpPrompt, isMcpResource } from '#lib/utils/index.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
|
||||
+4
-4
@@ -4,15 +4,15 @@
|
||||
ChatAttachmentsListItemMcpResource,
|
||||
ChatAttachmentsListItemThumbnailFile,
|
||||
ChatAttachmentsListItemThumbnailImage
|
||||
} from '$lib/components/app';
|
||||
import { AttachmentType } from '$lib/enums';
|
||||
} from '#lib/components/app/index.ts';
|
||||
import { AttachmentType } from '#lib/enums/index.ts';
|
||||
import type {
|
||||
ChatAttachmentDisplayItem,
|
||||
DatabaseMessageExtraMcpPrompt,
|
||||
DatabaseMessageExtraMcpResource,
|
||||
MCPResourceAttachment
|
||||
} from '$lib/types';
|
||||
import { isMcpPrompt, isMcpResource, isPdfFile } from '$lib/utils';
|
||||
} from '#lib/types/index.ts';
|
||||
import { isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
|
||||
+3
-3
@@ -1,8 +1,8 @@
|
||||
<script lang="ts">
|
||||
import { X } from '@lucide/svelte';
|
||||
import { ActionIcon, ChatMessageMcpPromptContent } from '$lib/components/app';
|
||||
import { McpPromptVariant } from '$lib/enums';
|
||||
import type { DatabaseMessageExtraMcpPrompt } from '$lib/types';
|
||||
import { ActionIcon, ChatMessageMcpPromptContent } from '#lib/components/app/index.ts';
|
||||
import { McpPromptVariant } from '#lib/enums/index.ts';
|
||||
import type { DatabaseMessageExtraMcpPrompt } from '#lib/types/index.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
|
||||
+11
-11
@@ -1,11 +1,11 @@
|
||||
<script lang="ts">
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { AlertCircle, Loader2 } from '@lucide/svelte';
|
||||
import { X } from '@lucide/svelte';
|
||||
import { ActionIcon } from '$lib/components/app';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { mcpStore } from '$lib/stores';
|
||||
import type { MCPResourceAttachment } from '$lib/types';
|
||||
import { getResourceDisplayName, getResourceIcon } from '$lib/utils';
|
||||
import { ActionIcon } from '#lib/components/app/index.ts';
|
||||
import { mcpStore } from '#lib/stores/index.ts';
|
||||
import type { MCPResourceAttachment } from '#lib/types/index.ts';
|
||||
import { getResourceDisplayName, getResourceIcon } from '#lib/utils/index.ts';
|
||||
|
||||
interface Props {
|
||||
attachment: MCPResourceAttachment;
|
||||
@@ -23,7 +23,7 @@
|
||||
const favicon = $derived(mcpStore.getServerFavicon(attachment.resource.serverName));
|
||||
|
||||
function getStatusClass(attachment: MCPResourceAttachment): string {
|
||||
if (attachment.error) return 'border-red-500/50 bg-red-500/10';
|
||||
if (attachment.error) return 'border-line-danger-soft bg-surface-danger';
|
||||
|
||||
if (attachment.loading) return 'border-border/50 bg-muted/30';
|
||||
|
||||
@@ -45,11 +45,11 @@
|
||||
type="button"
|
||||
>
|
||||
{#if attachment.loading}
|
||||
<Loader2 class="h-3 w-3 animate-spin text-muted-foreground" />
|
||||
<Loader2 class="size-icon-xs animate-spin text-muted-foreground" />
|
||||
{:else if attachment.error}
|
||||
<AlertCircle class="h-3 w-3 text-red-500" />
|
||||
<AlertCircle class="size-icon-xs text-fg-danger" />
|
||||
{:else}
|
||||
<ResourceIcon class="h-3 w-3 text-muted-foreground" />
|
||||
<ResourceIcon class="size-icon-xs text-muted-foreground" />
|
||||
{/if}
|
||||
|
||||
<span class="max-w-[150px] truncate text-xs">
|
||||
@@ -60,7 +60,7 @@
|
||||
<ActionIcon
|
||||
class="-my-2 -mr-1.5 bg-transparent"
|
||||
icon={X}
|
||||
iconSize="h-2 w-2"
|
||||
iconSize="size-icon-md"
|
||||
onclick={() => onRemove?.(attachment.id)}
|
||||
stopPropagationOnClick
|
||||
tooltip="Remove"
|
||||
@@ -74,7 +74,7 @@
|
||||
{#if favicon}
|
||||
<img
|
||||
alt={attachment.resource.serverName}
|
||||
class="h-3 w-3 shrink-0 rounded-sm"
|
||||
class="size-3 shrink-0 rounded-sm"
|
||||
onerror={(e) => {
|
||||
(e.currentTarget as HTMLImageElement).style.display = 'none';
|
||||
}}
|
||||
|
||||
+6
-6
@@ -1,8 +1,8 @@
|
||||
<script lang="ts">
|
||||
import { Music, Video, X } from '@lucide/svelte';
|
||||
import { ActionIcon } from '$lib/components/app';
|
||||
import { ICON_CLASS_DEFAULT } from '$lib/constants';
|
||||
import { AttachmentType } from '$lib/enums';
|
||||
import { ActionIcon } from '#lib/components/app/index.ts';
|
||||
import { ICON_CLASS_DEFAULT } from '#lib/constants/index.ts';
|
||||
import { AttachmentType } from '#lib/enums/index.ts';
|
||||
import {
|
||||
formatFileSize,
|
||||
getFileTypeLabel,
|
||||
@@ -11,7 +11,7 @@
|
||||
isPdfFile,
|
||||
isTextFile,
|
||||
isVideoFile
|
||||
} from '$lib/utils';
|
||||
} from '#lib/utils/index.ts';
|
||||
|
||||
interface Props {
|
||||
attachment?: DatabaseMessageExtra;
|
||||
@@ -107,7 +107,7 @@
|
||||
|
||||
{#snippet fileIcon()}
|
||||
<div
|
||||
class="flex h-8 w-8 items-center justify-center rounded bg-primary/10 text-xs font-medium text-primary"
|
||||
class="flex size-8 items-center justify-center rounded bg-primary/10 text-xs font-medium text-primary"
|
||||
>
|
||||
{#if isAudio}
|
||||
<Music class="{ICON_CLASS_DEFAULT} text-white/70" />
|
||||
@@ -129,7 +129,7 @@
|
||||
<button
|
||||
aria-label={readonly ? `Preview ${name}` : undefined}
|
||||
class="rounded-lg border border-border bg-muted p-3 {className} cursor-pointer {readonly
|
||||
? 'w-full max-w-2xl transition-shadow hover:shadow-md'
|
||||
? 'w-full max-w-2xl transition-shadow hover:shadow-(--elevation-surface)'
|
||||
: `group relative text-left ${textContent ? 'max-h-24 max-w-72' : 'max-w-36'}`} overflow-hidden"
|
||||
{onclick}
|
||||
type="button"
|
||||
|
||||
+4
-6
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { X } from '@lucide/svelte';
|
||||
import { ActionIcon } from '$lib/components/app';
|
||||
import { ActionIcon } from '#lib/components/app/index.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
@@ -35,13 +35,11 @@
|
||||
<img alt={name} class="{height} {width} cursor-pointer object-cover {imageClass}" src={preview} />
|
||||
{/snippet}
|
||||
|
||||
<div
|
||||
class="group relative min-w-0 overflow-hidden rounded-lg bg-muted shadow-lg dark:border dark:border-muted {className}"
|
||||
>
|
||||
<div class="group relative min-w-0 overflow-hidden rounded-lg bg-muted elevated-raised {className}">
|
||||
{#if onclick}
|
||||
<button
|
||||
aria-label="Preview {name}"
|
||||
class="block h-full w-full rounded-lg focus:ring-2 focus:ring-primary focus:ring-offset-2 focus:outline-none"
|
||||
class="block size-full rounded-lg focus:ring-2 focus:ring-primary focus:ring-offset-2 focus:outline-none"
|
||||
{onclick}
|
||||
type="button"
|
||||
>
|
||||
@@ -53,7 +51,7 @@
|
||||
|
||||
{#if label}
|
||||
<span
|
||||
class="pointer-events-none absolute bottom-1 left-1 rounded bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white"
|
||||
class="pointer-events-none absolute bottom-1 left-1 rounded-md bg-black/40 px-1.5 py-0.5 text-[10px] font-medium text-white matte-chip backdrop-blur-md [--badge-highlight:rgb(255_255_255/0.18)]"
|
||||
>
|
||||
{label}
|
||||
</span>
|
||||
|
||||
+5
-5
@@ -4,9 +4,9 @@
|
||||
ChatAttachmentsPreviewFileInfo,
|
||||
ChatAttachmentsPreviewNavButtons,
|
||||
ChatAttachmentsPreviewThumbnailStrip
|
||||
} from '$lib/components/app';
|
||||
import { UI_DATA_ATTRS } from '$lib/constants';
|
||||
import { modelsStore } from '$lib/stores';
|
||||
} from '#lib/components/app/index.ts';
|
||||
import { UI_DATA_ATTRS } from '#lib/constants/index.ts';
|
||||
import { modelsStore } from '#lib/stores/index.ts';
|
||||
import {
|
||||
createBase64DataUrl,
|
||||
formatFileSize,
|
||||
@@ -19,7 +19,7 @@
|
||||
isPdfFile,
|
||||
isTextFile,
|
||||
isVideoFile
|
||||
} from '$lib/utils';
|
||||
} from '#lib/utils/index.ts';
|
||||
|
||||
interface PreviewItem {
|
||||
id: string;
|
||||
@@ -189,7 +189,7 @@
|
||||
<div class="relative flex min-h-0 flex-1 items-center justify-center overflow-hidden">
|
||||
<ChatAttachmentsPreviewNavButtons onNext={next} onPrev={prev} show={allItems.length > 1} />
|
||||
|
||||
<div class="flex h-full w-full flex-col items-center justify-start overflow-auto py-4">
|
||||
<div class="flex size-full flex-col items-center justify-start overflow-auto py-4">
|
||||
{#if currentItem}
|
||||
<ChatAttachmentsPreviewFileInfo {displayName} {fileSize} />
|
||||
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
import ChatAttachmentsPreviewCurrentItemUnavailable from './ChatAttachmentsPreviewCurrentItemUnavailable.svelte';
|
||||
import ChatAttachmentsPreviewCurrentItemVideo from './ChatAttachmentsPreviewCurrentItemVideo.svelte';
|
||||
import { FileIcon, FileText, Image, Music, Video } from '@lucide/svelte';
|
||||
import type { ChatAttachmentDisplayItem } from '$lib/types';
|
||||
import type { ChatAttachmentDisplayItem } from '#lib/types/index.ts';
|
||||
|
||||
interface Props {
|
||||
currentItem: ChatAttachmentDisplayItem | null;
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
|
||||
<div class="flex flex-1 items-center justify-center p-8">
|
||||
<div class="w-full max-w-md text-center">
|
||||
<Music class="mx-auto mb-4 h-16 w-16 text-white/50" />
|
||||
<Music class="mx-auto mb-4 size-16 text-white/50" />
|
||||
|
||||
{#if audioSrc}
|
||||
<audio class="mb-4 w-full" controls src={audioSrc}>
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
<div class="flex flex-1 items-center justify-center">
|
||||
<img
|
||||
alt={currentItem?.name || 'preview'}
|
||||
class="max-h-[80vh] max-w-[80vw] rounded-lg object-contain shadow-lg"
|
||||
class="max-h-[80vh] max-w-[80vw] rounded-lg object-contain shadow-(--elevation-raised)"
|
||||
src={displayPreview}
|
||||
/>
|
||||
</div>
|
||||
|
||||
+15
-11
@@ -1,13 +1,13 @@
|
||||
<script lang="ts">
|
||||
import * as Alert from '@ggml-org/llama-ui/ui/alert';
|
||||
import { Button } from '@ggml-org/llama-ui/ui/button';
|
||||
import { Eye, FileText, Info } from '@lucide/svelte';
|
||||
import { SyntaxHighlightedCode } from '$lib/components/app';
|
||||
import * as Alert from '$lib/components/ui/alert';
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
import { ICON_CLASS_DEFAULT, PDF_AS_IMAGES_LEGACY_CONTENT_REGEX } from '$lib/constants';
|
||||
import { AttachmentType, MimeTypeApplication, PdfViewMode } from '$lib/enums';
|
||||
import type { ChatAttachmentDisplayItem } from '$lib/types';
|
||||
import { getLanguageFromFilename } from '$lib/utils';
|
||||
import { convertPDFToImage, convertPDFToText } from '$lib/utils/browser-only';
|
||||
import { SyntaxHighlightedCode } from '#lib/components/app/index.ts';
|
||||
import { ICON_CLASS_DEFAULT, PDF_AS_IMAGES_LEGACY_CONTENT_REGEX } from '#lib/constants/index.ts';
|
||||
import { AttachmentType, MimeTypeApplication, PdfViewMode } from '#lib/enums/index.ts';
|
||||
import type { ChatAttachmentDisplayItem } from '#lib/types/index.ts';
|
||||
import { convertPDFToImage, convertPDFToText } from '#lib/utils/browser-only.ts';
|
||||
import { getLanguageFromFilename } from '#lib/utils/index.ts';
|
||||
|
||||
interface Props {
|
||||
currentItem: ChatAttachmentDisplayItem | null;
|
||||
@@ -208,7 +208,11 @@
|
||||
{#each pdfImages as image, index (index)}
|
||||
<p class="mb-2 text-sm text-white/50">Page {index + 1}</p>
|
||||
|
||||
<img alt="PDF Page {index + 1}" class="mx-auto max-w-[85vw] rounded-lg shadow-lg" src={image} />
|
||||
<img
|
||||
alt="PDF Page {index + 1}"
|
||||
class="mx-auto max-w-[85vw] rounded-lg shadow-(--elevation-raised)"
|
||||
src={image}
|
||||
/>
|
||||
|
||||
<div class="h-4"></div>
|
||||
{/each}
|
||||
@@ -220,7 +224,7 @@
|
||||
<div class="flex flex-1 items-center justify-center p-8">
|
||||
<div class="text-center">
|
||||
<div
|
||||
class="mx-auto mb-4 h-8 w-8 animate-spin rounded-full border-4 border-white border-t-transparent"
|
||||
class="mx-auto mb-4 size-8 animate-spin rounded-full border-4 border-white border-t-transparent"
|
||||
></div>
|
||||
|
||||
<p class="text-white/70">{message}</p>
|
||||
@@ -231,7 +235,7 @@
|
||||
{#snippet status(message: string, detail?: string)}
|
||||
<div class="flex flex-1 items-center justify-center p-8">
|
||||
<div class="text-center">
|
||||
<FileText class="mx-auto mb-4 h-16 w-16 text-white/50" />
|
||||
<FileText class="mx-auto mb-4 size-16 text-white/50" />
|
||||
|
||||
<p class={['text-white/70', detail && 'mb-4']}>{message}</p>
|
||||
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
<script lang="ts">
|
||||
import { SyntaxHighlightedCode } from '$lib/components/app';
|
||||
import { SyntaxHighlightedCode } from '#lib/components/app/index.ts';
|
||||
|
||||
interface Props {
|
||||
displayTextContent: string | undefined;
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
|
||||
<div class="flex flex-1 items-center justify-center p-8">
|
||||
<div class="text-center">
|
||||
<IconComponent class="mx-auto mb-4 h-16 w-16 text-white/50" />
|
||||
<IconComponent class="mx-auto mb-4 size-16 text-white/50" />
|
||||
|
||||
<p class="text-white/70">Preview not available for this file type</p>
|
||||
</div>
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
|
||||
<div class="flex flex-1 items-center justify-center p-8">
|
||||
<div class="w-full max-w-md text-center">
|
||||
<Video class="mx-auto mb-4 h-16 w-16 text-white/50" />
|
||||
<Video class="mx-auto mb-4 size-16 text-white/50" />
|
||||
|
||||
{#if videoSrc}
|
||||
<video class="mb-4 w-full" controls src={videoSrc}>
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { Button } from '@ggml-org/llama-ui/ui/button';
|
||||
import { ChevronLeft, ChevronRight } from '@lucide/svelte';
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
|
||||
interface Props {
|
||||
onPrev: () => void;
|
||||
@@ -14,20 +14,20 @@
|
||||
{#if show}
|
||||
<Button
|
||||
aria-label="Previous"
|
||||
class="absolute top-1/2 left-4 z-10 h-8 w-8 -translate-y-1/2 rounded-full bg-background/5 p-0 text-white!"
|
||||
class="absolute top-1/2 left-4 z-10 size-8 -translate-y-1/2 rounded-full bg-background/5 p-0 text-white!"
|
||||
onclick={onPrev}
|
||||
size="icon"
|
||||
variant="secondary"
|
||||
variant="outline"
|
||||
>
|
||||
<ChevronLeft class="size-4" />
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
aria-label="Next"
|
||||
class="absolute top-1/2 right-4 z-10 h-8 w-8 -translate-y-1/2 rounded-full bg-background/5 p-0 text-white!"
|
||||
class="absolute top-1/2 right-4 z-10 size-8 -translate-y-1/2 rounded-full bg-background/5 p-0 text-white!"
|
||||
onclick={onNext}
|
||||
size="icon"
|
||||
variant="secondary"
|
||||
variant="outline"
|
||||
>
|
||||
<ChevronRight class="size-4" />
|
||||
</Button>
|
||||
|
||||
+5
-5
@@ -1,8 +1,8 @@
|
||||
<script lang="ts">
|
||||
import { FileText, Music, Video } from '@lucide/svelte';
|
||||
import { ScrollCarousel } from '$lib/components/app';
|
||||
import { ICON_CLASS_DEFAULT, UI_DATA_ATTRS } from '$lib/constants';
|
||||
import { ScrollCarouselVariant } from '$lib/enums';
|
||||
import { ScrollCarousel } from '#lib/components/app/index.ts';
|
||||
import { ICON_CLASS_DEFAULT, UI_DATA_ATTRS } from '#lib/constants/index.ts';
|
||||
import { ScrollCarouselVariant } from '#lib/enums/index.ts';
|
||||
|
||||
interface PreviewItem {
|
||||
id: string;
|
||||
@@ -48,10 +48,10 @@
|
||||
onclick={() => onNavigate(index)}
|
||||
>
|
||||
{#if (item.isImage || item.isPdf) && item.preview}
|
||||
<img alt={item.name} class="h-12 w-12 object-cover" src={item.preview} />
|
||||
<img alt={item.name} class="size-12 object-cover" src={item.preview} />
|
||||
{:else}
|
||||
<div
|
||||
class="bg-foreground-muted/50 flex h-12 w-12 flex-col items-center justify-center gap-0.5 py-1"
|
||||
class="bg-foreground-muted/50 flex size-12 flex-col items-center justify-center gap-0.5 py-1"
|
||||
>
|
||||
{#if item.isAudio}
|
||||
<Music class="{ICON_CLASS_DEFAULT} text-white/70" />
|
||||
|
||||
@@ -11,16 +11,15 @@
|
||||
ChatFormMcpResourcesList,
|
||||
ChatFormPickers,
|
||||
DialogMcpResourcesBrowser
|
||||
} from '$lib/components/app';
|
||||
} from '#lib/components/app/index.ts';
|
||||
import {
|
||||
CHAT_FORM_SINGLE_ROW_MAX_TEXT_PX,
|
||||
CLIPBOARD_CONTENT_QUOTE_PREFIX,
|
||||
ICON_CLASS_DEFAULT,
|
||||
INITIAL_FILE_SIZE,
|
||||
INPUT_CLASSES,
|
||||
PROMPT_CONTENT_SEPARATOR,
|
||||
SETTING_CONFIG_DEFAULT
|
||||
} from '$lib/constants';
|
||||
} from '#lib/constants/index.ts';
|
||||
import {
|
||||
ContentPartType,
|
||||
FileExtensionText,
|
||||
@@ -28,9 +27,9 @@
|
||||
MimeTypeText,
|
||||
SpecialFileType,
|
||||
ToolSource
|
||||
} from '$lib/enums';
|
||||
import { useChatFormExpansion } from '$lib/hooks/use-chat-form-expansion.svelte';
|
||||
import { useChatFormPickers } from '$lib/hooks/use-chat-form-pickers.svelte';
|
||||
} from '#lib/enums/index.ts';
|
||||
import { useChatFormExpansion } from '#lib/hooks/use-chat-form-expansion.svelte.ts';
|
||||
import { useChatFormPickers } from '#lib/hooks/use-chat-form-pickers.svelte.ts';
|
||||
import {
|
||||
chatStore,
|
||||
conversationsStore,
|
||||
@@ -41,14 +40,22 @@
|
||||
settingsStore,
|
||||
toolsStore,
|
||||
uiStore
|
||||
} from '$lib/stores';
|
||||
} from '#lib/stores/index.ts';
|
||||
import type {
|
||||
FileMentionEntry,
|
||||
GetPromptResult,
|
||||
MCPPromptInfo,
|
||||
MCPResourceInfo,
|
||||
PromptMessage
|
||||
} from '$lib/types';
|
||||
} from '#lib/types/index.ts';
|
||||
import {
|
||||
AudioRecorder,
|
||||
convertToWav,
|
||||
createAudioFile,
|
||||
describeMicrophoneError,
|
||||
isAudioRecordingSupported,
|
||||
microphoneUnavailableReason
|
||||
} from '#lib/utils/browser-only.ts';
|
||||
import {
|
||||
buildMentionInsertion,
|
||||
findCommandToken,
|
||||
@@ -57,15 +64,7 @@
|
||||
isIMEComposing,
|
||||
parseClipboardContent,
|
||||
uuid
|
||||
} from '$lib/utils';
|
||||
import {
|
||||
AudioRecorder,
|
||||
convertToWav,
|
||||
createAudioFile,
|
||||
describeMicrophoneError,
|
||||
isAudioRecordingSupported,
|
||||
microphoneUnavailableReason
|
||||
} from '$lib/utils/browser-only';
|
||||
} from '#lib/utils/index.ts';
|
||||
import { onMount, untrack } from 'svelte';
|
||||
import type { ClassValue } from 'svelte/elements';
|
||||
import { toast } from 'svelte-sonner';
|
||||
@@ -678,9 +677,10 @@
|
||||
<div
|
||||
bind:this={cardEl}
|
||||
class={[
|
||||
INPUT_CLASSES,
|
||||
'backdrop-blur-md group/chat-form relative flex shrink flex-col overflow-hidden transition-[min-height,padding,border-radius,border-color,background-color] duration-400 ease-[var(--chat-form-resize-ease,cubic-bezier(0.3,1.12,0.5,1))] motion-reduce:transition-none',
|
||||
isSingleRow ? 'rounded-(--chat-form-single-row-radius)' : 'rounded-4xl md:rounded-3xl',
|
||||
'chat-form-surface group/chat-form relative flex shrink flex-col overflow-hidden outline-none transition-[min-height,padding,border-radius,border-color,background-color] duration-400 ease-[var(--chat-form-resize-ease,cubic-bezier(0.3,1.12,0.5,1))] motion-reduce:transition-none',
|
||||
isSingleRow
|
||||
? 'rounded-(--chat-form-single-row-radius)'
|
||||
: 'rounded-(--chat-form-radius-compact) md:rounded-(--chat-form-radius)',
|
||||
// on a phone the sheet's animation moves it (see useChatFormExpansion): only its
|
||||
// border, hidden on the sheet, fades back in after
|
||||
'max-md:transition-[border-color] max-md:duration-200',
|
||||
|
||||
+4
-4
@@ -1,8 +1,8 @@
|
||||
<script lang="ts">
|
||||
import { Button } from '@ggml-org/llama-ui/ui/button';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { Plus } from '@lucide/svelte';
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { ATTACHMENT_TOOLTIP_TEXT, ICON_CLASS_DEFAULT } from '$lib/constants';
|
||||
import { ATTACHMENT_TOOLTIP_TEXT, ICON_CLASS_DEFAULT } from '#lib/constants/index.ts';
|
||||
|
||||
interface Props {
|
||||
disabled?: boolean;
|
||||
@@ -15,7 +15,7 @@
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger class="w-full">
|
||||
<Button
|
||||
class="file-upload-button bg-background h-9 w-9 rounded-full p-0 shadow-sm md:h-8 md:w-8 dark:bg-muted-foreground/15"
|
||||
class="file-upload-button bg-background size-9 rounded-full p-0 shadow-(--chat-form-key-shadow) md:size-8 dark:bg-muted-foreground/15"
|
||||
{disabled}
|
||||
{onclick}
|
||||
type="button"
|
||||
|
||||
+10
-10
@@ -1,4 +1,8 @@
|
||||
<script lang="ts">
|
||||
import { Checkbox } from '@ggml-org/llama-ui/ui/checkbox';
|
||||
import * as Collapsible from '@ggml-org/llama-ui/ui/collapsible';
|
||||
import * as Drawer from '@ggml-org/llama-ui/ui/drawer';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import {
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
@@ -10,21 +14,17 @@
|
||||
PencilRuler,
|
||||
Video
|
||||
} from '@lucide/svelte';
|
||||
import { Checkbox } from '$lib/components/ui/checkbox';
|
||||
import * as Collapsible from '$lib/components/ui/collapsible';
|
||||
import * as Drawer from '$lib/components/ui/drawer';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import {
|
||||
ATTACHMENT_FILE_ITEMS,
|
||||
ATTACHMENT_MENU_TEXT,
|
||||
ATTACHMENT_MODALITY_TEXT,
|
||||
ICON_CLASS_DEFAULT
|
||||
} from '$lib/constants';
|
||||
import { getChatFormActionsContext } from '$lib/contexts';
|
||||
import { AttachmentAction, AttachmentItemEnabledWhen } from '$lib/enums/attachment.enums';
|
||||
import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
|
||||
import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte';
|
||||
import type { ToolGroup } from '$lib/types';
|
||||
} from '#lib/constants/index.ts';
|
||||
import { getChatFormActionsContext } from '#lib/contexts/index.ts';
|
||||
import { AttachmentAction, AttachmentItemEnabledWhen } from '#lib/enums/attachment.enums.ts';
|
||||
import { useAttachmentMenu } from '#lib/hooks/use-attachment-menu.svelte.ts';
|
||||
import { useToolsPanel } from '#lib/hooks/use-tools-panel.svelte.ts';
|
||||
import type { ToolGroup } from '#lib/types/index.ts';
|
||||
import type { Snippet } from 'svelte';
|
||||
|
||||
interface Props {
|
||||
|
||||
+10
-10
@@ -1,20 +1,20 @@
|
||||
<script lang="ts">
|
||||
import { buttonVariants } from '@ggml-org/llama-ui/ui/button';
|
||||
import * as DropdownMenu from '@ggml-org/llama-ui/ui/dropdown-menu';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { cn } from '@ggml-org/llama-ui/utils';
|
||||
import { File, Folder, Image, MessageSquare, Mic, Plus, Video } from '@lucide/svelte';
|
||||
import { ChatFormActionAddToolsSubmenu } from '$lib/components/app';
|
||||
import { buttonVariants } from '$lib/components/ui/button';
|
||||
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { cn } from '$lib/components/ui/utils';
|
||||
import { ChatFormActionAddToolsSubmenu } from '#lib/components/app/index.ts';
|
||||
import {
|
||||
ATTACHMENT_FILE_ITEMS,
|
||||
ATTACHMENT_MENU_TEXT,
|
||||
ATTACHMENT_MODALITY_TEXT,
|
||||
ATTACHMENT_TOOLTIP_TEXT,
|
||||
ICON_CLASS_DEFAULT
|
||||
} from '$lib/constants';
|
||||
import { getChatFormActionsContext } from '$lib/contexts';
|
||||
import { AttachmentAction, AttachmentItemEnabledWhen } from '$lib/enums';
|
||||
import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
|
||||
} from '#lib/constants/index.ts';
|
||||
import { getChatFormActionsContext } from '#lib/contexts/index.ts';
|
||||
import { AttachmentAction, AttachmentItemEnabledWhen } from '#lib/enums/index.ts';
|
||||
import { useAttachmentMenu } from '#lib/hooks/use-attachment-menu.svelte.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
@@ -78,7 +78,7 @@
|
||||
class={cn(
|
||||
// ghost brings no fill of its own for the muted composer surface to fight over
|
||||
buttonVariants({ variant: 'ghost' }),
|
||||
'file-upload-button h-8 w-8 cursor-pointer rounded-full bg-background p-0 shadow-sm dark:bg-muted-foreground/15'
|
||||
'file-upload-button matte-key size-8 [--button-shadow:var(--chat-form-key-shadow)] cursor-pointer rounded-full bg-background p-0 dark:bg-muted-foreground/15'
|
||||
)}
|
||||
disabled={chatFormActions.disabled}
|
||||
>
|
||||
|
||||
+10
-10
@@ -1,13 +1,13 @@
|
||||
<script lang="ts">
|
||||
import { Checkbox } from '@ggml-org/llama-ui/ui/checkbox';
|
||||
import * as Collapsible from '@ggml-org/llama-ui/ui/collapsible';
|
||||
import * as DropdownMenu from '@ggml-org/llama-ui/ui/dropdown-menu';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { Check, ChevronDown, ChevronRight, Info, Loader2, PencilRuler } from '@lucide/svelte';
|
||||
import { Checkbox } from '$lib/components/ui/checkbox';
|
||||
import * as Collapsible from '$lib/components/ui/collapsible';
|
||||
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { ATTACHMENT_MENU_TEXT, CLI_FLAGS, ICON_CLASS_DEFAULT } from '$lib/constants';
|
||||
import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte';
|
||||
import { mcpStore, toolsStore } from '$lib/stores';
|
||||
import type { ToolGroup } from '$lib/types';
|
||||
import { ATTACHMENT_MENU_TEXT, CLI_FLAGS, ICON_CLASS_DEFAULT } from '#lib/constants/index.ts';
|
||||
import { useToolsPanel } from '#lib/hooks/use-tools-panel.svelte.ts';
|
||||
import { mcpStore, toolsStore } from '#lib/stores/index.ts';
|
||||
import type { ToolGroup } from '#lib/types/index.ts';
|
||||
|
||||
const toolsPanel = useToolsPanel();
|
||||
const hasMcpServersAvailable = $derived(mcpStore.getServers().length > 0);
|
||||
@@ -90,9 +90,9 @@
|
||||
class="flex min-w-0 flex-1 items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-muted/50"
|
||||
>
|
||||
{#if isExpanded}
|
||||
<ChevronDown class="h-3.5 w-3.5 shrink-0" />
|
||||
<ChevronDown class="size-icon-sm shrink-0" />
|
||||
{:else}
|
||||
<ChevronRight class="h-3.5 w-3.5 shrink-0" />
|
||||
<ChevronRight class="size-icon-sm shrink-0" />
|
||||
{/if}
|
||||
|
||||
<span class="inline-flex min-w-0 items-center gap-1.5 font-medium">
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
import ChatFormActionAddButton from './ChatFormActionAddButton.svelte';
|
||||
import ChatFormActionAddDrawer from './ChatFormActionAddDrawer.svelte';
|
||||
import ChatFormActionAddDropdown from './ChatFormActionAddDropdown.svelte';
|
||||
import { deviceStore } from '$lib/stores';
|
||||
import { deviceStore } from '#lib/stores/index.ts';
|
||||
</script>
|
||||
|
||||
{#if deviceStore.isMobile}
|
||||
|
||||
+13
-13
@@ -1,14 +1,14 @@
|
||||
<script lang="ts">
|
||||
import { Button } from '@ggml-org/llama-ui/ui/button';
|
||||
import * as Drawer from '@ggml-org/llama-ui/ui/drawer';
|
||||
import * as DropdownMenu from '@ggml-org/llama-ui/ui/dropdown-menu';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { Check, Info, Lightbulb, LightbulbOff } from '@lucide/svelte';
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
import * as Drawer from '$lib/components/ui/drawer';
|
||||
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { ICON_CLASS_DEFAULT, REASONING_EFFORT_LABELS } from '$lib/constants';
|
||||
import { ReasoningEffort } from '$lib/enums';
|
||||
import { useReasoningMenu } from '$lib/hooks/use-reasoning-menu.svelte';
|
||||
import { deviceStore } from '$lib/stores';
|
||||
import type { ReasoningEffortLevel } from '$lib/types';
|
||||
import { ICON_CLASS_DEFAULT, REASONING_EFFORT_LABELS } from '#lib/constants/index.ts';
|
||||
import { ReasoningEffort } from '#lib/enums/index.ts';
|
||||
import { useReasoningMenu } from '#lib/hooks/use-reasoning-menu.svelte.ts';
|
||||
import { deviceStore } from '#lib/stores/index.ts';
|
||||
import type { ReasoningEffortLevel } from '#lib/types/index.ts';
|
||||
|
||||
const reasoning = useReasoningMenu();
|
||||
|
||||
@@ -30,9 +30,9 @@
|
||||
{#snippet triggerLabel()}
|
||||
<span class="flex items-center gap-0.75 {reasoning.isOff ? 'text-muted-foreground' : ''}">
|
||||
{#if reasoning.isOff}
|
||||
<LightbulbOff class="size-3.25 max-md:size-4 shrink-0" />
|
||||
<LightbulbOff class="size-3.25 max-md:size-icon-md shrink-0" />
|
||||
{:else}
|
||||
<Lightbulb class="size-3.25 max-md:size-4 shrink-0" />
|
||||
<Lightbulb class="size-3.25 max-md:size-icon-md shrink-0" />
|
||||
{/if}
|
||||
|
||||
{#if !isDefault}
|
||||
@@ -64,7 +64,7 @@
|
||||
|
||||
{#if level.hasInfo}
|
||||
<!-- no hover on a phone: the hint rides on the icon's own title there -->
|
||||
<Info class="h-3.5 w-3.5 shrink-0 text-muted-foreground" title={MAX_EFFORT_HINT} />
|
||||
<Info class="size-icon-md shrink-0 text-muted-foreground" title={MAX_EFFORT_HINT} />
|
||||
{/if}
|
||||
</button>
|
||||
{/snippet}
|
||||
@@ -132,7 +132,7 @@
|
||||
{#if level.hasInfo}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Info class="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||
<Info class="size-icon-md shrink-0 text-muted-foreground" />
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content side="right">
|
||||
|
||||
+7
-6
@@ -1,8 +1,8 @@
|
||||
<script lang="ts">
|
||||
import { Button } from '@ggml-org/llama-ui/ui/button';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { Mic, MicOff, Square } from '@lucide/svelte';
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { ICON_CLASS_DEFAULT } from '$lib/constants';
|
||||
import { ICON_CLASS_DEFAULT } from '#lib/constants/index.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
@@ -37,14 +37,15 @@
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Button
|
||||
class="h-8 w-8 max-md:h-9 max-md:w-9 rounded-full p-0 {isRecording
|
||||
? 'animate-pulse bg-red-500 text-white hover:bg-red-600'
|
||||
class="size-8 max-md:size-9 rounded-full p-0 {isRecording
|
||||
? 'matte-orb orb-danger animate-pulse text-white! hover:bg-transparent'
|
||||
: showWarning
|
||||
? 'bg-amber-500/15 text-amber-600 hover:bg-amber-500/25 dark:text-amber-400'
|
||||
? 'bg-surface-warning text-fg-warning hover:bg-surface-warning'
|
||||
: ''}"
|
||||
disabled={disabled || isLoading || !hasAudioModality}
|
||||
onclick={onMicClick}
|
||||
type="button"
|
||||
variant={isRecording || showWarning ? 'ghost' : 'default'}
|
||||
>
|
||||
<span class="sr-only"
|
||||
>{isRecording
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user