mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-13 14:27:38 -06:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Georgi Gerganov
|
b07234f832 | ||
|
Aman Gupta
|
cbdab17ed6 | ||
|
Aman Gupta
|
5b1df74b58 | ||
|
Aman Gupta
|
33bc861866 | ||
|
Aman Gupta
|
ccb0a39926 |
+12
-12
@@ -1,4 +1,4 @@
|
||||
ARG ONEAPI_VERSION=2026.2.0-devel-ubuntu24.04
|
||||
ARG ONEAPI_VERSION=2026.1.1-devel-ubuntu24.04
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
ARG APP_REVISION=N/A
|
||||
@@ -22,20 +22,20 @@ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
|
||||
RUN echo "Download oneDNN installation package" && \
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh && \
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh && \
|
||||
echo "Install oneDNN components" && \
|
||||
bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
|
||||
ARG GGML_SYCL_F16=ON
|
||||
ARG LEVEL_ZERO_VERSION=1.34.0
|
||||
ARG LEVEL_ZERO_VERSION=1.28.2
|
||||
ARG LEVEL_ZERO_UBUNTU_VERSION=u24.04
|
||||
RUN apt-get update && \
|
||||
apt-get install -y git libssl-dev wget ca-certificates && \
|
||||
cd /tmp && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb && \
|
||||
apt-get -o Dpkg::Options::="--force-overwrite" install -y ./libze1.deb ./libze-dev.deb && \
|
||||
rm -f /tmp/libze1.deb /tmp/libze-dev.deb
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb && \
|
||||
apt-get -o Dpkg::Options::="--force-overwrite" install -y ./level-zero.deb ./level-zero-devel.deb && \
|
||||
rm -f /tmp/level-zero.deb /tmp/level-zero-devel.deb
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -91,11 +91,11 @@ COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl
|
||||
#ARG IGDGMM_VERSION=22.8.2
|
||||
|
||||
|
||||
ARG IGC_VERSION=v2.42.6
|
||||
ARG IGC_VERSION_FULL=2_2.42.6+23070
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0
|
||||
ARG IGDGMM_VERSION=22.10.2
|
||||
ARG IGC_VERSION=v2.34.4
|
||||
ARG IGC_VERSION_FULL=2_2.34.4+21428
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
|
||||
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \
|
||||
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \
|
||||
|
||||
@@ -22,6 +22,10 @@ let
|
||||
gitpython
|
||||
tabulate
|
||||
|
||||
# for examples/pydantic-models-to-grammar-examples.py
|
||||
docstring-parser
|
||||
pydantic
|
||||
|
||||
];
|
||||
|
||||
llama-python-test-deps = with python3Packages; [
|
||||
|
||||
@@ -100,7 +100,7 @@ jobs:
|
||||
run: |
|
||||
cd build
|
||||
echo "Fetch llama2c big-endian model"
|
||||
wget https://huggingface.co/ggml-org-dev/stories260K/resolve/main/stories260K-be.gguf
|
||||
wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
|
||||
./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
||||
|
||||
- name: Test quantize (s390x)
|
||||
|
||||
@@ -37,7 +37,7 @@ env:
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
# TODO: fix failing tests on OpenVINO backend
|
||||
CTEST_EXCLUDE: "test-llama-archs|test-llama-context|test-fp8-kv"
|
||||
CTEST_EXCLUDE: "test-llama-archs|test-llama-context"
|
||||
|
||||
jobs:
|
||||
ubuntu-24-openvino:
|
||||
|
||||
@@ -52,8 +52,8 @@ jobs:
|
||||
|
||||
env:
|
||||
ONEAPI_ROOT: /opt/intel/oneapi/
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
LEVEL_ZERO_VERSION: "1.34.0"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
LEVEL_ZERO_VERSION: "1.33.1"
|
||||
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
|
||||
|
||||
continue-on-error: true
|
||||
@@ -68,12 +68,12 @@ jobs:
|
||||
run: |
|
||||
cd /tmp
|
||||
echo "Download oneAPI Deep Learning Essentials installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/aef232d2-aac5-4cd6-bceb-911019bbb6a8/intel-deep-learning-essentials-2026.2.0.640_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b5c100a7-8052-4e88-90e2-63d9916a998b/intel-deep-learning-essentials-2026.1.4.17_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
echo "Install oneAPI Deep Learning Essentials components"
|
||||
sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
|
||||
|
||||
echo "Download oneDNN installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh
|
||||
echo "Install oneDNN components"
|
||||
sudo bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
|
||||
@@ -150,13 +150,13 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b177a006-5cd9-4cb2-807f-8199cc73d53c/intel-deep-learning-essentials-2026.2.0.638_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/8f905666-c85c-43fb-9bd0-2618c7ecd33b/intel-onednn-2026.1.0.622_offline.exe
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a51104eb-6f4a-4a0e-9dae-1b81b0c81ae1/intel-deep-learning-essentials-2026.1.4.16_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/34e28083-d04b-46a3-826c-8c1d00bf1e1c/intel-onednn-2026.0.2.47_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel
|
||||
WINDOWS_DNNL: intel.oneapi.win.dnnl
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.34.0/level-zero-win-sdk-1.34.0.zip
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
|
||||
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
|
||||
@@ -93,72 +93,13 @@ jobs:
|
||||
save: true
|
||||
|
||||
gpu-rocm:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Strix Halo (iGPU, RDNA3.5)
|
||||
- gpu: gfx1151
|
||||
runner: [self-hosted, Linux, strix-halo, devlab-dispatch]
|
||||
# async-execution correctness workaround, gfx1151 only (see Test step)
|
||||
hip_launch_blocking: "1"
|
||||
# Radeon AI PRO R9700 (dGPU, RDNA4)
|
||||
- gpu: gfx1201
|
||||
runner: [self-hosted, Linux, radeon-r9700, devlab-dispatch]
|
||||
hip_launch_blocking: "0"
|
||||
# HIP-graph capture fails on gfx1201 (hipMemset -> hipErrorStreamCaptureImplicit)
|
||||
disable_hip_graphs: true
|
||||
|
||||
name: gpu-rocm (${{ matrix.gpu }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 120
|
||||
|
||||
env:
|
||||
ROCM_VERSION: "10.0.0"
|
||||
runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Verify GPU device
|
||||
run: |
|
||||
ls /dev/kfd /dev/dri/renderD* || { echo "ERROR: no amdgpu compute device"; exit 1; }
|
||||
|
||||
# same TheRock wheels as the ubuntu-24-rocm release build, so the runner does not need a ROCm install
|
||||
- name: Setup TheRock with Wheels
|
||||
run: |
|
||||
python3 -m venv "$RUNNER_TEMP/rocm"
|
||||
"$RUNNER_TEMP/rocm/bin/python" -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel,device-${{ matrix.gpu }}]==${ROCM_VERSION}"
|
||||
ROCM_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --root)
|
||||
{
|
||||
echo "ROCM_PATH=$ROCM_PATH"
|
||||
echo "HIP_PATH=$ROCM_PATH"
|
||||
echo "CMAKE_PREFIX_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --cmake)"
|
||||
echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}"
|
||||
} >> "$GITHUB_ENV"
|
||||
"$RUNNER_TEMP/rocm/bin/rocm-sdk" path --bin >> "$GITHUB_PATH"
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
# ROCm is installed again on each run, so hash the compiler content, not its mtime
|
||||
- name: Tune ccache for reinstalled ROCm toolchain
|
||||
run: |
|
||||
ccache --set-config=compiler_check=content
|
||||
ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
|
||||
ccache --set-config=depend_mode=true
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-gpu-rocm-${{ matrix.gpu }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
# HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
|
||||
@@ -166,25 +107,10 @@ jobs:
|
||||
# incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
|
||||
# restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
|
||||
env:
|
||||
HIP_LAUNCH_BLOCKING: ${{ matrix.hip_launch_blocking }}
|
||||
HIP_LAUNCH_BLOCKING: "1"
|
||||
run: |
|
||||
rocminfo
|
||||
# Exported rather than set in env:, because ggml only checks that the
|
||||
# variable is defined: an empty value would disable graphs on gfx1151 too.
|
||||
if [ "${{ matrix.disable_hip_graphs }}" = "true" ]; then export GGML_CUDA_DISABLE_GRAPHS=1; fi
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=${{ matrix.gpu }} bash ./ci/run.sh "$RUNNER_TEMP/results/llama.cpp" "$RUNNER_TEMP/mnt/llama.cpp"
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-gpu-rocm-${{ matrix.gpu }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMD GPU machine
|
||||
# amd-rocm:
|
||||
|
||||
@@ -58,7 +58,12 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
|
||||
sudo apt install -y software-properties-common
|
||||
sudo add-apt-repository ppa:deadsnakes/ppa
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3.12 python3.12-venv
|
||||
sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
|
||||
sudo update-alternatives --config python3
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
pip-install: -r requirements/requirements-all.txt ty==0.0.100
|
||||
pip-install: -r requirements/requirements-all.txt ty==0.0.88
|
||||
# - name: Type-check with Pyright
|
||||
# uses: jakebailey/pyright-action@v2
|
||||
# with:
|
||||
|
||||
@@ -1239,7 +1239,6 @@ jobs:
|
||||
with:
|
||||
key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
|
||||
evict-old-files: 1d
|
||||
max-size: "1G"
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -1308,14 +1307,14 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b177a006-5cd9-4cb2-807f-8199cc73d53c/intel-deep-learning-essentials-2026.2.0.638_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/8f905666-c85c-43fb-9bd0-2618c7ecd33b/intel-onednn-2026.1.0.622_offline.exe
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a51104eb-6f4a-4a0e-9dae-1b81b0c81ae1/intel-deep-learning-essentials-2026.1.4.16_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/34e28083-d04b-46a3-826c-8c1d00bf1e1c/intel-onednn-2026.0.2.47_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel
|
||||
WINDOWS_DNNL: intel.oneapi.win.dnnl
|
||||
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.34.0/level-zero-win-sdk-1.34.0.zip
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
|
||||
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -1364,12 +1363,11 @@ jobs:
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_def.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_avx2.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_intel_thread.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/libimalloc.dll" ./build/bin
|
||||
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader_shared.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_opencl.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_win_proxy_loader.dll" ./build/bin
|
||||
ZE_LOADER_DLL=$(find "${{ env.ONEAPI_ROOT }}" "$LEVEL_ZERO_V1_SDK_PATH" -iname ze_loader.dll -print -quit 2>/dev/null || true)
|
||||
if [ -n "$ZE_LOADER_DLL" ]; then
|
||||
@@ -1423,8 +1421,8 @@ jobs:
|
||||
|
||||
env:
|
||||
ONEAPI_ROOT: /opt/intel/oneapi/
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
LEVEL_ZERO_VERSION: "1.34.0"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
LEVEL_ZERO_VERSION: "1.33.1"
|
||||
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
|
||||
|
||||
steps:
|
||||
@@ -1439,12 +1437,12 @@ jobs:
|
||||
run: |
|
||||
cd /tmp
|
||||
echo "Download oneAPI Deep Learning Essentials installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/aef232d2-aac5-4cd6-bceb-911019bbb6a8/intel-deep-learning-essentials-2026.2.0.640_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b5c100a7-8052-4e88-90e2-63d9916a998b/intel-deep-learning-essentials-2026.1.4.17_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
echo "Install oneAPI Deep Learning Essentials components"
|
||||
sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
|
||||
|
||||
echo "Download oneDNN installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh
|
||||
echo "Install oneDNN components"
|
||||
sudo bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
name: Server (speculative decoding)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sha:
|
||||
description: 'Commit SHA1 to build'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
server-speculative:
|
||||
runs-on: [self-hosted, Linux, gb10, perf]
|
||||
container: "nvidia/cuda:13.4.3-devel-ubuntu26.04"
|
||||
timeout-minutes: 60
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
apt update
|
||||
apt install -y cmake libssl-dev python3 python3-venv python3-pip
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
cmake --build build --config Release -j $(nproc) --target llama-server
|
||||
|
||||
- name: Python setup
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Test speculative decoding
|
||||
env:
|
||||
SPEC_METHODS: compact
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
|
||||
N_GPU_LAYERS: 99
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py
|
||||
@@ -15,7 +15,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-slim
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
|
||||
+2
-2
@@ -3,8 +3,8 @@ project("llama.cpp" C CXX)
|
||||
include(CheckIncludeFileCXX)
|
||||
|
||||
### llama.cpp version
|
||||
set(LLAMA_VERSION_MAJOR 1)
|
||||
set(LLAMA_VERSION_MINOR 0)
|
||||
set(LLAMA_VERSION_MAJOR 0)
|
||||
set(LLAMA_VERSION_MINOR 9)
|
||||
set(LLAMA_VERSION_PATCH 0)
|
||||
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# llama.cpp
|
||||
|
||||
|
||||
|
||||
|
||||
<div align="center">
|
||||
|
||||
<b>LLM inference in C/C++</b>
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v0)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=b)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
|
||||
@@ -180,7 +180,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
|
||||
|
||||
# TODO: fix failing tests on OpenVINO backend
|
||||
CTEST_EXTRA="-E test-llama-archs|test-llama-context|test-fp8-kv"
|
||||
CTEST_EXTRA="-E test-llama-archs|test-llama-context"
|
||||
fi
|
||||
|
||||
## helpers
|
||||
|
||||
@@ -2,11 +2,11 @@ get_filename_component(DEST_DIR "${DEST}" DIRECTORY)
|
||||
file(MAKE_DIRECTORY "${DEST_DIR}")
|
||||
|
||||
if(NOT EXISTS "${DEST}")
|
||||
message(STATUS "Downloading ${NAME} from ${REPO}...")
|
||||
message(STATUS "Downloading ${NAME} from ggml-org/models...")
|
||||
endif()
|
||||
|
||||
file(DOWNLOAD
|
||||
"https://huggingface.co/${REPO}/resolve/main/${NAME}?download=true"
|
||||
"https://huggingface.co/ggml-org/models/resolve/main/${NAME}?download=true"
|
||||
"${DEST}"
|
||||
TLS_VERIFY ON
|
||||
EXPECTED_HASH ${HASH}
|
||||
|
||||
@@ -66,8 +66,6 @@ add_library(${TARGET}
|
||||
chat-diff-analyzer.cpp
|
||||
chat-peg-parser.cpp
|
||||
chat-peg-parser.h
|
||||
chat-tts-norm.cpp
|
||||
chat-tts-norm.h
|
||||
chat.cpp
|
||||
chat.h
|
||||
common.cpp
|
||||
|
||||
+15
-32
@@ -282,7 +282,6 @@ const std::vector<ggml_type> kv_cache_types = {
|
||||
GGML_TYPE_F32,
|
||||
GGML_TYPE_F16,
|
||||
GGML_TYPE_BF16,
|
||||
GGML_TYPE_F8_E4M3,
|
||||
GGML_TYPE_Q8_0,
|
||||
GGML_TYPE_Q4_0,
|
||||
GGML_TYPE_Q4_1,
|
||||
@@ -943,8 +942,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
}
|
||||
|
||||
const bool mcp_enabled = !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty();
|
||||
if ((!params.server_tools.empty() || mcp_enabled || params.ui_mcp_proxy) && !params.cors_origins_explicit) {
|
||||
LOG_WRN("server tools, MCP servers or MCP proxy are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
|
||||
if ((!params.server_tools.empty() || mcp_enabled) && !params.cors_origins_explicit) {
|
||||
LOG_WRN("server tools or MCP servers are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
|
||||
params.cors_origins = "localhost";
|
||||
}
|
||||
|
||||
@@ -2501,6 +2500,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.ppl_output_type = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
|
||||
add_opt(common_arg(
|
||||
{"-dt", "--defrag-thold"}, "N",
|
||||
string_format("KV cache defragmentation threshold (DEPRECATED)"),
|
||||
[](common_params & params, const std::string & value) {
|
||||
GGML_UNUSED(params);
|
||||
GGML_UNUSED(value);
|
||||
LOG_WRN("DEPRECATED: --defrag-thold is deprecated and no longer necessary to specify\n");
|
||||
}
|
||||
).set_env("LLAMA_ARG_DEFRAG_THOLD"));
|
||||
if (ex == LLAMA_EXAMPLE_SERVER) {
|
||||
// this is to make sure this option appears in the server-specific section of the help message
|
||||
add_opt(common_arg(
|
||||
@@ -3024,16 +3032,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.model.path = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
|
||||
add_opt(common_arg(
|
||||
{"--ple", "--shard"}, "FNAME,...",
|
||||
"extra GGUF shards for the model, comma separated; may be given more than once\n"
|
||||
"paths may live anywhere, including on different drives; --ple is an alias for --shard",
|
||||
[](common_params & params, const std::string & value) {
|
||||
for (const auto & item : parse_csv_row(value)) {
|
||||
params.model.shards.push_back(item);
|
||||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_COMMON}).set_env("LLAMA_ARG_SHARD"));
|
||||
add_opt(common_arg(
|
||||
{"-mu", "--model-url"}, "MODEL_URL",
|
||||
"model download url (default: unused)",
|
||||
@@ -3352,7 +3350,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
{"--cors-origins"}, "ORIGINS",
|
||||
string_format(
|
||||
"comma-separated list of allowed origins for CORS (default: %s)\n"
|
||||
"if set to special value 'localhost', only allow same-origin requests (e.g. the built-in UI)",
|
||||
"if set to special value 'localhost', reflect the Origin header only if it is localhost",
|
||||
params.cors_origins.c_str()),
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.cors_origins = value;
|
||||
@@ -3408,8 +3406,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy", "--webui-mcp-proxy"},
|
||||
{"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"},
|
||||
"experimental: whether to enable MCP CORS proxy, for the Web UI to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
"experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)",
|
||||
[](common_params & params, bool value) {
|
||||
params.ui_mcp_proxy = value;
|
||||
}
|
||||
@@ -3435,13 +3432,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.server_tools_runtime = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME"));
|
||||
add_opt(common_arg(
|
||||
{"--allow-unauthenticated-tools"},
|
||||
"allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)",
|
||||
[](common_params & params) {
|
||||
params.server_tools_allow_unauth = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS"));
|
||||
add_opt(common_arg(
|
||||
{"--mcp-servers-config"}, "PATH",
|
||||
"experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n"
|
||||
@@ -3667,13 +3657,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--media-allow-private-urls"},
|
||||
"allow http(s) media URLs (e.g. image_url) that point to local or private addresses (default: disabled, only public addresses)",
|
||||
[](common_params & params) {
|
||||
params.media_allow_private_urls = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MEDIA_ALLOW_PRIVATE_URLS"));
|
||||
add_opt(common_arg(
|
||||
{"--models-dir"}, "PATH",
|
||||
"directory containing models for the router server (default: disabled)",
|
||||
@@ -4498,7 +4481,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_lang = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-voice"}, "VOICE",
|
||||
"voice for audio generation (model-dependent)\n"
|
||||
@@ -4506,7 +4489,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_voice = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-file"}, "FNAME",
|
||||
"speaker file path for audio generation",
|
||||
|
||||
+44
-48
@@ -131,12 +131,6 @@ static std::string normalize_quotes_to_json(const std::string & input) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Models can emit raw control characters inside string literals; escape them to keep the JSON valid.
|
||||
if ((in_single_quoted || in_double_quoted) && static_cast<unsigned char>(c) < 0x20) {
|
||||
result += escape_json_string_inner(std::string(1, c));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle quote characters
|
||||
if (c == '"') {
|
||||
if (in_single_quoted) {
|
||||
@@ -973,21 +967,6 @@ void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena,
|
||||
}
|
||||
}
|
||||
|
||||
using gemma4 = common_chat_peg_gemma4_mapper;
|
||||
|
||||
// Collects the nearest descendants carrying tag, looking through the rules the parser
|
||||
// wraps around the tagged nodes.
|
||||
static void gemma4_collect(const common_peg_ast_arena & arena, const common_peg_ast_node & node, const std::string & tag, std::vector<common_peg_ast_id> & out) {
|
||||
for (auto child_id : node.children) {
|
||||
const auto & child = arena.get(child_id);
|
||||
if (child.tag == tag) {
|
||||
out.push_back(child_id);
|
||||
} else {
|
||||
gemma4_collect(arena, child, tag, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) {
|
||||
const auto & node = arena.get(id);
|
||||
|
||||
@@ -995,45 +974,51 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
|
||||
return "";
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::NUMBER || node.tag == gemma4::BOOL || node.tag == gemma4::NULL_) {
|
||||
if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") {
|
||||
return std::string(node.text);
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::STRING_CONTENT) {
|
||||
if (node.rule == "gemma4-string-content") {
|
||||
return escape_json_string_inner(node.sanitized_text());
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::STRING) {
|
||||
if (node.rule == "gemma4-string") {
|
||||
std::string result = "\"";
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
if (!node.is_partial) {
|
||||
result += "\"";
|
||||
if (!node.children.empty()) {
|
||||
result += gemma4_to_json(arena, node.children[0]);
|
||||
if (!node.is_partial) {
|
||||
result += "\"";
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::ARRAY) {
|
||||
if (node.rule == "gemma4-array") {
|
||||
std::string result = "[";
|
||||
for (size_t i = 0; i < node.children.size(); i++) {
|
||||
if (i > 0) {
|
||||
|
||||
bool add_comma = false;
|
||||
for (auto child_id : node.children) {
|
||||
if (add_comma) {
|
||||
result += ',';
|
||||
}
|
||||
result += gemma4_to_json(arena, node.children[i]);
|
||||
add_comma = true;
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
|
||||
if (!node.is_partial) {
|
||||
result += ']';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::DICT_KEY) {
|
||||
auto name_id = arena.find_by_tag(node, gemma4::DICT_KEY_NAME);
|
||||
if (node.rule == "gemma4-dict-key-name") {
|
||||
return std::string(node.text);
|
||||
}
|
||||
|
||||
if (node.rule == "gemma4-dict-key") {
|
||||
std::string result = "\"";
|
||||
if (name_id != COMMON_PEG_INVALID_AST_ID) {
|
||||
result += escape_json_string_inner(std::string(arena.get(name_id).text));
|
||||
if (!node.children.empty()) {
|
||||
result += escape_json_string_inner(gemma4_to_json(arena, node.children[0]));
|
||||
}
|
||||
if (!node.is_partial) {
|
||||
result += "\":";
|
||||
@@ -1041,29 +1026,40 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::DICT) {
|
||||
std::vector<common_peg_ast_id> members;
|
||||
gemma4_collect(arena, node, gemma4::DICT_KV, members);
|
||||
if (node.rule == "gemma4-dict-kv") {
|
||||
std::string result;
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.rule == "gemma4-dict") {
|
||||
std::string result = "{";
|
||||
for (size_t i = 0; i < members.size(); i++) {
|
||||
if (i > 0) {
|
||||
|
||||
bool add_comma = false;
|
||||
for (auto child_id : node.children) {
|
||||
if (add_comma) {
|
||||
result += ',';
|
||||
}
|
||||
result += gemma4_to_json(arena, members[i]);
|
||||
add_comma = true;
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
|
||||
if (!node.is_partial) {
|
||||
result += '}';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// members, rules and other wrappers
|
||||
std::string result;
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
if (node.rule == "gemma4-value") {
|
||||
if (!node.children.empty()) {
|
||||
return gemma4_to_json(arena, node.children[0]);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
return result;
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {
|
||||
|
||||
@@ -34,18 +34,6 @@ class common_chat_peg_mapper {
|
||||
|
||||
class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper {
|
||||
public:
|
||||
// Tags of the Gemma 4 argument syntax. The parser may wrap them in any rules.
|
||||
static constexpr const char * STRING = "gemma4-string";
|
||||
static constexpr const char * STRING_CONTENT = "gemma4-string-content";
|
||||
static constexpr const char * NUMBER = "gemma4-number";
|
||||
static constexpr const char * BOOL = "gemma4-bool";
|
||||
static constexpr const char * NULL_ = "gemma4-null";
|
||||
static constexpr const char * ARRAY = "gemma4-array";
|
||||
static constexpr const char * DICT = "gemma4-dict";
|
||||
static constexpr const char * DICT_KV = "gemma4-dict-kv";
|
||||
static constexpr const char * DICT_KEY = "gemma4-dict-key";
|
||||
static constexpr const char * DICT_KEY_NAME = "gemma4-dict-key-name";
|
||||
|
||||
common_chat_peg_gemma4_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {}
|
||||
virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);
|
||||
private:
|
||||
|
||||
@@ -1,864 +0,0 @@
|
||||
#include "chat-tts-norm.h"
|
||||
|
||||
#include "jinja/runtime.h"
|
||||
#include "jinja/value.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification
|
||||
// keep this file in sync with the python reference: scripts/chat_tts_norm.py
|
||||
|
||||
using namespace jinja;
|
||||
|
||||
//
|
||||
// character classes
|
||||
// they are defined with explicit codepoint ranges, independent of any unicode database version,
|
||||
// so that the C++ and python implementations agree on every codepoint
|
||||
//
|
||||
|
||||
struct tts_norm_range {
|
||||
uint32_t lo;
|
||||
uint32_t hi;
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_SPACE[] = {
|
||||
{0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680},
|
||||
{0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_PUNCT[] = {
|
||||
{0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E},
|
||||
{0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7},
|
||||
{0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF},
|
||||
{0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB},
|
||||
{0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B},
|
||||
{0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65},
|
||||
{0x1F000, 0x1FAFF},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_CONTROL[] = {
|
||||
{0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_HAN[] = {
|
||||
{0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B},
|
||||
{0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF},
|
||||
};
|
||||
|
||||
template<size_t N>
|
||||
static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) {
|
||||
for (const auto & r : ranges) {
|
||||
if (c >= r.lo && c <= r.hi) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_alpha(uint32_t c) {
|
||||
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
|
||||
return true;
|
||||
}
|
||||
if (c == 0xAA || c == 0xB5 || c == 0xBA) {
|
||||
return true;
|
||||
}
|
||||
return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c);
|
||||
}
|
||||
|
||||
// kind is one of "dspawulhc"
|
||||
static bool tts_norm_class_match(char kind, uint32_t c) {
|
||||
switch (kind) {
|
||||
case 'd': return c >= '0' && c <= '9';
|
||||
case 's': return tts_norm_in(TTS_NORM_SPACE, c);
|
||||
case 'p': return tts_norm_in(TTS_NORM_PUNCT, c);
|
||||
case 'a': return tts_norm_is_alpha(c);
|
||||
case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9');
|
||||
case 'u': return c >= 'A' && c <= 'Z';
|
||||
case 'l': return c >= 'a' && c <= 'z';
|
||||
case 'h': return tts_norm_in(TTS_NORM_HAN, c);
|
||||
case 'c': return tts_norm_in(TTS_NORM_CONTROL, c);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_class_letter(uint32_t c) {
|
||||
if (c >= 'A' && c <= 'Z') {
|
||||
c += 'a' - 'A';
|
||||
}
|
||||
return c < 0x80 && std::string("dspawulhc").find(static_cast<char>(c)) != std::string::npos;
|
||||
}
|
||||
|
||||
static uint32_t tts_norm_ascii_lower(uint32_t c) {
|
||||
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
|
||||
}
|
||||
|
||||
static uint32_t tts_norm_ascii_swapcase(uint32_t c) {
|
||||
if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
|
||||
if (c >= 'a' && c <= 'z') return c - ('a' - 'A');
|
||||
return c;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_ascii_alnum(uint32_t c) {
|
||||
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||||
}
|
||||
|
||||
// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size
|
||||
static void tts_norm_decode_utf8(const std::string & s, std::vector<uint32_t> & cpts, std::vector<size_t> & offs) {
|
||||
cpts.clear();
|
||||
offs.clear();
|
||||
size_t i = 0;
|
||||
const size_t n = s.size();
|
||||
while (i < n) {
|
||||
offs.push_back(i);
|
||||
const unsigned char c = static_cast<unsigned char>(s[i]);
|
||||
uint32_t cp = 0;
|
||||
size_t len = 0;
|
||||
if (c < 0x80) {
|
||||
cp = c; len = 1;
|
||||
} else if ((c & 0xE0) == 0xC0) {
|
||||
cp = c & 0x1F; len = 2;
|
||||
} else if ((c & 0xF0) == 0xE0) {
|
||||
cp = c & 0x0F; len = 3;
|
||||
} else if ((c & 0xF8) == 0xF0) {
|
||||
cp = c & 0x07; len = 4;
|
||||
}
|
||||
bool ok = len > 0 && i + len <= n;
|
||||
for (size_t k = 1; ok && k < len; k++) {
|
||||
const unsigned char cc = static_cast<unsigned char>(s[i + k]);
|
||||
if ((cc & 0xC0) != 0x80) {
|
||||
ok = false;
|
||||
} else {
|
||||
cp = (cp << 6) | (cc & 0x3F);
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
cpts.push_back(0xFFFD);
|
||||
i += 1;
|
||||
} else {
|
||||
cpts.push_back(cp);
|
||||
i += len;
|
||||
}
|
||||
}
|
||||
offs.push_back(n);
|
||||
}
|
||||
|
||||
//
|
||||
// AST
|
||||
//
|
||||
|
||||
enum class tts_norm_op {
|
||||
LITERAL, // one codepoint
|
||||
SET, // one codepoint from a set; class escapes like %d are sets with a single class
|
||||
ANY, // any codepoint
|
||||
START, // ^
|
||||
END, // $
|
||||
BOUNDARY, // %b
|
||||
GROUP, // (...), (?:...), (?<name>...)
|
||||
LOOKAHEAD, // (?=...), (?!...)
|
||||
LOOKBEHIND, // (?<=c), (?<!c) - exactly one codepoint
|
||||
};
|
||||
|
||||
struct tts_norm_class {
|
||||
char kind;
|
||||
bool neg;
|
||||
};
|
||||
|
||||
struct tts_norm_set {
|
||||
bool neg = false;
|
||||
std::vector<std::pair<uint32_t, uint32_t>> ranges;
|
||||
std::vector<tts_norm_class> classes;
|
||||
};
|
||||
|
||||
struct tts_norm_item;
|
||||
using tts_norm_seq = std::vector<tts_norm_item>;
|
||||
|
||||
struct tts_norm_pattern_node {
|
||||
tts_norm_op op = tts_norm_op::GROUP;
|
||||
uint32_t cp = 0; // LITERAL
|
||||
tts_norm_set set; // SET
|
||||
std::vector<tts_norm_seq> alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item
|
||||
int capture = -1; // GROUP: capture index, -1 for non-capturing
|
||||
bool neg = false; // LOOKAHEAD, LOOKBEHIND
|
||||
};
|
||||
|
||||
static constexpr int TTS_NORM_INF = INT_MAX;
|
||||
static constexpr int TTS_NORM_MAX_REPEAT = 1000;
|
||||
static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096;
|
||||
|
||||
struct tts_norm_item {
|
||||
tts_norm_pattern_node atom;
|
||||
int min = 1;
|
||||
int max = 1;
|
||||
};
|
||||
|
||||
static bool tts_norm_is_zero_width(tts_norm_op op) {
|
||||
return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY
|
||||
|| op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND;
|
||||
}
|
||||
|
||||
//
|
||||
// parser
|
||||
//
|
||||
|
||||
struct tts_norm_parser {
|
||||
std::vector<uint32_t> s;
|
||||
size_t p = 0;
|
||||
size_t & n_caps;
|
||||
|
||||
tts_norm_parser(std::vector<uint32_t> src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {}
|
||||
|
||||
[[noreturn]] void fail(const std::string & msg) const {
|
||||
throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p));
|
||||
}
|
||||
|
||||
bool eof() const { return p >= s.size(); }
|
||||
uint32_t peek() const { return s[p]; }
|
||||
bool peek_is(uint32_t c) const { return !eof() && s[p] == c; }
|
||||
|
||||
void expect(uint32_t c, const char * what) {
|
||||
if (!peek_is(c)) {
|
||||
fail(std::string("expected ") + what);
|
||||
}
|
||||
p++;
|
||||
}
|
||||
|
||||
std::vector<tts_norm_seq> parse_alts() {
|
||||
std::vector<tts_norm_seq> alts;
|
||||
alts.push_back(parse_seq());
|
||||
while (peek_is('|')) {
|
||||
p++;
|
||||
alts.push_back(parse_seq());
|
||||
}
|
||||
return alts;
|
||||
}
|
||||
|
||||
tts_norm_seq parse_seq() {
|
||||
tts_norm_seq seq;
|
||||
while (!eof() && peek() != '|' && peek() != ')') {
|
||||
tts_norm_item item;
|
||||
item.atom = parse_atom();
|
||||
parse_quantifier(item);
|
||||
seq.push_back(std::move(item));
|
||||
}
|
||||
return seq;
|
||||
}
|
||||
|
||||
int parse_int() {
|
||||
if (eof() || peek() < '0' || peek() > '9') {
|
||||
fail("expected number in quantifier");
|
||||
}
|
||||
int v = 0;
|
||||
while (!eof() && peek() >= '0' && peek() <= '9') {
|
||||
v = v * 10 + static_cast<int>(peek() - '0');
|
||||
if (v > TTS_NORM_MAX_REPEAT) {
|
||||
fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT));
|
||||
}
|
||||
p++;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
void parse_quantifier(tts_norm_item & item) {
|
||||
if (eof()) {
|
||||
return;
|
||||
}
|
||||
const uint32_t c = peek();
|
||||
if (c != '?' && c != '*' && c != '+' && c != '{') {
|
||||
return;
|
||||
}
|
||||
if (tts_norm_is_zero_width(item.atom.op)) {
|
||||
fail("zero-width assertion cannot be repeated");
|
||||
}
|
||||
p++;
|
||||
if (c == '?') {
|
||||
item.min = 0; item.max = 1;
|
||||
} else if (c == '*') {
|
||||
item.min = 0; item.max = TTS_NORM_INF;
|
||||
} else if (c == '+') {
|
||||
item.min = 1; item.max = TTS_NORM_INF;
|
||||
} else {
|
||||
item.min = parse_int();
|
||||
item.max = item.min;
|
||||
if (peek_is(',')) {
|
||||
p++;
|
||||
item.max = peek_is('}') ? TTS_NORM_INF : parse_int();
|
||||
}
|
||||
expect('}', "'}' to close quantifier");
|
||||
if (item.max < item.min) {
|
||||
fail("quantifier max is smaller than min");
|
||||
}
|
||||
}
|
||||
if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) {
|
||||
fail("multiple quantifiers");
|
||||
}
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_group() {
|
||||
// '(' already consumed
|
||||
tts_norm_pattern_node node;
|
||||
node.op = tts_norm_op::GROUP;
|
||||
if (peek_is('?')) {
|
||||
p++;
|
||||
if (eof()) {
|
||||
fail("unterminated group");
|
||||
}
|
||||
const uint32_t c = peek();
|
||||
if (c == ':') {
|
||||
p++;
|
||||
} else if (c == '=' || c == '!') {
|
||||
p++;
|
||||
node.op = tts_norm_op::LOOKAHEAD;
|
||||
node.neg = c == '!';
|
||||
} else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) {
|
||||
node.op = tts_norm_op::LOOKBEHIND;
|
||||
node.neg = s[p + 1] == '!';
|
||||
p += 2;
|
||||
tts_norm_item item;
|
||||
if (eof() || peek() == ')') {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
item.atom = parse_atom();
|
||||
if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
if (!peek_is(')')) {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
p++;
|
||||
node.alts.push_back(tts_norm_seq{});
|
||||
node.alts[0].push_back(std::move(item));
|
||||
return node;
|
||||
} else if (c == 'i') {
|
||||
fail("(?i) is only allowed at the start of the pattern");
|
||||
} else {
|
||||
fail("unknown group type");
|
||||
}
|
||||
} else {
|
||||
node.capture = static_cast<int>(n_caps++);
|
||||
}
|
||||
node.alts = parse_alts();
|
||||
expect(')', "')' to close group");
|
||||
return node;
|
||||
}
|
||||
|
||||
// after '%' inside or outside of a set
|
||||
// returns true if it is a class (stored in cls), false if it is a literal (stored in lit)
|
||||
bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) {
|
||||
boundary = false;
|
||||
if (eof()) {
|
||||
fail("dangling '%'");
|
||||
}
|
||||
const uint32_t e = s[p++];
|
||||
if (tts_norm_is_class_letter(e)) {
|
||||
cls.kind = static_cast<char>(tts_norm_ascii_lower(e));
|
||||
cls.neg = e >= 'A' && e <= 'Z';
|
||||
return true;
|
||||
}
|
||||
if (e == 'b' && !in_set) {
|
||||
boundary = true;
|
||||
return false;
|
||||
}
|
||||
if (tts_norm_is_ascii_alnum(e)) {
|
||||
p--;
|
||||
fail(std::string("unknown escape '%") + static_cast<char>(e) + "'");
|
||||
}
|
||||
lit = e;
|
||||
return false;
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_set() {
|
||||
// '[' already consumed
|
||||
tts_norm_pattern_node node;
|
||||
node.op = tts_norm_op::SET;
|
||||
if (peek_is('^')) {
|
||||
node.set.neg = true;
|
||||
p++;
|
||||
}
|
||||
bool empty = true;
|
||||
while (true) {
|
||||
if (eof()) {
|
||||
fail("unterminated set");
|
||||
}
|
||||
if (peek() == ']') {
|
||||
p++;
|
||||
break;
|
||||
}
|
||||
empty = false;
|
||||
uint32_t lo = 0;
|
||||
if (peek() == '%') {
|
||||
p++;
|
||||
tts_norm_class cls{};
|
||||
bool boundary = false;
|
||||
if (parse_escape(true, cls, lo, boundary)) {
|
||||
node.set.classes.push_back(cls);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
lo = s[p++];
|
||||
}
|
||||
uint32_t hi = lo;
|
||||
if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') {
|
||||
p++;
|
||||
if (peek() == '%') {
|
||||
p++;
|
||||
tts_norm_class cls{};
|
||||
bool boundary = false;
|
||||
if (parse_escape(true, cls, hi, boundary)) {
|
||||
fail("character class cannot be used in a range");
|
||||
}
|
||||
} else {
|
||||
hi = s[p++];
|
||||
}
|
||||
if (hi < lo) {
|
||||
fail("invalid range in set");
|
||||
}
|
||||
}
|
||||
node.set.ranges.emplace_back(lo, hi);
|
||||
}
|
||||
if (empty) {
|
||||
fail("empty set");
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_atom() {
|
||||
const uint32_t c = s[p++];
|
||||
tts_norm_pattern_node node;
|
||||
switch (c) {
|
||||
case '(':
|
||||
return parse_group();
|
||||
case '[':
|
||||
return parse_set();
|
||||
case '.':
|
||||
node.op = tts_norm_op::ANY;
|
||||
return node;
|
||||
case '^':
|
||||
node.op = tts_norm_op::START;
|
||||
return node;
|
||||
case '$':
|
||||
node.op = tts_norm_op::END;
|
||||
return node;
|
||||
case '%': {
|
||||
tts_norm_class cls{};
|
||||
uint32_t lit = 0;
|
||||
bool boundary = false;
|
||||
if (parse_escape(false, cls, lit, boundary)) {
|
||||
node.op = tts_norm_op::SET;
|
||||
node.set.classes.push_back(cls);
|
||||
} else if (boundary) {
|
||||
node.op = tts_norm_op::BOUNDARY;
|
||||
} else {
|
||||
node.op = tts_norm_op::LITERAL;
|
||||
node.cp = lit;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
case '?':
|
||||
case '*':
|
||||
case '+':
|
||||
case '{':
|
||||
p--;
|
||||
fail("nothing to repeat");
|
||||
case ']':
|
||||
case '}':
|
||||
p--;
|
||||
fail(std::string("unescaped '") + static_cast<char>(c) + "', use '%" + static_cast<char>(c) + "'");
|
||||
default:
|
||||
node.op = tts_norm_op::LITERAL;
|
||||
node.cp = c;
|
||||
return node;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// matcher
|
||||
// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive
|
||||
// (they never give back what they consumed), so there is no backtracking
|
||||
//
|
||||
|
||||
struct tts_norm_matcher {
|
||||
const std::vector<uint32_t> & t;
|
||||
bool icase;
|
||||
std::vector<std::pair<int64_t, int64_t>> caps;
|
||||
|
||||
bool is_w(size_t i) const {
|
||||
return i < t.size() && tts_norm_class_match('w', t[i]);
|
||||
}
|
||||
|
||||
bool set_match(const tts_norm_set & st, uint32_t c) const {
|
||||
auto test = [&](uint32_t x) {
|
||||
for (const auto & r : st.ranges) {
|
||||
if (x >= r.first && x <= r.second) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for (const auto & k : st.classes) {
|
||||
if (tts_norm_class_match(k.kind, x) != k.neg) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
bool m = test(c);
|
||||
if (!m && icase) {
|
||||
const uint32_t o = tts_norm_ascii_swapcase(c);
|
||||
if (o != c) {
|
||||
m = test(o);
|
||||
}
|
||||
}
|
||||
return m != st.neg;
|
||||
}
|
||||
|
||||
bool match_alts(const std::vector<tts_norm_seq> & alts, size_t i, size_t & end) {
|
||||
for (const auto & seq : alts) {
|
||||
auto saved = caps;
|
||||
if (match_seq(seq, i, end)) {
|
||||
return true;
|
||||
}
|
||||
caps = std::move(saved);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) {
|
||||
size_t pos = i;
|
||||
for (const auto & it : seq) {
|
||||
int count = 0;
|
||||
while (count < it.max) {
|
||||
const bool is_group = it.atom.op == tts_norm_op::GROUP;
|
||||
std::vector<std::pair<int64_t, int64_t>> saved;
|
||||
if (is_group) {
|
||||
saved = caps;
|
||||
}
|
||||
size_t e = pos;
|
||||
if (!match_once(it.atom, pos, e)) {
|
||||
if (is_group) {
|
||||
caps = std::move(saved);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (e == pos) {
|
||||
// an empty iteration would repeat forever, it satisfies any remaining minimum
|
||||
count = std::max(count + 1, it.min);
|
||||
break;
|
||||
}
|
||||
pos = e;
|
||||
count++;
|
||||
}
|
||||
if (count < it.min) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
end = pos;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) {
|
||||
const size_t n = t.size();
|
||||
switch (node.op) {
|
||||
case tts_norm_op::LITERAL:
|
||||
if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::SET:
|
||||
if (i < n && set_match(node.set, t[i])) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::ANY:
|
||||
if (i < n) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::START:
|
||||
end = i;
|
||||
return i == 0;
|
||||
case tts_norm_op::END:
|
||||
end = i;
|
||||
return i == n;
|
||||
case tts_norm_op::BOUNDARY:
|
||||
end = i;
|
||||
return (i > 0 && is_w(i - 1)) != is_w(i);
|
||||
case tts_norm_op::GROUP: {
|
||||
size_t e = i;
|
||||
if (!match_alts(node.alts, i, e)) {
|
||||
return false;
|
||||
}
|
||||
if (node.capture >= 0) {
|
||||
caps[node.capture] = {static_cast<int64_t>(i), static_cast<int64_t>(e)};
|
||||
}
|
||||
end = e;
|
||||
return true;
|
||||
}
|
||||
case tts_norm_op::LOOKAHEAD: {
|
||||
// captures made inside a lookahead are discarded
|
||||
auto saved = caps;
|
||||
size_t e = i;
|
||||
const bool ok = match_alts(node.alts, i, e);
|
||||
caps = std::move(saved);
|
||||
end = i;
|
||||
return ok != node.neg;
|
||||
}
|
||||
case tts_norm_op::LOOKBEHIND: {
|
||||
size_t e = 0;
|
||||
const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i;
|
||||
end = i;
|
||||
return ok != node.neg;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// common_tts_norm_pattern
|
||||
//
|
||||
|
||||
common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) {
|
||||
if (src.size() > TTS_NORM_MAX_PATTERN_LEN) {
|
||||
throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes");
|
||||
}
|
||||
std::vector<uint32_t> cpts;
|
||||
std::vector<size_t> offs;
|
||||
tts_norm_decode_utf8(src, cpts, offs);
|
||||
|
||||
tts_norm_parser parser(std::move(cpts), n_caps);
|
||||
static const std::vector<uint32_t> icase_prefix = {'(', '?', 'i', ')'};
|
||||
if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) {
|
||||
icase = true;
|
||||
parser.p = 4;
|
||||
}
|
||||
|
||||
root = std::make_unique<tts_norm_pattern_node>();
|
||||
root->op = tts_norm_op::GROUP;
|
||||
root->alts = parser.parse_alts();
|
||||
if (!parser.eof()) {
|
||||
parser.fail("unbalanced ')'");
|
||||
}
|
||||
}
|
||||
|
||||
common_tts_norm_pattern::~common_tts_norm_pattern() = default;
|
||||
common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default;
|
||||
common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default;
|
||||
|
||||
bool common_tts_norm_pattern::match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const {
|
||||
tts_norm_matcher m{cpts, icase, std::vector<std::pair<int64_t, int64_t>>(n_caps, {-1, -1})};
|
||||
size_t e = pos;
|
||||
if (!m.match_alts(root->alts, pos, e)) {
|
||||
return false;
|
||||
}
|
||||
end = e;
|
||||
groups = std::move(m.caps);
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t common_tts_norm_pattern::n_groups() const {
|
||||
return n_caps;
|
||||
}
|
||||
|
||||
//
|
||||
// scan / sub
|
||||
//
|
||||
|
||||
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules) {
|
||||
std::vector<uint32_t> cpts;
|
||||
std::vector<size_t> offs;
|
||||
tts_norm_decode_utf8(text, cpts, offs);
|
||||
|
||||
std::vector<common_tts_norm_segment> out;
|
||||
const size_t n = cpts.size();
|
||||
size_t i = 0;
|
||||
size_t pending = 0; // start of the current unmatched run
|
||||
std::vector<std::pair<int64_t, int64_t>> groups;
|
||||
|
||||
while (i < n) {
|
||||
bool matched = false;
|
||||
for (size_t r = 0; r < rules.size(); r++) {
|
||||
size_t end = i;
|
||||
if (rules[r]->match_at(cpts, i, end, groups) && end > i) {
|
||||
if (pending < i) {
|
||||
out.push_back({-1, offs[pending], offs[i], {}});
|
||||
}
|
||||
common_tts_norm_segment seg;
|
||||
seg.rule = static_cast<int>(r);
|
||||
seg.start = offs[i];
|
||||
seg.end = offs[end];
|
||||
for (const auto & g : groups) {
|
||||
if (g.first < 0) {
|
||||
seg.groups.emplace_back(std::string::npos, std::string::npos);
|
||||
} else {
|
||||
seg.groups.emplace_back(offs[g.first], offs[g.second]);
|
||||
}
|
||||
}
|
||||
out.push_back(std::move(seg));
|
||||
i = end;
|
||||
pending = i;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
if (pending < n) {
|
||||
out.push_back({-1, offs[pending], offs[n], {}});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
struct tts_norm_repl_piece {
|
||||
std::string literal;
|
||||
int group = -1; // 0 = whole match, 1.. = groups
|
||||
};
|
||||
|
||||
static std::vector<tts_norm_repl_piece> tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) {
|
||||
std::vector<tts_norm_repl_piece> pieces;
|
||||
std::string lit;
|
||||
auto flush = [&]() {
|
||||
if (!lit.empty()) {
|
||||
pieces.push_back({lit, -1});
|
||||
lit.clear();
|
||||
}
|
||||
};
|
||||
for (size_t i = 0; i < repl.size(); i++) {
|
||||
if (repl[i] != '%') {
|
||||
lit += repl[i];
|
||||
continue;
|
||||
}
|
||||
if (i + 1 >= repl.size()) {
|
||||
throw std::runtime_error("invalid TTS norm replacement: dangling '%'");
|
||||
}
|
||||
const char c = repl[++i];
|
||||
if (c == '%') {
|
||||
lit += '%';
|
||||
} else if (c >= '0' && c <= '9') {
|
||||
const int g = c - '0';
|
||||
if (static_cast<size_t>(g) > pattern.n_groups()) {
|
||||
throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist");
|
||||
}
|
||||
flush();
|
||||
pieces.push_back({"", g});
|
||||
} else {
|
||||
throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'");
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return pieces;
|
||||
}
|
||||
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) {
|
||||
const auto pieces = tts_norm_parse_repl(repl, pattern);
|
||||
std::string out;
|
||||
for (const auto & seg : common_tts_norm_scan(text, {&pattern})) {
|
||||
if (seg.rule < 0) {
|
||||
out.append(text, seg.start, seg.end - seg.start);
|
||||
continue;
|
||||
}
|
||||
for (const auto & piece : pieces) {
|
||||
if (piece.group < 0) {
|
||||
out += piece.literal;
|
||||
} else if (piece.group == 0) {
|
||||
out.append(text, seg.start, seg.end - seg.start);
|
||||
} else {
|
||||
const auto & g = seg.groups[piece.group - 1];
|
||||
if (g.first != std::string::npos) {
|
||||
out.append(text, g.first, g.second - g.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
//
|
||||
// template builtins
|
||||
//
|
||||
|
||||
// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs
|
||||
static std::vector<std::pair<std::string, std::string>> tts_norm_rules_from_value(const value & v) {
|
||||
std::vector<std::pair<std::string, std::string>> rules;
|
||||
if (is_val<value_string>(v)) {
|
||||
rules.emplace_back("match", v->as_string().str());
|
||||
} else if (is_val<value_object>(v)) {
|
||||
for (const auto & kv : v->as_ordered_object()) {
|
||||
if (!is_val<value_string>(kv.second)) {
|
||||
throw std::runtime_error("tts_norm_scan: rule pattern must be a string");
|
||||
}
|
||||
rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str());
|
||||
}
|
||||
} else if (is_val<value_array>(v)) {
|
||||
for (const auto & item : v->as_array()) {
|
||||
if (!is_val<value_array>(item) || item->as_array().size() != 2
|
||||
|| !is_val<value_string>(item->as_array()[0]) || !is_val<value_string>(item->as_array()[1])) {
|
||||
throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings");
|
||||
}
|
||||
rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str());
|
||||
}
|
||||
} else {
|
||||
throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type());
|
||||
}
|
||||
return rules;
|
||||
}
|
||||
|
||||
static value tts_norm_builtin_scan(const func_args & args) {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_val<value_string>(args.get_pos(0));
|
||||
const string text_val = args.get_pos(0)->as_string();
|
||||
const std::string text = text_val.str();
|
||||
const bool is_input = text_val.all_parts_are_input();
|
||||
|
||||
const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1));
|
||||
std::vector<common_tts_norm_pattern> patterns;
|
||||
patterns.reserve(rule_srcs.size());
|
||||
std::vector<const common_tts_norm_pattern *> ptrs;
|
||||
for (const auto & r : rule_srcs) {
|
||||
patterns.emplace_back(r.second);
|
||||
}
|
||||
for (const auto & p : patterns) {
|
||||
ptrs.push_back(&p);
|
||||
}
|
||||
|
||||
auto mk_str = [&](size_t start, size_t end) -> value {
|
||||
return mk_val<value_string>(string(text.substr(start, end - start), is_input));
|
||||
};
|
||||
|
||||
auto out = mk_val<value_array>();
|
||||
for (const auto & seg : common_tts_norm_scan(text, ptrs)) {
|
||||
auto groups = mk_val<value_array>();
|
||||
for (const auto & g : seg.groups) {
|
||||
// groups that did not participate are empty strings, so they render as nothing and are falsy
|
||||
groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second));
|
||||
}
|
||||
auto obj = mk_val<value_object>();
|
||||
obj->insert("rule", seg.rule < 0 ? value(mk_val<value_none>()) : value(mk_val<value_string>(rule_srcs[seg.rule].first)));
|
||||
obj->insert("text", mk_str(seg.start, seg.end));
|
||||
obj->insert("groups", groups);
|
||||
out->push_back(obj);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static value tts_norm_builtin_sub(const func_args & args) {
|
||||
args.ensure_count(3, 3);
|
||||
args.ensure_vals<value_string, value_string, value_string>();
|
||||
const string text_val = args.get_pos(0)->as_string();
|
||||
const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str());
|
||||
const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str());
|
||||
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx) {
|
||||
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
|
||||
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
|
||||
}
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates
|
||||
//
|
||||
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
|
||||
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
|
||||
//
|
||||
// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
|
||||
// tts_norm_sub(text, pattern, repl) -> string
|
||||
|
||||
namespace jinja {
|
||||
struct context;
|
||||
}
|
||||
|
||||
struct tts_norm_pattern_node;
|
||||
|
||||
class common_tts_norm_pattern {
|
||||
public:
|
||||
// throws std::runtime_error on invalid pattern
|
||||
explicit common_tts_norm_pattern(const std::string & src);
|
||||
~common_tts_norm_pattern();
|
||||
common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept;
|
||||
common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept;
|
||||
|
||||
// match anchored at codepoint index pos; on success, end is the codepoint index after the match
|
||||
// groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate
|
||||
bool match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const;
|
||||
|
||||
size_t n_groups() const;
|
||||
|
||||
private:
|
||||
std::unique_ptr<tts_norm_pattern_node> root;
|
||||
size_t n_caps = 0;
|
||||
bool icase = false;
|
||||
};
|
||||
|
||||
// segment produced by common_tts_norm_scan(); byte offsets into the input text
|
||||
struct common_tts_norm_segment {
|
||||
int rule = -1; // index of the matched rule, -1 for unmatched text
|
||||
size_t start = 0;
|
||||
size_t end = 0;
|
||||
std::vector<std::pair<size_t, size_t>> groups; // byte offsets; {npos, npos} for groups that did not participate
|
||||
};
|
||||
|
||||
// split text into matched and unmatched segments
|
||||
// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped
|
||||
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules);
|
||||
|
||||
// replace every non-overlapping match of pattern with repl
|
||||
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
|
||||
|
||||
// add tts_norm_scan() and tts_norm_sub() to the context of a task template
|
||||
// they are not jinja global builtins on purpose: chat templates are not affected
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx);
|
||||
+18
-38
@@ -3,7 +3,6 @@
|
||||
#include "chat-auto-parser-helpers.h"
|
||||
#include "chat-auto-parser.h"
|
||||
#include "chat-peg-parser.h"
|
||||
#include "chat-tts-norm.h"
|
||||
#include "common.h"
|
||||
#include "ggml.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
@@ -808,8 +807,6 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
|
||||
// renders a task template (e.g. "tts") with the given variables
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
|
||||
jinja::context ctx(tmpl.source());
|
||||
// text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
jinja::global_from_json(ctx, vars, false);
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
@@ -822,29 +819,21 @@ static bool is_lfm3_template(const std::string & src) {
|
||||
return src.find("<additional_tools>") != std::string::npos;
|
||||
}
|
||||
|
||||
static std::string string_lower(std::string s) {
|
||||
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return (char) std::tolower(c); });
|
||||
return s;
|
||||
}
|
||||
|
||||
// LFM3-Audio: language code -> name
|
||||
static std::string lfm3_language_name(const std::string & language) {
|
||||
// LFM3-Audio: a known language code maps to its name, no language lets the model detect it
|
||||
static std::string lfm3_asr_system_prompt(const std::string & language) {
|
||||
static const std::map<std::string, std::string> names = {
|
||||
{ "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
|
||||
{ "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
|
||||
{ "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
|
||||
{ "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
|
||||
};
|
||||
const auto it = names.find(string_lower(language));
|
||||
return it != names.end() ? it->second : language;
|
||||
}
|
||||
|
||||
// LFM3-Audio: no language = auto detect
|
||||
static std::string lfm3_asr_system_prompt(const std::string & language) {
|
||||
if (language.empty()) {
|
||||
return "Perform ASR.";
|
||||
}
|
||||
return "Perform ASR in " + lfm3_language_name(language) + ".";
|
||||
std::string code = language;
|
||||
std::transform(code.begin(), code.end(), code.begin(), [](unsigned char c) { return (char) std::tolower(c); });
|
||||
const auto it = names.find(code);
|
||||
return "Perform ASR in " + (it != names.end() ? it->second : language) + ".";
|
||||
}
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
|
||||
@@ -879,27 +868,6 @@ std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templa
|
||||
});
|
||||
}
|
||||
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
const std::string & language, const std::string & voice) {
|
||||
if (!chat_templates || !chat_templates->template_default || !is_lfm3_template(chat_templates->template_default->source())) {
|
||||
return "";
|
||||
}
|
||||
// same voices as the "tts" template
|
||||
const std::string v = string_lower(voice);
|
||||
const bool has_voice = v == "female" || v == "male";
|
||||
if (language.empty() && !has_voice) {
|
||||
return "";
|
||||
}
|
||||
std::string out = "Speak";
|
||||
if (!language.empty()) {
|
||||
out += " in " + lfm3_language_name(language);
|
||||
}
|
||||
if (has_voice) {
|
||||
out += " using a " + v + " voice";
|
||||
}
|
||||
return out + ".";
|
||||
}
|
||||
|
||||
std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
|
||||
if (!variant.empty()) {
|
||||
if (variant == "tool_use") {
|
||||
@@ -1277,6 +1245,13 @@ static common_chat_params common_chat_params_init_lfm2_5(const common_chat_templ
|
||||
return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);
|
||||
}
|
||||
|
||||
// Older gemma4 templates need their tool responses rewritten before rendering
|
||||
static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
|
||||
auto adjusted = inputs;
|
||||
workaround::convert_tool_responses_gemma4(adjusted.messages);
|
||||
return common_chat_params_init_gemma4(tmpl, adjusted);
|
||||
}
|
||||
|
||||
// Pick the dedicated handler for a template from its source, or null for the autoparser.
|
||||
// Order matters: the first match wins, and later checks assume the earlier ones did not match.
|
||||
static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {
|
||||
@@ -1411,6 +1386,11 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
// Gemma4 format detection
|
||||
if (src.find("'<|tool_call>call:'") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Gemma4\n");
|
||||
if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
|
||||
LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "
|
||||
"Consider updating to the official template.\n", __func__);
|
||||
return common_chat_params_init_gemma4_legacy;
|
||||
}
|
||||
return common_chat_params_init_gemma4;
|
||||
}
|
||||
|
||||
|
||||
@@ -500,7 +500,3 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
|
||||
// language and voice are optional, the template defines their defaults
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language = "", const std::string & voice = "");
|
||||
|
||||
// system prompt line for the speech language and voice (e.g. LFM3-Audio), empty if not used
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
const std::string & language, const std::string & voice);
|
||||
|
||||
+3
-27
@@ -1258,13 +1258,13 @@ void common_fit_apply(common_params & params, llama_model_params & mparams, llam
|
||||
cparams_dft.n_rs_seq = 0;
|
||||
|
||||
const common_fit_extra_model extra = {
|
||||
/*.paths_model =*/ params_dft.model.get_paths(),
|
||||
/*.path_model =*/ params_dft.model.path.c_str(),
|
||||
/*.mparams =*/ &mparams_dft,
|
||||
/*.cparams =*/ &cparams_dft,
|
||||
/*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
|
||||
};
|
||||
|
||||
common_fit_params(params.model.get_paths(), &mparams, &cparams,
|
||||
common_fit_params(params.model.path.c_str(), &mparams, &cparams,
|
||||
params.tensor_split,
|
||||
params.tensor_buft_overrides.data(),
|
||||
params.fit_params_target.data(),
|
||||
@@ -1280,7 +1280,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
|
||||
common_fit_apply(params, mparams, cparams);
|
||||
|
||||
llama_model * model = common_model_load(params.model.get_paths(), mparams);
|
||||
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
|
||||
if (model == NULL) {
|
||||
return;
|
||||
}
|
||||
@@ -1722,19 +1722,6 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
|
||||
return mparams;
|
||||
}
|
||||
|
||||
struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams) {
|
||||
if (paths.size() == 1) {
|
||||
return llama_model_load_from_file(paths[0].c_str(), mparams);
|
||||
}
|
||||
|
||||
std::vector<const char *> c_paths;
|
||||
c_paths.reserve(paths.size());
|
||||
for (const auto & path : paths) {
|
||||
c_paths.push_back(path.c_str());
|
||||
}
|
||||
return llama_model_load_from_splits(c_paths.data(), c_paths.size(), mparams);
|
||||
}
|
||||
|
||||
struct llama_context_params common_context_params_to_llama(const common_params & params) {
|
||||
auto cparams = llama_context_default_params();
|
||||
|
||||
@@ -2255,14 +2242,6 @@ bool common_batch::set_embd_state(int32_t idx, llama_embd state) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool common_batch::set_embd_state_row(int32_t idx, int32_t row) {
|
||||
if (idx < 0 || idx >= size() || row < 0 || tokens[idx].state.data != nullptr || tokens[idx].state_row >= 0) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].state_row = row;
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) {
|
||||
token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, { nullptr, 0, 0 }, {} };
|
||||
for (int32_t j = 0; j < n_pos; ++j) {
|
||||
@@ -2328,9 +2307,6 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.state.data) {
|
||||
llama_batch_ext_set_embd_state(res, idx, t.state); // contexts without a state input ignore it
|
||||
}
|
||||
if (t.state_row >= 0 && !llama_batch_ext_set_embd_state_row(res, idx, t.state_row)) {
|
||||
GGML_ABORT("%s: failed to set the state row of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
if (t.decision_order != 0) {
|
||||
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
|
||||
}
|
||||
|
||||
@@ -311,18 +311,6 @@ struct common_params_model {
|
||||
std::string hf_file = ""; // HF file
|
||||
std::string docker_repo = ""; // Docker repo
|
||||
|
||||
std::vector<std::string> shards;
|
||||
|
||||
std::vector<std::string> get_paths() const {
|
||||
std::vector<std::string> paths = { path };
|
||||
for (const auto & shard : shards) {
|
||||
if (std::find(paths.begin(), paths.end(), shard) == paths.end()) {
|
||||
paths.push_back(shard);
|
||||
}
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
std::string get_name() const {
|
||||
if (!hf_repo.empty()) {
|
||||
return hf_repo;
|
||||
@@ -705,7 +693,6 @@ struct common_params {
|
||||
// enable built-in tools
|
||||
std::vector<std::string> server_tools;
|
||||
std::string server_tools_runtime;
|
||||
bool server_tools_allow_unauth = false; // allow tools/MCP on a non-loopback address without API key
|
||||
|
||||
// MCP server configs (Cursor-compatible JSON)
|
||||
std::string mcp_servers_config; // path to JSON file with MCP server definitions
|
||||
@@ -723,7 +710,6 @@ struct common_params {
|
||||
|
||||
std::string slot_save_path;
|
||||
std::string media_path; // path to directory for loading media files
|
||||
bool media_allow_private_urls = false; // allow http(s) media URLs on local or private addresses
|
||||
|
||||
float slot_prompt_similarity = 0.1f;
|
||||
|
||||
@@ -1033,8 +1019,6 @@ struct llama_context_params common_context_params_to_llama(const common_params &
|
||||
// apply --fit to mparams and cparams, no-op if params.fit_params is false
|
||||
void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams);
|
||||
|
||||
struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams);
|
||||
|
||||
// clear LoRA adapters from context, then apply new list of adapters
|
||||
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
|
||||
|
||||
@@ -1110,7 +1094,6 @@ struct common_batch {
|
||||
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
int32_t embd_row = -1; // see add_embd_row(), -1 if none
|
||||
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
|
||||
};
|
||||
|
||||
std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
|
||||
@@ -1149,9 +1132,6 @@ struct common_batch {
|
||||
// attach a state embedding (e.g. the target hidden state for MTP) to the entry at idx, can only be set once per entry
|
||||
bool set_embd_state(int32_t idx, llama_embd state);
|
||||
|
||||
// the state of entry idx is row `row` of the h_nextn device buffer of the state source context, see llama_set_ctx_state_src()
|
||||
bool set_embd_state_row(int32_t idx, int32_t row);
|
||||
|
||||
// add an embedding-only entry (no token id)
|
||||
// pos points to n_pos positions
|
||||
int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);
|
||||
|
||||
+29
-46
@@ -461,55 +461,38 @@ static int common_download_file_single_online(const std::string & url,
|
||||
return finish(success ? head->status : -1);
|
||||
}
|
||||
|
||||
std::pair<long, std::vector<char>> common_remote_get_content(const std::string & url_in,
|
||||
std::pair<long, std::vector<char>> common_remote_get_content(const std::string & url,
|
||||
const common_remote_params & params) {
|
||||
static constexpr int max_redirects = 5;
|
||||
std::string url = url_in;
|
||||
for (int n_redirects = 0; ; n_redirects++) {
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
|
||||
httplib::Headers headers;
|
||||
for (const auto & h : params.headers) {
|
||||
headers.emplace(h.first, h.second);
|
||||
}
|
||||
if (headers.find("User-Agent") == headers.end()) {
|
||||
headers.emplace("User-Agent", "llama-cpp/" + std::string(llama_build_info()));
|
||||
}
|
||||
|
||||
if (params.timeout > 0) {
|
||||
cli.set_read_timeout(params.timeout, 0);
|
||||
cli.set_write_timeout(params.timeout, 0);
|
||||
}
|
||||
if (params.resolve_host) {
|
||||
// pin checked address, follow redirects manually to check each hop
|
||||
cli.set_direct_hostname_addr_map({{parts.host, params.resolve_host(parts.host)}});
|
||||
cli.set_follow_location(false);
|
||||
}
|
||||
|
||||
std::vector<char> buf;
|
||||
auto res = cli.Get(parts.path, headers,
|
||||
[&](const char *data, size_t len) {
|
||||
buf.insert(buf.end(), data, data + len);
|
||||
return params.max_size == 0 ||
|
||||
buf.size() <= static_cast<size_t>(params.max_size);
|
||||
},
|
||||
nullptr
|
||||
);
|
||||
|
||||
if (!res) {
|
||||
throw std::runtime_error("error: cannot make GET request");
|
||||
}
|
||||
|
||||
if (params.resolve_host && res->status >= 300 && res->status < 400 && res->has_header("Location")) {
|
||||
if (n_redirects >= max_redirects) {
|
||||
throw std::runtime_error("error: too many redirects");
|
||||
}
|
||||
url = common_http_resolve_location(url, res->get_header_value("Location"));
|
||||
continue;
|
||||
}
|
||||
|
||||
return { res->status, std::move(buf) };
|
||||
httplib::Headers headers;
|
||||
for (const auto & h : params.headers) {
|
||||
headers.emplace(h.first, h.second);
|
||||
}
|
||||
if (headers.find("User-Agent") == headers.end()) {
|
||||
headers.emplace("User-Agent", "llama-cpp/" + std::string(llama_build_info()));
|
||||
}
|
||||
|
||||
if (params.timeout > 0) {
|
||||
cli.set_read_timeout(params.timeout, 0);
|
||||
cli.set_write_timeout(params.timeout, 0);
|
||||
}
|
||||
|
||||
std::vector<char> buf;
|
||||
auto res = cli.Get(parts.path, headers,
|
||||
[&](const char *data, size_t len) {
|
||||
buf.insert(buf.end(), data, data + len);
|
||||
return params.max_size == 0 ||
|
||||
buf.size() <= static_cast<size_t>(params.max_size);
|
||||
},
|
||||
nullptr
|
||||
);
|
||||
|
||||
if (!res) {
|
||||
throw std::runtime_error("error: cannot make GET request");
|
||||
}
|
||||
|
||||
return { res->status, std::move(buf) };
|
||||
}
|
||||
|
||||
int common_download_file_single(const std::string & url,
|
||||
|
||||
@@ -31,8 +31,6 @@ struct common_remote_params {
|
||||
common_header_list headers;
|
||||
long timeout = 0; // in seconds, 0 means no timeout
|
||||
long max_size = 0; // unlimited if 0
|
||||
// if set: called per hop (redirects too), returns the address to connect to, throws to refuse
|
||||
std::function<std::string(const std::string & host)> resolve_host;
|
||||
};
|
||||
|
||||
// get remote file content, returns <http_code, raw_response_body>
|
||||
|
||||
+20
-20
@@ -56,7 +56,7 @@ private:
|
||||
};
|
||||
|
||||
static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
@@ -71,7 +71,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
mparams_copy.no_alloc = true;
|
||||
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
|
||||
llama_model * model = common_model_load(paths_model, mparams_copy);
|
||||
llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
|
||||
if (model == nullptr) {
|
||||
throw std::runtime_error("failed to load model");
|
||||
}
|
||||
@@ -167,7 +167,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
}
|
||||
|
||||
std::vector<common_memory_breakdown_row> common_fit_measure(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
uint32_t * n_ctx_train,
|
||||
@@ -179,7 +179,7 @@ std::vector<common_memory_breakdown_row> common_fit_measure(
|
||||
mparams_copy.no_alloc = true;
|
||||
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
|
||||
llama_model_ptr model(common_model_load(paths_model, mparams_copy));
|
||||
llama_model_ptr model(llama_model_load_from_file(path_model, mparams_copy));
|
||||
if (!model) {
|
||||
throw std::runtime_error("failed to load model");
|
||||
}
|
||||
@@ -201,7 +201,7 @@ std::vector<common_memory_breakdown_row> common_fit_measure(
|
||||
|
||||
common_memory_reports common_measure_memory(common_params & params) {
|
||||
const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR;
|
||||
const std::vector<std::string> paths_model = params.model.get_paths();
|
||||
const char * path_model = params.model.path.c_str();
|
||||
|
||||
common_memory_reports res;
|
||||
|
||||
@@ -213,7 +213,7 @@ common_memory_reports common_measure_memory(common_params & params) {
|
||||
auto mparams_fit = mparams;
|
||||
auto cparams_fit = cparams;
|
||||
common_fit_apply(params, mparams_fit, cparams_fit);
|
||||
res.info_max = common_fit_measure(paths_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
|
||||
res.info_max = common_fit_measure(path_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
|
||||
}
|
||||
|
||||
// min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0)
|
||||
@@ -231,14 +231,14 @@ common_memory_reports common_measure_memory(common_params & params) {
|
||||
res.info_min = res.info_max;
|
||||
} else {
|
||||
cparams.n_ctx = n_ctx_min;
|
||||
res.info_min = common_fit_measure(paths_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
|
||||
res.info_min = common_fit_measure(path_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
common_device_memory_data_vec common_get_device_memory_data(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
@@ -248,7 +248,7 @@ common_device_memory_data_vec common_get_device_memory_data(
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level) {
|
||||
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
|
||||
paths_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
|
||||
|
||||
common_device_memory_data_vec ret(impl.size());
|
||||
for (size_t i = 0; i < impl.size(); i++) {
|
||||
@@ -262,7 +262,7 @@ common_device_memory_data_vec common_get_device_memory_data(
|
||||
}
|
||||
|
||||
static void common_params_fit_impl(
|
||||
const std::vector<std::string> & paths_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
|
||||
const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
|
||||
float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
|
||||
size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) {
|
||||
if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) {
|
||||
@@ -308,7 +308,7 @@ static void common_params_fit_impl(
|
||||
dmds_t measured;
|
||||
try {
|
||||
measured = common_get_device_memory_data_impl(
|
||||
extra->paths_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
|
||||
extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
|
||||
} catch (const std::runtime_error & e) {
|
||||
// the extra model is optional, fit the main model alone rather than giving up
|
||||
LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
|
||||
@@ -348,7 +348,7 @@ static void common_params_fit_impl(
|
||||
// step 1: get data for default parameters and check whether any changes are necessary in the first place
|
||||
|
||||
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
|
||||
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
|
||||
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
|
||||
@@ -360,7 +360,7 @@ static void common_params_fit_impl(
|
||||
if (n_streams > 1) {
|
||||
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
|
||||
__func__, n_ctx_max, n_streams);
|
||||
dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
}
|
||||
}
|
||||
add_extra_memory(dmds_full);
|
||||
@@ -499,7 +499,7 @@ static void common_params_fit_impl(
|
||||
|
||||
int64_t sum_projected_used_min_ctx = 0;
|
||||
cparams->n_ctx = n_ctx_min_total;
|
||||
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_min_ctx);
|
||||
if (nd == 0) {
|
||||
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
|
||||
@@ -695,7 +695,7 @@ static void common_params_fit_impl(
|
||||
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
|
||||
|
||||
dmds_t dmd_nl = common_get_device_memory_data_impl(
|
||||
paths_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmd_nl);
|
||||
|
||||
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
|
||||
@@ -724,7 +724,7 @@ static void common_params_fit_impl(
|
||||
|
||||
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
|
||||
dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
|
||||
paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_cpu_moe);
|
||||
|
||||
for (size_t id = 0; id < nd; id++) {
|
||||
@@ -978,7 +978,7 @@ static void common_params_fit_impl(
|
||||
}
|
||||
|
||||
enum common_params_fit_status common_fit_params(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams,
|
||||
float * tensor_split,
|
||||
@@ -990,7 +990,7 @@ enum common_params_fit_status common_fit_params(
|
||||
const int64_t t0_us = llama_time_us();
|
||||
common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS;
|
||||
try {
|
||||
common_params_fit_impl(paths_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
|
||||
common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
|
||||
LOG_TRC("%s: successfully fit params to free device memory\n", __func__);
|
||||
} catch (const common_params_fit_exception & e) {
|
||||
LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());
|
||||
@@ -1200,7 +1200,7 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
}
|
||||
|
||||
void common_fit_print(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams) {
|
||||
std::vector<ggml_backend_dev_t> devs;
|
||||
@@ -1209,7 +1209,7 @@ void common_fit_print(
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
|
||||
auto dmd = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
|
||||
auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
|
||||
GGML_ASSERT(dmd.size() == devs.size() + 1);
|
||||
|
||||
for (size_t id = 0; id < devs.size(); id++) {
|
||||
|
||||
+6
-6
@@ -16,7 +16,7 @@ enum common_params_fit_status {
|
||||
// - its context follows the context of the main model, so its memory is measured again whenever that context changes
|
||||
// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context
|
||||
struct common_fit_extra_model {
|
||||
std::vector<std::string> paths_model;
|
||||
const char * path_model;
|
||||
llama_model_params * mparams;
|
||||
llama_context_params * cparams;
|
||||
bool shares_model;
|
||||
@@ -28,19 +28,19 @@ struct common_fit_extra_model {
|
||||
// - only parameters that have the same value as in llama_default_model_params are modified
|
||||
// with the exception of the context size which is modified if and only if equal to 0
|
||||
common_params_fit_status common_fit_params(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams,
|
||||
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
|
||||
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
|
||||
size_t * margins, // margins of memory to leave per device in bytes
|
||||
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
|
||||
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
|
||||
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
|
||||
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
|
||||
|
||||
// print estimated memory to stdout
|
||||
void common_fit_print(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams);
|
||||
|
||||
@@ -75,7 +75,7 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llam
|
||||
// load model + context with no_alloc and return the memory breakdown, params are used as-is
|
||||
// n_ctx_train and n_ctx are optional outputs; throws on failure
|
||||
std::vector<common_memory_breakdown_row> common_fit_measure(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
uint32_t * n_ctx_train,
|
||||
@@ -99,7 +99,7 @@ using common_device_memory_data_vec = std::vector<common_device_memory_data>;
|
||||
|
||||
// Load a model + context with no_alloc and return the per-device memory breakdown.
|
||||
common_device_memory_data_vec common_get_device_memory_data(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
|
||||
@@ -99,26 +99,6 @@ static common_http_url common_http_parse_url(const std::string & url) {
|
||||
return parts;
|
||||
}
|
||||
|
||||
// resolve a redirect Location against the URL it came from
|
||||
static std::string common_http_resolve_location(const std::string & base_url, const std::string & location) {
|
||||
if (location.find("://") != std::string::npos) {
|
||||
return location;
|
||||
}
|
||||
const common_http_url base = common_http_parse_url(base_url);
|
||||
if (location.compare(0, 2, "//") == 0) {
|
||||
return base.scheme + ":" + location;
|
||||
}
|
||||
const std::string origin = base.scheme + "://" + common_http_format_host(base.host) + ":" + std::to_string(base.port);
|
||||
const std::string path = base.path.substr(0, base.path.find('?'));
|
||||
if (!location.empty() && location[0] == '/') {
|
||||
return origin + location;
|
||||
}
|
||||
if (!location.empty() && location[0] == '?') {
|
||||
return origin + path + location;
|
||||
}
|
||||
return origin + path.substr(0, path.rfind('/') + 1) + location;
|
||||
}
|
||||
|
||||
// match a loopback destination: a remote proxy can never reach the client machine's own
|
||||
// loopback, so these hosts are always contacted directly, before NO_PROXY is consulted
|
||||
// (similar to Go's net/http)
|
||||
|
||||
@@ -37,7 +37,6 @@ struct token {
|
||||
call_operator, // ()
|
||||
additive_binary_operator, // + - ~
|
||||
multiplicative_binary_operator, // * / %
|
||||
power_operator, // **
|
||||
comparison_binary_operator, // < > <= >= == !=
|
||||
unary_operator, // ! - +
|
||||
comment, // {# ... #}
|
||||
@@ -72,7 +71,6 @@ static std::string type_to_string(token::type t) {
|
||||
case token::call_operator: return "call_operator";
|
||||
case token::additive_binary_operator: return "additive_binary_operator";
|
||||
case token::multiplicative_binary_operator: return "multiplicative_binary_operator";
|
||||
case token::power_operator: return "power_operator";
|
||||
case token::comparison_binary_operator: return "comparison_binary_operator";
|
||||
case token::unary_operator: return "unary_operator";
|
||||
case token::comment: return "comment";
|
||||
@@ -139,7 +137,6 @@ struct lexer {
|
||||
{"+", token::additive_binary_operator},
|
||||
{"-", token::additive_binary_operator},
|
||||
{"~", token::additive_binary_operator},
|
||||
{"**", token::power_operator},
|
||||
{"*", token::multiplicative_binary_operator},
|
||||
{"/", token::multiplicative_binary_operator},
|
||||
{"%", token::multiplicative_binary_operator},
|
||||
|
||||
+12
-27
@@ -178,7 +178,7 @@ private:
|
||||
// bool has_caller_args = false;
|
||||
if (is(token::open_paren)) {
|
||||
// Optional caller arguments, e.g. {% call(user) dump_users(...) %}
|
||||
caller_args = parse_args(false);
|
||||
caller_args = parse_args();
|
||||
// has_caller_args = true;
|
||||
}
|
||||
auto callee = parse_primary_expression();
|
||||
@@ -283,7 +283,7 @@ private:
|
||||
|
||||
statement_ptr parse_macro_statement(size_t start_pos) {
|
||||
auto name = parse_primary_expression();
|
||||
auto args = parse_args(false);
|
||||
auto args = parse_args();
|
||||
expect(token::close_statement, "Expected %}");
|
||||
statements body;
|
||||
// Keep going until we hit {% endmacro
|
||||
@@ -397,13 +397,11 @@ private:
|
||||
|
||||
statement_ptr parse_comparison_expression() {
|
||||
// NOTE: membership has same precedence as comparison
|
||||
// e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in 'apple' and 'apple' == 'b' and 'b' in 'banana')
|
||||
statements operands;
|
||||
std::vector<token> ops;
|
||||
operands.push_back(parse_additive_expression());
|
||||
size_t pos = current;
|
||||
// e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana')))
|
||||
auto left = parse_additive_expression();
|
||||
while (true) {
|
||||
token op;
|
||||
size_t start_pos = current;
|
||||
if (is_identifier("not") && peek(1).t == token::identifier && peek(1).value == "in") {
|
||||
op = {token::identifier, "not in", tokens[current].pos};
|
||||
++current; // consume 'not'
|
||||
@@ -413,17 +411,9 @@ private:
|
||||
} else if (is(token::comparison_binary_operator)) {
|
||||
op = next();
|
||||
} else break;
|
||||
ops.push_back(std::move(op));
|
||||
operands.push_back(parse_additive_expression());
|
||||
left = mk_stmt<binary_expression>(start_pos, op, std::move(left), parse_additive_expression());
|
||||
}
|
||||
if (ops.empty()) {
|
||||
return std::move(operands[0]);
|
||||
}
|
||||
if (ops.size() == 1) {
|
||||
return mk_stmt<binary_expression>(pos, ops[0], std::move(operands[0]), std::move(operands[1]));
|
||||
}
|
||||
// chained comparison, e.g. 1 <= x <= 100
|
||||
return mk_stmt<comparison_chain_expression>(pos, std::move(operands), std::move(ops));
|
||||
return left;
|
||||
}
|
||||
|
||||
statement_ptr parse_additive_expression() {
|
||||
@@ -510,22 +500,17 @@ private:
|
||||
return parse_call_expression(std::move(member)); // foo.x()()
|
||||
}
|
||||
|
||||
statements parse_args(bool allow_spread = true) {
|
||||
statements parse_args() {
|
||||
// comma-separated arguments list
|
||||
expect(token::open_paren, "Expected (");
|
||||
statements args;
|
||||
while (!is(token::close_paren)) {
|
||||
statement_ptr arg;
|
||||
// unpacking: *expr or **expr
|
||||
const auto & t = peek();
|
||||
if (t.t == token::power_operator || (t.t == token::multiplicative_binary_operator && t.value == "*")) {
|
||||
if (!allow_spread) {
|
||||
throw std::runtime_error("Unexpected token: " + t.value + " of type " + std::to_string(t.t));
|
||||
}
|
||||
// unpacking: *expr
|
||||
if (peek().t == token::multiplicative_binary_operator && peek().value == "*") {
|
||||
size_t start_pos = current;
|
||||
++current; // consume * or **
|
||||
bool is_kwargs = t.t == token::power_operator;
|
||||
arg = mk_stmt<spread_expression>(start_pos, parse_expression(), is_kwargs);
|
||||
++current; // consume *
|
||||
arg = mk_stmt<spread_expression>(start_pos, parse_expression());
|
||||
} else {
|
||||
arg = parse_expression();
|
||||
if (is(token::equals)) {
|
||||
|
||||
+30
-66
@@ -117,8 +117,20 @@ value object_literal::execute_impl(context & ctx) const {
|
||||
return obj;
|
||||
}
|
||||
|
||||
static value exec_binary_op(context & ctx, const token & op, value left_val, value right_val) {
|
||||
value binary_expression::execute_impl(context & ctx) const {
|
||||
value left_val = left->execute(ctx);
|
||||
|
||||
// Logical operators
|
||||
if (op.value == "and") {
|
||||
JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str());
|
||||
return left_val->as_bool() ? right->execute(ctx) : std::move(left_val);
|
||||
} else if (op.value == "or") {
|
||||
JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str());
|
||||
return left_val->as_bool() ? std::move(left_val) : right->execute(ctx);
|
||||
}
|
||||
|
||||
// Equality operators
|
||||
value right_val = right->execute(ctx);
|
||||
JJ_DEBUG("Executing binary expression %s '%s' %s", left_val->type().c_str(), op.value.c_str(), right_val->type().c_str());
|
||||
if (op.value == "==") {
|
||||
return mk_val<value_bool>(*left_val == *right_val);
|
||||
@@ -295,67 +307,6 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
|
||||
}
|
||||
|
||||
value binary_expression::execute_impl(context & ctx) const {
|
||||
value left_val = left->execute(ctx);
|
||||
|
||||
// Logical operators
|
||||
if (op.value == "and") {
|
||||
JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str());
|
||||
return left_val->as_bool() ? right->execute(ctx) : std::move(left_val);
|
||||
} else if (op.value == "or") {
|
||||
JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str());
|
||||
return left_val->as_bool() ? std::move(left_val) : right->execute(ctx);
|
||||
}
|
||||
|
||||
return exec_binary_op(ctx, op, std::move(left_val), right->execute(ctx));
|
||||
}
|
||||
|
||||
value comparison_chain_expression::execute_impl(context & ctx) const {
|
||||
value left_val = operands[0]->execute(ctx);
|
||||
value res;
|
||||
for (size_t i = 0; i < ops.size(); ++i) {
|
||||
value right_val = operands[i + 1]->execute(ctx);
|
||||
res = exec_binary_op(ctx, ops[i], left_val, right_val);
|
||||
if (!res->as_bool()) {
|
||||
break; // same as "and", the next operands are not executed
|
||||
}
|
||||
left_val = std::move(right_val);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
static void exec_call_args(context & ctx, const statements & arg_stmts, func_args & args) {
|
||||
for (const auto & arg_stmt : arg_stmts) {
|
||||
if (!is_stmt<spread_expression>(arg_stmt)) {
|
||||
auto arg_val = arg_stmt->execute(ctx);
|
||||
JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
|
||||
args.push_back(arg_val);
|
||||
continue;
|
||||
}
|
||||
auto spread = cast_stmt<spread_expression>(arg_stmt);
|
||||
value val = spread->argument->execute(ctx);
|
||||
JJ_DEBUG(" Unpacked argument type: %s", val->type().c_str());
|
||||
if (spread->is_kwargs) {
|
||||
if (!is_val<value_object>(val)) {
|
||||
throw std::runtime_error("Argument after ** must be an object, got " + val->type());
|
||||
}
|
||||
for (const auto & pair : val->as_ordered_object()) {
|
||||
if (!is_val<value_string>(pair.first)) {
|
||||
throw std::runtime_error("Keyword argument key must be a string, got " + pair.first->type());
|
||||
}
|
||||
args.push_back(mk_val<value_kwarg>(pair.first->as_string().str(), pair.second));
|
||||
}
|
||||
} else {
|
||||
if (!is_val<value_array>(val)) {
|
||||
throw std::runtime_error("Argument after * must be an array, got " + val->type());
|
||||
}
|
||||
for (const auto & item : val->as_array()) {
|
||||
args.push_back(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
|
||||
JJ_DEBUG("Applying filter to %s", input->type().c_str());
|
||||
|
||||
@@ -373,7 +324,9 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
|
||||
|
||||
JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
|
||||
func_args args(ctx);
|
||||
exec_call_args(ctx, call->args, args);
|
||||
for (const auto & arg_expr : call->args) {
|
||||
args.push_back(arg_expr->execute(ctx));
|
||||
}
|
||||
|
||||
return apply_filter_by_name(ctx, filter_id, input, args);
|
||||
|
||||
@@ -416,7 +369,9 @@ value test_expression::execute_impl(context & ctx) const {
|
||||
test_id = cast_stmt<identifier>(call->callee)->val;
|
||||
|
||||
JJ_DEBUG("Applying test '%s' with arguments to %s", test_id.c_str(), input->type().c_str());
|
||||
exec_call_args(ctx, call->args, args);
|
||||
for (const auto & arg_expr : call->args) {
|
||||
args.push_back(arg_expr->execute(ctx));
|
||||
}
|
||||
|
||||
} else {
|
||||
throw std::runtime_error("Invalid test expression");
|
||||
@@ -807,7 +762,12 @@ value call_statement::execute_impl(context & ctx) const {
|
||||
call_ctx.set_val("caller", mk_val<value_func>("caller", func));
|
||||
|
||||
func_args args(call_ctx);
|
||||
exec_call_args(ctx, call_expr->args, args);
|
||||
|
||||
for (const auto & arg_expr : call_expr->args) {
|
||||
auto arg_val = arg_expr->execute(ctx);
|
||||
JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
|
||||
args.push_back(arg_val);
|
||||
}
|
||||
|
||||
JJ_DEBUG("Calling macro '%s' with %zu arguments", callee_func->name.c_str(), args.count());
|
||||
return callee_func->invoke(args);
|
||||
@@ -947,7 +907,11 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
value call_expression::execute_impl(context & ctx) const {
|
||||
// gather arguments
|
||||
func_args args(ctx);
|
||||
exec_call_args(ctx, this->args, args);
|
||||
for (auto & arg_stmt : this->args) {
|
||||
auto arg_val = arg_stmt->execute(ctx);
|
||||
JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
|
||||
args.push_back(arg_val);
|
||||
}
|
||||
// execute callee
|
||||
value callee_val = callee->execute(ctx);
|
||||
if (!is_val<value_func>(callee_val)) {
|
||||
|
||||
+1
-25
@@ -471,28 +471,6 @@ struct binary_expression : public expression {
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Two or more comparisons in a row, e.g. `a < b <= c`
|
||||
* It is the same as `a < b and b <= c`, but each operand is executed only one time.
|
||||
*/
|
||||
struct comparison_chain_expression : public expression {
|
||||
statements operands;
|
||||
std::vector<token> ops;
|
||||
|
||||
comparison_chain_expression(statements && operands, std::vector<token> && ops)
|
||||
: operands(std::move(operands)), ops(std::move(ops)) {
|
||||
assert(this->operands.size() == this->ops.size() + 1);
|
||||
for (const auto & operand : this->operands) chk_type<expression>(operand);
|
||||
}
|
||||
std::string type() const override { return "ComparisonChainExpression"; }
|
||||
value execute_impl(context & ctx) const override;
|
||||
void visit(context & ctx) const override {
|
||||
ctx.visitor(false, this, {
|
||||
{"operands", stmts_to_ptr(operands)}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* An operation with two sides, separated by the | operator.
|
||||
* Operator precedence: https://github.com/pallets/jinja/issues/379#issuecomment-168076202
|
||||
@@ -655,9 +633,7 @@ struct keyword_argument_expression : public expression {
|
||||
|
||||
struct spread_expression : public expression {
|
||||
statement_ptr argument;
|
||||
bool is_kwargs;
|
||||
spread_expression(statement_ptr && argument, bool is_kwargs)
|
||||
: argument(std::move(argument)), is_kwargs(is_kwargs) {
|
||||
explicit spread_expression(statement_ptr && argument) : argument(std::move(argument)) {
|
||||
chk_type<expression>(this->argument);
|
||||
}
|
||||
std::string type() const override { return "SpreadExpression"; }
|
||||
|
||||
@@ -860,7 +860,6 @@ public:
|
||||
std::unordered_set<std::string> required;
|
||||
std::vector<std::pair<std::string, const common_chat_schema *>> properties;
|
||||
std::map<std::string, size_t> enum_values;
|
||||
std::vector<const common_chat_schema *> leaves;
|
||||
std::function<void(const common_chat_schema &, bool, int)> add_component = [&](const common_chat_schema & comp, bool is_required, int depth) {
|
||||
if (depth >= COMMON_RECURSIVE_LIMIT) {
|
||||
throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
|
||||
@@ -880,10 +879,6 @@ public:
|
||||
for (const auto & v : as<common_chat_schema_enum>(comp).values) {
|
||||
enum_values[_generate_constant_rule(v)] += 1;
|
||||
}
|
||||
} else if (is_required && comp.kind() != common_chat_schema::KIND_ANY) {
|
||||
// Direct non-structural children are used for merging. anyOf alternatives are not merged.
|
||||
// Empty schemas are skipped.
|
||||
leaves.push_back(&comp);
|
||||
}
|
||||
};
|
||||
for (const auto & child : schema.children) {
|
||||
@@ -906,38 +901,6 @@ public:
|
||||
return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
|
||||
}
|
||||
}
|
||||
|
||||
if (properties.empty() && !leaves.empty()) {
|
||||
if (leaves.size() == 1) {
|
||||
return visit(*leaves[0], name);
|
||||
}
|
||||
bool all_numeric = true;
|
||||
bool any_integer = false;
|
||||
int64_t lo = std::numeric_limits<int64_t>::min();
|
||||
int64_t hi = std::numeric_limits<int64_t>::max();
|
||||
for (const auto * leaf : leaves) {
|
||||
if (leaf->kind() == common_chat_schema::KIND_INTEGER) {
|
||||
const auto & i = as<common_chat_schema_integer>(*leaf);
|
||||
any_integer = true;
|
||||
lo = std::max(lo, i.minimum);
|
||||
hi = std::min(hi, i.maximum);
|
||||
} else if (leaf->kind() != common_chat_schema::KIND_NUMBER) {
|
||||
all_numeric = false;
|
||||
}
|
||||
}
|
||||
if (all_numeric) {
|
||||
if (lo > hi) {
|
||||
throw std::invalid_argument("allOf has an empty numeric range");
|
||||
}
|
||||
if (any_integer) {
|
||||
common_chat_schema_integer merged;
|
||||
merged.minimum = lo;
|
||||
merged.maximum = hi;
|
||||
return visit(merged, name);
|
||||
}
|
||||
return visit(*leaves[0], name);
|
||||
}
|
||||
}
|
||||
return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr));
|
||||
}
|
||||
|
||||
|
||||
+14
-48
@@ -85,124 +85,92 @@ struct common_chat_schema {
|
||||
|
||||
static const char * kind_name(node_kind kind);
|
||||
static const char * type_name(value_type type);
|
||||
|
||||
template <typename T>
|
||||
const T & as() const {
|
||||
GGML_ASSERT(kind() == T::KIND);
|
||||
return static_cast<const T &>(*this);
|
||||
}
|
||||
};
|
||||
|
||||
using common_chat_schema_ptr = std::unique_ptr<common_chat_schema>;
|
||||
|
||||
struct common_chat_schema_any : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_ANY;
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_ANY; }
|
||||
};
|
||||
|
||||
// {"$ref": "#/..."}, only references into the same document are supported
|
||||
struct common_chat_schema_ref : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_REF;
|
||||
|
||||
std::string ref;
|
||||
const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs
|
||||
|
||||
explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {}
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_REF; }
|
||||
};
|
||||
|
||||
// oneOf / anyOf, or a "type" array expanded to one alternative per type
|
||||
struct common_chat_schema_any_of : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_ANY_OF;
|
||||
|
||||
std::vector<common_chat_schema_ptr> children;
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_ANY_OF; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_all_of : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_ALL_OF;
|
||||
|
||||
std::vector<common_chat_schema_ptr> children;
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_ALL_OF; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_const : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_CONST;
|
||||
|
||||
common_json value;
|
||||
|
||||
explicit common_chat_schema_const(common_json value) : value(std::move(value)) {}
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_CONST; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_enum : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_ENUM;
|
||||
|
||||
std::vector<common_json> values;
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_ENUM; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_null : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_NULL;
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_NULL; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_boolean : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_BOOLEAN;
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_BOOLEAN; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_number : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_NUMBER;
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_NUMBER; }
|
||||
};
|
||||
|
||||
// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in
|
||||
struct common_chat_schema_integer : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_INTEGER;
|
||||
|
||||
int64_t minimum = INT64_MIN; // INT64_MIN for unbounded
|
||||
int64_t maximum = INT64_MAX; // INT64_MAX for unbounded
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_INTEGER; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_string : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_STRING;
|
||||
|
||||
std::string pattern; // empty when absent
|
||||
string_format format = FORMAT_NONE;
|
||||
int min_length = 0;
|
||||
int max_length = -1; // -1 for unbounded
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_STRING; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_array : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_ARRAY;
|
||||
|
||||
common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent
|
||||
int min_items = 0;
|
||||
int max_items = -1; // -1 for unbounded
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_ARRAY; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_tuple : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_TUPLE;
|
||||
|
||||
std::vector<common_chat_schema_ptr> items;
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_TUPLE; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_property {
|
||||
@@ -212,12 +180,10 @@ struct common_chat_schema_property {
|
||||
};
|
||||
|
||||
struct common_chat_schema_object : common_chat_schema {
|
||||
static constexpr node_kind KIND = KIND_OBJECT;
|
||||
|
||||
std::vector<common_chat_schema_property> properties; // in schema order
|
||||
common_chat_schema_ptr additional_properties; // null when not allowed
|
||||
|
||||
node_kind kind() const override { return KIND; }
|
||||
node_kind kind() const override { return KIND_OBJECT; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_document {
|
||||
|
||||
@@ -300,7 +300,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && has_tools && !require_tools;
|
||||
data.grammar_lazy = has_tools && !require_tools;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
+169
-279
@@ -1,284 +1,152 @@
|
||||
#include "parsers.h"
|
||||
|
||||
#include "log.h"
|
||||
namespace workaround {
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
// Gemma4 uses a custom tool_responses field instead of role:tool messages.
|
||||
//
|
||||
// This will transform a sequence of messages:
|
||||
// assistant(tool_call+) -> tool+ -> assistant(content)
|
||||
//
|
||||
// Into a single assistant message containing a tool_responses field:
|
||||
// assistant(content + tool_call + tool_responses)
|
||||
//
|
||||
// This is necessary for the Gemma4 chat template to properly format the prompt.
|
||||
// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4
|
||||
struct gemma4_model_turn_builder {
|
||||
json & messages;
|
||||
size_t pos;
|
||||
json tool_calls = json::array();
|
||||
json tool_responses = json::array();
|
||||
json content;
|
||||
json reasoning_content;
|
||||
|
||||
namespace {
|
||||
gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}
|
||||
|
||||
using gemma4 = common_chat_peg_gemma4_mapper;
|
||||
void collect() {
|
||||
// Collect the first assistant message
|
||||
auto & msg = messages[pos];
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
// According to the prompt formatting guide, we need to preserve reasoning_content
|
||||
// between function calls. The current chat templates do not support this, but we will do it anyway.
|
||||
reasoning_content = msg.at("reasoning_content");
|
||||
}
|
||||
for (auto & tc : msg.at("tool_calls")) {
|
||||
tool_calls.push_back(tc);
|
||||
}
|
||||
pos++;
|
||||
|
||||
struct depth_guard {
|
||||
int & depth;
|
||||
depth_guard(int & depth) : depth(depth) {
|
||||
if (++depth > COMMON_RECURSIVE_LIMIT) {
|
||||
throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
|
||||
// Collect tool call results
|
||||
while (pos < messages.size() && messages[pos].value("role", "") == "tool") {
|
||||
collect_result(messages[pos]);
|
||||
pos++;
|
||||
}
|
||||
|
||||
// Check if the next assistant message is the final message
|
||||
if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {
|
||||
auto & next = messages[pos];
|
||||
if (!has_tool_calls(next) && has_content(next)) {
|
||||
content = next.at("content");
|
||||
pos++;
|
||||
}
|
||||
}
|
||||
}
|
||||
~depth_guard() { depth--; }
|
||||
|
||||
void collect_result(const json & curr) {
|
||||
json response;
|
||||
if (curr.contains("content")) {
|
||||
const auto & content = curr.at("content");
|
||||
if (content.is_string()) {
|
||||
// Try to parse the content as JSON; fall back to raw string
|
||||
try {
|
||||
response = json::parse(content.get<std::string>());
|
||||
} catch (...) {
|
||||
response = content;
|
||||
}
|
||||
} else {
|
||||
response = content;
|
||||
}
|
||||
}
|
||||
|
||||
std::string name;
|
||||
|
||||
// Match name with corresponding tool call
|
||||
size_t idx = tool_responses.size();
|
||||
if (idx < tool_calls.size()) {
|
||||
auto & tc = tool_calls[idx];
|
||||
if (tc.contains("function")) {
|
||||
name = tc.at("function").value("name", "");
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to the tool call id
|
||||
if (name.empty()) {
|
||||
name = curr.value("tool_call_id", "");
|
||||
}
|
||||
|
||||
tool_responses.push_back({{"name", name}, {"response", response}});
|
||||
}
|
||||
|
||||
json build() {
|
||||
collect();
|
||||
|
||||
json msg = {
|
||||
{"role", "assistant"},
|
||||
{"tool_calls", tool_calls},
|
||||
};
|
||||
if (!tool_responses.empty()) {
|
||||
msg["tool_responses"] = tool_responses;
|
||||
}
|
||||
if (!content.is_null()) {
|
||||
msg["content"] = content;
|
||||
}
|
||||
if (!reasoning_content.is_null()) {
|
||||
msg["reasoning_content"] = reasoning_content;
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
|
||||
static bool has_content(const json & msg) {
|
||||
if (!msg.contains("content") || msg.at("content").is_null()) {
|
||||
return false;
|
||||
}
|
||||
const auto & content = msg.at("content");
|
||||
if (content.is_string() && !content.get<std::string>().empty()) {
|
||||
return true;
|
||||
}
|
||||
if (content.is_array() && !content.empty()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool has_tool_calls(const json & msg) {
|
||||
return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();
|
||||
}
|
||||
};
|
||||
|
||||
// Gemma 4 value parsers
|
||||
void gemma4_value_rules(common_chat_peg_builder & p) {
|
||||
// The tags are what gemma4_to_json() in the mapper reads, the rules may wrap them freely.
|
||||
p.rule("gemma4-string", p.tag(gemma4::STRING, p.literal("<|\"|>") + p.tag(gemma4::STRING_CONTENT, p.until("<|\"|>")) + p.literal("<|\"|>")));
|
||||
void convert_tool_responses_gemma4(json & messages) {
|
||||
json result = json::array();
|
||||
size_t i = 0;
|
||||
|
||||
p.rule("gemma4-bool", p.tag(gemma4::BOOL, p.json_bool()));
|
||||
while (i < messages.size()) {
|
||||
auto & msg = messages[i];
|
||||
|
||||
p.rule("gemma4-null", p.tag(gemma4::NULL_, p.json_null()));
|
||||
if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||
|
||||
!msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {
|
||||
result.push_back(msg);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
p.rule("gemma4-number", p.tag(gemma4::NUMBER, p.json_number()));
|
||||
gemma4_model_turn_builder builder(messages, i);
|
||||
result.push_back(builder.build());
|
||||
i = builder.pos;
|
||||
}
|
||||
|
||||
// Keys are bare in the template, but Gemma 4 occasionally wraps them in quotes
|
||||
p.rule("gemma4-dict-key", p.tag(gemma4::DICT_KEY, p.choice({
|
||||
p.literal("<|\"|>") + p.tag(gemma4::DICT_KEY_NAME, p.until("<|\"|>")) + p.literal("<|\"|>"),
|
||||
p.literal("\"") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^\"]", 1, -1)) + p.literal("\""),
|
||||
p.literal("'") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^']", 1, -1)) + p.literal("'"),
|
||||
p.tag(gemma4::DICT_KEY_NAME, p.chars("[^:}]", 1, -1)),
|
||||
}) + p.literal(":")));
|
||||
|
||||
p.rule("gemma4-dict-kv", p.tag(gemma4::DICT_KV, p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")));
|
||||
|
||||
p.rule("gemma4-dict", [&]() {
|
||||
auto ws = p.space();
|
||||
auto member = p.ref("gemma4-dict-kv");
|
||||
auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
|
||||
return p.tag(gemma4::DICT, p.sequence({
|
||||
p.literal("{"), ws,
|
||||
p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
|
||||
}));
|
||||
});
|
||||
|
||||
p.rule("gemma4-array", [&]() {
|
||||
auto ws = p.space();
|
||||
auto value = p.ref("gemma4-value");
|
||||
auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
|
||||
return p.tag(gemma4::ARRAY, p.sequence({
|
||||
p.literal("["), ws,
|
||||
p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
|
||||
}));
|
||||
});
|
||||
|
||||
p.rule("gemma4-value", [&]() {
|
||||
return p.choice({
|
||||
p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
|
||||
p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
|
||||
});
|
||||
});
|
||||
messages = result;
|
||||
}
|
||||
|
||||
// Gemma 4 uses different delimiters and cannot leverage the json-schema-to-grammar implementation.
|
||||
// This implements a subset to enforce properties.
|
||||
class gemma4_schema_parser {
|
||||
common_chat_peg_builder & p_;
|
||||
std::string root_;
|
||||
int depth_ = 0;
|
||||
|
||||
std::map<const common_chat_schema *, std::string> refs_;
|
||||
|
||||
public:
|
||||
gemma4_schema_parser(common_chat_peg_builder & p, std::string root) : p_(p), root_(std::move(root)) {}
|
||||
|
||||
common_peg_parser visit(const common_chat_schema & node) { return visit(node, root_); }
|
||||
|
||||
private:
|
||||
common_peg_parser visit(const common_chat_schema & node, const std::string & name) {
|
||||
depth_guard guard(depth_);
|
||||
switch (node.kind()) {
|
||||
case common_chat_schema::KIND_REF: return visit_ref(node.as<common_chat_schema_ref>());
|
||||
case common_chat_schema::KIND_ANY_OF: return visit_any_of(node.as<common_chat_schema_any_of>(), name);
|
||||
case common_chat_schema::KIND_ALL_OF: return visit_all_of(node.as<common_chat_schema_all_of>(), name);
|
||||
case common_chat_schema::KIND_ARRAY: return visit_array(node.as<common_chat_schema_array>(), name);
|
||||
case common_chat_schema::KIND_TUPLE: return visit_tuple(node.as<common_chat_schema_tuple>(), name);
|
||||
case common_chat_schema::KIND_OBJECT: return visit_object(node.as<common_chat_schema_object>(), name);
|
||||
default: return p_.ref("gemma4-value");
|
||||
}
|
||||
}
|
||||
|
||||
// Whether a schema contributes a shape of its own, as opposed to accepting any value.
|
||||
static bool shaped(const common_chat_schema & node) {
|
||||
switch (node.kind()) {
|
||||
case common_chat_schema::KIND_REF:
|
||||
return node.as<common_chat_schema_ref>().target != nullptr;
|
||||
case common_chat_schema::KIND_ANY_OF:
|
||||
for (const auto & child : node.as<common_chat_schema_any_of>().children) {
|
||||
if (shaped(*child)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
case common_chat_schema::KIND_ALL_OF: {
|
||||
const auto & children = node.as<common_chat_schema_all_of>().children;
|
||||
return children.size() == 1 && shaped(*children[0]);
|
||||
}
|
||||
case common_chat_schema::KIND_ARRAY:
|
||||
case common_chat_schema::KIND_TUPLE:
|
||||
case common_chat_schema::KIND_OBJECT:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// The template writes keys bare. Models keep that for identifiers and quote anything else,
|
||||
// with single or double quotes or the string delimiter, so those are accepted as well.
|
||||
common_peg_parser key(const std::string & key) {
|
||||
auto name = [&]() { return p_.tag(gemma4::DICT_KEY_NAME, p_.literal(key)); };
|
||||
|
||||
bool identifier = !key.empty() && !std::isdigit(static_cast<unsigned char>(key[0]));
|
||||
for (char c : key) {
|
||||
if (!std::isalnum(static_cast<unsigned char>(c)) && c != '_') {
|
||||
identifier = false;
|
||||
}
|
||||
}
|
||||
if (identifier) {
|
||||
return p_.tag(gemma4::DICT_KEY, name() + p_.literal(":"));
|
||||
}
|
||||
|
||||
auto alts = p_.choice();
|
||||
if (!key.empty() && key.find_first_of(":}") == std::string::npos && std::string("'\"<").find(key[0]) == std::string::npos) {
|
||||
// Check if key is unambiguous
|
||||
alts |= name();
|
||||
}
|
||||
// Gemma 4 may emit keys in its string delimiters or quotes
|
||||
alts |= p_.literal("<|\"|>") + name() + p_.literal("<|\"|>");
|
||||
if (key.find('\'') == std::string::npos) {
|
||||
alts |= p_.literal("'") + name() + p_.literal("'");
|
||||
}
|
||||
if (key.find('"') == std::string::npos) {
|
||||
alts |= p_.literal("\"") + name() + p_.literal("\"");
|
||||
}
|
||||
return p_.tag(gemma4::DICT_KEY, alts + p_.literal(":"));
|
||||
}
|
||||
|
||||
common_peg_parser visit_ref(const common_chat_schema_ref & ref) {
|
||||
if (!ref.target) {
|
||||
return p_.ref("gemma4-value");
|
||||
}
|
||||
auto it = refs_.find(ref.target);
|
||||
if (it != refs_.end()) {
|
||||
return p_.ref(it->second);
|
||||
}
|
||||
// A target reached from several places gets one rule, named by the order it was first seen.
|
||||
auto rule = root_ + "-ref-" + std::to_string(refs_.size());
|
||||
refs_[ref.target] = rule;
|
||||
return p_.rule(rule, [&]() { return visit(*ref.target, rule + "-value"); });
|
||||
}
|
||||
|
||||
// Shaped alternatives are tried first, then anything else collapses into one generic value.
|
||||
common_peg_parser visit_any_of(const common_chat_schema_any_of & node, const std::string & name) {
|
||||
if (!shaped(node)) {
|
||||
return p_.ref("gemma4-value");
|
||||
}
|
||||
if (node.children.size() == 1) {
|
||||
return visit(*node.children[0], name);
|
||||
}
|
||||
auto alts = p_.choice();
|
||||
bool loose = false;
|
||||
for (size_t i = 0; i < node.children.size(); i++) {
|
||||
if (shaped(*node.children[i])) {
|
||||
alts |= visit(*node.children[i], name + "-" + std::to_string(i));
|
||||
} else {
|
||||
loose = true;
|
||||
}
|
||||
}
|
||||
if (loose) {
|
||||
alts |= p_.ref("gemma4-value");
|
||||
}
|
||||
return p_.rule(name, alts);
|
||||
}
|
||||
|
||||
common_peg_parser visit_all_of(const common_chat_schema_all_of & node, const std::string & name) {
|
||||
if (node.children.size() == 1) {
|
||||
return visit(*node.children[0], name);
|
||||
}
|
||||
return p_.ref("gemma4-value");
|
||||
}
|
||||
|
||||
common_peg_parser visit_array(const common_chat_schema_array & node, const std::string & name) {
|
||||
auto ws = p_.space();
|
||||
auto item = node.items && shaped(*node.items) ? visit(*node.items, name + "-item") : p_.ref("gemma4-value");
|
||||
auto elements = item + p_.zero_or_more(p_.literal(",") + ws + item);
|
||||
return p_.rule(name, p_.tag(gemma4::ARRAY,
|
||||
p_.literal("[") + ws + (p_.literal("]") | (elements + ws + p_.literal("]")))));
|
||||
}
|
||||
|
||||
common_peg_parser visit_tuple(const common_chat_schema_tuple & node, const std::string & name) {
|
||||
auto ws = p_.space();
|
||||
auto items = p_.sequence();
|
||||
for (size_t i = 0; i < node.items.size(); i++) {
|
||||
if (i > 0) {
|
||||
items = items + p_.literal(",") + ws;
|
||||
}
|
||||
items = items + visit(*node.items[i], name + "-" + std::to_string(i));
|
||||
}
|
||||
return p_.rule(name, p_.tag(gemma4::ARRAY, p_.literal("[") + ws + items + ws + p_.literal("]")));
|
||||
}
|
||||
|
||||
common_peg_parser visit_object(const common_chat_schema_object & node, const std::string & name) {
|
||||
auto ws = p_.space();
|
||||
|
||||
std::vector<const common_chat_schema_property *> props;
|
||||
for (const auto & prop : node.properties) {
|
||||
props.push_back(&prop);
|
||||
}
|
||||
|
||||
// The Gemma 4 jinja template sorts properties, so our rules need to match.
|
||||
auto lower = [](const std::string & s) {
|
||||
std::string out = s;
|
||||
for (auto & c : out) {
|
||||
c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
|
||||
}
|
||||
return out;
|
||||
};
|
||||
std::stable_sort(props.begin(), props.end(), [&](const auto * a, const auto * b) {
|
||||
return lower(a->name) < lower(b->name);
|
||||
});
|
||||
|
||||
if (props.empty()) {
|
||||
if (node.additional_properties) {
|
||||
return p_.ref("gemma4-dict");
|
||||
}
|
||||
return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + p_.literal("}")));
|
||||
}
|
||||
|
||||
std::optional<common_peg_parser> rest;
|
||||
std::optional<common_peg_parser> first;
|
||||
if (node.additional_properties) {
|
||||
auto kv = p_.ref("gemma4-dict-kv");
|
||||
rest = p_.zero_or_more(p_.literal(",") + ws + kv);
|
||||
first = p_.optional(kv + *rest);
|
||||
}
|
||||
|
||||
auto then_rest = [&](const common_peg_parser & parser) {
|
||||
return rest ? parser + *rest : parser;
|
||||
};
|
||||
|
||||
for (size_t i = props.size(); i-- > 0;) {
|
||||
const auto & prop = *props[i];
|
||||
auto kv = p_.tag(gemma4::DICT_KV, key(prop.name) + ws + visit(*prop.schema, name + "-" + std::to_string(i)));
|
||||
|
||||
common_peg_parser rest_i = then_rest(p_.literal(",") + ws + kv);
|
||||
common_peg_parser first_i = then_rest(kv);
|
||||
if (!prop.required) {
|
||||
rest_i = then_rest(p_.optional(p_.literal(",") + ws + kv));
|
||||
first_i = first ? (first_i | *first) : p_.optional(first_i);
|
||||
}
|
||||
|
||||
if (i > 0) {
|
||||
rest = p_.rule(name + "-rest-" + std::to_string(i), rest_i);
|
||||
}
|
||||
first = first_i;
|
||||
}
|
||||
|
||||
return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + *first + ws + p_.literal("}")));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
}
|
||||
|
||||
common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
@@ -350,29 +218,51 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
gemma4_value_rules(p);
|
||||
// Gemma4 tool calling syntax
|
||||
// Rules should match traversal logic in gemma4_to_json()
|
||||
p.rule("gemma4-string-content", p.until("<|\"|>"));
|
||||
p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));
|
||||
p.rule("gemma4-bool", p.json_bool());
|
||||
p.rule("gemma4-null", p.json_null());
|
||||
p.rule("gemma4-number", p.json_number());
|
||||
p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));
|
||||
p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));
|
||||
p.rule("gemma4-dict", [&]() {
|
||||
auto ws = p.space();
|
||||
auto member = p.ref("gemma4-dict-kv");
|
||||
auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
|
||||
return p.sequence({
|
||||
p.literal("{"), ws,
|
||||
p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
|
||||
});
|
||||
});
|
||||
p.rule("gemma4-array", [&]() {
|
||||
auto ws = p.space();
|
||||
auto value = p.ref("gemma4-value");
|
||||
auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
|
||||
return p.sequence({
|
||||
p.literal("["), ws,
|
||||
p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
|
||||
});
|
||||
});
|
||||
p.rule("gemma4-value", [&]() {
|
||||
return p.choice({
|
||||
p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
|
||||
p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
|
||||
});
|
||||
});
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
// Arguments follow the shape of the parameter schema when it is an object,
|
||||
// see gemma4_schema_parser, and are a generic dict otherwise.
|
||||
auto args = p.ref("gemma4-dict");
|
||||
try {
|
||||
auto doc = common_chat_schema_from_json(common_chat_tool_parameters(function));
|
||||
if (doc.root->kind() == common_chat_schema::KIND_OBJECT) {
|
||||
args = gemma4_schema_parser(p, "tool-" + std::to_string(tool_index) + "-args").visit(*doc.root);
|
||||
}
|
||||
} catch (const std::exception & e) {
|
||||
LOG_WRN("%s: tool %s uses the generic argument parser: %s\n", __func__, name.c_str(), e.what());
|
||||
}
|
||||
// TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules
|
||||
// const auto & params = function.at("parameters");
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.sequence({
|
||||
p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),
|
||||
p.tool_args(args),
|
||||
p.tool_args(p.ref("gemma4-dict")),
|
||||
})));
|
||||
});
|
||||
|
||||
|
||||
@@ -48,11 +48,6 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = { THINK_END };
|
||||
|
||||
// The models sometimes close medium/low reasoning with the generic </ifm|think>
|
||||
if (think != "ifm|think") {
|
||||
data.thinking_end_tags.push_back("</ifm|think>");
|
||||
}
|
||||
|
||||
data.preserved_tokens = data.thinking_end_tags;
|
||||
data.preserved_tokens.insert(data.preserved_tokens.end(), {
|
||||
THINK_START, SECTION_START, SECTION_END, CALL_START, CALL_END,
|
||||
|
||||
@@ -109,7 +109,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
|
||||
@@ -46,10 +46,9 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template
|
||||
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
// Constrained grammar whenever tools are offered or a response format is requested.
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
// Constrained grammar whenever tools are offered.
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
@@ -76,15 +75,6 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template
|
||||
auto final_msg = p.rule("final", recipient + p.literal("<|message|>") +
|
||||
p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_json = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));
|
||||
auto response_format = p.rule("response-format",
|
||||
recipient + p.literal("<|message|>") +
|
||||
((p.literal("```json") + p.space() + response_json + p.space() + p.literal("```")) | response_json));
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto string_value = p.ac(
|
||||
p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
|
||||
@@ -142,7 +132,7 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
@@ -41,6 +41,12 @@ std::string common_chat_template_generation_prompt_impl(
|
||||
|
||||
bool is_lfm2_template(const std::string & src);
|
||||
|
||||
namespace workaround {
|
||||
|
||||
void convert_tool_responses_gemma4(json & messages);
|
||||
|
||||
}
|
||||
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
@@ -191,7 +191,7 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
|
||||
+4
-55
@@ -10,6 +10,8 @@
|
||||
#include "ngram-mod.h"
|
||||
#include "sampling.h"
|
||||
|
||||
#include "../src/llama-ext.h" // staging API: llama_set_embeddings_nextn / llama_get_embeddings_nextn_ith (used by MTP)
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
@@ -1561,13 +1563,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
||||
std::vector<int> i_last;
|
||||
std::vector<std::vector<float>> chain_h;
|
||||
|
||||
// target h rows on the device (llama_set_embeddings_nextn_dev): the catch-up in process() reads row k - 1 of the last
|
||||
// target decode in place, and row n_batch_tgt + seq_id for the first token of seq_id, uploaded from pending_h
|
||||
// pending_h stays on the host: process() and accept() fetch it before the next target decode overwrites the rows
|
||||
bool dev_rows = false;
|
||||
int32_t n_batch_tgt = 0;
|
||||
std::vector<int32_t> verify_row0; // [n_seq] device row of the first verify row, see verify_h_rows
|
||||
|
||||
common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq)
|
||||
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max)
|
||||
, params(params.draft)
|
||||
@@ -1642,27 +1637,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
||||
|
||||
verify_h.assign(n_seq, {});
|
||||
verify_h_rows.assign(n_seq, 0);
|
||||
|
||||
n_batch_tgt = (int32_t) llama_n_batch(ctx_tgt);
|
||||
verify_row0.assign(n_seq, -1);
|
||||
|
||||
// shared memory and chained heads keep the host rows
|
||||
if (!is_mem_shared && !chain_heads) {
|
||||
dev_rows = llama_set_embeddings_nextn_dev(ctx_tgt, true) && llama_set_ctx_state_src(ctx_dft, ctx_tgt);
|
||||
if (!dev_rows) {
|
||||
llama_set_ctx_state_src(ctx_dft, nullptr);
|
||||
llama_set_embeddings_nextn_dev(ctx_tgt, false);
|
||||
}
|
||||
}
|
||||
SPC_TRC("- target h rows on the device: %s\n", dev_rows ? "yes" : "no");
|
||||
}
|
||||
|
||||
~common_speculative_impl_draft_mtp() override {
|
||||
auto * ctx_dft = this->params.ctx_dft;
|
||||
if (dev_rows) {
|
||||
llama_set_ctx_state_src(ctx_dft, nullptr);
|
||||
llama_set_embeddings_nextn_dev(this->params.ctx_tgt, false);
|
||||
}
|
||||
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) {
|
||||
if (backend_chains[seq_id] == nullptr) {
|
||||
continue;
|
||||
@@ -1737,16 +1715,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
||||
// i.e. we cannot have seq_id like this: [0, 0, 0, 1, 1, 0, 1, 1]
|
||||
// ^--- this is a problem
|
||||
// TODO:this is generally true, but would be nice to assert it
|
||||
if (dev_rows) {
|
||||
GGML_ASSERT(n_tokens <= n_batch_tgt && "the target batch exceeds the h_nextn device buffer");
|
||||
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
|
||||
if (i_batch_beg[seq_id] >= 0) {
|
||||
llama_set_embeddings_nextn_rows(ctx_tgt, n_batch_tgt + seq_id, 1, pending_h[seq_id].data());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const float * h_tgt = dev_rows ? nullptr : llama_get_embeddings_nextn(ctx_tgt);
|
||||
const float * h_tgt = llama_get_embeddings_nextn(ctx_tgt);
|
||||
|
||||
for (int k = 0; k < n_tokens; ++k) {
|
||||
const auto & t = batch_in.tokens[k];
|
||||
@@ -1758,11 +1727,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
||||
? batch.add(t.id, t.pos[0], seq_id, false)
|
||||
: batch.add_embd(t.embd, t.pos.data(), seq_id, false);
|
||||
|
||||
if (dev_rows) {
|
||||
batch.set_embd_state_row(idx, k == i_batch_beg[seq_id] ? n_batch_tgt + seq_id : k - 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
const float * h_row = k == i_batch_beg[seq_id]
|
||||
? pending_h[seq_id].data()
|
||||
: h_tgt + (size_t) (k - 1) * n_embd;
|
||||
@@ -1821,14 +1785,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
||||
const int32_t h_offset = is_mem_shared && !all_rows_output ? i_batch_end[seq_id] : i_batch_beg[seq_id];
|
||||
|
||||
verify_h_rows[seq_id] = h_rows;
|
||||
|
||||
if (dev_rows) {
|
||||
// the rows stay on the device until the next target decode, only the pending row comes to the host
|
||||
verify_row0[seq_id] = h_offset;
|
||||
llama_get_embeddings_nextn_rows(ctx_tgt, h_offset + h_rows - 1, 1, pending_h[seq_id].data());
|
||||
continue;
|
||||
}
|
||||
|
||||
verify_h[seq_id].resize((size_t) h_rows * n_embd);
|
||||
|
||||
for (int32_t i = 0; i < h_rows; ++i) {
|
||||
@@ -2020,13 +1976,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
||||
}
|
||||
|
||||
const int32_t i_h = std::min<int32_t>(n_accepted, n_rows - 1);
|
||||
if (dev_rows) {
|
||||
// process() already fetched the last row
|
||||
if (i_h != n_rows - 1) {
|
||||
llama_get_embeddings_nextn_rows(params.ctx_tgt, verify_row0[seq_id] + i_h, 1, pending_h[seq_id].data());
|
||||
}
|
||||
return;
|
||||
}
|
||||
const size_t row_bytes = (size_t) n_embd * sizeof(float);
|
||||
std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes);
|
||||
}
|
||||
@@ -2856,7 +2805,7 @@ common_speculative_init_result::common_speculative_init_result(
|
||||
model_path = params.speculative.draft.mparams.path;
|
||||
LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str());
|
||||
|
||||
llama_model * model_dft = common_model_load(params.speculative.draft.mparams.get_paths(), mparams);
|
||||
llama_model * model_dft = llama_model_load_from_file(model_path.c_str(), mparams);
|
||||
if (model_dft == NULL) {
|
||||
LOG_ERR("%s: failed to load draft model, '%s'\n", __func__, model_path.c_str());
|
||||
return;
|
||||
|
||||
@@ -371,7 +371,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"InternVisionModel": "internvl",
|
||||
"JanusForConditionalGeneration": "januspro",
|
||||
"KimiK25ForConditionalGeneration": "kimivl",
|
||||
"KimiK31ForConditionalGeneration": "kimivl",
|
||||
"KimiVLForConditionalGeneration": "kimivl",
|
||||
"Lfm2AudioForConditionalGeneration": "lfm2",
|
||||
"D1OmniModel": "lfm2",
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -199,18 +198,6 @@ class SarvamMoEModel(BailingMoeV2Model):
|
||||
# - full rotary (no partial_rotary_factor)
|
||||
# - expert bias is zero-mean normalized at load time
|
||||
|
||||
def set_vocab(self):
|
||||
# BPE with byte fallback: mark <0xXX> tokens as BYTE so they decode to raw bytes
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
toktypes = [gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-F]{2}>", tok) else typ for tok, typ in zip(tokens, toktypes)]
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(tokpre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
hparams = self.hparams
|
||||
|
||||
+37
-94
@@ -7,7 +7,6 @@ import ast
|
||||
import logging
|
||||
import contextlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
@@ -32,10 +31,10 @@ import gguf
|
||||
from gguf.vocab import MistralTokenizerType, MistralVocab
|
||||
|
||||
try:
|
||||
from mistral_common.tokens.tokenizers.base import TokenizerVersion # ty: ignore[import-not-found]
|
||||
from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # ty: ignore[import-not-found]
|
||||
from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found]
|
||||
from mistral_common.tokens.tokenizers.sentencepiece import ( # ty: ignore[import-not-found]
|
||||
from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import]
|
||||
from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # type: ignore[import-not-found, ty:unresolved-import]
|
||||
from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
|
||||
from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import]
|
||||
SentencePieceTokenizer,
|
||||
)
|
||||
|
||||
@@ -134,8 +133,7 @@ class ModelBase:
|
||||
target_model_dir: Path | None = None,
|
||||
fuse_gate_up_exps: bool = False,
|
||||
fp8_as_q8: bool = False,
|
||||
fuse_qkv: bool = False,
|
||||
ple_only: bool = False, no_ple: bool = False):
|
||||
fuse_qkv: bool = False):
|
||||
if type(self) is ModelBase or \
|
||||
type(self) is TextModel or \
|
||||
type(self) is MmprojModel:
|
||||
@@ -144,11 +142,6 @@ class ModelBase:
|
||||
if self.is_mistral_format and not _mistral_common_installed:
|
||||
raise ImportError(_mistral_import_error_msg)
|
||||
|
||||
if ple_only and no_ple:
|
||||
raise ValueError("ple_only and no_ple are mutually exclusive")
|
||||
self.ple_only = ple_only
|
||||
self.no_ple = no_ple
|
||||
self._has_ple = False
|
||||
self.dir_model = dir_model
|
||||
self.ftype = ftype
|
||||
self.fname_out = fname_out
|
||||
@@ -211,8 +204,7 @@ class ModelBase:
|
||||
|
||||
# Configure GGUF Writer
|
||||
self.gguf_writer = gguf.GGUFWriter(path=None, arch=gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=self.use_temp_file,
|
||||
split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard,
|
||||
tensor_filter=self._include_ple_tensor if ple_only or no_ple else None)
|
||||
split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard)
|
||||
|
||||
# Mistral specific
|
||||
self.disable_mistral_community_chat_template = disable_mistral_community_chat_template
|
||||
@@ -273,7 +265,7 @@ class ModelBase:
|
||||
if weight_map is None or not isinstance(weight_map, dict):
|
||||
raise ValueError(f"Can't load 'weight_map' from {index_name!r}")
|
||||
tensor_names_from_index.update(weight_map.keys())
|
||||
part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None)
|
||||
part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None) # ty: ignore[invalid-assignment]
|
||||
part_names = sorted(part_dict.keys())
|
||||
else:
|
||||
weight_map = {}
|
||||
@@ -344,36 +336,6 @@ class ModelBase:
|
||||
logger.info(f" + {scale_name} (per-expert scale, shape [{len(scales)}])")
|
||||
self.gguf_writer.add_tensor(scale_name, scale_vals)
|
||||
|
||||
def _prepare_kv_cache_scales(self):
|
||||
for name in list(self.model_tensors.keys()):
|
||||
if not name.endswith((".k_scale", ".v_scale")):
|
||||
continue
|
||||
|
||||
mapped = self.tensor_map.get_type_and_name(key=name, try_suffixes=(".k_scale", ".v_scale"))
|
||||
if mapped is None:
|
||||
continue
|
||||
|
||||
tensor_type, new_name = mapped
|
||||
if tensor_type not in (gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V, gguf.MODEL_TENSOR.ATTN_QKV):
|
||||
continue
|
||||
|
||||
bid = int(new_name.split(".")[1])
|
||||
scale_type = gguf.MODEL_TENSOR.ATTN_K if name.endswith(".k_scale") else gguf.MODEL_TENSOR.ATTN_V
|
||||
suffix = ".k_scale" if name.endswith(".k_scale") else ".v_scale"
|
||||
new_name = gguf.TENSOR_NAMES[scale_type].format(bid=bid) + suffix
|
||||
|
||||
scale = LazyTorchTensor.to_eager(self.model_tensors.pop(name)())
|
||||
if scale.dtype not in (torch.bfloat16, torch.float32) or scale.numel() != 1:
|
||||
raise ValueError(f"KV cache scale {name!r} must be a scalar BF16 or FP32 tensor")
|
||||
|
||||
scale = scale.float()
|
||||
value = float(scale.item())
|
||||
if not math.isfinite(value) or value == 0.0:
|
||||
raise ValueError(f"KV cache scale {name!r} must be finite and nonzero")
|
||||
|
||||
logger.info(f" + {new_name} (KV cache scale, shape [1])")
|
||||
self.gguf_writer.add_tensor(new_name, scale.flatten().numpy())
|
||||
|
||||
def dequant_model(self):
|
||||
# If all quantized tensors were already handled (e.g. pure NVFP4), skip
|
||||
if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv", ".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")) for k in self.model_tensors):
|
||||
@@ -1420,11 +1382,6 @@ class ModelBase:
|
||||
|
||||
del experts, merged
|
||||
|
||||
def _include_ple_tensor(self, name: str) -> bool:
|
||||
is_ple = name == gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD] + ".weight"
|
||||
self._has_ple |= is_ple
|
||||
return is_ple if self.ple_only else not is_ple
|
||||
|
||||
def prepare_tensors(self):
|
||||
# detect NVFP4 quantization (ModelOpt and Compressed-tensors formats)
|
||||
quantization_config = self.hparams.get("quantization_config") or {}
|
||||
@@ -1486,8 +1443,6 @@ class ModelBase:
|
||||
unmatched = layer_algo.keys() - matched
|
||||
assert not unmatched, f"quantized_layers entries not in the model tensors: {sorted(unmatched)}"
|
||||
|
||||
self._prepare_kv_cache_scales()
|
||||
|
||||
# NVFP4 weights are repacked and written directly to gguf_writer.
|
||||
# This must run before dequant_model so NVFP4 tensors are removed
|
||||
# from model_tensors, leaving only non-NVFP4 (e.g. FP8) for dequant.
|
||||
@@ -1558,8 +1513,6 @@ class ModelBase:
|
||||
break
|
||||
|
||||
for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):
|
||||
if (self.ple_only or self.no_ple) and not self._include_ple_tensor(new_name):
|
||||
continue
|
||||
n_dims = len(data_torch.shape)
|
||||
data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@@ -1692,8 +1645,6 @@ class ModelBase:
|
||||
|
||||
def write(self):
|
||||
self.prepare_tensors()
|
||||
if (self.ple_only or self.no_ple) and not self._has_ple:
|
||||
raise ValueError("model has no per-layer embedding table to export separately")
|
||||
self.prepare_metadata(vocab_only=False)
|
||||
self.gguf_writer.write_header_to_file(path=self.fname_out)
|
||||
self.gguf_writer.write_kv_data_to_file()
|
||||
@@ -1886,8 +1837,7 @@ class TextModel(ModelBase):
|
||||
fname_default: str = gguf.naming_convention(self.metadata.name, self.metadata.basename, self.metadata.finetune, self.metadata.version, size_label=None, output_type=None, model_type="vocab")
|
||||
|
||||
# Use the default filename
|
||||
prefix = "ple-" if self.ple_only else "no-ple-" if self.no_ple else ""
|
||||
self.fname_out = self.fname_out / f"{prefix}{fname_default}.gguf"
|
||||
self.fname_out = self.fname_out / f"{fname_default}.gguf"
|
||||
else:
|
||||
# Output path is a custom defined templated filename
|
||||
# Note: `not is_dir()` is used because `.is_file()` will not detect
|
||||
@@ -1896,9 +1846,8 @@ class TextModel(ModelBase):
|
||||
# Process templated file name with the output ftype, useful with the "auto" ftype
|
||||
self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type)
|
||||
|
||||
if not self.ple_only:
|
||||
logger.info("Set model tokenizer")
|
||||
self.set_vocab()
|
||||
logger.info("Set model tokenizer")
|
||||
self.set_vocab()
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
self.gguf_writer.add_block_count(self.block_count)
|
||||
@@ -2055,16 +2004,15 @@ class TextModel(ModelBase):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tokenizer is not None
|
||||
vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))
|
||||
assert max(tokenizer.vocab.values()) < vocab_size
|
||||
vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute]
|
||||
assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute]
|
||||
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
|
||||
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute]
|
||||
added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
|
||||
|
||||
added_tokens_decoder = tokenizer.added_tokens_decoder
|
||||
added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute]
|
||||
|
||||
for i in range(vocab_size):
|
||||
if i not in reverse_vocab:
|
||||
@@ -2077,7 +2025,7 @@ class TextModel(ModelBase):
|
||||
# To avoid unexpected issues - we make sure to normalize non-normalized tokens
|
||||
if not added_tokens_decoder[i].normalized:
|
||||
previous_token = token
|
||||
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment]
|
||||
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment]
|
||||
if previous_token != token:
|
||||
logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
|
||||
|
||||
@@ -2487,7 +2435,6 @@ class TextModel(ModelBase):
|
||||
def _set_vocab_hybriddna(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
vocab = tokenizer.get_vocab()
|
||||
vocab_size = self.hparams.get("vocab_size", len(vocab))
|
||||
assert max(vocab.values()) < vocab_size
|
||||
@@ -2501,11 +2448,11 @@ class TextModel(ModelBase):
|
||||
if tokenizer.k != 6 or tail not in ("pad_a", "pad_n"):
|
||||
raise NotImplementedError(f"HybridDNATokenizer with k={tokenizer.k}, tail={tail!r}")
|
||||
for kmer in [*tokenizer.kmers, *getattr(tokenizer, "npad_tokens", [])]:
|
||||
reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000"
|
||||
reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000" # ty: ignore[unresolved-attribute]
|
||||
# older HybridDNATokenizer versions (Carbon-500M/3B/8B) do not register the base tokenizer's added tokens (<|im_start|>, <think>, ...) as their own
|
||||
# but they are still encoded as special tokens: take their types from the base tokenizer
|
||||
base_tokenizer = getattr(tokenizer, "_base_tokenizer", tokenizer)
|
||||
added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder}
|
||||
added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder} # ty: ignore[unresolved-attribute]
|
||||
|
||||
tokens: list[str] = []
|
||||
toktypes: list[int] = []
|
||||
@@ -2552,25 +2499,24 @@ class TextModel(ModelBase):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
vocab_size = hparams["vocab_size"]
|
||||
assert max(tokenizer.get_vocab().values()) < vocab_size
|
||||
assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute]
|
||||
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
|
||||
merges = []
|
||||
vocab = {}
|
||||
mergeable_ranks = tokenizer.mergeable_ranks
|
||||
mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute]
|
||||
for token, rank in mergeable_ranks.items():
|
||||
vocab[QwenModel.token_bytes_to_string(token)] = rank
|
||||
if len(token) == 1:
|
||||
continue
|
||||
merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
|
||||
assert len(merged) == 2
|
||||
merges.append(str(' '.join(map(QwenModel.token_bytes_to_string, merged))))
|
||||
merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
|
||||
|
||||
# for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined
|
||||
added_vocab = tokenizer.special_tokens
|
||||
added_vocab = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
|
||||
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()}
|
||||
|
||||
for i in range(vocab_size):
|
||||
@@ -2593,10 +2539,10 @@ class TextModel(ModelBase):
|
||||
special_vocab.merges = merges
|
||||
# only add special tokens when they were not already loaded from config.json
|
||||
if len(special_vocab.special_token_ids) == 0:
|
||||
special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"])
|
||||
special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"])
|
||||
special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
# this one is usually not in config.json anyway
|
||||
special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"])
|
||||
special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
def _set_vocab_sentencepiece(self, add_to_gguf=True):
|
||||
@@ -2870,23 +2816,21 @@ class TextModel(ModelBase):
|
||||
def _set_vocab_glmedge(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tokenizer is not None
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(tokpre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
def _set_vocab_glm(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tokenizer is not None
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
@@ -2895,10 +2839,10 @@ class TextModel(ModelBase):
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
# Special tokens
|
||||
# Note: Using <|endoftext|> (151329) for eot causes endless generation
|
||||
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # 151331
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # 151336
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329
|
||||
special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338
|
||||
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # ty: ignore[unresolved-attribute] # 151331
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute] # 151336
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] # 151329
|
||||
special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # ty: ignore[unresolved-attribute] # 151338
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
def _set_vocab_interns1(self):
|
||||
@@ -2907,17 +2851,16 @@ class TextModel(ModelBase):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab())
|
||||
vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab()) # ty: ignore[unresolved-attribute]
|
||||
vocab_size = self.hparams.get("vocab_size", len(vocab))
|
||||
assert max(vocab.values()) < vocab_size
|
||||
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
|
||||
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()}
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
|
||||
|
||||
added_tokens_decoder = tokenizer.added_tokens_decoder
|
||||
added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute]
|
||||
|
||||
for i in range(vocab_size):
|
||||
if i not in reverse_vocab:
|
||||
@@ -2930,7 +2873,7 @@ class TextModel(ModelBase):
|
||||
# To avoid unexpected issues - we make sure to normalize non-normalized tokens
|
||||
if not added_tokens_decoder[i].normalized:
|
||||
previous_token = token
|
||||
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment]
|
||||
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment]
|
||||
if previous_token != token:
|
||||
logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
|
||||
|
||||
|
||||
+1
-2
@@ -60,8 +60,7 @@ class BeamModel(TextModel):
|
||||
# otherwise EOT is guessed from a hash-ordered scan
|
||||
from transformers import AutoTokenizer
|
||||
tok = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tok is not None
|
||||
ret_id = tok.convert_tokens_to_ids("<|return|>")
|
||||
ret_id = tok.convert_tokens_to_ids("<|return|>") # ty: ignore[unresolved-attribute]
|
||||
if isinstance(ret_id, int) and ret_id >= 0:
|
||||
self.gguf_writer.add_eot_token_id(ret_id)
|
||||
|
||||
|
||||
+10
-11
@@ -132,7 +132,6 @@ class BertModel(TextModel):
|
||||
from base64 import b64decode
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tokenizer is not None
|
||||
|
||||
with open(tokenizer_path, "r", encoding="utf-8") as fp:
|
||||
tokenizer_json = json.load(fp)
|
||||
@@ -141,13 +140,13 @@ class BertModel(TextModel):
|
||||
with open(tokenizer_config_path, "r", encoding="utf-8") as fp:
|
||||
tokenizer_config_json = json.load(fp)
|
||||
|
||||
add_prefix = tokenizer.add_prefix_space
|
||||
remove_whitespaces = tokenizer.clean_up_tokenization_spaces
|
||||
add_prefix = tokenizer.add_prefix_space # ty: ignore[unresolved-attribute]
|
||||
remove_whitespaces = tokenizer.clean_up_tokenization_spaces # ty: ignore[unresolved-attribute]
|
||||
precompiled_charsmap = b64decode(tokenizer_json["normalizer"]["precompiled_charsmap"])
|
||||
|
||||
vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size)
|
||||
vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size) # ty: ignore[unresolved-attribute]
|
||||
else:
|
||||
sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
|
||||
sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
|
||||
assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM
|
||||
|
||||
@@ -184,20 +183,20 @@ class BertModel(TextModel):
|
||||
scores[token_id] = score
|
||||
toktypes[token_id] = toktype
|
||||
else:
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
|
||||
unk_token = tokenizer_config_json.get("unk_token")
|
||||
unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3))
|
||||
unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3)) # ty: ignore[no-matching-overload]
|
||||
|
||||
for token_id in range(tokenizer.vocab_size):
|
||||
piece = tokenizer._convert_id_to_token(token_id)
|
||||
if (piece := tokenizer._convert_id_to_token(token_id)) is not None:
|
||||
for token_id in range(tokenizer.vocab_size): # ty: ignore[unresolved-attribute]
|
||||
piece = tokenizer._convert_id_to_token(token_id) # ty: ignore[unresolved-attribute]
|
||||
if (piece := tokenizer._convert_id_to_token(token_id)) is not None: # ty: ignore[unresolved-attribute]
|
||||
text = piece.encode("utf-8")
|
||||
score = tokenizer_json["model"]["vocab"][token_id][1]
|
||||
|
||||
toktype = SentencePieceTokenTypes.NORMAL
|
||||
if token_id == unk_token_id:
|
||||
toktype = SentencePieceTokenTypes.UNKNOWN
|
||||
elif token_id in tokenizer.all_special_ids:
|
||||
elif token_id in tokenizer.all_special_ids: # ty: ignore[unresolved-attribute]
|
||||
toktype = SentencePieceTokenTypes.CONTROL
|
||||
elif token_id in added_vocab.values():
|
||||
toktype = SentencePieceTokenTypes.USER_DEFINED
|
||||
|
||||
+16
-19
@@ -22,13 +22,12 @@ class ChatGLMModel(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab()))
|
||||
assert max(tokenizer.get_vocab().values()) < vocab_size
|
||||
vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab())) # ty: ignore[unresolved-attribute]
|
||||
assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute]
|
||||
role_special_tokens = ["<|system|>", "<|user|>", "<|assistant|>", "<|observation|>"]
|
||||
special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + role_special_tokens
|
||||
for token_id in range(vocab_size):
|
||||
piece = tokenizer._convert_id_to_token(token_id)
|
||||
piece = tokenizer._convert_id_to_token(token_id) # ty: ignore[unresolved-attribute]
|
||||
if token_id == 0:
|
||||
piece = "<unk>"
|
||||
elif token_id == 1:
|
||||
@@ -36,18 +35,17 @@ class ChatGLMModel(TextModel):
|
||||
elif token_id == 2:
|
||||
piece = "<eos>"
|
||||
|
||||
assert isinstance(piece, str)
|
||||
text = piece.encode("utf-8")
|
||||
text = piece.encode("utf-8") # ty: ignore[unresolved-attribute]
|
||||
score = 0.0
|
||||
# Referencing the tokenizer Python implementation(https://huggingface.co/THUDM/chatglm3-6b/blob/main/tokenization_chatglm.py),
|
||||
# it is only valid if it is less than tokenizer.tokenizer.sp_model.vocab_size()
|
||||
if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size():
|
||||
score = tokenizer.tokenizer.sp_model.get_score(token_id)
|
||||
if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size(): # ty: ignore[unresolved-attribute, invalid-argument-type]
|
||||
score = tokenizer.tokenizer.sp_model.get_score(token_id) # ty: ignore[unresolved-attribute]
|
||||
|
||||
if token_id >= tokenizer.tokenizer.sp_model.vocab_size():
|
||||
if token_id >= tokenizer.tokenizer.sp_model.vocab_size(): # ty: ignore[unresolved-attribute]
|
||||
if piece in special_tokens:
|
||||
toktype = SentencePieceTokenTypes.CONTROL
|
||||
elif len(piece) == 0:
|
||||
elif len(piece) == 0: # ty: ignore[invalid-argument-type]
|
||||
text = f"[PAD{token_id}]".encode("utf-8")
|
||||
toktype = SentencePieceTokenTypes.UNUSED
|
||||
else:
|
||||
@@ -58,13 +56,13 @@ class ChatGLMModel(TextModel):
|
||||
continue
|
||||
|
||||
toktype = SentencePieceTokenTypes.NORMAL
|
||||
if tokenizer.tokenizer.sp_model.is_unknown(token_id):
|
||||
if tokenizer.tokenizer.sp_model.is_unknown(token_id): # ty: ignore[unresolved-attribute]
|
||||
toktype = SentencePieceTokenTypes.UNKNOWN
|
||||
elif tokenizer.tokenizer.sp_model.is_control(token_id):
|
||||
elif tokenizer.tokenizer.sp_model.is_control(token_id): # ty: ignore[unresolved-attribute]
|
||||
toktype = SentencePieceTokenTypes.CONTROL
|
||||
elif tokenizer.tokenizer.sp_model.is_unused(token_id):
|
||||
elif tokenizer.tokenizer.sp_model.is_unused(token_id): # ty: ignore[unresolved-attribute]
|
||||
toktype = SentencePieceTokenTypes.UNUSED
|
||||
elif tokenizer.tokenizer.sp_model.is_byte(token_id):
|
||||
elif tokenizer.tokenizer.sp_model.is_byte(token_id): # ty: ignore[unresolved-attribute]
|
||||
toktype = SentencePieceTokenTypes.BYTE
|
||||
|
||||
tokens.append(text)
|
||||
@@ -117,9 +115,8 @@ class ChatGLMModel(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
vocab_size = hparams.get("padded_vocab_size",hparams["vocab_size"])
|
||||
assert max(tokenizer.get_vocab().values()) < vocab_size
|
||||
assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute]
|
||||
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
@@ -128,10 +125,10 @@ class ChatGLMModel(TextModel):
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
# only add special tokens when they were not already loaded from config.json
|
||||
special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
|
||||
special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute]
|
||||
# this one is usually not in config.json anyway
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
|
||||
@@ -277,14 +277,13 @@ class DeepseekV2Model(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
|
||||
if tokpre == "kimi-k2":
|
||||
# Build merges list using the approach similar to HunYuanMoE
|
||||
merges = []
|
||||
vocab = {}
|
||||
mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[missing-union-attribute]
|
||||
mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[unresolved-attribute]
|
||||
for token, rank in mergeable_ranks.items():
|
||||
vocab[QwenModel.token_bytes_to_string(token)] = rank
|
||||
if len(token) == 1:
|
||||
@@ -295,7 +294,7 @@ class DeepseekV2Model(TextModel):
|
||||
|
||||
# Build token list
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
special_tokens = tokenizer.special_tokens
|
||||
special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
|
||||
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
|
||||
tokens: list[str] = []
|
||||
toktypes: list[int] = []
|
||||
|
||||
@@ -62,7 +62,6 @@ def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
|
||||
assert tokenizer is not None
|
||||
sentinel = "\ue000"
|
||||
normalizer = normalizers.Sequence([
|
||||
normalizers.NFKC(),
|
||||
@@ -81,7 +80,6 @@ def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
|
||||
text = backend.decode([token_id], skip_special_tokens=False)
|
||||
if "\ufffd" in text:
|
||||
key = backend.id_to_token(token_id)
|
||||
assert key is not None
|
||||
else:
|
||||
normalized = normalizer.normalize_str(text)
|
||||
key = normalized if normalized else text
|
||||
|
||||
+1
-2
@@ -75,14 +75,13 @@ class Dots3NoteModel(DeepseekV2Model):
|
||||
def set_vocab(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tokenizer is not None
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(tokpre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"])
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
@classmethod
|
||||
|
||||
+2
-3
@@ -19,16 +19,15 @@ class DreamModel(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
|
||||
vocab_dict = tokenizer.get_vocab()
|
||||
vocab_dict = tokenizer.get_vocab() # ty: ignore[unresolved-attribute]
|
||||
vocab_size = self.hparams.get("vocab_size", len(vocab_dict))
|
||||
assert max(vocab_dict.values()) < vocab_size
|
||||
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
|
||||
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
|
||||
|
||||
for i in range(vocab_size):
|
||||
if i not in reverse_vocab:
|
||||
|
||||
+8
-10
@@ -31,7 +31,6 @@ class Glm4Model(TextModel):
|
||||
def set_vocab(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
@@ -39,10 +38,10 @@ class Glm4Model(TextModel):
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
@@ -392,17 +391,16 @@ class SolarOpenModel(Glm4MoeModel):
|
||||
def set_vocab(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tokenizer is not None
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(tokpre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"])
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"])
|
||||
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"])
|
||||
special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"]) # ty: ignore[unresolved-attribute]
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
|
||||
|
||||
@@ -85,16 +85,6 @@ class GptOssModel(TextModel):
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
# Normalize erroneous GPT-OSS MXFP4 tensor naming.
|
||||
# https://github.com/p-e-w/heretic/pull/211#issuecomment-5094370390
|
||||
if "mlp.experts." in name:
|
||||
for proj in ("down_proj", "gate_up_proj"):
|
||||
name = name.replace(
|
||||
f"{proj}_blocks_blocks", f"{proj}_blocks"
|
||||
).replace(
|
||||
f"{proj}_blocks_scales", f"{proj}_scales"
|
||||
)
|
||||
|
||||
if "sinks" in name:
|
||||
name += ".weight"
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ class HunYuanMoEModel(TextModel):
|
||||
def set_vocab(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
|
||||
# 1. Get the pre-tokenizer identifier hash
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
@@ -32,7 +31,7 @@ class HunYuanMoEModel(TextModel):
|
||||
# 2. Reverse-engineer the merges list from mergeable_ranks
|
||||
merges = []
|
||||
vocab = {}
|
||||
mergeable_ranks = tokenizer.mergeable_ranks
|
||||
mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute]
|
||||
for token, rank in mergeable_ranks.items():
|
||||
vocab[QwenModel.token_bytes_to_string(token)] = rank
|
||||
if len(token) == 1:
|
||||
@@ -43,8 +42,8 @@ class HunYuanMoEModel(TextModel):
|
||||
|
||||
# 3. Generate the tokens and toktypes lists
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
assert tokenizer.vocab_size == vocab_size
|
||||
special_tokens = tokenizer.special_tokens
|
||||
assert tokenizer.vocab_size == vocab_size # ty: ignore[unresolved-attribute]
|
||||
special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
|
||||
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
|
||||
tokens: list[str] = []
|
||||
toktypes: list[int] = []
|
||||
@@ -185,7 +184,6 @@ class HunYuanModel(TextModel):
|
||||
else:
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
|
||||
# 1. Get the pre-tokenizer identifier hash
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
@@ -193,7 +191,7 @@ class HunYuanModel(TextModel):
|
||||
# 2. Reverse-engineer the merges list from mergeable_ranks
|
||||
merges = []
|
||||
vocab = {}
|
||||
mergeable_ranks = tokenizer.mergeable_ranks
|
||||
mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute]
|
||||
for token, rank in mergeable_ranks.items():
|
||||
vocab[QwenModel.token_bytes_to_string(token)] = rank
|
||||
if len(token) == 1:
|
||||
@@ -204,8 +202,8 @@ class HunYuanModel(TextModel):
|
||||
|
||||
# 3. Generate the tokens and toktypes lists
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
assert tokenizer.vocab_size == vocab_size
|
||||
special_tokens = tokenizer.special_tokens
|
||||
assert tokenizer.vocab_size == vocab_size # ty: ignore[unresolved-attribute]
|
||||
special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
|
||||
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
|
||||
tokens: list[str] = []
|
||||
toktypes: list[int] = []
|
||||
|
||||
@@ -36,7 +36,7 @@ class InternLM2Model(TextModel):
|
||||
logger.error(f'Error: Missing {tokenizer_path}')
|
||||
sys.exit(1)
|
||||
|
||||
sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
|
||||
sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
|
||||
add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix
|
||||
|
||||
|
||||
@@ -29,14 +29,13 @@ class KimiLinearModel(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
|
||||
if tokpre == "kimi-k2":
|
||||
# Build merges list using the approach similar to HunYuanMoE
|
||||
merges = []
|
||||
vocab = {}
|
||||
mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[missing-union-attribute]
|
||||
mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[unresolved-attribute]
|
||||
for token, rank in mergeable_ranks.items():
|
||||
vocab[QwenModel.token_bytes_to_string(token)] = rank
|
||||
if len(token) == 1:
|
||||
@@ -46,7 +45,7 @@ class KimiLinearModel(TextModel):
|
||||
merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
|
||||
# Build token list
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
special_tokens = tokenizer.special_tokens
|
||||
special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
|
||||
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
|
||||
tokens: list[str] = []
|
||||
toktypes: list[int] = []
|
||||
@@ -72,7 +71,7 @@ class KimiLinearModel(TextModel):
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
# override eos id in config.json with tiktoken eos id
|
||||
self.gguf_writer.add_eos_token_id(tokenizer.eos_id)
|
||||
self.gguf_writer.add_eos_token_id(tokenizer.eos_id) # ty: ignore[unresolved-attribute]
|
||||
else:
|
||||
raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!")
|
||||
|
||||
|
||||
+3
-25
@@ -57,8 +57,6 @@ class KimiVLModel(MmprojModel):
|
||||
class KimiK25Model(MmprojModel):
|
||||
"""Kimi-K2.5 with MoonViT3d vision encoder"""
|
||||
|
||||
projector_type = gguf.VisionProjectorType.KIMIK25
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -80,7 +78,7 @@ class KimiK25Model(MmprojModel):
|
||||
super().set_gguf_parameters()
|
||||
assert self.hparams_vision is not None
|
||||
|
||||
self.gguf_writer.add_clip_projector_type(self.projector_type)
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.KIMIK25)
|
||||
|
||||
# Position embedding parameters (for interpolation)
|
||||
self.gguf_writer.add_uint32("vision.pos_emb_height", self.hparams_vision.get("init_pos_emb_height", 64))
|
||||
@@ -89,11 +87,7 @@ class KimiK25Model(MmprojModel):
|
||||
|
||||
# Projector parameters
|
||||
self.gguf_writer.add_vision_use_gelu(self.hparams_vision.get("projector_hidden_act", "gelu") == "gelu")
|
||||
if self.hparams_vision.get("norm_type", "layernorm") == "rmsnorm":
|
||||
# nn.RMSNorm without eps uses finfo(float32).eps, also for bf16 inputs
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(torch.finfo(torch.float32).eps)
|
||||
else:
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("projector_ln_eps", 1e-5))
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("projector_ln_eps", 1e-5))
|
||||
self.gguf_writer.add_vision_projector_scale_factor(self.merge_kernel_size[0])
|
||||
|
||||
# Image size limits
|
||||
@@ -126,7 +120,7 @@ class KimiK25Model(MmprojModel):
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
assert self.hparams_vision is not None
|
||||
n_head = self.find_vparam(["num_attention_heads", "vt_num_attention_heads"], optional=True) or 16
|
||||
n_head = self.hparams_vision.get("num_attention_heads", 16)
|
||||
|
||||
# Permute Q/K weights/biases from interleaved to split RoPE format
|
||||
# This allows using build_rope_2d at runtime without post-permutation.
|
||||
@@ -177,19 +171,3 @@ class Glm5vModel(KimiK25Model):
|
||||
name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1)
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("KimiK31ForConditionalGeneration")
|
||||
@ModelBase.example("moonshotai/Kimi-K3.1-Pro")
|
||||
class KimiK31VisionModel(KimiK25Model):
|
||||
"""Kimi-K2.5 MoonViT3d with RMSNorm, a wider QKV and the patchmergerv2 projector"""
|
||||
|
||||
projector_type = gguf.VisionProjectorType.KIMIK31
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
assert self.hparams_vision is not None
|
||||
assert self.hparams_vision["norm_type"] == "rmsnorm"
|
||||
assert self.hparams_vision["mm_projector_type"] == "patchmergerv2"
|
||||
n_head = self.find_vparam(["vt_num_attention_heads"])
|
||||
self.gguf_writer.add_vision_head_dim(self.hparams_vision["qkv_hidden_size"] // n_head)
|
||||
|
||||
+2
-2
@@ -77,10 +77,10 @@ class _DecisionLoraMixin:
|
||||
delta = self.lora_scale * (lora["B"] @ lora["A"])
|
||||
data_torch = data_torch.float() + LazyTorchTensor.from_eager(delta)
|
||||
self.lora_merged.add(name[name.index("layers."):])
|
||||
yield from super().modify_tensors(data_torch, name, bid) # ty: ignore[missing-attribute]
|
||||
yield from super().modify_tensors(data_torch, name, bid) # ty: ignore[unresolved-attribute]
|
||||
|
||||
def prepare_tensors(self):
|
||||
super().prepare_tensors() # ty: ignore[missing-attribute]
|
||||
super().prepare_tensors() # ty: ignore[unresolved-attribute]
|
||||
if len(self.lora_merged) != len(self.lora):
|
||||
raise ValueError(f"only {len(self.lora_merged)} of {len(self.lora)} LoRA tensors were merged into the base model")
|
||||
|
||||
|
||||
+2
-3
@@ -22,16 +22,15 @@ class LLaDAModel(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
|
||||
vocab_dict = tokenizer.get_vocab()
|
||||
vocab_dict = tokenizer.get_vocab() # ty: ignore[unresolved-attribute]
|
||||
vocab_size = self.hparams.get("vocab_size", len(vocab_dict))
|
||||
assert max(vocab_dict.values()) < vocab_size
|
||||
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
|
||||
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
|
||||
|
||||
for i in range(vocab_size):
|
||||
if i not in reverse_vocab:
|
||||
|
||||
@@ -263,7 +263,6 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
|
||||
# the tile separators are text tokens; clip appends their embeddings so that one chunk holds the whole image
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tokenizer is not None
|
||||
ids = []
|
||||
for text in self.tok_embd_sep:
|
||||
tok = tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
@@ -25,8 +25,7 @@ class MiniMaxText01Model(TextModel):
|
||||
# they get in the way of the token sampling process and must be suppressed
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
tokenizer_vocab_size = tokenizer.vocab_size
|
||||
tokenizer_vocab_size = tokenizer.vocab_size # ty: ignore[unresolved-attribute]
|
||||
|
||||
with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
|
||||
@@ -12,9 +12,9 @@ from .deepseek import DeepseekV2Model
|
||||
from .llama import LlamaModel
|
||||
|
||||
if _mistral_common_installed:
|
||||
from mistral_common.tokens.tokenizers.base import TokenizerVersion # ty: ignore[import-not-found]
|
||||
from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found]
|
||||
from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer # ty: ignore[import-not-found]
|
||||
from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import]
|
||||
from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
|
||||
from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer # type: ignore[import-not-found, ty:unresolved-import]
|
||||
else:
|
||||
TokenizerVersion = None # type: ignore[assignment]
|
||||
Tekkenizer = None # type: ignore[assignment]
|
||||
|
||||
@@ -37,8 +37,7 @@ class MuseGlimmerModel(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tok = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tok is not None
|
||||
eot_id = tok.convert_tokens_to_ids("<|eot|>")
|
||||
eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute]
|
||||
if isinstance(eot_id, int) and eot_id >= 0:
|
||||
self.gguf_writer.add_eot_token_id(eot_id)
|
||||
|
||||
@@ -153,7 +152,7 @@ class MuseGlimmerAssistantModel(TextModel):
|
||||
target_arch = json.load(f)["architectures"][0]
|
||||
target_cls = get_model_class(target_arch)
|
||||
if target_cls is not type(self):
|
||||
target_cls.set_vocab(self) # ty: ignore[missing-attribute]
|
||||
target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
|
||||
else:
|
||||
super().set_vocab()
|
||||
|
||||
|
||||
@@ -44,8 +44,7 @@ class MuseSparkModel(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tok = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tok is not None
|
||||
eot_id = tok.convert_tokens_to_ids("<|eot|>")
|
||||
eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute]
|
||||
if isinstance(eot_id, int) and eot_id >= 0:
|
||||
self.gguf_writer.add_eot_token_id(eot_id)
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ class LiLiCorrDraftModel(DFlashModel):
|
||||
try:
|
||||
if target_arch == "NemotronHForCausalLM":
|
||||
self.is_moe = "num_experts_per_tok" in target_text_config
|
||||
target_cls.set_vocab(self) # ty: ignore[missing-attribute]
|
||||
target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
|
||||
finally:
|
||||
self.dir_model = original_dir
|
||||
self.gguf_writer.add_mask_token_id(self.hparams["dflash_config"]["mask_token_id"])
|
||||
@@ -609,7 +609,6 @@ class NemotronHModel(GraniteHybridModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
assert tokenizer is not None
|
||||
|
||||
# Pad vocab size (from Mamba2Model/GraniteHybridModel)
|
||||
self.hparams["pad_vocab_size_multiple"] = 8 # Setting this here since GraniteHybridModel.set_vocab() isn't being invoked now.
|
||||
@@ -624,14 +623,14 @@ class NemotronHModel(GraniteHybridModel):
|
||||
vocab_skip = getattr(self, "_vocab_skip", ())
|
||||
user_defined_tokens = getattr(self, "_user_defined_tokens", None)
|
||||
|
||||
assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size
|
||||
assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size # ty: ignore[unresolved-attribute]
|
||||
|
||||
tokpre = self.get_vocab_base_pre(tokenizer)
|
||||
|
||||
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute]
|
||||
added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
|
||||
|
||||
added_tokens_decoder = tokenizer.added_tokens_decoder
|
||||
added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute]
|
||||
|
||||
for i in range(vocab_size):
|
||||
if i not in reverse_vocab:
|
||||
@@ -642,7 +641,7 @@ class NemotronHModel(GraniteHybridModel):
|
||||
if token in added_vocab:
|
||||
if not added_tokens_decoder[i].normalized:
|
||||
previous_token = token
|
||||
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment]
|
||||
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment]
|
||||
if previous_token != token:
|
||||
logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
|
||||
|
||||
|
||||
@@ -191,7 +191,7 @@ class PocketTTSModel(TextModel):
|
||||
# unigram segmentation, so use the UGM tokenizer instead
|
||||
from sentencepiece import sentencepiece_model_pb2 as model
|
||||
|
||||
proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
|
||||
proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
proto.ParseFromString(open(self.dir_model / "tokenizer.model", "rb").read())
|
||||
assert proto.trainer_spec.model_type == 1, "expected a unigram tokenizer"
|
||||
|
||||
|
||||
+12
-58
@@ -225,13 +225,12 @@ class Qwen3Model(Qwen2Model):
|
||||
def _find_rerank_config(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
assert tokenizer is not None
|
||||
|
||||
self.is_rerank = True
|
||||
self.is_tied_embeddings = self.hparams.get("tie_word_embeddings", False)
|
||||
self.token_false_id = tokenizer.convert_tokens_to_ids("no") # ty: ignore[invalid-assignment]
|
||||
self.token_true_id = tokenizer.convert_tokens_to_ids("yes") # ty: ignore[invalid-assignment]
|
||||
self.sep_token_id = tokenizer.convert_tokens_to_ids("|")
|
||||
self.token_false_id = tokenizer.convert_tokens_to_ids("no") # ty: ignore[unresolved-attribute, invalid-assignment]
|
||||
self.token_true_id = tokenizer.convert_tokens_to_ids("yes") # ty: ignore[unresolved-attribute, invalid-assignment]
|
||||
self.sep_token_id = tokenizer.convert_tokens_to_ids("|") # ty: ignore[unresolved-attribute]
|
||||
|
||||
assert self.token_false_id is not None and self.token_true_id is not None
|
||||
|
||||
@@ -330,7 +329,7 @@ class _QwenMtpMixin:
|
||||
key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None)
|
||||
type(self)._original_block_count = hparams.get(key)
|
||||
type(self).opt_num_mtp_layers = 0
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[missing-attribute]
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute]
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
@@ -367,7 +366,7 @@ class _QwenMtpMixin:
|
||||
return name, gen
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters() # ty: ignore[missing-attribute]
|
||||
super().set_gguf_parameters() # ty: ignore[unresolved-attribute]
|
||||
if self.no_mtp:
|
||||
return
|
||||
if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0:
|
||||
@@ -375,15 +374,15 @@ class _QwenMtpMixin:
|
||||
|
||||
def prepare_metadata(self, vocab_only: bool):
|
||||
from_dir = self.fname_out.is_dir()
|
||||
super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[missing-attribute]
|
||||
super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute]
|
||||
|
||||
if not self.mtp_only or not from_dir:
|
||||
return
|
||||
|
||||
output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
|
||||
output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
fname_default: str = gguf.naming_convention(
|
||||
self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
|
||||
self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
|
||||
self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
|
||||
|
||||
|
||||
@@ -493,45 +492,6 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
|
||||
interleaved repeat: [G0_v0, G1_v0, ..., G0_v1, G1_v1, ...].
|
||||
"""
|
||||
|
||||
_SHARED_INPUT_MERGES: tuple[tuple[gguf.MODEL_TENSOR, tuple[gguf.MODEL_TENSOR, gguf.MODEL_TENSOR], bool], ...] = ()
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {}
|
||||
|
||||
# caches one half of a merged weight until the other half arrives, then yields [a | b]
|
||||
def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold
|
||||
if not self._SHARED_INPUT_MERGES or bid is None or self.hparams.get("quantization_config") or self._fp8_e4m3_preserved or self.hadamard_folded_names() or hasattr(data, "get_lora_A_B"):
|
||||
yield (new_name, data)
|
||||
return
|
||||
|
||||
layer_types = self.hparams.get("layer_types")
|
||||
if layer_types is None:
|
||||
is_linear = bid < self.hparams["num_hidden_layers"] and (bid + 1) % self.hparams.get("full_attention_interval", 4) != 0
|
||||
else:
|
||||
is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention"
|
||||
|
||||
match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES
|
||||
if merged in gguf.MODEL_TENSORS[self.model_arch]
|
||||
for part in parts
|
||||
if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None)
|
||||
if match is None:
|
||||
yield (new_name, data)
|
||||
return
|
||||
|
||||
merged, idx = match
|
||||
halves = self._merge_parts.setdefault((bid, merged), {})
|
||||
halves[idx] = data
|
||||
if len(halves) == 2:
|
||||
del self._merge_parts[(bid, merged)]
|
||||
yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0))
|
||||
|
||||
def prepare_tensors(self):
|
||||
super().prepare_tensors()
|
||||
if self._merge_parts:
|
||||
raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}")
|
||||
|
||||
@staticmethod
|
||||
def _reorder_v_heads(tensor: Tensor, dim: int, num_k_heads: int, num_v_per_k: int, head_dim: int) -> Tensor:
|
||||
"""Reorder V heads from grouped (by K head) to tiled order along the given dimension."""
|
||||
@@ -711,8 +671,7 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
|
||||
# Out projection weight: reorder columns (input dimension)
|
||||
data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim)
|
||||
|
||||
for new_name, data in super().modify_tensors(data_torch, name, bid):
|
||||
yield from self._merge_shared_input(new_name, data, bid)
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
class _Qwen35MRopeMixin:
|
||||
@@ -726,7 +685,7 @@ class _Qwen35MRopeMixin:
|
||||
rope_parameters: dict
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters() # ty: ignore[missing-attribute]
|
||||
super().set_gguf_parameters() # ty: ignore[unresolved-attribute]
|
||||
if "mrope_section" not in self.rope_parameters:
|
||||
self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION)
|
||||
|
||||
@@ -736,11 +695,6 @@ class _Qwen35MRopeMixin:
|
||||
class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35
|
||||
|
||||
_SHARED_INPUT_MERGES = (
|
||||
(gguf.MODEL_TENSOR.ATTN_QKV_GATE, (gguf.MODEL_TENSOR.ATTN_QKV, gguf.MODEL_TENSOR.ATTN_GATE), True),
|
||||
(gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT, (gguf.MODEL_TENSOR.SSM_BETA, gguf.MODEL_TENSOR.SSM_ALPHA), True),
|
||||
)
|
||||
|
||||
def __init__(self, dir_model, *args, **kwargs):
|
||||
# Inner TextModel does not own mtp.*. Set no_mtp before mixin bumps block_count.
|
||||
hparams = kwargs.pop("hparams", None)
|
||||
@@ -889,7 +843,7 @@ class DFlashModel(Qwen3Model):
|
||||
if target_cls is not type(self):
|
||||
if target_arch == "NemotronHForCausalLM":
|
||||
setattr(self, "is_moe", "num_experts_per_tok" in target_hparams)
|
||||
target_cls.set_vocab(self) # ty: ignore[missing-attribute]
|
||||
target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
|
||||
else:
|
||||
super().set_vocab()
|
||||
|
||||
|
||||
+30
-1
@@ -41,6 +41,8 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
self._ple_shards: dict[int, str] = {}
|
||||
self._ple_row_dim: int | None = None
|
||||
self._mtp_fc: dict[str, Tensor] = {}
|
||||
# halves of the merged weights, keyed by (bid, merged tensor), see _merge_shared_input
|
||||
self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {}
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
@@ -169,7 +171,8 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
yield (self.map_tensor_name(name), data_torch.squeeze())
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
for new_name, data in super().modify_tensors(data_torch, name, bid):
|
||||
yield from self._merge_shared_input(new_name, data, bid)
|
||||
|
||||
# matmuls that read the same input, written as one weight [a | b] so the runtime does one matmul:
|
||||
# (merged tensor, (a, b), linear-attention layers only)
|
||||
@@ -180,6 +183,30 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
(gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT, (gguf.MODEL_TENSOR.SSM_BETA, gguf.MODEL_TENSOR.SSM_ALPHA), True),
|
||||
)
|
||||
|
||||
# caches one half of a merged weight until the other half arrives, then yields [a | b]
|
||||
def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold
|
||||
if bid is None or self.hparams.get("quantization_config"):
|
||||
yield (new_name, data)
|
||||
return
|
||||
|
||||
layer_types = self.hparams.get("layer_types") or []
|
||||
is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention"
|
||||
|
||||
match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES
|
||||
for part in parts
|
||||
if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None)
|
||||
if match is None:
|
||||
yield (new_name, data)
|
||||
return
|
||||
|
||||
merged, idx = match
|
||||
halves = self._merge_parts.setdefault((bid, merged), {})
|
||||
halves[idx] = data
|
||||
if len(halves) == 2:
|
||||
del self._merge_parts[(bid, merged)]
|
||||
yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0))
|
||||
|
||||
# the shards concatenate into a tensor of well over 100 GB
|
||||
# use LazyChunkedTensor here, a single shard resident at a time
|
||||
def _place_ple_shard(self, data_torch: Tensor, name: str) -> Iterable[tuple[str, Tensor]]:
|
||||
@@ -224,6 +251,8 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
super().prepare_tensors()
|
||||
if self._mtp_fc:
|
||||
raise ValueError(f"MTP projection missing its other half: {sorted(self._mtp_fc)}")
|
||||
if self._merge_parts:
|
||||
raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}")
|
||||
n_parts = self.hparams.get("split_ngram_parts", 0)
|
||||
if self._ple_shards and len(self._ple_shards) != n_parts:
|
||||
raise ValueError(
|
||||
|
||||
+2
-2
@@ -40,7 +40,7 @@ class T5Model(TextModel):
|
||||
if not tokenizer_path.is_file():
|
||||
raise FileNotFoundError(f"File not found: {tokenizer_path}")
|
||||
|
||||
sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
|
||||
sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
|
||||
|
||||
# some models like Pile-T5 family use BPE tokenizer instead of Unigram
|
||||
@@ -178,7 +178,7 @@ class T5EncoderModel(TextModel):
|
||||
if not tokenizer_path.is_file():
|
||||
raise FileNotFoundError(f"File not found: {tokenizer_path}")
|
||||
|
||||
sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
|
||||
sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
|
||||
|
||||
# some models like Pile-T5 family use BPE tokenizer instead of Unigram
|
||||
|
||||
@@ -25,16 +25,15 @@ class XverseModel(TextModel):
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(dir_model)
|
||||
assert tokenizer is not None
|
||||
vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
|
||||
vocab_size = hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute]
|
||||
# Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size,
|
||||
# because vocab_size is the count of items, and indexes start at 0.
|
||||
max_vocab_index = max(tokenizer.get_vocab().values())
|
||||
max_vocab_index = max(tokenizer.get_vocab().values()) # ty: ignore[unresolved-attribute]
|
||||
if max_vocab_index >= vocab_size:
|
||||
raise ValueError("Vocabulary size exceeds expected maximum size.")
|
||||
|
||||
reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute]
|
||||
added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
|
||||
|
||||
for token_id in range(vocab_size):
|
||||
token_text = reverse_vocab[token_id].encode('utf-8')
|
||||
@@ -45,7 +44,7 @@ class XverseModel(TextModel):
|
||||
elif re.fullmatch(br"<0x[0-9A-Fa-f]{2}>", token_text):
|
||||
toktype = gguf.TokenType.BYTE # special
|
||||
elif reverse_vocab[token_id] in added_vocab:
|
||||
if tokenizer.added_tokens_decoder[token_id].special:
|
||||
if tokenizer.added_tokens_decoder[token_id].special: # ty: ignore[unresolved-attribute]
|
||||
toktype = gguf.TokenType.CONTROL
|
||||
else:
|
||||
toktype = gguf.TokenType.USER_DEFINED
|
||||
|
||||
@@ -125,15 +125,6 @@ def parse_args() -> argparse.Namespace:
|
||||
"--no-nextn", "--no-mtp", dest="no_mtp", action="store_true",
|
||||
help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.",
|
||||
)
|
||||
ple_group = parser.add_mutually_exclusive_group()
|
||||
ple_group.add_argument(
|
||||
"--ple-only", action="store_true",
|
||||
help="Export only the per-layer embedding table as a separate GGUF for use with --shard (alias: --ple).",
|
||||
)
|
||||
ple_group.add_argument(
|
||||
"--no-ple", action="store_true",
|
||||
help="Exclude the per-layer embedding table while retaining its metadata. Load the table separately with --shard (alias: --ple).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dspark", action="store_true",
|
||||
help="Export only the DeepSeek-V4 or Mistral Large 4 DSpark draft tensors as a separate GGUF.",
|
||||
@@ -182,10 +173,6 @@ def parse_args() -> argparse.Namespace:
|
||||
args = parser.parse_args()
|
||||
if not args.print_supported_models and args.model is None:
|
||||
parser.error("the following arguments are required: model")
|
||||
if (args.ple_only or args.no_ple) and (args.vocab_only or args.mmproj or args.mtp or args.dspark):
|
||||
parser.error("--ple-only / --no-ple cannot be combined with --vocab-only, --mmproj, --mtp, or --dspark")
|
||||
if args.ple_only and args.no_tensor_first_split:
|
||||
parser.error("--ple-only cannot be combined with --no-tensor-first-split")
|
||||
return args
|
||||
|
||||
|
||||
@@ -321,7 +308,6 @@ def main() -> None:
|
||||
fuse_gate_up_exps=args.fuse_gate_up_exps,
|
||||
fp8_as_q8=args.fp8_as_q8,
|
||||
fuse_qkv=args.fuse_qkv,
|
||||
ple_only=args.ple_only, no_ple=args.no_ple,
|
||||
)
|
||||
|
||||
if args.vocab_only:
|
||||
|
||||
@@ -325,8 +325,7 @@ for model in [*pre_computed_hashes, *all_models]:
|
||||
except Exception as e:
|
||||
raise OSError(f"Error loading tokenizer for model {name}.") from e
|
||||
|
||||
assert tokenizer is not None
|
||||
chktok = tokenizer.encode(CHK_TXT)
|
||||
chktok = tokenizer.encode(CHK_TXT) # ty: ignore[unresolved-attribute]
|
||||
chkhsh = sha256(str(chktok).encode()).hexdigest()
|
||||
|
||||
logger.info(f"model: {name}")
|
||||
@@ -487,7 +486,6 @@ for model in models:
|
||||
logger.error(f"Failed to load tokenizer for model {name}. Error: {e}")
|
||||
continue # Skip this model and continue with the next one in the loop
|
||||
|
||||
assert tokenizer is not None
|
||||
if not os.path.exists(f"models/ggml-vocab-{name}.gguf"):
|
||||
logger.info(f"Skip vocab files for model {name}, no GGUF file found")
|
||||
continue
|
||||
@@ -499,7 +497,7 @@ for model in models:
|
||||
|
||||
with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:
|
||||
for text in tests:
|
||||
res = tokenizer.encode(text, add_special_tokens=False)
|
||||
res = tokenizer.encode(text, add_special_tokens=False) # ty: ignore[unresolved-attribute]
|
||||
for r in res:
|
||||
f.write(f" {r}")
|
||||
f.write("\n")
|
||||
|
||||
@@ -443,8 +443,7 @@ if __name__ == '__main__':
|
||||
# the invocation string includes the "<|start_of_turn|>"
|
||||
# token, but the adapters themselves were trained to
|
||||
# activate _after_ that first token, so we drop it here.
|
||||
assert tokenizer is not None
|
||||
alora_invocation_tokens = tokenizer(invocation_string)["input_ids"][1:]
|
||||
alora_invocation_tokens = tokenizer(invocation_string)["input_ids"][1:] # ty: ignore[call-non-callable]
|
||||
if alora_invocation_tokens:
|
||||
logger.debug("GGUF KV: %s = %s", gguf.Keys.Adapter.ALORA_INVOCATION_TOKENS, alora_invocation_tokens)
|
||||
self.gguf_writer.add_key_value(
|
||||
|
||||
@@ -52,9 +52,6 @@ The packages for FP32 and FP16 would have different accuracy and performance on
|
||||
|
||||
## News
|
||||
|
||||
- 2026.11
|
||||
- Update the CI and release build for oneAPI 2026.2 and Level Zero SDK 1.34.0. oneDNN is updated to the 2026.1 stand-alone package.
|
||||
|
||||
- 2026.09
|
||||
- Update the CI build environment for oneAPI 2026.1 (unified oneAPI Toolkit). oneDNN is removed from the Deep Learning Essentials package in 2026.0, so the CI now uses the oneAPI Toolkit installer which still includes oneDNN.
|
||||
- oneAPI 2026.1 improves the SYCL build performance: measured with the same code on Arc B570, prompt processing 1331 vs 434 t/s (3.1x) vs the 2025.3-based release build.
|
||||
@@ -288,7 +285,6 @@ Upon a successful installation, SYCL is enabled for the available Intel devices,
|
||||
|
||||
|Verified release|
|
||||
|-|
|
||||
|2026.2 |
|
||||
|2026.1 |
|
||||
|2025.3.3 |
|
||||
|2025.2.1|
|
||||
@@ -829,8 +825,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
|
||||
| GGML_SYCL_ENABLE_XMX_ESIMD | 0 or 1 (default) | Use the XMX (DPAS) multi-column mat-vec kernels for reordered Q2_K to Q6_K and Q8_0 weights on Xe2 GPUs. Speeds up speculative decoding and short prompt batches. |
|
||||
| GGML_SYCL_XMX_MAX_COLS | non-negative integer, 80 (default) | Most activation columns (batch size) the `GGML_SYCL_ENABLE_XMX_ESIMD` kernels take; wider batches use the other mat-mul kernels. A larger value keeps XMX for bigger batches (as repeated launches), `0` never uses it. |
|
||||
| GGML_SYCL_XMX_GLU_MAX_COLS | 0 to 16, 16 (default) | Most columns the fused gate and up XMX kernel takes; wider batches run gate and up as separate mat-vecs. 16 is the most the kernel is built for, larger values are clamped to 16 and `0` turns the fused kernel off. |
|
||||
| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:<br>* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M<br>* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K, 8192: Q2_K, 16384: Q3_K (MoE `MUL_MAT_ID` only)<br>Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert use the library GEMM unless the device schedule width heuristic is overridden with `GGML_SYCL_MMID_SCHED=3`. Row lengths must be a multiple of 256 (32 for IQ4_NL and Q8_0). |
|
||||
| GGML_SYCL_MMID_SCHED | decimal bitmask, 1 (default) | Build the grouped MoE routing and tile schedule on the GPU, avoiding the expert-ID host readback and CPU sort.<br>* 1: enable the device schedule.<br>* 2: override the fallback for models with few routed experts by disabling the performance heuristic that rejects batches averaging more than 64 routed rows per expert. This may reduce performance; type, shape, precision and device capability checks still apply.<br>Use `3` to enable the device schedule and override this heuristic, or `0` to use the host schedule. Bit 2 alone does not enable the device schedule. Higher bits are ignored. |
|
||||
| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:<br>* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M<br>* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K (MoE `MUL_MAT_ID` only)<br>Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert or row lengths that are not a multiple of 256 (32 for IQ4_NL and Q8_0) use the library GEMM. |
|
||||
| GGML_SYCL_XMX_GATHER_SHAPES | decimal bitmask, 255 (default) | XMX `joint_matrix` combinations the paths of `GGML_SYCL_XMX_GATHER_TYPES` may use; the operand type comes from `GGML_SYCL_DYNAMIC_PRECISION` and the best supported combination is picked automatically (logged as `fg_pick_combo`). Bits:<br>* Xe2, Xe3, Xe-HPC: 1: f16 8x16x16, 2: f16 16x16x16, 4: f16 32x64x16, 8: f16 32x64x32, 32: tf32 8x16x8, 64: bf16 8x16x16<br>* Xe-HPG (Arc A770, ARL-H): 16: f16 8x8x16, 128: bf16 8x8x16<br>Clear a bit to exclude a combination, or set a single bit to force one for testing. |
|
||||
| GGML_SYCL_DYNAMIC_PRECISION | `F16` (default with `GGML_SYCL_F16=ON`), `BF16`, `TF32` or `F32` (default otherwise) | Operand type of the XMX dequant-GEMM paths (`GGML_SYCL_XMX_GATHER_TYPES`); accumulation is always f32. `F16` is the fastest, but activations above 65504 overflow. `BF16` keeps the f32 range at a 7-bit mantissa, `TF32` keeps the range and the f16 mantissa but is about 30% slower and needs Xe2, Xe3 or Xe-HPC, and `F32` turns the XMX paths off. Ops that request a higher src1 precision ([TAG_GGML_PREC]) get it regardless of this setting. |
|
||||
| GGML_SYCL_DYNAMIC_REQUIRED_PRECISION | `F32` (default), `TF32`, `BF16` or `F16` | Lowest type the XMX paths may use for an op that requests an F32 src1, such as Mistral 4 `ffn_down_exps`. The default runs such ops on the library f32 GEMM; `TF32` or `BF16` trade mantissa for speed while keeping the f32 range. `F16` ignores the request and can overflow; it is meant for testing only. |
|
||||
|
||||
@@ -60,7 +60,7 @@ Implementation: [`htp-repack.cpp`](../../../ggml/src/ggml-hexagon/htp-repack.cpp
|
||||
### Tile Dimensions and Grid Layout
|
||||
|
||||
Each tile represents a submatrix of `N_rows = 32` rows and `K_cols` columns:
|
||||
- Standard types (`Q4_0`, `Q4_1`, `Q5_0`, `Q5_1`, `Q8_0`, `IQ4_NL`, `MXFP4`, and K-quants): `K_cols = 32` (`HTP_TENSOR_TILE_N_COLS = 32`).
|
||||
- Standard types (`Q4_0`, `Q4_1`, `Q8_0`, `IQ4_NL`, `MXFP4`, and K-quants): `K_cols = 32` (`HTP_TENSOR_TILE_N_COLS = 32`).
|
||||
- `Q2_0`: `K_cols = 64` (`HTP_TENSOR_TILE_K_Q2_0 = 64`).
|
||||
- `Q1_0`: `K_cols = 128` (`HTP_TENSOR_TILE_K_Q1_0 = 128`).
|
||||
|
||||
@@ -123,17 +123,6 @@ Block-32 formats use a tile size of 32 rows x 32 columns (`K_cols = 32`):
|
||||
- **Scales** (bytes 512..639, 128 bytes):
|
||||
32 pairs of FP16 values (`ggml_half`): `scale[2 * row + 0] = d` (scale) and `scale[2 * row + 1] = m` (min) for each row `row = 0..31`.
|
||||
|
||||
- **Q5_0** (704 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q5_0`):
|
||||
- **Quants** (bytes 0..511): same 4-bit nibble transposition and storage layout as `Q4_0`, holding the low 4 bits.
|
||||
- **5th-bit plane** (bytes 512..639): byte `32*c + row` (`c = 0..3`) holds a flag for `k = 2*(4i + c)` at bit `2i` and for `k = 2*(4i + c) + 1` at bit `2i + 1`, for `i = 0..3`. The flag is set when the 5th bit of the quant is clear.
|
||||
- **Scales** (bytes 640..703, 64 bytes): 32 FP16 values, storing scale `d` for each row `row = 0..31`.
|
||||
- **Dequantization**: `0xF0` is OR-ed into the nibble where the flag is set, giving the signed value `q - 16` (-16..15), scaled by `d`.
|
||||
|
||||
- **Q5_1** (768 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q5_1`, same layout as the Q5_K tile):
|
||||
- **Quants** (bytes 0..511) and **scales** (bytes 512..639): same as `Q4_1`, holding the low 4 bits and `(d, m)` pairs.
|
||||
- **5th-bit plane** (bytes 640..767): same as `Q5_K`; a bit is set when the 5th bit of the quant is set.
|
||||
- Uses the `Q5_K` HVX/HMX kernels (Q8_1 activations with block sums).
|
||||
|
||||
### Q1_0 Format (1-Bit Quantization)
|
||||
|
||||
`Q1_0` packs 128 columns per tile (`K_cols = 128`, `HTP_TENSOR_TILE_K_Q1_0 = 128`):
|
||||
|
||||
+1
-1
@@ -608,7 +608,7 @@ You can test with:
|
||||
If the following info is output on screen, you are using `llama.cpp` with the CANN backend:
|
||||
```bash
|
||||
llm_load_tensors: CANN model buffer size = 13313.00 MiB
|
||||
llama_init_from_model: CANN compute buffer size = 1260.81 MiB
|
||||
llama_new_context_with_model: CANN compute buffer size = 1260.81 MiB
|
||||
```
|
||||
|
||||
For detailed info, such as model/device supports, CANN install, please refer to [llama.cpp for CANN](./backend/CANN.md).
|
||||
|
||||
@@ -24,23 +24,3 @@ The Hugging Face platform provides a variety of online tools for converting, qua
|
||||
- Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123)
|
||||
- Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268)
|
||||
- Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669)
|
||||
|
||||
## Exporting per-layer embeddings separately
|
||||
|
||||
For models with a per-layer embedding (PLE) table, export the backbone and table from the same checkpoint:
|
||||
|
||||
```sh
|
||||
python3 convert_hf_to_gguf.py /path/to/model --no-ple --outfile model-no-ple.gguf
|
||||
python3 convert_hf_to_gguf.py /path/to/model --ple-only --outfile model-ple.gguf
|
||||
llama-cli -m model-no-ple.gguf --shard model-ple.gguf
|
||||
```
|
||||
|
||||
`--no-ple` excludes only `per_layer_token_embd.weight` and keeps the PLE metadata and projection weights. `--ple-only` exports only that table, without a tokenizer. Each export supports `--outtype`, so the table and backbone can use different precisions. Default output names start with `no-ple-` and `ple-` respectively.
|
||||
|
||||
These modes are mutually exclusive and require a model with a PLE table, such as Qwen3.8 Flash Next or Gemma 4. They cannot be combined with vocabulary, multimodal projector, MTP-only, or DSpark-only export. `--no-ple` can be combined with `--no-mtp`.
|
||||
|
||||
`--ple` is an alias for `--shard`. Extra shards can be comma separated or supplied with repeated options, for example `-m backbone.gguf --shard /fast/ple.gguf,/other/token-embd.gguf`. Paths may be on different drives. Tensor names must be unique across all files, except for the PLE case below.
|
||||
|
||||
A numbered backbone is auto-discovered and checked as usual; explicitly supplied paths outside that numbered set are appended as extra shards. Their split numbering is ignored, and the loader uses the actual combined tensor count. Paths already in the discovered set are loaded once. If the main filename does not follow the numbered naming scheme, an explicit backbone list must still match its declared split count and order.
|
||||
|
||||
If an extra shard holds only `per_layer_token_embd.weight`, it replaces the PLE table of the model. If the model already has the table, the extra shard overrides it. A backbone split that holds only that tensor is skipped, and it can be absent from disk. This lets one PLE file be shared across quants and releases, for example `-m model-Q4_K_M-00001-of-00003.gguf --shard /fast/ple-Q8_0.gguf`.
|
||||
|
||||
+2
-1
@@ -40,6 +40,7 @@ Legend:
|
||||
| CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
@@ -85,7 +86,7 @@ Legend:
|
||||
| OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
|
||||
@@ -8741,6 +8741,9 @@
|
||||
"BLAS","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","BLAS"
|
||||
"BLAS","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","BLAS"
|
||||
"BLAS","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","BLAS"
|
||||
"BLAS","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","BLAS"
|
||||
"BLAS","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","BLAS"
|
||||
"BLAS","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","BLAS"
|
||||
"BLAS","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","BLAS"
|
||||
"BLAS","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","BLAS"
|
||||
"BLAS","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","BLAS"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -8764,6 +8764,9 @@
|
||||
"CANN0","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","CANN"
|
||||
"CANN0","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","CANN"
|
||||
"CANN0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","CANN"
|
||||
"CANN0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","CANN"
|
||||
"CANN0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","CANN"
|
||||
"CANN0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","CANN"
|
||||
"CANN0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CANN"
|
||||
"CANN0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CANN"
|
||||
"CANN0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CANN"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -12962,6 +12962,9 @@
|
||||
"CPU","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","CPU"
|
||||
"CPU","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","CPU"
|
||||
"CPU","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","CPU"
|
||||
"CPU","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","CPU"
|
||||
"CPU","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","CPU"
|
||||
"CPU","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","CPU"
|
||||
"CPU","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CPU"
|
||||
"CPU","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CPU"
|
||||
"CPU","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -8725,6 +8725,9 @@
|
||||
"CUDA0","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","CUDA"
|
||||
"CUDA0","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","CUDA"
|
||||
"CUDA0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","CUDA"
|
||||
"CUDA0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","CUDA"
|
||||
"CUDA0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","CUDA"
|
||||
"CUDA0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","CUDA"
|
||||
"CUDA0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CUDA"
|
||||
"CUDA0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CUDA"
|
||||
"CUDA0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CUDA"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -9637,6 +9637,9 @@
|
||||
"ET","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","ET"
|
||||
"ET","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","ET"
|
||||
"ET","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","ET"
|
||||
"ET","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","ET"
|
||||
"ET","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","ET"
|
||||
"ET","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","ET"
|
||||
"ET","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","ET"
|
||||
"ET","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","ET"
|
||||
"ET","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","ET"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -14564,6 +14564,9 @@
|
||||
"HTP0","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","HTP"
|
||||
"HTP0","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","HTP"
|
||||
"HTP0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","HTP"
|
||||
"HTP0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","HTP"
|
||||
"HTP0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","HTP"
|
||||
"HTP0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","HTP"
|
||||
"HTP0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","HTP"
|
||||
"HTP0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","HTP"
|
||||
"HTP0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","HTP"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -9628,6 +9628,9 @@
|
||||
"MTL0","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","MTL"
|
||||
"MTL0","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","MTL"
|
||||
"MTL0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","MTL"
|
||||
"MTL0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","MTL"
|
||||
"MTL0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","MTL"
|
||||
"MTL0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","MTL"
|
||||
"MTL0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","MTL"
|
||||
"MTL0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","MTL"
|
||||
"MTL0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","MTL"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -14887,6 +14887,9 @@
|
||||
"OpenCL","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","OpenCL"
|
||||
"OpenCL","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","OpenCL"
|
||||
"OpenCL","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","OpenCL"
|
||||
"OpenCL","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","OpenCL"
|
||||
"OpenCL","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","OpenCL"
|
||||
"OpenCL","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","OpenCL"
|
||||
"OpenCL","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","OpenCL"
|
||||
"OpenCL","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","OpenCL"
|
||||
"OpenCL","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","OpenCL"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -14534,6 +14534,9 @@
|
||||
"SYCL0","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","SYCL"
|
||||
"SYCL0","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","SYCL"
|
||||
"SYCL0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","SYCL"
|
||||
"SYCL0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","SYCL"
|
||||
"SYCL0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","SYCL"
|
||||
"SYCL0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","SYCL"
|
||||
"SYCL0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL"
|
||||
"SYCL0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL"
|
||||
"SYCL0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -12313,6 +12313,9 @@
|
||||
"Vulkan0","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","Vulkan"
|
||||
"Vulkan0","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","Vulkan"
|
||||
"Vulkan0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","Vulkan"
|
||||
"Vulkan0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","Vulkan"
|
||||
"Vulkan0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","Vulkan"
|
||||
"Vulkan0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","Vulkan"
|
||||
"Vulkan0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","Vulkan"
|
||||
|
||||
|
Can't render this file because it is too large.
|
+5
-7
@@ -10560,6 +10560,9 @@
|
||||
"WebGPU: WebGPU","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","WebGPU"
|
||||
@@ -12333,13 +12336,8 @@
|
||||
"WebGPU: WebGPU","PAD","type=f32,ne_a=[100,1,1,1],pad_0=100,pad_1=0,circular=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD","type=f32,ne_a=[100,1,1,1],pad_0=0,pad_1=100,circular=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD","type=f32,ne_a=[100,100,1,1],pad_0=50,pad_1=50,circular=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[512,34,2,1],pad_0=10,pad_1=9,view=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[1,3,2,2],pad_0=0,pad_1=0,view=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[5,3,2,2],pad_0=4,pad_1=1,view=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[281,128,1,1],pad_0=2,pad_1=2,view=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[257,7,1,1],pad_0=0,pad_1=256,view=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[2,16777217,1,1],pad_0=1,pad_1=1,view=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[3000,384,4,1],pad_0=10,pad_1=9,view=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[512,34,2,1],pad_0=10,pad_1=9","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[3000,384,4,1],pad_0=10,pad_1=9","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","ROLL","shift0=3,shift1=-2,shift3=1,shift4=-1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","ARANGE","type=f32,start=3.000000,stop=3.500000,step=1.000000","support","1","yes","WebGPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -9319,6 +9319,9 @@
|
||||
"ZenDNN","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","ZenDNN"
|
||||
"ZenDNN","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","ZenDNN"
|
||||
"ZenDNN","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","ZenDNN"
|
||||
"ZenDNN","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","ZenDNN"
|
||||
"ZenDNN","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","ZenDNN"
|
||||
"ZenDNN","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","ZenDNN"
|
||||
"ZenDNN","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","ZenDNN"
|
||||
"ZenDNN","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","ZenDNN"
|
||||
"ZenDNN","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","ZenDNN"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -14550,6 +14550,9 @@
|
||||
"zDNN","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","zDNN"
|
||||
"zDNN","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","zDNN"
|
||||
"zDNN","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","zDNN"
|
||||
"zDNN","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","zDNN"
|
||||
"zDNN","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","zDNN"
|
||||
"zDNN","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","zDNN"
|
||||
"zDNN","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","zDNN"
|
||||
"zDNN","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","zDNN"
|
||||
"zDNN","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","zDNN"
|
||||
|
||||
|
Can't render this file because it is too large.
|
+1
-1
@@ -189,7 +189,7 @@ llama-server ... --spec-type ngram-mod --spec-ngram-mod-n-match 24 --spec-ngram-
|
||||
|
||||
Applications:
|
||||
|
||||
- Iterating over a block of text/code (e.g. in [llama.vim](https://github.com/ggml-org/llama.vim))
|
||||
- Iterating over a block of text/code (e.g. in llama.vim)
|
||||
- Reasoning models (when they have to repeat their thinking in the final answer)
|
||||
- Summarization
|
||||
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
# TTS normalization patterns
|
||||
|
||||
Text-to-speech models often need their input normalized before tokenization: numbers, currencies, dates and symbols spelled out, punctuation mapped, spacing fixed. The exact rules are model-specific (they must match what the model was trained on), so they live in the model's `tts` task template rather than in C++.
|
||||
|
||||
Jinja has no regex support, and exposing `std::regex` / Python `re` would not help: the two differ in many details (`\d`, `\w`, `\b`, lookbehind, case folding, backtracking limits), so a template could give different results in llama.cpp and in Python. Instead, task templates get two functions with a small matching language whose semantics are defined here:
|
||||
|
||||
- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments
|
||||
- `tts_norm_sub(text, pattern, repl)`: replace every match
|
||||
|
||||
Implementations:
|
||||
- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
|
||||
- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
|
||||
- Both are checked against each other by `tests/test-jinja.cpp` (`test-jinja -py "tts norm patterns.*"`)
|
||||
|
||||
## Example
|
||||
|
||||
```jinja
|
||||
{%- for s in tts_norm_scan(text, [
|
||||
['money', '%$(%d+)(?:%.(%d%d))?'],
|
||||
['percent', '(%d+)%%'],
|
||||
['time', '(%d%d?):(%d%d)(?!%d)'],
|
||||
['neg', '(?<!%w)-(%d+)'],
|
||||
['number', '%d+'],
|
||||
]) -%}
|
||||
{%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole) }} dollars{{ ' and ' ~ num(cents) ~ ' cents' if cents else '' }}
|
||||
{%- elif s.rule == 'percent' -%}{{ num(s.groups[0]) }} percent
|
||||
{%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h) }} {{ num(m) }}
|
||||
{%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0]) }}
|
||||
{%- elif s.rule == 'number' -%}{{ num(s.text) }}
|
||||
{%- else -%}{{ s.text }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
```
|
||||
|
||||
`It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats.` becomes `It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.` (with a `num` macro that spells numbers, see the "mini text normalizer" test in `tests/test-jinja.cpp`).
|
||||
|
||||
## Functions
|
||||
|
||||
### `tts_norm_scan(text, rules)`
|
||||
|
||||
`rules` is one of:
|
||||
- a list of `[name, pattern]` pairs (recommended, the order is explicit)
|
||||
- a dict `{name: pattern}`, in insertion order
|
||||
- a single pattern string, the rule is named `match`
|
||||
|
||||
Scanning starts at the first codepoint. At each position, the rules are tried in order and the first one that produces a **non-empty** match wins; the scan continues after the match. If no rule matches, one codepoint is skipped. Matches never overlap.
|
||||
|
||||
The result is a list of segments covering the whole text, in order. Consecutive unmatched codepoints are merged into one segment. Each segment is a dict:
|
||||
|
||||
| key | matched segment | unmatched segment |
|
||||
|---|---|---|
|
||||
| `rule` | rule name | `none` |
|
||||
| `text` | matched text | unmatched text |
|
||||
| `groups` | list of capture groups, in order of their `(` | `[]` |
|
||||
|
||||
There are no named groups, unpack `groups` instead: `{% set whole, cents = s.groups %}`.
|
||||
|
||||
A group that did not participate in the match (e.g. inside a `?` that matched nothing) is an empty string, so it renders as nothing and is falsy.
|
||||
|
||||
### `tts_norm_sub(text, pattern, repl)`
|
||||
|
||||
Replaces every match of `pattern`, with the same scanning rules as `tts_norm_scan`. `repl` is a literal string with references:
|
||||
|
||||
| reference | value |
|
||||
|---|---|
|
||||
| `%0` | whole match |
|
||||
| `%1` .. `%9` | group |
|
||||
| `%%` | a literal `%` |
|
||||
|
||||
Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.
|
||||
|
||||
## Pattern syntax
|
||||
|
||||
Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
|
||||
|
||||
| syntax | meaning |
|
||||
|---|---|
|
||||
| `x` | the literal codepoint `x` |
|
||||
| `%x` | escape: `%` followed by any non-alphanumeric codepoint is that literal codepoint (`%.`, `%$`, `%%`, `%(`, ...) |
|
||||
| `.` | any codepoint, including newlines |
|
||||
| `%d` `%s` `%p` `%a` `%w` `%u` `%l` `%h` `%c` | character classes, see below; the uppercase form (`%D`, ...) is the negation |
|
||||
| `[...]` | set: literal codepoints, ranges `a-z`, classes `%d`; `[^...]` is the negation |
|
||||
| `(...)` | capture group |
|
||||
| `(?:...)` | group without capture |
|
||||
| `a\|b` | ordered alternation |
|
||||
| `?` `*` `+` `{m}` `{m,}` `{m,n}` | quantifiers, greedy and possessive; `m`, `n` up to 1000 |
|
||||
| `^` `$` | start / end of the whole text |
|
||||
| `%b` | boundary: between a `%w` codepoint and a non-`%w` codepoint (the edges of the text count as non-`%w`) |
|
||||
| `(?=...)` `(?!...)` | lookahead, positive / negative |
|
||||
| `(?<=c)` `(?<!c)` | lookbehind of exactly one codepoint: `c` is a literal, a set, a class or `.` |
|
||||
| `(?i)` | at the very start of the pattern only: ignore ASCII case |
|
||||
|
||||
These characters must be escaped with `%` to be matched literally: `( ) [ ] { } . | ? * + ^ $ %`. Inside a set, only `]`, `%`, `-` (between two codepoints) and `^` (first position) are special.
|
||||
|
||||
### Matching semantics
|
||||
|
||||
The matcher is a PEG, not a backtracking regex:
|
||||
|
||||
- **Ordered alternation**: alternatives are tried left to right and the first one that matches is committed to. If what follows fails, the other alternatives are not retried. `(?:a|ab)c` never matches `abc`; write `(?:ab|a)c`.
|
||||
- **Possessive quantifiers**: quantifiers match as many repetitions as possible and never give any back. `%d+0` never matches; `.*x` never matches. Write patterns that describe exactly what to consume.
|
||||
- A repetition that matches the empty string ends the loop.
|
||||
- Captures inside lookaheads are discarded. A capture inside a repeated group holds its last repetition.
|
||||
|
||||
This gives a simple and linear-time implementation with identical results in C++ and Python.
|
||||
|
||||
### Character classes
|
||||
|
||||
Classes are defined by explicit codepoint ranges, not by a Unicode database, so the result does not depend on the Unicode version of the platform.
|
||||
|
||||
| class | definition |
|
||||
|---|---|
|
||||
| `%d` | ASCII digits `0-9` only |
|
||||
| `%s` | U+0009..U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000..U+200A, U+2028, U+2029, U+202F, U+205F, U+3000 |
|
||||
| `%p` | punctuation and symbols: ASCII punctuation, U+00A1..U+00BF (except U+00AA, U+00B5, U+00BA), U+00D7, U+00F7, U+2010..U+2027, U+2030..U+205E, U+20A0..U+20CF (currency), U+2100..U+2BFF (letterlike, number forms, arrows, math, technical, box drawing, shapes, misc symbols, dingbats), CJK punctuation (U+3001..U+3003, U+3008..U+3020, U+3030, U+303D, U+30FB), U+FE10..U+FE19, U+FE30..U+FE6B, fullwidth punctuation (U+FF01..U+FF0F, U+FF1A..U+FF20, U+FF3B..U+FF40, U+FF5B..U+FF65), U+1F000..U+1FAFF (emoji) |
|
||||
| `%c` | control and invisible format characters: U+0000..U+001F, U+007F..U+009F, U+200B..U+200F, U+2060..U+206F, U+FEFF |
|
||||
| `%a` | ASCII letters, U+00AA, U+00B5, U+00BA, and every codepoint from U+00C0 that is not in `%s`, `%p` or `%c` |
|
||||
| `%w` | `%a` or `%d` |
|
||||
| `%u` / `%l` | ASCII uppercase / lowercase letters only |
|
||||
| `%h` | Han: U+2E80..U+2FDF, U+3005, U+3007, U+3021..U+3029, U+3038..U+303B, U+3400..U+4DBF, U+4E00..U+9FFF, U+F900..U+FAFF, U+20000..U+323AF |
|
||||
|
||||
Classes may overlap (e.g. U+0085 is both `%s` and `%c`). `%a` intentionally includes all scripts, combining marks and CJK, so `(?<!%w)` and `%b` work as "not inside a word" for any language.
|
||||
|
||||
## Errors
|
||||
|
||||
Invalid patterns are rejected when the function is called, with a message and a position, for example: unbalanced parentheses, a quantifier with nothing to repeat, multiple quantifiers (`a+?`), repeating a zero-width assertion (`%b+`), unknown escapes (`%q`), unescaped `]` or `}`, an empty set, a lookbehind wider than one codepoint and `(?i)` not at the start.
|
||||
@@ -6,16 +6,15 @@ target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||
|
||||
if(LLAMA_BUILD_TESTS)
|
||||
if(NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
|
||||
set(MODEL_NAME "stories15M-q4_0.gguf")
|
||||
set(MODEL_NAME "tinyllamas/stories15M-q4_0.gguf")
|
||||
set(MODEL_HASH "SHA256=66967fbece6dbe97886593fdbb73589584927e29119ec31f08090732d1861739")
|
||||
else()
|
||||
set(MODEL_NAME "stories15M-be.Q4_0.gguf")
|
||||
set(MODEL_NAME "tinyllamas/stories15M-be.Q4_0.gguf")
|
||||
set(MODEL_HASH "SHA256=9aec857937849d976f30397e97eb1cabb53eb9dcb1ce4611ba8247fb5f44c65d")
|
||||
endif()
|
||||
set(MODEL_DEST "${CMAKE_BINARY_DIR}/${MODEL_NAME}")
|
||||
set(TEST_TARGET test-eval-callback)
|
||||
add_test(NAME ${TEST_TARGET}-download-model COMMAND ${CMAKE_COMMAND}
|
||||
-DREPO=ggml-org-dev/stories15M
|
||||
-DDEST=${MODEL_DEST}
|
||||
-DNAME=${MODEL_NAME}
|
||||
-DHASH=${MODEL_HASH}
|
||||
|
||||
@@ -20,12 +20,12 @@ Will print:
|
||||
```shell
|
||||
llm_load_tensors: offloaded 33/33 layers to GPU
|
||||
...
|
||||
llama_init_from_model: n_ctx = 512
|
||||
llama_new_context_with_model: n_ctx = 512
|
||||
...
|
||||
llama_init_from_model: CUDA0 compute buffer size = 105.00 MiB
|
||||
llama_init_from_model: CUDA_Host compute buffer size = 6.01 MiB
|
||||
llama_init_from_model: graph nodes = 1225
|
||||
llama_init_from_model: graph splits = 2
|
||||
llama_new_context_with_model: CUDA0 compute buffer size = 105.00 MiB
|
||||
llama_new_context_with_model: CUDA_Host compute buffer size = 6.01 MiB
|
||||
llama_new_context_with_model: graph nodes = 1225
|
||||
llama_new_context_with_model: graph splits = 2
|
||||
ggml_debug: inp_embd = (f32) GET_ROWS(token_embd.weight{2560, 51200, 1, 1}, inp_tokens{1, 1, 1, 1}}) = {2560, 1, 1, 1}
|
||||
[
|
||||
[
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# Usage:
|
||||
#! ./llama-server -m some-model.gguf &
|
||||
#! pip install pydantic
|
||||
#! python json_schema_pydantic_example.py
|
||||
|
||||
from pydantic import BaseModel, Field, TypeAdapter
|
||||
from annotated_types import MinLen
|
||||
from typing import Annotated, List, Optional
|
||||
import json, requests
|
||||
|
||||
if True:
|
||||
|
||||
def create_completion(*, response_model=None, endpoint="http://localhost:9931/v1/chat/completions", messages, **kwargs):
|
||||
'''
|
||||
Creates a chat completion using an OpenAI-compatible endpoint w/ JSON schema support
|
||||
(llama.cpp server, llama-cpp-python, Anyscale / Together...)
|
||||
|
||||
The response_model param takes a type (+ supports Pydantic) and behaves just as w/ Instructor (see below)
|
||||
'''
|
||||
response_format = None
|
||||
type_adapter = None
|
||||
|
||||
if response_model:
|
||||
type_adapter = TypeAdapter(response_model)
|
||||
schema = type_adapter.json_schema()
|
||||
messages = [{
|
||||
"role": "system",
|
||||
"content": f"You respond in JSON format with the following schema: {json.dumps(schema, indent=2)}"
|
||||
}] + messages
|
||||
response_format={"type": "json_object", "schema": schema}
|
||||
|
||||
data = requests.post(endpoint, headers={"Content-Type": "application/json"},
|
||||
json=dict(messages=messages, response_format=response_format, **kwargs)).json()
|
||||
if 'error' in data:
|
||||
raise Exception(data['error']['message'])
|
||||
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
return type_adapter.validate_json(content) if type_adapter else content
|
||||
|
||||
else:
|
||||
|
||||
# This alternative branch uses Instructor + OpenAI client lib.
|
||||
# Instructor support streamed iterable responses, retry & more.
|
||||
# (see https://python.useinstructor.com/)
|
||||
#! pip install instructor openai
|
||||
import instructor, openai
|
||||
client = instructor.patch(
|
||||
openai.OpenAI(api_key="123", base_url="http://localhost:9931"),
|
||||
mode=instructor.Mode.JSON_SCHEMA)
|
||||
create_completion = client.chat.completions.create
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
class QAPair(BaseModel):
|
||||
class Config:
|
||||
extra = 'forbid' # triggers additionalProperties: false in the JSON schema
|
||||
question: str
|
||||
concise_answer: str
|
||||
justification: str
|
||||
stars: Annotated[int, Field(ge=1, le=5)]
|
||||
|
||||
class PyramidalSummary(BaseModel):
|
||||
class Config:
|
||||
extra = 'forbid' # triggers additionalProperties: false in the JSON schema
|
||||
title: str
|
||||
summary: str
|
||||
question_answers: Annotated[List[QAPair], MinLen(2)]
|
||||
sub_sections: Optional[Annotated[List['PyramidalSummary'], MinLen(2)]]
|
||||
|
||||
print("# Summary\n", create_completion(
|
||||
model="...",
|
||||
response_model=PyramidalSummary,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": f"""
|
||||
You are a highly efficient corporate document summarizer.
|
||||
Create a pyramidal summary of an imaginary internal document about our company processes
|
||||
(starting high-level, going down to each sub sections).
|
||||
Keep questions short, and answers even shorter (trivia / quizz style).
|
||||
"""
|
||||
}]))
|
||||
@@ -99,7 +99,7 @@ static llama_context *init_context(llama_model *model, const int n_ctx = DEFAULT
|
||||
ctx_params.n_threads_batch = n_threads;
|
||||
auto *context = llama_init_from_model(g_model, ctx_params);
|
||||
if (context == nullptr) {
|
||||
LOGe("%s: llama_init_from_model() returned null)", __func__);
|
||||
LOGe("%s: llama_new_context_with_model() returned null)", __func__);
|
||||
}
|
||||
return context;
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user