llama.cpp/.github/workflows/release.yml

name: Release

on:
  workflow_dispatch: # allows manual triggering
    inputs:
      create_release:
        description: 'Create new release'
        required: true
        type: boolean
  push:
    branches:
      - master
    paths: [
      '.github/workflows/release.yml',
      '**/CMakeLists.txt',
      '**/.cmake',
      '**/*.h',
      '**/*.hpp',
      '**/*.c',
      '**/*.cpp',
      '**/*.cu',
      '**/*.cuh',
      '**/*.swift',
      '**/*.m',
      '**/*.metal',
      '**/*.comp',
      '**/*.glsl'
    ]

env:
  GH_TOKEN: ${{ github.token }}
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"

# note: run this workflow one at a time for better cache reuse
concurrency:
  group: release
  queue: max

jobs:
  check-release:
    runs-on: ubuntu-slim

    outputs:
      should_release: ${{ steps.check.outputs.should_release }}

    steps:
      - id: check
        env:
          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
        run: |
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            echo "should_release=true" >> $GITHUB_OUTPUT
          elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/master" ]]; then
            if echo "$COMMIT_MESSAGE" | grep -q '\[no release\]'; then
              echo "should_release=false" >> $GITHUB_OUTPUT
            else
              echo "should_release=true" >> $GITHUB_OUTPUT
            fi
          else
            echo "should_release=false" >> $GITHUB_OUTPUT
          fi

  macos-cpu:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}
    strategy:
      matrix:
        include:
          - build: 'arm64'
            arch: 'arm64'
            os: macos-26
            defines: "-DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"
          # TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23780)
          #       in order to enable it again, we have to provision dedicated runners  to run it
          #- build: 'arm64-kleidiai'
          #  arch: 'arm64'
          #  os: macos-14
          #  defines: "-DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 -DGGML_CPU_KLEIDIAI=ON"
          - build: 'x64'
            arch: 'x64'
            os: macos-15-intel
            # Metal is disabled on x64 due to intermittent failures with Github runners not having a GPU:
            # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
            defines: "-DGGML_METAL=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"

    runs-on: ${{ matrix.os }}

    permissions:
      actions: write

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-${{ matrix.os }}-${{ matrix.arch }}
          evict-old-files: 1d

      - name: Build
        id: cmake_build
        run: |
          sysctl -a
          cmake -B build \
            ${{ matrix.defines }} \
            -DCMAKE_INSTALL_RPATH='@loader_path' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            -DLLAMA_BUILD_BORINGSSL=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz -s ",^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz
          name: llama-bin-macos-${{ matrix.build }}.tar.gz

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-${{ matrix.os }}-${{ matrix.arch }}

  ubuntu-cpu:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}
    strategy:
      matrix:
        include:
          - build: 'x64'
            os: ubuntu-22.04
          - build: 'arm64'
            os: ubuntu-24.04-arm
          - build: 's390x'
            os: ubuntu-24.04-s390x

    runs-on: ${{ matrix.os }}

    permissions:
      actions: write

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: Dependencies
        id: depends
        run: |
          sudo apt-get update
          sudo apt-get install build-essential libssl-dev

      - name: Toolchain workaround (GCC 14)
        if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
        run: |
          sudo apt-get install -y gcc-14 g++-14
          echo "CC=gcc-14" >> "$GITHUB_ENV"
          echo "CXX=g++-14" >> "$GITHUB_ENV"

      - name: ccache
        if: ${{ matrix.build != 's390x' }}
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-${{ matrix.os }}-cpu
          evict-old-files: 1d

      - name: Build
        id: cmake_build
        run: |
          cmake -B build \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j $(nproc)

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz
          name: llama-bin-ubuntu-${{ matrix.build }}.tar.gz

      - name: ccache-clear
        if: ${{ matrix.build != 's390x' }}
        uses: ./.github/actions/ccache-clear
        with:
          key: release-${{ matrix.os }}-cpu

  ubuntu-vulkan:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    strategy:
      matrix:
        include:
          - build: 'x64'
            os: ubuntu-22.04
          - build: 'arm64'
            os: ubuntu-24.04-arm

    runs-on: ${{ matrix.os }}

    permissions:
      actions: write

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: Dependencies
        id: depends
        run: |
          if [[ "${{ matrix.os }}" =~ "ubuntu-22.04" ]]; then
            wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -
            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
            sudo apt-get update -y
            sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev
          else
            sudo apt-get update -y
            sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build
            echo "CC=gcc-14" >> "$GITHUB_ENV"
            echo "CXX=g++-14" >> "$GITHUB_ENV"
          fi

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-${{ matrix.os }}-vulkan
          evict-old-files: 1d

      - name: Build
        id: cmake_build
        run: |
          cmake -B build \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_VULKAN=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j $(nproc)

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz
          name: llama-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-${{ matrix.os }}-vulkan

  ubuntu-cuda:
    name: ubuntu-cuda (${{ matrix.label }}, ${{ matrix.build }})
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    strategy:
      matrix:
        include:
          # label = short version used in artifact names / release body
          # cuda  = full container image tag
          - build: 'x64'
            os: ubuntu-24.04
            cuda: '12.8.2'
            label: '12.8'
            defines: '-DGGML_CUDA_CUB_3DOT2=ON'
          - build: 'x64'
            os: ubuntu-24.04
            cuda: '13.4.1'
            label: '13.4'
            defines: ''
          - build: 'arm64'
            os: ubuntu-24.04-arm
            cuda: '13.4.1'
            label: '13.4'
            defines: ''

    runs-on: ${{ matrix.os }}
    container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04

    permissions:
      actions: write

    steps:
      # the container has no git; install it before checkout so that a real git
      # repository is created (the get-tag-name action and the build both need it)
      - name: Install git
        run: |
          apt-get update
          apt-get install -y --no-install-recommends git

      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      # checkout runs as the host user; in-container steps run as root, so git
      # refuses to touch a repo it does not own. Mark the workspace as safe.
      # use the env var: the github.workspace context holds the HOST path,
      # GITHUB_WORKSPACE the container path
      - name: Git safe directory
        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: Dependencies
        id: depends
        # container jobs default to sh (dash); need bash for the [[ ]] below
        shell: bash
        run: |
          apt-get update
          apt-get install -y --no-install-recommends build-essential cmake ninja-build libssl-dev jq python3-venv
          # the container ships GCC 13, which does not know the 'sme' march
          # feature used by the armv9.2 CPU variant of GGML_CPU_ALL_VARIANTS
          if [[ "${{ matrix.build }}" == "arm64" ]]; then
            apt-get install -y --no-install-recommends gcc-14 g++-14
            echo "CC=gcc-14" >> "$GITHUB_ENV"
            echo "CXX=g++-14" >> "$GITHUB_ENV"
          fi

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }}
          evict-old-files: 1d
          max-size: "1G"

      - name: Build
        id: cmake_build
        # no CMAKE_CUDA_ARCHITECTURES: use the broad default arch set from
        # ggml/src/ggml-cuda/CMakeLists.txt so the release binary covers many GPUs
        run: |
          cmake -B build \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_CUDA=ON \
            -DGGML_CUDA_NCCL=OFF \
            ${{ env.CMAKE_ARGS }} ${{ matrix.defines }}
          cmake --build build --config Release -j $(nproc)

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
          name: llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz

      # ship the CUDA runtime libraries the backend links against, mirroring
      # the windows-cuda cudart zip - extract next to the binaries ($ORIGIN rpath)
      - name: Pack CUDA runtime
        id: pack_cuda_runtime
        run: |
          major="${{ matrix.label }}"
          major="${major%%.*}"
          mkdir -p ./cudart
          # cp -L dereferences the SONAME symlinks into plain files, so the
          # tarball holds exactly 3 files with no versioned duplicates
          cp -L /usr/local/cuda/lib64/libcudart.so.${major} ./cudart/
          cp -L /usr/local/cuda/lib64/libcublas.so.${major} ./cudart/
          cp -L /usr/local/cuda/lib64/libcublasLt.so.${major} ./cudart/
          tar -czvf cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}," -C ./cudart .

      - name: Upload CUDA runtime
        uses: actions/upload-artifact@v6
        with:
          path: cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
          name: cudart-llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }}

  android-arm64:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: ubuntu-24.04  # previously ubuntu-latest

    #permissions:
    #  actions: write

    env:
      NDK_VERSION: "29.0.14206865"

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: Set up JDK
        uses: actions/setup-java@v5
        with:
          java-version: 17
          distribution: temurin

      - name: Setup Android SDK
        uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
        with:
          log-accepted-android-sdk-licenses: false

      - name: Install NDK
        run: |
          sdkmanager "ndk;${{ env.NDK_VERSION }}"
          echo "ANDROID_NDK=${ANDROID_SDK_ROOT}/ndk/${{ env.NDK_VERSION }}" >> $GITHUB_ENV

      # note : disabled to spare some cache space (https://github.com/ggml-org/llama.cpp/pull/23789)
      #        for some reason, the ccache does not improve the build time in this case
      # example:
      #   cache off: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78160400831
      #   cache on:  https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
      #
      #- name: ccache
      #  uses: ggml-org/ccache-action@v1.2.24
      #  with:
      #    key: release-android-arm64
      #    evict-old-files: 1d

      - name: Build
        id: cmake_build
        run: |
          cmake -B build \
            -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \
            -DANDROID_ABI=arm64-v8a \
            -DANDROID_PLATFORM=android-28 \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            -DGGML_OPENMP=OFF \
            -DLLAMA_BUILD_BORINGSSL=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j $(nproc)

      #- name: ccache-clear
      #  uses: ./.github/actions/ccache-clear
      #  with:
      #    key: release-android-arm64

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz
          name: llama-bin-android-arm64.tar.gz

  android-arm64-snapdragon:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: ubuntu-latest
    container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'

    defaults:
      run:
        shell: bash

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      # checkout runs as the host user; in-container steps run as root, so git
      # refuses to touch a repo it does not own. Mark the workspace as safe.
      - name: Git safe directory
        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: Build
        id: cmake_build
        run: |
          cp docs/backend/snapdragon/CMakeUserPresets.json .
          cmake --preset arm64-android-snapdragon-release -B build \
            -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DLLAMA_BUILD_BORINGSSL=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build -j $(nproc)
          cmake --install build --prefix pkg-snapdragon/llama.cpp

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE pkg-snapdragon/llama.cpp/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C pkg-snapdragon/llama.cpp .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz
          name: llama-bin-android-arm64-snapdragon.tar.gz

  linux-arm64-snapdragon:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: ubuntu-latest
    container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'

    defaults:
      run:
        shell: bash

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      # checkout runs as the host user; in-container steps run as root, so git
      # refuses to touch a repo it does not own. Mark the workspace as safe.
      - name: Git safe directory
        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: Build
        id: cmake_build
        run: |
          cp docs/backend/snapdragon/CMakeUserPresets.json .
          cmake --preset arm64-linux-snapdragon-release -B build -DGGML_OPENCL=ON \
            -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DLLAMA_BUILD_BORINGSSL=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build -j $(nproc)
          cmake --install build --prefix pkg-snapdragon/llama.cpp

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE pkg-snapdragon/llama.cpp/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C pkg-snapdragon/llama.cpp .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz
          name: llama-bin-linux-arm64-snapdragon.tar.gz

  ubuntu-24-openvino:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: ubuntu-24.04

    permissions:
      actions: write

    outputs:
      openvino_version: ${{ steps.openvino_version.outputs.value }}

    env:
      # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
      OPENVINO_VERSION_MAJOR: "2026.4"
      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"

    steps:
      - name: Set OpenVINO version output
        id: openvino_version
        run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT

      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-ubuntu-24.04-openvino-release-no-preset-v1
          evict-old-files: 1d

      - name: Dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential libssl-dev libtbb12 cmake ninja-build python3-pip
          sudo apt install ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd

      - name: Use OpenVINO Toolkit Cache
        uses: actions/cache@v5
        id: cache-openvino
        with:
          path: ./openvino_toolkit
          key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}

      - name: Setup OpenVINO Toolkit
        if: steps.cache-openvino.outputs.cache-hit != 'true'
        uses: ./.github/actions/linux-setup-openvino
        with:
          path: ./openvino_toolkit
          version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
          version_full: ${{ env.OPENVINO_VERSION_FULL }}

      - name: Install OpenVINO dependencies
        run: |
          cd ./openvino_toolkit
          chmod +x ./install_dependencies/install_openvino_dependencies.sh
          echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh

      - name: Build
        id: cmake_build
        run: |
          source ./openvino_toolkit/setupvars.sh
          cmake -B build/ReleaseOV -G Ninja \
            -DCMAKE_BUILD_TYPE=Release \
            -DGGML_OPENVINO=ON \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build/ReleaseOV --config Release --parallel

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          dest=./build/ReleaseOV/bin
          OPENVINO_ROOT=./openvino_toolkit
          ov_lib="$OPENVINO_ROOT/runtime/lib/intel64"

          # Bundle OpenVINO runtime libs + TBB. Binaries built with RPATH=$ORIGIN
          # load these siblings without setupvars.sh / LD_LIBRARY_PATH.
          cp -P "$ov_lib"/libopenvino.so* \
                "$ov_lib"/libopenvino_c.so* \
                "$ov_lib"/libopenvino_*_plugin.so \
                "$ov_lib"/libopenvino_intel_npu_compiler*.so \
                "$OPENVINO_ROOT"/runtime/3rdparty/tbb/lib/*.so* \
                "$dest"
          cp -P /usr/lib/x86_64-linux-gnu/libOpenCL.so.1* "$dest" 2>/dev/null || true
          cp "$ov_lib"/cache.json "$dest" 2>/dev/null || true

          # OpenVINO licensing
          cp -r "$OPENVINO_ROOT"/docs/licensing "$dest"/openvino-licensing

          cp LICENSE "$dest"
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C "$dest" .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz
          name: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-ubuntu-24.04-openvino-release-no-preset-v1

  windows-openvino:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: windows-2022

    outputs:
      openvino_version: ${{ steps.openvino_version.outputs.value }}

    env:
      # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
      OPENVINO_VERSION_MAJOR: "2026.4"
      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"

    steps:
      - name: Set OpenVINO version output
        id: openvino_version
        shell: bash
        run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT

      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
            fetch-depth: 0

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-windows-2022-openvino
          variant: ccache
          evict-old-files: 1d

      - name: Setup Cache
        uses: actions/cache@v5
        id: cache-openvino
        with:
          path: ./openvino_toolkit
          key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}

      - name: Setup OpenVINO Toolkit
        if: steps.cache-openvino.outputs.cache-hit != 'true'
        uses: ./.github/actions/windows-setup-openvino
        with:
          path: ./openvino_toolkit
          version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
          version_full: ${{ env.OPENVINO_VERSION_FULL }}

      - name: Install OpenCL using vcpkg
        shell: powershell
        run: |
          git clone https://github.com/microsoft/vcpkg C:\vcpkg
          C:\vcpkg\bootstrap-vcpkg.bat
          C:\vcpkg\vcpkg install opencl

      - name: Build
        id: cmake_build
        shell: cmd
        run: |
          REM Find extracted OpenVINO folder dynamically
          for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i

          if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" (
              echo ERROR: OpenVINOConfig.cmake not found
              exit /b 1
          )

          call "%OPENVINO_ROOT%\setupvars.bat"

          cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^
            -A x64 ^
            -DCMAKE_BUILD_TYPE=Release ^
            -DGGML_OPENVINO=ON ^
            -DLLAMA_BUILD_BORINGSSL=ON ^
            -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
            ${{ env.CMAKE_ARGS }}

          cmake --build build\ReleaseOV --config Release -- /m

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        shell: powershell
        run: |
          # Locate the extracted OpenVINO toolkit root (same pattern as the Build step).
          $OPENVINO_ROOT = (Get-ChildItem -Directory openvino_toolkit | Select-Object -First 1).FullName
          if (-not $OPENVINO_ROOT) {
            Write-Error "OpenVINO toolkit folder not found under .\openvino_toolkit"
            exit 1
          }

          $dest = ".\build\ReleaseOV\bin\Release"

          $ovBin = Join-Path $OPENVINO_ROOT 'runtime\bin\intel64\Release'
          Copy-Item -Path (Join-Path $ovBin '*.dll')       -Destination $dest -Force
          Copy-Item -Path (Join-Path $ovBin 'cache.json')  -Destination $dest -Force

          $tbbBin = Join-Path $OPENVINO_ROOT 'runtime\3rdparty\tbb\bin'
          Copy-Item -Path (Join-Path $tbbBin 'tbb*.dll') -Destination $dest -Force

          # OpenVINO licensing
          $licensingDest = Join-Path $dest 'openvino-licensing'
          New-Item -ItemType Directory -Force -Path $licensingDest | Out-Null
          Copy-Item -Path (Join-Path $OPENVINO_ROOT 'docs\licensing\*') -Destination $licensingDest -Recurse -Force

          Copy-Item LICENSE $dest
          7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip $dest\*

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
          name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-windows-2022-openvino

  windows-cpu:
    name: windows-cpu / ${{ matrix.arch }}
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: windows-2025-vs2026

    permissions:
      actions: write

    strategy:
      matrix:
        include:
          - arch: 'x64'
          - arch: 'arm64'

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: Install Ninja
        run: |
          choco install ninja

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
          evict-old-files: 1d

      - name: Build
        shell: cmd
        run: |
          call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}
          cmake -S . -B build -G "Ninja Multi-Config" ^
            -D CMAKE_TOOLCHAIN_FILE=cmake/${{ matrix.arch }}-windows-llvm.cmake ^
            -DLLAMA_BUILD_BORINGSSL=ON ^
            -DGGML_NATIVE=OFF ^
            -DGGML_BACKEND_DL=ON ^
            -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^
            -DGGML_OPENMP=ON ^
            -DGGML_OPENMP_FETCH=ON ^
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          7z a -snl llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\*

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-bin-win-cpu-${{ matrix.arch }}.zip
          name: llama-bin-win-cpu-${{ matrix.arch }}.zip

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu

  # note: builds only the ggml-hip backend - llama-server is injected from the
  #       windows-cpu zip during the release "Merge artifacts" step
  windows-rocm:
    needs: [check-release]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: windows-2022

    strategy:
      matrix:
        include:
          - ROCM_VERSION: "10.0.0"
            gpu_targets: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201"
            build: x64

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Install Ninja
        run: |
          choco install ninja

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
          evict-old-files: 1d
          max-size: "1G"

      # - name: Cache ROCm Installation
      #   id: cache-rocm
      #   uses: actions/cache@v5
      #   with:
      #     path: C:\TheRock\build
      #     key: rocm-wheels-${{ matrix.ROCM_VERSION }}-multi-arch-${{ runner.os }}

      - name: Setup ROCm
        # if: steps.cache-rocm.outputs.cache-hit != 'true'
        uses: ./.github/actions/windows-setup-rocm
        with:
          version: ${{ matrix.ROCM_VERSION }}

      - name: Setup ROCm Environment
        run: |
          $ErrorActionPreference = "Stop"

          # Activate venv from cache or fresh install
          & C:\TheRock\build\.venv\Scripts\Activate.ps1

          # Expand the devel tree (idempotent; no-op if already done during install)
          rocm-sdk init
          if ($LASTEXITCODE -ne 0) { throw "rocm-sdk init failed with exit code $LASTEXITCODE" }

          # Get ROCm installation paths using the rocm-sdk CLI tool
          $rocmPath = (rocm-sdk path --root)
          if (-not $rocmPath) { throw "rocm-sdk path --root returned empty - devel package may not be installed" }
          $rocmPath = $rocmPath.Trim()
          $cmakePath = (rocm-sdk path --cmake).Trim()
          $binPath = (rocm-sdk path --bin).Trim()
          write-host "ROCm root: $rocmPath"
          write-host "CMake path: $cmakePath"
          write-host "Bin path: $binPath"

          echo "HIP_PATH=$rocmPath" >> $env:GITHUB_ENV
          echo "CMAKE_PREFIX_PATH=$cmakePath" >> $env:GITHUB_ENV
          echo "HIP_DEVICE_LIB_PATH=$rocmPath\lib\llvm\amdgcn\bitcode" >> $env:GITHUB_ENV
          echo "HIP_PLATFORM=amd" >> $env:GITHUB_ENV
          echo "LLVM_PATH=$rocmPath\lib\llvm" >> $env:GITHUB_ENV
          echo "$binPath" >> $env:GITHUB_PATH

          # Keep venv in PATH for subsequent steps
          echo "C:\TheRock\build\.venv\Scripts" >> $env:GITHUB_PATH

      - name: Build
        run: |
          cmake -S . -B build `
            -G "Ninja Multi-Config" `
            -DCMAKE_PREFIX_PATH="${env:HIP_PATH}" `
            -DGGML_BACKEND_DL=ON `
            -DGGML_NATIVE=OFF `
            -DGGML_CPU=OFF `
            -DGGML_HIP=ON `
            -DCMAKE_C_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
            -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang++.exe" `
            -DCMAKE_C_FLAGS="-Wno-error=incompatible-pointer-types" `
            -DCMAKE_HIP_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
            -DHIP_PATH="${env:HIP_PATH}" `
            -DAMDGPU_TARGETS="${{ matrix.gpu_targets }}"
          cmake --build build --config Release --parallel ${env:NUMBER_OF_PROCESSORS} --target ggml-hip

      - name: Verify HIP backend was built
        run: |
          $hipDll = Get-ChildItem -Path build\bin\Release -Filter "ggml-hip*.dll" -ErrorAction SilentlyContinue
          if (-not $hipDll) {
            Write-Host "##[error]ggml-hip*.dll was NOT produced. The HIP backend silently failed to build."
            Write-Host "Contents of build\bin\Release:"
            Get-ChildItem build\bin\Release | Format-Table -AutoSize
            exit 1
          }
          Write-Host "HIP backend artifact found:"
          $hipDll | Format-Table FullName, Length -AutoSize

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Get ROCm short version
        run: |
          $rocmVersionShort = ('${{ matrix.ROCM_VERSION }}'.Split('.')[0..1] -join '.')
          echo "ROCM_VERSION_SHORT=$rocmVersionShort" >> $env:GITHUB_ENV

      - name: Bundle HIP runtime DLLs (amdhip64_7.dll, rocm_kpack.dll, amd_comgr.dll)
        run: |
          $ErrorActionPreference = "Stop"
          # See issue https://github.com/ggml-org/llama.cpp/issues/26929.
          # ggml-hip.dll loads amdhip64_7.dll at run time. The Adrenalin driver
          # ships an amdhip64_7.dll in System32, which the loader searches before PATH,
          # so a matching DLL from PATH cannot win. Copy amdhip64 next to the
          # binaries (exe directory is searched before System32) so the correct
          # runtime is used. rocm_kpack.dll is amdhip64_7's direct dependency, so
          # copy the matching version too. amd_comgr is copied as well to keep it
          # in sync with the bundled amdhip64, avoiding a version mismatch with a
          # amd_comgr from System32.
          # rocblas/hipblaslt kernels resolve fine via PATH and are not copied.
          $binPath = (rocm-sdk path --bin).Trim()
          if (-not $binPath) { throw "rocm-sdk path --bin returned empty" }
          write-host "ROCm bin path: $binPath"

          $patterns = @("amdhip64_7.dll", "rocm_kpack.dll", "amd_comgr.dll")
          foreach ($pattern in $patterns) {
            $files = Get-ChildItem -Path $binPath -Filter $pattern -ErrorAction SilentlyContinue
            if (-not $files) { throw "no match for $pattern in $binPath" }
            foreach ($f in $files) {
              Copy-Item $f.FullName -Destination build\bin\Release -Force
              write-host "  copied $($f.Name)"
            }
          }

      - name: Pack artifacts
        run: |
          7z a -snl llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip `
            .\build\bin\Release\ggml-hip.dll `
            .\build\bin\Release\amdhip64_7.dll `
            .\build\bin\Release\rocm_kpack.dll `
            .\build\bin\Release\amd_comgr.dll

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip
          name: llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}

  # note: builds only the backend library - llama-server (with the embedded UI)
  #       is injected from the windows-cpu zip during the release "Merge artifacts" step
  windows:
    needs: [check-release]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: windows-2025

    permissions:
      actions: write

    env:
      OPENBLAS_VERSION: 0.3.23
      VULKAN_VERSION: 1.4.357.0

    strategy:
      matrix:
        include:
          - backend: 'vulkan'
            arch: 'x64'
            defines: '-DGGML_VULKAN=ON'
            target: 'ggml-vulkan'
          - backend: 'opencl-adreno'
            arch: 'arm64'
            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON'
            target: 'ggml-opencl'

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6

      - name: Install Vulkan SDK
        id: get_vulkan
        if: ${{ matrix.backend == 'vulkan' }}
        run: |
          curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
          & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
          Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
          Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"

      - name: Install Ninja
        id: install_ninja
        run: |
          choco install ninja

      # TODO: these jobs need to use llvm toolchain in order to utilize the ccache
      #- name: ccache
      #  uses: ggml-org/ccache-action@v1.2.24
      #  with:
      #    key: release-windows-2025-${{ matrix.arch }}-${{ matrix.backend }}
      #    evict-old-files: 1d

      - name: Install OpenCL Headers and Libs
        id: install_opencl
        if: ${{ matrix.backend == 'opencl-adreno' && matrix.arch == 'arm64' }}
        run: |
          git clone https://github.com/KhronosGroup/OpenCL-Headers
          cd OpenCL-Headers
          cmake -B build `
            -DBUILD_TESTING=OFF `
            -DOPENCL_HEADERS_BUILD_TESTING=OFF `
            -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
            -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
          cmake --build build --target install
          git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader
          cd OpenCL-ICD-Loader
          cmake -B build-arm64-release `
            -A arm64 `
            -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `
            -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
          cmake --build build-arm64-release --target install --config release

      - name: Build
        id: cmake_build
        run: |
          cmake -S . -B build ${{ matrix.defines }} -DGGML_NATIVE=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_BUILD_BORINGSSL=ON
          cmake --build build --config Release --target ${{ matrix.target }}

      #- name: ccache-clear
      #  uses: ./.github/actions/ccache-clear
      #  with:
      #    key: release-windows-2025-${{ matrix.arch }}-${{ matrix.backend }}

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          7z a -snl llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip .\build\bin\Release\${{ matrix.target }}.dll

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
          name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip

  # note: builds only the ggml-cuda backend - llama-server is injected from the
  #       windows-cpu zip during the release "Merge artifacts" step
  windows-cuda:
    name: windows-cuda (${{ matrix.cuda }}, ${{ matrix.arch }})
    needs: [check-release]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: windows-2022

    permissions:
      actions: write

    strategy:
      matrix:
        include:
          - cuda: '12.4'
            arch: x64
            defines: '-DGGML_CUDA_CUB_3DOT2=ON'
          - cuda: '13.4'
            arch: x64
            defines: ''
          - cuda: '13.4'
            arch: arm64
            defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6

      - name: Install Cuda Toolkit
        uses: ./.github/actions/windows-setup-cuda
        with:
          cuda_version: ${{ matrix.cuda }}
          cuda_arch: ${{ matrix.arch }}

      - name: Install Ninja
        id: install_ninja
        run: |
          choco install ninja

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
          evict-old-files: 1d

      - name: Build
        id: cmake_build
        shell: cmd
        # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
        run: |
          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}
          cmake -S . -B build -G "Ninja Multi-Config" ^
            -DGGML_BACKEND_DL=ON ^
            -DGGML_NATIVE=OFF ^
            -DGGML_CPU=OFF ^
            -DGGML_CUDA=ON ^
            -DLLAMA_BUILD_BORINGSSL=ON ${{ matrix.defines }}
          set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
          cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          7z a -snl llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip .\build\bin\Release\ggml-cuda.dll

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
          name: llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip

      - name: Copy and pack Cuda runtime (x64)
        if: ${{ matrix.arch == 'x64' }}
        run: |
          echo "Cuda install location: ${{ env.CUDA_PATH }}"
          $dst='.\build\bin\cudart\'
          robocopy "${{env.CUDA_PATH}}\bin" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll
          robocopy "${{env.CUDA_PATH}}\lib" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll
          robocopy "${{env.CUDA_PATH}}\bin\x64" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll
          7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip $dst\*

      - name: Copy and pack Cuda runtime (ARM64)
        if: ${{ matrix.arch == 'arm64' }}
        run: |
          echo "Cuda install location: ${{ env.CUDA_PATH }}"
          $dst='.\build\bin\cudart\'
          robocopy "${{env.CUDA_PATH}}\bin\arm64" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll
          7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip $dst\*

      - name: Upload Cuda runtime
        uses: actions/upload-artifact@v6
        with:
          path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
          name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}

  # note: builds only the ggml-sycl backend - llama-server is injected from the
  #       windows-cpu zip during the release "Merge artifacts" step
  windows-sycl:
    needs: [check-release]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: windows-2022

    defaults:
      run:
        shell: bash

    env:
      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
      ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
      ONEAPI_INSTALLER_VERSION: "2025.3.3"

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6

      - name: Download & Install oneAPI
        shell: bash
        run: |
          scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL

      - name: Install Level Zero SDK
        shell: pwsh
        run: |
          Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile "level-zero-win-sdk.zip"
          Expand-Archive -Path "level-zero-win-sdk.zip" -DestinationPath "C:/level-zero-sdk" -Force
          "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-windows-2022-x64-sycl
          evict-old-files: 1d

      - name: Build
        id: cmake_build
        shell: cmd
        run: |
          call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
          cmake -G "Ninja" -B build ^
            -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx ^
            -DCMAKE_BUILD_TYPE=Release ^
            -DGGML_BACKEND_DL=ON -DBUILD_SHARED_LIBS=ON ^
            -DGGML_CPU=OFF -DGGML_SYCL=ON ^
            -DLLAMA_BUILD_BORINGSSL=ON
          cmake --build build --target ggml-sycl -j %NUMBER_OF_PROCESSORS%

      - name: Build the release package
        id: pack_artifacts
        run: |
          echo "cp oneAPI running time dll files in ${{ env.ONEAPI_ROOT }} to ./build/bin"

          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_sycl_blas.5.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_core.2.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.2.dll" ./build/bin

          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_opencl.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_win_proxy_loader.dll" ./build/bin
          ZE_LOADER_DLL=$(find "${{ env.ONEAPI_ROOT }}" "$LEVEL_ZERO_V1_SDK_PATH" -iname ze_loader.dll -print -quit 2>/dev/null || true)
          if [ -n "$ZE_LOADER_DLL" ]; then
            echo "Using Level Zero loader: $ZE_LOADER_DLL"
            cp "$ZE_LOADER_DLL" ./build/bin
          else
            echo "Level Zero loader DLL not found in oneAPI or SDK; relying on system driver/runtime"
          fi

          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl8.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/svml_dispmd.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libmmd.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libiomp5md.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl-ls.exe" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libsycl-fallback-bfloat16.spv" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libsycl-native-bfloat16.spv" ./build/bin

          cp "${{ env.ONEAPI_ROOT }}/dnnl/latest/bin/dnnl.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/tbb/latest/bin/tbb12.dll" ./build/bin

          cp "${{ env.ONEAPI_ROOT }}/tcm/latest/bin/tcm.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/tcm/latest/bin/libhwloc-15.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/umf/latest/bin/umf.dll" ./build/bin

          echo "cp oneAPI running time dll files to ./build/bin done"
          7z a -snl llama-bin-win-sycl-x64.zip ./build/bin/*

      - name: Upload the release package
        uses: actions/upload-artifact@v6
        with:
          path: llama-bin-win-sycl-x64.zip
          name: llama-bin-win-sycl-x64.zip

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-windows-2022-x64-sycl

  ubuntu-24-sycl:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    strategy:
      matrix:
        build: [fp32, fp16]
        include:
          - build: fp32
            fp16: OFF
          - build: fp16
            fp16: ON

    runs-on: ubuntu-24.04

    env:
      ONEAPI_ROOT: /opt/intel/oneapi/
      ONEAPI_INSTALLER_VERSION: "2025.3.3"
      LEVEL_ZERO_VERSION: "1.28.2"
      LEVEL_ZERO_UBUNTU_VERSION: "u24.04"

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download & Install oneAPI
        shell: bash
        run: |
          cd /tmp
          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
          sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept

      - name: Install Level Zero SDK
        shell: bash
        run: |
          cd /tmp
          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
          sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-ubuntu-24.04-sycl-${{ matrix.build }}
          evict-old-files: 1d

      - name: Build
        id: cmake_build
        run: |
          source /opt/intel/oneapi/setvars.sh
          cmake -B build \
            -G "Ninja" \
            -DCMAKE_BUILD_TYPE=Release \
            -DGGML_SYCL=ON \
            -DCMAKE_C_COMPILER=icx \
            -DCMAKE_CXX_COMPILER=icpx \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DLLAMA_OPENSSL=OFF \
            -DGGML_NATIVE=OFF \
            -DGGML_SYCL_F16=${{ matrix.fp16 }}
          time cmake --build build --config Release -j $(nproc)

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
          name: llama-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-ubuntu-24.04-sycl-${{ matrix.build }}

  ubuntu-24-rocm:
    needs: [check-release, ui-build]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}

    runs-on: ubuntu-24.04

    permissions:
      actions: write

    strategy:
      matrix:
        include:
          - ROCM_VERSION: "10.0.0"
            gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
            build: 'x64'

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download UI build
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: tools/ui/dist

      - name: Free up disk space
        uses: ggml-org/free-disk-space@v1.3.1
        with:
          tool-cache: true

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
          evict-old-files: 1d
          max-size: "1G"

      - name: Tune ccache for reinstalled ROCm toolchain
        run: |
          # ROCm is pip-installed fresh each run, so the clang binary's mtime
          # changes every time. With the default compiler_check=mtime that
          # invalidates the cache; hash compiler contents instead so warm
          # builds hit.
          ccache --set-config=compiler_check=content
          ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime

      - name: Dependencies
        id: depends
        run: |
          sudo apt install -y build-essential git cmake wget

      - name: Setup TheRock with Wheels
        id: therock_env
        run: |
          # Create Python virtual environment
          python3 -m venv .venv
          source .venv/bin/activate

          # Install ROCm wheels for build
          # libraries = HIP runtime and CMake configs needed for linking
          # devel = compilers, headers, static libs
          python -m pip install --upgrade pip
          python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"

          # Get ROCm installation paths using the rocm-sdk CLI tool
          ROCM_PATH=$(rocm-sdk path --root)
          CMAKE_PATH=$(rocm-sdk path --cmake)
          BIN_PATH=$(rocm-sdk path --bin)
          echo "ROCM_PATH=$ROCM_PATH"
          echo "CMAKE_PATH=$CMAKE_PATH"
          echo "BIN_PATH=$BIN_PATH"

          # Set environment variables
          echo "ROCM_PATH=$ROCM_PATH" >> $GITHUB_ENV
          echo "CMAKE_PREFIX_PATH=$CMAKE_PATH" >> $GITHUB_ENV
          echo "HIP_PATH=$ROCM_PATH" >> $GITHUB_ENV
          echo "PATH=$BIN_PATH:${PATH}" >> $GITHUB_ENV
          echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV

          # Keep venv activated for subsequent steps
          echo "$(pwd)/.venv/bin" >> $GITHUB_PATH

      - name: Build with native CMake HIP support
        id: cmake_build
        run: |
          cmake -B build -S . \
            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
            -DCMAKE_BUILD_TYPE=Release \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
            -DGGML_HIP=ON \
            -DHIP_PLATFORM=amd \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j $(nproc)

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Get ROCm short version
        run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
          name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz

      - name: ccache-clear
        uses: ./.github/actions/ccache-clear
        with:
          key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}

  ios-xcode:
    needs: [check-release]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}
    runs-on: macos-26

    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup Xcode
        run: |
          sudo xcode-select -s /Applications/Xcode_26.4.app

      - name: Build
        id: cmake_build
        run: |
          sysctl -a
          cmake -B build -G Xcode \
            -DGGML_METAL_EMBED_LIBRARY=ON \
            -DLLAMA_OPENSSL=OFF \
            -DLLAMA_BUILD_APP=OFF \
            -DLLAMA_BUILD_EXAMPLES=OFF \
            -DLLAMA_BUILD_TOOLS=OFF \
            -DLLAMA_BUILD_TESTS=OFF \
            -DLLAMA_BUILD_SERVER=OFF \
            -DCMAKE_SYSTEM_NAME=iOS \
            -DCMAKE_OSX_DEPLOYMENT_TARGET=16.0 \
            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO

      - name: xcodebuild for swift package
        id: xcodebuild
        run: |
          # note: only macos and ios-device due to long build time
          # ref: https://github.com/ggml-org/llama.cpp/pull/27252
          ./build-xcframework.sh macos ios-device

      - name: Build Xcode project
        run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Pack artifacts
        id: pack_artifacts
        run: |
          # Zip file is required for Swift Package Manager, which does not support tar.gz for binary targets.
          # For more details, see https://developer.apple.com/documentation/xcode/distributing-binary-frameworks-as-swift-packages
          zip -r -y llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: llama-${{ steps.tag.outputs.name }}-xcframework.zip
          name: llama-${{ steps.tag.outputs.name }}-xcframework.zip

# TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23705)
#       in order to enable it again, we have to provision dedicated runners  to run it
#  openEuler-cann:
#    strategy:
#      matrix:
#        include:
#          # 910b with aclgraph (both architectures)
#          - arch: x86
#            chip_type: '910b'
#            build: 'Release'
#            use_acl_graph: 'on'
#          - arch: aarch64
#            chip_type: '910b'
#            build: 'Release'
#            use_acl_graph: 'on'
#          # 310p without aclgraph (both architectures)
#          - arch: x86
#            chip_type: '310p'
#            build: 'Release'
#            use_acl_graph: 'off'
#          - arch: aarch64
#            chip_type: '310p'
#            build: 'Release'
#            use_acl_graph: 'off'
#    runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
#    steps:
#      - name: Checkout
#        uses: actions/checkout@v6
#        with:
#          fetch-depth: 0
#
#      - name: Free up disk space
#        uses: ggml-org/free-disk-space@v1.3.1
#        with:
#          tool-cache: true
#
#      - name: Set container image
#        id: cann-image
#        run: |
#          image="ascendai/cann:${{ matrix.chip_type == '910b' &&  '8.5.0-910b-openeuler24.03-py3.11' || '8.5.0-310p-openeuler24.03-py3.11' }}"
#          echo "image=${image}" >> "${GITHUB_OUTPUT}"
#
#      - name: Pull container image
#        run: docker pull "${{ steps.cann-image.outputs.image }}"
#
#      - name: Build
#        env:
#          BUILD_TYPE: ${{ matrix.build }}
#          SOC_TYPE: ascend${{ matrix.chip_type }}
#          USE_ACL_GRAPH: ${{ matrix.use_acl_graph }}
#        run: |
#          HOST_UID=$(id -u)
#          HOST_GID=$(id -g)
#
#          docker run --rm \
#            -v "${PWD}:/workspace" \
#            -w /workspace \
#            -e SOC_TYPE=${SOC_TYPE} \
#            -e BUILD_TYPE=${BUILD_TYPE} \
#            -e USE_ACL_GRAPH=${USE_ACL_GRAPH} \
#            "${{ steps.cann-image.outputs.image }}" \
#            bash -lc '
#              set -e
#              yum install -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs git gcc gcc-c++ make cmake openssl-devel
#              yum clean all && rm -rf /var/cache/yum
#              git config --global --add safe.directory "/workspace"
#              export LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/lib64:${ASCEND_TOOLKIT_HOME}/$(uname -m)-linux/devlib/:${LD_LIBRARY_PATH}
#              cmake -S . -B build \
#                  -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
#                  -DGGML_CANN=on \
#                  -DSOC_TYPE=${SOC_TYPE} \
#                  -DUSE_ACL_GRAPH=${USE_ACL_GRAPH}
#              cmake --build build -j $(nproc)
#
#              chown -R '"${HOST_UID}"':'"${HOST_GID}"' /workspace/build
#            '
#
#      - name: Determine tag name
#        id: tag
#        uses: ./.github/actions/get-tag-name
#
#      - name: Pack artifacts
#        run: |
#          cp LICENSE ./build/bin/
#          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
#
#      - name: Upload artifacts
#        uses: actions/upload-artifact@v6
#        with:
#          path: llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz
#          name: llama-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz

  ui-build:
    needs: [check-release]
    if: ${{ needs.check-release.outputs.should_release == 'true' }}
    uses: ./.github/workflows/ui-build.yml

  release:
    if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}

    # Fine-grant permission
    # https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
    permissions:
        contents: write # for creating release
        id-token: write
        attestations: write

    runs-on: ubuntu-slim

    needs:
      - windows
      - windows-cpu
      - windows-cuda
      - windows-sycl
      - windows-rocm
      - windows-openvino
      - ubuntu-24-rocm
      - ubuntu-cpu
      - ubuntu-vulkan
      - ubuntu-cuda
      - ubuntu-24-openvino
      - ubuntu-24-sycl
      - android-arm64
      - android-arm64-snapdragon
      - linux-arm64-snapdragon
      - macos-cpu
      - ios-xcode
      #- openEuler-cann
      - ui-build

    outputs:
      tag_name: ${{ steps.tag.outputs.name }}

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}

      - name: Determine tag name
        id: tag
        uses: ./.github/actions/get-tag-name

      - name: Download artifacts
        id: download-artifact
        uses: actions/download-artifact@v7
        with:
          path: ./artifact
          merge-multiple: true

      - name: Merge artifacts
        id: move_artifacts
        run: |
          mkdir -p release

          # the windows-cpu zip contains the full toolset (llama-server with the embedded
          # UI, ggml-cpu) - inject it into the other windows zips so that every archive
          # ships the same binaries, only with a different backend library on top
          echo "Injecting windows-cpu binaries (llama-server + CPU backend) into the backend zips..."
          for arch in x64 arm64; do
            cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip"
            temp_dir=$(mktemp -d)
            echo "Extracting windows-cpu-${arch} package..."
            unzip "$cpu_zip" -d "$temp_dir"

            echo "Merging into $arch zips..."
            for target_zip in artifact/llama-bin-win-*-${arch}.zip; do
              if [[ "$target_zip" == "$cpu_zip" ]]; then
                continue
              fi
              echo "Injecting into $(basename "$target_zip")"
              realpath_target_zip=$(realpath "$target_zip")
              (cd "$temp_dir" && zip -r "$realpath_target_zip" .)
            done

            rm -rf "$temp_dir"
          done

          echo "Renaming and moving zips to release..."
          for zip_file in artifact/llama-bin-win-*.zip; do
            base_name=$(basename "$zip_file" .zip)
            zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip"
            echo "Moving $zip_file to release/$zip_name"
            mv "$zip_file" "release/$zip_name"
          done

          echo "Moving other artifacts..."
          mv -v artifact/*.zip release
          mv -v artifact/*.tar.gz release

      - name: Download UI build
        id: download_ui
        uses: actions/download-artifact@v7
        with:
          name: llama-ui.zip
          path: ./ui-dist

      - name: Package UI
        id: package_ui
        run: |
          tar -czvf release/llama-${{ steps.tag.outputs.name }}-ui.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./ui-dist .

      - name: Attest release artifacts
        id: attest
        uses: actions/attest@v4
        with:
          subject-path: 'release/*'

      - name: Create and push git tag
        run: |
          TAG="${{ steps.tag.outputs.name }}"
          if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null 2>&1; then
            echo "Tag ${TAG} already exists, skipping creation"
          else
            git tag "${TAG}"
            git push origin "${TAG}"
          fi

      - name: Create release
        id: create_release
        uses: ggml-org/action-create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ${{ steps.tag.outputs.name }}
          prerelease: true
          body: |
            <details open>

            ${{ github.event.head_commit.message }}

            </details>

            **Website:**
            - <https://llama.app>

            **Attestations:**
            - <${{ steps.attest.outputs.attestation-url }}>

            **macOS/iOS:**
            - [macOS Apple Silicon (arm64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.tar.gz)
            - macOS Apple Silicon (arm64, KleidiAI enabled) [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23780)
            - [macOS Intel (x64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-x64.tar.gz)
            - [iOS XCFramework](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-xcframework.zip)

            **Linux:**
            - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
            - [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
            - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
            - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
            - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
            - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
            - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz)
            - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz)
            - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
            - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
            - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
            - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)
            - [Linux arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/linux.md)

            **Android:**
            - [Android arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz)
            - [Android arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/README.md)

            **Windows:**
            - [Windows x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-x64.zip)
            - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
            - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
            - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
            - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
            - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
            - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
            - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)
            - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)
            - [Windows x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-10.0-x64.zip)

            **openEuler:**
            - [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23705)
            - openEuler x86 (310p)
            - openEuler x86 (910b, ACL Graph)
            - openEuler aarch64 (310p)
            - openEuler aarch64 (910b, ACL Graph)

            **UI:**
            - [UI](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-ui.tar.gz)

      - name: Upload release
        id: upload_release
        uses: actions/github-script@v8
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const path = require('path');
            const fs = require('fs');
            const release_id = '${{ steps.create_release.outputs.id }}';
            for (let file of await fs.readdirSync('./release')) {
              if (path.extname(file) === '.zip' || file.endsWith('.tar.gz')) {
                console.log('uploadReleaseAsset', file);
                await github.rest.repos.uploadReleaseAsset({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  release_id: release_id,
                  name: file,
                  data: await fs.readFileSync(`./release/${file}`)
                });
              }
            }

  ui-publish:
    if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}

    needs:
      - release

    uses: ./.github/workflows/ui-publish.yml
    with:
      version_tag: ${{ needs.release.outputs.tag_name }}
    secrets:
      hf_token: ${{ secrets.HF_TOKEN_UI_STATIC_OUTPUT }}