From 205a0426905d45f5ed0d120f3c671e3573e01b41 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Tue, 1 Sep 2026 12:59:48 -0700 Subject: [PATCH] build: go deps (#18161) * build: go deps Gather go dep licenses * address comments --- .github/workflows/release.yaml | 12 +++ .github/workflows/test-llamacpp-update.yaml | 15 +++- .github/workflows/test.yaml | 20 +++++ Dockerfile | 8 ++ cmake/generate_go_license.cmake | 98 +++++++++++++++++++++ cmake/local.cmake | 36 ++++++++ go.mod | 2 +- go.sum | 4 +- scripts/build_darwin.sh | 1 + 9 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 cmake/generate_go_license.cmake diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d2f046b92..22e099cb2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -77,6 +77,8 @@ jobs: MLX_VERSION MLX_C_VERSION - run: | + cmake -S . -B build/go-license -DOLLAMA_LLAMA_BACKENDS= -DOLLAMA_MLX_BACKENDS= -DOLLAMA_PAYLOAD_INSTALL_PREFIX=dist/darwin-arm64 "-DOLLAMA_GO_LICENSE_TARGETS=darwin/amd64;darwin/arm64" + cmake --build build/go-license --target ollama-go-license ./scripts/build_darwin.sh - name: Log build results run: | @@ -446,6 +448,16 @@ jobs: do [ -f "$payload" ] || { echo "missing $payload"; exit 1; } done + - name: Collect Go licenses + shell: bash + run: | + set -euo pipefail + for arch in amd64 arm64; do + cmake -S . -B build/go-license -DOLLAMA_LLAMA_BACKENDS= -DOLLAMA_MLX_BACKENDS= \ + "-DOLLAMA_PAYLOAD_INSTALL_PREFIX=dist/windows-${arch}" \ + "-DOLLAMA_GO_LICENSE_TARGETS=windows/${arch}" + cmake --build build/go-license --target ollama-go-license + done - run: | ./scripts/build_windows.ps1 deps sign installer zip - name: Log contents after build diff --git a/.github/workflows/test-llamacpp-update.yaml b/.github/workflows/test-llamacpp-update.yaml index 0ae4f32b8..6b6f73e05 100644 --- a/.github/workflows/test-llamacpp-update.yaml +++ b/.github/workflows/test-llamacpp-update.yaml @@ -57,7 +57,10 @@ jobs: MLX_VERSION MLX_C_VERSION - name: Build unsigned Darwin runtime - run: ./scripts/build_darwin.sh build package + run: | + cmake -S . -B build/go-license -DOLLAMA_LLAMA_BACKENDS= -DOLLAMA_MLX_BACKENDS= -DOLLAMA_PAYLOAD_INSTALL_PREFIX=dist/darwin-arm64 "-DOLLAMA_GO_LICENSE_TARGETS=darwin/amd64;darwin/arm64" + cmake --build build/go-license --target ollama-go-license + ./scripts/build_darwin.sh build package - name: Log build results run: ls -l dist/ - uses: actions/upload-artifact@v4 @@ -578,6 +581,16 @@ jobs: do [ -f "$payload" ] || { echo "missing $payload"; exit 1; } done + - name: Collect Go licenses + shell: bash + run: | + set -euo pipefail + for arch in amd64 arm64; do + cmake -S . -B build/go-license -DOLLAMA_LLAMA_BACKENDS= -DOLLAMA_MLX_BACKENDS= \ + "-DOLLAMA_PAYLOAD_INSTALL_PREFIX=dist/windows-${arch}" \ + "-DOLLAMA_GO_LICENSE_TARGETS=windows/${arch}" + cmake --build build/go-license --target ollama-go-license + done - name: Build unsigned Windows installer and zips run: ./scripts/build_windows.ps1 deps installer zip - name: Log contents after build diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 50620f491..430a20e0e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,6 +23,7 @@ jobs: outputs: changed: ${{ steps.changes.outputs.changed }} app_changed: ${{ steps.changes.outputs.app_changed }} + go_mod_changed: ${{ steps.changes.outputs.go_mod_changed }} enginehash: ${{ steps.changes.outputs.enginehash }} steps: - uses: actions/checkout@v4 @@ -56,6 +57,7 @@ jobs: 'x/mlxrunner/xgrammar/native/**/*' \ '.github/**/*') | tee -a $GITHUB_OUTPUT echo app_changed=$(changed 'app/**' 'app/**/*') | tee -a $GITHUB_OUTPUT + echo go_mod_changed=$(changed 'go.mod') | tee -a $GITHUB_OUTPUT echo enginehash=$(cat LLAMA_CPP_VERSION)-$(cat MLX_VERSION)-$(cat MLX_C_VERSION) | tee -a $GITHUB_OUTPUT patches: @@ -368,6 +370,24 @@ jobs: - name: check that 'go mod tidy' is clean run: go mod tidy --diff || (echo "Please run 'go mod tidy'." && exit 1) + go_license: + needs: [changes] + if: needs.changes.outputs.go_mod_changed == 'True' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Verify Go dependency licenses + run: | + # See cmake/generate_go_license.cmake for special case handling. + cmake -S . -B build/go-license \ + -DOLLAMA_LLAMA_BACKENDS= \ + -DOLLAMA_MLX_BACKENDS= \ + "-DOLLAMA_GO_LICENSE_TARGETS=linux/amd64;linux/arm64;darwin/amd64;darwin/arm64;windows/amd64;windows/arm64" + cmake --build build/go-license --target ollama-go-license + test: needs: [changes] strategy: diff --git a/Dockerfile b/Dockerfile index 862f656a7..1771dc197 100644 --- a/Dockerfile +++ b/Dockerfile @@ -262,9 +262,15 @@ ENV CGO_CFLAGS="${CGO_CFLAGS}" ENV CGO_CXXFLAGS="${CGO_CXXFLAGS}" RUN --mount=type=cache,target=/root/.cache/go-build \ go build -trimpath -buildmode=pie -o /bin/ollama . +RUN --mount=type=cache,target=/root/.cache/go-build \ + cmake -S . -B build/go-license \ + -DOLLAMA_LLAMA_BACKENDS= \ + -DOLLAMA_MLX_BACKENDS= \ + && cmake --build build/go-license --target ollama-go-license FROM scratch AS publish-go COPY --from=build /bin/ollama /bin/ollama +COPY --from=build /go/src/github.com/ollama/ollama/build/go-license/lib/ollama/GO_LICENSE /lib/ollama/GO_LICENSE # # Assembly stages — combine llama-server variants + GPU runtime libs @@ -297,9 +303,11 @@ COPY --from=arm64 /lib/ollama /lib/ollama/ FROM ${TARGETARCH}-archive AS archive COPY --from=build /bin/ollama /bin/ollama +COPY --from=build /go/src/github.com/ollama/ollama/build/go-license/lib/ollama/GO_LICENSE /lib/ollama/GO_LICENSE FROM ${FLAVOR} AS image-archive COPY --from=build /bin/ollama /bin/ollama +COPY --from=build /go/src/github.com/ollama/ollama/build/go-license/lib/ollama/GO_LICENSE /lib/ollama/GO_LICENSE FROM ubuntu:24.04 ARG APT_MIRROR=http://archive.ubuntu.com/ubuntu diff --git a/cmake/generate_go_license.cmake b/cmake/generate_go_license.cmake new file mode 100644 index 000000000..b352f3867 --- /dev/null +++ b/cmake/generate_go_license.cmake @@ -0,0 +1,98 @@ +foreach(_variable IN ITEMS GO_EXECUTABLE SOURCE_DIR BINARY_DIR OUTPUT_DIR TARGETS) + if(NOT DEFINED ${_variable}) + message(FATAL_ERROR "${_variable} is required") + endif() +endforeach() +if(NOT TARGETS) + message(FATAL_ERROR "At least one GOOS/GOARCH target is required") +endif() + +execute_process( + COMMAND "${GO_EXECUTABLE}" tool dist list + OUTPUT_VARIABLE _supported_targets + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND_ERROR_IS_FATAL ANY) +string(REPLACE "\r" "" _supported_targets "${_supported_targets}") +string(REPLACE "\n" ";" _supported_targets "${_supported_targets}") + +set(_version v2.0.1) +set(_tool_dir "${BINARY_DIR}/go-licenses-${_version}") +set(_tool "${_tool_dir}/go-licenses") +if(CMAKE_HOST_WIN32) + string(APPEND _tool ".exe") +endif() + +if(NOT EXISTS "${_tool}") + file(MAKE_DIRECTORY "${_tool_dir}") + execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + --unset=GOOS --unset=GOARCH --unset=GOARM --unset=GOAMD64 + "GOBIN=${_tool_dir}" "GOFLAGS=" + "${GO_EXECUTABLE}" install "github.com/google/go-licenses/v2@${_version}" + WORKING_DIRECTORY "${SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY) +endif() + +set(_staging_dir "${BINARY_DIR}/go-license-files") +file(REMOVE_RECURSE "${_staging_dir}") + +foreach(_target IN LISTS TARGETS) + list(FIND _supported_targets "${_target}" _target_index) + if(_target_index EQUAL -1) + message(FATAL_ERROR "Unsupported Go license target '${_target}'; expected a GOOS/GOARCH from 'go tool dist list'") + endif() + string(REPLACE "/" ";" _target_parts "${_target}") + list(GET _target_parts 0 _goos) + list(GET _target_parts 1 _goarch) + + set(_packages .) + if(_goos STREQUAL "darwin" OR _goos STREQUAL "windows") + list(APPEND _packages ./app/cmd/app) + endif() + + set(_target_staging_dir "${BINARY_DIR}/go-license-files-${_goos}-${_goarch}") + message(STATUS "Collecting Go licenses for ${_target}") + execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + "GOOS=${_goos}" "GOARCH=${_goarch}" "CGO_ENABLED=1" + "${_tool}" save ${_packages} + --save_path "${_target_staging_dir}" --force + --ignore github.com/apache/arrow/go/arrow + WORKING_DIRECTORY "${SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY) + + file(COPY "${_target_staging_dir}/" DESTINATION "${_staging_dir}") +endforeach() + +# Arrow's aggregate license includes a license that go-licenses cannot classify. +execute_process( + COMMAND "${GO_EXECUTABLE}" list -m -f "{{.Dir}}" github.com/apache/arrow/go/arrow + WORKING_DIRECTORY "${SOURCE_DIR}" + OUTPUT_VARIABLE _arrow_dir + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND_ERROR_IS_FATAL ANY) +if(NOT EXISTS "${_arrow_dir}/LICENSE.txt") + message(FATAL_ERROR "failed to locate the Apache Arrow license") +endif() + +set(_arrow_output_dir "${_staging_dir}/github.com/apache/arrow/go/arrow") +file(MAKE_DIRECTORY "${_arrow_output_dir}") +file(COPY "${_arrow_dir}/LICENSE.txt" DESTINATION "${_arrow_output_dir}") + +file(GLOB_RECURSE _license_files + LIST_DIRECTORIES FALSE + RELATIVE "${_staging_dir}" + "${_staging_dir}/*") +list(SORT _license_files) + +file(MAKE_DIRECTORY "${OUTPUT_DIR}") +set(_output "${OUTPUT_DIR}/GO_LICENSE") +file(WRITE "${_output}" "Go licenses for Ollama and its dependencies.\n") +foreach(_license_file IN LISTS _license_files) + file(READ "${_staging_dir}/${_license_file}" _license_text) + file(APPEND "${_output}" + "\n================================================================================\n" + "${_license_file}\n" + "================================================================================\n" + "${_license_text}\n") +endforeach() diff --git a/cmake/local.cmake b/cmake/local.cmake index e136d9e0b..88be7fff3 100644 --- a/cmake/local.cmake +++ b/cmake/local.cmake @@ -568,6 +568,42 @@ endfunction() find_program(GO_EXECUTABLE go) +if(GO_EXECUTABLE) + if(NOT DEFINED OLLAMA_GO_LICENSE_TARGETS) + execute_process( + COMMAND "${GO_EXECUTABLE}" env GOOS + OUTPUT_VARIABLE _go_license_goos + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND_ERROR_IS_FATAL ANY) + execute_process( + COMMAND "${GO_EXECUTABLE}" env GOARCH + OUTPUT_VARIABLE _go_license_goarch + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND_ERROR_IS_FATAL ANY) + set(OLLAMA_GO_LICENSE_TARGETS "${_go_license_goos}/${_go_license_goarch}" CACHE STRING + "Semicolon-separated GOOS/GOARCH targets included in GO_LICENSE") + endif() + + add_custom_target(ollama-go-license + COMMAND ${CMAKE_COMMAND} + "-DGO_EXECUTABLE=${GO_EXECUTABLE}" + "-DSOURCE_DIR=${CMAKE_SOURCE_DIR}" + "-DBINARY_DIR=${CMAKE_BINARY_DIR}" + "-DOUTPUT_DIR=${OLLAMA_PAYLOAD_INSTALL_PREFIX}/${OLLAMA_LIB_DIR}" + "-DTARGETS=${OLLAMA_GO_LICENSE_TARGETS}" + -P "${CMAKE_SOURCE_DIR}/cmake/generate_go_license.cmake" + BYPRODUCTS "${OLLAMA_PAYLOAD_INSTALL_PREFIX}/${OLLAMA_LIB_DIR}/GO_LICENSE" + COMMENT "Collecting Go licenses" + VERBATIM) +else() + add_custom_target(ollama-go-license + COMMAND ${CMAKE_COMMAND} -E echo + "Go executable not found. Install Go or set GO_EXECUTABLE to collect Go licenses." + COMMAND ${CMAKE_COMMAND} -E false + COMMENT "Collecting Go licenses" + VERBATIM) +endif() + if(OLLAMA_MLX_BACKENDS) if(GO_EXECUTABLE AND (NOT APPLE OR CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR)) add_custom_target(ollama-mlx-generate-wrappers diff --git a/go.mod b/go.mod index c1525d23c..b26cd0ed9 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-localereader v0.0.2-0.20220822084737-6bae6c923850 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect diff --git a/go.sum b/go.sum index 5c57a64b2..8e9950992 100644 --- a/go.sum +++ b/go.sum @@ -170,8 +170,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= -github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-localereader v0.0.2-0.20220822084737-6bae6c923850 h1:3wTbcd6IAtyC45hpBhisgmnAl/+twARH7FyVSh5NbcY= +github.com/mattn/go-localereader v0.0.2-0.20220822084737-6bae6c923850/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= diff --git a/scripts/build_darwin.sh b/scripts/build_darwin.sh index 82164f1c5..62c2e2ad2 100755 --- a/scripts/build_darwin.sh +++ b/scripts/build_darwin.sh @@ -171,6 +171,7 @@ _sign_darwin() { if [ -n "$APPLE_IDENTITY" ]; then for F in dist/darwin/ollama dist/darwin/llama-server dist/darwin/llama-quantize dist/darwin/lib/ollama/* dist/darwin/lib/ollama/mlx_metal_v*/*; do [ -f "$F" ] && [ ! -L "$F" ] || continue + case "$F" in *_LICENSE|*_NOTICE) continue ;; esac codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier ai.ollama.ollama --options=runtime "$F" done