ollama/server/create_split_gguf.go

package server

import (
	"fmt"
	"path"
	"regexp"
	"strconv"

	"github.com/ollama/ollama/fs/gguf"
	"github.com/ollama/ollama/manifest"
)

const (
	maxSplitGGUFParts         = maxCreateFiles
	splitGGUFMinOllamaVersion = "0.35.0"
)

var splitGGUFNameRe = regexp.MustCompile(`^(.*)-(\d{5})-of-(\d{5})\.gguf$`)

type splitGGUFSlot struct {
	layer    *modelLayer
	splitKey splitGGUFKey
}

type splitGGUFKey struct {
	mediaType string
	dir       string
	prefix    string
	count     uint16
}

type splitGGUFCollector struct {
	slots  []splitGGUFSlot
	groups map[splitGGUFKey][]*modelLayer
}

func newSplitGGUFCollector() *splitGGUFCollector {
	return &splitGGUFCollector{
		groups: make(map[splitGGUFKey][]*modelLayer),
	}
}

func (c *splitGGUFCollector) Add(layer *modelLayer) error {
	key, ok, err := splitGGUFGroupKey(layer)
	if err != nil {
		return invalidSplitGGUF(err)
	}
	if !ok {
		c.slots = append(c.slots, splitGGUFSlot{layer: layer})
		return nil
	}

	if _, ok := c.groups[key]; !ok {
		c.slots = append(c.slots, splitGGUFSlot{splitKey: key})
	}
	c.groups[key] = append(c.groups[key], layer)
	return nil
}

func (c *splitGGUFCollector) Layers() ([]*modelLayer, error) {
	layers := make([]*modelLayer, 0, len(c.slots))
	for _, slot := range c.slots {
		if slot.layer != nil {
			layers = append(layers, slot.layer)
			continue
		}

		layer, err := groupSplitGGUFLayers(c.groups[slot.splitKey])
		if err != nil {
			return nil, invalidSplitGGUF(err)
		}
		layers = append(layers, layer)
	}

	return layers, nil
}

func invalidSplitGGUF(err error) error {
	if err == nil {
		return nil
	}
	return fmt.Errorf("%w: %w", errInvalidSplitGGUF, err)
}

func splitGGUFName(name string) (prefix string, index, count uint16, ok bool) {
	matches := splitGGUFNameRe.FindStringSubmatch(path.Base(name))
	if len(matches) != 4 {
		return "", 0, 0, false
	}

	idx, err := strconv.ParseUint(matches[2], 10, 16)
	if err != nil || idx == 0 {
		return "", 0, 0, false
	}
	n, err := strconv.ParseUint(matches[3], 10, 16)
	if err != nil || n == 0 {
		return "", 0, 0, false
	}
	return matches[1], uint16(idx - 1), uint16(n), true
}

func splitGGUFGroupKey(layer *modelLayer) (splitGGUFKey, bool, error) {
	if layer == nil || layer.GGUF == nil {
		return splitGGUFKey{}, false, nil
	}
	count, ok := layer.GGUF.UintOK("split.count")
	if !ok || count <= 1 {
		return splitGGUFKey{}, false, nil
	}
	if count > maxSplitGGUFParts {
		return splitGGUFKey{}, false, fmt.Errorf("split GGUF %q has too many shards: %d", layer.From, count)
	}

	filename := layer.splitFile
	if filename == "" {
		filename = layer.From
	}
	prefix, index, nameCount, ok := splitGGUFName(filename)
	if !ok {
		return splitGGUFKey{}, false, fmt.Errorf("split GGUF %q must use llama.cpp split filename pattern", filename)
	}
	if uint64(nameCount) != count {
		return splitGGUFKey{}, false, fmt.Errorf("split GGUF %q filename count %d does not match metadata count %d", filename, nameCount, count)
	}
	if uint64(index) >= count {
		return splitGGUFKey{}, false, fmt.Errorf("split GGUF %q filename index %d exceeds metadata count %d", filename, index, count)
	}
	splitNo, ok := layer.GGUF.UintOK("split.no")
	if !ok {
		return splitGGUFKey{}, false, fmt.Errorf("split GGUF %q is missing split.no metadata", filename)
	}
	if splitNo != uint64(index) {
		return splitGGUFKey{}, false, fmt.Errorf("split GGUF %q filename index %d does not match metadata index %d", filename, index, splitNo)
	}

	return splitGGUFKey{
		mediaType: layer.MediaType,
		dir:       path.Dir(filename),
		prefix:    prefix,
		count:     nameCount,
	}, true, nil
}

func groupManifestSplitGGUFLayers(layers []*modelLayer) ([]*modelLayer, error) {
	grouped := make([]*modelLayer, 0, len(layers))
	for i := 0; i < len(layers); {
		layer := layers[i]
		if layer == nil || layer.GGUF == nil {
			grouped = append(grouped, layer)
			i++
			continue
		}

		count, ok := layer.GGUF.UintOK("split.count")
		if !ok || count <= 1 {
			grouped = append(grouped, layer)
			i++
			continue
		}
		if count > uint64(len(layers)-i) {
			return nil, invalidSplitGGUF(fmt.Errorf("split GGUF has %d remaining shards, expected %d", len(layers)-i, count))
		}
		logicalLayer, err := groupSplitGGUFLayers(layers[i : i+int(count)])
		if err != nil {
			return nil, invalidSplitGGUF(err)
		}
		grouped = append(grouped, logicalLayer)
		i += int(count)
	}
	return grouped, nil
}

func groupSplitGGUFLayers(layers []*modelLayer) (*modelLayer, error) {
	if len(layers) == 0 {
		return nil, fmt.Errorf("split GGUF has no shards")
	}
	if layers[0] == nil || layers[0].GGUF == nil {
		return nil, fmt.Errorf("split GGUF has an invalid shard")
	}

	countValue, ok := layers[0].GGUF.UintOK("split.count")
	if !ok || countValue <= 1 {
		return nil, fmt.Errorf("split GGUF has invalid shard count")
	}
	if countValue > maxSplitGGUFParts {
		return nil, fmt.Errorf("split GGUF %q has too many shards: %d", layers[0].From, countValue)
	}
	count := uint16(countValue)
	if len(layers) != int(count) {
		return nil, fmt.Errorf("split GGUF %q has %d shards, expected %d", layers[0].From, len(layers), count)
	}

	parts := make([]manifest.Layer, count)
	seen := make([]bool, count)
	architecture := layers[0].GGUF.Architecture()
	fileType := layers[0].GGUF.FileType()
	mediaType := layers[0].MediaType
	if mediaType != "application/vnd.ollama.image.model" && mediaType != manifest.MediaTypeImageDraft {
		return nil, fmt.Errorf("split GGUF %q has unsupported media type %q", layers[0].From, mediaType)
	}
	var expectedTensors uint64
	var hasExpectedTensors bool
	var actualTensors, parameterCount uint64
	var primary *modelLayer
	tensorNames := make(map[string]struct{})

	for _, layer := range layers {
		if layer == nil || layer.GGUF == nil {
			return nil, fmt.Errorf("split GGUF has an invalid shard")
		}
		if layer.MediaType != mediaType {
			return nil, fmt.Errorf("split GGUF %q media type does not match %q", layer.From, layers[0].From)
		}
		layerCount, ok := layer.GGUF.UintOK("split.count")
		if !ok || layerCount != uint64(count) {
			return nil, fmt.Errorf("split GGUF %q shard count does not match %q", layer.From, layers[0].From)
		}
		indexValue, ok := layer.GGUF.UintOK("split.no")
		if !ok || indexValue >= uint64(count) {
			return nil, fmt.Errorf("split GGUF %q has invalid shard index", layer.From)
		}
		layerArchitecture := layer.GGUF.Architecture()
		if architecture != "unknown" && layerArchitecture != "unknown" && layerArchitecture != architecture {
			return nil, fmt.Errorf("split GGUF %q architecture does not match %q", layer.From, layers[0].From)
		}
		layerFileType := layer.GGUF.FileType()
		if fileType != gguf.FileTypeUnknown && layerFileType != gguf.FileTypeUnknown && layerFileType != fileType {
			return nil, fmt.Errorf("split GGUF %q file type does not match %q", layer.From, layers[0].From)
		}

		index := uint16(indexValue)
		if layer.splitFile != "" {
			_, filenameIndex, filenameCount, ok := splitGGUFName(layer.splitFile)
			if !ok {
				return nil, fmt.Errorf("split GGUF %q must use llama.cpp split filename pattern", layer.splitFile)
			}
			if filenameCount != count {
				return nil, fmt.Errorf("split GGUF %q filename count %d does not match expected count %d", layer.splitFile, filenameCount, count)
			}
			if filenameIndex != index {
				return nil, fmt.Errorf("split GGUF %q filename index %d does not match metadata index %d", layer.splitFile, filenameIndex, index)
			}
		}
		if seen[index] {
			return nil, fmt.Errorf("split GGUF %q duplicate shard index %d", layer.From, index)
		}
		seen[index] = true
		if index == 0 {
			primary = layer
		}
		if layerTensorCount, ok := layer.GGUF.UintOK("split.tensors.count"); ok {
			if !hasExpectedTensors {
				expectedTensors = layerTensorCount
				hasExpectedTensors = true
			} else if expectedTensors != layerTensorCount {
				return nil, fmt.Errorf("split GGUF %q tensor count does not match %q", layer.From, layers[0].From)
			}
		}
		numTensors := uint64(layer.GGUF.NumTensors())
		if numTensors > ^uint64(0)-actualTensors {
			return nil, fmt.Errorf("split GGUF %q tensor count overflows", layer.From)
		}
		actualTensors += numTensors
		for _, tensor := range layer.GGUF.TensorInfos() {
			if _, ok := tensorNames[tensor.Name]; ok {
				return nil, fmt.Errorf("split GGUF %q contains duplicate tensor %q", layer.From, tensor.Name)
			}
			tensorNames[tensor.Name] = struct{}{}
		}
		if layer.parameterCount > ^uint64(0)-parameterCount {
			return nil, fmt.Errorf("split GGUF %q parameter count overflows", layer.From)
		}
		parameterCount += layer.parameterCount
		parts[index] = layer.Layer
	}

	if primary == nil {
		return nil, fmt.Errorf("split GGUF %q is missing first shard", layers[0].From)
	}
	if hasExpectedTensors && expectedTensors != actualTensors {
		return nil, fmt.Errorf("split GGUF %q has %d tensors, expected %d", primary.From, actualTensors, expectedTensors)
	}
	primary.parameterCount = parameterCount
	primary.splitLayers = parts
	return primary, nil
}