diff --git a/convert/convert_qwen3next.go b/convert/convert_qwen3next.go index e4b653ba5..eb00187aa 100644 --- a/convert/convert_qwen3next.go +++ b/convert/convert_qwen3next.go @@ -934,8 +934,11 @@ func safetensorFloat32Data(st safetensor) ([]float32, error) { func (q *qwen3NextModel) Tensors(ts []Tensor) []*ggml.Tensor { var out []*ggml.Tensor - merges := make([]merge, q.NumHiddenLayers*3) - for i := range q.NumHiddenLayers { + ts = q.renameMTPLayerTensors(ts) + + blockCount := q.NumHiddenLayers + q.NumNextNPredictLayers + merges := make([]merge, blockCount*3) + for i := range blockCount { merges[i*3+0] = merge{ fmt.Sprintf("blk.%d.mlp.experts.*.gate_proj.weight", i), fmt.Sprintf("blk.%d.ffn_gate_exps.weight", i), @@ -1065,6 +1068,57 @@ func (q *qwen3NextModel) Tensors(ts []Tensor) []*ggml.Tensor { return out } +func (q *qwen3NextModel) renameMTPLayerTensors(ts []Tensor) []Tensor { + var out []Tensor + for i, t := range ts { + name, ok := q.mtpLayerTensorName(t.Name()) + if !ok { + continue + } + if out == nil { + out = slices.Clone(ts) + } + out[i] = &renamedTensor{Tensor: t, name: name} + } + if out != nil { + return out + } + return ts +} + +func (q *qwen3NextModel) mtpLayerTensorName(name string) (string, bool) { + rest := strings.TrimPrefix(name, "mtp.layers.") + if rest == name { + return "", false + } + layer, suffix, ok := strings.Cut(rest, ".") + if !ok { + return "", false + } + idx, err := strconv.ParseUint(layer, 10, 32) + if err != nil { + return "", false + } + return fmt.Sprintf("blk.%d.%s", q.NumHiddenLayers+uint32(idx), suffix), true +} + +type renamedTensor struct { + Tensor + name string +} + +func (t *renamedTensor) Name() string { + return t.name +} + +func (t *renamedTensor) Clone() Tensor { + return &renamedTensor{Tensor: t.Tensor.Clone(), name: t.name} +} + +func (t *renamedTensor) SourceDType() string { + return sourceDType(t.Tensor) +} + func (q *qwen3NextModel) appendDirectTensor(out []*ggml.Tensor, t Tensor, name string) []*ggml.Tensor { if qwen3NextShouldShiftNorm(name) { t = t.Clone() @@ -1093,18 +1147,6 @@ func (q *qwen3NextModel) mtpTensorNames(name string) []string { nextn = 1 } - if rest := strings.TrimPrefix(name, "mtp.layers."); rest != name { - layer, suffix, ok := strings.Cut(rest, ".") - if !ok { - return nil - } - idx, err := strconv.ParseUint(layer, 10, 32) - if err != nil { - return nil - } - return []string{fmt.Sprintf("blk.%d.%s", base+uint32(idx), suffix)} - } - var suffix string switch name { case "mtp.fc.weight": diff --git a/convert/convert_qwen3next_test.go b/convert/convert_qwen3next_test.go index 4b286eaa0..233fd6288 100644 --- a/convert/convert_qwen3next_test.go +++ b/convert/convert_qwen3next_test.go @@ -819,6 +819,101 @@ func TestQwen35MoePackedExperts(t *testing.T) { } } +func TestQwen35MTPMoePackedExperts(t *testing.T) { + m := &qwen3NextModel{ + qwen3NextTextConfig: qwen3NextTextConfig{ + NumHiddenLayers: 40, + NumNextNPredictLayers: 1, + }, + } + + out := m.Tensors([]Tensor{ + &fakeTensor{ + name: "mtp.layers.0.mlp.experts.gate_up_proj", + shape: []uint64{2, 4, 3}, + data: []float32{ + 0, 1, 2, + 3, 4, 5, + 6, 7, 8, + 9, 10, 11, + 12, 13, 14, + 15, 16, 17, + 18, 19, 20, + 21, 22, 23, + }, + }, + &fakeTensor{ + name: "mtp.layers.0.mlp.experts.down_proj", + shape: []uint64{2, 5, 3}, + data: make([]float32, 2*5*3), + }, + }) + + byName := map[string]*ggml.Tensor{} + for _, tensor := range out { + if strings.Contains(tensor.Name, ".mlp.experts.") { + t.Fatalf("unexpected raw expert tensor %q", tensor.Name) + } + byName[tensor.Name] = tensor + } + + gate := byName["blk.40.ffn_gate_exps.weight"] + if gate == nil { + t.Fatalf("missing tensor %q", "blk.40.ffn_gate_exps.weight") + } + if got, want := gate.Shape, []uint64{2, 2, 3}; !slices.Equal(got, want) { + t.Fatalf("unexpected gate shape: got %v want %v", got, want) + } + if got, want := readTensorData(t, gate), []float32{ + 0, 1, 2, 3, 4, 5, + 12, 13, 14, 15, 16, 17, + }; !slices.Equal(got, want) { + t.Fatalf("unexpected gate values: got %v want %v", got, want) + } + + if _, ok := byName["blk.40.ffn_up_exps.weight"]; !ok { + t.Fatalf("missing tensor %q", "blk.40.ffn_up_exps.weight") + } + if _, ok := byName["blk.40.ffn_down_exps.weight"]; !ok { + t.Fatalf("missing tensor %q", "blk.40.ffn_down_exps.weight") + } +} + +func TestQwen35MTPMoePerExpertTensors(t *testing.T) { + m := &qwen3NextModel{ + qwen3NextTextConfig: qwen3NextTextConfig{ + NumHiddenLayers: 40, + NumNextNPredictLayers: 1, + }, + } + + out := m.Tensors([]Tensor{ + &fakeTensor{ + name: "mtp.layers.0.mlp.experts.1.gate_proj.weight", + shape: []uint64{2, 2}, + data: []float32{10, 11, 12, 13}, + }, + &fakeTensor{ + name: "mtp.layers.0.mlp.experts.0.gate_proj.weight", + shape: []uint64{2, 2}, + data: []float32{0, 1, 2, 3}, + }, + }) + + if len(out) != 1 { + t.Fatalf("unexpected output tensor count: got %d want 1", len(out)) + } + if got, want := out[0].Name, "blk.40.ffn_gate_exps.weight"; got != want { + t.Fatalf("unexpected tensor name: got %q want %q", got, want) + } + if got, want := out[0].Shape, []uint64{2, 2, 2}; !slices.Equal(got, want) { + t.Fatalf("unexpected tensor shape: got %v want %v", got, want) + } + if got, want := readTensorData(t, out[0]), []float32{0, 1, 2, 3, 10, 11, 12, 13}; !slices.Equal(got, want) { + t.Fatalf("unexpected tensor values: got %v want %v", got, want) + } +} + func TestQwen35SharedExpertGateKeepsMatrixShape(t *testing.T) { m := &qwen3NextModel{} diff --git a/server/quantization.go b/server/quantization.go index 78b467330..0044aff47 100644 --- a/server/quantization.go +++ b/server/quantization.go @@ -284,6 +284,15 @@ func llamaQuantizeArgs(arch string, newFileType fsggml.FileType, input, output, if typeName == "COPY" { return append(args, input, output, typeName) } + // Qwen3.5 MTP uses this projection to combine hidden and embedding states + // for the draft layer. Keep it at least Q8 when quantizing lower than Q8, + // while preserving unquantized outputs such as F16/BF16. + if arch == "qwen35" || arch == "qwen35moe" { + switch newFileType { + case fsggml.FileTypeQ4_K_S, fsggml.FileTypeQ4_K_M: + args = append(args, "--tensor-type", `^blk\.[0-9]+\.nextn\.eh_proj\.weight$=q8_0`) + } + } // gemma3n's per_layer_token_embd is read on every layer for every token // (not just once at input like token_embd), so it's far more quality-sensitive // than a normal token embedding. Keep it at F16 on K-quants via an anchored diff --git a/server/quantization_test.go b/server/quantization_test.go index 2925df824..62cccadc0 100644 --- a/server/quantization_test.go +++ b/server/quantization_test.go @@ -25,6 +25,44 @@ func TestLlamaQuantizeArgs(t *testing.T) { fileType: fsggml.FileTypeQ4_K_M, want: []string{"--allow-requantize", "in.gguf", "out.gguf", "Q4_K_M"}, }, + { + name: "qwen35moe k quant keeps mtp projection q8", + arch: "qwen35moe", + fileType: fsggml.FileTypeQ4_K_M, + want: []string{ + "--allow-requantize", + "--tensor-type", `^blk\.[0-9]+\.nextn\.eh_proj\.weight$=q8_0`, + "in.gguf", "out.gguf", "Q4_K_M", + }, + }, + { + name: "qwen35 k quant keeps mtp projection q8", + arch: "qwen35", + fileType: fsggml.FileTypeQ4_K_S, + want: []string{ + "--allow-requantize", + "--tensor-type", `^blk\.[0-9]+\.nextn\.eh_proj\.weight$=q8_0`, + "in.gguf", "out.gguf", "Q4_K_S", + }, + }, + { + name: "qwen35moe f16 keeps mtp projection unquantized", + arch: "qwen35moe", + fileType: fsggml.FileTypeF16, + want: []string{"--allow-requantize", "in.gguf", "out.gguf", "F16"}, + }, + { + name: "qwen35moe bf16 keeps mtp projection unquantized", + arch: "qwen35moe", + fileType: fsggml.FileTypeBF16, + want: []string{"--allow-requantize", "in.gguf", "out.gguf", "BF16"}, + }, + { + name: "qwen35moe q8 already satisfies mtp projection floor", + arch: "qwen35moe", + fileType: fsggml.FileTypeQ8_0, + want: []string{"--allow-requantize", "in.gguf", "out.gguf", "Q8_0"}, + }, { name: "gemma3n k quant keeps per layer token embedding f16", arch: "gemma3n", @@ -84,6 +122,13 @@ func TestLlamaQuantizeArgs(t *testing.T) { typeName: "COPY", want: []string{"--allow-requantize", "in.gguf", "out.gguf", "COPY"}, }, + { + name: "qwen35moe copy does not add mtp projection override", + arch: "qwen35moe", + fileType: fsggml.FileTypeQ4_K_M, + typeName: "COPY", + want: []string{"--allow-requantize", "in.gguf", "out.gguf", "COPY"}, + }, } for _, tt := range tests {