refactor: require external Gemma 2 and GPT-OSS tokenizers (#1974)

This commit is contained in:
leejet
2026-09-15 01:25:13 +08:00
committed by GitHub
parent 4964abdfc5
commit f9ddc0f388
19 changed files with 73 additions and 337 deletions
+1 -1
View File
@@ -412,7 +412,7 @@ ArgOptions SDContextParams::get_options() {
&llm_path},
{"",
"--tokenizer",
"tokenizer.json path, or comma-separated main=FILE,clip-l=FILE,clip-g=FILE assignments",
"tokenizer.json path, or comma-separated main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens",
(int)',',
&tokenizer},
{"",