highperfocused/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-11-12 10:57:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

model: fix issues with spm tokenizer for Gemma 3 (#10081)

Browse Source
This commit is contained in:
Jeffrey Morgan
2025-04-02 13:22:56 -07:00
committed by GitHub
parent b42970063d
commit b51e0f397c
5 changed files with 175 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
model/models/gemma3/model_text.go
View File

@@ -45,7 +45,6 @@ func newTextModel(c ml.Config) *TextModel {
m := TextModel{
SentencePieceModel: model.NewSentencePieceModel(
c.String("tokenizer.ggml.pretokenizer", `(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+`),
&model.Vocabulary{
Values: c.Strings("tokenizer.ggml.tokens"),
Scores: c.Floats("tokenizer.ggml.scores"),