highperfocused/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-04-15 15:19:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

wip: test fixes

Browse Source
This commit is contained in:
Bruce MacDonald 2025-03-20 11:19:42 -07:00 committed by jmorganca
parent edac05387f
commit 9a12fd1067
3 changed files with 1226971 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
model/process_text_test.go
View File

@ -222,14 +222,10 @@ func tekken(t testing.TB) TextProcessor {
defer configFile.Close()
var config struct {
AddBosToken bool `json:"add_bos_token"`
AddEosToken bool `json:"add_eos_token"`
BosToken struct {
Content string `json:"content"`
} `json:"bos_token"`
EosToken struct {
Content string `json:"content"`
} `json:"eos_token"`
AddBosToken bool `json:"add_bos_token"`
AddEosToken bool `json:"add_eos_token"`
BosToken string `json:"bos_token"`
EosToken string `json:"eos_token"`
}
if err := json.NewDecoder(configFile).Decode(&config); err != nil {
t.Fatal(err)
@ -319,8 +315,8 @@ func tekken(t testing.TB) TextProcessor {
Types: types,
Scores: scores,
Merges: merges,
BOS: vocab[config.BosToken.Content],
EOS: vocab[config.EosToken.Content],
BOS: vocab[config.BosToken],
EOS: vocab[config.EosToken],
AddBOS: config.AddBosToken,
AddEOS: config.AddEosToken,
}

1217945
model/testdata/mistral-small/tokenizer.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff

9020
model/testdata/mistral-small/tokenizer_config.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff