highperfocused/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-07-28 12:44:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

skip tokenizer.model if possible (#11050)

Browse Source 
if tokenizer.json is already copied, skip tokenizer.model
This commit is contained in:
Michael Yang
2025-06-11 12:10:35 -07:00
committed by GitHub
parent 2e77aa1ae7
commit 0dabb4ef6a
1 changed files with 12 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
parser/parser.go
View File

@@ -292,6 +292,10 @@ func filesForModel(path string) ([]string, error) {
} }
files = append(files, js...) files = append(files, js...)
// only include tokenizer.model is tokenizer.json is not present
if !slices.ContainsFunc(files, func(s string) bool {
return slices.Contains(strings.Split(s, string(os.PathSeparator)), "tokenizer.json")
}) {
if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 { if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 {
// add tokenizer.model if it exists, tokenizer.json is automatically picked up by the previous glob // add tokenizer.model if it exists, tokenizer.json is automatically picked up by the previous glob
// tokenizer.model might be a unresolved git lfs reference; error if it is // tokenizer.model might be a unresolved git lfs reference; error if it is
@@ -300,6 +304,7 @@ func filesForModel(path string) ([]string, error) {
// some times tokenizer.model is in a subdirectory (e.g. meta-llama/Meta-Llama-3-8B) // some times tokenizer.model is in a subdirectory (e.g. meta-llama/Meta-Llama-3-8B)
files = append(files, tks...) files = append(files, tks...)
} }
}
return files, nil return files, nil
} }