highperfocused/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-11-12 05:17:08 +01:00
Code Issues Packages Projects Releases Wiki Activity

llama: remove model loading for grammar (#10096)

Browse Source
This commit is contained in:
Parth Sareen
2025-04-24 11:51:19 -07:00
committed by GitHub
parent 40b10eee6d
commit a53d744b01
13 changed files with 521 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
model/process_text.go
View File

@@ -32,6 +32,7 @@ type TextProcessor interface {
Encode(s string, addSpecial bool) ([]int32, error)
Decode([]int32) (string, error)
Is(int32, Special) bool
Vocabulary() *Vocabulary
}
type Vocabulary struct {
@@ -117,6 +118,8 @@ type BytePairEncoding struct {
vocab *Vocabulary
}
var _ TextProcessor = (*BytePairEncoding)(nil)
func NewBytePairEncoding(pre string, vocab *Vocabulary) BytePairEncoding {
return BytePairEncoding{
pre: regexp2.MustCompile(pre, regexp2.Unicode|regexp2.RE2),
@@ -124,6 +127,10 @@ func NewBytePairEncoding(pre string, vocab *Vocabulary) BytePairEncoding {
}
}
func (bpe BytePairEncoding) Vocabulary() *Vocabulary {
return bpe.vocab
}
func (bpe BytePairEncoding) Is(id int32, special Special) bool {
return bpe.vocab.Is(id, special)
}