mirror of
https://github.com/ollama/ollama.git
synced 2025-07-05 09:41:05 +02:00
333e360422
* get eos_token_id from generation_config.json * refactor * include both ids and strings in trace * comments * remove special case for gemma3 special vocab (#10743)
18 lines
326 B
Go
18 lines
326 B
Go
package model
|
|
|
|
const (
|
|
TOKEN_TYPE_NORMAL = iota + 1
|
|
TOKEN_TYPE_UNKNOWN
|
|
TOKEN_TYPE_CONTROL
|
|
TOKEN_TYPE_USER_DEFINED
|
|
TOKEN_TYPE_UNUSED
|
|
TOKEN_TYPE_BYTE
|
|
)
|
|
|
|
type TextProcessor interface {
|
|
Encode(s string, addSpecial bool) ([]int32, error)
|
|
Decode([]int32) (string, error)
|
|
Is(int32, Special) bool
|
|
Vocabulary() *Vocabulary
|
|
}
|