highperfocused/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-03-29 11:11:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1971 from jmorganca/mxyng/max-context-length

Browse Source 
add max context length check
This commit is contained in:
Michael Yang 2024-01-12 15:10:25 -08:00 committed by GitHub
commit 356d178f6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
llm/ggml.go
View File

@ -83,6 +83,7 @@ type model interface {
NumEmbed() uint32
NumHead() uint32
NumHeadKv() uint32
NumCtx() uint32
}
type container interface {

9
llm/gguf.go
View File

@ -308,6 +308,15 @@ func (llm *ggufModel) NumHeadKv() uint32 {
return value.(uint32)
}
func (llm *ggufModel) NumCtx() uint32 {
value, exists := llm.kv[fmt.Sprintf("%s.context_length", llm.ModelFamily())]
if !exists {
return 0
}
return value.(uint32)
}
func (llm *ggufModel) NumGQA() uint32 {
numHeadKv := llm.NumHeadKv()
if numHeadKv == 0 {

5
llm/llm.go
View File

@ -35,6 +35,11 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options)
return nil, err
}
if opts.NumCtx > int(ggml.NumCtx()) {
log.Printf("WARNING: requested context length is greater than model's max context length (%d > %d), using %d instead", opts.NumCtx, ggml.NumCtx(), ggml.NumCtx())
opts.NumCtx = int(ggml.NumCtx())
}
if opts.NumCtx < 4 {
opts.NumCtx = 4
}