mirror of
https://github.com/ollama/ollama.git
synced 2025-04-07 03:18:24 +02:00
better estimate scratch buffer size
This commit is contained in:
parent
18ddf6d57d
commit
58ce2d8273
@ -62,8 +62,8 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options)
|
||||
|
||||
// this amount is the overhead + tensors in memory
|
||||
// TODO: get this from the llama.cpp's graph calcluations instead of
|
||||
// guessing it's ~1/7th of the kv cache times gqa
|
||||
requiredAlloc := int64(ggml.NumGQA()) * requiredKv / 7
|
||||
// estimating it's 1/6 * kv_cache_size * num_gqa
|
||||
requiredAlloc := int64(ggml.NumGQA()) * requiredKv / 6
|
||||
|
||||
requiredTotal := requiredModel + requiredKv + requiredAlloc
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user