highperfocused/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-07-12 16:22:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

skip quantizing per_layer_token_embd (#11207)

Browse Source 
this tensor isn't compatible with cuda when quantized to q4_K so skip it
This commit is contained in:
Michael Yang
2025-06-26 21:49:35 -07:00
committed by GitHub
parent 11ffc36157
commit d0b32def60
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/quantization.go
View File

@ -231,6 +231,8 @@ func newType(t *fsggml.Tensor, kv fsggml.KV, qs *quantizeState, ftype fsggml.Fil
// do not quantize relative position bias (T5)
quantize = quantize && !strings.Contains(name, "attn_rel_b.weight")
quantize = quantize && !strings.Contains(name, "per_layer_token_embd.weight")
newType := fsggml.TensorType(t.Kind)
if quantize {
// get more optimal quantization type based on the tensor shape, layer, etc.