From a9220da3b699432ff4fff7a7aa94191f3694f10d Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 2 Apr 2025 16:13:15 -0700 Subject: [PATCH] s/gelu/silu/ --- model/models/mistral3/model_vision.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/models/mistral3/model_vision.go b/model/models/mistral3/model_vision.go index 6f0a7a0d7..2a52bf7c4 100644 --- a/model/models/mistral3/model_vision.go +++ b/model/models/mistral3/model_vision.go @@ -50,7 +50,7 @@ type VisionMLP struct { } func (mlp *VisionMLP) Forward(ctx ml.Context, hiddenStates ml.Tensor, opts *VisionModelOptions) ml.Tensor { - hiddenStates = mlp.Gate.Forward(ctx, hiddenStates).GELU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenStates)) + hiddenStates = mlp.Gate.Forward(ctx, hiddenStates).SILU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenStates)) return mlp.Down.Forward(ctx, hiddenStates) }