From 386169205c94aef035456ecfea8047f9cb76cf0a Mon Sep 17 00:00:00 2001
From: Michael Yang <mxyng@pm.me>
Date: Fri, 20 Oct 2023 18:17:14 -0700
Subject: [PATCH] update runtime options (#864)
---
server/routes.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/server/routes.go b/server/routes.go
index cdd3caf58..5c52dbfef 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -112,6 +112,10 @@ func load(ctx context.Context, workDir string, model *Model, reqOpts map[string]
loaded.Options = &opts
}
+ // update options for the loaded llm
+ // TODO(mxyng): this isn't thread safe, but it should be fine for now
+ loaded.runner.SetOptions(opts)
+
loaded.expireAt = time.Now().Add(sessionDuration)
if loaded.expireTimer == nil {