highperfocused/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-11-13 08:17:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

client updates

Browse Source
This commit is contained in:
Jeffrey Morgan
2023-07-04 00:47:00 -04:00
parent 6292f4b64c
commit fd962a36e5
21 changed files with 198 additions and 3137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
server/routes.go
View File

@@ -17,26 +17,14 @@ import (
func Serve(ln net.Listener) error {
r := gin.Default()
var l *llama.LLama
gpulayers := 1
// TODO: these should be request parameters
gpulayers := 0
tokens := 512
threads := runtime.NumCPU()
model := "/Users/pdevine/.cache/gpt4all/GPT4All-13B-snoozy.ggmlv3.q4_0.bin"
r.POST("/api/load", func(c *gin.Context) {
var err error
l, err = llama.New(model, llama.EnableF16Memory, llama.SetContext(128), llama.EnableEmbeddings, llama.SetGPULayers(gpulayers))
if err != nil {
fmt.Println("Loading the model failed:", err.Error())
}
})
r.POST("/api/unload", func(c *gin.Context) {
})
r.POST("/api/generate", func(c *gin.Context) {
// TODO: set prompt from template
fmt.Println("Generating text...")
var req api.GenerateRequest
if err := c.ShouldBindJSON(&req); err != nil {
@@ -44,6 +32,14 @@ func Serve(ln net.Listener) error {
return
}
fmt.Println(req)
l, err := llama.New(req.Model, llama.EnableF16Memory, llama.SetContext(128), llama.EnableEmbeddings, llama.SetGPULayers(gpulayers))
if err != nil {
fmt.Println("Loading the model failed:", err.Error())
return
}
ch := make(chan string)
go func() {
@@ -55,7 +51,7 @@ func Serve(ln net.Listener) error {
if err != nil {
panic(err)
}
}()
}()
c.Stream(func(w io.Writer) bool {
tok, ok := <-ch
@@ -65,11 +61,6 @@ func Serve(ln net.Listener) error {
c.SSEvent("token", tok)
return true
})
// embeds, err := l.Embeddings(text)
// if err != nil {
// fmt.Printf("Embeddings: error %s \n", err.Error())
// }
})
log.Printf("Listening on %s", ln.Addr())