return more info in generate response

This commit is contained in:
Michael Yang
2023-07-12 18:18:06 -07:00
parent 31590284a7
commit 05e08d2310
4 changed files with 116 additions and 31 deletions

View File

@@ -13,6 +13,7 @@ import (
"path"
"strings"
"text/template"
"time"
"github.com/gin-gonic/gin"
"github.com/lithammer/fuzzysearch/fuzzy"
@@ -35,6 +36,8 @@ func cacheDir() string {
}
func generate(c *gin.Context) {
start := time.Now()
req := api.GenerateRequest{
Options: api.DefaultOptions(),
}
@@ -81,8 +84,14 @@ func generate(c *gin.Context) {
}
defer llm.Close()
fn := func(s string) {
ch <- api.GenerateResponse{Response: s}
fn := func(r api.GenerateResponse) {
r.Model = req.Model
r.CreatedAt = time.Now().UTC()
if r.Done {
r.TotalDuration = time.Since(start)
}
ch <- r
}
if err := llm.Predict(req.Prompt, fn); err != nil {