highperfocused/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-11-11 21:17:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make maximum pending request configurable

Browse Source 
This also bumps up the default to be 50 queued requests
instead of 10.
This commit is contained in:
Daniel Hiltgen
2024-05-03 16:25:57 -07:00
parent 371f5e52aa
commit 20f6c06569
3 changed files with 37 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
server/routes.go
View File

@@ -146,12 +146,7 @@ func (s *Server) GenerateHandler(c *gin.Context) {
select {
case runner = <-rCh:
case err = <-eCh:
if errors.Is(err, context.Canceled) {
c.JSON(499, gin.H{"error": "request canceled"})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
handleErrorResponse(c, err)
return
}
@@ -394,12 +389,7 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) {
select {
case runner = <-rCh:
case err = <-eCh:
if errors.Is(err, context.Canceled) {
c.JSON(499, gin.H{"error": "request canceled"})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
handleErrorResponse(c, err)
return
}
@@ -1212,12 +1202,7 @@ func (s *Server) ChatHandler(c *gin.Context) {
select {
case runner = <-rCh:
case err = <-eCh:
if errors.Is(err, context.Canceled) {
c.JSON(499, gin.H{"error": "request canceled"})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
handleErrorResponse(c, err)
return
}
@@ -1338,3 +1323,15 @@ func (s *Server) ChatHandler(c *gin.Context) {
streamResponse(c, ch)
}
func handleErrorResponse(c *gin.Context, err error) {
if errors.Is(err, context.Canceled) {
c.JSON(499, gin.H{"error": "request canceled"})
return
}
if errors.Is(err, ErrMaxQueue) {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}