From c7f4ae7b9c8976b4d50c59eb87e9582ea9c5c82f Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 12 May 2025 20:41:42 -0700 Subject: [PATCH] server: add webp image input support (#10653) --- cmd/interactive.go | 6 +++--- cmd/interactive_test.go | 19 +++++++++++++++---- server/routes.go | 6 ++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cmd/interactive.go b/cmd/interactive.go index 702227116..d7e6fbcfb 100644 --- a/cmd/interactive.go +++ b/cmd/interactive.go @@ -44,7 +44,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { fmt.Fprintln(os.Stderr, "Use \"\"\" to begin a multi-line message.") if opts.MultiModal { - fmt.Fprintf(os.Stderr, "Use %s to include .jpg or .png images.\n", filepath.FromSlash("/path/to/file")) + fmt.Fprintf(os.Stderr, "Use %s to include .jpg, .png, or .webp images.\n", filepath.FromSlash("/path/to/file")) } fmt.Fprintln(os.Stderr, "") @@ -511,7 +511,7 @@ func extractFileNames(input string) []string { // Regex to match file paths starting with optional drive letter, / ./ \ or .\ and include escaped or unescaped spaces (\ or %20) // and followed by more characters and a file extension // This will capture non filename strings, but we'll check for file existence to remove mismatches - regexPattern := `(?:[a-zA-Z]:)?(?:\./|/|\\)[\S\\ ]+?\.(?i:jpg|jpeg|png)\b` + regexPattern := `(?:[a-zA-Z]:)?(?:\./|/|\\)[\S\\ ]+?\.(?i:jpg|jpeg|png|webp)\b` re := regexp.MustCompile(regexPattern) return re.FindAllString(input, -1) @@ -553,7 +553,7 @@ func getImageData(filePath string) ([]byte, error) { } contentType := http.DetectContentType(buf) - allowedTypes := []string{"image/jpeg", "image/jpg", "image/png"} + allowedTypes := []string{"image/jpeg", "image/jpg", "image/png", "image/webp"} if !slices.Contains(allowedTypes, contentType) { return nil, fmt.Errorf("invalid image type: %s", contentType) } diff --git a/cmd/interactive_test.go b/cmd/interactive_test.go index ba8331918..809f53ff9 100644 --- a/cmd/interactive_test.go +++ b/cmd/interactive_test.go @@ -12,14 +12,17 @@ func TestExtractFilenames(t *testing.T) { // Unix style paths input := ` some preamble ./relative\ path/one.png inbetween1 ./not a valid two.jpg inbetween2 ./1.svg -/unescaped space /three.jpeg inbetween3 /valid\ path/dir/four.png "./quoted with spaces/five.JPG` +/unescaped space /three.jpeg inbetween3 /valid\ path/dir/four.png "./quoted with spaces/five.JPG +/unescaped space /six.webp inbetween6 /valid\ path/dir/seven.WEBP` res := extractFileNames(input) - assert.Len(t, res, 5) + assert.Len(t, res, 7) assert.Contains(t, res[0], "one.png") assert.Contains(t, res[1], "two.jpg") assert.Contains(t, res[2], "three.jpeg") assert.Contains(t, res[3], "four.png") assert.Contains(t, res[4], "five.JPG") + assert.Contains(t, res[5], "six.webp") + assert.Contains(t, res[6], "seven.WEBP") assert.NotContains(t, res[4], '"') assert.NotContains(t, res, "inbetween1") assert.NotContains(t, res, "./1.svg") @@ -30,10 +33,12 @@ func TestExtractFilenames(t *testing.T) { /absolute/nospace/three.jpeg inbetween3 /absolute/with space/four.png inbetween4 ./relative\ path/five.JPG inbetween5 "./relative with/spaces/six.png inbetween6 d:\path with\spaces\seven.JPEG inbetween7 c:\users\jdoe\eight.png inbetween8 - d:\program files\someplace\nine.png inbetween9 "E:\program files\someplace\ten.PNG some ending + d:\program files\someplace\nine.png inbetween9 "E:\program files\someplace\ten.PNG +c:/users/jdoe/eleven.webp inbetween11 c:/program files/someplace/twelve.WebP inbetween12 +d:\path with\spaces\thirteen.WEBP some ending ` res = extractFileNames(input) - assert.Len(t, res, 10) + assert.Len(t, res, 13) assert.NotContains(t, res, "inbetween2") assert.Contains(t, res[0], "one.png") assert.Contains(t, res[0], "c:") @@ -51,6 +56,12 @@ d:\path with\spaces\seven.JPEG inbetween7 c:\users\jdoe\eight.png inbetween8 assert.Contains(t, res[8], "d:") assert.Contains(t, res[9], "ten.PNG") assert.Contains(t, res[9], "E:") + assert.Contains(t, res[10], "eleven.webp") + assert.Contains(t, res[10], "c:") + assert.Contains(t, res[11], "twelve.WebP") + assert.Contains(t, res[11], "c:") + assert.Contains(t, res[12], "thirteen.WEBP") + assert.Contains(t, res[12], "d:") } // Ensure that file paths wrapped in single quotes are removed with the quotes. diff --git a/server/routes.go b/server/routes.go index f1706f74b..fd65669a2 100644 --- a/server/routes.go +++ b/server/routes.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "image" "io" "io/fs" "log/slog" @@ -25,6 +26,7 @@ import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" + "golang.org/x/image/webp" "golang.org/x/sync/errgroup" "github.com/ollama/ollama/api" @@ -1304,6 +1306,10 @@ func Serve(ln net.Listener) error { s.sched.Run(schedCtx) + // register the experimental webp decoder + // so webp images can be used in multimodal inputs + image.RegisterFormat("webp", "RIFF????WEBP", webp.Decode, webp.DecodeConfig) + // At startup we retrieve GPU information so we can get log messages before loading a model // This will log warnings to the log in case we have problems with detected GPUs gpus := discover.GetGPUInfo()