From 1f91cb0c8ccf734f060a7ed065e991233daa0448 Mon Sep 17 00:00:00 2001 From: Parth Sareen Date: Mon, 7 Jul 2025 15:53:42 -0700 Subject: [PATCH] template: add tool result compatibility (#11294) --- api/types.go | 1 + docs/api.md | 241 +++++++++++++++++++++++++++++++++++++- template/template.go | 16 +-- template/template_test.go | 98 ++++++++++++++++ 4 files changed, 348 insertions(+), 8 deletions(-) diff --git a/api/types.go b/api/types.go index 94d492006a..f1e47c5928 100644 --- a/api/types.go +++ b/api/types.go @@ -143,6 +143,7 @@ type Message struct { Thinking string `json:"thinking,omitempty"` Images []ImageData `json:"images,omitempty"` ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolName string `json:"tool_name,omitempty"` } func (m *Message) UnmarshalJSON(b []byte) error { diff --git a/docs/api.md b/docs/api.md index 11eaf73ab8..2460e6ced7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -508,13 +508,21 @@ Advanced parameters (optional): - `stream`: if `false` the response will be returned as a single response object, rather than a stream of objects - `keep_alive`: controls how long the model will stay loaded into memory following the request (default: `5m`) +### Tool calling + +Tool calling is supported by providing a list of tools in the `tools` parameter. The model will generate a response that includes a list of tool calls. See the [Chat request (Streaming with tools)](#chat-request-streaming-with-tools) example below. + +Models can also explain the result of the tool call in the response. See the [Chat request (With history, with tools)](#chat-request-with-history-with-tools) example below. + +[See models with tool calling capabilities](https://ollama.com/search?c=tool). + ### Structured outputs Structured outputs are supported by providing a JSON schema in the `format` parameter. The model will generate a response that matches the schema. See the [Chat request (Structured outputs)](#chat-request-structured-outputs) example below. ### Examples -#### Chat Request (Streaming) +#### Chat request (Streaming) ##### Request @@ -569,6 +577,88 @@ Final response: } ``` +#### Chat request (Streaming with tools) + +##### Request + +```shell +curl http://localhost:11434/api/chat -d '{ + "model": "llama3.2", + "messages": [ + { + "role": "user", + "content": "what is the weather in tokyo?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather in a given city", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to get the weather for" + } + }, + "required": ["city"] + } + } + } + ], + "stream": true +}' +``` + +##### Response + +A stream of JSON objects is returned: +```json +{ + "model": "llama3.2", + "created_at": "2025-07-07T20:22:19.184789Z", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "function": { + "name": "get_weather", + "arguments": { + "city": "Tokyo" + } + }, + } + ] + }, + "done": false +} +``` + +Final response: + +```json +{ + "model":"llama3.2", + "created_at":"2025-07-07T20:22:19.19314Z", + "message": { + "role": "assistant", + "content": "" + }, + "done_reason": "stop", + "done": true, + "total_duration": 182242375, + "load_duration": 41295167, + "prompt_eval_count": 169, + "prompt_eval_duration": 24573166, + "eval_count": 15, + "eval_duration": 115959084 +} +``` + #### Chat request (No streaming) ##### Request @@ -606,6 +696,74 @@ curl http://localhost:11434/api/chat -d '{ } ``` +#### Chat request (No streaming, with tools) + +##### Request + + +```shell +curl http://localhost:11434/api/chat -d '{ + "model": "llama3.2", + "messages": [ + { + "role": "user", + "content": "what is the weather in tokyo?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather in a given city", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to get the weather for" + } + }, + "required": ["city"] + } + } + } + ], + "stream": false +}' +``` + +##### Response + +```json +{ + "model": "llama3.2", + "created_at": "2025-07-07T20:32:53.844124Z", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "function": { + "name": "get_weather", + "arguments": { + "city": "Tokyo" + } + }, + } + ] + }, + "done_reason": "stop", + "done": true, + "total_duration": 3244883583, + "load_duration": 2969184542, + "prompt_eval_count": 169, + "prompt_eval_duration": 141656333, + "eval_count": 18, + "eval_duration": 133293625 +} +``` + #### Chat request (Structured outputs) ##### Request @@ -712,6 +870,87 @@ Final response: } ``` + +#### Chat request (With history, with tools) + +##### Request + +```shell +curl http://localhost:11434/api/chat -d '{ + "model": "llama3.2", + "messages": [ + { + "role": "user", + "content": "what is the weather in Toronto?" + }, + // the message from the model appended to history + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "function": { + "name": "get_temperature", + "arguments": { + "city": "Toronto" + } + }, + } + ] + }, + // the tool call result appended to history + { + "role": "tool", + "content": "11 degrees celsius", + "tool_name": "get_temperature", + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather in a given city", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to get the weather for" + } + }, + "required": ["city"] + } + } + } + ] +}' +``` + +##### Response + +```json +{ + "model": "llama3.2", + "created_at": "2025-07-07T20:43:37.688511Z", + "message": { + "role": "assistant", + "content": "The current temperature in Toronto is 11°C." + }, + "done_reason": "stop", + "done": true, + "total_duration": 890771750, + "load_duration": 707634750, + "prompt_eval_count": 94, + "prompt_eval_duration": 91703208, + "eval_count": 11, + "eval_duration": 90282125 +} + +``` + + #### Chat request (with images) ##### Request diff --git a/template/template.go b/template/template.go index da910afbd5..242708f162 100644 --- a/template/template.go +++ b/template/template.go @@ -310,21 +310,23 @@ func (t *Template) Execute(w io.Writer, v Values) error { } // collate messages based on role. consecutive messages of the same role are merged -// into a single message. collate also collects and returns all system messages. +// into a single message (except for tool messages which preserve individual metadata). +// collate also collects and returns all system messages. // collate mutates message content adding image tags ([img-%d]) as needed +// todo(parthsareen): revisit for contextual image support func collate(msgs []api.Message) (string, []*api.Message) { var system []string var collated []*api.Message for i := range msgs { - msg := msgs[i] - if msg.Role == "system" { - system = append(system, msg.Content) + if msgs[i].Role == "system" { + system = append(system, msgs[i].Content) } - if len(collated) > 0 && collated[len(collated)-1].Role == msg.Role { - collated[len(collated)-1].Content += "\n\n" + msg.Content + // merges consecutive messages of the same role into a single message (except for tool messages) + if len(collated) > 0 && collated[len(collated)-1].Role == msgs[i].Role && msgs[i].Role != "tool" { + collated[len(collated)-1].Content += "\n\n" + msgs[i].Content } else { - collated = append(collated, &msg) + collated = append(collated, &msgs[i]) } } diff --git a/template/template_test.go b/template/template_test.go index ba1046500b..3d4eb99149 100644 --- a/template/template_test.go +++ b/template/template_test.go @@ -163,10 +163,12 @@ func TestParse(t *testing.T) { {"{{ .System }} {{ .Prompt }} {{ .Response }}", []string{"prompt", "response", "system"}}, {"{{ with .Tools }}{{ . }}{{ end }} {{ .System }} {{ .Prompt }}", []string{"prompt", "response", "system", "tools"}}, {"{{ range .Messages }}{{ .Role }} {{ .Content }}{{ end }}", []string{"content", "messages", "role"}}, + {"{{ range .Messages }}{{ if eq .Role \"tool\" }}Tool Result: {{ .ToolName }} {{ .Content }}{{ end }}{{ end }}", []string{"content", "messages", "role", "toolname"}}, {`{{- range .Messages }} {{- if eq .Role "system" }}SYSTEM: {{- else if eq .Role "user" }}USER: {{- else if eq .Role "assistant" }}ASSISTANT: +{{- else if eq .Role "tool" }}TOOL: {{- end }} {{ .Content }} {{- end }}`, []string{"content", "messages", "role"}}, {`{{- if .Messages }} @@ -376,3 +378,99 @@ func TestExecuteWithSuffix(t *testing.T) { }) } } + +func TestCollate(t *testing.T) { + cases := []struct { + name string + msgs []api.Message + expected []*api.Message + system string + }{ + { + name: "consecutive user messages are merged", + msgs: []api.Message{ + {Role: "user", Content: "Hello"}, + {Role: "user", Content: "How are you?"}, + }, + expected: []*api.Message{ + {Role: "user", Content: "Hello\n\nHow are you?"}, + }, + system: "", + }, + { + name: "consecutive tool messages are NOT merged", + msgs: []api.Message{ + {Role: "tool", Content: "sunny", ToolName: "get_weather"}, + {Role: "tool", Content: "72F", ToolName: "get_temperature"}, + }, + expected: []*api.Message{ + {Role: "tool", Content: "sunny", ToolName: "get_weather"}, + {Role: "tool", Content: "72F", ToolName: "get_temperature"}, + }, + system: "", + }, + { + name: "tool messages preserve all fields", + msgs: []api.Message{ + {Role: "user", Content: "What's the weather?"}, + {Role: "tool", Content: "sunny", ToolName: "get_conditions"}, + {Role: "tool", Content: "72F", ToolName: "get_temperature"}, + }, + expected: []*api.Message{ + {Role: "user", Content: "What's the weather?"}, + {Role: "tool", Content: "sunny", ToolName: "get_conditions"}, + {Role: "tool", Content: "72F", ToolName: "get_temperature"}, + }, + system: "", + }, + { + name: "mixed messages with system", + msgs: []api.Message{ + {Role: "system", Content: "You are helpful"}, + {Role: "user", Content: "Hello"}, + {Role: "assistant", Content: "Hi there!"}, + {Role: "user", Content: "What's the weather?"}, + {Role: "tool", Content: "sunny", ToolName: "get_weather"}, + {Role: "tool", Content: "72F", ToolName: "get_temperature"}, + {Role: "user", Content: "Thanks"}, + }, + expected: []*api.Message{ + {Role: "system", Content: "You are helpful"}, + {Role: "user", Content: "Hello"}, + {Role: "assistant", Content: "Hi there!"}, + {Role: "user", Content: "What's the weather?"}, + {Role: "tool", Content: "sunny", ToolName: "get_weather"}, + {Role: "tool", Content: "72F", ToolName: "get_temperature"}, + {Role: "user", Content: "Thanks"}, + }, + system: "You are helpful", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + system, collated := collate(tt.msgs) + if diff := cmp.Diff(system, tt.system); diff != "" { + t.Errorf("system mismatch (-got +want):\n%s", diff) + } + + // Compare the messages + if len(collated) != len(tt.expected) { + t.Errorf("expected %d messages, got %d", len(tt.expected), len(collated)) + return + } + + for i := range collated { + if collated[i].Role != tt.expected[i].Role { + t.Errorf("message %d role mismatch: got %q, want %q", i, collated[i].Role, tt.expected[i].Role) + } + if collated[i].Content != tt.expected[i].Content { + t.Errorf("message %d content mismatch: got %q, want %q", i, collated[i].Content, tt.expected[i].Content) + } + if collated[i].ToolName != tt.expected[i].ToolName { + t.Errorf("message %d tool name mismatch: got %q, want %q", i, collated[i].ToolName, tt.expected[i].ToolName) + } + } + }) + } +}