mirror of
https://github.com/ollama/ollama.git
synced 2025-11-11 00:57:53 +01:00
5f57b0ef42
- Both `/api/generate` and `/api/chat` now accept a `"think"` option that allows specifying whether thinking mode should be on or not - Templates get passed this new option so, e.g., qwen3's template can put `/think` or `/no_think` in the system prompt depending on the value of the setting - Models' thinking support is inferred by inspecting model templates. The prefix and suffix the parser uses to identify thinking support is also automatically inferred from templates - Thinking control & parsing is opt-in via the API to prevent breaking existing API consumers. If the `"think"` option is not specified, the behavior is unchanged from previous versions of ollama - Add parsing for thinking blocks in both streaming/non-streaming mode in both `/generate` and `/chat` - Update the CLI to make use of these changes. Users can pass `--think` or `--think=false` to control thinking, or during an interactive session they can use the commands `/set think` or `/set nothink` - A `--hidethinking` option has also been added to the CLI. This makes it easy to use thinking in scripting scenarios like `ollama run qwen3 --think --hidethinking "my question here"` where you just want to see the answer but still want the benefits of thinking models
113 lines
2.9 KiB
Go
113 lines
2.9 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"strings"
|
|
|
|
"github.com/ollama/ollama/api"
|
|
"github.com/ollama/ollama/llm"
|
|
"github.com/ollama/ollama/template"
|
|
)
|
|
|
|
type tokenizeFunc func(context.Context, string) ([]int, error)
|
|
|
|
// chatPrompt accepts a list of messages and returns the prompt and images that should be used for the next chat turn.
|
|
// chatPrompt truncates any messages that exceed the context window of the model, making sure to always include 1) the
|
|
// latest message and 2) system messages
|
|
func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.Options, msgs []api.Message, tools []api.Tool, think *bool) (prompt string, images []llm.ImageData, _ error) {
|
|
var system []api.Message
|
|
|
|
// TODO: Ideally we would compute this from the projector metadata but some pieces are implementation dependent
|
|
// Clip images are represented as 768 tokens, each an embedding
|
|
imageNumTokens := 768
|
|
|
|
n := len(msgs) - 1
|
|
// in reverse, find all messages that fit into context window
|
|
for i := n; i >= 0; i-- {
|
|
// always include the last message
|
|
if i == n {
|
|
continue
|
|
}
|
|
|
|
system = make([]api.Message, 0)
|
|
for j := range i {
|
|
if msgs[j].Role == "system" {
|
|
system = append(system, msgs[j])
|
|
}
|
|
}
|
|
|
|
thinkVal := false
|
|
if think != nil {
|
|
thinkVal = *think
|
|
}
|
|
var b bytes.Buffer
|
|
if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[i:]...), Tools: tools, Think: thinkVal, IsThinkSet: think != nil}); err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
s, err := tokenize(ctx, b.String())
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
ctxLen := len(s)
|
|
if m.ProjectorPaths != nil {
|
|
for _, m := range msgs[i:] {
|
|
ctxLen += imageNumTokens * len(m.Images)
|
|
}
|
|
}
|
|
|
|
if ctxLen > opts.NumCtx {
|
|
slog.Debug("truncating input messages which exceed context length", "truncated", len(msgs[i:]))
|
|
break
|
|
} else {
|
|
n = i
|
|
}
|
|
}
|
|
|
|
currMsgIdx := n
|
|
|
|
for cnt, msg := range msgs[currMsgIdx:] {
|
|
if slices.Contains(m.Config.ModelFamilies, "mllama") && len(msg.Images) > 1 {
|
|
return "", nil, errors.New("this model only supports one image while more than one image requested")
|
|
}
|
|
|
|
var prefix string
|
|
prompt := msg.Content
|
|
|
|
for _, i := range msg.Images {
|
|
imgData := llm.ImageData{
|
|
ID: len(images),
|
|
Data: i,
|
|
}
|
|
|
|
imgTag := fmt.Sprintf("[img-%d]", imgData.ID)
|
|
if !strings.Contains(prompt, "[img]") {
|
|
prefix += imgTag
|
|
} else {
|
|
prompt = strings.Replace(prompt, "[img]", imgTag, 1)
|
|
}
|
|
|
|
images = append(images, imgData)
|
|
}
|
|
msgs[currMsgIdx+cnt].Content = prefix + prompt
|
|
}
|
|
|
|
// truncate any messages that do not fit into the context window
|
|
var b bytes.Buffer
|
|
thinkVal := false
|
|
if think != nil {
|
|
thinkVal = *think
|
|
}
|
|
if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[currMsgIdx:]...), Tools: tools, Think: thinkVal, IsThinkSet: think != nil}); err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
return b.String(), images, nil
|
|
}
|