fix(server): follow the user's language in chat quick actions (MUL-5689) (#6345)

Quick actions never pinned their output language: the system prompt named Chinese in a rule about button width, and ALREADY SUGGESTED replayed the previous turn's labels, so one bad pass seeded the next. Neither prompt names a language now, and the user message closes with an explicit rule anchored on the most recent [user] turn, disowning the agent's reply, older turns, the instructions, and the replayed labels.

MUL-5689
This commit is contained in:
Bohan Jiang
2026-08-04 14:15:33 +08:00
committed by GitHub
parent fe62be266d
commit 2f431cd74a
2 changed files with 101 additions and 6 deletions

View File

@@ -100,6 +100,12 @@ type ChatQuickActionsLLM interface {
// agent-operations actions; this pass has no such context and must be told the
// frame explicitly.
//
// Output language is settled by chatQuickActionsLanguageRule, which closes the
// user message; this prompt only says that rule wins. Naming a specific
// language in these stable instructions is what this file must avoid — a model
// reading "…in Chinese" in a rule about button width will sometimes take it as
// permission to answer in Chinese (MUL-5689).
//
// The word "JSON" must stay in this text: response_format=json_object is
// rejected upstream without it.
const chatQuickActionsSystemPrompt = `You generate follow-up suggestions for a chat between a user and an AI agent.
@@ -122,22 +128,48 @@ Quality bar:
rewordings of the same request.
Field rules:
- "label": the button text. A short verb phrase — at most 6 words in English, at
most 12 characters in Chinese. No trailing punctuation, no quotes, no emoji.
It is a button, not a sentence.
- "label": the button text. A short verb phrase — at most 6 words, or at most 12
characters in a script that does not put spaces between words. No trailing
punctuation, no quotes, no emoji. It is a button, not a sentence.
- "prompt": the full message sent on the user's behalf. First person, the user's
own voice, and SELF-CONTAINED — the agent never sees the label, so the prompt
must carry every detail itself. One or two sentences.
- "primary": true on exactly one suggestion, the single most likely next step.
false on all others.
Write both fields in the same language the USER has been writing in, regardless
of what language the agent replied in.
Language: the user message ends with a LANGUAGE RULE line. It is authoritative;
follow it exactly.
Output JSON only, exactly this shape:
{"actions":[{"label":"...","prompt":"...","primary":true}]}
No prose, no markdown, no code fences.`
// chatQuickActionsLanguageRule closes the pass's user message and is the whole
// language policy. It is last on purpose: everything above it is conversation
// that may be in a language the pills must NOT be written in, and the rule has
// to be the final thing read before the task line.
//
// It anchors on the MOST RECENT user turn, not on the window as a whole. A pill
// sends a message as the user, so it follows how they are writing right now —
// which also means switching language takes effect on the very next turn
// instead of waiting for the window to tip.
//
// Everything else is named and excluded explicitly, because each one has been
// observed to pull the output the wrong way (MUL-5689): the agent may reply in
// another language, these instructions are English, and ALREADY SUGGESTED
// replays the previous turn's labels — which is what made one bad pass stick,
// each Chinese label seeding the next round.
//
// Deliberately NOT done here: resolving the language server-side from
// `"user".language` or from Unicode script detection. The stored preference is
// the UI locale, not the language of this conversation, so an English UI would
// silently override someone chatting in Chinese. Script detection fares worse —
// it cannot tell one Latin-script language from another (so the model still has
// to infer), it reads an earlier Korean turn over the latest Japanese one, and
// a kana sentence carrying enough English identifiers classifies as Latin, at
// which point a "never emit CJK" clause forbids the user's own script.
const chatQuickActionsLanguageRule = `LANGUAGE RULE: Write every "label" and "prompt" in the same language as the most recent [user] message above. Ignore the agent's reply, older messages, these instructions, and ALREADY SUGGESTED when choosing the language. If there is no [user] message, use the latest [agent] message.`
// GenerateChatQuickActionsForTask runs one suggestion pass for a completed chat
// turn and attaches the result to that turn's assistant row, broadcasting
// chat:quick_actions. It is the synchronous core shared by the automatic
@@ -344,6 +376,11 @@ func (s *TaskService) buildChatQuickActionsPrompt(ctx context.Context, target db
// included: on an explicit refresh it holds the pills the user is asking to
// replace, and offering the same three back is the one outcome a refresh must
// never produce.
//
// These labels are a de-duplication list ONLY. They are replayed verbatim, so
// on a session that once drifted they are the wrong language — which is exactly
// how one bad pass used to seed the next. chatQuickActionsLanguageRule names
// them explicitly as something to ignore when choosing the output language.
func collectPreviousChatQuickActions(msgs []db.ChatMessage) []string {
labels := make([]string, 0, chatQuickActionsPreviousMax)
seen := make(map[string]struct{}, chatQuickActionsPreviousMax)
@@ -407,7 +444,10 @@ func renderChatQuickActionsContext(msgs []db.ChatMessage, previous []string) str
}
}
b.WriteString("\nProduce the follow-up suggestions for the latest agent reply.")
b.WriteString("\n")
b.WriteString(chatQuickActionsLanguageRule)
b.WriteString("\n\nProduce the follow-up suggestions for the latest agent reply.")
return b.String()
}

View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"strings"
"testing"
"unicode"
db "github.com/multica-ai/multica/server/pkg/db/generated"
"github.com/multica-ai/multica/server/pkg/protocol"
@@ -187,3 +188,57 @@ func TestParseChatQuickActionsOutputAcceptsFencedObject(t *testing.T) {
t.Fatalf("actions = %+v", actions)
}
}
// The MUL-5689 shape, with every pull toward the wrong language present at
// once: an older Chinese turn, a Chinese agent reply, Chinese labels replayed
// under ALREADY SUGGESTED — and the user's newest turn in English. The rendered
// prompt must close by pointing at that newest [user] turn and disowning the
// rest.
func TestRenderChatQuickActionsContextClosesWithTheLanguageRule(t *testing.T) {
out := renderChatQuickActionsContext([]db.ChatMessage{
chatMsg("user", "帮我看下这个 PR"),
chatMsg("user", "review this PR for simplifications"),
chatMsg("assistant", "已创建工单 EFF-359并分配给了 Claude Engineer。"),
}, []string{"查看工单详情", "补充审查范围"})
// Last thing before the task line: the conversation above it is Chinese, so
// anything earlier would be read through that.
if !strings.Contains(out, chatQuickActionsLanguageRule+"\n\nProduce the follow-up") {
t.Fatalf("language rule must be the final constraint:\n%s", out)
}
// Anchored on the newest user turn — not the window, not the agent.
if !strings.Contains(out, "same language as the most recent [user] message") {
t.Fatalf("rule must anchor on the most recent user turn:\n%s", out)
}
for _, disowned := range []string{"agent's reply", "older messages", "these instructions", "ALREADY SUGGESTED"} {
if !strings.Contains(chatQuickActionsLanguageRule, disowned) {
t.Fatalf("rule must explicitly exclude %q: %s", disowned, chatQuickActionsLanguageRule)
}
}
// The Chinese context is still delivered verbatim; the rule governs output,
// it does not scrub the input.
if !strings.Contains(out, "已创建工单 EFF-359") || !strings.Contains(out, "- 查看工单详情") {
t.Fatalf("conversation and previous labels must survive intact:\n%s", out)
}
}
// Neither prompt may name or contain a language: that is what taught the model
// Chinese was on the table in the first place.
func TestChatQuickActionsPromptsNameNoLanguage(t *testing.T) {
for name, text := range map[string]string{
"system prompt": chatQuickActionsSystemPrompt,
"language rule": chatQuickActionsLanguageRule,
} {
for _, r := range text {
if unicode.Is(unicode.Han, r) || unicode.Is(unicode.Hiragana, r) ||
unicode.Is(unicode.Katakana, r) || unicode.Is(unicode.Hangul, r) {
t.Fatalf("%s must contain no CJK, found %q", name, r)
}
}
for _, named := range []string{"Chinese", "Japanese", "Korean", "English"} {
if strings.Contains(text, named) {
t.Fatalf("%s must not name a language, found %q", name, named)
}
}
}
}