feat: add image upload support in chat

- Add click-to-select and Cmd+V paste for image upload - Support multiple images per message (max 10MB each) - Display image previews before sending - Update message pipeline to handle MessageContent array - Add backward compatibility for old session user_message format - Remove Gemini agent from defaults Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-07-23 10:08:38 +02:00 · 2026-01-15 13:45:28 +08:00
parent d5a58890be
commit 913cccc8aa
10 changed files with 307 additions and 61 deletions
--- a/src/main/cli.ts
+++ b/src/main/cli.ts
@@ -464,7 +464,9 @@ async function sendPrompt(state: CLIState, content: string): Promise<void> {
  print(`\n${c.blue}🤖 Agent:${c.reset}`)

  try {
-    const stopReason = await state.conductor.sendPrompt(state.currentSession.id, content)
+    // Convert string to MessageContent format
+    const messageContent = [{ type: 'text' as const, text: content }]
+    const stopReason = await state.conductor.sendPrompt(state.currentSession.id, messageContent)
    print(`\n\n${c.dim}[${stopReason}]${c.reset}\n`)
  } catch (err) {
    printError(`${err}`)
--- a/src/main/conductor/Conductor.ts
+++ b/src/main/conductor/Conductor.ts
@@ -24,6 +24,7 @@ import type {
  SessionData,
  ListSessionsOptions,
 } from '../../shared/types'
+import type { MessageContent, MessageContentItem } from '../../shared/types/message'
 import { formatHistoryForReplay, hasReplayableHistory } from './historyReplay'

 export interface SessionUpdateCallback {
@@ -290,15 +291,27 @@ export class Conductor {
  }

  /**
-   * Send a prompt to the agent
+   * Send a prompt to the agent (supports text and images)
   */
-  async sendPrompt(sessionId: string, content: string): Promise<string> {
+  async sendPrompt(sessionId: string, content: MessageContent): Promise<string> {
    // Ensure agent is running (lazy start if needed)
    const sessionAgent = await this.ensureAgentForSession(sessionId)
    const { connection, agentSessionId } = sessionAgent

-    // Prepare the final prompt content
-    let finalContent = content
+    // Convert MessageContent to ACP SDK format
+    const convertToAcpFormat = (items: MessageContent): Array<{ type: string; text?: string; data?: string; mimeType?: string }> => {
+      return items.map((item: MessageContentItem) => {
+        if (item.type === 'text') {
+          return { type: 'text', text: item.text }
+        } else if (item.type === 'image') {
+          return { type: 'image', data: item.data, mimeType: item.mimeType }
+        }
+        return { type: 'text', text: '' } // fallback
+      })
+    }
+
+    // Build prompt content array
+    let promptContent = convertToAcpFormat(content)

    // If this is a resumed session, prepend conversation history to first prompt
    if (sessionAgent.needsHistoryReplay && this.sessionStore) {
@@ -308,7 +321,8 @@ export class Conductor {
          const history = formatHistoryForReplay(data.updates)
          if (history) {
            console.log(`[Conductor] Prepending conversation history (${data.updates.length} updates)`)
-            finalContent = history + content
+            // Prepend history as text block before other content
+            promptContent = [{ type: 'text', text: history }, ...promptContent]
          }
        }
      } catch (error) {
@@ -320,17 +334,24 @@ export class Conductor {
      }
    }

+    // Log prompt info
+    const textContent = content.find((c: MessageContentItem) => c.type === 'text')
+    const imageCount = content.filter((c: MessageContentItem) => c.type === 'image').length
    console.log(`[Conductor] Sending prompt to session ${agentSessionId}`)
-    console.log(`[Conductor]   Content: ${finalContent.slice(0, 100)}${finalContent.length > 100 ? '...' : ''}`)
+    if (textContent && textContent.type === 'text') {
+      console.log(`[Conductor]   Text: ${textContent.text.slice(0, 100)}${textContent.text.length > 100 ? '...' : ''}`)
+    }
+    if (imageCount > 0) {
+      console.log(`[Conductor]   Images: ${imageCount}`)
+    }

    // Store user message before sending (so it appears in history)
-    // Note: We store the original content, not the history-prepended version
    if (this.sessionStore) {
      const userUpdate = {
        sessionId: agentSessionId,
        update: {
          sessionUpdate: 'user_message',
-          content: { type: 'text', text: content },
+          content: content, // Store full MessageContent array
        },
      }
      await this.sessionStore.appendUpdate(sessionId, userUpdate as any)
@@ -343,7 +364,7 @@ export class Conductor {
    try {
      const result = await connection.prompt({
        sessionId: agentSessionId,
-        prompt: [{ type: 'text', text: finalContent }],
+        prompt: promptContent as any,
      })

      console.log(`[Conductor] Prompt completed with stopReason: ${result.stopReason}`)
--- a/src/main/config/defaults.ts
+++ b/src/main/config/defaults.ts
@@ -9,7 +9,7 @@ export const DEFAULT_AGENTS: Record<string, AgentConfig> = {
    name: 'Claude Code',
    command: 'claude-code-acp',
    args: [],
-    enabled: true,
+    enabled: true
    // Note: Uses https://github.com/zed-industries/claude-code-acp
    // Requires ANTHROPIC_API_KEY environment variable
  },
@@ -18,24 +18,17 @@ export const DEFAULT_AGENTS: Record<string, AgentConfig> = {
    name: 'opencode',
    command: 'opencode',
    args: ['acp'],
-    enabled: true,
+    enabled: true
  },
  codex: {
    id: 'codex',
    name: 'Codex CLI (ACP)',
    command: 'codex-acp',
    args: [],
-    enabled: true,
+    enabled: true
    // Note: Uses https://github.com/zed-industries/codex-acp
    // Official Codex CLI doesn't support ACP
-  },
-  gemini: {
-    id: 'gemini',
-    name: 'Gemini CLI',
-    command: 'gemini',
-    args: ['acp'],
-    enabled: true,
-  },
+  }
 }

 export const DEFAULT_CONFIG: AppConfig = {
@@ -44,6 +37,6 @@ export const DEFAULT_CONFIG: AppConfig = {
  agents: DEFAULT_AGENTS,
  ui: {
    theme: 'system',
-    fontSize: 14,
-  },
+    fontSize: 14
+  }
 }
--- a/src/main/ipc/handlers.ts
+++ b/src/main/ipc/handlers.ts
@@ -9,6 +9,7 @@ import { checkAgents } from '../utils/agent-check'
 import type { Conductor } from '../conductor/Conductor'
 import type { ListSessionsOptions, MulticaSession } from '../../shared/types'
 import type { FileTreeNode, DetectedApp } from '../../shared/electron-api'
+import type { MessageContent } from '../../shared/types/message'
 import * as fs from 'fs'
 import * as path from 'path'
 import { spawn } from 'child_process'
@@ -48,7 +49,7 @@ function isValidPath(inputPath: string): boolean {
 export function registerIPCHandlers(conductor: Conductor): void {
  // --- Agent handlers (per-session) ---

-  ipcMain.handle(IPC_CHANNELS.AGENT_PROMPT, async (_event, sessionId: string, content: string) => {
+  ipcMain.handle(IPC_CHANNELS.AGENT_PROMPT, async (_event, sessionId: string, content: MessageContent) => {
    const stopReason = await conductor.sendPrompt(sessionId, content)
    return { stopReason }
  })
--- a/src/preload/index.ts
+++ b/src/preload/index.ts
@@ -2,6 +2,7 @@ import { contextBridge, ipcRenderer } from 'electron'
 import { IPC_CHANNELS } from '../shared/ipc-channels'
 import type { ElectronAPI, OpenWithOptions } from '../shared/electron-api'
 import type { ListSessionsOptions, MulticaSession } from '../shared/types'
+import type { MessageContent } from '../shared/types/message'

 // Electron API exposed to renderer process
 const electronAPI: ElectronAPI = {
@@ -9,7 +10,7 @@ const electronAPI: ElectronAPI = {
  getAgentStatus: () => ipcRenderer.invoke(IPC_CHANNELS.AGENT_STATUS),

  // Agent communication
-  sendPrompt: (sessionId: string, content: string) =>
+  sendPrompt: (sessionId: string, content: MessageContent) =>
    ipcRenderer.invoke(IPC_CHANNELS.AGENT_PROMPT, sessionId, content),

  cancelRequest: (sessionId: string) => ipcRenderer.invoke(IPC_CHANNELS.AGENT_CANCEL, sessionId),
--- a/src/renderer/src/components/ChatView.tsx
+++ b/src/renderer/src/components/ChatView.tsx
@@ -81,6 +81,12 @@ interface TextBlock {
  content: string
 }

+interface ImageBlock {
+  type: 'image'
+  data: string
+  mimeType: string
+}
+
 interface ThoughtBlock {
  type: 'thought'
  content: string
@@ -91,7 +97,7 @@ interface ToolCallBlock {
  toolCall: ToolCall
 }

-type ContentBlock = TextBlock | ThoughtBlock | ToolCallBlock
+type ContentBlock = TextBlock | ImageBlock | ThoughtBlock | ToolCallBlock

 interface Message {
  role: 'user' | 'assistant'
@@ -148,13 +154,38 @@ function groupUpdatesIntoMessages(updates: StoredSessionUpdate[]): Message[] {
      case 'user_message' as string:
        // Flush any pending assistant message
        flushAssistantMessage()
-        // Add user message
+        // Add user message - supports multiple formats for backward compatibility
        {
-          const userUpdate = update as { content?: { type: string; text: string } }
-          if (userUpdate.content?.type === 'text') {
+          const userUpdate = update as { content?: unknown }
+          const userBlocks: ContentBlock[] = []
+          const content = userUpdate.content
+
+          if (Array.isArray(content)) {
+            // New format: MessageContent array (e.g., [{ type: 'text', text: '...' }, { type: 'image', ... }])
+            for (const item of content) {
+              if (item && typeof item === 'object') {
+                if (item.type === 'text' && typeof item.text === 'string') {
+                  userBlocks.push({ type: 'text', content: item.text })
+                } else if (item.type === 'image' && typeof item.data === 'string') {
+                  userBlocks.push({ type: 'image', data: item.data, mimeType: item.mimeType || 'image/png' })
+                }
+              }
+            }
+          } else if (content && typeof content === 'object' && 'type' in content && 'text' in content) {
+            // Old format: single text content object { type: 'text', text: '...' }
+            const textContent = content as { type: string; text: unknown }
+            if (textContent.type === 'text' && typeof textContent.text === 'string') {
+              userBlocks.push({ type: 'text', content: textContent.text })
+            }
+          } else if (typeof content === 'string') {
+            // Fallback: plain string content
+            userBlocks.push({ type: 'text', content: content })
+          }
+
+          if (userBlocks.length > 0) {
            messages.push({
              role: 'user',
-              blocks: [{ type: 'text', content: userUpdate.content.text }],
+              blocks: userBlocks,
            })
          }
        }
@@ -269,14 +300,31 @@ interface MessageBubbleProps {
 function MessageBubble({ message }: MessageBubbleProps) {
  const isUser = message.role === 'user'

-  // User message - bubble style
+  // User message - bubble style with support for images
  if (isUser) {
-    // Get the text content from blocks
+    const imageBlocks = message.blocks.filter((b): b is ImageBlock => b.type === 'image')
    const textBlock = message.blocks.find((b): b is TextBlock => b.type === 'text')
+
    return (
      <div className="flex justify-end">
-        <div className="max-w-[85%] whitespace-pre-wrap rounded-lg bg-muted px-4 py-3 text-[15px]">
-          {textBlock?.content || ''}
+        <div className="max-w-[85%] rounded-lg bg-muted px-4 py-3 text-[15px]">
+          {/* Render images first */}
+          {imageBlocks.length > 0 && (
+            <div className="flex flex-wrap gap-2 mb-2">
+              {imageBlocks.map((img, idx) => (
+                <img
+                  key={idx}
+                  src={`data:${img.mimeType};base64,${img.data}`}
+                  alt={`Uploaded image ${idx + 1}`}
+                  className="max-w-[200px] max-h-[200px] rounded-md object-cover"
+                />
+              ))}
+            </div>
+          )}
+          {/* Render text content */}
+          {textBlock && (
+            <div className="whitespace-pre-wrap">{textBlock.content}</div>
+          )}
        </div>
      </div>
    )
@@ -293,6 +341,15 @@ function MessageBubble({ message }: MessageBubbleProps) {
            return <ToolCallItem key={block.toolCall.id} toolCall={block.toolCall} />
          case 'text':
            return <TextContentBlock key={`text-${idx}`} content={block.content} />
+          case 'image':
+            return (
+              <img
+                key={`image-${idx}`}
+                src={`data:${block.mimeType};base64,${block.data}`}
+                alt={`Image ${idx + 1}`}
+                className="max-w-[300px] max-h-[300px] rounded-md object-cover"
+              />
+            )
          default:
            return null
        }
--- a/src/renderer/src/components/MessageInput.tsx
+++ b/src/renderer/src/components/MessageInput.tsx
@@ -1,13 +1,14 @@
 /**
- * Message input component
+ * Message input component with image upload support
 */
-import { useState, useRef, useEffect } from 'react'
-import { ArrowUp, Square, Folder } from 'lucide-react'
+import { useState, useRef, useEffect, useCallback } from 'react'
+import { ArrowUp, Square, Folder, Paperclip, X } from 'lucide-react'
 import { Button } from '@/components/ui/button'
 import { Tooltip, TooltipTrigger, TooltipContent } from '@/components/ui/tooltip'
+import type { MessageContent, ImageContentItem } from '../../../shared/types/message'

 interface MessageInputProps {
-  onSend: (content: string) => void
+  onSend: (content: MessageContent) => void
  onCancel: () => void
  isProcessing: boolean
  disabled: boolean
@@ -16,6 +17,9 @@ interface MessageInputProps {
  onSelectFolder: () => Promise<void>
 }

+const MAX_IMAGE_SIZE = 10 * 1024 * 1024 // 10MB
+const SUPPORTED_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp']
+
 export function MessageInput({
  onSend,
  onCancel,
@@ -27,7 +31,9 @@ export function MessageInput({
 }: MessageInputProps) {
  const [value, setValue] = useState('')
  const [isComposing, setIsComposing] = useState(false)
+  const [images, setImages] = useState<ImageContentItem[]>([])
  const textareaRef = useRef<HTMLTextAreaElement>(null)
+  const fileInputRef = useRef<HTMLInputElement>(null)

  // Get folder name from path
  const folderName = workingDirectory
@@ -48,13 +54,107 @@ export function MessageInput({
    textareaRef.current?.focus()
  }, [])

-  const handleSubmit = () => {
-    const trimmed = value.trim()
-    if (!trimmed || disabled || isProcessing) return
+  // Convert file to base64
+  const fileToBase64 = useCallback((file: File): Promise<string> => {
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader()
+      reader.onload = () => {
+        const result = reader.result as string
+        // Remove data URL prefix (e.g., "data:image/png;base64,")
+        const base64 = result.split(',')[1]
+        resolve(base64)
+      }
+      reader.onerror = reject
+      reader.readAsDataURL(file)
+    })
+  }, [])

-    onSend(trimmed)
+  // Process and add image
+  const addImage = useCallback(async (file: File) => {
+    // Validate file type
+    if (!SUPPORTED_IMAGE_TYPES.includes(file.type)) {
+      console.warn('Unsupported image type:', file.type)
+      return
+    }
+
+    // Validate file size
+    if (file.size > MAX_IMAGE_SIZE) {
+      console.warn('Image too large:', file.size)
+      return
+    }
+
+    try {
+      const base64 = await fileToBase64(file)
+      const imageItem: ImageContentItem = {
+        type: 'image',
+        data: base64,
+        mimeType: file.type,
+      }
+      setImages((prev) => [...prev, imageItem])
+    } catch (err) {
+      console.error('Failed to process image:', err)
+    }
+  }, [fileToBase64])
+
+  // Handle paste event
+  const handlePaste = useCallback(async (e: React.ClipboardEvent) => {
+    const items = e.clipboardData?.items
+    if (!items) return
+
+    for (const item of items) {
+      if (item.type.startsWith('image/')) {
+        e.preventDefault()
+        const file = item.getAsFile()
+        if (file) {
+          await addImage(file)
+        }
+        break
+      }
+    }
+  }, [addImage])
+
+  // Handle file selection
+  const handleFileSelect = useCallback(async (e: React.ChangeEvent<HTMLInputElement>) => {
+    const files = e.target.files
+    if (!files) return
+
+    for (const file of files) {
+      await addImage(file)
+    }
+
+    // Reset input
+    if (fileInputRef.current) {
+      fileInputRef.current.value = ''
+    }
+  }, [addImage])
+
+  // Remove image
+  const removeImage = useCallback((index: number) => {
+    setImages((prev) => prev.filter((_, i) => i !== index))
+  }, [])
+
+  // Handle submit
+  const handleSubmit = useCallback(() => {
+    const trimmed = value.trim()
+    if ((!trimmed && images.length === 0) || disabled || isProcessing) return
+
+    // Build message content array
+    const content: MessageContent = []
+
+    // Add images first
+    for (const img of images) {
+      content.push(img)
+    }
+
+    // Add text if present
+    if (trimmed) {
+      content.push({ type: 'text', text: trimmed })
+    }
+
+    onSend(content)
    setValue('')
-  }
+    setImages([])
+  }, [value, images, disabled, isProcessing, onSend])

  const handleKeyDown = (e: React.KeyboardEvent) => {
    // Don't submit while IME is composing
@@ -64,7 +164,7 @@ export function MessageInput({
    }
  }

-  const canSubmit = !disabled && value.trim().length > 0
+  const canSubmit = !disabled && (value.trim().length > 0 || images.length > 0)
  const hasFolder = !!workingDirectory

  // Render folder selection mode when no folder is selected
@@ -104,12 +204,37 @@ export function MessageInput({
    <div className="p-4">
      <div className="mx-auto max-w-3xl">
        <div className="bg-secondary/50 hover:bg-secondary focus-within:bg-secondary transition-colors duration-200 rounded-xl p-3 border border-border">
+          {/* Image previews */}
+          {images.length > 0 && (
+            <div className="flex flex-wrap gap-2 mb-3 pb-3 border-b border-border/50">
+              {images.map((img, index) => (
+                <div
+                  key={index}
+                  className="relative group w-16 h-16 rounded-lg overflow-hidden border border-border bg-background"
+                >
+                  <img
+                    src={`data:${img.mimeType};base64,${img.data}`}
+                    alt={`Upload ${index + 1}`}
+                    className="w-full h-full object-cover"
+                  />
+                  <button
+                    onClick={() => removeImage(index)}
+                    className="absolute top-0.5 right-0.5 p-0.5 rounded-full bg-background/80 hover:bg-background text-muted-foreground hover:text-foreground opacity-0 group-hover:opacity-100 transition-opacity"
+                  >
+                    <X className="h-3 w-3" />
+                  </button>
+                </div>
+              ))}
+            </div>
+          )}
+
          {/* Text input */}
          <textarea
            ref={textareaRef}
            value={value}
            onChange={(e) => setValue(e.target.value)}
            onKeyDown={handleKeyDown}
+            onPaste={handlePaste}
            onCompositionStart={() => setIsComposing(true)}
            onCompositionEnd={() => setIsComposing(false)}
            placeholder={placeholder}
@@ -118,21 +243,48 @@ export function MessageInput({
            className="w-full resize-none bg-transparent px-1 py-1 text-sm text-foreground outline-none placeholder:text-muted-foreground disabled:cursor-not-allowed disabled:opacity-50"
          />

+          {/* Hidden file input */}
+          <input
+            ref={fileInputRef}
+            type="file"
+            accept={SUPPORTED_IMAGE_TYPES.join(',')}
+            multiple
+            onChange={handleFileSelect}
+            className="hidden"
+          />
+
          {/* Bottom toolbar */}
          <div className="flex items-center justify-between pt-2">
-            {/* Folder indicator */}
-            <Tooltip>
-              <TooltipTrigger asChild>
-                <button
-                  onClick={onSelectFolder}
-                  className="flex items-center gap-1.5 text-xs text-muted-foreground hover:text-foreground transition-colors px-2 py-1 rounded-md hover:bg-background/50"
-                >
-                  <Folder className="h-3.5 w-3.5" />
-                  <span className="max-w-[150px] truncate">{folderName}</span>
-                </button>
-              </TooltipTrigger>
-              <TooltipContent side="top">Change folder</TooltipContent>
-            </Tooltip>
+            {/* Left side: folder and image button */}
+            <div className="flex items-center gap-1">
+              {/* Folder indicator */}
+              <Tooltip>
+                <TooltipTrigger asChild>
+                  <button
+                    onClick={onSelectFolder}
+                    className="flex items-center gap-1.5 text-xs text-muted-foreground hover:text-foreground transition-colors px-2 py-1 rounded-md hover:bg-background/50"
+                  >
+                    <Folder className="h-3.5 w-3.5" />
+                    <span className="max-w-[150px] truncate">{folderName}</span>
+                  </button>
+                </TooltipTrigger>
+                <TooltipContent side="top">Change folder</TooltipContent>
+              </Tooltip>
+
+              {/* Image upload button */}
+              <Tooltip>
+                <TooltipTrigger asChild>
+                  <button
+                    onClick={() => fileInputRef.current?.click()}
+                    disabled={disabled}
+                    className="flex items-center justify-center text-muted-foreground hover:text-foreground transition-colors p-1.5 rounded-md hover:bg-background/50 disabled:opacity-50 disabled:cursor-not-allowed"
+                  >
+                    <Paperclip className="h-4 w-4" />
+                  </button>
+                </TooltipTrigger>
+                <TooltipContent side="top">Attach image (or paste with Cmd+V)</TooltipContent>
+              </Tooltip>
+            </div>

            {/* Send button */}
            <Tooltip>
--- a/src/renderer/src/hooks/useApp.ts
+++ b/src/renderer/src/hooks/useApp.ts
@@ -7,6 +7,7 @@ import type {
  StoredSessionUpdate,
 } from '../../../shared/types'
 import type { RunningSessionsStatus } from '../../../shared/electron-api'
+import type { MessageContent } from '../../../shared/types/message'
 import { usePermissionStore } from '../stores/permissionStore'
 import { useFileChangeStore } from '../stores/fileChangeStore'

@@ -34,7 +35,7 @@ export interface AppActions {
  clearCurrentSession: () => void

  // Agent actions (per-session)
-  sendPrompt: (content: string) => Promise<void>
+  sendPrompt: (content: MessageContent) => Promise<void>
  cancelRequest: () => Promise<void>

  // UI actions
@@ -258,7 +259,7 @@ export function useApp(): AppState & AppActions {
    setSessionUpdates([])
  }, [])

-  const sendPrompt = useCallback(async (content: string) => {
+  const sendPrompt = useCallback(async (content: MessageContent) => {
    if (!currentSession) {
      setError('No active session')
      return
@@ -275,7 +276,7 @@ export function useApp(): AppState & AppActions {
          sessionId: currentSession.agentSessionId,
          update: {
            sessionUpdate: 'user_message',
-            content: { type: 'text', text: content },
+            content: content, // Now stores full MessageContent array
          },
        },
      } as unknown as StoredSessionUpdate
--- a/src/shared/electron-api.d.ts
+++ b/src/shared/electron-api.d.ts
@@ -8,6 +8,7 @@ import type {
  SessionData,
  ListSessionsOptions,
 } from './types'
+import type { MessageContent } from './types/message'

 export interface AgentMessage {
  sessionId: string
@@ -93,7 +94,7 @@ export interface ElectronAPI {
  getAgentStatus(): Promise<RunningSessionsStatus>

  // Agent communication
-  sendPrompt(sessionId: string, content: string): Promise<{ stopReason: string }>
+  sendPrompt(sessionId: string, content: MessageContent): Promise<{ stopReason: string }>
  cancelRequest(sessionId: string): Promise<{ success: boolean }>

  // Session management (agent starts when session is created)
--- a/src/shared/types/message.ts
+++ b/src/shared/types/message.ts
@@ -0,0 +1,17 @@
+/**
+ * Message content types for multi-modal chat messages
+ */
+
+export interface TextContentItem {
+  type: 'text'
+  text: string
+}
+
+export interface ImageContentItem {
+  type: 'image'
+  data: string // base64 encoded image data
+  mimeType: string // image/png, image/jpeg, image/gif, image/webp
+}
+
+export type MessageContentItem = TextContentItem | ImageContentItem
+export type MessageContent = MessageContentItem[]