diff --git a/skills/whisper/SKILL.md b/skills/whisper/SKILL.md index 7edbb77260..1748a79d15 100644 --- a/skills/whisper/SKILL.md +++ b/skills/whisper/SKILL.md @@ -1,10 +1,9 @@ --- name: Audio Transcription -description: Transcribe audio files using OpenAI Whisper CLI +description: Transcribe audio files using local Whisper CLI (fallback when API is unavailable) version: 1.0.0 metadata: emoji: "🎙️" - always: true requires: anyBins: - whisper @@ -16,11 +15,6 @@ metadata: bins: [whisper] label: "Install OpenAI Whisper via Homebrew" os: [darwin] - - id: pip-whisper - kind: uv - package: openai-whisper - bins: [whisper] - label: "Install OpenAI Whisper via pip/uv" tags: - audio - transcription @@ -29,26 +23,14 @@ userInvocable: false disableModelInvocation: false --- -## Audio Transcription +## Audio Transcription (Local Fallback) -When you receive a message indicating an audio or voice message file (e.g., `[audio message received]` with a `File:` path), you should transcribe it. +Voice messages from channels are normally transcribed automatically via the OpenAI Whisper API before reaching you. This skill is only needed when the API is unavailable. -### How to Transcribe - -Run the following command using the `exec` tool: +If you receive `[audio message received]` with a `File:` path (instead of `[Voice Message]` with a transcript), it means the API transcription was not available. Use local whisper to transcribe: ``` -whisper "" --model turbo --output_format txt --output_dir /tmp +whisper "" --model base --output_format txt --output_dir /tmp ``` -Then read the resulting `.txt` file (same name as input, in `/tmp/`) to get the transcript. - -### Response Format - -After transcription, respond naturally based on the transcribed content. If the user said something in the voice message, respond to it as if they had typed it. - -If transcription fails, let the user know and suggest they check their Whisper installation. - -### Supported Formats - -Whisper supports: mp3, mp4, mpeg, mpga, m4a, wav, webm, ogg, oga, flac +Then read the `.txt` file from `/tmp/` and respond based on the transcribed content. diff --git a/src/channels/manager.ts b/src/channels/manager.ts index 8ecd1da78f..4250b02466 100644 --- a/src/channels/manager.ts +++ b/src/channels/manager.ts @@ -20,6 +20,7 @@ import { listChannels } from "./registry.js"; import { loadChannelsConfig } from "./config.js"; import { MessageAggregator, DEFAULT_CHUNKER_CONFIG } from "../hub/message-aggregator.js"; import type { AsyncAgent } from "../agent/async-agent.js"; +import { transcribeAudio } from "../media/transcribe.js"; interface AccountHandle { channelId: string; @@ -275,7 +276,7 @@ export class ChannelManager { } } - /** Download media file and forward to agent */ + /** Download media file, process it, and forward result to agent */ private async routeMedia( plugin: ChannelPlugin, accountId: string, @@ -294,8 +295,23 @@ export class ChannelManager { const mimeType = media.mimeType ?? "image/jpeg"; const caption = media.caption || "User sent an image."; agent.writeWithImages(caption, [{ type: "image", data: base64, mimeType }]); + } else if (media.type === "audio") { + // Audio: transcribe via Whisper API before reaching agent + const transcript = await transcribeAudio(filePath); + if (transcript) { + const parts = ["[Voice Message]", `Transcript: ${transcript}`]; + if (media.caption) parts.push(`Caption: ${media.caption}`); + agent.write(parts.join("\n")); + } else { + // No API key configured — fall back to file path + const parts = ["[audio message received]", `File: ${filePath}`]; + if (media.mimeType) parts.push(`Type: ${media.mimeType}`); + if (media.duration) parts.push(`Duration: ${media.duration}s`); + if (media.caption) parts.push(`Caption: ${media.caption}`); + agent.write(parts.join("\n")); + } } else { - // Audio/video/document: tell agent the file path, let it handle via skills + // Video/document: tell agent the file path const parts: string[] = []; parts.push(`[${media.type} message received]`); parts.push(`File: ${filePath}`); @@ -306,9 +322,8 @@ export class ChannelManager { } } catch (err) { const msg = err instanceof Error ? err.message : String(err); - console.error(`[Channels] Failed to download media: ${msg}`); - // Fallback: send text-only if download fails - agent.write(message.text || `[Failed to download ${media.type}]`); + console.error(`[Channels] Failed to process media: ${msg}`); + agent.write(message.text || `[Failed to process ${media.type}]`); } } diff --git a/src/media/transcribe.ts b/src/media/transcribe.ts new file mode 100644 index 0000000000..d301a4e5f5 --- /dev/null +++ b/src/media/transcribe.ts @@ -0,0 +1,63 @@ +/** + * Audio transcription via OpenAI Whisper API. + * + * Called by ChannelManager before the message reaches the Agent, + * so the Agent only ever sees text. + */ + +import { readFile } from "node:fs/promises"; +import { basename } from "node:path"; +import { credentialManager } from "../agent/credentials.js"; + +/** + * Transcribe an audio file using OpenAI Whisper API. + * + * @param filePath - Local path to the audio file + * @returns Transcribed text, or null if no API key configured + */ +export async function transcribeAudio(filePath: string): Promise { + const config = credentialManager.getLlmProviderConfig("openai"); + const apiKey = config?.apiKey; + if (!apiKey) return null; + + const fileBuffer = await readFile(filePath); + const fileName = basename(filePath); + + // Build multipart form data manually (no external dependency) + const boundary = `----FormBoundary${Date.now()}`; + const parts: Buffer[] = []; + + // file field + parts.push(Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${fileName}"\r\nContent-Type: application/octet-stream\r\n\r\n`, + )); + parts.push(fileBuffer); + parts.push(Buffer.from("\r\n")); + + // model field + parts.push(Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\nwhisper-1\r\n`, + )); + + // closing boundary + parts.push(Buffer.from(`--${boundary}--\r\n`)); + + const body = Buffer.concat(parts); + + const res = await fetch("https://api.openai.com/v1/audio/transcriptions", { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": `multipart/form-data; boundary=${boundary}`, + }, + body, + }); + + if (!res.ok) { + const errText = await res.text().catch(() => ""); + throw new Error(`Whisper API error: HTTP ${res.status} ${errText}`); + } + + const result = (await res.json()) as { text: string }; + return result.text; +}