highperfocused/multica
1
0
Fork 0
mirror of https://github.com/multica-ai/multica.git synced 2026-07-24 11:10:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(lark): channel-backed Feishu store + fix base64 backfill wrapping

Browse Source 
Cutover step 1 of switching the lark Go code from lark_* onto the channel_*
tables (MUL-3515). Introduces the JSONB config boundary the rest of the
cutover sits on, and fixes a latent backfill bug surfaced while building it.

- migration 123: strip newlines from the app_secret_encrypted base64 backfill.
  PostgreSQL encode(...,'base64') MIME-wraps at 76 chars, and a secretbox-
  sealed ~72-byte secret exceeds that. Go's encoding/json decodes a JSON
  string into []byte with base64.StdEncoding, which rejects embedded newlines,
  so without the strip every migrated installation would fail to decrypt its
  app secret once reads move to channel_installation.config.

- store.go: flat domain types (Installation / UserBinding / ChatSessionBinding)
  with field parity to the retired db.Lark* rows, plus the feishu config codec.
  Row->domain mappers decode the JSONB config; the secret decoder is
  whitespace-tolerant so legacy MIME-wrapped data still round-trips, while the
  encoder emits unwrapped base64. Binding config encodes an absent union_id as
  "{}" so the upsert's jsonb_strip_nulls merge never clobbers a stored union_id.

- store_test.go: 72-byte secret round-trip, MIME-wrapped tolerance, optional
  null-strip, and flat-column preservation. Verified on PG17.

Field parity keeps the upcoming ~190 db.LarkInstallation call sites a
mechanical rename. No call sites switched yet; behavior unchanged.

Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
J
2026-06-23 00:19:00 +08:00
parent c6ab060977
commit 0901e2077d
3 changed files with 460 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
server/internal/integrations/lark/store.go Normal file
View File

@@ -0,0 +1,221 @@
package lark
// Channel-backed store for the Feishu integration.
//
// MUL-3515 generalized the lark_* tables into channel_* (a channel_type
// discriminator + a JSONB `config` blob for the platform-specific
// identifiers/credentials). This file owns the one boundary where that JSONB
// is (de)serialized: the rest of the package keeps working with flat domain
// structs whose fields mirror the retired db.Lark* rows one-for-one, so the
// call sites are a mechanical rename rather than a reshape.
//
// The feishu config blob carries exactly the columns that used to be flat on
// lark_installation / lark_user_binding:
//
// installation: app_id, app_secret_encrypted (base64), tenant_key,
// bot_open_id, bot_union_id, region
// user binding: union_id
//
// app_secret_encrypted is secretbox ciphertext stored as a base64 string.
// The decoder is whitespace-tolerant on purpose: the migration backfill writes
// it via PostgreSQL encode(...,'base64'), which MIME-wraps every 76 chars, and
// a sealed ~72-byte secret exceeds that. The encoder always emits unwrapped
// base64, so rows written by Go are already clean; stripping on read keeps
// both sources interchangeable.
import (
"encoding/base64"
"encoding/json"
"fmt"
"strings"
"github.com/jackc/pgx/v5/pgtype"
db "github.com/multica-ai/multica/server/pkg/db/generated"
)
// Installation is the flat, feishu-shaped view of a channel_installation row.
// Field parity with the retired db.LarkInstallation is intentional: it keeps
// the cutover a rename at the ~190 call sites. The feishu-specific fields
// (AppID, AppSecretEncrypted, TenantKey, BotOpenID, BotUnionID, Region) come
// from the JSONB config; the rest are flat columns.
type Installation struct {
ID pgtype.UUID
WorkspaceID pgtype.UUID
AgentID pgtype.UUID
AppID string
AppSecretEncrypted []byte
TenantKey pgtype.Text
BotOpenID string
InstallerUserID pgtype.UUID
Status string
WsLeaseToken pgtype.Text
WsLeaseExpiresAt pgtype.Timestamptz
InstalledAt pgtype.Timestamptz
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
BotUnionID pgtype.Text
Region string
}
// UserBinding is the flat view of a channel_user_binding row. ChannelUserID is
// the feishu open_id; UnionID (secondary identity) lives in the JSONB config.
type UserBinding struct {
ID pgtype.UUID
WorkspaceID pgtype.UUID
MulticaUserID pgtype.UUID
InstallationID pgtype.UUID
ChannelUserID string
UnionID pgtype.Text
BoundAt pgtype.Timestamptz
}
// ChatSessionBinding is the flat view of a channel_chat_session_binding row.
// Every field is a flat column (config is unused for feishu today), so this is
// a pure copy with no JSON involved.
type ChatSessionBinding struct {
ID pgtype.UUID
ChatSessionID pgtype.UUID
InstallationID pgtype.UUID
ChannelChatID string
ChatType string
CreatedAt pgtype.Timestamptz
LastMessageID pgtype.Text
LastThreadID pgtype.Text
}
// feishuInstallConfig is the JSON shape of channel_installation.config for the
// feishu channel. app_secret_encrypted is decoded by hand (see decodeSecret)
// rather than as a json []byte field, so MIME-wrapped base64 from the SQL
// backfill round-trips too. omitempty mirrors the migration's jsonb_strip_nulls.
type feishuInstallConfig struct {
AppID string `json:"app_id"`
AppSecretEncrypted string `json:"app_secret_encrypted,omitempty"`
TenantKey string `json:"tenant_key,omitempty"`
BotOpenID string `json:"bot_open_id,omitempty"`
BotUnionID string `json:"bot_union_id,omitempty"`
Region string `json:"region,omitempty"`
}
// feishuBindingConfig is the JSON shape of channel_user_binding.config.
type feishuBindingConfig struct {
UnionID string `json:"union_id,omitempty"`
}
// installationFromRow decodes a channel_installation row (flat columns + JSONB
// config) into the flat Installation domain struct.
func installationFromRow(row db.ChannelInstallation) (Installation, error) {
var cfg feishuInstallConfig
if len(row.Config) > 0 {
if err := json.Unmarshal(row.Config, &cfg); err != nil {
return Installation{}, fmt.Errorf("decode installation config: %w", err)
}
}
secret, err := decodeSecret(cfg.AppSecretEncrypted)
if err != nil {
return Installation{}, fmt.Errorf("decode app_secret_encrypted: %w", err)
}
return Installation{
ID: row.ID,
WorkspaceID: row.WorkspaceID,
AgentID: row.AgentID,
AppID: cfg.AppID,
AppSecretEncrypted: secret,
TenantKey: textOrNull(cfg.TenantKey),
BotOpenID: cfg.BotOpenID,
InstallerUserID: row.InstallerUserID,
Status: row.Status,
WsLeaseToken: row.WsLeaseToken,
WsLeaseExpiresAt: row.WsLeaseExpiresAt,
InstalledAt: row.InstalledAt,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
BotUnionID: textOrNull(cfg.BotUnionID),
Region: cfg.Region,
}, nil
}
// encodeInstallConfig builds the channel_installation.config JSONB from the
// feishu fields of an Installation. The secret is emitted as unwrapped base64.
func encodeInstallConfig(inst Installation) ([]byte, error) {
cfg := feishuInstallConfig{
AppID: inst.AppID,
TenantKey: inst.TenantKey.String,
BotOpenID: inst.BotOpenID,
BotUnionID: inst.BotUnionID.String,
Region: inst.Region,
}
if len(inst.AppSecretEncrypted) > 0 {
cfg.AppSecretEncrypted = base64.StdEncoding.EncodeToString(inst.AppSecretEncrypted)
}
return json.Marshal(cfg)
}
// userBindingFromRow decodes a channel_user_binding row into UserBinding.
func userBindingFromRow(row db.ChannelUserBinding) (UserBinding, error) {
var cfg feishuBindingConfig
if len(row.Config) > 0 {
if err := json.Unmarshal(row.Config, &cfg); err != nil {
return UserBinding{}, fmt.Errorf("decode user binding config: %w", err)
}
}
return UserBinding{
ID: row.ID,
WorkspaceID: row.WorkspaceID,
MulticaUserID: row.MulticaUserID,
InstallationID: row.InstallationID,
ChannelUserID: row.ChannelUserID,
UnionID: textOrNull(cfg.UnionID),
BoundAt: row.BoundAt,
}, nil
}
// encodeBindingConfig builds channel_user_binding.config from a UserBinding.
// Returns the JSON null-stripped (an absent union_id is "{}"), so the
// upsert's `config || jsonb_strip_nulls(EXCLUDED.config)` merge never clobbers
// a previously-captured union_id with this write.
func encodeBindingConfig(b UserBinding) ([]byte, error) {
return json.Marshal(feishuBindingConfig{UnionID: b.UnionID.String})
}
// chatSessionBindingFromRow copies a channel_chat_session_binding row into the
// flat domain struct. No JSON: every feishu field is already a flat column.
func chatSessionBindingFromRow(row db.ChannelChatSessionBinding) ChatSessionBinding {
return ChatSessionBinding{
ID: row.ID,
ChatSessionID: row.ChatSessionID,
InstallationID: row.InstallationID,
ChannelChatID: row.ChannelChatID,
ChatType: row.ChatType,
CreatedAt: row.CreatedAt,
LastMessageID: row.LastMessageID,
LastThreadID: row.LastThreadID,
}
}
// decodeSecret base64-decodes the stored app secret ciphertext. It tolerates
// the newline wrapping PostgreSQL's encode(...,'base64') inserts, so secrets
// written by the SQL backfill and by encodeInstallConfig both decode. An empty
// string yields a nil slice (an installation mid-registration before the
// secret is sealed).
func decodeSecret(s string) ([]byte, error) {
if s == "" {
return nil, nil
}
return base64.StdEncoding.DecodeString(stripWhitespace(s))
}
// stripWhitespace removes the ASCII whitespace MIME base64 wrapping introduces.
func stripWhitespace(s string) string {
if !strings.ContainsAny(s, "\n\r \t") {
return s
}
return strings.Map(func(r rune) rune {
switch r {
case '\n', '\r', ' ', '\t':
return -1
default:
return r
}
}, s)
}

231
server/internal/integrations/lark/store_test.go Normal file
View File

@@ -0,0 +1,231 @@
package lark
import (
"bytes"
"encoding/base64"
"encoding/json"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgtype"
db "github.com/multica-ai/multica/server/pkg/db/generated"
)
func uuidFrom(b byte) pgtype.UUID {
var u pgtype.UUID
for i := range u.Bytes {
u.Bytes[i] = b
}
u.Valid = true
return u
}
// sealedSecret returns an n-byte value standing in for a secretbox-sealed app
// secret. A real sealed Lark secret is ~72 bytes, which base64-encodes to >76
// chars and so triggers PostgreSQL's MIME line wrapping.
func sealedSecret(n int) []byte {
b := make([]byte, n)
for i := range b {
b[i] = byte(i % 251)
}
return b
}
// mimeWrap reproduces PostgreSQL encode(...,'base64') line wrapping: a newline
// every 76 characters.
func mimeWrap(s string) string {
var out strings.Builder
for i := 0; i < len(s); i += 76 {
end := i + 76
if end > len(s) {
end = len(s)
}
out.WriteString(s[i:end])
if end < len(s) {
out.WriteByte('\n')
}
}
return out.String()
}
func TestInstallationConfigRoundTrip(t *testing.T) {
secret := sealedSecret(72)
in := Installation{
ID: uuidFrom(0x11),
WorkspaceID: uuidFrom(0x22),
AgentID: uuidFrom(0x33),
AppID: "cli_app_123",
AppSecretEncrypted: secret,
TenantKey: pgtype.Text{String: "tenant_xyz", Valid: true},
BotOpenID: "ou_bot_open",
InstallerUserID: uuidFrom(0x44),
Region: "lark",
BotUnionID: pgtype.Text{String: "on_union_999", Valid: true},
}
cfg, err := encodeInstallConfig(in)
if err != nil {
t.Fatalf("encodeInstallConfig: %v", err)
}
// The encoder must emit unwrapped base64 — embedded newlines are exactly
// what breaks Go's json []byte decode path.
if bytes.ContainsAny(cfg, "\n\r") {
t.Fatalf("encoded config carries wrapped base64 / newlines: %s", cfg)
}
row := db.ChannelInstallation{
ID: in.ID,
WorkspaceID: in.WorkspaceID,
AgentID: in.AgentID,
ChannelType: "feishu",
Config: cfg,
Status: "active",
InstallerUserID: in.InstallerUserID,
WsLeaseToken: pgtype.Text{String: "node-1-g3", Valid: true},
}
got, err := installationFromRow(row)
if err != nil {
t.Fatalf("installationFromRow: %v", err)
}
if !bytes.Equal(got.AppSecretEncrypted, secret) {
t.Fatalf("secret round-trip mismatch:\n got %x\nwant %x", got.AppSecretEncrypted, secret)
}
if got.AppID != in.AppID || got.BotOpenID != in.BotOpenID || got.Region != in.Region {
t.Fatalf("scalar config mismatch: %+v", got)
}
if got.TenantKey != in.TenantKey || got.BotUnionID != in.BotUnionID {
t.Fatalf("optional config mismatch: tenant=%+v union=%+v", got.TenantKey, got.BotUnionID)
}
// Flat columns must come straight from the row, not the config.
if got.Status != "active" || got.WsLeaseToken.String != "node-1-g3" {
t.Fatalf("flat columns not preserved: status=%q lease=%q", got.Status, got.WsLeaseToken.String)
}
if got.ID != in.ID || got.WorkspaceID != in.WorkspaceID || got.InstallerUserID != in.InstallerUserID {
t.Fatalf("flat id columns not preserved: %+v", got)
}
}
// TestInstallationToleratesMimeWrappedSecret simulates the migration backfill:
// channel_installation.config.app_secret_encrypted holds base64 wrapped at 76
// chars (PostgreSQL encode default). Decoding must still recover the bytes.
func TestInstallationToleratesMimeWrappedSecret(t *testing.T) {
secret := sealedSecret(72)
wrapped := mimeWrap(base64.StdEncoding.EncodeToString(secret))
if !strings.Contains(wrapped, "\n") {
t.Fatal("test setup: expected wrapped base64 to contain a newline")
}
cfg, err := json.Marshal(map[string]string{
"app_id": "cli_app_123",
"app_secret_encrypted": wrapped,
"region": "feishu",
})
if err != nil {
t.Fatalf("marshal config: %v", err)
}
got, err := installationFromRow(db.ChannelInstallation{Config: cfg, Status: "active"})
if err != nil {
t.Fatalf("installationFromRow with wrapped secret: %v", err)
}
if !bytes.Equal(got.AppSecretEncrypted, secret) {
t.Fatalf("wrapped secret round-trip mismatch:\n got %x\nwant %x", got.AppSecretEncrypted, secret)
}
}
func TestInstallConfigOmitsEmptyOptionalFields(t *testing.T) {
cfg, err := encodeInstallConfig(Installation{
AppID: "cli_app_123",
BotOpenID: "ou_bot",
Region: "feishu",
// TenantKey, BotUnionID, AppSecretEncrypted all empty.
})
if err != nil {
t.Fatalf("encodeInstallConfig: %v", err)
}
var raw map[string]json.RawMessage
if err := json.Unmarshal(cfg, &raw); err != nil {
t.Fatalf("unmarshal: %v", err)
}
for _, k := range []string{"tenant_key", "bot_union_id", "app_secret_encrypted"} {
if _, ok := raw[k]; ok {
t.Fatalf("expected %q to be omitted, config=%s", k, cfg)
}
}
// And they decode back to the zero/invalid pgtype values.
got, err := installationFromRow(db.ChannelInstallation{Config: cfg})
if err != nil {
t.Fatalf("installationFromRow: %v", err)
}
if got.TenantKey.Valid || got.BotUnionID.Valid || got.AppSecretEncrypted != nil {
t.Fatalf("empty optionals should decode invalid/nil: %+v", got)
}
}
func TestUserBindingConfigRoundTrip(t *testing.T) {
in := UserBinding{
ID: uuidFrom(0x55),
WorkspaceID: uuidFrom(0x22),
MulticaUserID: uuidFrom(0x66),
InstallationID: uuidFrom(0x11),
ChannelUserID: "ou_sender",
UnionID: pgtype.Text{String: "on_union_777", Valid: true},
}
cfg, err := encodeBindingConfig(in)
if err != nil {
t.Fatalf("encodeBindingConfig: %v", err)
}
got, err := userBindingFromRow(db.ChannelUserBinding{
ID: in.ID,
WorkspaceID: in.WorkspaceID,
MulticaUserID: in.MulticaUserID,
InstallationID: in.InstallationID,
ChannelUserID: in.ChannelUserID,
Config: cfg,
})
if err != nil {
t.Fatalf("userBindingFromRow: %v", err)
}
if got.UnionID != in.UnionID || got.ChannelUserID != in.ChannelUserID || got.MulticaUserID != in.MulticaUserID {
t.Fatalf("user binding round-trip mismatch: %+v", got)
}
}
// An absent union_id must serialize to "{}" so the upsert's
// `config || jsonb_strip_nulls(EXCLUDED.config)` merge cannot wipe an
// already-stored union_id when this bind does not carry one.
func TestBindingConfigNullStrip(t *testing.T) {
cfg, err := encodeBindingConfig(UserBinding{ChannelUserID: "ou_sender"})
if err != nil {
t.Fatalf("encodeBindingConfig: %v", err)
}
if got := strings.TrimSpace(string(cfg)); got != "{}" {
t.Fatalf("expected empty union_id to encode as {}, got %q", got)
}
}
func TestChatSessionBindingFromRow(t *testing.T) {
row := db.ChannelChatSessionBinding{
ID: uuidFrom(0x77),
ChatSessionID: uuidFrom(0x88),
InstallationID: uuidFrom(0x11),
ChannelType: "feishu",
ChannelChatID: "oc_chat_1",
ChatType: "group",
LastMessageID: pgtype.Text{String: "om_last", Valid: true},
LastThreadID: pgtype.Text{String: "omt_thread", Valid: true},
}
got := chatSessionBindingFromRow(row)
if got.ChannelChatID != "oc_chat_1" || got.ChatType != "group" {
t.Fatalf("chat fields mismatch: %+v", got)
}
if got.LastMessageID != row.LastMessageID || got.LastThreadID != row.LastThreadID {
t.Fatalf("reply-target fields mismatch: %+v", got)
}
if got.ChatSessionID != row.ChatSessionID || got.InstallationID != row.InstallationID {
t.Fatalf("id fields mismatch: %+v", got)
}
}

12
server/migrations/123_channel_generalization.up.sql
View File

@@ -21,9 +21,13 @@
-- copies the data forward.
--
-- app_secret_encrypted is BYTEA; it is carried into the JSONB config as a
-- base64 string (encode(...,'base64')). Go's encoding/json decodes a
-- base64 string straight back into a []byte field, so the round-trip is
-- symmetric and the ciphertext is never stored in plaintext.
-- base64 string. PostgreSQL's encode(...,'base64') MIME-wraps the output
-- with a newline every 76 chars, and a secretbox-sealed app secret (~72
-- bytes) exceeds that, so we strip the newlines: Go's encoding/json decodes
-- a base64 string into a []byte field with base64.StdEncoding, which rejects
-- embedded newlines. Stripping keeps the bytea -> JSON -> []byte round-trip
-- symmetric (the Go writer emits unwrapped base64 too) and the ciphertext is
-- never stored in plaintext.
-- =====================
-- channel_installation
@@ -78,7 +82,7 @@ SELECT
id, workspace_id, agent_id, 'feishu',
jsonb_strip_nulls(jsonb_build_object(
'app_id', app_id,
'app_secret_encrypted', encode(app_secret_encrypted, 'base64'),
'app_secret_encrypted', replace(encode(app_secret_encrypted, 'base64'), E'\n', ''),
'tenant_key', tenant_key,
'bot_open_id', bot_open_id,
'bot_union_id', bot_union_id,