From 0901e2077df5fe474fb261433d596a2ddd944a4f Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Jun 2026 00:19:00 +0800 Subject: [PATCH] feat(lark): channel-backed Feishu store + fix base64 backfill wrapping Cutover step 1 of switching the lark Go code from lark_* onto the channel_* tables (MUL-3515). Introduces the JSONB config boundary the rest of the cutover sits on, and fixes a latent backfill bug surfaced while building it. - migration 123: strip newlines from the app_secret_encrypted base64 backfill. PostgreSQL encode(...,'base64') MIME-wraps at 76 chars, and a secretbox- sealed ~72-byte secret exceeds that. Go's encoding/json decodes a JSON string into []byte with base64.StdEncoding, which rejects embedded newlines, so without the strip every migrated installation would fail to decrypt its app secret once reads move to channel_installation.config. - store.go: flat domain types (Installation / UserBinding / ChatSessionBinding) with field parity to the retired db.Lark* rows, plus the feishu config codec. Row->domain mappers decode the JSONB config; the secret decoder is whitespace-tolerant so legacy MIME-wrapped data still round-trips, while the encoder emits unwrapped base64. Binding config encodes an absent union_id as "{}" so the upsert's jsonb_strip_nulls merge never clobbers a stored union_id. - store_test.go: 72-byte secret round-trip, MIME-wrapped tolerance, optional null-strip, and flat-column preservation. Verified on PG17. Field parity keeps the upcoming ~190 db.LarkInstallation call sites a mechanical rename. No call sites switched yet; behavior unchanged. Co-authored-by: multica-agent --- server/internal/integrations/lark/store.go | 221 +++++++++++++++++ .../internal/integrations/lark/store_test.go | 231 ++++++++++++++++++ .../123_channel_generalization.up.sql | 12 +- 3 files changed, 460 insertions(+), 4 deletions(-) create mode 100644 server/internal/integrations/lark/store.go create mode 100644 server/internal/integrations/lark/store_test.go diff --git a/server/internal/integrations/lark/store.go b/server/internal/integrations/lark/store.go new file mode 100644 index 0000000000..a912ee4581 --- /dev/null +++ b/server/internal/integrations/lark/store.go @@ -0,0 +1,221 @@ +package lark + +// Channel-backed store for the Feishu integration. +// +// MUL-3515 generalized the lark_* tables into channel_* (a channel_type +// discriminator + a JSONB `config` blob for the platform-specific +// identifiers/credentials). This file owns the one boundary where that JSONB +// is (de)serialized: the rest of the package keeps working with flat domain +// structs whose fields mirror the retired db.Lark* rows one-for-one, so the +// call sites are a mechanical rename rather than a reshape. +// +// The feishu config blob carries exactly the columns that used to be flat on +// lark_installation / lark_user_binding: +// +// installation: app_id, app_secret_encrypted (base64), tenant_key, +// bot_open_id, bot_union_id, region +// user binding: union_id +// +// app_secret_encrypted is secretbox ciphertext stored as a base64 string. +// The decoder is whitespace-tolerant on purpose: the migration backfill writes +// it via PostgreSQL encode(...,'base64'), which MIME-wraps every 76 chars, and +// a sealed ~72-byte secret exceeds that. The encoder always emits unwrapped +// base64, so rows written by Go are already clean; stripping on read keeps +// both sources interchangeable. + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgtype" + + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// Installation is the flat, feishu-shaped view of a channel_installation row. +// Field parity with the retired db.LarkInstallation is intentional: it keeps +// the cutover a rename at the ~190 call sites. The feishu-specific fields +// (AppID, AppSecretEncrypted, TenantKey, BotOpenID, BotUnionID, Region) come +// from the JSONB config; the rest are flat columns. +type Installation struct { + ID pgtype.UUID + WorkspaceID pgtype.UUID + AgentID pgtype.UUID + AppID string + AppSecretEncrypted []byte + TenantKey pgtype.Text + BotOpenID string + InstallerUserID pgtype.UUID + Status string + WsLeaseToken pgtype.Text + WsLeaseExpiresAt pgtype.Timestamptz + InstalledAt pgtype.Timestamptz + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz + BotUnionID pgtype.Text + Region string +} + +// UserBinding is the flat view of a channel_user_binding row. ChannelUserID is +// the feishu open_id; UnionID (secondary identity) lives in the JSONB config. +type UserBinding struct { + ID pgtype.UUID + WorkspaceID pgtype.UUID + MulticaUserID pgtype.UUID + InstallationID pgtype.UUID + ChannelUserID string + UnionID pgtype.Text + BoundAt pgtype.Timestamptz +} + +// ChatSessionBinding is the flat view of a channel_chat_session_binding row. +// Every field is a flat column (config is unused for feishu today), so this is +// a pure copy with no JSON involved. +type ChatSessionBinding struct { + ID pgtype.UUID + ChatSessionID pgtype.UUID + InstallationID pgtype.UUID + ChannelChatID string + ChatType string + CreatedAt pgtype.Timestamptz + LastMessageID pgtype.Text + LastThreadID pgtype.Text +} + +// feishuInstallConfig is the JSON shape of channel_installation.config for the +// feishu channel. app_secret_encrypted is decoded by hand (see decodeSecret) +// rather than as a json []byte field, so MIME-wrapped base64 from the SQL +// backfill round-trips too. omitempty mirrors the migration's jsonb_strip_nulls. +type feishuInstallConfig struct { + AppID string `json:"app_id"` + AppSecretEncrypted string `json:"app_secret_encrypted,omitempty"` + TenantKey string `json:"tenant_key,omitempty"` + BotOpenID string `json:"bot_open_id,omitempty"` + BotUnionID string `json:"bot_union_id,omitempty"` + Region string `json:"region,omitempty"` +} + +// feishuBindingConfig is the JSON shape of channel_user_binding.config. +type feishuBindingConfig struct { + UnionID string `json:"union_id,omitempty"` +} + +// installationFromRow decodes a channel_installation row (flat columns + JSONB +// config) into the flat Installation domain struct. +func installationFromRow(row db.ChannelInstallation) (Installation, error) { + var cfg feishuInstallConfig + if len(row.Config) > 0 { + if err := json.Unmarshal(row.Config, &cfg); err != nil { + return Installation{}, fmt.Errorf("decode installation config: %w", err) + } + } + secret, err := decodeSecret(cfg.AppSecretEncrypted) + if err != nil { + return Installation{}, fmt.Errorf("decode app_secret_encrypted: %w", err) + } + return Installation{ + ID: row.ID, + WorkspaceID: row.WorkspaceID, + AgentID: row.AgentID, + AppID: cfg.AppID, + AppSecretEncrypted: secret, + TenantKey: textOrNull(cfg.TenantKey), + BotOpenID: cfg.BotOpenID, + InstallerUserID: row.InstallerUserID, + Status: row.Status, + WsLeaseToken: row.WsLeaseToken, + WsLeaseExpiresAt: row.WsLeaseExpiresAt, + InstalledAt: row.InstalledAt, + CreatedAt: row.CreatedAt, + UpdatedAt: row.UpdatedAt, + BotUnionID: textOrNull(cfg.BotUnionID), + Region: cfg.Region, + }, nil +} + +// encodeInstallConfig builds the channel_installation.config JSONB from the +// feishu fields of an Installation. The secret is emitted as unwrapped base64. +func encodeInstallConfig(inst Installation) ([]byte, error) { + cfg := feishuInstallConfig{ + AppID: inst.AppID, + TenantKey: inst.TenantKey.String, + BotOpenID: inst.BotOpenID, + BotUnionID: inst.BotUnionID.String, + Region: inst.Region, + } + if len(inst.AppSecretEncrypted) > 0 { + cfg.AppSecretEncrypted = base64.StdEncoding.EncodeToString(inst.AppSecretEncrypted) + } + return json.Marshal(cfg) +} + +// userBindingFromRow decodes a channel_user_binding row into UserBinding. +func userBindingFromRow(row db.ChannelUserBinding) (UserBinding, error) { + var cfg feishuBindingConfig + if len(row.Config) > 0 { + if err := json.Unmarshal(row.Config, &cfg); err != nil { + return UserBinding{}, fmt.Errorf("decode user binding config: %w", err) + } + } + return UserBinding{ + ID: row.ID, + WorkspaceID: row.WorkspaceID, + MulticaUserID: row.MulticaUserID, + InstallationID: row.InstallationID, + ChannelUserID: row.ChannelUserID, + UnionID: textOrNull(cfg.UnionID), + BoundAt: row.BoundAt, + }, nil +} + +// encodeBindingConfig builds channel_user_binding.config from a UserBinding. +// Returns the JSON null-stripped (an absent union_id is "{}"), so the +// upsert's `config || jsonb_strip_nulls(EXCLUDED.config)` merge never clobbers +// a previously-captured union_id with this write. +func encodeBindingConfig(b UserBinding) ([]byte, error) { + return json.Marshal(feishuBindingConfig{UnionID: b.UnionID.String}) +} + +// chatSessionBindingFromRow copies a channel_chat_session_binding row into the +// flat domain struct. No JSON: every feishu field is already a flat column. +func chatSessionBindingFromRow(row db.ChannelChatSessionBinding) ChatSessionBinding { + return ChatSessionBinding{ + ID: row.ID, + ChatSessionID: row.ChatSessionID, + InstallationID: row.InstallationID, + ChannelChatID: row.ChannelChatID, + ChatType: row.ChatType, + CreatedAt: row.CreatedAt, + LastMessageID: row.LastMessageID, + LastThreadID: row.LastThreadID, + } +} + +// decodeSecret base64-decodes the stored app secret ciphertext. It tolerates +// the newline wrapping PostgreSQL's encode(...,'base64') inserts, so secrets +// written by the SQL backfill and by encodeInstallConfig both decode. An empty +// string yields a nil slice (an installation mid-registration before the +// secret is sealed). +func decodeSecret(s string) ([]byte, error) { + if s == "" { + return nil, nil + } + return base64.StdEncoding.DecodeString(stripWhitespace(s)) +} + +// stripWhitespace removes the ASCII whitespace MIME base64 wrapping introduces. +func stripWhitespace(s string) string { + if !strings.ContainsAny(s, "\n\r \t") { + return s + } + return strings.Map(func(r rune) rune { + switch r { + case '\n', '\r', ' ', '\t': + return -1 + default: + return r + } + }, s) +} diff --git a/server/internal/integrations/lark/store_test.go b/server/internal/integrations/lark/store_test.go new file mode 100644 index 0000000000..00d8473a45 --- /dev/null +++ b/server/internal/integrations/lark/store_test.go @@ -0,0 +1,231 @@ +package lark + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +func uuidFrom(b byte) pgtype.UUID { + var u pgtype.UUID + for i := range u.Bytes { + u.Bytes[i] = b + } + u.Valid = true + return u +} + +// sealedSecret returns an n-byte value standing in for a secretbox-sealed app +// secret. A real sealed Lark secret is ~72 bytes, which base64-encodes to >76 +// chars and so triggers PostgreSQL's MIME line wrapping. +func sealedSecret(n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = byte(i % 251) + } + return b +} + +// mimeWrap reproduces PostgreSQL encode(...,'base64') line wrapping: a newline +// every 76 characters. +func mimeWrap(s string) string { + var out strings.Builder + for i := 0; i < len(s); i += 76 { + end := i + 76 + if end > len(s) { + end = len(s) + } + out.WriteString(s[i:end]) + if end < len(s) { + out.WriteByte('\n') + } + } + return out.String() +} + +func TestInstallationConfigRoundTrip(t *testing.T) { + secret := sealedSecret(72) + in := Installation{ + ID: uuidFrom(0x11), + WorkspaceID: uuidFrom(0x22), + AgentID: uuidFrom(0x33), + AppID: "cli_app_123", + AppSecretEncrypted: secret, + TenantKey: pgtype.Text{String: "tenant_xyz", Valid: true}, + BotOpenID: "ou_bot_open", + InstallerUserID: uuidFrom(0x44), + Region: "lark", + BotUnionID: pgtype.Text{String: "on_union_999", Valid: true}, + } + + cfg, err := encodeInstallConfig(in) + if err != nil { + t.Fatalf("encodeInstallConfig: %v", err) + } + + // The encoder must emit unwrapped base64 — embedded newlines are exactly + // what breaks Go's json []byte decode path. + if bytes.ContainsAny(cfg, "\n\r") { + t.Fatalf("encoded config carries wrapped base64 / newlines: %s", cfg) + } + + row := db.ChannelInstallation{ + ID: in.ID, + WorkspaceID: in.WorkspaceID, + AgentID: in.AgentID, + ChannelType: "feishu", + Config: cfg, + Status: "active", + InstallerUserID: in.InstallerUserID, + WsLeaseToken: pgtype.Text{String: "node-1-g3", Valid: true}, + } + + got, err := installationFromRow(row) + if err != nil { + t.Fatalf("installationFromRow: %v", err) + } + if !bytes.Equal(got.AppSecretEncrypted, secret) { + t.Fatalf("secret round-trip mismatch:\n got %x\nwant %x", got.AppSecretEncrypted, secret) + } + if got.AppID != in.AppID || got.BotOpenID != in.BotOpenID || got.Region != in.Region { + t.Fatalf("scalar config mismatch: %+v", got) + } + if got.TenantKey != in.TenantKey || got.BotUnionID != in.BotUnionID { + t.Fatalf("optional config mismatch: tenant=%+v union=%+v", got.TenantKey, got.BotUnionID) + } + // Flat columns must come straight from the row, not the config. + if got.Status != "active" || got.WsLeaseToken.String != "node-1-g3" { + t.Fatalf("flat columns not preserved: status=%q lease=%q", got.Status, got.WsLeaseToken.String) + } + if got.ID != in.ID || got.WorkspaceID != in.WorkspaceID || got.InstallerUserID != in.InstallerUserID { + t.Fatalf("flat id columns not preserved: %+v", got) + } +} + +// TestInstallationToleratesMimeWrappedSecret simulates the migration backfill: +// channel_installation.config.app_secret_encrypted holds base64 wrapped at 76 +// chars (PostgreSQL encode default). Decoding must still recover the bytes. +func TestInstallationToleratesMimeWrappedSecret(t *testing.T) { + secret := sealedSecret(72) + wrapped := mimeWrap(base64.StdEncoding.EncodeToString(secret)) + if !strings.Contains(wrapped, "\n") { + t.Fatal("test setup: expected wrapped base64 to contain a newline") + } + + cfg, err := json.Marshal(map[string]string{ + "app_id": "cli_app_123", + "app_secret_encrypted": wrapped, + "region": "feishu", + }) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + + got, err := installationFromRow(db.ChannelInstallation{Config: cfg, Status: "active"}) + if err != nil { + t.Fatalf("installationFromRow with wrapped secret: %v", err) + } + if !bytes.Equal(got.AppSecretEncrypted, secret) { + t.Fatalf("wrapped secret round-trip mismatch:\n got %x\nwant %x", got.AppSecretEncrypted, secret) + } +} + +func TestInstallConfigOmitsEmptyOptionalFields(t *testing.T) { + cfg, err := encodeInstallConfig(Installation{ + AppID: "cli_app_123", + BotOpenID: "ou_bot", + Region: "feishu", + // TenantKey, BotUnionID, AppSecretEncrypted all empty. + }) + if err != nil { + t.Fatalf("encodeInstallConfig: %v", err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(cfg, &raw); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, k := range []string{"tenant_key", "bot_union_id", "app_secret_encrypted"} { + if _, ok := raw[k]; ok { + t.Fatalf("expected %q to be omitted, config=%s", k, cfg) + } + } + // And they decode back to the zero/invalid pgtype values. + got, err := installationFromRow(db.ChannelInstallation{Config: cfg}) + if err != nil { + t.Fatalf("installationFromRow: %v", err) + } + if got.TenantKey.Valid || got.BotUnionID.Valid || got.AppSecretEncrypted != nil { + t.Fatalf("empty optionals should decode invalid/nil: %+v", got) + } +} + +func TestUserBindingConfigRoundTrip(t *testing.T) { + in := UserBinding{ + ID: uuidFrom(0x55), + WorkspaceID: uuidFrom(0x22), + MulticaUserID: uuidFrom(0x66), + InstallationID: uuidFrom(0x11), + ChannelUserID: "ou_sender", + UnionID: pgtype.Text{String: "on_union_777", Valid: true}, + } + cfg, err := encodeBindingConfig(in) + if err != nil { + t.Fatalf("encodeBindingConfig: %v", err) + } + got, err := userBindingFromRow(db.ChannelUserBinding{ + ID: in.ID, + WorkspaceID: in.WorkspaceID, + MulticaUserID: in.MulticaUserID, + InstallationID: in.InstallationID, + ChannelUserID: in.ChannelUserID, + Config: cfg, + }) + if err != nil { + t.Fatalf("userBindingFromRow: %v", err) + } + if got.UnionID != in.UnionID || got.ChannelUserID != in.ChannelUserID || got.MulticaUserID != in.MulticaUserID { + t.Fatalf("user binding round-trip mismatch: %+v", got) + } +} + +// An absent union_id must serialize to "{}" so the upsert's +// `config || jsonb_strip_nulls(EXCLUDED.config)` merge cannot wipe an +// already-stored union_id when this bind does not carry one. +func TestBindingConfigNullStrip(t *testing.T) { + cfg, err := encodeBindingConfig(UserBinding{ChannelUserID: "ou_sender"}) + if err != nil { + t.Fatalf("encodeBindingConfig: %v", err) + } + if got := strings.TrimSpace(string(cfg)); got != "{}" { + t.Fatalf("expected empty union_id to encode as {}, got %q", got) + } +} + +func TestChatSessionBindingFromRow(t *testing.T) { + row := db.ChannelChatSessionBinding{ + ID: uuidFrom(0x77), + ChatSessionID: uuidFrom(0x88), + InstallationID: uuidFrom(0x11), + ChannelType: "feishu", + ChannelChatID: "oc_chat_1", + ChatType: "group", + LastMessageID: pgtype.Text{String: "om_last", Valid: true}, + LastThreadID: pgtype.Text{String: "omt_thread", Valid: true}, + } + got := chatSessionBindingFromRow(row) + if got.ChannelChatID != "oc_chat_1" || got.ChatType != "group" { + t.Fatalf("chat fields mismatch: %+v", got) + } + if got.LastMessageID != row.LastMessageID || got.LastThreadID != row.LastThreadID { + t.Fatalf("reply-target fields mismatch: %+v", got) + } + if got.ChatSessionID != row.ChatSessionID || got.InstallationID != row.InstallationID { + t.Fatalf("id fields mismatch: %+v", got) + } +} diff --git a/server/migrations/123_channel_generalization.up.sql b/server/migrations/123_channel_generalization.up.sql index 4fbd5192f2..54b3fdc812 100644 --- a/server/migrations/123_channel_generalization.up.sql +++ b/server/migrations/123_channel_generalization.up.sql @@ -21,9 +21,13 @@ -- copies the data forward. -- -- app_secret_encrypted is BYTEA; it is carried into the JSONB config as a --- base64 string (encode(...,'base64')). Go's encoding/json decodes a --- base64 string straight back into a []byte field, so the round-trip is --- symmetric and the ciphertext is never stored in plaintext. +-- base64 string. PostgreSQL's encode(...,'base64') MIME-wraps the output +-- with a newline every 76 chars, and a secretbox-sealed app secret (~72 +-- bytes) exceeds that, so we strip the newlines: Go's encoding/json decodes +-- a base64 string into a []byte field with base64.StdEncoding, which rejects +-- embedded newlines. Stripping keeps the bytea -> JSON -> []byte round-trip +-- symmetric (the Go writer emits unwrapped base64 too) and the ciphertext is +-- never stored in plaintext. -- ===================== -- channel_installation @@ -78,7 +82,7 @@ SELECT id, workspace_id, agent_id, 'feishu', jsonb_strip_nulls(jsonb_build_object( 'app_id', app_id, - 'app_secret_encrypted', encode(app_secret_encrypted, 'base64'), + 'app_secret_encrypted', replace(encode(app_secret_encrypted, 'base64'), E'\n', ''), 'tenant_key', tenant_key, 'bot_open_id', bot_open_id, 'bot_union_id', bot_union_id,