mirror of
https://github.com/multica-ai/multica.git
synced 2026-08-03 11:10:23 +02:00
* fix(storage): support custom S3 endpoints for self-hosted deployments When AWS_ENDPOINT_URL is set, the S3 client now uses path-style addressing and routes requests to the custom endpoint (e.g. MinIO). Returns path-style URLs (endpoint/bucket/key) instead of virtual-hosted URLs so attachments are accessible on local setups. Also falls back to STANDARD storage class for custom endpoints since MinIO and other S3-compatible stores do not support INTELLIGENT_TIERING. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix(storage): handle custom endpoint URLs in KeyFromURL --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
193 lines
5.5 KiB
Go
193 lines
5.5 KiB
Go
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/config"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
)
|
|
|
|
type S3Storage struct {
|
|
client *s3.Client
|
|
bucket string
|
|
cdnDomain string // if set, returned URLs use this instead of bucket name
|
|
endpointURL string // if set, use path-style URLs (e.g. MinIO)
|
|
}
|
|
|
|
// NewS3StorageFromEnv creates an S3Storage from environment variables.
|
|
// Returns nil if S3_BUCKET is not set.
|
|
//
|
|
// Environment variables:
|
|
// - S3_BUCKET (required)
|
|
// - S3_REGION (default: us-west-2)
|
|
// - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (optional; falls back to default credential chain)
|
|
func NewS3StorageFromEnv() *S3Storage {
|
|
bucket := os.Getenv("S3_BUCKET")
|
|
if bucket == "" {
|
|
slog.Info("S3_BUCKET not set, file upload disabled")
|
|
return nil
|
|
}
|
|
|
|
region := os.Getenv("S3_REGION")
|
|
if region == "" {
|
|
region = "us-west-2"
|
|
}
|
|
|
|
opts := []func(*config.LoadOptions) error{
|
|
config.WithRegion(region),
|
|
}
|
|
|
|
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
|
|
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
|
|
if accessKey != "" && secretKey != "" {
|
|
opts = append(opts, config.WithCredentialsProvider(
|
|
credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
|
))
|
|
}
|
|
|
|
cfg, err := config.LoadDefaultConfig(context.Background(), opts...)
|
|
if err != nil {
|
|
slog.Error("failed to load AWS config", "error", err)
|
|
return nil
|
|
}
|
|
|
|
cdnDomain := os.Getenv("CLOUDFRONT_DOMAIN")
|
|
|
|
endpointURL := os.Getenv("AWS_ENDPOINT_URL")
|
|
s3Opts := []func(*s3.Options){}
|
|
if endpointURL != "" {
|
|
s3Opts = append(s3Opts, func(o *s3.Options) {
|
|
o.BaseEndpoint = aws.String(endpointURL)
|
|
o.UsePathStyle = true
|
|
})
|
|
}
|
|
|
|
slog.Info("S3 storage initialized", "bucket", bucket, "region", region, "cdn_domain", cdnDomain, "endpoint_url", endpointURL)
|
|
return &S3Storage{
|
|
client: s3.NewFromConfig(cfg, s3Opts...),
|
|
bucket: bucket,
|
|
cdnDomain: cdnDomain,
|
|
endpointURL: endpointURL,
|
|
}
|
|
}
|
|
|
|
// storageClass returns the appropriate S3 storage class.
|
|
// Custom endpoints (e.g. MinIO) only support STANDARD; real AWS defaults to INTELLIGENT_TIERING.
|
|
func (s *S3Storage) storageClass() types.StorageClass {
|
|
if s.endpointURL != "" {
|
|
return types.StorageClassStandard
|
|
}
|
|
return types.StorageClassIntelligentTiering
|
|
}
|
|
|
|
// sanitizeFilename removes characters that could cause header injection in Content-Disposition.
|
|
func sanitizeFilename(name string) string {
|
|
var b strings.Builder
|
|
b.Grow(len(name))
|
|
for _, r := range name {
|
|
// Strip control chars, newlines, null bytes, quotes, semicolons, backslashes
|
|
if r < 0x20 || r == 0x7f || r == '"' || r == ';' || r == '\\' || r == '\x00' {
|
|
b.WriteRune('_')
|
|
} else {
|
|
b.WriteRune(r)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// KeyFromURL extracts the S3 object key from a CDN or bucket URL.
|
|
// e.g. "https://multica-static.copilothub.ai/abc123.png" → "abc123.png"
|
|
func (s *S3Storage) KeyFromURL(rawURL string) string {
|
|
if s.endpointURL != "" {
|
|
prefix := strings.TrimRight(s.endpointURL, "/") + "/" + s.bucket + "/"
|
|
if strings.HasPrefix(rawURL, prefix) {
|
|
return strings.TrimPrefix(rawURL, prefix)
|
|
}
|
|
}
|
|
|
|
// Strip the "https://domain/" prefix.
|
|
for _, prefix := range []string{
|
|
"https://" + s.cdnDomain + "/",
|
|
"https://" + s.bucket + "/",
|
|
} {
|
|
if strings.HasPrefix(rawURL, prefix) {
|
|
return strings.TrimPrefix(rawURL, prefix)
|
|
}
|
|
}
|
|
// Fallback: take everything after the last "/".
|
|
if i := strings.LastIndex(rawURL, "/"); i >= 0 {
|
|
return rawURL[i+1:]
|
|
}
|
|
return rawURL
|
|
}
|
|
|
|
// Delete removes an object from S3. Errors are logged but not fatal.
|
|
func (s *S3Storage) Delete(ctx context.Context, key string) {
|
|
if key == "" {
|
|
return
|
|
}
|
|
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
if err != nil {
|
|
slog.Error("s3 DeleteObject failed", "key", key, "error", err)
|
|
}
|
|
}
|
|
|
|
// DeleteKeys removes multiple objects from S3. Best-effort, errors are logged.
|
|
func (s *S3Storage) DeleteKeys(ctx context.Context, keys []string) {
|
|
for _, key := range keys {
|
|
s.Delete(ctx, key)
|
|
}
|
|
}
|
|
|
|
// isInlineContentType returns true for media types that browsers should
|
|
// display inline (images, video, audio, PDF). Everything else triggers a
|
|
// download via Content-Disposition: attachment.
|
|
func isInlineContentType(ct string) bool {
|
|
return strings.HasPrefix(ct, "image/") ||
|
|
strings.HasPrefix(ct, "video/") ||
|
|
strings.HasPrefix(ct, "audio/") ||
|
|
ct == "application/pdf"
|
|
}
|
|
|
|
func (s *S3Storage) Upload(ctx context.Context, key string, data []byte, contentType string, filename string) (string, error) {
|
|
safe := sanitizeFilename(filename)
|
|
disposition := "attachment"
|
|
if isInlineContentType(contentType) {
|
|
disposition = "inline"
|
|
}
|
|
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
Body: bytes.NewReader(data),
|
|
ContentType: aws.String(contentType),
|
|
ContentDisposition: aws.String(fmt.Sprintf(`%s; filename="%s"`, disposition, safe)),
|
|
CacheControl: aws.String("max-age=432000,public"),
|
|
StorageClass: s.storageClass(),
|
|
})
|
|
if err != nil {
|
|
return "", fmt.Errorf("s3 PutObject: %w", err)
|
|
}
|
|
|
|
if s.endpointURL != "" {
|
|
link := fmt.Sprintf("%s/%s/%s", strings.TrimRight(s.endpointURL, "/"), s.bucket, key)
|
|
return link, nil
|
|
}
|
|
domain := s.bucket
|
|
if s.cdnDomain != "" {
|
|
domain = s.cdnDomain
|
|
}
|
|
link := fmt.Sprintf("https://%s/%s", domain, key)
|
|
return link, nil
|
|
}
|