Files
multica/server/internal/storage/s3.go
Bohan Jiang cc1ccedaf3 test(storage): lock S3 upload URL behavior across all env combos (#1421)
Extract the URL assembly at the end of S3Storage.Upload into a helper
(uploadedURL) so the four env-var combinations can be covered by a
table-driven test without mocking s3.PutObject. This locks in the fix
from #1300 — cdn > endpoint > bucket — so future refactors can't
silently regress the CDN-wins-over-custom-endpoint case.

No behavior change.
2026-04-21 12:57:36 +08:00

177 lines
5.2 KiB
Go

package storage
import (
"bytes"
"context"
"fmt"
"log/slog"
"os"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type S3Storage struct {
client *s3.Client
bucket string
cdnDomain string // if set, returned URLs use this instead of bucket name
endpointURL string // if set, use path-style URLs (e.g. MinIO)
}
// NewS3StorageFromEnv creates an S3Storage from environment variables.
// Returns nil if S3_BUCKET is not set.
//
// Environment variables:
// - S3_BUCKET (required)
// - S3_REGION (default: us-west-2)
// - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (optional; falls back to default credential chain)
func NewS3StorageFromEnv() *S3Storage {
bucket := os.Getenv("S3_BUCKET")
if bucket == "" {
slog.Info("S3_BUCKET not set, cloud upload disabled")
return nil
}
region := os.Getenv("S3_REGION")
if region == "" {
region = "us-west-2"
}
opts := []func(*config.LoadOptions) error{
config.WithRegion(region),
}
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if accessKey != "" && secretKey != "" {
opts = append(opts, config.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
))
}
cfg, err := config.LoadDefaultConfig(context.Background(), opts...)
if err != nil {
slog.Error("failed to load AWS config", "error", err)
return nil
}
cdnDomain := os.Getenv("CLOUDFRONT_DOMAIN")
endpointURL := os.Getenv("AWS_ENDPOINT_URL")
s3Opts := []func(*s3.Options){}
if endpointURL != "" {
s3Opts = append(s3Opts, func(o *s3.Options) {
o.BaseEndpoint = aws.String(endpointURL)
o.UsePathStyle = true
})
}
slog.Info("S3 storage initialized", "bucket", bucket, "region", region, "cdn_domain", cdnDomain, "endpoint_url", endpointURL)
return &S3Storage{
client: s3.NewFromConfig(cfg, s3Opts...),
bucket: bucket,
cdnDomain: cdnDomain,
endpointURL: endpointURL,
}
}
func (s *S3Storage) CdnDomain() string {
return s.cdnDomain
}
// storageClass returns the appropriate S3 storage class.
// Custom endpoints (e.g. MinIO) only support STANDARD; real AWS defaults to INTELLIGENT_TIERING.
func (s *S3Storage) storageClass() types.StorageClass {
if s.endpointURL != "" {
return types.StorageClassStandard
}
return types.StorageClassIntelligentTiering
}
// KeyFromURL extracts the S3 object key from a CDN or bucket URL.
// e.g. "https://multica-static.copilothub.ai/abc123.png" → "abc123.png"
func (s *S3Storage) KeyFromURL(rawURL string) string {
if s.endpointURL != "" {
prefix := strings.TrimRight(s.endpointURL, "/") + "/" + s.bucket + "/"
if strings.HasPrefix(rawURL, prefix) {
return strings.TrimPrefix(rawURL, prefix)
}
}
// Strip the "https://domain/" prefix.
for _, prefix := range []string{
"https://" + s.cdnDomain + "/",
"https://" + s.bucket + "/",
} {
if strings.HasPrefix(rawURL, prefix) {
return strings.TrimPrefix(rawURL, prefix)
}
}
// Fallback: take everything after the last "/".
if i := strings.LastIndex(rawURL, "/"); i >= 0 {
return rawURL[i+1:]
}
return rawURL
}
// Delete removes an object from S3. Errors are logged but not fatal.
func (s *S3Storage) Delete(ctx context.Context, key string) {
if key == "" {
return
}
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
})
if err != nil {
slog.Error("s3 DeleteObject failed", "key", key, "error", err)
}
}
// DeleteKeys removes multiple objects from S3. Best-effort, errors are logged.
func (s *S3Storage) DeleteKeys(ctx context.Context, keys []string) {
for _, key := range keys {
s.Delete(ctx, key)
}
}
func (s *S3Storage) Upload(ctx context.Context, key string, data []byte, contentType string, filename string) (string, error) {
safe := sanitizeFilename(filename)
disposition := "attachment"
if isInlineContentType(contentType) {
disposition = "inline"
}
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
Body: bytes.NewReader(data),
ContentType: aws.String(contentType),
ContentDisposition: aws.String(fmt.Sprintf(`%s; filename="%s"`, disposition, safe)),
CacheControl: aws.String("max-age=432000,public"),
StorageClass: s.storageClass(),
})
if err != nil {
return "", fmt.Errorf("s3 PutObject: %w", err)
}
return s.uploadedURL(key), nil
}
// uploadedURL returns the URL stored for client consumption after an upload.
// Priority: CDN domain > custom endpoint > bucket. The CDN domain wins even when
// a custom endpoint is set so S3-compatible backends (MinIO, R2, B2, Wasabi, etc.)
// can be paired with a separate public-read domain — writes still go through the
// SDK with the custom endpoint; only the reader-facing URL changes.
func (s *S3Storage) uploadedURL(key string) string {
if s.cdnDomain != "" {
return fmt.Sprintf("https://%s/%s", s.cdnDomain, key)
}
if s.endpointURL != "" {
return fmt.Sprintf("%s/%s/%s", strings.TrimRight(s.endpointURL, "/"), s.bucket, key)
}
return fmt.Sprintf("https://%s/%s", s.bucket, key)
}