mirror of
https://github.com/multica-ai/multica.git
synced 2026-07-14 13:49:18 +02:00
* feat(storage): add local file storage fallback
- Add local storage implementation for file uploads
- Update .env.example with LOCAL_UPLOAD_DIR and LOCAL_UPLOAD_BASE_URL
- Integrate local storage into server router and handlers
- Add storage abstraction layer with util functions
* ♻️ refactor(storage): improve path handling and file serving
switch from path to filepath for better cross-platform support and replace manual file serving logic with http.ServeFile to enhance security against path traversal. update unit tests to use t.Setenv for cleaner environment variable management.
168 lines
4.7 KiB
Go
168 lines
4.7 KiB
Go
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/config"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
)
|
|
|
|
type S3Storage struct {
|
|
client *s3.Client
|
|
bucket string
|
|
cdnDomain string // if set, returned URLs use this instead of bucket name
|
|
endpointURL string // if set, use path-style URLs (e.g. MinIO)
|
|
}
|
|
|
|
// NewS3StorageFromEnv creates an S3Storage from environment variables.
|
|
// Returns nil if S3_BUCKET is not set.
|
|
//
|
|
// Environment variables:
|
|
// - S3_BUCKET (required)
|
|
// - S3_REGION (default: us-west-2)
|
|
// - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (optional; falls back to default credential chain)
|
|
func NewS3StorageFromEnv() *S3Storage {
|
|
bucket := os.Getenv("S3_BUCKET")
|
|
if bucket == "" {
|
|
slog.Info("S3_BUCKET not set, cloud upload disabled")
|
|
return nil
|
|
}
|
|
|
|
region := os.Getenv("S3_REGION")
|
|
if region == "" {
|
|
region = "us-west-2"
|
|
}
|
|
|
|
opts := []func(*config.LoadOptions) error{
|
|
config.WithRegion(region),
|
|
}
|
|
|
|
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
|
|
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
|
|
if accessKey != "" && secretKey != "" {
|
|
opts = append(opts, config.WithCredentialsProvider(
|
|
credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
|
))
|
|
}
|
|
|
|
cfg, err := config.LoadDefaultConfig(context.Background(), opts...)
|
|
if err != nil {
|
|
slog.Error("failed to load AWS config", "error", err)
|
|
return nil
|
|
}
|
|
|
|
cdnDomain := os.Getenv("CLOUDFRONT_DOMAIN")
|
|
|
|
endpointURL := os.Getenv("AWS_ENDPOINT_URL")
|
|
s3Opts := []func(*s3.Options){}
|
|
if endpointURL != "" {
|
|
s3Opts = append(s3Opts, func(o *s3.Options) {
|
|
o.BaseEndpoint = aws.String(endpointURL)
|
|
o.UsePathStyle = true
|
|
})
|
|
}
|
|
|
|
slog.Info("S3 storage initialized", "bucket", bucket, "region", region, "cdn_domain", cdnDomain, "endpoint_url", endpointURL)
|
|
return &S3Storage{
|
|
client: s3.NewFromConfig(cfg, s3Opts...),
|
|
bucket: bucket,
|
|
cdnDomain: cdnDomain,
|
|
endpointURL: endpointURL,
|
|
}
|
|
}
|
|
|
|
// storageClass returns the appropriate S3 storage class.
|
|
// Custom endpoints (e.g. MinIO) only support STANDARD; real AWS defaults to INTELLIGENT_TIERING.
|
|
func (s *S3Storage) storageClass() types.StorageClass {
|
|
if s.endpointURL != "" {
|
|
return types.StorageClassStandard
|
|
}
|
|
return types.StorageClassIntelligentTiering
|
|
}
|
|
|
|
// KeyFromURL extracts the S3 object key from a CDN or bucket URL.
|
|
// e.g. "https://multica-static.copilothub.ai/abc123.png" → "abc123.png"
|
|
func (s *S3Storage) KeyFromURL(rawURL string) string {
|
|
if s.endpointURL != "" {
|
|
prefix := strings.TrimRight(s.endpointURL, "/") + "/" + s.bucket + "/"
|
|
if strings.HasPrefix(rawURL, prefix) {
|
|
return strings.TrimPrefix(rawURL, prefix)
|
|
}
|
|
}
|
|
|
|
// Strip the "https://domain/" prefix.
|
|
for _, prefix := range []string{
|
|
"https://" + s.cdnDomain + "/",
|
|
"https://" + s.bucket + "/",
|
|
} {
|
|
if strings.HasPrefix(rawURL, prefix) {
|
|
return strings.TrimPrefix(rawURL, prefix)
|
|
}
|
|
}
|
|
// Fallback: take everything after the last "/".
|
|
if i := strings.LastIndex(rawURL, "/"); i >= 0 {
|
|
return rawURL[i+1:]
|
|
}
|
|
return rawURL
|
|
}
|
|
|
|
// Delete removes an object from S3. Errors are logged but not fatal.
|
|
func (s *S3Storage) Delete(ctx context.Context, key string) {
|
|
if key == "" {
|
|
return
|
|
}
|
|
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
if err != nil {
|
|
slog.Error("s3 DeleteObject failed", "key", key, "error", err)
|
|
}
|
|
}
|
|
|
|
// DeleteKeys removes multiple objects from S3. Best-effort, errors are logged.
|
|
func (s *S3Storage) DeleteKeys(ctx context.Context, keys []string) {
|
|
for _, key := range keys {
|
|
s.Delete(ctx, key)
|
|
}
|
|
}
|
|
|
|
func (s *S3Storage) Upload(ctx context.Context, key string, data []byte, contentType string, filename string) (string, error) {
|
|
safe := sanitizeFilename(filename)
|
|
disposition := "attachment"
|
|
if isInlineContentType(contentType) {
|
|
disposition = "inline"
|
|
}
|
|
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
Body: bytes.NewReader(data),
|
|
ContentType: aws.String(contentType),
|
|
ContentDisposition: aws.String(fmt.Sprintf(`%s; filename="%s"`, disposition, safe)),
|
|
CacheControl: aws.String("max-age=432000,public"),
|
|
StorageClass: s.storageClass(),
|
|
})
|
|
if err != nil {
|
|
return "", fmt.Errorf("s3 PutObject: %w", err)
|
|
}
|
|
|
|
if s.endpointURL != "" {
|
|
link := fmt.Sprintf("%s/%s/%s", strings.TrimRight(s.endpointURL, "/"), s.bucket, key)
|
|
return link, nil
|
|
}
|
|
domain := s.bucket
|
|
if s.cdnDomain != "" {
|
|
domain = s.cdnDomain
|
|
}
|
|
link := fmt.Sprintf("https://%s/%s", domain, key)
|
|
return link, nil
|
|
}
|