package storage import ( "bytes" "context" "fmt" "io" "log/slog" "os" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" ) type S3Storage struct { client *s3.Client bucket string region string // used to construct virtual-hosted-style public URLs when no CDN/endpoint is set cdnDomain string // if set, returned URLs use this instead of bucket name endpointURL string // if set, use path-style URLs (e.g. MinIO) } // NewS3StorageFromEnv creates an S3Storage from environment variables. // Returns nil if S3_BUCKET is not set. // // Environment variables: // - S3_BUCKET (required) // - S3_REGION (default: us-west-2) // - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (optional; falls back to default credential chain) func NewS3StorageFromEnv() *S3Storage { bucket := os.Getenv("S3_BUCKET") if bucket == "" { slog.Info("S3_BUCKET not set, cloud upload disabled") return nil } if looksLikeS3Hostname(bucket) { slog.Warn( "S3_BUCKET looks like a hostname rather than a bucket name — uploads and public URLs will likely both fail. Use only the bucket name (e.g. \"my-bucket\"), not \".s3..amazonaws.com\".", "value", bucket, ) } region := os.Getenv("S3_REGION") if region == "" { region = "us-west-2" } opts := []func(*config.LoadOptions) error{ config.WithRegion(region), } accessKey := os.Getenv("AWS_ACCESS_KEY_ID") secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") if accessKey != "" && secretKey != "" { opts = append(opts, config.WithCredentialsProvider( credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""), )) } cfg, err := config.LoadDefaultConfig(context.Background(), opts...) if err != nil { slog.Error("failed to load AWS config", "error", err) return nil } cdnDomain := os.Getenv("CLOUDFRONT_DOMAIN") endpointURL := os.Getenv("AWS_ENDPOINT_URL") s3Opts := []func(*s3.Options){} if endpointURL != "" { s3Opts = append(s3Opts, func(o *s3.Options) { o.BaseEndpoint = aws.String(endpointURL) o.UsePathStyle = true }) } slog.Info("S3 storage initialized", "bucket", bucket, "region", region, "cdn_domain", cdnDomain, "endpoint_url", endpointURL) return &S3Storage{ client: s3.NewFromConfig(cfg, s3Opts...), bucket: bucket, region: region, cdnDomain: cdnDomain, endpointURL: endpointURL, } } func (s *S3Storage) CdnDomain() string { return s.cdnDomain } // looksLikeS3Hostname returns true when the configured S3_BUCKET value looks // like an S3 endpoint hostname rather than a bucket name. Real bucket names // can never legitimately contain "amazonaws.com", so this is an unambiguous // misconfiguration signal — the most common form being users pasting // ".s3..amazonaws.com" into S3_BUCKET. func looksLikeS3Hostname(bucket string) bool { return strings.Contains(bucket, "amazonaws.com") } // storageClass returns the appropriate S3 storage class. // Custom endpoints (e.g. MinIO) only support STANDARD; real AWS defaults to INTELLIGENT_TIERING. func (s *S3Storage) storageClass() types.StorageClass { if s.endpointURL != "" { return types.StorageClassStandard } return types.StorageClassIntelligentTiering } // KeyFromURL extracts the S3 object key from a CDN or bucket URL. // e.g. "https://multica-static.copilothub.ai/abc123.png" → "abc123.png" // // "https://my-bucket.s3.us-east-1.amazonaws.com/uploads/x/y.png" → "uploads/x/y.png" func (s *S3Storage) KeyFromURL(rawURL string) string { if s.endpointURL != "" { prefix := strings.TrimRight(s.endpointURL, "/") + "/" + s.bucket + "/" if strings.HasPrefix(rawURL, prefix) { return strings.TrimPrefix(rawURL, prefix) } } // Strip known "https://host/" prefixes. Order matters: the more specific // region-qualified hosts come first so they win over the legacy bucket-only // prefix that we used to write before the suffix bug was fixed. prefixes := make([]string, 0, 5) if s.cdnDomain != "" { prefixes = append(prefixes, "https://"+s.cdnDomain+"/") } if s.region != "" { // virtual-hosted-style: https://.s3..amazonaws.com/ prefixes = append(prefixes, "https://"+s.bucket+".s3."+s.region+".amazonaws.com/", // path-style: https://s3..amazonaws.com// "https://s3."+s.region+".amazonaws.com/"+s.bucket+"/", ) } // Legacy / fallback: the buggy "https:///" form that older // records may still hold, plus a generic bucket-host prefix. prefixes = append(prefixes, "https://"+s.bucket+"/") for _, prefix := range prefixes { if strings.HasPrefix(rawURL, prefix) { return strings.TrimPrefix(rawURL, prefix) } } // Fallback: take everything after the last "/". if i := strings.LastIndex(rawURL, "/"); i >= 0 { return rawURL[i+1:] } return rawURL } // GetReader streams the object body back to the caller. The returned // ReadCloser must be closed; closing it terminates the underlying HTTP // connection to S3. A missing key surfaces as an *types.NoSuchKey error // wrapped in the SDK's smithy wrapper — callers can use errors.As to // distinguish "not found" from a transport failure. func (s *S3Storage) GetReader(ctx context.Context, key string) (io.ReadCloser, error) { if key == "" { return nil, fmt.Errorf("s3 GetReader: empty key") } out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), }) if err != nil { return nil, fmt.Errorf("s3 GetObject: %w", err) } return out.Body, nil } func (s *S3Storage) PresignGet(ctx context.Context, key string, ttl time.Duration) (string, error) { return s.PresignGetWithContentDisposition(ctx, key, ttl, "") } func (s *S3Storage) PresignGetWithContentDisposition(ctx context.Context, key string, ttl time.Duration, contentDisposition string) (string, error) { if key == "" { return "", fmt.Errorf("s3 PresignGet: empty key") } if ttl <= 0 { ttl = 30 * time.Minute } input := &s3.GetObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), } if contentDisposition != "" { input.ResponseContentDisposition = aws.String(contentDisposition) } out, err := s3.NewPresignClient(s.client).PresignGetObject(ctx, input, func(opts *s3.PresignOptions) { opts.Expires = ttl }) if err != nil { return "", fmt.Errorf("s3 PresignGetObject: %w", err) } return out.URL, nil } // Delete removes an object from S3. Errors are logged but not fatal. func (s *S3Storage) Delete(ctx context.Context, key string) { if key == "" { return } _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), }) if err != nil { slog.Error("s3 DeleteObject failed", "key", key, "error", err) } } // DeleteKeys removes multiple objects from S3. Best-effort, errors are logged. func (s *S3Storage) DeleteKeys(ctx context.Context, keys []string) { for _, key := range keys { s.Delete(ctx, key) } } func (s *S3Storage) Upload(ctx context.Context, key string, data []byte, contentType string, filename string) (string, error) { _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), Body: bytes.NewReader(data), ContentType: aws.String(contentType), ContentDisposition: aws.String(ContentDisposition(contentType, filename)), CacheControl: aws.String("max-age=432000,public"), StorageClass: s.storageClass(), }) if err != nil { return "", fmt.Errorf("s3 PutObject: %w", err) } return s.uploadedURL(key), nil } // uploadedURL returns the URL stored for client consumption after an upload. // Priority: CDN domain > custom endpoint > AWS S3 region-qualified host. The CDN // domain wins even when a custom endpoint is set so S3-compatible backends // (MinIO, R2, B2, Wasabi, etc.) can be paired with a separate public-read // domain — writes still go through the SDK with the custom endpoint; only the // reader-facing URL changes. // // For the default AWS S3 case, virtual-hosted-style is preferred: // https://.s3..amazonaws.com/. When the bucket name // contains dots, the AWS-issued wildcard TLS certificate (`*.s3.amazonaws.com`) // fails to validate the host, so we fall back to path-style: // https://s3..amazonaws.com//. func (s *S3Storage) uploadedURL(key string) string { if s.cdnDomain != "" { return fmt.Sprintf("https://%s/%s", s.cdnDomain, key) } if s.endpointURL != "" { return fmt.Sprintf("%s/%s/%s", strings.TrimRight(s.endpointURL, "/"), s.bucket, key) } if strings.Contains(s.bucket, ".") { return fmt.Sprintf("https://s3.%s.amazonaws.com/%s/%s", s.region, s.bucket, key) } return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", s.bucket, s.region, key) }