diff --git a/.gitignore b/.gitignore index a22238de..001cf9c2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +*.swp imgproxy tmp/ diff --git a/README.md b/README.md index c98beac8..a33fcf07 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Massive processing of remote images is a potentially dangerous thing, security-w * [WebP support detection](./docs/configuration.md#webp-support-detection) * [Presets](./docs/configuration.md#presets) * [Serving local files](./docs/configuration.md#serving-local-files) + * [Serving files from Amazon S3](./docs/configuration.md#serving-files-from-amazon-s3) * [Miscellaneous](./docs/configuration.md#miscellaneous) 4. [Generating the URL](./docs/generating_the_url_basic.md) * [Basic](./docs/generating_the_url_basic.md) @@ -69,6 +70,7 @@ Massive processing of remote images is a potentially dangerous thing, security-w * [Signing the URL](./docs/signing_the_url.md) 5. [Presets](./docs/presets.md) 6. [Serving local files](./docs/serving_local_files.md) +7. [Serving files from Amazon S3](./docs/serving_files_from_s3.md) 7. [Source image formats support](./docs/source_image_formats_support.md) 8. [About processing pipeline](./docs/about_processing_pipeline.md) 9. [Health check](./docs/healthcheck.md) diff --git a/config.go b/config.go index 9ed322ee..3118c49f 100644 --- a/config.go +++ b/config.go @@ -137,6 +137,7 @@ type config struct { IgnoreSslVerification bool LocalFileSystemRoot string + S3Enabled bool ETagEnabled bool @@ -159,6 +160,7 @@ var conf = config{ Quality: 80, GZipCompression: 5, ETagEnabled: false, + S3Enabled: false, } func init() { @@ -211,6 +213,8 @@ func init() { strEnvConfig(&conf.LocalFileSystemRoot, "IMGPROXY_LOCAL_FILESYSTEM_ROOT") + boolEnvConfig(&conf.S3Enabled, "IMGPROXY_USE_S3") + boolEnvConfig(&conf.ETagEnabled, "IMGPROXY_USE_ETAG") strEnvConfig(&conf.BaseURL, "IMGPROXY_BASE_URL") diff --git a/docs/configuration.md b/docs/configuration.md index 37e0bbe8..b44650b0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -30,7 +30,7 @@ $ echo $(xxd -g 2 -l 64 -p /dev/random | tr -d '\n') * `IMGPROXY_CONCURRENCY` — the maximum number of image requests to be processed simultaneously. Default: double number of CPU cores; * `IMGPROXY_MAX_CLIENTS` — the maximum number of simultaneous active connections. Default: `IMGPROXY_CONCURRENCY * 10`; * `IMGPROXY_TTL` — duration in seconds sent in `Expires` and `Cache-Control: max-age` headers. Default: `3600` (1 hour); -* `IMGPROXY_USE_ETAG` — when true, enables using [ETag](https://en.wikipedia.org/wiki/HTTP_ETag) header for the cache control. Default: false; +* `IMGPROXY_USE_ETAG` — when `true`, enables using [ETag](https://en.wikipedia.org/wiki/HTTP_ETag) header for the cache control. Default: false; ### Security @@ -104,6 +104,14 @@ imgproxy can serve your local images, but this feature is disabled by default. T Check out [Serving local files](../docs/serving_local_files.md) guide to get more info. +### Serving files from Amazon S3 + +imgproxy can process files from Amazon S3 buckets, but this feature is disabled by default. To enable it, set `IMGPROXY_USE_S3` as `true`: + +* `IMGPROXY_USE_S3` — when `true`, enables fetching the images from Amazon S3 buckets. Default: false. + +Check out [Serving files from S3](../docs/serving_files_from_s3.md) guide to get more info. + ### Miscellaneous * `IMGPROXY_BASE_URL` - base URL part which will be added to every requested image URL. For example, if base URL is `http://example.com/images` and `/path/to/image.png` is requested, imgproxy will download the image from `http://example.com/images/path/to/image.png`. Default: blank. diff --git a/docs/serving_files_from_s3.md b/docs/serving_files_from_s3.md new file mode 100644 index 00000000..a4abdd03 --- /dev/null +++ b/docs/serving_files_from_s3.md @@ -0,0 +1,40 @@ +# Serving files from S3 + +imgproxy can process files from S3 buckets. To use this feature do the following: + +1. Set `IMGPROXY_USE_S3` environment variable as `true`; +2. [Setup credentials](#setup-credentials) to grant access to your bucket; +3. Use `s3://%bucket_name/%file_key` as the source image url. + +### Setup credentials + +There are three ways to specify your AWS credentials. The credentials need to be able to read from any of the buckets given in the source URLs. + +#### Environment variables + +You can specify AWS Acces Key ID and Secret Access Key by setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. + +``` bash +$ AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy + +# same for Docker +docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy +``` + +It's recommended to use this way when you run dockerized imgproxy. + +#### Shared credentials file + +Create `.aws/credentials` file in your home directory with the following content: + +```ini +[default] +aws_access_key_id = %access_key_id +aws_secret_access_key = %secret_access_key +``` + +#### IAM Roles for Amazon EC2 Instances + +If you are running imgproxy on an Amazon EC2 instance, you can use the instance's [IAM role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) to get security credentials to make calls to AWS. + +You can get more info about credentials in the [Configuring the AWS SDK for Go](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) guide. diff --git a/download.go b/download.go index 51971601..58e93486 100644 --- a/download.go +++ b/download.go @@ -69,6 +69,10 @@ func initDownloading() { transport.RegisterProtocol("local", http.NewFileTransport(http.Dir(conf.LocalFileSystemRoot))) } + if conf.S3Enabled { + transport.RegisterProtocol("s3", newS3Transport()) + } + downloadClient = &http.Client{ Timeout: time.Duration(conf.DownloadTimeout) * time.Second, Transport: transport, diff --git a/s3transport.go b/s3transport.go new file mode 100644 index 00000000..c4079c45 --- /dev/null +++ b/s3transport.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + http "net/http" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +// s3Transport implements RoundTripper for the 's3' protocol. +type s3Transport struct { + svc *s3.S3 +} + +func newS3Transport() http.RoundTripper { + return s3Transport{s3.New(session.New())} +} + +func (t s3Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) { + input := &s3.GetObjectInput{ + Bucket: aws.String(req.URL.Host), + Key: aws.String(req.URL.Path), + } + s3req, _ := t.svc.GetObjectRequest(input) + + s3err := s3req.Send() + if s3err == nil { // resp is now filled + return s3req.HTTPResponse, nil + } + fmt.Println("s3 error", s3err) + return nil, s3err +}