diff --git a/etag.go b/etag.go deleted file mode 100644 index cffb9189..00000000 --- a/etag.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "hash" - "sync" - - "github.com/imgproxy/imgproxy/v2/imagedata" - "github.com/imgproxy/imgproxy/v2/options" - "github.com/imgproxy/imgproxy/v2/version" -) - -type eTagCalc struct { - hash hash.Hash - enc *json.Encoder -} - -var eTagCalcPool = sync.Pool{ - New: func() interface{} { - h := sha256.New() - - enc := json.NewEncoder(h) - enc.SetEscapeHTML(false) - enc.SetIndent("", "") - - return &eTagCalc{h, enc} - }, -} - -func calcETag(ctx context.Context, imgdata *imagedata.ImageData, po *options.ProcessingOptions) string { - c := eTagCalcPool.Get().(*eTagCalc) - defer eTagCalcPool.Put(c) - - c.hash.Reset() - c.hash.Write(imgdata.Data) - footprint := c.hash.Sum(nil) - - c.hash.Reset() - c.hash.Write(footprint) - c.hash.Write([]byte(version.Version())) - c.enc.Encode(po) - - return hex.EncodeToString(c.hash.Sum(nil)) -} diff --git a/etag/etag.go b/etag/etag.go new file mode 100644 index 00000000..268523d1 --- /dev/null +++ b/etag/etag.go @@ -0,0 +1,153 @@ +package etag + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "hash" + "net/textproto" + "strings" + "sync" + + "github.com/imgproxy/imgproxy/v2/imagedata" + "github.com/imgproxy/imgproxy/v2/options" + "github.com/imgproxy/imgproxy/v2/version" +) + +type eTagCalc struct { + hash hash.Hash + enc *json.Encoder +} + +var eTagCalcPool = sync.Pool{ + New: func() interface{} { + h := sha256.New() + + enc := json.NewEncoder(h) + enc.SetEscapeHTML(false) + enc.SetIndent("", "") + + return &eTagCalc{h, enc} + }, +} + +type Handler struct { + poHashActual, poHashExpected string + + imgEtagActual, imgEtagExpected string + imgHashActual, imgHashExpected string +} + +func (h *Handler) ParseExpectedETag(etag string) { + // We suuport only a single ETag value + if i := strings.IndexByte(etag, ','); i >= 0 { + etag = textproto.TrimString(etag[:i]) + } + + etagLen := len(etag) + + // ETag is empty or invalid + if etagLen < 2 { + return + } + + // We support strong ETags only + if etag[0] != '"' || etag[etagLen-1] != '"' { + return + } + + // Remove quotes + etag = etag[1 : etagLen-1] + + i := strings.Index(etag, "/") + if i < 0 || i > etagLen-3 { + // Doesn't look like imgproxy ETag + return + } + + poPart, imgPartMark, imgPart := etag[:i], etag[i+1], etag[i+2:] + + switch imgPartMark { + case 'R': + imgPartDec, err := base64.RawStdEncoding.DecodeString(imgPart) + if err == nil { + h.imgEtagExpected = string(imgPartDec) + } + case 'D': + h.imgHashExpected = imgPart + default: + // Unknown image part mark + return + } + + h.poHashExpected = poPart +} + +func (h *Handler) ProcessingOptionsMatch() bool { + return h.poHashActual == h.poHashExpected +} + +func (h *Handler) SetActualProcessingOptions(po *options.ProcessingOptions) bool { + c := eTagCalcPool.Get().(*eTagCalc) + defer eTagCalcPool.Put(c) + + c.hash.Reset() + c.hash.Write([]byte(version.Version())) + c.enc.Encode(po) + + h.poHashActual = base64.RawURLEncoding.EncodeToString(c.hash.Sum(nil)) + + return h.ProcessingOptionsMatch() +} + +func (h *Handler) ImageEtagExpected() string { + return h.imgEtagExpected +} + +func (h *Handler) SetActualImageData(imgdata *imagedata.ImageData) bool { + var haveActualImgETag bool + h.imgEtagActual, haveActualImgETag = imgdata.Headers["ETag"] + haveActualImgETag = haveActualImgETag && len(h.imgEtagActual) > 0 + + // Just in case server didn't check ETag properly and returned the same one + // as we expected + if haveActualImgETag && h.imgEtagExpected == h.imgEtagActual { + return true + } + + haveExpectedImgHash := len(h.imgHashExpected) != 0 + + if !haveActualImgETag || haveExpectedImgHash { + c := eTagCalcPool.Get().(*eTagCalc) + defer eTagCalcPool.Put(c) + + c.hash.Reset() + c.hash.Write(imgdata.Data) + + h.imgHashActual = base64.RawURLEncoding.EncodeToString(c.hash.Sum(nil)) + + return haveExpectedImgHash && h.imgHashActual == h.imgHashExpected + } + + return false +} + +func (h *Handler) GenerateActualETag() string { + return h.generate(h.poHashActual, h.imgEtagActual, h.imgHashActual) +} + +func (h *Handler) GenerateExpectedETag() string { + return h.generate(h.poHashExpected, h.imgEtagExpected, h.imgHashExpected) +} + +func (h *Handler) generate(poHash, imgEtag, imgHash string) string { + imgPartMark := 'D' + imgPart := imgHash + if len(imgEtag) != 0 { + imgPartMark = 'R' + imgPart = base64.RawURLEncoding.EncodeToString([]byte(imgEtag)) + } + + return fmt.Sprintf(`"%s/%c%s"`, poHash, imgPartMark, imgPart) +} diff --git a/etag/etag_test.go b/etag/etag_test.go new file mode 100644 index 00000000..73720927 --- /dev/null +++ b/etag/etag_test.go @@ -0,0 +1,135 @@ +package etag + +import ( + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/imgproxy/imgproxy/v2/imagedata" + "github.com/imgproxy/imgproxy/v2/options" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +var ( + po = options.NewProcessingOptions() + + imgWithETag = imagedata.ImageData{ + Data: []byte("Hello Test"), + Headers: map[string]string{"ETag": `"loremipsumdolor"`}, + } + imgWithoutETag = imagedata.ImageData{ + Data: []byte("Hello Test"), + } + + etagReq = `"ATeSQpxYMfaZVBSmCh-zpE8682vBUrZ1qxXgQkxtntA/RImxvcmVtaXBzdW1kb2xvciI"` + etagData = `"ATeSQpxYMfaZVBSmCh-zpE8682vBUrZ1qxXgQkxtntA/DvyChhMNu_sFX7jrjoyrgQbnFwfoOVv7kzp_Fbs6hQBg"` +) + +type EtagTestSuite struct { + suite.Suite + + h Handler +} + +func (s *EtagTestSuite) SetupSuite() { + logrus.SetOutput(ioutil.Discard) +} + +func (s *EtagTestSuite) TeardownSuite() { + logrus.SetOutput(os.Stdout) +} + +func (s *EtagTestSuite) SetupTest() { + s.h = Handler{} +} + +func (s *EtagTestSuite) TestGenerateActualReq() { + s.h.SetActualProcessingOptions(po) + s.h.SetActualImageData(&imgWithETag) + + assert.Equal(s.T(), etagReq, s.h.GenerateActualETag()) +} + +func (s *EtagTestSuite) TestGenerateActualData() { + s.h.SetActualProcessingOptions(po) + s.h.SetActualImageData(&imgWithoutETag) + + assert.Equal(s.T(), etagData, s.h.GenerateActualETag()) +} + +func (s *EtagTestSuite) TestGenerateExpectedReq() { + s.h.ParseExpectedETag(etagReq) + assert.Equal(s.T(), etagReq, s.h.GenerateExpectedETag()) +} + +func (s *EtagTestSuite) TestGenerateExpectedData() { + s.h.ParseExpectedETag(etagData) + assert.Equal(s.T(), etagData, s.h.GenerateExpectedETag()) +} + +func (s *EtagTestSuite) TestProcessingOptionsCheckSuccess() { + s.h.ParseExpectedETag(etagReq) + + assert.True(s.T(), s.h.SetActualProcessingOptions(po)) + assert.True(s.T(), s.h.ProcessingOptionsMatch()) +} + +func (s *EtagTestSuite) TestProcessingOptionsCheckFailure() { + i := strings.Index(etagReq, "/") + wrongEtag := `"wrongpohash` + etagReq[i:] + + s.h.ParseExpectedETag(wrongEtag) + + assert.False(s.T(), s.h.SetActualProcessingOptions(po)) + assert.False(s.T(), s.h.ProcessingOptionsMatch()) +} + +func (s *EtagTestSuite) TestImageETagExpectedPresent() { + s.h.ParseExpectedETag(etagReq) + + assert.Equal(s.T(), imgWithETag.Headers["ETag"], s.h.ImageEtagExpected()) +} + +func (s *EtagTestSuite) TestImageETagExpectedBlank() { + s.h.ParseExpectedETag(etagData) + + assert.Empty(s.T(), s.h.ImageEtagExpected()) +} + +func (s *EtagTestSuite) TestImageDataCheckDataToDataSuccess() { + s.h.ParseExpectedETag(etagData) + assert.True(s.T(), s.h.SetActualImageData(&imgWithoutETag)) +} + +func (s *EtagTestSuite) TestImageDataCheckDataToDataFailure() { + i := strings.Index(etagData, "/") + wrongEtag := etagData[:i] + `/Dwrongimghash"` + + s.h.ParseExpectedETag(wrongEtag) + assert.False(s.T(), s.h.SetActualImageData(&imgWithoutETag)) +} + +func (s *EtagTestSuite) TestImageDataCheckDataToReqSuccess() { + s.h.ParseExpectedETag(etagData) + assert.True(s.T(), s.h.SetActualImageData(&imgWithETag)) +} + +func (s *EtagTestSuite) TestImageDataCheckDataToReqFailure() { + i := strings.Index(etagData, "/") + wrongEtag := etagData[:i] + `/Dwrongimghash"` + + s.h.ParseExpectedETag(wrongEtag) + assert.False(s.T(), s.h.SetActualImageData(&imgWithETag)) +} + +func (s *EtagTestSuite) TestImageDataCheckReqToDataFailure() { + s.h.ParseExpectedETag(etagReq) + assert.False(s.T(), s.h.SetActualImageData(&imgWithoutETag)) +} + +func TestEtag(t *testing.T) { + suite.Run(t, new(EtagTestSuite)) +} diff --git a/ierrors/errors.go b/ierrors/errors.go index 0b586dd5..cf6cc3ef 100644 --- a/ierrors/errors.go +++ b/ierrors/errors.go @@ -62,6 +62,15 @@ func Wrap(err error, skip int) *Error { return NewUnexpected(err.Error(), skip+1) } +func WrapWithMessage(err error, skip int, msg string) *Error { + if ierr, ok := err.(*Error); ok { + newErr := *ierr + ierr.Message = msg + return &newErr + } + return NewUnexpected(err.Error(), skip+1) +} + func callers(skip int) []uintptr { stack := make([]uintptr, 10) n := runtime.Callers(skip, stack) @@ -78,3 +87,10 @@ func formatStack(stack []uintptr) string { return strings.Join(lines, "\n") } + +func StatusCode(err error) int { + if ierr, ok := err.(*Error); ok { + return ierr.StatusCode + } + return 0 +} diff --git a/imagedata/download.go b/imagedata/download.go index 0fce5520..06148996 100644 --- a/imagedata/download.go +++ b/imagedata/download.go @@ -24,10 +24,13 @@ var ( imageHeadersToStore = []string{ "Cache-Control", "Expires", + "ETag", } // For tests redirectAllRequestsTo string + + ErrNotModified = ierrors.New(http.StatusNotModified, "Not Modified", "Not Modified") ) const msgSourceImageIsUnreachable = "Source image is unreachable" @@ -81,7 +84,7 @@ func initDownloading() error { return nil } -func requestImage(imageURL string) (*http.Response, error) { +func requestImage(imageURL string, header http.Header) (*http.Response, error) { req, err := http.NewRequest("GET", imageURL, nil) if err != nil { return nil, ierrors.New(404, err.Error(), msgSourceImageIsUnreachable).SetUnexpected(config.ReportDownloadingErrors) @@ -89,29 +92,39 @@ func requestImage(imageURL string) (*http.Response, error) { req.Header.Set("User-Agent", config.UserAgent) + for k, v := range header { + if len(v) > 0 { + req.Header.Set(k, v[0]) + } + } + res, err := downloadClient.Do(req) if err != nil { return res, ierrors.New(404, checkTimeoutErr(err).Error(), msgSourceImageIsUnreachable).SetUnexpected(config.ReportDownloadingErrors) } + if res.StatusCode == http.StatusNotModified { + return nil, ErrNotModified + } + if res.StatusCode != 200 { body, _ := ioutil.ReadAll(res.Body) res.Body.Close() - msg := fmt.Sprintf("Can't download image; Status: %d; %s", res.StatusCode, string(body)) + msg := fmt.Sprintf("Status: %d; %s", res.StatusCode, string(body)) return res, ierrors.New(404, msg, msgSourceImageIsUnreachable).SetUnexpected(config.ReportDownloadingErrors) } return res, nil } -func download(imageURL string) (*ImageData, error) { +func download(imageURL string, header http.Header) (*ImageData, error) { // We use this for testing if len(redirectAllRequestsTo) > 0 { imageURL = redirectAllRequestsTo } - res, err := requestImage(imageURL) + res, err := requestImage(imageURL, header) if res != nil { defer res.Body.Close() } diff --git a/imagedata/image_data.go b/imagedata/image_data.go index 811b9112..43bef89f 100644 --- a/imagedata/image_data.go +++ b/imagedata/image_data.go @@ -4,11 +4,13 @@ import ( "context" "encoding/base64" "fmt" + "net/http" "os" "strings" "sync" "github.com/imgproxy/imgproxy/v2/config" + "github.com/imgproxy/imgproxy/v2/ierrors" "github.com/imgproxy/imgproxy/v2/imagetype" ) @@ -68,7 +70,7 @@ func loadWatermark() (err error) { } if len(config.WatermarkURL) > 0 { - Watermark, err = Download(config.WatermarkURL, "watermark") + Watermark, err = Download(config.WatermarkURL, "watermark", nil) return } @@ -87,7 +89,7 @@ func loadFallbackImage() (err error) { } if len(config.FallbackImageURL) > 0 { - FallbackImage, err = Download(config.FallbackImageURL, "fallback image") + FallbackImage, err = Download(config.FallbackImageURL, "fallback image", nil) return } @@ -125,10 +127,10 @@ func FromFile(path, desc string) (*ImageData, error) { return imgdata, nil } -func Download(imageURL, desc string) (*ImageData, error) { - imgdata, err := download(imageURL) +func Download(imageURL, desc string, header http.Header) (*ImageData, error) { + imgdata, err := download(imageURL, header) if err != nil { - return nil, fmt.Errorf("Can't download %s: %s", desc, err) + return nil, ierrors.WrapWithMessage(err, 1, fmt.Sprintf("Can't download %s: %s", desc, err)) } return imgdata, nil diff --git a/processing_handler.go b/processing_handler.go index 537b2825..9e797816 100644 --- a/processing_handler.go +++ b/processing_handler.go @@ -12,6 +12,7 @@ import ( "github.com/imgproxy/imgproxy/v2/config" "github.com/imgproxy/imgproxy/v2/errorreport" + "github.com/imgproxy/imgproxy/v2/etag" "github.com/imgproxy/imgproxy/v2/ierrors" "github.com/imgproxy/imgproxy/v2/imagedata" "github.com/imgproxy/imgproxy/v2/imagetype" @@ -113,6 +114,17 @@ func respondWithImage(reqID string, r *http.Request, rw http.ResponseWriter, res ) } +func respondWithNotModified(reqID string, r *http.Request, rw http.ResponseWriter, po *options.ProcessingOptions, originURL string) { + rw.WriteHeader(304) + router.LogResponse( + reqID, r, 304, nil, + log.Fields{ + "image_url": originURL, + "processing_options": po, + }, + ) +} + func handleProcessing(reqID string, rw http.ResponseWriter, r *http.Request) { ctx, timeoutCancel := context.WithTimeout(r.Context(), time.Duration(config.WriteTimeout)*time.Second) defer timeoutCancel() @@ -162,6 +174,20 @@ func handleProcessing(reqID string, rw http.ResponseWriter, r *http.Request) { )) } + imgRequestHeader := make(http.Header) + + var etagHandler etag.Handler + + if config.ETagEnabled { + etagHandler.ParseExpectedETag(r.Header.Get("If-None-Match")) + + if etagHandler.SetActualProcessingOptions(po) { + if imgEtag := etagHandler.ImageEtagExpected(); len(imgEtag) != 0 { + imgRequestHeader.Set("If-None-Match", imgEtag) + } + } + } + // The heavy part start here, so we need to restrict concurrency select { case processingSem <- struct{}{}: @@ -175,21 +201,26 @@ func handleProcessing(reqID string, rw http.ResponseWriter, r *http.Request) { originData, err := func() (*imagedata.ImageData, error) { defer metrics.StartDownloadingSegment(ctx)() - return imagedata.Download(imageURL, "source image") + return imagedata.Download(imageURL, "source image", imgRequestHeader) }() - if err == nil { + switch { + case err == nil: defer originData.Close() - } else { + case ierrors.StatusCode(err) == http.StatusNotModified: + rw.Header().Set("ETag", etagHandler.GenerateExpectedETag()) + respondWithNotModified(reqID, r, rw, po, imageURL) + return + default: + if ierr, ok := err.(*ierrors.Error); !ok || ierr.Unexpected { + errorreport.Report(err, r) + } + metrics.SendError(ctx, "download", err) if imagedata.FallbackImage == nil { panic(err) } - if ierr, ok := err.(*ierrors.Error); !ok || ierr.Unexpected { - errorreport.Report(err, r) - } - log.Warningf("Could not load image %s. Using fallback image. %s", imageURL, err.Error()) r = r.WithContext(setFallbackImageUsedCtx(r.Context())) originData = imagedata.FallbackImage @@ -198,12 +229,12 @@ func handleProcessing(reqID string, rw http.ResponseWriter, r *http.Request) { router.CheckTimeout(ctx) if config.ETagEnabled && !getFallbackImageUsed(ctx) { - eTag := calcETag(ctx, originData, po) - rw.Header().Set("ETag", eTag) + imgDataMatch := etagHandler.SetActualImageData(originData) - if eTag == r.Header.Get("If-None-Match") { - rw.WriteHeader(304) - router.LogResponse(reqID, r, 304, nil, log.Fields{"image_url": imageURL}) + rw.Header().Set("ETag", etagHandler.GenerateActualETag()) + + if imgDataMatch && etagHandler.ProcessingOptionsMatch() { + respondWithNotModified(reqID, r, rw, po, imageURL) return } } diff --git a/processing_handler_test.go b/processing_handler_test.go index a501e8d6..72285e82 100644 --- a/processing_handler_test.go +++ b/processing_handler_test.go @@ -289,11 +289,10 @@ func (s *ProcessingHandlerTestSuite) TestCacheControlPassthrough() { config.CacheControlPassthrough = true ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - data := s.readTestFile("test1.png") rw.Header().Set("Cache-Control", "fake-cache-control") rw.Header().Set("Expires", "fake-expires") rw.WriteHeader(200) - rw.Write(data) + rw.Write(s.readTestFile("test1.png")) })) defer ts.Close() @@ -308,11 +307,10 @@ func (s *ProcessingHandlerTestSuite) TestCacheControlPassthroughDisabled() { config.CacheControlPassthrough = false ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - data := s.readTestFile("test1.png") rw.Header().Set("Cache-Control", "fake-cache-control") rw.Header().Set("Expires", "fake-expires") rw.WriteHeader(200) - rw.Write(data) + rw.Write(s.readTestFile("test1.png")) })) defer ts.Close() @@ -323,6 +321,183 @@ func (s *ProcessingHandlerTestSuite) TestCacheControlPassthroughDisabled() { assert.NotEqual(s.T(), "fake-expires", res.Header.Get("Expires")) } +func (s *ProcessingHandlerTestSuite) TestETagDisabled() { + config.ETagEnabled = false + + rw := s.send("/unsafe/rs:fill:4:4/plain/local:///test1.png") + res := rw.Result() + + assert.Equal(s.T(), 200, res.StatusCode) + assert.Empty(s.T(), res.Header.Get("ETag")) +} + +func (s *ProcessingHandlerTestSuite) TestETagReqNoIfNotModified() { + config.ETagEnabled = true + + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Empty(s.T(), r.Header.Get("If-None-Match")) + + rw.Header().Set("ETag", `"loremipsumdolor"`) + rw.WriteHeader(200) + rw.Write(s.readTestFile("test1.png")) + })) + defer ts.Close() + + rw := s.send("/unsafe/rs:fill:4:4/plain/" + ts.URL) + res := rw.Result() + + assert.Equal(s.T(), 200, res.StatusCode) + assert.Equal( + s.T(), + `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/RImxvcmVtaXBzdW1kb2xvciI"`, + res.Header.Get("ETag"), + ) +} + +func (s *ProcessingHandlerTestSuite) TestETagDataNoIfNotModified() { + config.ETagEnabled = true + + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Empty(s.T(), r.Header.Get("If-None-Match")) + + rw.WriteHeader(200) + rw.Write(s.readTestFile("test1.png")) + })) + defer ts.Close() + + rw := s.send("/unsafe/rs:fill:4:4/plain/" + ts.URL) + res := rw.Result() + + assert.Equal(s.T(), 200, res.StatusCode) + assert.Equal( + s.T(), + `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/Dl29MNvkqdLEqPyFvMlh_NlPbRrsC0GDG_AUlmMdX6HA"`, + res.Header.Get("ETag"), + ) +} + +func (s *ProcessingHandlerTestSuite) TestETagReqMatch() { + config.ETagEnabled = true + + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Equal(s.T(), `"loremipsumdolor"`, r.Header.Get("If-None-Match")) + + rw.WriteHeader(304) + })) + defer ts.Close() + + etag := `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/RImxvcmVtaXBzdW1kb2xvciI"` + + header := make(http.Header) + header.Set("If-None-Match", etag) + + rw := s.send("/unsafe/rs:fill:4:4/plain/"+ts.URL, header) + res := rw.Result() + + assert.Equal(s.T(), 304, res.StatusCode) + assert.Equal(s.T(), etag, res.Header.Get("ETag")) +} + +func (s *ProcessingHandlerTestSuite) TestETagDataMatch() { + config.ETagEnabled = true + + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Empty(s.T(), r.Header.Get("If-None-Match")) + + rw.WriteHeader(200) + rw.Write(s.readTestFile("test1.png")) + })) + defer ts.Close() + + etag := `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/Dl29MNvkqdLEqPyFvMlh_NlPbRrsC0GDG_AUlmMdX6HA"` + + header := make(http.Header) + header.Set("If-None-Match", etag) + + rw := s.send("/unsafe/rs:fill:4:4/plain/"+ts.URL, header) + res := rw.Result() + + assert.Equal(s.T(), 304, res.StatusCode) + assert.Equal(s.T(), etag, res.Header.Get("ETag")) +} + +func (s *ProcessingHandlerTestSuite) TestETagReqNotMatch() { + config.ETagEnabled = true + + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Equal(s.T(), `"loremipsum"`, r.Header.Get("If-None-Match")) + + rw.Header().Set("ETag", `"loremipsumdolor"`) + rw.WriteHeader(200) + rw.Write(s.readTestFile("test1.png")) + })) + defer ts.Close() + + header := make(http.Header) + header.Set("If-None-Match", `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/RImxvcmVtaXBzdW0i"`) + + rw := s.send("/unsafe/rs:fill:4:4/plain/"+ts.URL, header) + res := rw.Result() + + assert.Equal(s.T(), 200, res.StatusCode) + assert.Equal( + s.T(), + `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/RImxvcmVtaXBzdW1kb2xvciI"`, + res.Header.Get("ETag"), + ) +} + +func (s *ProcessingHandlerTestSuite) TestETagDataNotMatch() { + config.ETagEnabled = true + + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Empty(s.T(), r.Header.Get("If-None-Match")) + + rw.WriteHeader(200) + rw.Write(s.readTestFile("test1.png")) + })) + defer ts.Close() + + header := make(http.Header) + header.Set("If-None-Match", `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/Dl29MNvkqdLEq"`) + + rw := s.send("/unsafe/rs:fill:4:4/plain/"+ts.URL, header) + res := rw.Result() + + assert.Equal(s.T(), 200, res.StatusCode) + assert.Equal( + s.T(), + `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/Dl29MNvkqdLEqPyFvMlh_NlPbRrsC0GDG_AUlmMdX6HA"`, + res.Header.Get("ETag"), + ) +} + +func (s *ProcessingHandlerTestSuite) TestETagProcessingOptionsNotMatch() { + config.ETagEnabled = true + + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + assert.Empty(s.T(), r.Header.Get("If-None-Match")) + + rw.Header().Set("ETag", `"loremipsumdolor"`) + rw.WriteHeader(200) + rw.Write(s.readTestFile("test1.png")) + })) + defer ts.Close() + + header := make(http.Header) + header.Set("If-None-Match", `"1Uuny6YTSUO08MMVZ/Dl29MNvkqdLEq"`) + + rw := s.send("/unsafe/rs:fill:4:4/plain/"+ts.URL, header) + res := rw.Result() + + assert.Equal(s.T(), 200, res.StatusCode) + assert.Equal( + s.T(), + `"1Uuny6YTSUO08MMVZyantp9oaoOcsaYibQoNbBTJEzk/RImxvcmVtaXBzdW1kb2xvciI"`, + res.Header.Get("ETag"), + ) +} + func TestProcessingHandler(t *testing.T) { suite.Run(t, new(ProcessingHandlerTestSuite)) } diff --git a/transport/azure/azuret.go b/transport/azure/azuret.go index 4f1b2dcb..6de55c66 100644 --- a/transport/azure/azuret.go +++ b/transport/azure/azuret.go @@ -46,5 +46,27 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error) return nil, err } + if config.ETagEnabled { + etag := string(get.ETag()) + + if etag == req.Header.Get("If-None-Match") { + if body := get.Response().Body; body != nil { + get.Response().Body.Close() + } + + return &http.Response{ + StatusCode: http.StatusNotModified, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Header: make(http.Header), + ContentLength: 0, + Body: nil, + Close: false, + Request: req, + }, nil + } + } + return get.Response(), nil } diff --git a/transport/fs/fs.go b/transport/fs/fs.go index 90193dcd..3a860155 100644 --- a/transport/fs/fs.go +++ b/transport/fs/fs.go @@ -1,7 +1,10 @@ package fs import ( + "crypto/md5" + "encoding/base64" "fmt" + "io/fs" "net/http" "github.com/imgproxy/imgproxy/v2/config" @@ -31,16 +34,43 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error) return nil, fmt.Errorf("%s is a directory", req.URL.Path) } + header := make(http.Header) + + if config.ETagEnabled { + etag := BuildEtag(req.URL.Path, fi) + header.Set("ETag", etag) + + if etag == req.Header.Get("If-None-Match") { + return &http.Response{ + StatusCode: http.StatusNotModified, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Header: header, + ContentLength: 0, + Body: nil, + Close: false, + Request: req, + }, nil + } + } + return &http.Response{ Status: "200 OK", StatusCode: 200, Proto: "HTTP/1.0", ProtoMajor: 1, ProtoMinor: 0, - Header: make(http.Header), + Header: header, ContentLength: fi.Size(), Body: f, Close: true, Request: req, }, nil } + +func BuildEtag(path string, fi fs.FileInfo) string { + tag := fmt.Sprintf("%s__%d__%d", path, fi.Size(), fi.ModTime().UnixNano()) + hash := md5.Sum([]byte(tag)) + return `"` + string(base64.RawURLEncoding.EncodeToString(hash[:])) + `"` +} diff --git a/transport/gcs/gcs.go b/transport/gcs/gcs.go index 06919341..8e31cb18 100644 --- a/transport/gcs/gcs.go +++ b/transport/gcs/gcs.go @@ -43,13 +43,35 @@ func (t transport) RoundTrip(req *http.Request) (*http.Response, error) { obj = obj.Generation(g) } - reader, err := obj.NewReader(context.Background()) + header := make(http.Header) + if config.ETagEnabled { + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, err + } + header.Set("ETag", attrs.Etag) + + if attrs.Etag == req.Header.Get("If-None-Match") { + return &http.Response{ + StatusCode: http.StatusNotModified, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Header: header, + ContentLength: 0, + Body: nil, + Close: false, + Request: req, + }, nil + } + } + + reader, err := obj.NewReader(context.Background()) if err != nil { return nil, err } - header := make(http.Header) header.Set("Cache-Control", reader.Attrs.CacheControl) return &http.Response{ diff --git a/transport/s3/s3.go b/transport/s3/s3.go index 33f17e27..4dcc88f5 100644 --- a/transport/s3/s3.go +++ b/transport/s3/s3.go @@ -50,6 +50,12 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error) input.VersionId = aws.String(req.URL.RawQuery) } + if config.ETagEnabled { + if ifNoneMatch := req.Header.Get("If-None-Match"); len(ifNoneMatch) > 0 { + input.IfNoneMatch = aws.String(ifNoneMatch) + } + } + s3req, _ := t.svc.GetObjectRequest(input) if err := s3req.Send(); err != nil {