vf_unsharp: fix out-of-buffer read

In apply_unsharp(), when y is >= height, prevent out-of-buffer reading
from src, read from the last buffer line in src2 instead.

The check was implemented in the original unsharp libmpcodecs code and
lost in the port.

This also fixes output discrepancy between the two filters.

Signed-off-by: Anton Khirnov <anton@khirnov.net>
(cherry picked from commit 998e8519ef)

Signed-off-by: Anton Khirnov <anton@khirnov.net>
Signed-off-by: Reinhard Tartler <siretart@tauware.de>
This commit is contained in:
Stefano Sabatini
2011-08-12 08:42:35 +02:00
committed by Reinhard Tartler
parent 7f62cf120b
commit 734a9bb05f

View File

@@ -70,6 +70,7 @@ static void unsharpen(uint8_t *dst, uint8_t *src, int dst_stride, int src_stride
int32_t res;
int x, y, z;
const uint8_t *src2;
if (!fp->amount) {
if (dst_stride == src_stride)
@@ -84,9 +85,12 @@ static void unsharpen(uint8_t *dst, uint8_t *src, int dst_stride, int src_stride
memset(sc[y], 0, sizeof(sc[y][0]) * (width + 2 * fp->steps_x));
for (y = -fp->steps_y; y < height + fp->steps_y; y++) {
if (y < height)
src2 = src;
memset(sr, 0, sizeof(sr[0]) * (2 * fp->steps_x - 1));
for (x = -fp->steps_x; x < width + fp->steps_x; x++) {
tmp1 = x <= 0 ? src[0] : x >= width ? src[width-1] : src[x];
tmp1 = x <= 0 ? src2[0] : x >= width ? src2[width-1] : src2[x];
for (z = 0; z < fp->steps_x * 2; z += 2) {
tmp2 = sr[z + 0] + tmp1; sr[z + 0] = tmp1;
tmp1 = sr[z + 1] + tmp2; sr[z + 1] = tmp2;