vf_unsharp: fix out-of-buffer read

In apply_unsharp(), when y is >= height, prevent out-of-buffer reading
from src, read from the last buffer line in src2 instead.

The check was implemented in the original unsharp libmpcodecs code and
lost in the port.

This also fixes output discrepancy between the two filters.

Signed-off-by: Anton Khirnov <anton@khirnov.net>
(cherry picked from commit 998e8519efbc772994c5ba19c0d39573998be9db)
This commit is contained in:
Stefano Sabatini 2011-08-12 08:42:35 +02:00 committed by Michael Niedermayer
parent d414c77ded
commit d155fdefb8

View File

@ -70,6 +70,7 @@ static void unsharpen(uint8_t *dst, const uint8_t *src, int dst_stride, int src_
int32_t res;
int x, y, z;
const uint8_t *src2;
if (!fp->amount) {
if (dst_stride == src_stride)
@ -84,9 +85,12 @@ static void unsharpen(uint8_t *dst, const uint8_t *src, int dst_stride, int src_
memset(sc[y], 0, sizeof(sc[y][0]) * (width + 2 * fp->steps_x));
for (y = -fp->steps_y; y < height + fp->steps_y; y++) {
if (y < height)
src2 = src;
memset(sr, 0, sizeof(sr[0]) * (2 * fp->steps_x - 1));
for (x = -fp->steps_x; x < width + fp->steps_x; x++) {
tmp1 = x <= 0 ? src[0] : x >= width ? src[width-1] : src[x];
tmp1 = x <= 0 ? src2[0] : x >= width ? src2[width-1] : src2[x];
for (z = 0; z < fp->steps_x * 2; z += 2) {
tmp2 = sr[z + 0] + tmp1; sr[z + 0] = tmp1;
tmp1 = sr[z + 1] + tmp2; sr[z + 1] = tmp2;