vmdav: Try to fix unpack_rle()

This fixes out of array accesses
The code prior to this commit could not have worked, thus obviously
was untested. I was also not able to find a valid sample that uses this
code.
This fix is thus only based on the description of the format

If someone has a sample that uses unpack_rle(), please mail me.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit c1f2c4c3b4)

Conflicts:

	libavcodec/vmdav.c

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer
2013-05-01 23:46:38 +02:00
parent 039f6921c2
commit 0baa0a5a02

View File

@@ -162,7 +162,7 @@ static int rle_unpack(const unsigned char *src, int src_len, int src_count,
const unsigned char *ps; const unsigned char *ps;
const unsigned char *ps_end; const unsigned char *ps_end;
unsigned char *pd; unsigned char *pd;
int i, l; int i, j, l;
unsigned char *dest_end = dest + dest_len; unsigned char *dest_end = dest + dest_len;
ps = src; ps = src;
@@ -188,9 +188,9 @@ static int rle_unpack(const unsigned char *src, int src_len, int src_count,
ps += l; ps += l;
pd += l; pd += l;
} else { } else {
if (dest_end - pd < i || ps_end - ps < 2) if (dest_end - pd < 2*l || ps_end - ps < 2)
return ps - src; return ps - src;
for (i = 0; i < l; i++) { for (j = 0; j < l; j++) {
*pd++ = ps[0]; *pd++ = ps[0];
*pd++ = ps[1]; *pd++ = ps[1];
} }