diff --git a/libavutil/imgutils.c b/libavutil/imgutils.c index 3103efbe4a..8581150771 100644 --- a/libavutil/imgutils.c +++ b/libavutil/imgutils.c @@ -43,27 +43,41 @@ void av_image_fill_max_pixsteps(int max_pixsteps[4], int max_pixstep_comps[4], } } -int av_image_get_linesize(enum PixelFormat pix_fmt, int width, int plane) +static inline +int image_get_linesize(int width, int plane, + int max_step, int max_step_comp, + const AVPixFmtDescriptor *desc) { - const AVPixFmtDescriptor *desc = &av_pix_fmt_descriptors[pix_fmt]; - int max_step [4]; /* max pixel step for each plane */ - int max_step_comp[4]; /* the component for each plane which has the max pixel step */ - int s, linesize; + int s, shifted_w, linesize; - if ((unsigned)pix_fmt >= PIX_FMT_NB || desc->flags & PIX_FMT_HWACCEL) + if (width < 0) return AVERROR(EINVAL); - - av_image_fill_max_pixsteps(max_step, max_step_comp, desc); - s = (max_step_comp[plane] == 1 || max_step_comp[plane] == 2) ? desc->log2_chroma_w : 0; - linesize = max_step[plane] * (((width + (1 << s) - 1)) >> s); + s = (max_step_comp == 1 || max_step_comp == 2) ? desc->log2_chroma_w : 0; + shifted_w = ((width + (1 << s) - 1)) >> s; + if (shifted_w && max_step > INT_MAX / shifted_w) + return AVERROR(EINVAL); + linesize = max_step * shifted_w; if (desc->flags & PIX_FMT_BITSTREAM) linesize = (linesize + 7) >> 3; return linesize; } +int av_image_get_linesize(enum PixelFormat pix_fmt, int width, int plane) +{ + const AVPixFmtDescriptor *desc = &av_pix_fmt_descriptors[pix_fmt]; + int max_step [4]; /* max pixel step for each plane */ + int max_step_comp[4]; /* the component for each plane which has the max pixel step */ + + if ((unsigned)pix_fmt >= PIX_FMT_NB || desc->flags & PIX_FMT_HWACCEL) + return AVERROR(EINVAL); + + av_image_fill_max_pixsteps(max_step, max_step_comp, desc); + return image_get_linesize(width, plane, max_step[plane], max_step_comp[plane], desc); +} + int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int width) { - int i; + int i, ret; const AVPixFmtDescriptor *desc = &av_pix_fmt_descriptors[pix_fmt]; int max_step [4]; /* max pixel step for each plane */ int max_step_comp[4]; /* the component for each plane which has the max pixel step */ @@ -75,13 +89,9 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt av_image_fill_max_pixsteps(max_step, max_step_comp, desc); for (i = 0; i < 4; i++) { - int s = (max_step_comp[i] == 1 || max_step_comp[i] == 2) ? desc->log2_chroma_w : 0; - int shifted_w = ((width + (1 << s) - 1)) >> s; - if (max_step[i] > INT_MAX / shifted_w) - return AVERROR(EINVAL); - linesizes[i] = max_step[i] * shifted_w; - if (desc->flags & PIX_FMT_BITSTREAM) - linesizes[i] = (linesizes[i] + 7) >> 3; + if ((ret = image_get_linesize(width, i, max_step[i], max_step_comp[i], desc)) < 0) + return ret; + linesizes[i] = ret; } return 0;