diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index 43d35934c8..cde060aa1f 100644
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -25,7 +25,6 @@
 #include "libavutil/imgutils.h"
 
 #include "avcodec.h"
-#include "motion_est.h"
 #include "mpegpicture.h"
 #include "refstruct.h"
 
@@ -136,8 +135,8 @@ void ff_mpv_workpic_from_pic(MPVWorkPicture *wpic, MPVPicture *pic)
     set_workpic_from_pic(wpic, pic);
 }
 
-int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
-                            ScratchpadContext *sc, int linesize)
+int ff_mpv_framesize_alloc(AVCodecContext *avctx,
+                           ScratchpadContext *sc, int linesize)
 {
 #   define EMU_EDGE_HEIGHT (4 * 70)
     int linesizeabs = FFABS(linesize);
@@ -158,7 +157,7 @@ int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
         return AVERROR(ENOMEM);
 
     av_freep(&sc->edge_emu_buffer);
-    av_freep(&me->scratchpad);
+    av_freep(&sc->scratchpad_buf);
 
     // edge emu needs blocksize + filter length - 1
     // (= 17x17 for  halfpel / 21x21 for H.264)
@@ -167,16 +166,14 @@ int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
     // linesize * interlaced * MBsize
     // we also use this buffer for encoding in encode_mb_internal() needig an additional 32 lines
     if (!FF_ALLOCZ_TYPED_ARRAY(sc->edge_emu_buffer, alloc_size * EMU_EDGE_HEIGHT) ||
-        !FF_ALLOCZ_TYPED_ARRAY(me->scratchpad,      alloc_size * 4 * 16 * 2)) {
+        !FF_ALLOCZ_TYPED_ARRAY(sc->scratchpad_buf,  alloc_size * 4 * 16 * 2)) {
         sc->linesize = 0;
         av_freep(&sc->edge_emu_buffer);
         return AVERROR(ENOMEM);
     }
     sc->linesize = linesizeabs;
 
-    me->temp            = me->scratchpad;
-    sc->b_scratchpad    = me->scratchpad;
-    sc->obmc_scratchpad = me->scratchpad + 16;
+    sc->obmc_scratchpad = sc->scratchpad_buf + 16;
 
     return 0;
 }
@@ -238,14 +235,13 @@ static int alloc_picture_tables(BufferPoolContext *pools, MPVPicture *pic,
 }
 
 int ff_mpv_alloc_pic_accessories(AVCodecContext *avctx, MPVWorkPicture *wpic,
-                                 MotionEstContext *me, ScratchpadContext *sc,
+                                 ScratchpadContext *sc,
                                  BufferPoolContext *pools, int mb_height)
 {
     MPVPicture *pic = wpic->ptr;
     int ret;
 
-    ret = ff_mpeg_framesize_alloc(avctx, me, sc,
-                                  pic->f->linesize[0]);
+    ret = ff_mpv_framesize_alloc(avctx, sc, pic->f->linesize[0]);
     if (ret < 0)
         goto fail;
 
diff --git a/libavcodec/mpegpicture.h b/libavcodec/mpegpicture.h
index ddb3ac5a2b..86504fe8ca 100644
--- a/libavcodec/mpegpicture.h
+++ b/libavcodec/mpegpicture.h
@@ -25,7 +25,6 @@
 #include <stdint.h>
 
 #include "avcodec.h"
-#include "motion_est.h"
 #include "threadprogress.h"
 
 #define MPV_MAX_PLANES 3
@@ -35,6 +34,7 @@ typedef struct ScratchpadContext {
     uint8_t *edge_emu_buffer;     ///< temporary buffer for if MVs point to out-of-frame data
     uint8_t *obmc_scratchpad;
     union {
+        uint8_t *scratchpad_buf;  ///< the other *_scratchpad point into this buffer
         uint8_t *b_scratchpad;    ///< scratchpad used for writing into write only buffers
         uint8_t *rd_scratchpad;   ///< scratchpad for rate distortion mb decision
     };
@@ -121,7 +121,7 @@ struct FFRefStructPool *ff_mpv_alloc_pic_pool(int init_progress);
  * and set the MPVWorkPicture's fields.
  */
 int ff_mpv_alloc_pic_accessories(AVCodecContext *avctx, MPVWorkPicture *pic,
-                                 MotionEstContext *me, ScratchpadContext *sc,
+                                 ScratchpadContext *sc,
                                  BufferPoolContext *pools, int mb_height);
 
 /**
@@ -133,8 +133,8 @@ int ff_mpv_alloc_pic_accessories(AVCodecContext *avctx, MPVWorkPicture *pic,
 int ff_mpv_pic_check_linesize(void *logctx, const struct AVFrame *f,
                               ptrdiff_t *linesizep, ptrdiff_t *uvlinesizep);
 
-int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
-                            ScratchpadContext *sc, int linesize);
+int ff_mpv_framesize_alloc(AVCodecContext *avctx,
+                           ScratchpadContext *sc, int linesize);
 
 void ff_mpv_unref_picture(MPVWorkPicture *pic);
 void ff_mpv_workpic_from_pic(MPVWorkPicture *wpic, MPVPicture *pic);
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 36947c6e31..89d19a743a 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -438,9 +438,8 @@ static void free_duplicate_context(MpegEncContext *s)
         return;
 
     av_freep(&s->sc.edge_emu_buffer);
-    av_freep(&s->me.scratchpad);
-    s->me.temp =
-    s->sc.b_scratchpad =
+    av_freep(&s->sc.scratchpad_buf);
+    s->me.temp = s->me.scratchpad =
     s->sc.obmc_scratchpad = NULL;
     s->sc.linesize = 0;
 
@@ -465,8 +464,6 @@ static void backup_duplicate_context(MpegEncContext *bak, MpegEncContext *src)
 {
 #define COPY(a) bak->a = src->a
     COPY(sc);
-    COPY(me.scratchpad);
-    COPY(me.temp);
     COPY(me.map);
     COPY(me.score_map);
     COPY(blocks);
@@ -500,8 +497,7 @@ int ff_update_duplicate_context(MpegEncContext *dst, const MpegEncContext *src)
         // exchange uv
         FFSWAP(void *, dst->pblocks[4], dst->pblocks[5]);
     }
-    ret = ff_mpeg_framesize_alloc(dst->avctx, &dst->me,
-                                  &dst->sc, dst->linesize);
+    ret = ff_mpv_framesize_alloc(dst->avctx, &dst->sc, dst->linesize);
     if (ret < 0) {
         av_log(dst->avctx, AV_LOG_ERROR, "failed to allocate context "
                "scratch buffers.\n");
diff --git a/libavcodec/mpegvideo_dec.c b/libavcodec/mpegvideo_dec.c
index b7f72ad460..9d2b7671e3 100644
--- a/libavcodec/mpegvideo_dec.c
+++ b/libavcodec/mpegvideo_dec.c
@@ -155,8 +155,7 @@ int ff_mpeg_update_thread_context(AVCodecContext *dst,
     }
 
     // linesize-dependent scratch buffer allocation
-    ret = ff_mpeg_framesize_alloc(s->avctx, &s->me,
-                                  &s->sc, s1->linesize);
+    ret = ff_mpv_framesize_alloc(s->avctx, &s->sc, s1->linesize);
     if (ret < 0) {
         av_log(s->avctx, AV_LOG_ERROR, "Failed to allocate context "
                "scratch buffers.\n");
@@ -264,7 +263,7 @@ static int alloc_picture(MpegEncContext *s, MPVWorkPicture *dst, int reference)
     av_assert1(s->mb_height == s->buffer_pools.alloc_mb_height ||
                FFALIGN(s->mb_height, 2) == s->buffer_pools.alloc_mb_height);
     av_assert1(s->mb_stride == s->buffer_pools.alloc_mb_stride);
-    ret = ff_mpv_alloc_pic_accessories(s->avctx, dst, &s->me, &s->sc,
+    ret = ff_mpv_alloc_pic_accessories(s->avctx, dst, &s->sc,
                                        &s->buffer_pools, s->mb_height);
     if (ret < 0)
         goto fail;
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index d131be964b..eaaf366624 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -1663,12 +1663,13 @@ static int select_input_picture(MpegEncContext *s)
         av_assert1(s->mb_width  == s->buffer_pools.alloc_mb_width);
         av_assert1(s->mb_height == s->buffer_pools.alloc_mb_height);
         av_assert1(s->mb_stride == s->buffer_pools.alloc_mb_stride);
-        ret = ff_mpv_alloc_pic_accessories(s->avctx, &s->cur_pic, &s->me,
+        ret = ff_mpv_alloc_pic_accessories(s->avctx, &s->cur_pic,
                                            &s->sc, &s->buffer_pools, s->mb_height);
         if (ret < 0) {
             ff_mpv_unref_picture(&s->cur_pic);
             return ret;
         }
+        s->me.temp = s->me.scratchpad = s->sc.scratchpad_buf;
         s->picture_number = s->cur_pic.ptr->display_picture_number;
 
     }
@@ -3616,9 +3617,11 @@ static int encode_picture(MpegEncContext *s)
 
     s->mb_intra=0; //for the rate distortion & bit compare functions
     for(i=1; i<context_count; i++){
-        ret = ff_update_duplicate_context(s->thread_context[i], s);
+        MpegEncContext *const slice = s->thread_context[i];
+        ret = ff_update_duplicate_context(slice, s);
         if (ret < 0)
             return ret;
+        slice->me.temp = slice->me.scratchpad = slice->sc.scratchpad_buf;
     }
 
     /* Estimate motion for every MB */
diff --git a/libavcodec/svq1enc.c b/libavcodec/svq1enc.c
index 9c9be8c6b3..35413b8afd 100644
--- a/libavcodec/svq1enc.c
+++ b/libavcodec/svq1enc.c
@@ -543,15 +543,15 @@ static av_cold int svq1_encode_end(AVCodecContext *avctx)
                s->rd_total / (double)(avctx->width * avctx->height *
                                       avctx->frame_num));
 
-    s->m.mb_type = NULL;
-    ff_mpv_common_end(&s->m);
-
     av_freep(&s->m.me.scratchpad);
     av_freep(&s->m.me.map);
     av_freep(&s->mb_type);
     av_freep(&s->dummy);
     av_freep(&s->scratchbuf);
 
+    s->m.mb_type = NULL;
+    ff_mpv_common_end(&s->m);
+
     for (i = 0; i < 3; i++) {
         av_freep(&s->motion_val8[i]);
         av_freep(&s->motion_val16[i]);