shithub: dav1d

Info  •  Files  •  Log  •  Branches

Fix handling of some memory allocation failures

Continuing trying to decode after a memory allocation failure could
cause null pointer dereferences in certain scenarios.

--- a/src/decode.c

+++ b/src/decode.c

@@ -2623,6 +2623,20 @@

     const int n_ts = f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows;

     if (n_ts != f->n_ts) {

+        if (c->n_fc > 1) {

+            freep(&f->frame_thread.tile_start_off);

+            f->frame_thread.tile_start_off =

+                malloc(sizeof(*f->frame_thread.tile_start_off) * n_ts);

+            if (!f->frame_thread.tile_start_off) {

+                for (int n = 0; n < f->n_ts; n++) {

+                    Dav1dTileState *const ts = &f->ts[n];

+                    pthread_cond_destroy(&ts->tile_thread.cond);

+                    pthread_mutex_destroy(&ts->tile_thread.lock);

+                }

+                f->n_ts = 0;

+                goto error;

+            }

+        }

         if (n_ts > f->n_ts) {

             Dav1dTileState *ts_new = realloc(f->ts, sizeof(*f->ts) * n_ts);

             if (!ts_new) goto error;

@@ -2646,20 +2660,6 @@

             if (!ts_new) goto error;

             f->ts = ts_new;

         }

-        if (c->n_fc > 1) {

-            freep(&f->frame_thread.tile_start_off);

-            f->frame_thread.tile_start_off =

-                malloc(sizeof(*f->frame_thread.tile_start_off) * n_ts);

-            if (!f->frame_thread.tile_start_off) {

-                for (int n = 0; n < f->n_ts; n++) {

-                    Dav1dTileState *const ts = &f->ts[n];

-                    pthread_cond_destroy(&ts->tile_thread.cond);

-                    pthread_mutex_destroy(&ts->tile_thread.lock);

-                }

-                f->n_ts = 0;

-                goto error;

-            }

-        }

     }

     const int a_sz = f->sb128w * f->frame_hdr->tiling.rows;

@@ -3284,7 +3284,7 @@

         assert(c->n_tile_data < INT_MAX / (int)sizeof(*f->tile));

         f->tile = malloc(c->n_tile_data * sizeof(*f->tile));

         if (!f->tile) {

-            f->n_tile_data_alloc = 0;

+            f->n_tile_data_alloc = f->n_tile_data = 0;

             res = DAV1D_ERR(ENOMEM);

             goto error;

         }

--- a/src/ref_mvs.c

+++ b/src/ref_mvs.c

@@ -1988,7 +1988,10 @@

         const int align_h = (h8 + 15) & ~15;

         if (cm->tpl_mvs) free(cm->tpl_mvs);

         cm->tpl_mvs = malloc(sizeof(*cm->tpl_mvs) * (stride >> 1) * align_h);

-        if (!cm->tpl_mvs) return DAV1D_ERR(ENOMEM);

+        if (!cm->tpl_mvs) {

+            cm->mi_cols = cm->mi_rows = 0;

+            return DAV1D_ERR(ENOMEM);

+        }

         for (int i = 0; i < 7; i++)

             cm->frame_refs[i].idx = i;

         cm->mi_cols = w8 << 1;