shithub: dav1d

Info  •  Files  •  Log  •  Branches

coef/dequant: clip coefs before and after dequantization

Fixes #142 and a signed overflow in decode_coefs during dequantization
with /clusterfuzz-testcase-minimized-dav1d_fuzzer-5691270664552448.
Credits to oss-fuzz and Thierry.

--- a/src/recon_tmpl.c

+++ b/src/recon_tmpl.c

@@ -241,11 +241,15 @@

                    i, rc, tok - 15, tok, ts->msac.rng);

         }

-        // dequant

+        // coefficient parsing, see 5.11.39

+        tok &= 0xfffff;

+

+        // dequant, see 7.12.3

         cul_level += tok;

-        tok *= dq;

-        tok >>= dq_shift;

-        cf[rc] = sign ? -tok : tok;

+        tok = (((int64_t)dq * tok) & 0xffffff) >> dq_shift;

+        cf[rc] = iclip(sign ? -tok : tok,

+                       -(1 << (7 + BITDEPTH)),

+                       (1 << (7 + BITDEPTH)) - 1);

     }

     // context