libtheora: cherry-picked some fixes from upstream:

- Sanity improvements to oc_dec_headerin() error returns. (50fd339b4)
- Fix mismatched oc_mb_fill_cmapping11 signature. (12f20c7a5)
- Correct an operator precedence error in the visualization code.  (6b94bf905)
- Remove some redundant NULL checks (de386d3ea)

media/libtheora/lib/decinfo.c

@@ -174,7 +174,23 @@ static int oc_dec_headerin(oc_pack_buf *_opb,th_info *_info,
   packtype=(int)val;
   /*If we're at a data packet and we have received all three headers, we're
      done.*/
-  if(!(packtype&0x80)&&_info->frame_width>0&&_tc->vendor!=NULL&&*_setup!=NULL){
+  /*If we're at a data packet...*/
+  if(!(packtype&0x80)){
+    /*Check to make sure we received all three headers...
+      If we haven't seen any valid headers, assume this is not actually
+       Theora.*/
+    if(_info->frame_width<=0)return TH_ENOTFORMAT;
+    /*Follow our documentation, which says we'll return TH_EFAULT if this
+       are NULL (_info was checked by our caller).*/
+    if(_tc==NULL)return TH_EFAULT;
+    /*And if any other headers were missing, declare this packet "out of
+       sequence" instead.*/
+    if(_tc->vendor==NULL)return TH_EBADHEADER;
+    /*Don't check this until it's needed, since we allow passing NULL for the
+       arguments that we're not expecting the next header to fill in yet.*/
+    if(_setup==NULL)return TH_EFAULT;
+    if(*_setup==NULL)return TH_EBADHEADER;
+    /*If we got everything, we're done.*/
     return 0;
   }
   /*Check the codec string.*/