Bug 1388020. r=nical, a=RyanVM

gfx/gl/GLTextureImage.cpp

@@ -149,6 +149,9 @@ BasicTextureImage::DirectUpdate(gfx::DataSourceSurface* aSurf, const nsIntRegion
                                &uploadSize,
                                needInit,
                                aFrom);
+    if (mTextureFormat == SurfaceFormat::UNKNOWN) {
+        return false;
+    }
 
     if (uploadSize > 0) {
         UpdateUploadSize(uploadSize);

gfx/gl/GLUploadHelpers.cpp

@@ -27,6 +27,23 @@ DataOffset(const IntPoint& aPoint, int32_t aStride, SurfaceFormat aFormat)
   return data;
 }
 
+static bool
+CheckUploadBounds(const IntSize& aDst, const IntSize& aSrc, const IntPoint& aOffset)
+{
+  if (aOffset.x < 0 || aOffset.y < 0 ||
+      aOffset.x >= aSrc.width ||
+      aOffset.y >= aSrc.height) {
+    MOZ_ASSERT_UNREACHABLE("Offset outside source bounds");
+    return false;
+  }
+  if (aDst.width > (aSrc.width - aOffset.x) ||
+      aDst.height > (aSrc.height - aOffset.y)) {
+    MOZ_ASSERT_UNREACHABLE("Source has insufficient data");
+    return false;
+  }
+  return true;
+}
+
 static GLint GetAddressAlignment(ptrdiff_t aAddress)
 {
     if (!(aAddress & 0x7)) {
@@ -375,6 +392,7 @@ TexImage2DHelper(GLContext* gl,
 SurfaceFormat
 UploadImageDataToTexture(GLContext* gl,
                          unsigned char* aData,
+                         const gfx::IntSize& aDataSize,
                          int32_t aStride,
                          SurfaceFormat aFormat,
                          const nsIntRegion& aDstRegion,
@@ -498,6 +516,10 @@ UploadImageDataToTexture(GLContext* gl,
         // Upload each rect in the region to the texture
         for (auto iter = aDstRegion.RectIter(); !iter.Done(); iter.Next()) {
             const IntRect& rect = iter.Get();
+            if (!CheckUploadBounds(rect.Size(), aDataSize, rect.TopLeft())) {
+                return SurfaceFormat::UNKNOWN;
+            }
+
             const unsigned char* rectData =
                 aData + DataOffset(rect.TopLeft(), aStride, aFormat);
 
@@ -534,10 +556,17 @@ UploadSurfaceToTexture(GLContext* gl,
 
     int32_t stride = aSurface->Stride();
     SurfaceFormat format = aSurface->GetFormat();
+    gfx::IntSize size = aSurface->GetSize();
+    if (!CheckUploadBounds(aSize, size, aSrcPoint)) {
+        return SurfaceFormat::UNKNOWN;
+    }
+
     unsigned char* data = aSurface->GetData() +
         DataOffset(aSrcPoint, stride, format);
+    size.width -= aSrcPoint.x;
+    size.height -= aSrcPoint.y;
 
-    return UploadImageDataToTexture(gl, data, stride, format,
+    return UploadImageDataToTexture(gl, data, size, stride, format,
                                     aDstRegion, aTexture, aSize,
                                     aOutUploadSize, aNeedInit,
                                     aTextureUnit, aTextureTarget);

gfx/gl/GLUploadHelpers.h

@@ -28,6 +28,7 @@ class GLContext;
  * \param gl The GL Context to use.
  * \param aData Start of image data of surface to upload.
  *              Corresponds to the first pixel of the texture.
+ * \param aDataSize The image data's size.
  * \param aStride The image data's stride.
  * \param aFormat The image data's format.
  * \param aDstRegion Region of the texture to upload.
@@ -46,6 +47,7 @@ class GLContext;
 gfx::SurfaceFormat
 UploadImageDataToTexture(GLContext* gl,
                          unsigned char* aData,
+                         const gfx::IntSize& aDataSize,
                          int32_t aStride,
                          gfx::SurfaceFormat aFormat,
                          const nsIntRegion& aDstRegion,

gfx/gl/TextureImageEGL.cpp

@@ -119,6 +119,10 @@ TextureImageEGL::DirectUpdate(gfx::DataSourceSurface* aSurf, const nsIntRegion&
                              &uploadSize,
                              needInit,
                              aFrom);
+    if (mTextureFormat == SurfaceFormat::UNKNOWN) {
+        return false;
+    }
+
     if (uploadSize > 0) {
         UpdateUploadSize(uploadSize);
     }

gfx/layers/composite/TextureHost.cpp

@@ -888,8 +888,7 @@ BufferTextureHost::Upload(nsIntRegion *aRegion)
         mFirstSource = mCompositor->CreateDataTextureSource(mFlags|TextureFlags::RGB_FROM_YCBCR);
         mFirstSource->SetOwner(this);
       }
-      mFirstSource->Update(surf, aRegion);
+      return mFirstSource->Update(surf, aRegion);
-      return true;
     }
 
     RefPtr<DataTextureSource> srcY;

gfx/layers/opengl/TextureHostOGL.cpp

@@ -161,9 +161,7 @@ TextureImageTextureSourceOGL::Update(gfx::DataSourceSurface* aSurface,
     }
   }
 
-  mTexImage->UpdateFromDataSource(aSurface, aDestRegion, aSrcOffset);
+  return mTexImage->UpdateFromDataSource(aSurface, aDestRegion, aSrcOffset);
-
-  return true;
 }
 
 void