From b848a924bca8a89dd55644e463e56ba27b062618 Mon Sep 17 00:00:00 2001
From: Moonchild <moonchild@palemoon.org>
Date: Wed, 2 Oct 2024 10:34:38 +0200
Subject: [PATCH] [GMP] Factor out more detailed CheckDimensions function for
 CreateFrame.

---
 dom/media/gmp/GMPVideoi420FrameImpl.cpp | 35 +++++++++++++++++++++----
 dom/media/gmp/GMPVideoi420FrameImpl.h   |  3 +++
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/dom/media/gmp/GMPVideoi420FrameImpl.cpp b/dom/media/gmp/GMPVideoi420FrameImpl.cpp
index fdbb9a9624..ed1f8cfea5 100644
--- a/dom/media/gmp/GMPVideoi420FrameImpl.cpp
+++ b/dom/media/gmp/GMPVideoi420FrameImpl.cpp
@@ -89,6 +89,34 @@ GMPVideoi420FrameImpl::CheckFrameData(const GMPVideoi420FrameData& aFrameData)
   return true;
 }
 
+bool
+GMPVideoi420FrameImpl::CheckDimensions(int32_t aWidth, int32_t aHeight,
+                                       int32_t aStride_y,
+                                       int32_t aStride_u,
+                                       int32_t aStride_v, int32_t aSize_y,
+                                       int32_t aSize_u, int32_t aSize_v) {
+  if (aWidth < 1 || aHeight < 1 || aStride_y < aWidth || aSize_y < 1 ||
+      aSize_u < 1 || aSize_v < 1) {
+    return false;
+  }
+  auto halfWidth = (CheckedInt<int32_t>(aWidth) + 1) / 2;
+  if (!halfWidth.isValid() || aStride_u < halfWidth.value() ||
+      aStride_v < halfWidth.value()) {
+    return false;
+  }
+  auto height = CheckedInt<int32_t>(aHeight);
+  auto halfHeight = (height + 1) / 2;
+  auto minSizeY = height * aStride_y;
+  auto minSizeU = halfHeight * aStride_u;
+  auto minSizeV = halfHeight * aStride_v;
+  if (!minSizeY.isValid() || !minSizeU.isValid() || !minSizeV.isValid() ||
+      minSizeY.value() > aSize_y || minSizeU.value() > aSize_u ||
+      minSizeV.value() > aSize_v) {
+    return false;
+  }
+  return true;
+}
+
 bool
 GMPVideoi420FrameImpl::CheckDimensions(int32_t aWidth, int32_t aHeight,
                                        int32_t aStride_y, int32_t aStride_u, int32_t aStride_v)
@@ -179,11 +207,8 @@ GMPVideoi420FrameImpl::CreateFrame(int32_t aSize_y, const uint8_t* aBuffer_y,
   MOZ_ASSERT(aBuffer_u);
   MOZ_ASSERT(aBuffer_v);
 
-  if (aSize_y < 1 || aSize_u < 1 || aSize_v < 1) {
-    return GMPGenericErr;
-  }
-
-  if (!CheckDimensions(aWidth, aHeight, aStride_y, aStride_u, aStride_v)) {
+  if (!CheckDimensions(aWidth, aHeight, aStride_y, aStride_u, aStride_v,
+                       aSize_y, aSize_u, aSize_v)) {
     return GMPGenericErr;
   }
 
diff --git a/dom/media/gmp/GMPVideoi420FrameImpl.h b/dom/media/gmp/GMPVideoi420FrameImpl.h
index f5cb0254b9..700162255b 100644
--- a/dom/media/gmp/GMPVideoi420FrameImpl.h
+++ b/dom/media/gmp/GMPVideoi420FrameImpl.h
@@ -65,6 +65,9 @@ public:
   void ResetSize() override;
 
 private:
+  bool CheckDimensions(int32_t aWidth, int32_t aHeight, int32_t aStride_y,
+                       int32_t aStride_u, int32_t aStride_v, int32_t aSize_y,
+                       int32_t aSize_u, int32_t aSize_v);
   bool CheckDimensions(int32_t aWidth, int32_t aHeight,
                        int32_t aStride_y, int32_t aStride_u, int32_t aStride_v);