Bug 1444668 - Avoid allocating large AssemblerBuffers. r=luke, r=bbouvier, a=RyanVM

js/src/jit/MacroAssembler.cpp

@@ -2214,6 +2214,12 @@ MacroAssembler::finish()
     }
 
     MacroAssemblerSpecific::finish();
+
+    MOZ_RELEASE_ASSERT(size() <= MaxCodeBytesPerProcess,
+                       "AssemblerBuffer should ensure we don't exceed MaxCodeBytesPerProcess");
+
+    if (bytesNeeded() > MaxCodeBytesPerProcess)
+        setOOM();
 }
 
 void

js/src/jit/ProcessExecutableMemory.cpp

@@ -385,14 +385,6 @@ class PageBitSet
 #endif
 };
 
-// Limit on the number of bytes of executable memory to prevent JIT spraying
-// attacks.
-#if JS_BITS_PER_WORD == 32
-static const size_t MaxCodeBytesPerProcess = 128 * 1024 * 1024;
-#else
-static const size_t MaxCodeBytesPerProcess = 1 * 1024 * 1024 * 1024;
-#endif
-
 // Per-process executable memory allocator. It reserves a block of memory of
 // MaxCodeBytesPerProcess bytes, then allocates/deallocates pages from that.
 //

js/src/jit/ProcessExecutableMemory.h

@@ -17,6 +17,14 @@ namespace jit {
 // alignment though.
 static const size_t ExecutableCodePageSize = 64 * 1024;
 
+// Limit on the number of bytes of executable memory to prevent JIT spraying
+// attacks.
+#if JS_BITS_PER_WORD == 32
+static const size_t MaxCodeBytesPerProcess = 128 * 1024 * 1024;
+#else
+static const size_t MaxCodeBytesPerProcess = 1 * 1024 * 1024 * 1024;
+#endif
+
 enum class ProtectionSetting {
     Protected, // Not readable, writable, or executable.
     Writable,

js/src/jit/shared/IonAssemblerBuffer.h

@@ -181,6 +181,10 @@ class AssemblerBuffer
 
   protected:
     virtual Slice* newSlice(LifoAlloc& a) {
+        if (size() > MaxCodeBytesPerProcess - sizeof(Slice)) {
+            fail_oom();
+            return nullptr;
+        }
         Slice* tmp = static_cast<Slice*>(a.alloc(sizeof(Slice)));
         if (!tmp) {
             fail_oom();

js/src/jit/x86-shared/AssemblerBuffer-x86-shared.h

@@ -68,6 +68,33 @@ namespace js {
 
 namespace jit {
 
+    // AllocPolicy for AssemblerBuffer. OOMs when trying to allocate more than
+    // MaxCodeBytesPerProcess bytes. Use private inheritance to make sure we
+    // explicitly have to expose SystemAllocPolicy methods.
+    class AssemblerBufferAllocPolicy : private SystemAllocPolicy
+    {
+      public:
+        using SystemAllocPolicy::checkSimulatedOOM;
+        using SystemAllocPolicy::reportAllocOverflow;
+        using SystemAllocPolicy::free_;
+
+        template <typename T> T* pod_realloc(T* p, size_t oldSize, size_t newSize) {
+            static_assert(sizeof(T) == 1,
+                          "AssemblerBufferAllocPolicy should only be used with byte vectors");
+            MOZ_ASSERT(oldSize <= MaxCodeBytesPerProcess);
+            if (MOZ_UNLIKELY(newSize > MaxCodeBytesPerProcess))
+                return nullptr;
+            return SystemAllocPolicy::pod_realloc<T>(p, oldSize, newSize);
+        }
+        template <typename T> T* pod_malloc(size_t numElems) {
+            static_assert(sizeof(T) == 1,
+                          "AssemblerBufferAllocPolicy should only be used with byte vectors");
+            if (MOZ_UNLIKELY(numElems > MaxCodeBytesPerProcess))
+                return nullptr;
+            return SystemAllocPolicy::pod_malloc<T>(numElems);
+        }
+    };
+
     class AssemblerBuffer
     {
         template<size_t size, typename T>
@@ -93,6 +120,9 @@ namespace jit {
 
         void ensureSpace(size_t space)
         {
+            // This should only be called with small |space| values to ensure
+            // we don't overflow below.
+            MOZ_ASSERT(space <= 16);
             if (MOZ_UNLIKELY(!m_buffer.reserve(m_buffer.length() + space)))
                 oomDetected();
         }
@@ -168,7 +198,7 @@ namespace jit {
             m_buffer.clear();
         }
 
-        PageProtectingVector<unsigned char, 256, SystemAllocPolicy> m_buffer;
+        PageProtectingVector<unsigned char, 256, AssemblerBufferAllocPolicy> m_buffer;
         bool m_oom;
     };