Revamp SSE2 optimisations, moving them to one place so all of JS engine gets SSE2 optimizations

js/src/jsutil.h

@@ -19,6 +19,20 @@
 
 #include <limits.h>
 
+#if (defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64))
+# if defined(_MSC_VER)
+#  if defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
+#   define JS_HAVE_SSE2_INTRINSICS 1
+#  endif
+# elif defined(__SSE2__)
+#  define JS_HAVE_SSE2_INTRINSICS 1
+# endif
+#endif
+
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+# include <emmintrin.h>
+#endif
+
 #include "js/Utility.h"
 #include "js/Value.h"
 
@@ -41,9 +55,117 @@ js_memcpy(void* dst_, const void* src_, size_t len)
     MOZ_ASSERT_IF(dst >= src, (size_t) (dst - src) >= len);
     MOZ_ASSERT_IF(src >= dst, (size_t) (src - dst) >= len);
 
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    if (len >= 64) {
+        uint8_t* d = (uint8_t*)dst;
+        const uint8_t* s = (const uint8_t*)src;
+
+        while (len >= 64) {
+            __m128i v0 = _mm_loadu_si128((const __m128i*)(s + 0));
+            __m128i v1 = _mm_loadu_si128((const __m128i*)(s + 16));
+            __m128i v2 = _mm_loadu_si128((const __m128i*)(s + 32));
+            __m128i v3 = _mm_loadu_si128((const __m128i*)(s + 48));
+            _mm_storeu_si128((__m128i*)(d + 0), v0);
+            _mm_storeu_si128((__m128i*)(d + 16), v1);
+            _mm_storeu_si128((__m128i*)(d + 32), v2);
+            _mm_storeu_si128((__m128i*)(d + 48), v3);
+            d += 64;
+            s += 64;
+            len -= 64;
+        }
+
+        while (len >= 16) {
+            __m128i v = _mm_loadu_si128((const __m128i*)s);
+            _mm_storeu_si128((__m128i*)d, v);
+            d += 16;
+            s += 16;
+            len -= 16;
+        }
+
+        if (len) {
+            memcpy(d, s, len);
+        }
+        return dst;
+    }
+#endif
+
     return memcpy(dst, src, len);
 }
 
+static MOZ_ALWAYS_INLINE void*
+js_memmove(void* dst_, const void* src_, size_t len)
+{
+    char* dst = (char*) dst_;
+    const char* src = (const char*) src_;
+
+    if (dst == src || !len) {
+        return dst;
+    }
+
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    if (len >= 64) {
+        uint8_t* d = (uint8_t*)dst;
+        const uint8_t* s = (const uint8_t*)src;
+
+        if (d < s || d >= s + len) {
+            return js_memcpy(dst, src, len);
+        }
+
+        d += len;
+        s += len;
+        while (len >= 16) {
+            d -= 16;
+            s -= 16;
+            __m128i v = _mm_loadu_si128((const __m128i*)s);
+            _mm_storeu_si128((__m128i*)d, v);
+            len -= 16;
+        }
+
+        while (len--) {
+            *--d = *--s;
+        }
+        return dst;
+    }
+#endif
+
+    return memmove(dst, src, len);
+}
+
+static MOZ_ALWAYS_INLINE void*
+js_memset(void* dst_, uint8_t value, size_t len)
+{
+    char* dst = (char*) dst_;
+
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    if (len >= 64) {
+        uint8_t* d = (uint8_t*)dst;
+        __m128i v = _mm_set1_epi8((char)value);
+
+        while (len >= 64) {
+            _mm_storeu_si128((__m128i*)(d + 0), v);
+            _mm_storeu_si128((__m128i*)(d + 16), v);
+            _mm_storeu_si128((__m128i*)(d + 32), v);
+            _mm_storeu_si128((__m128i*)(d + 48), v);
+            d += 64;
+            len -= 64;
+        }
+
+        while (len >= 16) {
+            _mm_storeu_si128((__m128i*)d, v);
+            d += 16;
+            len -= 16;
+        }
+
+        while (len--) {
+            *d++ = (char)value;
+        }
+        return dst;
+    }
+#endif
+
+    return memset(dst, value, len);
+}
+
 namespace js {
 
 template <class T>

js/src/vm/ArrayBufferObject.cpp

@@ -3,8 +3,6 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#include <emmintrin.h> // SSE2
-
 #include "vm/ArrayBufferObject-inl.h"
 #include "vm/ArrayBufferObject.h"
 
@@ -67,58 +65,6 @@ using mozilla::Nothing;
 using namespace js;
 using namespace js::gc;
 
-// SSE2 version of memcpy_sse2_sse2 and memset_sse2_sse2 for faster JS n shit
-static inline void
-memcpy_sse2(void* dst, const void* src, size_t n)
-{
-    uint8_t* d = (uint8_t*)dst;
-    const uint8_t* s = (const uint8_t*)src;
-
-    // Align to 16 bytes
-    while (((uintptr_t)d & 15) && n) {
-        *d++ = *s++;
-        n--;
-    }
-
-    // 16‑byte chunks
-    while (n >= 16) {
-        __m128i v = _mm_loadu_si128((__m128i*)s);
-        _mm_storeu_si128((__m128i*)d, v);
-        s += 16;
-        d += 16;
-        n -= 16;
-    }
-
-    // tail
-    while (n--) {
-        *d++ = *s++;
-    }
-}
-
-static inline void
-memset_sse2(void* dst, uint8_t value, size_t n)
-{
-    uint8_t* d = (uint8_t*)dst;
-
-    // Align
-    while (((uintptr_t)d & 15) && n) {
-        *d++ = value;
-        n--;
-    }
-
-    __m128i v = _mm_set1_epi8(value);
-
-    while (n >= 16) {
-        _mm_storeu_si128((__m128i*)d, v);
-        d += 16;
-        n -= 16;
-    }
-
-    while (n--) {
-        *d++ = value;
-    }
-}
-
 /*
  * Convert |v| to an array index for an array of length |length| per
  * the Typed Array Specification section 7.0, |subarray|. If successful,
@@ -850,7 +796,7 @@ ArrayBufferObject::prepareForAsmJS(JSContext* cx, Handle<ArrayBufferObject*> buf
         }
 
         void* data = wasmBuf->dataPointer();
-        memcpy_sse2(data, buffer->dataPointer(), length);
+        js_memcpy(data, buffer->dataPointer(), length);
 
         // Swap the new elements into the ArrayBufferObject. Mark the
         // ArrayBufferObject so we don't do this again.
@@ -872,7 +818,7 @@ ArrayBufferObject::prepareForAsmJS(JSContext* cx, Handle<ArrayBufferObject*> buf
         BufferContents contents = AllocateArrayBufferContents(cx, buffer->byteLength());
         if (!contents)
             return false;
-        memcpy_sse2(contents.data(), buffer->dataPointer(), buffer->byteLength());
+        js_memcpy(contents.data(), buffer->dataPointer(), buffer->byteLength());
         buffer->changeContents(cx, contents, OwnsData);
     }
 
@@ -1049,7 +995,7 @@ ArrayBufferObject::wasmMovingGrowToSize(uint32_t newSize,
     BufferContents contents = BufferContents::create<WASM>(newRawBuf->dataPointer());
     newBuf->initialize(newSize, contents, OwnsData);
 
-    memcpy_sse2(newBuf->dataPointer(), oldBuf->dataPointer(), oldBuf->byteLength());
+    js_memcpy(newBuf->dataPointer(), oldBuf->dataPointer(), oldBuf->byteLength());
     ArrayBufferObject::detach(cx, oldBuf, BufferContents::createPlain(nullptr));
     return true;
 }
@@ -1151,7 +1097,7 @@ ArrayBufferObject::create(JSContext* cx, uint32_t nbytes, BufferContents content
 
     if (!contents) {
         void* data = obj->inlineDataPointer();
-        memset_sse2(data, 0, nbytes);
+        js_memset(data, 0, nbytes);
         obj->initialize(nbytes, BufferContents::createPlain(data), DoesntOwnData);
     } else {
         obj->initialize(nbytes, contents, ownsState);
@@ -1236,7 +1182,7 @@ ArrayBufferObject::externalizeContents(JSContext* cx, Handle<ArrayBufferObject*>
     BufferContents newContents = AllocateArrayBufferContents(cx, buffer->byteLength());
     if (!newContents)
         return BufferContents::createPlain(nullptr);
-    memcpy_sse2(newContents.data(), contents.data(), buffer->byteLength());
+    js_memcpy(newContents.data(), contents.data(), buffer->byteLength());
     buffer->changeContents(cx, newContents, DoesntOwnData);
 
     return newContents;
@@ -1271,7 +1217,7 @@ ArrayBufferObject::stealContents(JSContext* cx, Handle<ArrayBufferObject*> buffe
         return BufferContents::createPlain(nullptr);
 
     if (buffer->byteLength() > 0)
-        memcpy_sse2(contentsCopy.data(), oldContents.data(), buffer->byteLength());
+        js_memcpy(contentsCopy.data(), oldContents.data(), buffer->byteLength());
     ArrayBufferObject::detach(cx, buffer, oldContents);
     return contentsCopy;
 }
@@ -1324,7 +1270,7 @@ ArrayBufferObject::copyData(Handle<ArrayBufferObject*> toBuffer, uint32_t toInde
     MOZ_ASSERT(fromBuffer->byteLength() >= fromIndex);
     MOZ_ASSERT(fromBuffer->byteLength() >= fromIndex + count);
 
-    memcpy_sse2(toBuffer->dataPointer() + toIndex, fromBuffer->dataPointer() + fromIndex, count);
+    js_memcpy(toBuffer->dataPointer() + toIndex, fromBuffer->dataPointer() + fromIndex, count);
 }
 
 /* static */ void

js/src/vm/TypedArrayObject.cpp

@@ -3,8 +3,6 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#include <emmintrin.h> // SSE2
-
 #include "vm/TypedArrayObject.h"
 
 #include "mozilla/Alignment.h"
@@ -59,58 +57,6 @@ using JS::CanonicalizeNaN;
 using JS::ToInt32;
 using JS::ToUint32;
 
-// SSE2 version of memcpy_sse2 and memset_sse2 for faster JS n shit
-static inline void
-memcpy_sse2(void* dst, const void* src, size_t n)
-{
-    uint8_t* d = (uint8_t*)dst;
-    const uint8_t* s = (const uint8_t*)src;
-
-    // Align to 16 bytes
-    while (((uintptr_t)d & 15) && n) {
-        *d++ = *s++;
-        n--;
-    }
-
-    // 16‑byte chunks
-    while (n >= 16) {
-        __m128i v = _mm_loadu_si128((__m128i*)s);
-        _mm_storeu_si128((__m128i*)d, v);
-        s += 16;
-        d += 16;
-        n -= 16;
-    }
-
-    // tail
-    while (n--) {
-        *d++ = *s++;
-    }
-}
-
-static inline void
-memset_sse2(void* dst, uint8_t value, size_t n)
-{
-    uint8_t* d = (uint8_t*)dst;
-
-    // Align
-    while (((uintptr_t)d & 15) && n) {
-        *d++ = value;
-        n--;
-    }
-
-    __m128i v = _mm_set1_epi8(value);
-
-    while (n >= 16) {
-        _mm_storeu_si128((__m128i*)d, v);
-        d += 16;
-        n -= 16;
-    }
-
-    while (n--) {
-        *d++ = value;
-    }
-}
-
 
 /*
  * TypedArrayObject
@@ -198,7 +144,7 @@ TypedArrayObject::ensureHasBuffer(JSContext* cx, Handle<TypedArrayObject*> tarra
         return false;
 
     // tarray is not shared, because if it were it would have a buffer.
-    memcpy_sse2(buffer->dataPointer(), tarray->viewDataUnshared(), tarray->byteLength());
+    js_memcpy(buffer->dataPointer(), tarray->viewDataUnshared(), tarray->byteLength());
 
     // If the object is in the nursery, the buffer will be freed by the next
     // nursery GC. Free the data slot pointer if the object has no inline data.
@@ -593,7 +539,7 @@ class TypedArrayObjectTemplate : public TypedArrayObject
         } else {
             void* data = obj->fixedData(FIXED_DATA_START);
             obj->initPrivate(data);
-            memset_sse2(data, 0, len * sizeof(NativeType));
+            js_memset(data, 0, len * sizeof(NativeType));
 #ifdef DEBUG
             if (len == 0) {
                 uint8_t* elements = static_cast<uint8_t*>(data);
@@ -722,7 +668,7 @@ class TypedArrayObjectTemplate : public TypedArrayObject
 
             void* data = tarray->fixedData(FIXED_DATA_START);
             tarray->initPrivate(data);
-            memset_sse2(data, 0, nbytes);
+            js_memset(data, 0, nbytes);
         }
     }
 
@@ -759,7 +705,7 @@ class TypedArrayObjectTemplate : public TypedArrayObject
                 return nullptr;
             }
 
-            memset_sse2(buf, 0, nbytes);
+            js_memset(buf, 0, nbytes);
          }
 
         RootedObject tmp(cx, NewObjectWithGroup<TypedArrayObject>(cx, group, allocKind, newKind));
@@ -2212,7 +2158,7 @@ struct DataViewIO
     static void fromBuffer(DataType* dest, const uint8_t* unalignedBuffer, bool wantSwap)
     {
         MOZ_ASSERT((reinterpret_cast<uintptr_t>(dest) & (Min<size_t>(MOZ_ALIGNOF(void*), sizeof(DataType)) - 1)) == 0);
-        memcpy_sse2((void*) dest, unalignedBuffer, sizeof(ReadWriteType));
+        js_memcpy((void*) dest, unalignedBuffer, sizeof(ReadWriteType));
         if (wantSwap) {
             ReadWriteType* rwDest = reinterpret_cast<ReadWriteType*>(dest);
             *rwDest = swapBytes(*rwDest);
@@ -2225,7 +2171,7 @@ struct DataViewIO
         ReadWriteType temp = *reinterpret_cast<const ReadWriteType*>(src);
         if (wantSwap)
             temp = swapBytes(temp);
-        memcpy_sse2(unalignedBuffer, (void*) &temp, sizeof(ReadWriteType));
+        js_memcpy(unalignedBuffer, (void*) &temp, sizeof(ReadWriteType));
     }
 };