diff --git a/js/src/jsutil.h b/js/src/jsutil.h index daf056ad2f..00c2806c1d 100644 --- a/js/src/jsutil.h +++ b/js/src/jsutil.h @@ -19,6 +19,20 @@ #include +#if (defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)) +# if defined(_MSC_VER) +# if defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +# define JS_HAVE_SSE2_INTRINSICS 1 +# endif +# elif defined(__SSE2__) +# define JS_HAVE_SSE2_INTRINSICS 1 +# endif +#endif + +#if defined(JS_HAVE_SSE2_INTRINSICS) +# include +#endif + #include "js/Utility.h" #include "js/Value.h" @@ -41,9 +55,117 @@ js_memcpy(void* dst_, const void* src_, size_t len) MOZ_ASSERT_IF(dst >= src, (size_t) (dst - src) >= len); MOZ_ASSERT_IF(src >= dst, (size_t) (src - dst) >= len); +#if defined(JS_HAVE_SSE2_INTRINSICS) + if (len >= 64) { + uint8_t* d = (uint8_t*)dst; + const uint8_t* s = (const uint8_t*)src; + + while (len >= 64) { + __m128i v0 = _mm_loadu_si128((const __m128i*)(s + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(s + 16)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(s + 32)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(s + 48)); + _mm_storeu_si128((__m128i*)(d + 0), v0); + _mm_storeu_si128((__m128i*)(d + 16), v1); + _mm_storeu_si128((__m128i*)(d + 32), v2); + _mm_storeu_si128((__m128i*)(d + 48), v3); + d += 64; + s += 64; + len -= 64; + } + + while (len >= 16) { + __m128i v = _mm_loadu_si128((const __m128i*)s); + _mm_storeu_si128((__m128i*)d, v); + d += 16; + s += 16; + len -= 16; + } + + if (len) { + memcpy(d, s, len); + } + return dst; + } +#endif + return memcpy(dst, src, len); } +static MOZ_ALWAYS_INLINE void* +js_memmove(void* dst_, const void* src_, size_t len) +{ + char* dst = (char*) dst_; + const char* src = (const char*) src_; + + if (dst == src || !len) { + return dst; + } + +#if defined(JS_HAVE_SSE2_INTRINSICS) + if (len >= 64) { + uint8_t* d = (uint8_t*)dst; + const uint8_t* s = (const uint8_t*)src; + + if (d < s || d >= s + len) { + return js_memcpy(dst, src, len); + } + + d += len; + s += len; + while (len >= 16) { + d -= 16; + s -= 16; + __m128i v = _mm_loadu_si128((const __m128i*)s); + _mm_storeu_si128((__m128i*)d, v); + len -= 16; + } + + while (len--) { + *--d = *--s; + } + return dst; + } +#endif + + return memmove(dst, src, len); +} + +static MOZ_ALWAYS_INLINE void* +js_memset(void* dst_, uint8_t value, size_t len) +{ + char* dst = (char*) dst_; + +#if defined(JS_HAVE_SSE2_INTRINSICS) + if (len >= 64) { + uint8_t* d = (uint8_t*)dst; + __m128i v = _mm_set1_epi8((char)value); + + while (len >= 64) { + _mm_storeu_si128((__m128i*)(d + 0), v); + _mm_storeu_si128((__m128i*)(d + 16), v); + _mm_storeu_si128((__m128i*)(d + 32), v); + _mm_storeu_si128((__m128i*)(d + 48), v); + d += 64; + len -= 64; + } + + while (len >= 16) { + _mm_storeu_si128((__m128i*)d, v); + d += 16; + len -= 16; + } + + while (len--) { + *d++ = (char)value; + } + return dst; + } +#endif + + return memset(dst, value, len); +} + namespace js { template diff --git a/js/src/vm/ArrayBufferObject.cpp b/js/src/vm/ArrayBufferObject.cpp index cb4f607505..e43edab690 100644 --- a/js/src/vm/ArrayBufferObject.cpp +++ b/js/src/vm/ArrayBufferObject.cpp @@ -3,8 +3,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include // SSE2 - #include "vm/ArrayBufferObject-inl.h" #include "vm/ArrayBufferObject.h" @@ -67,58 +65,6 @@ using mozilla::Nothing; using namespace js; using namespace js::gc; -// SSE2 version of memcpy_sse2_sse2 and memset_sse2_sse2 for faster JS n shit -static inline void -memcpy_sse2(void* dst, const void* src, size_t n) -{ - uint8_t* d = (uint8_t*)dst; - const uint8_t* s = (const uint8_t*)src; - - // Align to 16 bytes - while (((uintptr_t)d & 15) && n) { - *d++ = *s++; - n--; - } - - // 16‑byte chunks - while (n >= 16) { - __m128i v = _mm_loadu_si128((__m128i*)s); - _mm_storeu_si128((__m128i*)d, v); - s += 16; - d += 16; - n -= 16; - } - - // tail - while (n--) { - *d++ = *s++; - } -} - -static inline void -memset_sse2(void* dst, uint8_t value, size_t n) -{ - uint8_t* d = (uint8_t*)dst; - - // Align - while (((uintptr_t)d & 15) && n) { - *d++ = value; - n--; - } - - __m128i v = _mm_set1_epi8(value); - - while (n >= 16) { - _mm_storeu_si128((__m128i*)d, v); - d += 16; - n -= 16; - } - - while (n--) { - *d++ = value; - } -} - /* * Convert |v| to an array index for an array of length |length| per * the Typed Array Specification section 7.0, |subarray|. If successful, @@ -850,7 +796,7 @@ ArrayBufferObject::prepareForAsmJS(JSContext* cx, Handle buf } void* data = wasmBuf->dataPointer(); - memcpy_sse2(data, buffer->dataPointer(), length); + js_memcpy(data, buffer->dataPointer(), length); // Swap the new elements into the ArrayBufferObject. Mark the // ArrayBufferObject so we don't do this again. @@ -872,7 +818,7 @@ ArrayBufferObject::prepareForAsmJS(JSContext* cx, Handle buf BufferContents contents = AllocateArrayBufferContents(cx, buffer->byteLength()); if (!contents) return false; - memcpy_sse2(contents.data(), buffer->dataPointer(), buffer->byteLength()); + js_memcpy(contents.data(), buffer->dataPointer(), buffer->byteLength()); buffer->changeContents(cx, contents, OwnsData); } @@ -1049,7 +995,7 @@ ArrayBufferObject::wasmMovingGrowToSize(uint32_t newSize, BufferContents contents = BufferContents::create(newRawBuf->dataPointer()); newBuf->initialize(newSize, contents, OwnsData); - memcpy_sse2(newBuf->dataPointer(), oldBuf->dataPointer(), oldBuf->byteLength()); + js_memcpy(newBuf->dataPointer(), oldBuf->dataPointer(), oldBuf->byteLength()); ArrayBufferObject::detach(cx, oldBuf, BufferContents::createPlain(nullptr)); return true; } @@ -1151,7 +1097,7 @@ ArrayBufferObject::create(JSContext* cx, uint32_t nbytes, BufferContents content if (!contents) { void* data = obj->inlineDataPointer(); - memset_sse2(data, 0, nbytes); + js_memset(data, 0, nbytes); obj->initialize(nbytes, BufferContents::createPlain(data), DoesntOwnData); } else { obj->initialize(nbytes, contents, ownsState); @@ -1236,7 +1182,7 @@ ArrayBufferObject::externalizeContents(JSContext* cx, Handle BufferContents newContents = AllocateArrayBufferContents(cx, buffer->byteLength()); if (!newContents) return BufferContents::createPlain(nullptr); - memcpy_sse2(newContents.data(), contents.data(), buffer->byteLength()); + js_memcpy(newContents.data(), contents.data(), buffer->byteLength()); buffer->changeContents(cx, newContents, DoesntOwnData); return newContents; @@ -1271,7 +1217,7 @@ ArrayBufferObject::stealContents(JSContext* cx, Handle buffe return BufferContents::createPlain(nullptr); if (buffer->byteLength() > 0) - memcpy_sse2(contentsCopy.data(), oldContents.data(), buffer->byteLength()); + js_memcpy(contentsCopy.data(), oldContents.data(), buffer->byteLength()); ArrayBufferObject::detach(cx, buffer, oldContents); return contentsCopy; } @@ -1324,7 +1270,7 @@ ArrayBufferObject::copyData(Handle toBuffer, uint32_t toInde MOZ_ASSERT(fromBuffer->byteLength() >= fromIndex); MOZ_ASSERT(fromBuffer->byteLength() >= fromIndex + count); - memcpy_sse2(toBuffer->dataPointer() + toIndex, fromBuffer->dataPointer() + fromIndex, count); + js_memcpy(toBuffer->dataPointer() + toIndex, fromBuffer->dataPointer() + fromIndex, count); } /* static */ void diff --git a/js/src/vm/TypedArrayObject.cpp b/js/src/vm/TypedArrayObject.cpp index e3fec2f894..ebe785aa2e 100644 --- a/js/src/vm/TypedArrayObject.cpp +++ b/js/src/vm/TypedArrayObject.cpp @@ -3,8 +3,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include // SSE2 - #include "vm/TypedArrayObject.h" #include "mozilla/Alignment.h" @@ -59,58 +57,6 @@ using JS::CanonicalizeNaN; using JS::ToInt32; using JS::ToUint32; -// SSE2 version of memcpy_sse2 and memset_sse2 for faster JS n shit -static inline void -memcpy_sse2(void* dst, const void* src, size_t n) -{ - uint8_t* d = (uint8_t*)dst; - const uint8_t* s = (const uint8_t*)src; - - // Align to 16 bytes - while (((uintptr_t)d & 15) && n) { - *d++ = *s++; - n--; - } - - // 16‑byte chunks - while (n >= 16) { - __m128i v = _mm_loadu_si128((__m128i*)s); - _mm_storeu_si128((__m128i*)d, v); - s += 16; - d += 16; - n -= 16; - } - - // tail - while (n--) { - *d++ = *s++; - } -} - -static inline void -memset_sse2(void* dst, uint8_t value, size_t n) -{ - uint8_t* d = (uint8_t*)dst; - - // Align - while (((uintptr_t)d & 15) && n) { - *d++ = value; - n--; - } - - __m128i v = _mm_set1_epi8(value); - - while (n >= 16) { - _mm_storeu_si128((__m128i*)d, v); - d += 16; - n -= 16; - } - - while (n--) { - *d++ = value; - } -} - /* * TypedArrayObject @@ -198,7 +144,7 @@ TypedArrayObject::ensureHasBuffer(JSContext* cx, Handle tarra return false; // tarray is not shared, because if it were it would have a buffer. - memcpy_sse2(buffer->dataPointer(), tarray->viewDataUnshared(), tarray->byteLength()); + js_memcpy(buffer->dataPointer(), tarray->viewDataUnshared(), tarray->byteLength()); // If the object is in the nursery, the buffer will be freed by the next // nursery GC. Free the data slot pointer if the object has no inline data. @@ -593,7 +539,7 @@ class TypedArrayObjectTemplate : public TypedArrayObject } else { void* data = obj->fixedData(FIXED_DATA_START); obj->initPrivate(data); - memset_sse2(data, 0, len * sizeof(NativeType)); + js_memset(data, 0, len * sizeof(NativeType)); #ifdef DEBUG if (len == 0) { uint8_t* elements = static_cast(data); @@ -722,7 +668,7 @@ class TypedArrayObjectTemplate : public TypedArrayObject void* data = tarray->fixedData(FIXED_DATA_START); tarray->initPrivate(data); - memset_sse2(data, 0, nbytes); + js_memset(data, 0, nbytes); } } @@ -759,7 +705,7 @@ class TypedArrayObjectTemplate : public TypedArrayObject return nullptr; } - memset_sse2(buf, 0, nbytes); + js_memset(buf, 0, nbytes); } RootedObject tmp(cx, NewObjectWithGroup(cx, group, allocKind, newKind)); @@ -2212,7 +2158,7 @@ struct DataViewIO static void fromBuffer(DataType* dest, const uint8_t* unalignedBuffer, bool wantSwap) { MOZ_ASSERT((reinterpret_cast(dest) & (Min(MOZ_ALIGNOF(void*), sizeof(DataType)) - 1)) == 0); - memcpy_sse2((void*) dest, unalignedBuffer, sizeof(ReadWriteType)); + js_memcpy((void*) dest, unalignedBuffer, sizeof(ReadWriteType)); if (wantSwap) { ReadWriteType* rwDest = reinterpret_cast(dest); *rwDest = swapBytes(*rwDest); @@ -2225,7 +2171,7 @@ struct DataViewIO ReadWriteType temp = *reinterpret_cast(src); if (wantSwap) temp = swapBytes(temp); - memcpy_sse2(unalignedBuffer, (void*) &temp, sizeof(ReadWriteType)); + js_memcpy(unalignedBuffer, (void*) &temp, sizeof(ReadWriteType)); } };