From 6cc90c7009511ab43ca7952d162a9845a4a20095 Mon Sep 17 00:00:00 2001 From: wuggy Date: Fri, 18 Sep 2026 13:11:49 -0700 Subject: [PATCH] JS optimisations pt.1 --- js/src/builtin/TypedObject.cpp | 6 +- js/src/irregexp/RegExpInterpreter.cpp | 4 +- js/src/jit-test/tests/latin1/sse2-search.js | 18 ++ js/src/jit/BaselineBailouts.cpp | 2 +- js/src/jit/BaselineJIT.cpp | 3 +- js/src/jit/CompileInfo.h | 16 +- js/src/jit/Ion.cpp | 23 +- js/src/jit/IonAnalysis.cpp | 7 +- js/src/jit/shared/CodeGenerator-shared.cpp | 5 +- js/src/jsapi.cpp | 2 +- js/src/jsarray.cpp | 17 +- js/src/jsatominlines.h | 5 +- js/src/jsfriendapi.h | 43 +++- js/src/jsobjinlines.h | 3 +- js/src/jsscript.cpp | 2 +- js/src/jsstr.cpp | 163 ++++++++++--- js/src/jsstr.h | 245 ++++++++++++++++++++ js/src/jsutil.h | 15 ++ js/src/vm/ArgumentsObject.cpp | 9 +- js/src/vm/ArrayBufferObject.cpp | 6 +- js/src/vm/CharacterEncoding.cpp | 73 +++++- js/src/vm/CharacterOperations.h | 155 +++++++++++++ js/src/vm/Interpreter.cpp | 5 +- js/src/vm/NativeObject.h | 11 +- js/src/vm/String-inl.h | 6 +- js/src/vm/String.cpp | 79 +++++-- js/src/vm/TypedArrayObject.cpp | 16 ++ js/src/vm/UnboxedObject-inl.h | 12 +- js/src/vm/UnboxedObject.cpp | 2 +- mfbt/HashFunctions.cpp | 12 +- mfbt/HashFunctions.h | 13 +- 31 files changed, 869 insertions(+), 109 deletions(-) diff --git a/js/src/builtin/TypedObject.cpp b/js/src/builtin/TypedObject.cpp index 10ae8902dd..c2517abde2 100644 --- a/js/src/builtin/TypedObject.cpp +++ b/js/src/builtin/TypedObject.cpp @@ -2040,7 +2040,7 @@ InlineTypedObject::createCopy(JSContext* cx, Handle template if (!res) return nullptr; - memcpy(res->inlineTypedMem(), templateObject->inlineTypedMem(), templateObject->size()); + js_memcpy(res->inlineTypedMem(), templateObject->inlineTypedMem(), templateObject->size()); return res; } @@ -2767,7 +2767,7 @@ TypeDescr::initInstances(const JSRuntime* rt, uint8_t* mem, size_t length) MemoryInitVisitor visitor(rt); // Initialize the 0th instance - memset(mem, 0, size()); + js_memset(mem, 0, size()); if (opaque()) visitReferences(*this, mem, visitor); @@ -2775,7 +2775,7 @@ TypeDescr::initInstances(const JSRuntime* rt, uint8_t* mem, size_t length) uint8_t* target = mem; for (size_t i = 1; i < length; i++) { target += size(); - memcpy(target, mem, size()); + js_memcpy(target, mem, size()); } } diff --git a/js/src/irregexp/RegExpInterpreter.cpp b/js/src/irregexp/RegExpInterpreter.cpp index f53acfb606..f7f7f6eeb7 100644 --- a/js/src/irregexp/RegExpInterpreter.cpp +++ b/js/src/irregexp/RegExpInterpreter.cpp @@ -31,6 +31,7 @@ #include "irregexp/RegExpBytecode.h" #include "irregexp/RegExpMacroAssembler.h" +#include "jsutil.h" #include "vm/MatchPairs.h" using namespace js; @@ -197,7 +198,8 @@ irregexp::InterpretCode(JSContext* cx, const uint8_t* byteCode, const CharT* cha return RegExpRunStatus_Success_NotFound; BYTECODE(SUCCEED) if (matches) - memcpy(matches->pairsRaw(), registers.begin(), matches->length() * 2 * sizeof(int32_t)); + js_memcpy(matches->pairsRaw(), registers.begin(), + matches->length() * 2 * sizeof(int32_t)); else if (endIndex) *endIndex = registers[1]; return RegExpRunStatus_Success; diff --git a/js/src/jit-test/tests/latin1/sse2-search.js b/js/src/jit-test/tests/latin1/sse2-search.js index 064e802b87..1bf2f70ea1 100644 --- a/js/src/jit-test/tests/latin1/sse2-search.js +++ b/js/src/jit-test/tests/latin1/sse2-search.js @@ -19,3 +19,21 @@ for (var length of [0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65]) { assertEq(wide.indexOf("\xff"), length + 1); assertEq(wide.indexOf(latin1), 1); } + +// Mixed-encoding searches should use the same SIMD first-character scan, and +// an impossible UTF-16 character should reject a Latin-1 haystack immediately. +var latin1Haystack = "x".repeat(4096); +assertEq(latin1Haystack.indexOf("x\u0100x"), -1); +var wideHaystack = "\u0100" + "x".repeat(4096) + "needle"; +assertEq(wideHaystack.indexOf("needle"), 4097); + +// Exercise the mixed-width EqualChars fast path through string equality. +var wideLatin1 = ("\u0100" + latin1Haystack).slice(1); +assertEq(wideLatin1, latin1Haystack); +assertEq(wideLatin1 + "y", latin1Haystack + "z"); +assertEq(wideLatin1 + "\u0100" > latin1Haystack + "z", true); +assertEq(latin1Haystack + "z" < wideLatin1 + "\u0100", true); + +assertEq(latin1Haystack.lastIndexOf("x\u0100x"), -1); +assertEq(wideHaystack.lastIndexOf("needle"), 4097); +assertEq(wideHaystack.lastIndexOf("x"), 4096); diff --git a/js/src/jit/BaselineBailouts.cpp b/js/src/jit/BaselineBailouts.cpp index be69f3f1ee..073f198288 100644 --- a/js/src/jit/BaselineBailouts.cpp +++ b/js/src/jit/BaselineBailouts.cpp @@ -150,7 +150,7 @@ struct BaselineStackBuilder uint8_t* newBuffer = reinterpret_cast(js_calloc(newSize)); if (!newBuffer) return false; - memcpy((newBuffer + newSize) - bufferUsed_, header_->copyStackBottom, bufferUsed_); + js_memcpy((newBuffer + newSize) - bufferUsed_, header_->copyStackBottom, bufferUsed_); memcpy(newBuffer, header_, sizeof(BaselineBailoutInfo)); js_free(buffer_); buffer_ = newBuffer; diff --git a/js/src/jit/BaselineJIT.cpp b/js/src/jit/BaselineJIT.cpp index 70f7c79eb8..adf6f7590c 100644 --- a/js/src/jit/BaselineJIT.cpp +++ b/js/src/jit/BaselineJIT.cpp @@ -18,6 +18,7 @@ #include "vm/Interpreter.h" #include "vm/TraceLogging.h" #include "wasm/WasmInstance.h" +#include "jsutil.h" #include "jsobjinlines.h" #include "jsopcodeinlines.h" @@ -812,7 +813,7 @@ BaselineScript::copyPCMappingEntries(const CompactBufferWriter& entries) MOZ_ASSERT(entries.length() > 0); MOZ_ASSERT(entries.length() == pcMappingSize_); - memcpy(pcMappingData(), entries.buffer(), entries.length()); + js_memcpy(pcMappingData(), entries.buffer(), entries.length()); } void diff --git a/js/src/jit/CompileInfo.h b/js/src/jit/CompileInfo.h index f10d09e410..36b06571db 100644 --- a/js/src/jit/CompileInfo.h +++ b/js/src/jit/CompileInfo.h @@ -442,14 +442,18 @@ class CompileInfo // the frame is active on the stack. This implies that these definitions // would have to be executed and that they cannot be removed even if they // are unused. - bool isObservableSlot(uint32_t slot) const { - if (isObservableFrameSlot(slot)) - return true; + inline bool isObservableSlot(uint32_t slot) const { + if (slot >= firstLocalSlot()) { + // The |this| slot for a derived class constructor is a local slot. + if (thisSlotForDerivedClassConstructor_) + return *thisSlotForDerivedClassConstructor_ == slot; + return false; + } - if (isObservableArgumentSlot(slot)) - return true; + if (slot < firstArgSlot()) + return isObservableFrameSlot(slot); - return false; + return isObservableArgumentSlot(slot); } bool isObservableFrameSlot(uint32_t slot) const { diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp index 8e28a93c0e..4ee44ba576 100644 --- a/js/src/jit/Ion.cpp +++ b/js/src/jit/Ion.cpp @@ -13,6 +13,7 @@ #include "jscompartment.h" #include "jsgc.h" #include "jsprf.h" +#include "jsutil.h" #include "gc/Marking.h" #include "jit/AliasAnalysis.h" @@ -1042,33 +1043,33 @@ void IonScript::copySnapshots(const SnapshotWriter* writer) { MOZ_ASSERT(writer->listSize() == snapshotsListSize_); - memcpy((uint8_t*)this + snapshots_, - writer->listBuffer(), snapshotsListSize_); + js_memcpy((uint8_t*)this + snapshots_, + writer->listBuffer(), snapshotsListSize_); MOZ_ASSERT(snapshotsRVATableSize_); MOZ_ASSERT(writer->RVATableSize() == snapshotsRVATableSize_); - memcpy((uint8_t*)this + snapshots_ + snapshotsListSize_, - writer->RVATableBuffer(), snapshotsRVATableSize_); + js_memcpy((uint8_t*)this + snapshots_ + snapshotsListSize_, + writer->RVATableBuffer(), snapshotsRVATableSize_); } void IonScript::copyRecovers(const RecoverWriter* writer) { MOZ_ASSERT(writer->size() == recoversSize_); - memcpy((uint8_t*)this + recovers_, writer->buffer(), recoversSize_); + js_memcpy((uint8_t*)this + recovers_, writer->buffer(), recoversSize_); } void IonScript::copySafepoints(const SafepointWriter* writer) { MOZ_ASSERT(writer->size() == safepointsSize_); - memcpy((uint8_t*)this + safepointsStart_, writer->buffer(), safepointsSize_); + js_memcpy((uint8_t*)this + safepointsStart_, writer->buffer(), safepointsSize_); } void IonScript::copyBailoutTable(const SnapshotOffset* table) { - memcpy(bailoutTable(), table, bailoutEntries_ * sizeof(uint32_t)); + js_memcpy(bailoutTable(), table, bailoutEntries_ * sizeof(uint32_t)); } void @@ -1114,25 +1115,25 @@ IonScript::copySafepointIndices(const SafepointIndex* si, MacroAssembler& masm) // code, not the absolute positions of the jumps. Update according to the // final code address now. SafepointIndex* table = safepointIndices(); - memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex)); + js_memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex)); } void IonScript::copyOsiIndices(const OsiIndex* oi, MacroAssembler& masm) { - memcpy(osiIndices(), oi, osiIndexEntries_ * sizeof(OsiIndex)); + js_memcpy(osiIndices(), oi, osiIndexEntries_ * sizeof(OsiIndex)); } void IonScript::copyRuntimeData(const uint8_t* data) { - memcpy(runtimeData(), data, runtimeSize()); + js_memcpy(runtimeData(), data, runtimeSize()); } void IonScript::copyCacheEntries(const uint32_t* caches, MacroAssembler& masm) { - memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t)); + js_memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t)); // Jumps in the caches reflect the offset of those jumps in the compiled // code, not the absolute positions of the jumps. Update according to the diff --git a/js/src/jit/IonAnalysis.cpp b/js/src/jit/IonAnalysis.cpp index 6f424098e8..6d0b8deaf1 100644 --- a/js/src/jit/IonAnalysis.cpp +++ b/js/src/jit/IonAnalysis.cpp @@ -196,6 +196,8 @@ FlagPhiInputsAsHavingRemovedUses(MIRGenerator* mir, MBasicBlock* block, MBasicBl static bool FlagAllOperandsAsHavingRemovedUses(MIRGenerator* mir, MBasicBlock* block) { + const CompileInfo& info = block->info(); + // Flag all instructions operands as having removed uses. MInstructionIterator end = block->end(); for (MInstructionIterator it = block->begin(); it != end; it++) { @@ -214,7 +216,7 @@ FlagAllOperandsAsHavingRemovedUses(MIRGenerator* mir, MBasicBlock* block) if (mir->shouldCancel("FlagAllOperandsAsHavingRemovedUses inner loop")) return false; - if (!rp->isObservableOperand(i)) + if (!info.isObservableSlot(i)) continue; rp->getOperand(i)->setUseRemovedUnchecked(); } @@ -227,8 +229,9 @@ FlagAllOperandsAsHavingRemovedUses(MIRGenerator* mir, MBasicBlock* block) if (mir->shouldCancel("FlagAllOperandsAsHavingRemovedUses loop 2")) return false; + const CompileInfo& info = rp->block()->info(); for (size_t i = 0, e = rp->numOperands(); i < e; i++) { - if (!rp->isObservableOperand(i)) + if (!info.isObservableSlot(i)) continue; rp->getOperand(i)->setUseRemovedUnchecked(); } diff --git a/js/src/jit/shared/CodeGenerator-shared.cpp b/js/src/jit/shared/CodeGenerator-shared.cpp index 78f66bb9da..3e86fe87b6 100644 --- a/js/src/jit/shared/CodeGenerator-shared.cpp +++ b/js/src/jit/shared/CodeGenerator-shared.cpp @@ -16,6 +16,7 @@ #include "jit/MIR.h" #include "jit/MIRGenerator.h" #include "jit/OptimizationTracking.h" +#include "jsutil.h" #include "js/Conversions.h" #include "vm/TraceLogging.h" @@ -754,7 +755,7 @@ CodeGeneratorShared::generateCompactNativeToBytecodeMap(JSContext* cx, JitCode* return false; } - memcpy(data, writer.buffer(), writer.length()); + js_memcpy(data, writer.buffer(), writer.length()); nativeToBytecodeMap_ = data; nativeToBytecodeMapSize_ = writer.length(); nativeToBytecodeTableOffset_ = tableOffset; @@ -908,7 +909,7 @@ CodeGeneratorShared::generateCompactTrackedOptimizationsMap(JSContext* cx, JitCo if (!data) return false; - memcpy(data, writer.buffer(), writer.length()); + js_memcpy(data, writer.buffer(), writer.length()); trackedOptimizationsMap_ = data; trackedOptimizationsMapSize_ = writer.length(); trackedOptimizationsRegionTableOffset_ = regionTableOffset; diff --git a/js/src/jsapi.cpp b/js/src/jsapi.cpp index 3ea147eaff..07a80026ca 100644 --- a/js/src/jsapi.cpp +++ b/js/src/jsapi.cpp @@ -6005,7 +6005,7 @@ EncodeLatin1(ExclusiveContext* cx, JSString* str) return nullptr; } - mozilla::PodCopy(buf, linear->latin1Chars(nogc), len); + js_memcpy(buf, linear->latin1Chars(nogc), len); buf[len] = '\0'; return reinterpret_cast(buf); } diff --git a/js/src/jsarray.cpp b/js/src/jsarray.cpp index 15cb23a213..fc2871c70f 100644 --- a/js/src/jsarray.cpp +++ b/js/src/jsarray.cpp @@ -153,6 +153,21 @@ StringIsArrayIndex(const CharT* s, uint32_t length, uint32_t* indexp) if (length == 0 || length > (sizeof("4294967294") - 1) || !JS7_ISDEC(*s)) return false; + // Small indices are by far the most common property keys. Handle them + // without entering the general overflow-checking loop below. + if (length == 1) { + *indexp = JS7_UNDEC(*s); + return true; + } + + if (length == 2) { + uint32_t first = JS7_UNDEC(s[0]); + if (first == 0 || !JS7_ISDEC(s[1])) + return false; + *indexp = first * 10 + JS7_UNDEC(s[1]); + return true; + } + uint32_t c = 0, previous = 0; uint32_t index = JS7_UNDEC(*s++); @@ -2197,7 +2212,7 @@ ShiftMoveBoxedOrUnboxedDenseElements(JSObject* obj) } else { uint8_t* data = obj->as().elements(); size_t elementSize = UnboxedTypeSize(Type); - memmove(data, data + elementSize, initlen * elementSize); + js_memmove(data, data + elementSize, initlen * elementSize); } return DenseElementResult::Success; diff --git a/js/src/jsatominlines.h b/js/src/jsatominlines.h index ab91f974d1..2af4b7355d 100644 --- a/js/src/jsatominlines.h +++ b/js/src/jsatominlines.h @@ -8,7 +8,6 @@ #include "jsatom.h" -#include "mozilla/PodOperations.h" #include "mozilla/RangedPtr.h" #include "jscntxt.h" @@ -177,14 +176,14 @@ AtomHasher::match(const AtomStateEntry& entry, const Lookup& lookup) if (key->hasLatin1Chars()) { const Latin1Char* keyChars = key->latin1Chars(lookup.nogc); if (lookup.isLatin1) - return mozilla::PodEqual(keyChars, lookup.latin1Chars, lookup.length); + return EqualChars(keyChars, lookup.latin1Chars, lookup.length); return EqualChars(keyChars, lookup.twoByteChars, lookup.length); } const char16_t* keyChars = key->twoByteChars(lookup.nogc); if (lookup.isLatin1) return EqualChars(lookup.latin1Chars, keyChars, lookup.length); - return mozilla::PodEqual(keyChars, lookup.twoByteChars, lookup.length); + return EqualChars(keyChars, lookup.twoByteChars, lookup.length); } inline Handle diff --git a/js/src/jsfriendapi.h b/js/src/jsfriendapi.h index 2697cb5337..ef89efc8b8 100644 --- a/js/src/jsfriendapi.h +++ b/js/src/jsfriendapi.h @@ -12,6 +12,8 @@ #include "mozilla/MemoryReporting.h" #include "mozilla/UniquePtr.h" +#include + #include "jsapi.h" // For JSAutoByteString. See bug 1033916. #include "jsbytecode.h" #include "jspubtd.h" @@ -21,6 +23,12 @@ #include "js/Class.h" #include "js/Utility.h" +#if defined(__SSE2__) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +# define JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS +# include +#endif + #if JS_STACK_GROWTH_DIRECTION > 0 # define JS_CHECK_STACK_SIZE(limit, sp) (MOZ_LIKELY((uintptr_t)(sp) < (limit))) #else @@ -877,8 +885,21 @@ CopyLinearStringChars(char16_t* dest, JSLinearString* s, size_t len, size_t star JS::AutoCheckCannotGC nogc; if (LinearStringHasLatin1Chars(s)) { const JS::Latin1Char* src = GetLatin1LinearStringChars(nogc, s); +#if defined(JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS) + size_t i = 0; + const __m128i zero = _mm_setzero_si128(); + for (; i + 8 <= len; i += 8) { + const __m128i bytes8 = _mm_loadl_epi64( + reinterpret_cast(src + start + i)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dest + i), + _mm_unpacklo_epi8(bytes8, zero)); + } + for (; i < len; i++) + dest[i] = src[start + i]; +#else for (size_t i = 0; i < len; i++) dest[i] = src[start + i]; +#endif } else { const char16_t* src = GetTwoByteLinearStringChars(nogc, s); mozilla::PodCopy(dest, src + start, len); @@ -892,12 +913,26 @@ CopyLinearStringChars(char* dest, JSLinearString* s, size_t len, size_t start = JS::AutoCheckCannotGC nogc; if (LinearStringHasLatin1Chars(s)) { const JS::Latin1Char* src = GetLatin1LinearStringChars(nogc, s); - for (size_t i = 0; i < len; i++) - dest[i] = char(src[start + i]); + memcpy(dest, src + start, len); } else { const char16_t* src = GetTwoByteLinearStringChars(nogc, s); +#if defined(JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS) + size_t i = 0; + const __m128i lowByteMask = _mm_set1_epi16(0xff); + const __m128i zero = _mm_setzero_si128(); + for (; i + 8 <= len; i += 8) { + const __m128i wide = _mm_loadu_si128( + reinterpret_cast(src + start + i)); + const __m128i lowBytes = _mm_and_si128(wide, lowByteMask); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i), + _mm_packus_epi16(lowBytes, zero)); + } + for (; i < len; i++) + dest[i] = char(src[start + i]); +#else for (size_t i = 0; i < len; i++) dest[i] = char(src[start + i]); +#endif } } @@ -3039,4 +3074,8 @@ class MemProfiler } }; +#ifdef JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS +# undef JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS +#endif + #endif /* jsfriendapi_h */ diff --git a/js/src/jsobjinlines.h b/js/src/jsobjinlines.h index a27a13fd6c..bb29474d34 100644 --- a/js/src/jsobjinlines.h +++ b/js/src/jsobjinlines.h @@ -12,6 +12,7 @@ #include "jsfriendapi.h" #include "jsfun.h" +#include "jsutil.h" #include "builtin/MapObject.h" #include "builtin/TypedObject.h" @@ -401,7 +402,7 @@ JSObject::create(js::ExclusiveContext* cx, js::gc::AllocKind kind, js::gc::Initi kind == js::gc::AllocKind::FUNCTION_EXTENDED); size_t size = kind == js::gc::AllocKind::FUNCTION ? sizeof(JSFunction) : sizeof(js::FunctionExtended); - memset(obj->as().fixedSlots(), 0, size - sizeof(js::NativeObject)); + js_memset(obj->as().fixedSlots(), 0, size - sizeof(js::NativeObject)); if (kind == js::gc::AllocKind::FUNCTION_EXTENDED) { // SetNewObjectMetadata may gc, which will be unhappy if flags & // EXTENDED doesn't match the arena's AllocKind. diff --git a/js/src/jsscript.cpp b/js/src/jsscript.cpp index ca51bb1b19..3e76e39522 100644 --- a/js/src/jsscript.cpp +++ b/js/src/jsscript.cpp @@ -3530,7 +3530,7 @@ js::detail::CopyScript(JSContext* cx, HandleScript src, HandleScript dst, dst->dataSize_ = size; MOZ_ASSERT(bool(dst->data) == bool(src->data)); if (dst->data) - memcpy(dst->data, src->data, size); + js_memcpy(dst->data, src->data, size); /* Script filenames, bytecodes and atoms are runtime-wide. */ dst->setScriptData(src->scriptData()); diff --git a/js/src/jsstr.cpp b/js/src/jsstr.cpp index d8374e8e43..e78d626834 100644 --- a/js/src/jsstr.cpp +++ b/js/src/jsstr.cpp @@ -68,7 +68,6 @@ using mozilla::IsNegativeZero; using mozilla::IsSame; using mozilla::Move; using mozilla::PodCopy; -using mozilla::PodEqual; using mozilla::RangedPtr; using JS::AutoCheckCannotGC; @@ -1085,6 +1084,18 @@ ToUpperCaseLength(const CharT* chars, size_t startIndex, size_t length) return upperLength; } +static inline void +CopyChars(char16_t* destChars, const char* srcChars, size_t length) +{ + CopyAndInflateChars(destChars, srcChars, length); +} + +static inline void +CopyChars(char16_t* destChars, const Latin1Char* srcChars, size_t length) +{ + CopyAndInflateChars(destChars, srcChars, length); +} + template static inline void CopyChars(DestChar* destChars, const SrcChar* srcChars, size_t length) @@ -1706,6 +1717,16 @@ template static int Matcher(const TextChar* text, uint32_t textlen, const PatChar* pat, uint32_t patlen) { + // A Latin-1 string can never contain a UTF-16 code unit above 0xff. Do + // this check once instead of repeatedly testing every candidate position + // in the mixed-encoding matcher. This is particularly useful for search + // strings containing supplementary-plane or otherwise non-Latin-1 text. + if (sizeof(TextChar) == 1 && sizeof(PatChar) == 2 && + !CharactersFitInLatin1(reinterpret_cast(pat), patlen)) + { + return -1; + } + const typename InnerMatch::Extent extent = InnerMatch::computeExtent(pat, patlen); uint32_t i = 0; @@ -1717,6 +1738,16 @@ Matcher(const TextChar* text, uint32_t textlen, const PatChar* pat, uint32_t pat pos = (TextChar*) FirstCharMatcher16bit((char16_t*)text + i, n - i, pat[0]); else if (sizeof(TextChar) == 1 && sizeof(PatChar) == 1) pos = (TextChar*) FirstCharMatcher8bit((char*) text + i, n - i, pat[0]); + else if (sizeof(TextChar) == 1 && sizeof(PatChar) == 2) + // The complete pattern was checked above, so this narrowing is + // lossless and keeps the other mixed-width direction on SIMD. + pos = FindCharacter(text + i, n - i, TextChar(pat[0])); + else if (sizeof(TextChar) == 2 && sizeof(PatChar) == 1) + // FindCharacter is encoding-independent for the text and keeps + // mixed Latin-1/UTF-16 searches on the SSE2 fast path. + pos = reinterpret_cast( + FindCharacter(reinterpret_cast(text) + i, + n - i, char16_t(pat[0]))); else pos = (TextChar*) FirstCharMatcherUnrolled(text + i, n - i, pat[0]); @@ -1742,9 +1773,9 @@ StringMatch(const TextChar* text, uint32_t textLen, const PatChar* pat, uint32_t if (textLen < patLen) return -1; -#if defined(__i386__) || defined(_M_IX86) || defined(__i386) - // Avoid the generic substring matcher for a single character on x86. - // FindCharacter uses SSE2 where available, including mixed encodings. +#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS + // Avoid the generic substring matcher for a single character when the + // bounded SSE2 search helper is available, including mixed encodings. if (patLen == 1) { // A two-byte needle cannot match Latin1 text if it exceeds 0xff. if (sizeof(TextChar) == 1 && uint32_t(*pat) > 0xff) @@ -2162,17 +2193,35 @@ LastIndexOfImpl(const TextChar* text, size_t textLen, const PatChar* pat, size_t const PatChar* patNext = pat + 1; const PatChar* patEnd = pat + patLen; - for (const TextChar* t = text + start; t >= text; --t) { - if (*t == p0) { - const TextChar* t1 = t + 1; - for (const PatChar* p1 = patNext; p1 < patEnd; ++p1, ++t1) { - if (*t1 != *p1) - goto break_continue; - } - - return static_cast(t - text); + // Search candidate first characters backwards in SIMD-sized blocks. The + // bounded helper keeps the scan safe at allocation and page boundaries, + // while the scalar comparison below still verifies the rest of the + // pattern exactly. + size_t searchLength = start + 1; + while (searchLength) { + const TextChar* t; + if (sizeof(TextChar) == 1 && sizeof(PatChar) == 2) { + if (uint32_t(p0) > 0xff) + return -1; + t = FindCharacterReverse(text, searchLength, TextChar(p0)); + } else { + t = FindCharacterReverse(text, searchLength, TextChar(p0)); } - break_continue:; + if (!t) + return -1; + + const TextChar* t1 = t + 1; + bool match = true; + for (const PatChar* p1 = patNext; p1 < patEnd; ++p1, ++t1) { + if (*t1 != *p1) { + match = false; + break; + } + } + + if (match) + return static_cast(t - text); + searchLength = static_cast(t - text); } return -1; @@ -2271,14 +2320,14 @@ js::HasSubstringAt(JSLinearString* text, JSLinearString* pat, size_t start) if (text->hasLatin1Chars()) { const Latin1Char* textChars = text->latin1Chars(nogc) + start; if (pat->hasLatin1Chars()) - return PodEqual(textChars, pat->latin1Chars(nogc), patLen); + return EqualChars(textChars, pat->latin1Chars(nogc), patLen); return EqualChars(textChars, pat->twoByteChars(nogc), patLen); } const char16_t* textChars = text->twoByteChars(nogc) + start; if (pat->hasTwoByteChars()) - return PodEqual(textChars, pat->twoByteChars(nogc), patLen); + return EqualChars(textChars, pat->twoByteChars(nogc), patLen); return EqualChars(pat->latin1Chars(nogc), textChars, patLen); } @@ -3988,13 +4037,13 @@ js::EqualChars(JSLinearString* str1, JSLinearString* str2) AutoCheckCannotGC nogc; if (str1->hasTwoByteChars()) { if (str2->hasTwoByteChars()) - return PodEqual(str1->twoByteChars(nogc), str2->twoByteChars(nogc), len); + return EqualChars(str1->twoByteChars(nogc), str2->twoByteChars(nogc), len); return EqualChars(str2->latin1Chars(nogc), str1->twoByteChars(nogc), len); } if (str2->hasLatin1Chars()) - return PodEqual(str1->latin1Chars(nogc), str2->latin1Chars(nogc), len); + return EqualChars(str1->latin1Chars(nogc), str2->latin1Chars(nogc), len); return EqualChars(str1->latin1Chars(nogc), str2->twoByteChars(nogc), len); } @@ -4110,7 +4159,7 @@ js::StringEqualsAscii(JSLinearString* str, const char* asciiBytes) AutoCheckCannotGC nogc; return str->hasLatin1Chars() - ? PodEqual(latin1, str->latin1Chars(nogc), length) + ? EqualChars(latin1, str->latin1Chars(nogc), length) : EqualChars(latin1, str->twoByteChars(nogc), length); } @@ -4207,12 +4256,15 @@ template const CharT* js_strchr_limit(const CharT* s, char16_t c, const CharT* limit) { - while (s < limit) { - if (*s == c) - return s; - s++; - } - return nullptr; + MOZ_ASSERT(limit >= s); + + // A Latin-1 buffer cannot contain a UTF-16 code unit above 0xff. Apart + // from avoiding a scan, this guard is required before narrowing |c| for + // the SIMD helper. + if (sizeof(CharT) == 1 && c > 0xff) + return nullptr; + + return FindCharacter(s, size_t(limit - s), CharT(c)); } template const Latin1Char* @@ -4232,8 +4284,20 @@ js::InflateString(ExclusiveContext* cx, const char* bytes, size_t* lengthp) chars = cx->pod_malloc(nchars + 1); if (!chars) goto bad; +#if defined(JS_HAVE_SSE2_INTRINSICS) + size_t i = 0; + const __m128i zero = _mm_setzero_si128(); + for (; i + 8 <= nchars; i += 8) { + const __m128i bytes8 = _mm_loadl_epi64(reinterpret_cast(bytes + i)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(chars + i), + _mm_unpacklo_epi8(bytes8, zero)); + } + for (; i < nchars; i++) + chars[i] = (unsigned char) bytes[i]; +#else for (size_t i = 0; i < nchars; i++) chars[i] = (unsigned char) bytes[i]; +#endif *lengthp = nchars; chars[nchars] = 0; return chars; @@ -4245,6 +4309,49 @@ js::InflateString(ExclusiveContext* cx, const char* bytes, size_t* lengthp) return nullptr; } +template +static inline void +DeflateChars(char* dst, const CharT* src, size_t length) +{ + static_assert(sizeof(CharT) == 1 || sizeof(CharT) == 2, "character width"); + + if (sizeof(CharT) == 1) { + memcpy(dst, src, length); + return; + } + +#if defined(JS_HAVE_SSE2_INTRINSICS) + size_t i = 0; + const __m128i lowByteMask = _mm_set1_epi16(0xff); + const __m128i zero = _mm_setzero_si128(); + for (; i + 32 <= length; i += 32) { + const __m128i wide0 = _mm_loadu_si128(reinterpret_cast(src + i)); + const __m128i wide1 = _mm_loadu_si128(reinterpret_cast(src + i + 8)); + const __m128i wide2 = _mm_loadu_si128(reinterpret_cast(src + i + 16)); + const __m128i wide3 = _mm_loadu_si128(reinterpret_cast(src + i + 24)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i), + _mm_packus_epi16(_mm_and_si128(wide0, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i + 8), + _mm_packus_epi16(_mm_and_si128(wide1, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i + 16), + _mm_packus_epi16(_mm_and_si128(wide2, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i + 24), + _mm_packus_epi16(_mm_and_si128(wide3, lowByteMask), zero)); + } + for (; i + 8 <= length; i += 8) { + const __m128i wide = _mm_loadu_si128(reinterpret_cast(src + i)); + const __m128i lowBytes = _mm_and_si128(wide, lowByteMask); + const __m128i packed = _mm_packus_epi16(lowBytes, zero); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i), packed); + } + for (; i < length; i++) + dst[i] = char(src[i]); +#else + for (size_t i = 0; i < length; i++) + dst[i] = char(src[i]); +#endif +} + template bool js::DeflateStringToBuffer(JSContext* maybecx, const CharT* src, size_t srclen, @@ -4252,8 +4359,7 @@ js::DeflateStringToBuffer(JSContext* maybecx, const CharT* src, size_t srclen, { size_t dstlen = *dstlenp; if (srclen > dstlen) { - for (size_t i = 0; i < dstlen; i++) - dst[i] = char(src[i]); + DeflateChars(dst, src, dstlen); if (maybecx) { AutoSuppressGC suppress(maybecx); JS_ReportErrorNumberASCII(maybecx, GetErrorMessage, nullptr, @@ -4261,8 +4367,7 @@ js::DeflateStringToBuffer(JSContext* maybecx, const CharT* src, size_t srclen, } return false; } - for (size_t i = 0; i < srclen; i++) - dst[i] = char(src[i]); + DeflateChars(dst, src, srclen); *dstlenp = srclen; return true; } diff --git a/js/src/jsstr.h b/js/src/jsstr.h index cd2be4e59b..7162559e94 100644 --- a/js/src/jsstr.h +++ b/js/src/jsstr.h @@ -53,7 +53,100 @@ template inline int32_t CompareChars(const Char1* s1, size_t len1, const Char2* s2, size_t len2) { + if (mozilla::IsSame::value && + reinterpret_cast(s1) == reinterpret_cast(s2)) + { + return int32_t(len1 - len2); + } + size_t n = Min(len1, len2); + +#if defined(JS_HAVE_SSE2_INTRINSICS) + if (sizeof(Char1) == 1 && sizeof(Char2) == 1) { + const uint8_t* left = reinterpret_cast(s1); + const uint8_t* right = reinterpret_cast(s2); + while (n >= 16) { + const __m128i leftBlock = _mm_loadu_si128(reinterpret_cast(left)); + const __m128i rightBlock = _mm_loadu_si128(reinterpret_cast(right)); + const uint32_t equalMask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(leftBlock, rightBlock))); + if (equalMask != 0xffff) { + const uint32_t lane = mozilla::CountTrailingZeroes32((~equalMask) & 0xffff); + return int32_t(left[lane]) - int32_t(right[lane]); + } + left += 16; + right += 16; + n -= 16; + } + s1 = reinterpret_cast(left); + s2 = reinterpret_cast(right); + } else if (sizeof(Char1) == 2 && sizeof(Char2) == 2) { + const char16_t* left = reinterpret_cast(s1); + const char16_t* right = reinterpret_cast(s2); + while (n >= 8) { + const __m128i leftBlock = _mm_loadu_si128(reinterpret_cast(left)); + const __m128i rightBlock = _mm_loadu_si128(reinterpret_cast(right)); + const uint32_t equalMask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi16(leftBlock, rightBlock))); + if (equalMask != 0xffff) { + const uint32_t lane = mozilla::CountTrailingZeroes32((~equalMask) & 0xffff) / 2; + return int32_t(left[lane]) - int32_t(right[lane]); + } + left += 8; + right += 8; + n -= 8; + } + s1 = reinterpret_cast(left); + s2 = reinterpret_cast(right); + } + + // Find the first differing code unit in eight mixed-width characters at + // once. The scalar result is still used for the first mismatch, so this + // preserves CompareChars' ordering semantics rather than merely testing + // equality. + if (sizeof(Char1) == 1 && sizeof(Char2) == 2) { + const uint8_t* bytes = reinterpret_cast(s1); + const char16_t* wide = reinterpret_cast(s2); + const __m128i zero = _mm_setzero_si128(); + while (n >= 8) { + const __m128i byteBlock = _mm_loadl_epi64(reinterpret_cast(bytes)); + const __m128i wideBlock = _mm_loadu_si128(reinterpret_cast(wide)); + const __m128i expanded = _mm_unpacklo_epi8(byteBlock, zero); + const uint32_t equalMask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi16(expanded, wideBlock))); + if (equalMask != 0xffff) { + const uint32_t lane = mozilla::CountTrailingZeroes32((~equalMask) & 0xffff) / 2; + return int32_t(bytes[lane]) - int32_t(wide[lane]); + } + bytes += 8; + wide += 8; + n -= 8; + } + s1 = reinterpret_cast(bytes); + s2 = reinterpret_cast(wide); + } else if (sizeof(Char1) == 2 && sizeof(Char2) == 1) { + const char16_t* wide = reinterpret_cast(s1); + const uint8_t* bytes = reinterpret_cast(s2); + const __m128i zero = _mm_setzero_si128(); + while (n >= 8) { + const __m128i wideBlock = _mm_loadu_si128(reinterpret_cast(wide)); + const __m128i byteBlock = _mm_loadl_epi64(reinterpret_cast(bytes)); + const __m128i expanded = _mm_unpacklo_epi8(byteBlock, zero); + const uint32_t equalMask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi16(wideBlock, expanded))); + if (equalMask != 0xffff) { + const uint32_t lane = mozilla::CountTrailingZeroes32((~equalMask) & 0xffff) / 2; + return int32_t(wide[lane]) - int32_t(bytes[lane]); + } + wide += 8; + bytes += 8; + n -= 8; + } + s1 = reinterpret_cast(wide); + s2 = reinterpret_cast(bytes); + } +#endif + for (size_t i = 0; i < n; i++) { if (int32_t cmp = s1[i] - s2[i]) return cmp; @@ -252,6 +345,67 @@ template inline bool EqualChars(const Char1* s1, const Char1* s2, size_t len) { + if (s1 == s2) + return true; + +#if defined(JS_HAVE_SSE2_INTRINSICS) + if (sizeof(Char1) == 1) { + const uint8_t* left = reinterpret_cast(s1); + const uint8_t* right = reinterpret_cast(s2); + while (len >= 64) { + for (unsigned block = 0; block < 4; block++) { + const __m128i leftBlock = _mm_loadu_si128( + reinterpret_cast(left + block * 16)); + const __m128i rightBlock = _mm_loadu_si128( + reinterpret_cast(right + block * 16)); + if (_mm_movemask_epi8(_mm_cmpeq_epi8(leftBlock, rightBlock)) != 0xffff) + return false; + } + left += 64; + right += 64; + len -= 64; + } + while (len >= 16) { + const __m128i leftBlock = _mm_loadu_si128(reinterpret_cast(left)); + const __m128i rightBlock = _mm_loadu_si128(reinterpret_cast(right)); + if (_mm_movemask_epi8(_mm_cmpeq_epi8(leftBlock, rightBlock)) != 0xffff) + return false; + left += 16; + right += 16; + len -= 16; + } + s1 = reinterpret_cast(left); + s2 = reinterpret_cast(right); + } else if (sizeof(Char1) == 2) { + const char16_t* left = reinterpret_cast(s1); + const char16_t* right = reinterpret_cast(s2); + while (len >= 32) { + for (unsigned block = 0; block < 4; block++) { + const __m128i leftBlock = _mm_loadu_si128( + reinterpret_cast(left + block * 8)); + const __m128i rightBlock = _mm_loadu_si128( + reinterpret_cast(right + block * 8)); + if (_mm_movemask_epi8(_mm_cmpeq_epi16(leftBlock, rightBlock)) != 0xffff) + return false; + } + left += 32; + right += 32; + len -= 32; + } + while (len >= 8) { + const __m128i leftBlock = _mm_loadu_si128(reinterpret_cast(left)); + const __m128i rightBlock = _mm_loadu_si128(reinterpret_cast(right)); + if (_mm_movemask_epi8(_mm_cmpeq_epi16(leftBlock, rightBlock)) != 0xffff) + return false; + left += 8; + right += 8; + len -= 8; + } + s1 = reinterpret_cast(left); + s2 = reinterpret_cast(right); + } +#endif + return mozilla::PodEqual(s1, s2, len); } @@ -259,6 +413,45 @@ template inline bool EqualChars(const Char1* s1, const Char2* s2, size_t len) { +#if defined(JS_HAVE_SSE2_INTRINSICS) + // Compare eight mixed-width characters at a time. Widening the Latin-1 + // bytes before comparing also makes values above 0xff fail naturally, + // preserving the scalar implementation's semantics. + if (sizeof(Char1) == 1 && sizeof(Char2) == 2) { + const uint8_t* bytes = reinterpret_cast(s1); + const char16_t* wide = reinterpret_cast(s2); + const __m128i zero = _mm_setzero_si128(); + while (len >= 8) { + const __m128i byteBlock = _mm_loadl_epi64(reinterpret_cast(bytes)); + const __m128i wideBlock = _mm_loadu_si128(reinterpret_cast(wide)); + const __m128i expanded = _mm_unpacklo_epi8(byteBlock, zero); + if (_mm_movemask_epi8(_mm_cmpeq_epi16(expanded, wideBlock)) != 0xffff) + return false; + bytes += 8; + wide += 8; + len -= 8; + } + s1 = reinterpret_cast(bytes); + s2 = reinterpret_cast(wide); + } else if (sizeof(Char1) == 2 && sizeof(Char2) == 1) { + const char16_t* wide = reinterpret_cast(s1); + const uint8_t* bytes = reinterpret_cast(s2); + const __m128i zero = _mm_setzero_si128(); + while (len >= 8) { + const __m128i wideBlock = _mm_loadu_si128(reinterpret_cast(wide)); + const __m128i byteBlock = _mm_loadl_epi64(reinterpret_cast(bytes)); + const __m128i expanded = _mm_unpacklo_epi8(byteBlock, zero); + if (_mm_movemask_epi8(_mm_cmpeq_epi16(wideBlock, expanded)) != 0xffff) + return false; + wide += 8; + bytes += 8; + len -= 8; + } + s1 = reinterpret_cast(wide); + s2 = reinterpret_cast(bytes); + } +#endif + for (const Char1* s1end = s1 + len; s1 < s1end; s1++, s2++) { if (*s1 != *s2) return false; @@ -290,15 +483,67 @@ InflateString(ExclusiveContext* cx, const char* bytes, size_t* length); inline void CopyAndInflateChars(char16_t* dst, const char* src, size_t srclen) { +#if defined(JS_HAVE_SSE2_INTRINSICS) + size_t i = 0; + const __m128i zero = _mm_setzero_si128(); + for (; i + 32 <= srclen; i += 32) { + const __m128i bytes0 = _mm_loadl_epi64(reinterpret_cast(src + i)); + const __m128i bytes1 = _mm_loadl_epi64(reinterpret_cast(src + i + 8)); + const __m128i bytes2 = _mm_loadl_epi64(reinterpret_cast(src + i + 16)); + const __m128i bytes3 = _mm_loadl_epi64(reinterpret_cast(src + i + 24)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), + _mm_unpacklo_epi8(bytes0, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 8), + _mm_unpacklo_epi8(bytes1, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16), + _mm_unpacklo_epi8(bytes2, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 24), + _mm_unpacklo_epi8(bytes3, zero)); + } + for (; i + 8 <= srclen; i += 8) { + const __m128i bytes8 = _mm_loadl_epi64(reinterpret_cast(src + i)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), + _mm_unpacklo_epi8(bytes8, zero)); + } + for (; i < srclen; i++) + dst[i] = (unsigned char) src[i]; +#else for (size_t i = 0; i < srclen; i++) dst[i] = (unsigned char) src[i]; +#endif } inline void CopyAndInflateChars(char16_t* dst, const JS::Latin1Char* src, size_t srclen) { +#if defined(JS_HAVE_SSE2_INTRINSICS) + size_t i = 0; + const __m128i zero = _mm_setzero_si128(); + for (; i + 32 <= srclen; i += 32) { + const __m128i bytes0 = _mm_loadl_epi64(reinterpret_cast(src + i)); + const __m128i bytes1 = _mm_loadl_epi64(reinterpret_cast(src + i + 8)); + const __m128i bytes2 = _mm_loadl_epi64(reinterpret_cast(src + i + 16)); + const __m128i bytes3 = _mm_loadl_epi64(reinterpret_cast(src + i + 24)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), + _mm_unpacklo_epi8(bytes0, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 8), + _mm_unpacklo_epi8(bytes1, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16), + _mm_unpacklo_epi8(bytes2, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 24), + _mm_unpacklo_epi8(bytes3, zero)); + } + for (; i + 8 <= srclen; i += 8) { + const __m128i bytes8 = _mm_loadl_epi64(reinterpret_cast(src + i)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), + _mm_unpacklo_epi8(bytes8, zero)); + } + for (; i < srclen; i++) + dst[i] = src[i]; +#else for (size_t i = 0; i < srclen; i++) dst[i] = src[i]; +#endif } /* diff --git a/js/src/jsutil.h b/js/src/jsutil.h index 1969a2e823..3de1bbeff6 100644 --- a/js/src/jsutil.h +++ b/js/src/jsutil.h @@ -113,6 +113,21 @@ js_memmove(void* dst_, const void* src_, size_t len) d += len; s += len; + while (len >= 64) { + d -= 64; + s -= 64; + // Load the complete chunk before storing it: the ranges may + // overlap, so an early store must not destroy a later load. + __m128i v0 = _mm_loadu_si128((const __m128i*)(s + 0)); + __m128i v1 = _mm_loadu_si128((const __m128i*)(s + 16)); + __m128i v2 = _mm_loadu_si128((const __m128i*)(s + 32)); + __m128i v3 = _mm_loadu_si128((const __m128i*)(s + 48)); + _mm_storeu_si128((__m128i*)(d + 0), v0); + _mm_storeu_si128((__m128i*)(d + 16), v1); + _mm_storeu_si128((__m128i*)(d + 32), v2); + _mm_storeu_si128((__m128i*)(d + 48), v3); + len -= 64; + } while (len >= 16) { d -= 16; s -= 16; diff --git a/js/src/vm/ArgumentsObject.cpp b/js/src/vm/ArgumentsObject.cpp index e23de30d66..2fd0f1de61 100644 --- a/js/src/vm/ArgumentsObject.cpp +++ b/js/src/vm/ArgumentsObject.cpp @@ -11,6 +11,7 @@ #include "vm/AsyncFunction.h" #include "vm/GlobalObject.h" #include "vm/Stack.h" +#include "jsutil.h" #include "jsobjinlines.h" @@ -36,7 +37,7 @@ RareArgumentsData::create(JSContext* cx, ArgumentsObject* obj) if (!data) return nullptr; - mozilla::PodZero(data, bytes); + js_memset(data, 0, bytes); return new(data) RareArgumentsData(); } @@ -299,7 +300,7 @@ ArgumentsObject::create(JSContext* cx, HandleFunction callee, unsigned numActual // Zero the argument Values. This sets each value to DoubleValue(0), which // is safe for GC tracing. - memset(data->args, 0, numArgs * sizeof(Value)); + js_memset(data->args, 0, numArgs * sizeof(Value)); MOZ_ASSERT(DoubleValue(0).asRawBits() == 0x0); MOZ_ASSERT_IF(numArgs > 0, data->args[0].asRawBits() == 0x0); @@ -815,7 +816,7 @@ ArgumentsObject::objectMovedDuringMinorGC(JSTracer* trc, JSObject* dst, JSObject oomUnsafe.crash("Failed to allocate ArgumentsObject data while tenuring."); ndst->initFixedSlot(DATA_SLOT, PrivateValue(data)); - mozilla::PodCopy(data, reinterpret_cast(nsrc->data()), nbytes); + js_memcpy(data, reinterpret_cast(nsrc->data()), nbytes); nbytesTotal += nbytes; } @@ -830,7 +831,7 @@ ArgumentsObject::objectMovedDuringMinorGC(JSTracer* trc, JSObject* dst, JSObject oomUnsafe.crash("Failed to allocate RareArgumentsData data while tenuring."); ndst->data()->rareData = (RareArgumentsData*)dstRareData; - mozilla::PodCopy(dstRareData, reinterpret_cast(srcRareData), nbytes); + js_memcpy(dstRareData, reinterpret_cast(srcRareData), nbytes); nbytesTotal += nbytes; } } diff --git a/js/src/vm/ArrayBufferObject.cpp b/js/src/vm/ArrayBufferObject.cpp index 666fc774df..b0c0037169 100644 --- a/js/src/vm/ArrayBufferObject.cpp +++ b/js/src/vm/ArrayBufferObject.cpp @@ -656,7 +656,7 @@ ResizeArrayBuffer(JSContext* cx, Handle buffer, uint32_t new uint32_t copyLength = std::min(newByteLength, buffer->byteLength()); if (copyLength > 0) - memcpy(newContents.data(), buffer->dataPointer(), copyLength); + js_memcpy(newContents.data(), buffer->dataPointer(), copyLength); buffer->changeContentsForResize(cx, newContents, ArrayBufferObject::OwnsData, newByteLength); return true; @@ -729,7 +729,7 @@ ArrayBufferTransfer(JSContext* cx, const CallArgs& args, bool preserveResizabili uint32_t copyLength = std::min(newByteLength, buffer->byteLength()); if (copyLength > 0) - memcpy(newBuffer->dataPointer(), buffer->dataPointer(), copyLength); + js_memcpy(newBuffer->dataPointer(), buffer->dataPointer(), copyLength); ArrayBufferObject::BufferContents detachedContents = buffer->hasStealableContents() ? ArrayBufferObject::BufferContents::createPlain(nullptr) @@ -1428,7 +1428,7 @@ ArrayBufferObject::create(JSContext* cx, uint32_t nbytes, BufferContents content if (!contents) { void* data = obj->inlineDataPointer(); - memset(data, 0, nbytes); + js_memset(data, 0, nbytes); obj->initialize(nbytes, BufferContents::createPlain(data), DoesntOwnData, maxByteLength, resizable); } else { diff --git a/js/src/vm/CharacterEncoding.cpp b/js/src/vm/CharacterEncoding.cpp index b126e8a05a..585f6a150a 100644 --- a/js/src/vm/CharacterEncoding.cpp +++ b/js/src/vm/CharacterEncoding.cpp @@ -9,10 +9,12 @@ #include "mozilla/Sprintf.h" #include +#include #include #include "jscntxt.h" #include "jsprf.h" +#include "vm/CharacterOperations.h" using namespace js; @@ -25,8 +27,40 @@ JS::LossyTwoByteCharsToNewLatin1CharsZ(js::ExclusiveContext* cx, unsigned char* latin1 = cx->pod_malloc(len + 1); if (!latin1) return Latin1CharsZ(); +#if defined(JS_HAS_SSE2_CHARACTER_OPERATIONS) + size_t i = 0; + const __m128i lowByteMask = _mm_set1_epi16(0xff); + const __m128i zero = _mm_setzero_si128(); + for (; i + 32 <= len; i += 32) { + const __m128i wide0 = _mm_loadu_si128( + reinterpret_cast(tbchars.begin().get() + i)); + const __m128i wide1 = _mm_loadu_si128( + reinterpret_cast(tbchars.begin().get() + i + 8)); + const __m128i wide2 = _mm_loadu_si128( + reinterpret_cast(tbchars.begin().get() + i + 16)); + const __m128i wide3 = _mm_loadu_si128( + reinterpret_cast(tbchars.begin().get() + i + 24)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i), + _mm_packus_epi16(_mm_and_si128(wide0, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i + 8), + _mm_packus_epi16(_mm_and_si128(wide1, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i + 16), + _mm_packus_epi16(_mm_and_si128(wide2, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i + 24), + _mm_packus_epi16(_mm_and_si128(wide3, lowByteMask), zero)); + } + for (; i + 8 <= len; i += 8) { + const __m128i wide = _mm_loadu_si128(reinterpret_cast(tbchars.begin().get() + i)); + const __m128i lowBytes = _mm_and_si128(wide, lowByteMask); + const __m128i packed = _mm_packus_epi16(lowBytes, zero); + _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i), packed); + } + for (; i < len; ++i) + latin1[i] = static_cast(tbchars[i]); +#else for (size_t i = 0; i < len; ++i) latin1[i] = static_cast(tbchars[i]); +#endif latin1[len] = '\0'; return Latin1CharsZ(latin1, len); } @@ -423,8 +457,43 @@ InflateUTF8StringHelper(ContextT* cx, const UTF8Chars src, size_t* outlen) if (encoding == JS::SmallestEncoding::ASCII) { size_t srclen = src.length(); MOZ_ASSERT(*outlen == srclen); - for (uint32_t i = 0; i < srclen; i++) - dst[i] = CharT(src[i]); + if (sizeof(CharT) == 1) { + memcpy(dst, src.begin().get(), srclen); + } else { +#if defined(JS_HAS_SSE2_CHARACTER_OPERATIONS) + size_t i = 0; + const __m128i zero = _mm_setzero_si128(); + for (; i + 32 <= srclen; i += 32) { + const __m128i bytes0 = _mm_loadl_epi64( + reinterpret_cast(src.begin().get() + i)); + const __m128i bytes1 = _mm_loadl_epi64( + reinterpret_cast(src.begin().get() + i + 8)); + const __m128i bytes2 = _mm_loadl_epi64( + reinterpret_cast(src.begin().get() + i + 16)); + const __m128i bytes3 = _mm_loadl_epi64( + reinterpret_cast(src.begin().get() + i + 24)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), + _mm_unpacklo_epi8(bytes0, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 8), + _mm_unpacklo_epi8(bytes1, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16), + _mm_unpacklo_epi8(bytes2, zero)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 24), + _mm_unpacklo_epi8(bytes3, zero)); + } + for (; i + 8 <= srclen; i += 8) { + const __m128i bytes8 = _mm_loadl_epi64( + reinterpret_cast(src.begin().get() + i)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), + _mm_unpacklo_epi8(bytes8, zero)); + } + for (; i < srclen; i++) + dst[i] = CharT(src[i]); +#else + for (size_t i = 0; i < srclen; i++) + dst[i] = CharT(src[i]); +#endif + } } else { MOZ_ALWAYS_TRUE((InflateUTF8StringToBuffer(cx, src, dst, outlen, &encoding))); } diff --git a/js/src/vm/CharacterOperations.h b/js/src/vm/CharacterOperations.h index 591a3216bf..ecc976237b 100644 --- a/js/src/vm/CharacterOperations.h +++ b/js/src/vm/CharacterOperations.h @@ -31,6 +31,54 @@ FindCharacter(const CharT* chars, size_t length, CharT match) const __m128i needle = sizeof(CharT) == 1 ? _mm_set1_epi8(static_cast(match)) : _mm_set1_epi16(static_cast(match)); + while (length >= 4 * lanes) { + end -= 4 * lanes; + length -= 4 * lanes; + + const __m128i block3 = _mm_loadu_si128( + reinterpret_cast(end + 3 * lanes)); + const uint32_t mask3 = static_cast( + _mm_movemask_epi8(sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block3, needle) + : _mm_cmpeq_epi16(block3, needle))); + if (mask3) { + const uint32_t highest = 31 - mozilla::CountLeadingZeroes32(mask3); + return end + 3 * lanes + highest / sizeof(CharT); + } + + const __m128i block2 = _mm_loadu_si128( + reinterpret_cast(end + 2 * lanes)); + const uint32_t mask2 = static_cast( + _mm_movemask_epi8(sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block2, needle) + : _mm_cmpeq_epi16(block2, needle))); + if (mask2) { + const uint32_t highest = 31 - mozilla::CountLeadingZeroes32(mask2); + return end + 2 * lanes + highest / sizeof(CharT); + } + + const __m128i block1 = _mm_loadu_si128( + reinterpret_cast(end + lanes)); + const uint32_t mask1 = static_cast( + _mm_movemask_epi8(sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block1, needle) + : _mm_cmpeq_epi16(block1, needle))); + if (mask1) { + const uint32_t highest = 31 - mozilla::CountLeadingZeroes32(mask1); + return end + lanes + highest / sizeof(CharT); + } + + const __m128i block0 = _mm_loadu_si128( + reinterpret_cast(end)); + const uint32_t mask0 = static_cast( + _mm_movemask_epi8(sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block0, needle) + : _mm_cmpeq_epi16(block0, needle))); + if (mask0) { + const uint32_t highest = 31 - mozilla::CountLeadingZeroes32(mask0); + return end + highest / sizeof(CharT); + } + } do { // Never read beyond the supplied span, even at a page boundary. const __m128i block = _mm_loadu_si128(reinterpret_cast(chars)); @@ -52,6 +100,85 @@ FindCharacter(const CharT* chars, size_t length, CharT match) return nullptr; } +template +inline const CharT* +FindCharacterReverse(const CharT* chars, size_t length, CharT match) +{ + static_assert(sizeof(CharT) == 1 || sizeof(CharT) == 2, "character width"); +#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS + const size_t lanes = 16 / sizeof(CharT); + const CharT* end = chars + length; + if (length >= lanes) { + const __m128i needle = sizeof(CharT) == 1 + ? _mm_set1_epi8(static_cast(match)) + : _mm_set1_epi16(static_cast(match)); + while (length >= 4 * lanes) { + const __m128i block0 = _mm_loadu_si128( + reinterpret_cast(chars)); + const uint32_t mask0 = static_cast( + _mm_movemask_epi8(sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block0, needle) + : _mm_cmpeq_epi16(block0, needle))); + if (mask0) + return chars + mozilla::CountTrailingZeroes32(mask0) / sizeof(CharT); + + const __m128i block1 = _mm_loadu_si128( + reinterpret_cast(chars + lanes)); + const uint32_t mask1 = static_cast( + _mm_movemask_epi8(sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block1, needle) + : _mm_cmpeq_epi16(block1, needle))); + if (mask1) + return chars + lanes + mozilla::CountTrailingZeroes32(mask1) / sizeof(CharT); + + const __m128i block2 = _mm_loadu_si128( + reinterpret_cast(chars + 2 * lanes)); + const uint32_t mask2 = static_cast( + _mm_movemask_epi8(sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block2, needle) + : _mm_cmpeq_epi16(block2, needle))); + if (mask2) + return chars + 2 * lanes + mozilla::CountTrailingZeroes32(mask2) / sizeof(CharT); + + const __m128i block3 = _mm_loadu_si128( + reinterpret_cast(chars + 3 * lanes)); + const uint32_t mask3 = static_cast( + _mm_movemask_epi8(sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block3, needle) + : _mm_cmpeq_epi16(block3, needle))); + if (mask3) + return chars + 3 * lanes + mozilla::CountTrailingZeroes32(mask3) / sizeof(CharT); + + chars += 4 * lanes; + length -= 4 * lanes; + } + do { + end -= lanes; + length -= lanes; + const __m128i block = _mm_loadu_si128(reinterpret_cast(end)); + const __m128i equal = sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block, needle) + : _mm_cmpeq_epi16(block, needle); + const uint32_t mask = static_cast(_mm_movemask_epi8(equal)); + if (mask) { + const uint32_t highest = 31 - mozilla::CountLeadingZeroes32(mask); + return end + highest / sizeof(CharT); + } + } while (length >= lanes); + } +#else + const CharT* end = chars + length; +#endif + + while (length) { + --end; + --length; + if (*end == match) + return end; + } + return nullptr; +} + inline bool CharactersFitInLatin1(const char16_t* chars, size_t length) { @@ -59,6 +186,34 @@ CharactersFitInLatin1(const char16_t* chars, size_t length) if (length >= 8) { const __m128i highBytes = _mm_set1_epi16(static_cast(0xff00)); const __m128i zero = _mm_setzero_si128(); + while (length >= 32) { + const __m128i block0 = _mm_loadu_si128( + reinterpret_cast(chars)); + if (_mm_movemask_epi8(_mm_cmpeq_epi16( + _mm_and_si128(block0, highBytes), zero)) != 0xffff) + return false; + + const __m128i block1 = _mm_loadu_si128( + reinterpret_cast(chars + 8)); + if (_mm_movemask_epi8(_mm_cmpeq_epi16( + _mm_and_si128(block1, highBytes), zero)) != 0xffff) + return false; + + const __m128i block2 = _mm_loadu_si128( + reinterpret_cast(chars + 16)); + if (_mm_movemask_epi8(_mm_cmpeq_epi16( + _mm_and_si128(block2, highBytes), zero)) != 0xffff) + return false; + + const __m128i block3 = _mm_loadu_si128( + reinterpret_cast(chars + 24)); + if (_mm_movemask_epi8(_mm_cmpeq_epi16( + _mm_and_si128(block3, highBytes), zero)) != 0xffff) + return false; + + chars += 32; + length -= 32; + } do { const __m128i block = _mm_loadu_si128(reinterpret_cast(chars)); const __m128i fits = _mm_cmpeq_epi16(_mm_and_si128(block, highBytes), zero); diff --git a/js/src/vm/Interpreter.cpp b/js/src/vm/Interpreter.cpp index ad52234a31..50ce7a1728 100644 --- a/js/src/vm/Interpreter.cpp +++ b/js/src/vm/Interpreter.cpp @@ -31,6 +31,7 @@ #include "jsprf.h" #include "jsscript.h" #include "jsstr.h" +#include "jsutil.h" #include "builtin/Eval.h" #include "builtin/ModuleObject.h" @@ -2147,7 +2148,7 @@ CASE(JSOP_PICK) unsigned i = GET_UINT8(REGS.pc); MOZ_ASSERT(REGS.stackDepth() >= i + 1); Value lval = REGS.sp[-int(i + 1)]; - memmove(REGS.sp - (i + 1), REGS.sp - i, sizeof(Value) * i); + js_memmove(REGS.sp - (i + 1), REGS.sp - i, sizeof(Value) * i); REGS.sp[-1] = lval; } END_CASE(JSOP_PICK) @@ -2157,7 +2158,7 @@ CASE(JSOP_UNPICK) int i = GET_UINT8(REGS.pc); MOZ_ASSERT(REGS.stackDepth() >= unsigned(i) + 1); Value lval = REGS.sp[-1]; - memmove(REGS.sp - i, REGS.sp - (i + 1), sizeof(Value) * i); + js_memmove(REGS.sp - i, REGS.sp - (i + 1), sizeof(Value) * i); REGS.sp[-(i + 1)] = lval; } END_CASE(JSOP_UNPICK) diff --git a/js/src/vm/NativeObject.h b/js/src/vm/NativeObject.h index 67fd3a7a46..63779d8142 100644 --- a/js/src/vm/NativeObject.h +++ b/js/src/vm/NativeObject.h @@ -13,6 +13,7 @@ #include "jsfriendapi.h" #include "jsobj.h" +#include "jsutil.h" #include "NamespaceImports.h" #include "gc/Barrier.h" @@ -1122,8 +1123,8 @@ class NativeObject : public ShapedObject for (uint32_t i = 0; i < count; ++i) elements_[dstStart + i].set(this, HeapSlot::Element, dstStart + i, src[i]); } else { - memcpy(reinterpret_cast(&elements_[dstStart]), src, - count * sizeof(Value)); + js_memcpy(reinterpret_cast(&elements_[dstStart]), src, + count * sizeof(Value)); elementsRangeWriteBarrierPost(dstStart, count); } } @@ -1132,7 +1133,7 @@ class NativeObject : public ShapedObject MOZ_ASSERT(dstStart + count <= getDenseCapacity()); MOZ_ASSERT(!denseElementsAreCopyOnWrite()); MOZ_ASSERT(!denseElementsAreFrozen()); - memcpy(reinterpret_cast(&elements_[dstStart]), src, count * sizeof(Value)); + js_memcpy(reinterpret_cast(&elements_[dstStart]), src, count * sizeof(Value)); elementsRangeWriteBarrierPost(dstStart, count); } @@ -1167,7 +1168,7 @@ class NativeObject : public ShapedObject dst->set(this, HeapSlot::Element, dst - elements_, *src); } } else { - memmove(elements_ + dstStart, elements_ + srcStart, count * sizeof(HeapSlot)); + js_memmove(elements_ + dstStart, elements_ + srcStart, count * sizeof(HeapSlot)); elementsRangeWriteBarrierPost(dstStart, count); } } @@ -1180,7 +1181,7 @@ class NativeObject : public ShapedObject MOZ_ASSERT(!denseElementsAreCopyOnWrite()); MOZ_ASSERT(!denseElementsAreFrozen()); - memmove(elements_ + dstStart, elements_ + srcStart, count * sizeof(HeapSlot)); + js_memmove(elements_ + dstStart, elements_ + srcStart, count * sizeof(HeapSlot)); elementsRangeWriteBarrierPost(dstStart, count); } diff --git a/js/src/vm/String-inl.h b/js/src/vm/String-inl.h index 52261d0f75..f186f02afc 100644 --- a/js/src/vm/String-inl.h +++ b/js/src/vm/String-inl.h @@ -8,11 +8,11 @@ #include "vm/String.h" -#include "mozilla/PodOperations.h" #include "mozilla/Range.h" #include "jscntxt.h" #include "jscompartment.h" +#include "jsutil.h" #include "gc/Allocator.h" #include "gc/Marking.h" @@ -57,7 +57,7 @@ NewInlineString(ExclusiveContext* cx, mozilla::Range chars) if (!str) return nullptr; - mozilla::PodCopy(storage, chars.begin().get(), len); + js_memcpy(storage, chars.begin().get(), len); storage[len] = 0; return str; } @@ -75,7 +75,7 @@ NewInlineString(ExclusiveContext* cx, HandleLinearString base, size_t start, siz return nullptr; JS::AutoCheckCannotGC nogc; - mozilla::PodCopy(chars, base->chars(nogc) + start, length); + js_memcpy(chars, base->chars(nogc) + start, length * sizeof(CharT)); chars[length] = 0; return s; } diff --git a/js/src/vm/String.cpp b/js/src/vm/String.cpp index 3ef6fb229d..2a292c9927 100644 --- a/js/src/vm/String.cpp +++ b/js/src/vm/String.cpp @@ -7,7 +7,6 @@ #include "mozilla/MathAlgorithms.h" #include "mozilla/MemoryReporting.h" -#include "mozilla/PodOperations.h" #include "mozilla/RangedPtr.h" #include "mozilla/SizePrintfMacros.h" #include "mozilla/TypeTraits.h" @@ -20,11 +19,11 @@ #include "jscntxtinlines.h" #include "jscompartmentinlines.h" +#include "jsutil.h" using namespace js; using mozilla::IsSame; -using mozilla::PodCopy; using mozilla::RangedPtr; using mozilla::RoundUpPow2; @@ -346,7 +345,7 @@ CopyChars(char16_t* dest, const JSLinearString& str) { AutoCheckCannotGC nogc; if (str.hasTwoByteChars()) - PodCopy(dest, str.twoByteChars(nogc), str.length()); + js_memcpy(dest, str.twoByteChars(nogc), str.length() * sizeof(char16_t)); else CopyAndInflateChars(dest, str.latin1Chars(nogc), str.length()); } @@ -357,7 +356,7 @@ CopyChars(Latin1Char* dest, const JSLinearString& str) { AutoCheckCannotGC nogc; if (str.hasLatin1Chars()) { - PodCopy(dest, str.latin1Chars(nogc), str.length()); + js_memcpy(dest, str.latin1Chars(nogc), str.length()); } else { /* * When we flatten a TwoByte rope, we turn child ropes (including Latin1 @@ -369,10 +368,24 @@ CopyChars(Latin1Char* dest, const JSLinearString& str) */ size_t len = str.length(); const char16_t* chars = str.twoByteChars(nogc); +#if defined(JS_HAS_SSE2_CHARACTER_OPERATIONS) + size_t i = 0; + const __m128i zero = _mm_setzero_si128(); + for (; i + 8 <= len; i += 8) { + const __m128i wide = _mm_loadu_si128(reinterpret_cast(chars + i)); + const __m128i packed = _mm_packus_epi16(wide, zero); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i), packed); + } + for (; i < len; i++) { + MOZ_ASSERT(chars[i] <= JSString::MAX_LATIN1_CHAR); + dest[i] = chars[i]; + } +#else for (size_t i = 0; i < len; i++) { MOZ_ASSERT(chars[i] <= JSString::MAX_LATIN1_CHAR); dest[i] = chars[i]; } +#endif } } @@ -639,16 +652,17 @@ js::ConcatStrings(ExclusiveContext* cx, return nullptr; if (isLatin1) { - PodCopy(latin1Buf, leftLinear->latin1Chars(nogc), leftLen); - PodCopy(latin1Buf + leftLen, rightLinear->latin1Chars(nogc), rightLen); + js_memcpy(latin1Buf, leftLinear->latin1Chars(nogc), leftLen); + js_memcpy(latin1Buf + leftLen, rightLinear->latin1Chars(nogc), rightLen); latin1Buf[wholeLength] = 0; } else { if (leftLinear->hasTwoByteChars()) - PodCopy(twoByteBuf, leftLinear->twoByteChars(nogc), leftLen); + js_memcpy(twoByteBuf, leftLinear->twoByteChars(nogc), leftLen * sizeof(char16_t)); else CopyAndInflateChars(twoByteBuf, leftLinear->latin1Chars(nogc), leftLen); if (rightLinear->hasTwoByteChars()) - PodCopy(twoByteBuf + leftLen, rightLinear->twoByteChars(nogc), rightLen); + js_memcpy(twoByteBuf + leftLen, rightLinear->twoByteChars(nogc), + rightLen * sizeof(char16_t)); else CopyAndInflateChars(twoByteBuf + leftLen, rightLinear->latin1Chars(nogc), rightLen); twoByteBuf[wholeLength] = 0; @@ -676,7 +690,7 @@ JSDependentString::undependInternal(JSContext* cx) return nullptr; AutoCheckCannotGC nogc; - PodCopy(s, nonInlineChars(nogc), n); + js_memcpy(s, nonInlineChars(nogc), n * sizeof(CharT)); s[n] = '\0'; setNonInlineChars(s); @@ -1026,7 +1040,7 @@ AutoStableStringChars::copyLatin1Chars(JSContext* cx, HandleLinearString linearS if (!chars) return false; - PodCopy(chars, linearString->rawLatin1Chars(), length); + js_memcpy(chars, linearString->rawLatin1Chars(), length); chars[length] = 0; state_ = Latin1; @@ -1043,7 +1057,7 @@ AutoStableStringChars::copyTwoByteChars(JSContext* cx, HandleLinearString linear if (!chars) return false; - PodCopy(chars, linearString->rawTwoByteChars(), length); + js_memcpy(chars, linearString->rawTwoByteChars(), length * sizeof(char16_t)); chars[length] = 0; state_ = TwoByte; @@ -1077,7 +1091,7 @@ JSExternalString::ensureFlat(JSContext* cx) // Copy the chars before finalizing the string. { AutoCheckCannotGC nogc; - PodCopy(s, nonInlineChars(nogc), n); + js_memcpy(s, nonInlineChars(nogc), n * sizeof(char16_t)); s[n] = '\0'; } @@ -1158,6 +1172,41 @@ CanStoreCharsAsLatin1(const Latin1Char* s, size_t length) MOZ_CRASH("Shouldn't be called for Latin1 chars"); } +static MOZ_ALWAYS_INLINE void +CopyAndDeflateLatin1Chars(Latin1Char* dest, const char16_t* src, size_t length) +{ +#if defined(JS_HAS_SSE2_CHARACTER_OPERATIONS) + size_t i = 0; + const __m128i lowByteMask = _mm_set1_epi16(0xff); + const __m128i zero = _mm_setzero_si128(); + for (; i + 32 <= length; i += 32) { + const __m128i wide0 = _mm_loadu_si128(reinterpret_cast(src + i)); + const __m128i wide1 = _mm_loadu_si128(reinterpret_cast(src + i + 8)); + const __m128i wide2 = _mm_loadu_si128(reinterpret_cast(src + i + 16)); + const __m128i wide3 = _mm_loadu_si128(reinterpret_cast(src + i + 24)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i), + _mm_packus_epi16(_mm_and_si128(wide0, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i + 8), + _mm_packus_epi16(_mm_and_si128(wide1, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i + 16), + _mm_packus_epi16(_mm_and_si128(wide2, lowByteMask), zero)); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i + 24), + _mm_packus_epi16(_mm_and_si128(wide3, lowByteMask), zero)); + } + for (; i + 8 <= length; i += 8) { + const __m128i wide = _mm_loadu_si128(reinterpret_cast(src + i)); + const __m128i lowBytes = _mm_and_si128(wide, lowByteMask); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i), + _mm_packus_epi16(lowBytes, zero)); + } + for (; i < length; i++) + dest[i] = Latin1Char(src[i]); +#else + for (size_t i = 0; i < length; i++) + dest[i] = Latin1Char(src[i]); +#endif +} + template static MOZ_ALWAYS_INLINE JSInlineString* NewInlineStringDeflated(ExclusiveContext* cx, mozilla::Range chars) @@ -1170,8 +1219,8 @@ NewInlineStringDeflated(ExclusiveContext* cx, mozilla::Range cha for (size_t i = 0; i < len; i++) { MOZ_ASSERT(chars[i] <= JSString::MAX_LATIN1_CHAR); - storage[i] = Latin1Char(chars[i]); } + CopyAndDeflateLatin1Chars(storage, chars.begin().get(), len); storage[len] = '\0'; return str; } @@ -1210,8 +1259,8 @@ NewStringDeflated(ExclusiveContext* cx, const char16_t* s, size_t n) for (size_t i = 0; i < n; i++) { MOZ_ASSERT(s[i] <= JSString::MAX_LATIN1_CHAR); - news.get()[i] = Latin1Char(s[i]); } + CopyAndDeflateLatin1Chars(news.get(), s, n); news[n] = '\0'; JSFlatString* str = JSFlatString::new_(cx, news.get(), n); @@ -1313,7 +1362,7 @@ NewStringCopyNDontDeflate(ExclusiveContext* cx, const CharT* s, size_t n) return nullptr; } - PodCopy(news.get(), s, n); + js_memcpy(news.get(), s, n * sizeof(CharT)); news[n] = 0; JSFlatString* str = JSFlatString::new_(cx, news.get(), n); diff --git a/js/src/vm/TypedArrayObject.cpp b/js/src/vm/TypedArrayObject.cpp index 22d21c8550..5a759db27f 100644 --- a/js/src/vm/TypedArrayObject.cpp +++ b/js/src/vm/TypedArrayObject.cpp @@ -3413,6 +3413,22 @@ js::StringIsTypedArrayIndex(const CharT* s, size_t length, uint64_t* indexp) index = digit; + // Most typed-array accesses use one- or two-digit indices. Once the + // digits have been validated, these forms cannot overflow uint64_t and + // need no general-purpose accumulation loop. + if (s == end) { + *indexp = negative ? UINT64_MAX : index; + return true; + } + + if (end - s == 1) { + if (!JS7_ISDEC(*s)) + return false; + digit = JS7_UNDEC(*s); + *indexp = negative ? UINT64_MAX : index * 10 + digit; + return true; + } + for (; s < end; s++) { if (!JS7_ISDEC(*s)) return false; diff --git a/js/src/vm/UnboxedObject-inl.h b/js/src/vm/UnboxedObject-inl.h index fa986a7575..3a983e2f96 100644 --- a/js/src/vm/UnboxedObject-inl.h +++ b/js/src/vm/UnboxedObject-inl.h @@ -583,9 +583,9 @@ MoveBoxedOrUnboxedDenseElements(JSContext* cx, JSObject* obj, uint32_t dstStart, obj->as().triggerPreBarrier(dstStart + i); } - memmove(data + dstStart * elementSize, - data + srcStart * elementSize, - length * elementSize); + js_memmove(data + dstStart * elementSize, + data + srcStart * elementSize, + length * elementSize); } return DenseElementResult::Success; @@ -619,9 +619,9 @@ CopyBoxedOrUnboxedDenseElements(JSContext* cx, JSObject* dst, JSObject* src, uint8_t* srcData = src->as().elements(); size_t elementSize = UnboxedTypeSize(DstType); - memcpy(dstData + dstStart * elementSize, - srcData + srcStart * elementSize, - length * elementSize); + js_memcpy(dstData + dstStart * elementSize, + srcData + srcStart * elementSize, + length * elementSize); // Add a store buffer entry if we might have copied a nursery pointer to dst. if (UnboxedTypeNeedsPostBarrier(DstType) && !IsInsideNursery(dst)) diff --git a/js/src/vm/UnboxedObject.cpp b/js/src/vm/UnboxedObject.cpp index 2ed89e32e4..bfb5bbc7bc 100644 --- a/js/src/vm/UnboxedObject.cpp +++ b/js/src/vm/UnboxedObject.cpp @@ -1895,7 +1895,7 @@ UnboxedPlainObject::fillAfterConvert(ExclusiveContext* cx, Handle> values, size_t* valueCursor) { initExpando(); - memset(data(), 0, layout().size()); + js_memset(data(), 0, layout().size()); for (size_t i = 0; i < layout().properties().length(); i++) JS_ALWAYS_TRUE(setValue(cx, layout().properties()[i], NextValue(values, valueCursor))); } diff --git a/mfbt/HashFunctions.cpp b/mfbt/HashFunctions.cpp index 6ba3c2d6e9..63418401a2 100644 --- a/mfbt/HashFunctions.cpp +++ b/mfbt/HashFunctions.cpp @@ -20,7 +20,17 @@ HashBytes(const void* aBytes, size_t aLength) /* Walk word by word. */ size_t i = 0; - for (; i < aLength - (aLength % sizeof(size_t)); i += sizeof(size_t)) { + const size_t wordLength = aLength - (aLength % sizeof(size_t)); + const size_t doubleWordLength = wordLength - (wordLength % (2 * sizeof(size_t))); + for (; i < doubleWordLength; i += 2 * sizeof(size_t)) { + size_t data0; + size_t data1; + memcpy(&data0, b + i, sizeof(data0)); + memcpy(&data1, b + i + sizeof(data0), sizeof(data1)); + hash = AddToHash(hash, data0, sizeof(data0)); + hash = AddToHash(hash, data1, sizeof(data1)); + } + for (; i < wordLength; i += sizeof(size_t)) { /* Do an explicitly unaligned load of the data. */ size_t data; memcpy(&data, b + i, sizeof(size_t)); diff --git a/mfbt/HashFunctions.h b/mfbt/HashFunctions.h index d287081174..fa123d1085 100644 --- a/mfbt/HashFunctions.h +++ b/mfbt/HashFunctions.h @@ -230,8 +230,17 @@ uint32_t HashKnownLength(const T* aStr, size_t aLength) { uint32_t hash = 0; - for (size_t i = 0; i < aLength; i++) { - hash = AddToHash(hash, aStr[i]); + while (aLength >= 4) { + hash = AddToHash(hash, aStr[0]); + hash = AddToHash(hash, aStr[1]); + hash = AddToHash(hash, aStr[2]); + hash = AddToHash(hash, aStr[3]); + aStr += 4; + aLength -= 4; + } + while (aLength) { + hash = AddToHash(hash, *aStr++); + --aLength; } return hash; }