From b7c0ebfcd4c4a44562342e940048fd3f5bdc6aae Mon Sep 17 00:00:00 2001 From: wuggy Date: Sat, 12 Sep 2026 06:30:01 -0700 Subject: [PATCH] Optimize string operations with SSE2 --- js/src/jit-test/tests/latin1/sse2-search.js | 21 +++ js/src/jsapi-tests/moz.build | 1 + .../jsapi-tests/testCharacterOperations.cpp | 145 ++++++++++++++++++ js/src/jsstr.cpp | 29 ++-- js/src/vm/CharacterOperations.h | 81 ++++++++++ js/src/vm/String.cpp | 9 +- 6 files changed, 268 insertions(+), 18 deletions(-) create mode 100644 js/src/jit-test/tests/latin1/sse2-search.js create mode 100644 js/src/jsapi-tests/testCharacterOperations.cpp create mode 100644 js/src/vm/CharacterOperations.h diff --git a/js/src/jit-test/tests/latin1/sse2-search.js b/js/src/jit-test/tests/latin1/sse2-search.js new file mode 100644 index 0000000000..064e802b87 --- /dev/null +++ b/js/src/jit-test/tests/latin1/sse2-search.js @@ -0,0 +1,21 @@ +// Exercise vector-sized spans, scalar tails, and mixed character encodings. +for (var length of [0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65]) { + for (var needle of ["\0", "\x7f", "\x80", "\xff", "\u0100", "\ud800", "\uffff"]) { + for (var position = 0; position <= length; ++position) { + var text = "a".repeat(position) + needle + "a".repeat(length - position); + for (var start of [0, position, position + 1, text.length]) { + var expected = start <= position ? position : -1; + assertEq(text.indexOf(needle, start), expected); + assertEq(text.includes(needle, start), expected !== -1); + } + assertEq(text.indexOf(needle + "b"), -1); + assertEq(text.indexOf(needle + "a"), position < length ? position : -1); + } + } + var latin1 = "\0".repeat(length) + "\xff"; + assertEq(latin1.indexOf("\u0100"), -1); // Must not narrow to NUL on x86. + assertEq(latin1.indexOf("\uffff"), -1); // Must not narrow to 0xff on x86. + var wide = "\u0100" + latin1; + assertEq(wide.indexOf("\xff"), length + 1); + assertEq(wide.indexOf(latin1), 1); +} diff --git a/js/src/jsapi-tests/moz.build b/js/src/jsapi-tests/moz.build index 1b8730c49d..f5d32d6c8f 100644 --- a/js/src/jsapi-tests/moz.build +++ b/js/src/jsapi-tests/moz.build @@ -17,6 +17,7 @@ UNIFIED_SOURCES += [ 'testBug604087.cpp', 'testCallArgs.cpp', 'testCallNonGenericMethodOnProxy.cpp', + 'testCharacterOperations.cpp', 'testChromeBuffer.cpp', 'testClassGetter.cpp', 'testCloneScript.cpp', diff --git a/js/src/jsapi-tests/testCharacterOperations.cpp b/js/src/jsapi-tests/testCharacterOperations.cpp new file mode 100644 index 0000000000..eb8fdb5ba8 --- /dev/null +++ b/js/src/jsapi-tests/testCharacterOperations.cpp @@ -0,0 +1,145 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "vm/CharacterOperations.h" + +#include + +#ifndef JS_CHARACTER_OPERATIONS_STANDALONE +# include "jsapi-tests/tests.h" +# include "jswin.h" +#elif defined(_WIN32) +# include +#endif + +template +static bool +CheckCharacterSearch() +{ + CharT buffer[128]; + const uint32_t needles[] = {0, 0x7f, 0x80, 0xff, 0x100, 0xd800, 0xdc00, 0xffff}; + for (uint32_t value : needles) { + if (sizeof(CharT) == 1 && value > 0xff) + continue; + const CharT needle = CharT(value); + const CharT other = CharT(value ^ 1); + for (size_t offset = 0; offset < 16 / sizeof(CharT); ++offset) { + CharT* text = buffer + offset; + for (size_t length = 0; length <= 96; ++length) { + for (size_t i = 0; i <= length; ++i) + text[i] = other; + // A match just outside the span must never be returned. + text[length] = needle; + if (js::FindCharacter(text, length, needle)) + return false; + for (size_t position = 0; position < length; ++position) { + text[position] = needle; + text[length - 1] = needle; + if (js::FindCharacter(text, length, needle) != text + position) + return false; + text[position] = other; + text[length - 1] = other; + } + } + } + } + return true; +} + +static bool +CheckLatin1Detection() +{ + char16_t buffer[128]; + const char16_t nonLatin1[] = {0x100, 0x8000, 0xd800, 0xdc00, 0xffff}; + for (size_t offset = 0; offset < 8; ++offset) { + char16_t* text = buffer + offset; + for (size_t length = 0; length <= 96; ++length) { + for (size_t i = 0; i < length; ++i) + text[i] = (i & 1) ? 0xff : 0x80; + text[length] = 0xffff; + if (!js::CharactersFitInLatin1(text, length)) + return false; + for (char16_t invalid : nonLatin1) { + for (size_t position = 0; position < length; ++position) { + const char16_t saved = text[position]; + text[position] = invalid; + if (js::CharactersFitInLatin1(text, length)) + return false; + text[position] = saved; + } + } + } + } + return true; +} + +#ifdef _WIN32 +static bool +CheckCharacterPageBoundary() +{ + SYSTEM_INFO info; + GetSystemInfo(&info); + char* pages = static_cast(VirtualAlloc(nullptr, 2 * info.dwPageSize, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + if (!pages) + return false; + DWORD oldProtection; + char* end = pages + info.dwPageSize; + bool ok = !!VirtualProtect(end, info.dwPageSize, PAGE_NOACCESS, &oldProtection); + if (ok) { + for (size_t length = 0; length <= 64; ++length) { + char* bytes = end - length; + for (size_t i = 0; i < length; ++i) + bytes[i] = 'a'; + ok = ok && !js::FindCharacter(bytes, length, 'b'); + if (length) { + bytes[length - 1] = 'b'; + ok = ok && js::FindCharacter(bytes, length, 'b') == bytes + length - 1; + } + char16_t* wide = reinterpret_cast(end) - length; + for (size_t i = 0; i < length; ++i) + wide[i] = 0xff; + ok = ok && !js::FindCharacter(wide, length, char16_t(0x100)); + ok = ok && js::CharactersFitInLatin1(wide, length); + if (length) { + wide[length - 1] = 0x100; + ok = ok && js::FindCharacter(wide, length, char16_t(0x100)) == wide + length - 1; + ok = ok && !js::CharactersFitInLatin1(wide, length); + } + } + } + VirtualFree(pages, 0, MEM_RELEASE); + return ok; +} +#endif + +static bool +CheckCharacterOperations() +{ + return CheckCharacterSearch() && CheckCharacterSearch() && + CheckCharacterSearch() && CheckLatin1Detection() +#ifdef _WIN32 + && CheckCharacterPageBoundary() +#endif + ; +} + +// Allow testing these native helpers without rebuilding/linking SpiderMonkey. +#ifdef JS_CHARACTER_OPERATIONS_STANDALONE +int main() +{ + if (!CheckCharacterOperations()) { + fprintf(stderr, "Character operation regression test failed\n"); + return 1; + } + return 0; +} +#else +BEGIN_TEST(testCharacterOperations) +{ + CHECK(CheckCharacterOperations()); + return true; +} +END_TEST(testCharacterOperations) +#endif diff --git a/js/src/jsstr.cpp b/js/src/jsstr.cpp index 67de5d7236..d8374e8e43 100644 --- a/js/src/jsstr.cpp +++ b/js/src/jsstr.cpp @@ -38,6 +38,7 @@ #include "js/UniquePtr.h" #include "unicode/uchar.h" #include "unicode/unorm2.h" +#include "vm/CharacterOperations.h" #include "vm/GlobalObject.h" #include "vm/Interpreter.h" #include "vm/Opcodes.h" @@ -1640,7 +1641,11 @@ FirstCharMatcherUnrolled(const TextChar* text, uint32_t n, const PatChar pat) static const char* FirstCharMatcher8bit(const char* text, uint32_t n, const char pat) { -#if defined(__clang__) +#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS + if (n >= 16) + return FindCharacter(text, n, pat); +#endif +#if defined(__clang__) return FirstCharMatcherUnrolled(text, n, pat); #else return reinterpret_cast(memchr(text, pat, n)); @@ -1650,6 +1655,10 @@ FirstCharMatcher8bit(const char* text, uint32_t n, const char pat) static const char16_t* FirstCharMatcher16bit(const char16_t* text, uint32_t n, const char16_t pat) { +#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS + if (n >= 8) + return FindCharacter(text, n, pat); +#endif #if defined(XP_DARWIN) || defined(XP_WIN) /* * Performance of memchr is horrible in OSX. Windows is better, @@ -1734,18 +1743,14 @@ StringMatch(const TextChar* text, uint32_t textLen, const PatChar* pat, uint32_t return -1; #if defined(__i386__) || defined(_M_IX86) || defined(__i386) - /* - * Given enough registers, the unrolled loop below is faster than the - * following loop. 32-bit x86 does not have enough registers. - */ + // Avoid the generic substring matcher for a single character on x86. + // FindCharacter uses SSE2 where available, including mixed encodings. if (patLen == 1) { - const PatChar p0 = *pat; - const TextChar* end = text + textLen; - for (const TextChar* c = text; c != end; ++c) { - if (*c == p0) - return c - text; - } - return -1; + // A two-byte needle cannot match Latin1 text if it exceeds 0xff. + if (sizeof(TextChar) == 1 && uint32_t(*pat) > 0xff) + return -1; + const TextChar* match = FindCharacter(text, textLen, TextChar(*pat)); + return match ? int(match - text) : -1; } #endif diff --git a/js/src/vm/CharacterOperations.h b/js/src/vm/CharacterOperations.h new file mode 100644 index 0000000000..591a3216bf --- /dev/null +++ b/js/src/vm/CharacterOperations.h @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef vm_CharacterOperations_h +#define vm_CharacterOperations_h + +#include "mozilla/MathAlgorithms.h" + +#include +#include + +// Use intrinsics only when SSE2 is part of the compiler's target baseline. +// Builds for other architectures (or pre-SSE2 x86) retain scalar operations. +#if defined(__SSE2__) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +# define JS_HAS_SSE2_CHARACTER_OPERATIONS +# include +#endif + +namespace js { + +template +inline const CharT* +FindCharacter(const CharT* chars, size_t length, CharT match) +{ + static_assert(sizeof(CharT) == 1 || sizeof(CharT) == 2, "character width"); +#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS + const size_t lanes = 16 / sizeof(CharT); + if (length >= lanes) { + const __m128i needle = sizeof(CharT) == 1 + ? _mm_set1_epi8(static_cast(match)) + : _mm_set1_epi16(static_cast(match)); + do { + // Never read beyond the supplied span, even at a page boundary. + const __m128i block = _mm_loadu_si128(reinterpret_cast(chars)); + const __m128i equal = sizeof(CharT) == 1 + ? _mm_cmpeq_epi8(block, needle) + : _mm_cmpeq_epi16(block, needle); + const uint32_t mask = static_cast(_mm_movemask_epi8(equal)); + if (mask) + return chars + mozilla::CountTrailingZeroes32(mask) / sizeof(CharT); + chars += lanes; + length -= lanes; + } while (length >= lanes); + } +#endif + for (; length; --length, ++chars) { + if (*chars == match) + return chars; + } + return nullptr; +} + +inline bool +CharactersFitInLatin1(const char16_t* chars, size_t length) +{ +#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS + if (length >= 8) { + const __m128i highBytes = _mm_set1_epi16(static_cast(0xff00)); + const __m128i zero = _mm_setzero_si128(); + do { + const __m128i block = _mm_loadu_si128(reinterpret_cast(chars)); + const __m128i fits = _mm_cmpeq_epi16(_mm_and_si128(block, highBytes), zero); + if (_mm_movemask_epi8(fits) != 0xffff) + return false; + chars += 8; + length -= 8; + } while (length >= 8); + } +#endif + for (; length; --length, ++chars) { + if (*chars > 0xff) + return false; + } + return true; +} + +} // namespace js + +#endif // vm_CharacterOperations_h diff --git a/js/src/vm/String.cpp b/js/src/vm/String.cpp index 538e9c09eb..3ef6fb229d 100644 --- a/js/src/vm/String.cpp +++ b/js/src/vm/String.cpp @@ -15,6 +15,7 @@ #include "gc/Marking.h" #include "js/UbiNode.h" +#include "vm/CharacterOperations.h" #include "vm/SPSProfiler.h" #include "jscntxtinlines.h" @@ -1147,12 +1148,8 @@ js::NewDependentString(JSContext* cx, JSString* baseArg, size_t start, size_t le static bool CanStoreCharsAsLatin1(const char16_t* s, size_t length) { - for (const char16_t* end = s + length; s < end; ++s) { - if (*s > JSString::MAX_LATIN1_CHAR) - return false; - } - - return true; + static_assert(JSString::MAX_LATIN1_CHAR == 0xff, "Latin1 character range"); + return CharactersFitInLatin1(s, length); } static bool