Optimize string operations with SSE2

js/src/jit-test/tests/latin1/sse2-search.js

@@ -0,0 +1,21 @@
+// Exercise vector-sized spans, scalar tails, and mixed character encodings.
+for (var length of [0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65]) {
+    for (var needle of ["\0", "\x7f", "\x80", "\xff", "\u0100", "\ud800", "\uffff"]) {
+        for (var position = 0; position <= length; ++position) {
+            var text = "a".repeat(position) + needle + "a".repeat(length - position);
+            for (var start of [0, position, position + 1, text.length]) {
+                var expected = start <= position ? position : -1;
+                assertEq(text.indexOf(needle, start), expected);
+                assertEq(text.includes(needle, start), expected !== -1);
+            }
+            assertEq(text.indexOf(needle + "b"), -1);
+            assertEq(text.indexOf(needle + "a"), position < length ? position : -1);
+        }
+    }
+    var latin1 = "\0".repeat(length) + "\xff";
+    assertEq(latin1.indexOf("\u0100"), -1); // Must not narrow to NUL on x86.
+    assertEq(latin1.indexOf("\uffff"), -1); // Must not narrow to 0xff on x86.
+    var wide = "\u0100" + latin1;
+    assertEq(wide.indexOf("\xff"), length + 1);
+    assertEq(wide.indexOf(latin1), 1);
+}

js/src/jsapi-tests/moz.build

@@ -17,6 +17,7 @@ UNIFIED_SOURCES += [
     'testBug604087.cpp',
     'testCallArgs.cpp',
     'testCallNonGenericMethodOnProxy.cpp',
+    'testCharacterOperations.cpp',
     'testChromeBuffer.cpp',
     'testClassGetter.cpp',
     'testCloneScript.cpp',

js/src/jsapi-tests/testCharacterOperations.cpp

@@ -0,0 +1,145 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "vm/CharacterOperations.h"
+
+#include <stdio.h>
+
+#ifndef JS_CHARACTER_OPERATIONS_STANDALONE
+# include "jsapi-tests/tests.h"
+# include "jswin.h"
+#elif defined(_WIN32)
+# include <windows.h>
+#endif
+
+template <typename CharT>
+static bool
+CheckCharacterSearch()
+{
+    CharT buffer[128];
+    const uint32_t needles[] = {0, 0x7f, 0x80, 0xff, 0x100, 0xd800, 0xdc00, 0xffff};
+    for (uint32_t value : needles) {
+        if (sizeof(CharT) == 1 && value > 0xff)
+            continue;
+        const CharT needle = CharT(value);
+        const CharT other = CharT(value ^ 1);
+        for (size_t offset = 0; offset < 16 / sizeof(CharT); ++offset) {
+            CharT* text = buffer + offset;
+            for (size_t length = 0; length <= 96; ++length) {
+                for (size_t i = 0; i <= length; ++i)
+                    text[i] = other;
+                // A match just outside the span must never be returned.
+                text[length] = needle;
+                if (js::FindCharacter(text, length, needle))
+                    return false;
+                for (size_t position = 0; position < length; ++position) {
+                    text[position] = needle;
+                    text[length - 1] = needle;
+                    if (js::FindCharacter(text, length, needle) != text + position)
+                        return false;
+                    text[position] = other;
+                    text[length - 1] = other;
+                }
+            }
+        }
+    }
+    return true;
+}
+
+static bool
+CheckLatin1Detection()
+{
+    char16_t buffer[128];
+    const char16_t nonLatin1[] = {0x100, 0x8000, 0xd800, 0xdc00, 0xffff};
+    for (size_t offset = 0; offset < 8; ++offset) {
+        char16_t* text = buffer + offset;
+        for (size_t length = 0; length <= 96; ++length) {
+            for (size_t i = 0; i < length; ++i)
+                text[i] = (i & 1) ? 0xff : 0x80;
+            text[length] = 0xffff;
+            if (!js::CharactersFitInLatin1(text, length))
+                return false;
+            for (char16_t invalid : nonLatin1) {
+                for (size_t position = 0; position < length; ++position) {
+                    const char16_t saved = text[position];
+                    text[position] = invalid;
+                    if (js::CharactersFitInLatin1(text, length))
+                        return false;
+                    text[position] = saved;
+                }
+            }
+        }
+    }
+    return true;
+}
+
+#ifdef _WIN32
+static bool
+CheckCharacterPageBoundary()
+{
+    SYSTEM_INFO info;
+    GetSystemInfo(&info);
+    char* pages = static_cast<char*>(VirtualAlloc(nullptr, 2 * info.dwPageSize,
+                                                MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+    if (!pages)
+        return false;
+    DWORD oldProtection;
+    char* end = pages + info.dwPageSize;
+    bool ok = !!VirtualProtect(end, info.dwPageSize, PAGE_NOACCESS, &oldProtection);
+    if (ok) {
+        for (size_t length = 0; length <= 64; ++length) {
+            char* bytes = end - length;
+            for (size_t i = 0; i < length; ++i)
+                bytes[i] = 'a';
+            ok = ok && !js::FindCharacter(bytes, length, 'b');
+            if (length) {
+                bytes[length - 1] = 'b';
+                ok = ok && js::FindCharacter(bytes, length, 'b') == bytes + length - 1;
+            }
+            char16_t* wide = reinterpret_cast<char16_t*>(end) - length;
+            for (size_t i = 0; i < length; ++i)
+                wide[i] = 0xff;
+            ok = ok && !js::FindCharacter(wide, length, char16_t(0x100));
+            ok = ok && js::CharactersFitInLatin1(wide, length);
+            if (length) {
+                wide[length - 1] = 0x100;
+                ok = ok && js::FindCharacter(wide, length, char16_t(0x100)) == wide + length - 1;
+                ok = ok && !js::CharactersFitInLatin1(wide, length);
+            }
+        }
+    }
+    VirtualFree(pages, 0, MEM_RELEASE);
+    return ok;
+}
+#endif
+
+static bool
+CheckCharacterOperations()
+{
+    return CheckCharacterSearch<char>() && CheckCharacterSearch<unsigned char>() &&
+           CheckCharacterSearch<char16_t>() && CheckLatin1Detection()
+#ifdef _WIN32
+           && CheckCharacterPageBoundary()
+#endif
+           ;
+}
+
+// Allow testing these native helpers without rebuilding/linking SpiderMonkey.
+#ifdef JS_CHARACTER_OPERATIONS_STANDALONE
+int main()
+{
+    if (!CheckCharacterOperations()) {
+        fprintf(stderr, "Character operation regression test failed\n");
+        return 1;
+    }
+    return 0;
+}
+#else
+BEGIN_TEST(testCharacterOperations)
+{
+    CHECK(CheckCharacterOperations());
+    return true;
+}
+END_TEST(testCharacterOperations)
+#endif

js/src/jsstr.cpp

@@ -38,6 +38,7 @@
 #include "js/UniquePtr.h"
 #include "unicode/uchar.h"
 #include "unicode/unorm2.h"
+#include "vm/CharacterOperations.h"
 #include "vm/GlobalObject.h"
 #include "vm/Interpreter.h"
 #include "vm/Opcodes.h"
@@ -1640,7 +1641,11 @@ FirstCharMatcherUnrolled(const TextChar* text, uint32_t n, const PatChar pat)
 static const char*
 FirstCharMatcher8bit(const char* text, uint32_t n, const char pat)
 {
-#if  defined(__clang__)
+#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS
+    if (n >= 16)
+        return FindCharacter(text, n, pat);
+#endif
+#if defined(__clang__)
     return FirstCharMatcherUnrolled<char, char>(text, n, pat);
 #else
     return reinterpret_cast<const char*>(memchr(text, pat, n));
@@ -1650,6 +1655,10 @@ FirstCharMatcher8bit(const char* text, uint32_t n, const char pat)
 static const char16_t*
 FirstCharMatcher16bit(const char16_t* text, uint32_t n, const char16_t pat)
 {
+#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS
+    if (n >= 8)
+        return FindCharacter(text, n, pat);
+#endif
 #if defined(XP_DARWIN) || defined(XP_WIN)
     /*
      * Performance of memchr is horrible in OSX. Windows is better,
@@ -1734,18 +1743,14 @@ StringMatch(const TextChar* text, uint32_t textLen, const PatChar* pat, uint32_t
         return -1;
 
 #if defined(__i386__) || defined(_M_IX86) || defined(__i386)
-    /*
+    // Avoid the generic substring matcher for a single character on x86.
-     * Given enough registers, the unrolled loop below is faster than the
+    // FindCharacter uses SSE2 where available, including mixed encodings.
-     * following loop. 32-bit x86 does not have enough registers.
-     */
     if (patLen == 1) {
-        const PatChar p0 = *pat;
+        // A two-byte needle cannot match Latin1 text if it exceeds 0xff.
-        const TextChar* end = text + textLen;
+        if (sizeof(TextChar) == 1 && uint32_t(*pat) > 0xff)
-        for (const TextChar* c = text; c != end; ++c) {
+            return -1;
-            if (*c == p0)
+        const TextChar* match = FindCharacter(text, textLen, TextChar(*pat));
-                return c - text;
+        return match ? int(match - text) : -1;
-        }
-        return -1;
     }
 #endif
 

js/src/vm/CharacterOperations.h

@@ -0,0 +1,81 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef vm_CharacterOperations_h
+#define vm_CharacterOperations_h
+
+#include "mozilla/MathAlgorithms.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+// Use intrinsics only when SSE2 is part of the compiler's target baseline.
+// Builds for other architectures (or pre-SSE2 x86) retain scalar operations.
+#if defined(__SSE2__) || defined(_M_X64) || \
+    (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
+# define JS_HAS_SSE2_CHARACTER_OPERATIONS
+# include <emmintrin.h>
+#endif
+
+namespace js {
+
+template <typename CharT>
+inline const CharT*
+FindCharacter(const CharT* chars, size_t length, CharT match)
+{
+    static_assert(sizeof(CharT) == 1 || sizeof(CharT) == 2, "character width");
+#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS
+    const size_t lanes = 16 / sizeof(CharT);
+    if (length >= lanes) {
+        const __m128i needle = sizeof(CharT) == 1
+                              ? _mm_set1_epi8(static_cast<char>(match))
+                              : _mm_set1_epi16(static_cast<short>(match));
+        do {
+            // Never read beyond the supplied span, even at a page boundary.
+            const __m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(chars));
+            const __m128i equal = sizeof(CharT) == 1
+                                 ? _mm_cmpeq_epi8(block, needle)
+                                 : _mm_cmpeq_epi16(block, needle);
+            const uint32_t mask = static_cast<uint32_t>(_mm_movemask_epi8(equal));
+            if (mask)
+                return chars + mozilla::CountTrailingZeroes32(mask) / sizeof(CharT);
+            chars += lanes;
+            length -= lanes;
+        } while (length >= lanes);
+    }
+#endif
+    for (; length; --length, ++chars) {
+        if (*chars == match)
+            return chars;
+    }
+    return nullptr;
+}
+
+inline bool
+CharactersFitInLatin1(const char16_t* chars, size_t length)
+{
+#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS
+    if (length >= 8) {
+        const __m128i highBytes = _mm_set1_epi16(static_cast<short>(0xff00));
+        const __m128i zero = _mm_setzero_si128();
+        do {
+            const __m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(chars));
+            const __m128i fits = _mm_cmpeq_epi16(_mm_and_si128(block, highBytes), zero);
+            if (_mm_movemask_epi8(fits) != 0xffff)
+                return false;
+            chars += 8;
+            length -= 8;
+        } while (length >= 8);
+    }
+#endif
+    for (; length; --length, ++chars) {
+        if (*chars > 0xff)
+            return false;
+    }
+    return true;
+}
+
+} // namespace js
+
+#endif // vm_CharacterOperations_h

js/src/vm/String.cpp

@@ -15,6 +15,7 @@
 
 #include "gc/Marking.h"
 #include "js/UbiNode.h"
+#include "vm/CharacterOperations.h"
 #include "vm/SPSProfiler.h"
 
 #include "jscntxtinlines.h"
@@ -1147,12 +1148,8 @@ js::NewDependentString(JSContext* cx, JSString* baseArg, size_t start, size_t le
 static bool
 CanStoreCharsAsLatin1(const char16_t* s, size_t length)
 {
-    for (const char16_t* end = s + length; s < end; ++s) {
+    static_assert(JSString::MAX_LATIN1_CHAR == 0xff, "Latin1 character range");
-        if (*s > JSString::MAX_LATIN1_CHAR)
+    return CharactersFitInLatin1(s, length);
-            return false;
-    }
-
-    return true;
 }
 
 static bool