JS optimisations pt.3

build/moz.configure/windows.configure

@@ -90,8 +90,10 @@ def get_sdk_dirs(sdk, subdir):
             return int(parts[2]) if len(parts) >= 3 else 0
         except ValueError:
             return 0
+    # Keep compatibility with the current Windows 11 SDK while retaining an
+    # upper bound for SDK layouts this older tree has not been audited against.
     valid_versions = sorted((v for v in set(include_dirs) & set(lib_dirs)
-                             if build_num(v) <= 19041), reverse=True)
+                             if build_num(v) <= 26100), reverse=True)
     if valid_versions:
         return namespace(
             path=sdk,
@@ -149,7 +151,7 @@ def valid_windows_sdk_dir(compiler, windows_sdk_dir, target_version,
             return int(parts[2]) if len(parts) >= 3 else 0
         except ValueError:
             return 0
-    sdks = {k: v for k, v in sdks.items() if sdk_build(k) <= 19041}
+    sdks = {k: v for k, v in sdks.items() if sdk_build(k) <= 26100}
     valid_sdks = sorted(sdks, key=lambda x: sdks[x][0], reverse=True)
     if valid_sdks:
         biggest_version, sdk = sdks[valid_sdks[0]]
@@ -240,7 +242,8 @@ def valid_ucrt_sdk_dir(windows_sdk_dir, windows_sdk_dir_env):
                 'The SDK in WINDOWSSDKDIR (%s) does not contain the Universal '
                 'CRT.' % windows_sdk_dir_env)
 
-    sdks = {k: v for k, v in sdks.items() if int(str(v[0]).split('.')[2]) <= 19041}
+    sdks = {k: v for k, v in sdks.items()
+            if int(str(v[0]).split('.')[2]) <= 26100}
     valid_sdks = sorted(sdks, key=lambda x: sdks[x][0], reverse=True)
     if not valid_sdks:
         raise FatalCheckError('Cannot find the Universal CRT SDK. '

gfx/layers/composite/ContentHost.cpp

@@ -10,12 +10,14 @@
 #include "mozilla/Assertions.h"         // for MOZ_ASSERT, etc
 #include "mozilla/gfx/BaseRect.h"       // for BaseRect
 #include "mozilla/layers/Compositor.h"  // for Compositor
+#include "mozilla/layers/CompositorOGL.h"  // for CompositorOGL
 #include "mozilla/layers/Effects.h"     // for TexturedEffect, Effect, etc
 #include "mozilla/layers/LayersMessages.h"  // for ThebesBufferData
 #include "nsAString.h"
 #include "nsPrintfCString.h"            // for nsPrintfCString
 #include "nsString.h"                   // for nsAutoCString
 #include "mozilla/layers/TextureHostOGL.h"  // for TextureHostOGL
+#include "gfxPrefs.h"                   // for gfxPrefs::WebRenderEnabled
 
 namespace mozilla {
 using namespace gfx;
@@ -69,6 +71,15 @@ ContentHostTexture::Composite(LayerComposite* aLayer,
 
   aEffectChain.mPrimaryEffect = effect;
 
+  CompositorOGL* compositorOGL = GetCompositor()->AsCompositorOGL();
+  bool canUseWebRender =
+    gfxPrefs::WebRenderEnabled() && compositorOGL &&
+    effect->mType == EffectTypes::RGB && !mTextureSourceOnWhite &&
+    !mTextureSource->AsBigImageIterator() && aTransform.Is2D() &&
+    !aEffectChain.mSecondaryEffects[EffectTypes::MASK] &&
+    !aEffectChain.mSecondaryEffects[EffectTypes::BLEND_MODE] &&
+    aSamplingFilter == SamplingFilter::LINEAR;
+
   nsIntRegion tmpRegion;
   const nsIntRegion* renderRegion;
   if (PaintWillResample()) {
@@ -176,7 +187,13 @@ ContentHostTexture::Composite(LayerComposite* aLayer,
                                         Float(tileRegionRect.y) / texRect.height,
                                         Float(tileRegionRect.width) / texRect.width,
                                         Float(tileRegionRect.height) / texRect.height);
-          GetCompositor()->DrawQuad(rect, aClipRect, aEffectChain, aOpacity, aTransform);
+          if (!canUseWebRender ||
+              !compositorOGL->DrawWebRenderImage(
+                effect->mTexture, rect, effect->mTextureCoords, aOpacity,
+                aTransform, aClipRect, effect->mPremultiplied)) {
+            GetCompositor()->DrawQuad(rect, aClipRect, aEffectChain,
+                                      aOpacity, aTransform);
+          }
           if (usingTiles) {
             DiagnosticFlags diagnostics = DiagnosticFlags::CONTENT | DiagnosticFlags::BIGIMAGE;
             if (iterOnWhite) {

gfx/layers/composite/ImageHost.cpp

@@ -9,6 +9,7 @@
 #include "composite/CompositableHost.h"  // for CompositableHost, etc
 #include "ipc/IPCMessageUtils.h"        // for null_t
 #include "mozilla/layers/Compositor.h"  // for Compositor
+#include "mozilla/layers/CompositorOGL.h"  // for CompositorOGL
 #include "mozilla/layers/Effects.h"     // for TexturedEffect, Effect, etc
 #include "mozilla/layers/ImageContainerParent.h"
 #include "mozilla/layers/LayerManagerComposite.h"     // for TexturedEffect, Effect, etc
@@ -16,6 +17,7 @@
 #include "nsDebug.h"                    // for NS_WARNING, NS_ASSERTION
 #include "nsPrintfCString.h"            // for nsPrintfCString
 #include "nsString.h"                   // for nsAutoCString
+#include "gfxPrefs.h"                   // for gfxPrefs::WebRenderEnabled
 
 #define BIAS_TIME_MS 1.0
 
@@ -381,6 +383,16 @@ ImageHost::Composite(LayerComposite* aLayer,
       mLastProducerID = img->mProducerID;
     }
     aEffectChain.mPrimaryEffect = effect;
+    CompositorOGL* compositorOGL = GetCompositor()->AsCompositorOGL();
+    auto drawWebRenderImage = [&](const gfx::Rect& rect) {
+      return gfxPrefs::WebRenderEnabled() && compositorOGL &&
+             effect->mType == EffectTypes::RGB && aTransform.Is2D() &&
+             !aEffectChain.mSecondaryEffects[EffectTypes::MASK] &&
+             !aEffectChain.mSecondaryEffects[EffectTypes::BLEND_MODE] &&
+             compositorOGL->DrawWebRenderImage(
+               effect->mTexture, rect, effect->mTextureCoords, aOpacity,
+               aTransform, aClipRect, effect->mPremultiplied);
+    };
     gfx::Rect pictureRect(0, 0, img->mPictureRect.width, img->mPictureRect.height);
     BigImageIterator* it = mCurrentTextureSource->AsBigImageIterator();
     if (it) {
@@ -414,8 +426,10 @@ ImageHost::Composite(LayerComposite* aLayer,
           effect->mTextureCoords.y = effect->mTextureCoords.YMost();
           effect->mTextureCoords.height = -effect->mTextureCoords.height;
         }
-        GetCompositor()->DrawQuad(rect, aClipRect, aEffectChain,
-                                  aOpacity, aTransform);
+        if (!drawWebRenderImage(rect)) {
+          GetCompositor()->DrawQuad(rect, aClipRect, aEffectChain,
+                                    aOpacity, aTransform);
+        }
         GetCompositor()->DrawDiagnostics(diagnosticFlags | DiagnosticFlags::BIGIMAGE,
                                          rect, aClipRect, aTransform, mFlashCounter);
       } while (it->NextTile());
@@ -435,8 +449,10 @@ ImageHost::Composite(LayerComposite* aLayer,
         effect->mTextureCoords.height = -effect->mTextureCoords.height;
       }
 
-      GetCompositor()->DrawQuad(pictureRect, aClipRect, aEffectChain,
-                                aOpacity, aTransform);
+      if (!drawWebRenderImage(pictureRect)) {
+        GetCompositor()->DrawQuad(pictureRect, aClipRect, aEffectChain,
+                                  aOpacity, aTransform);
+      }
       GetCompositor()->DrawDiagnostics(diagnosticFlags,
                                        pictureRect, aClipRect,
                                        aTransform, mFlashCounter);

gfx/layers/composite/TiledContentHost.cpp

@@ -10,6 +10,7 @@
 #include "mozilla/gfx/Matrix.h"         // for Matrix4x4
 #include "mozilla/gfx/Point.h"          // for IntSize
 #include "mozilla/layers/Compositor.h"  // for Compositor
+#include "mozilla/layers/CompositorOGL.h"  // for CompositorOGL
 //#include "mozilla/layers/CompositorBridgeParent.h"  // for CompositorBridgeParent
 #include "mozilla/layers/Effects.h"     // for TexturedEffect, Effect, etc
 #include "mozilla/layers/LayerMetricsWrapper.h" // for LayerMetricsWrapper
@@ -486,6 +487,15 @@ TiledContentHost::RenderTile(TileHost& aTile,
   float opacity = aTile.GetFadeInOpacity(aOpacity);
   aEffectChain.mPrimaryEffect = effect;
 
+  CompositorOGL* compositorOGL = mCompositor->AsCompositorOGL();
+  bool canUseWebRender =
+    gfxPrefs::WebRenderEnabled() && compositorOGL &&
+    effect->mType == EffectTypes::RGB && !aTile.mTextureHostOnWhite &&
+    aTransform.Is2D() &&
+    !aEffectChain.mSecondaryEffects[EffectTypes::MASK] &&
+    !aEffectChain.mSecondaryEffects[EffectTypes::BLEND_MODE] &&
+    aSamplingFilter == gfx::SamplingFilter::LINEAR;
+
   for (auto iter = aScreenRegion.RectIter(); !iter.Done(); iter.Next()) {
     const IntRect& rect = iter.Get();
     Rect graphicsRect(rect.x, rect.y, rect.width, rect.height);
@@ -496,7 +506,13 @@ TiledContentHost::RenderTile(TileHost& aTile,
                                   textureRect.y / aTextureBounds.height,
                                   textureRect.width / aTextureBounds.width,
                                   textureRect.height / aTextureBounds.height);
-    mCompositor->DrawQuad(graphicsRect, aClipRect, aEffectChain, opacity, aTransform, aVisibleRect);
+    if (!canUseWebRender ||
+        !compositorOGL->DrawWebRenderImage(
+          effect->mTexture, graphicsRect, effect->mTextureCoords, opacity,
+          aTransform, aClipRect, effect->mPremultiplied)) {
+      mCompositor->DrawQuad(graphicsRect, aClipRect, aEffectChain, opacity,
+                            aTransform, aVisibleRect);
+    }
   }
   DiagnosticFlags flags = DiagnosticFlags::CONTENT | DiagnosticFlags::TILE;
   if (aTile.mTextureHostOnWhite) {
@@ -562,6 +578,10 @@ TiledContentHost::RenderLayerBuffer(TiledLayerBufferComposite& aLayerBuffer,
     return;
   }
 
+  CompositorOGL* compositorOGL = mCompositor->AsCompositorOGL();
+  bool canUseWebRenderSolid =
+    gfxPrefs::WebRenderEnabled() && compositorOGL && aTransform.Is2D();
+
   if (aBackgroundColor) {
     nsIntRegion backgroundRegion = compositeRegion;
     backgroundRegion.ScaleRoundOut(resolution, resolution);
@@ -570,7 +590,12 @@ TiledContentHost::RenderLayerBuffer(TiledLayerBufferComposite& aLayerBuffer,
     for (auto iter = backgroundRegion.RectIter(); !iter.Done(); iter.Next()) {
       const IntRect& rect = iter.Get();
       Rect graphicsRect(rect.x, rect.y, rect.width, rect.height);
-      mCompositor->DrawQuad(graphicsRect, aClipRect, effect, 1.0, aTransform);
+      if (!canUseWebRenderSolid ||
+          !compositorOGL->DrawWebRenderRect(
+            graphicsRect, *aBackgroundColor, 1.0, aTransform, aClipRect)) {
+        mCompositor->DrawQuad(graphicsRect, aClipRect, effect, 1.0,
+                              aTransform);
+      }
     }
   }
 

gfx/layers/opengl/CompositorOGL.cpp

@@ -87,6 +87,9 @@ CompositorOGL::CompositorOGL(CompositorBridgeParent* aParent,
   , mWidgetSize(-1, -1)
   , mSurfaceSize(aSurfaceWidth, aSurfaceHeight)
   , mWebRenderContext(nullptr)
+  , mWebRenderViewport(0, 0, 0, 0)
+  , mWebRenderSceneHasCommands(false)
+  , mNextWebRenderImageKey(1)
   , mHasBGRA(0)
   , mUseExternalSurfaceSize(aUseExternalSurfaceSize)
   , mFrameInProgress(false)
@@ -170,6 +173,8 @@ CompositorOGL::CleanupResources()
     wr_context_destroy(mWebRenderContext);
     mWebRenderContext = nullptr;
   }
+  mWebRenderSceneHasCommands = false;
+  mWebRenderSceneImageSources.Clear();
 
   if (!mGLContext)
     return;
@@ -651,6 +656,22 @@ CompositorOGL::BeginFrame(const nsIntRegion& aInvalidRegion,
 
   // We're about to actually draw a frame.
   mFrameInProgress = true;
+  mWebRenderViewport = rect;
+  mWebRenderSceneHasCommands = false;
+  mWebRenderSceneImageSources.Clear();
+  mNextWebRenderImageKey = 1;
+  if (gfxPrefs::WebRenderEnabled()) {
+    if (!mWebRenderContext) {
+      mWebRenderContext = wr_context_create(16);
+    }
+    if (mWebRenderContext) {
+      wr_context_clear(mWebRenderContext);
+      wr_context_clear_images(mWebRenderContext);
+      wr_context_set_viewport(mWebRenderContext,
+                              wr_rect{ Float(rect.x), Float(rect.y),
+                                       Float(rect.width), Float(rect.height) });
+    }
+  }
 
   // If the widget size changed, we have to force a MakeCurrent
   // to make sure that GL sees the updated widget size.
@@ -981,6 +1002,34 @@ CompositorOGL::DrawQuad(const Rect& aRect,
   PROFILER_LABEL("CompositorOGL", "DrawQuad",
     js::ProfileEntry::Category::GRAPHICS);
 
+  // Give the C WebRender scene first refusal for the primitive types it can
+  // represent.  Layer hosts use the same entry points directly, but this
+  // covers compositor callers that submit through the generic DrawQuad path.
+  if (gfxPrefs::WebRenderEnabled() && aEffectChain.mPrimaryEffect &&
+      aTransform.Is2D() &&
+      !aEffectChain.mSecondaryEffects[EffectTypes::MASK] &&
+      !aEffectChain.mSecondaryEffects[EffectTypes::BLEND_MODE] &&
+      !aEffectChain.mSecondaryEffects[EffectTypes::COLOR_MATRIX]) {
+    if (aEffectChain.mPrimaryEffect->mType == EffectTypes::SOLID_COLOR) {
+      EffectSolidColor* effect =
+        static_cast<EffectSolidColor*>(aEffectChain.mPrimaryEffect.get());
+      if (DrawWebRenderRect(aRect, effect->mColor, aOpacity, aTransform,
+                            aClipRect)) {
+        return;
+      }
+    } else if (aEffectChain.mPrimaryEffect->mType == EffectTypes::RGB ||
+               aEffectChain.mPrimaryEffect->mType == EffectTypes::RENDER_TARGET) {
+      TexturedEffect* effect =
+        static_cast<TexturedEffect*>(aEffectChain.mPrimaryEffect.get());
+      if (effect->mTexture &&
+          DrawWebRenderImage(effect->mTexture, aRect, effect->mTextureCoords,
+                             aOpacity, aTransform, aClipRect,
+                             effect->mPremultiplied)) {
+        return;
+      }
+    }
+  }
+
   DrawGeometry(aRect, aClipRect, aEffectChain,
                aOpacity, aTransform, aVisibleRect);
 }
@@ -1002,7 +1051,10 @@ CompositorOGL::DrawTriangle(const gfx::TexturedTriangle& aTriangle,
 
 bool
 CompositorOGL::DrawWebRenderContext(wr_context* aContext,
-                                    const IntRect& aClipRect)
+                                    const IntRect& aClipRect,
+                                    TextureSource* aImageSource,
+                                    uint32_t aImageKey,
+                                    bool aImagePremultiplied)
 {
   if (!aContext) {
     return false;
@@ -1010,7 +1062,13 @@ CompositorOGL::DrawWebRenderContext(wr_context* aContext,
 
   const wr_frame* frame = wr_context_build_frame(aContext);
   if (frame) {
-    DrawWebRenderFrame(*frame, aClipRect);
+    WebRenderImageSource imageSource;
+    imageSource.mSource = aImageSource;
+    imageSource.mKey = aImageKey;
+    imageSource.mPremultiplied = aImagePremultiplied;
+    DrawWebRenderFrame(*frame, aClipRect,
+                       aImageSource && aImageKey ? &imageSource : nullptr,
+                       aImageSource && aImageKey ? 1 : 0);
     return true;
   }
   return false;
@@ -1023,42 +1081,169 @@ CompositorOGL::DrawWebRenderRect(const Rect& aRect,
                                  const Matrix4x4& aTransform,
                                  const IntRect& aClipRect)
 {
-  if (!mFrameInProgress || aClipRect.IsEmpty() || !aTransform.Is2D()) {
+  if (!gfxPrefs::WebRenderEnabled() || !mFrameInProgress ||
+      aClipRect.IsEmpty() || !aTransform.Is2D()) {
     return false;
   }
   if (!mWebRenderContext) {
-    mWebRenderContext = wr_context_create(8);
+    mWebRenderContext = wr_context_create(16);
   }
   if (!mWebRenderContext) {
     return false;
   }
 
-  wr_context_clear(mWebRenderContext);
   wr_context_set_viewport(mWebRenderContext,
-                          wr_rect{ Float(aClipRect.x), Float(aClipRect.y),
-                                   Float(aClipRect.width), Float(aClipRect.height) });
+                          wr_rect{ Float(mWebRenderViewport.x),
+                                   Float(mWebRenderViewport.y),
+                                   Float(mWebRenderViewport.width),
+                                   Float(mWebRenderViewport.height) });
   Matrix transform = aTransform.As2D();
   wr_context_set_transform(mWebRenderContext,
                            wr_transform{ transform._11, transform._12,
                                          transform._21, transform._22,
                                          transform._31, transform._32 });
   wr_context_set_opacity(mWebRenderContext, aOpacity);
+  if (!wr_context_begin_transaction(mWebRenderContext)) {
+    return false;
+  }
+  if (!wr_display_list_push_clip(
+        mWebRenderContext,
+        wr_rect{ Float(aClipRect.x), Float(aClipRect.y),
+                 Float(aClipRect.width), Float(aClipRect.height) })) {
+    wr_context_abort_transaction(mWebRenderContext);
+    return false;
+  }
   if (!wr_display_list_push_rect(mWebRenderContext,
                                  wr_rect{ aRect.x, aRect.y,
                                           aRect.width, aRect.height },
                                  wr_color{ aColor.r, aColor.g,
                                            aColor.b, aColor.a })) {
+    wr_context_abort_transaction(mWebRenderContext);
+    return false;
+  }
+  wr_display_list_pop_clip(mWebRenderContext);
+  if (!wr_context_commit_transaction(mWebRenderContext)) {
+    wr_context_abort_transaction(mWebRenderContext);
     return false;
   }
 
-  bool drawn = DrawWebRenderContext(mWebRenderContext, aClipRect);
-  wr_context_clear(mWebRenderContext);
-  return drawn;
+  mWebRenderSceneHasCommands = true;
+  return true;
+}
+
+uint32_t
+CompositorOGL::RegisterWebRenderImage(TextureSource* aImageSource,
+                                      bool aPremultiplied)
+{
+  IntSize imageSize;
+  uint32_t key;
+
+  for (const WebRenderImageSource& image : mWebRenderSceneImageSources) {
+    if (image.mSource == aImageSource &&
+        image.mPremultiplied == aPremultiplied) {
+      return image.mKey;
+    }
+  }
+
+  if (!aImageSource) {
+    return 0;
+  }
+  imageSize = aImageSource->GetSize();
+  if (imageSize.width <= 0 || imageSize.height <= 0) {
+    return 0;
+  }
+
+  key = mNextWebRenderImageKey++;
+  if (!key) {
+    key = mNextWebRenderImageKey++;
+  }
+  if (!wr_context_register_image(mWebRenderContext, key,
+                                 uint32_t(imageSize.width),
+                                 uint32_t(imageSize.height))) {
+    return 0;
+  }
+
+  WebRenderImageSource* image =
+    mWebRenderSceneImageSources.AppendElement();
+  image->mSource = aImageSource;
+  image->mKey = key;
+  image->mPremultiplied = aPremultiplied;
+  return key;
+}
+
+bool
+CompositorOGL::DrawWebRenderImage(TextureSource* aImageSource,
+                                  const Rect& aRect,
+                                  const Rect& aTexRect,
+                                  Float aOpacity,
+                                  const Matrix4x4& aTransform,
+                                  const IntRect& aClipRect,
+                                  bool aPremultiplied)
+{
+  if (!gfxPrefs::WebRenderEnabled() || !aImageSource ||
+      !mFrameInProgress || aClipRect.IsEmpty() ||
+      !aTransform.Is2D()) {
+    return false;
+  }
+  if (aImageSource->AsSourceOGL() == nullptr) {
+    return false;
+  }
+  if (!mWebRenderContext) {
+    mWebRenderContext = wr_context_create(16);
+  }
+  if (!mWebRenderContext) {
+    return false;
+  }
+
+  wr_context_set_viewport(mWebRenderContext,
+                          wr_rect{ Float(mWebRenderViewport.x),
+                                   Float(mWebRenderViewport.y),
+                                   Float(mWebRenderViewport.width),
+                                   Float(mWebRenderViewport.height) });
+  Matrix transform = aTransform.As2D();
+  wr_context_set_transform(mWebRenderContext,
+                           wr_transform{ transform._11, transform._12,
+                                         transform._21, transform._22,
+                                         transform._31, transform._32 });
+  wr_context_set_opacity(mWebRenderContext, aOpacity);
+  uint32_t imageKey = RegisterWebRenderImage(aImageSource, aPremultiplied);
+  if (!imageKey) {
+    return false;
+  }
+  if (!wr_context_begin_transaction(mWebRenderContext)) {
+    return false;
+  }
+  if (!wr_display_list_push_clip(
+        mWebRenderContext,
+        wr_rect{ Float(aClipRect.x), Float(aClipRect.y),
+                 Float(aClipRect.width), Float(aClipRect.height) })) {
+    wr_context_abort_transaction(mWebRenderContext);
+    return false;
+  }
+  if (!wr_display_list_push_image(
+        mWebRenderContext,
+        wr_rect{ aRect.x, aRect.y, aRect.width, aRect.height },
+        imageKey, wr_rect{ aTexRect.x, aTexRect.y,
+                    aTexRect.width, aTexRect.height },
+        wr_color{ 1, 1, 1, 1 })) {
+    wr_context_abort_transaction(mWebRenderContext);
+    return false;
+  }
+  wr_display_list_pop_clip(mWebRenderContext);
+  if (!wr_context_commit_transaction(mWebRenderContext)) {
+    wr_context_abort_transaction(mWebRenderContext);
+    return false;
+  }
+
+  mWebRenderSceneHasCommands = true;
+  return true;
 }
 
 void
 CompositorOGL::DrawWebRenderFrame(const wr_frame& aFrame,
-                                  const IntRect& aClipRect)
+                                  const IntRect& aClipRect,
+                                  const WebRenderImageSource* aImageSources,
+                                  size_t aImageSourceCount)
 {
   MOZ_ASSERT(mFrameInProgress, "frame not started");
   MOZ_ASSERT(mCurrentRenderTarget, "No destination");
@@ -1070,21 +1255,10 @@ CompositorOGL::DrawWebRenderFrame(const wr_frame& aFrame,
 
   MakeCurrent();
 
-  // This first adapter handles solid rectangles. It deliberately reuses the
-  // compositor's existing four-rect uniform and triangle VBO path, so the
-  // retained C display list is submitted to the GPU without a CPU raster pass.
-  EffectSolidColor effect(Color(0, 0, 0, 0));
-  ShaderConfigOGL config = GetShaderConfigFor(&effect);
-  ShaderProgramOGL* program = GetShaderProgramFor(config);
-  if (!program) {
-    return;
-  }
-
   IntPoint offset = mCurrentRenderTarget->GetOrigin();
-  ActivateProgram(program);
-  program->SetProjectionMatrix(mProjMatrix);
-  program->SetLayerTransform(Matrix4x4());
-  program->SetRenderOffset(offset.x, offset.y);
+  EffectSolidColor solidEffect(Color(0, 0, 0, 0));
+  ShaderConfigOGL solidConfig = GetShaderConfigFor(&solidEffect);
+  ShaderProgramOGL* solidProgram = GetShaderProgramFor(solidConfig);
 
   ScopedGLState scopedScissorTestState(mGLContext, LOCAL_GL_SCISSOR_TEST, true);
   ScopedScissorRect autoScissorRect(mGLContext,
@@ -1115,15 +1289,71 @@ CompositorOGL::DrawWebRenderFrame(const wr_frame& aFrame,
       mGLContext, batchClip.x, FlipY(batchClip.y + batchClip.height),
       batchClip.width, batchClip.height);
 
+    ShaderProgramOGL* program = solidProgram;
+    TextureSourceOGL* imageSource = nullptr;
+    bool imagePremultiplied = true;
+    gfx::Rect textureRects[4] = {};
+    if (batch.kind == WR_PRIMITIVE_IMAGE) {
+      TextureSource* source = nullptr;
+      for (size_t sourceIndex = 0; sourceIndex < aImageSourceCount;
+           ++sourceIndex) {
+        if (aImageSources[sourceIndex].mKey == batch.image_key) {
+          source = aImageSources[sourceIndex].mSource;
+          imagePremultiplied = aImageSources[sourceIndex].mPremultiplied;
+          break;
+        }
+      }
+      if (!source) {
+        continue;
+      }
+      imageSource = source->AsSourceOGL();
+      if (!imageSource) {
+        continue;
+      }
+      EffectRGB imageEffect(source, imagePremultiplied,
+                            SamplingFilter::LINEAR);
+      ShaderConfigOGL imageConfig = GetShaderConfigFor(&imageEffect);
+      imageConfig.SetTextureTint(true);
+      imageConfig.SetOpacity(batch.color.a != 1.0f);
+      program = GetShaderProgramFor(imageConfig);
+      if (!program) {
+        continue;
+      }
+      ActivateProgram(program);
+      imageSource->BindTexture(LOCAL_GL_TEXTURE0, SamplingFilter::LINEAR);
+      program->SetTextureUnit(0);
+      program->SetTextureTransform(imageSource->GetTextureTransform());
+      if (imageConfig.mFeatures & ENABLE_TEXTURE_RECT) {
+        program->SetTexCoordMultiplier(imageSource->GetSize().width,
+                                       imageSource->GetSize().height);
+      }
+      if (batch.color.a != 1.0f) {
+        program->SetLayerOpacity(batch.color.a);
+      }
+      program->SetTextureTint(Color(batch.color.r, batch.color.g,
+                                    batch.color.b, 1.0f));
+    } else {
+      if (!program) {
+        continue;
+      }
+      ActivateProgram(program);
+      SetBlendMode(gl(), CompositionOp::OP_OVER);
+      program->SetRenderColor(Color(batch.color.r, batch.color.g,
+                                    batch.color.b, batch.color.a));
+    }
+
+    program->SetProjectionMatrix(mProjMatrix);
     program->SetLayerTransform(Matrix4x4::From2D(
       Matrix(batch.transform.m11, batch.transform.m12,
              batch.transform.m21, batch.transform.m22,
              batch.transform.m31, batch.transform.m32)));
+    program->SetRenderOffset(offset.x, offset.y);
+    if (imageSource) {
+      SetBlendMode(gl(), CompositionOp::OP_OVER, imagePremultiplied);
+    }
 
     uint32_t remaining = batch.quad_count;
     uint32_t quadIndex = batch.first_quad;
-    program->SetRenderColor(Color(batch.color.r, batch.color.g,
-                                  batch.color.b, batch.color.a));
 
     while (remaining) {
       Rect rects[4] = {};
@@ -1132,14 +1362,38 @@ CompositorOGL::DrawWebRenderFrame(const wr_frame& aFrame,
         const wr_gpu_quad& quad = aFrame.quads[quadIndex + i];
         rects[i] = Rect(quad.rect.x, quad.rect.y,
                         quad.rect.width, quad.rect.height);
+        if (imageSource) {
+          textureRects[i] = Rect(quad.tex_rect.x, quad.tex_rect.y,
+                                 quad.tex_rect.width, quad.tex_rect.height);
+        }
       }
-      BindAndDrawQuads(program, count, rects, nullptr);
+      BindAndDrawQuads(program, count, rects,
+                       imageSource ? textureRects : nullptr);
       quadIndex += count;
       remaining -= count;
     }
   }
 }
 
+void
+CompositorOGL::FlushWebRenderScene()
+{
+  if (!mWebRenderSceneHasCommands || !mWebRenderContext ||
+      !mFrameInProgress) {
+    return;
+  }
+
+  const wr_frame* frame = wr_context_build_frame(mWebRenderContext);
+  if (frame) {
+    DrawWebRenderFrame(*frame, mWebRenderViewport,
+                       mWebRenderSceneImageSources.Elements(),
+                       mWebRenderSceneImageSources.Length());
+  }
+  wr_context_clear(mWebRenderContext);
+  mWebRenderSceneHasCommands = false;
+  mWebRenderSceneImageSources.Clear();
+}
+
 template<typename Geometry>
 void
 CompositorOGL::DrawGeometry(const Geometry& aGeometry,
@@ -1152,6 +1406,8 @@ CompositorOGL::DrawGeometry(const Geometry& aGeometry,
   MOZ_ASSERT(mFrameInProgress, "frame not started");
   MOZ_ASSERT(mCurrentRenderTarget, "No destination");
 
+  FlushWebRenderScene();
+
   MakeCurrent();
 
   IntPoint offset = mCurrentRenderTarget->GetOrigin();
@@ -1720,6 +1976,8 @@ CompositorOGL::EndFrame()
 
   MOZ_ASSERT(mCurrentRenderTarget == mWindowRenderTarget, "Rendering target not properly restored");
 
+  FlushWebRenderScene();
+
 #ifdef MOZ_DUMP_PAINTING
   if (gfxEnv::DumpCompositorTextures()) {
     LayoutDeviceIntSize size;
@@ -1773,6 +2031,7 @@ void
 CompositorOGL::EndFrameForExternalComposition(const gfx::Matrix& aTransform)
 {
   MOZ_ASSERT(!mTarget);
+  FlushWebRenderScene();
   if (mTexturePool) {
     mTexturePool->EndFrame();
   }

gfx/layers/opengl/CompositorOGL.h

@@ -34,6 +34,8 @@
 #include "nsXULAppAPI.h"                // for XRE_GetProcessType
 #include "nscore.h"                     // for NS_IMETHOD
 
+#include <stddef.h>
+
 class nsIWidget;
 
 namespace mozilla {
@@ -169,6 +171,12 @@ protected:
 public:
   virtual CompositorOGL* AsCompositorOGL() override { return this; }
 
+  struct WebRenderImageSource {
+    RefPtr<TextureSource> mSource;
+    uint32_t mKey;
+    bool mPremultiplied;
+  };
+
   virtual already_AddRefed<DataTextureSource>
   CreateDataTextureSource(TextureFlags aFlags = TextureFlags::NO_FLAGS) override;
 
@@ -214,14 +222,29 @@ public:
 
   // Consume solid retained-display-list quads through the existing GPU path.
   bool DrawWebRenderContext(wr_context* aContext,
-                            const gfx::IntRect& aClipRect);
+                            const gfx::IntRect& aClipRect,
+                            TextureSource* aImageSource = nullptr,
+                            uint32_t aImageKey = 0,
+                            bool aImagePremultiplied = true);
   bool DrawWebRenderRect(const gfx::Rect& aRect,
                          const gfx::Color& aColor,
                          gfx::Float aOpacity,
                          const gfx::Matrix4x4& aTransform,
                          const gfx::IntRect& aClipRect);
+  bool DrawWebRenderImage(TextureSource* aImageSource,
+                          const gfx::Rect& aRect,
+                          const gfx::Rect& aTexRect,
+                          gfx::Float aOpacity,
+                          const gfx::Matrix4x4& aTransform,
+                          const gfx::IntRect& aClipRect,
+                          bool aPremultiplied);
   void DrawWebRenderFrame(const wr_frame& aFrame,
-                          const gfx::IntRect& aClipRect);
+                          const gfx::IntRect& aClipRect,
+                          const WebRenderImageSource* aImageSources,
+                          size_t aImageSourceCount);
+  void FlushWebRenderScene();
+  uint32_t RegisterWebRenderImage(TextureSource* aImageSource,
+                                  bool aPremultiplied);
 
   virtual void EndFrame() override;
   virtual void EndFrameForExternalComposition(const gfx::Matrix& aTransform) override;
@@ -340,6 +363,10 @@ private:
   /** Currently bound render target */
   RefPtr<CompositingRenderTargetOGL> mCurrentRenderTarget;
   wr_context* mWebRenderContext;
+  gfx::IntRect mWebRenderViewport;
+  bool mWebRenderSceneHasCommands;
+  nsTArray<WebRenderImageSource> mWebRenderSceneImageSources;
+  uint32_t mNextWebRenderImageKey;
 #ifdef DEBUG
   CompositingRenderTargetOGL* mWindowRenderTarget;
 #endif

gfx/layers/opengl/OGLShaderProgram.cpp

@@ -60,6 +60,7 @@ AddUniforms(ProgramProfileOGL& aProfile)
         "uViewportSize",
         "uVisibleCenter",
         "uYuvColorMatrix",
+        "uTextureTint",
         nullptr
     };
 
@@ -75,6 +76,12 @@ ShaderConfigOGL::SetRenderColor(bool aEnabled)
   SetFeature(ENABLE_RENDER_COLOR, aEnabled);
 }
 
+void
+ShaderConfigOGL::SetTextureTint(bool aEnabled)
+{
+  SetFeature(ENABLE_TEXTURE_TINT, aEnabled);
+}
+
 void
 ShaderConfigOGL::SetTextureTarget(GLenum aTarget)
 {
@@ -351,6 +358,9 @@ ProgramProfileOGL::GetProfileFor(ShaderConfigOGL aConfig)
     if (aConfig.mFeatures & ENABLE_OPACITY) {
       fs << "uniform COLOR_PRECISION float uLayerOpacity;" << endl;
     }
+    if (aConfig.mFeatures & ENABLE_TEXTURE_TINT) {
+      fs << "uniform COLOR_PRECISION vec4 uTextureTint;" << endl;
+    }
   }
   if (BlendOpIsMixBlendMode(blendOp)) {
     fs << "varying vec2 vBackdropCoord;" << endl;
@@ -506,6 +516,9 @@ ProgramProfileOGL::GetProfileFor(ShaderConfigOGL aConfig)
     if (aConfig.mFeatures & ENABLE_OPACITY) {
       fs << "  color *= uLayerOpacity;" << endl;
     }
+    if (aConfig.mFeatures & ENABLE_TEXTURE_TINT) {
+      fs << "  color *= uTextureTint;" << endl;
+    }
   }
   if (aConfig.mFeatures & ENABLE_DEAA) {
     // Calculate the sub-pixel coverage of the pixel and modulate its opacity

gfx/layers/opengl/OGLShaderProgram.h

@@ -42,7 +42,8 @@ enum ShaderFeatures {
   ENABLE_MASK=0x800,
   ENABLE_NO_PREMUL_ALPHA=0x1000,
   ENABLE_DEAA=0x2000,
-  ENABLE_DYNAMIC_GEOMETRY=0x4000
+  ENABLE_DYNAMIC_GEOMETRY=0x4000,
+  ENABLE_TEXTURE_TINT=0x8000
 };
 
 class KnownUniform {
@@ -83,6 +84,7 @@ public:
     ViewportSize,
     VisibleCenter,
     YuvColorMatrix,
+    TextureTint,
 
     KnownUniformCount
   };
@@ -218,6 +220,7 @@ public:
   {}
 
   void SetRenderColor(bool aEnabled);
+  void SetTextureTint(bool aEnabled);
   void SetTextureTarget(GLenum aTarget);
   void SetRBSwap(bool aEnabled);
   void SetNoAlpha(bool aEnabled);
@@ -463,6 +466,10 @@ public:
     SetUniform(KnownUniform::RenderColor, aColor);
   }
 
+  void SetTextureTint(const gfx::Color& aColor) {
+    SetUniform(KnownUniform::TextureTint, aColor);
+  }
+
   void SetColorMatrix(const gfx::Matrix5x4& aColorMatrix)
   {
     SetMatrixUniform(KnownUniform::ColorMatrix, &aColorMatrix._11);

gfx/wr/moz.build

@@ -7,7 +7,14 @@ EXPORTS.mozilla.gfx += [
 ]
 
 SOURCES += [
-    'webrender.c',
+    'wr_context.c',
+    'wr_display_list.c',
+    'wr_frame.c',
+    'wr_frame_packet.c',
+    'wr_retained.c',
+    'wr_util.c',
 ]
 
 FINAL_LIBRARY = 'xul'
+
+TEST_DIRS += ['tests']

gfx/wr/tests/moz.build

@@ -0,0 +1,5 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+
+# This is a standalone C regression executable. It includes the C renderer
+# implementation directly so it does not require another language's harness.
+SimplePrograms(['webrender_test'], ext='.c')

gfx/wr/tests/webrender_test.c

@@ -0,0 +1,274 @@
+#include "../wr_util.c"
+#include "../wr_context.c"
+#include "../wr_display_list.c"
+#include "../wr_frame.c"
+#include "../wr_frame_packet.c"
+#include "../wr_retained.c"
+
+#include <assert.h>
+
+static void
+test_retained_frame(void)
+{
+  wr_context* context = wr_context_create(4);
+  const wr_frame* first;
+  const wr_frame* cached;
+  const wr_frame* culled;
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  assert(wr_display_list_push_rect(context, (wr_rect){ 0, 0, 10, 10 },
+                                   (wr_color){ 1, 0, 0, 1 }));
+  first = wr_context_build_frame(context);
+  cached = wr_context_build_frame(context);
+  assert(first == cached);
+  assert(cached->quad_count == 1 && cached->batch_count == 1);
+
+  wr_context_set_viewport(context, (wr_rect){ 200, 200, 10, 10 });
+  culled = wr_context_build_frame(context);
+  assert(culled && culled->quad_count == 0);
+  wr_context_destroy(context);
+}
+
+static void
+test_stacking_and_image(void)
+{
+  wr_context* context = wr_context_create(4);
+  const wr_frame* frame;
+  const wr_transform translate = { 1, 0, 0, 1, 20, 0 };
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  assert(wr_context_register_image(context, 7, 64, 64));
+  assert(wr_display_list_push_stacking_context(context, translate, 0.5f));
+  assert(wr_display_list_push_clip(context, (wr_rect){ 15, 0, 30, 30 }));
+  assert(wr_display_list_push_rect(context, (wr_rect){ 0, 0, 10, 10 },
+                                   (wr_color){ 1, 0, 0, 1 }));
+  wr_display_list_pop_stacking_context(context);
+  assert(wr_display_list_push_image(
+    context, (wr_rect){ 30, 0, 20, 20 }, 7,
+    (wr_rect){ 0, 0, 0.5f, 0.5f }, (wr_color){ 0.25f, 0.5f, 0.75f, 1 }));
+
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 2 && frame->batch_count == 2);
+  assert(frame->quads[0].color.a == 0.5f);
+  assert(frame->batches[0].has_clip);
+  assert(frame->batches[0].transform.m31 == 20);
+  assert(frame->batches[1].kind == WR_PRIMITIVE_IMAGE);
+  assert(frame->batches[1].image_key == 7);
+  assert(frame->quads[1].color.r == 0.25f);
+  assert(frame->quads[1].color.g == 0.5f);
+  assert(frame->quads[1].color.b == 0.75f);
+  assert(frame->quads[1].tex_rect.width == 0.5f);
+  wr_context_unregister_image(context, 7);
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 1 && frame->batch_count == 1);
+  wr_context_clear(context);
+  assert(!wr_display_list_push_image(
+    context, (wr_rect){ 30, 0, 20, 20 }, 7,
+    (wr_rect){ 0, 0, 0.5f, 0.5f }, (wr_color){ 1, 1, 1, 1 }));
+  wr_context_destroy(context);
+}
+
+static void
+test_scene_transactions(void)
+{
+  wr_context* context = wr_context_create(2);
+  const wr_frame* frame;
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  assert(wr_display_list_push_rect(context, (wr_rect){ 0, 0, 10, 10 },
+                                   (wr_color){ 1, 0, 0, 1 }));
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 1);
+
+  assert(wr_context_begin_transaction(context));
+  assert(wr_display_list_push_rect(context, (wr_rect){ 20, 0, 10, 10 },
+                                   (wr_color){ 0, 1, 0, 1 }));
+  wr_context_abort_transaction(context);
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 1);
+
+  assert(wr_context_begin_transaction(context));
+  assert(wr_display_list_push_rect(context, (wr_rect){ 20, 0, 10, 10 },
+                                   (wr_color){ 0, 1, 0, 1 }));
+  assert(wr_context_begin_transaction(context));
+  assert(wr_display_list_push_rect(context, (wr_rect){ 40, 0, 10, 10 },
+                                   (wr_color){ 0, 0, 1, 1 }));
+  wr_context_abort_transaction(context);
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 2);
+  assert(wr_context_commit_transaction(context));
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 2);
+
+  assert(!wr_context_commit_transaction(context));
+  wr_context_destroy(context);
+}
+
+static void
+test_multiple_image_resources(void)
+{
+  wr_context* context = wr_context_create(4);
+  const wr_frame* frame;
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  assert(wr_context_register_image(context, 11, 32, 32));
+  assert(wr_context_register_image(context, 12, 64, 64));
+  assert(wr_display_list_push_image(
+    context, (wr_rect){ 0, 0, 20, 20 }, 11,
+    (wr_rect){ 0, 0, 1, 1 }, (wr_color){ 1, 1, 1, 1 }));
+  assert(wr_display_list_push_image(
+    context, (wr_rect){ 20, 0, 20, 20 }, 12,
+    (wr_rect){ 0, 0, 1, 1 }, (wr_color){ 1, 1, 1, 1 }));
+
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 2 && frame->batch_count == 2);
+  assert(frame->batches[0].image_key == 11);
+  assert(frame->batches[1].image_key == 12);
+  wr_context_clear_images(context);
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 0 && frame->batch_count == 0);
+  wr_context_destroy(context);
+}
+
+static void
+test_stable_item_updates(void)
+{
+  wr_context* context = wr_context_create(4);
+  const wr_frame* frame;
+  const wr_transform identity = { 1, 0, 0, 1, 0, 0 };
+  wr_item_id item;
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  item = wr_display_list_push_rect_with_id(
+    context, 42, (wr_rect){ 0, 0, 10, 10 }, (wr_color){ 1, 0, 0, 1 });
+  assert(item == 42);
+  assert(wr_display_list_push_rect_with_id(
+    context, 42, (wr_rect){ 20, 0, 10, 10 }, (wr_color){ 0, 1, 0, 1 }) == 0);
+
+  assert(wr_context_begin_transaction(context));
+  assert(wr_display_list_update_rect(
+    context, item, (wr_rect){ 20, 0, 10, 10 },
+    (wr_color){ 0, 1, 0, 1 }, identity));
+  wr_context_abort_transaction(context);
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 1);
+  assert(frame->quads[0].rect.x == 0);
+
+  assert(wr_context_begin_transaction(context));
+  assert(wr_display_list_update_rect(
+    context, item, (wr_rect){ 20, 0, 10, 10 },
+    (wr_color){ 0, 1, 0, 1 }, identity));
+  assert(wr_context_commit_transaction(context));
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 1);
+  assert(frame->quads[0].rect.x == 20);
+  assert(frame->quads[0].item_id == 42);
+
+  assert(wr_context_remove_item(context, item));
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 0);
+  wr_context_destroy(context);
+}
+
+static void
+test_opaque_occlusion(void)
+{
+  wr_context* context = wr_context_create(2);
+  const wr_frame* frame;
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  assert(wr_display_list_push_rect(
+    context, (wr_rect){ 0, 0, 20, 20 }, (wr_color){ 1, 0, 0, 1 }));
+  assert(wr_display_list_push_rect(
+    context, (wr_rect){ 0, 0, 20, 20 }, (wr_color){ 0, 0, 1, 1 }));
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 1 && frame->batch_count == 1);
+  assert(frame->quads[0].color.b == 1.0f);
+  wr_context_destroy(context);
+}
+
+static void
+test_gpu_tessellated_primitives(void)
+{
+  wr_context* context = wr_context_create(64);
+  const wr_frame* frame;
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  assert(wr_display_list_push_linear_gradient(
+    context, (wr_rect){ 0, 0, 40, 20 },
+    (wr_color){ 1, 0, 0, 1 }, (wr_color){ 0, 0, 1, 1 }, 1));
+  assert(wr_display_list_push_rounded_rect(
+    context, (wr_rect){ 0, 20, 40, 20 }, (wr_color){ 0, 1, 0, 1 }, 8));
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 48);
+  wr_context_destroy(context);
+}
+
+static void
+test_glyph_run(void)
+{
+  wr_context* context = wr_context_create(4);
+  const wr_frame* frame;
+  const wr_glyph glyphs[2] = {
+    { { 0, 0, 8, 12 }, { 0, 0, 0.25f, 1 } },
+    { { 8, 0, 8, 12 }, { 0.25f, 0, 0.25f, 1 } }
+  };
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  assert(wr_context_register_image(context, 21, 256, 256));
+  assert(wr_display_list_push_glyph_run(
+    context, 21, glyphs, 2, (wr_color){ 1, 1, 1, 1 }));
+  frame = wr_context_build_frame(context);
+  assert(frame && frame->quad_count == 2 && frame->batch_count == 1);
+  assert(frame->batches[0].kind == WR_PRIMITIVE_IMAGE);
+  wr_context_destroy(context);
+}
+
+static void
+test_frame_packet(void)
+{
+  wr_context* context = wr_context_create(2);
+  const wr_frame* frame;
+  size_t packet_size;
+  unsigned char* packet;
+  wr_frame_packet_header header;
+
+  assert(context);
+  wr_context_set_viewport(context, (wr_rect){ 0, 0, 100, 100 });
+  assert(wr_display_list_push_rect(
+    context, (wr_rect){ 0, 0, 10, 10 }, (wr_color){ 1, 0, 0, 1 }));
+  frame = wr_context_build_frame(context);
+  packet_size = wr_frame_packet_size(frame);
+  assert(packet_size > sizeof(header));
+  packet = (unsigned char*)malloc(packet_size);
+  assert(packet);
+  assert(wr_frame_serialize(frame, packet, packet_size));
+  memcpy(&header, packet, sizeof(header));
+  assert(header.version == WR_FRAME_PACKET_VERSION);
+  assert(header.quad_count == 1 && header.batch_count == 1);
+  free(packet);
+  wr_context_destroy(context);
+}
+
+int
+main(void)
+{
+  test_retained_frame();
+  test_stacking_and_image();
+  test_scene_transactions();
+  test_multiple_image_resources();
+  test_stable_item_updates();
+  test_opaque_occlusion();
+  test_gpu_tessellated_primitives();
+  test_glyph_run();
+  test_frame_packet();
+  return 0;
+}

gfx/wr/webrender.c

@@ -1,447 +0,0 @@
-#include "webrender.h"
-
-#include <float.h>
-#include <math.h>
-#include <stdlib.h>
-
-typedef struct {
-  wr_rect rect;
-  wr_color color;
-  wr_transform transform;
-  wr_rect clip_rect;
-  uint8_t has_clip;
-} wr_rect_command;
-
-typedef struct {
-  wr_transform transform;
-  float opacity;
-  size_t clip_depth;
-} wr_display_state;
-
-struct wr_context {
-  wr_rect_command* commands;
-  size_t command_count;
-  size_t command_capacity;
-
-  wr_gpu_quad* quads;
-  size_t quad_count;
-  size_t quad_capacity;
-
-  wr_gpu_batch* batches;
-  size_t batch_count;
-  size_t batch_capacity;
-
-  wr_rect viewport;
-  wr_transform transform;
-  float opacity;
-  wr_rect* clip_stack;
-  size_t clip_depth;
-  size_t clip_capacity;
-  wr_display_state* state_stack;
-  size_t state_depth;
-  size_t state_capacity;
-  wr_frame frame;
-};
-
-static int
-wr_size_mul_overflow(size_t a, size_t b)
-{
-  return b != 0 && a > SIZE_MAX / b;
-}
-
-static int
-wr_reserve(void** data, size_t* capacity, size_t count, size_t element_size)
-{
-  size_t new_capacity;
-  void* new_data;
-
-  if (count <= *capacity)
-    return 1;
-  if (element_size == 0 || wr_size_mul_overflow(count, element_size))
-    return 0;
-
-  new_capacity = *capacity ? *capacity : 8;
-  while (new_capacity < count) {
-    if (new_capacity > SIZE_MAX / 2) {
-      new_capacity = count;
-      break;
-    }
-    new_capacity *= 2;
-  }
-  if (wr_size_mul_overflow(new_capacity, element_size))
-    return 0;
-
-  new_data = realloc(*data, new_capacity * element_size);
-  if (!new_data)
-    return 0;
-
-  *data = new_data;
-  *capacity = new_capacity;
-  return 1;
-}
-
-static int
-wr_valid_rect(wr_rect rect)
-{
-  return isfinite(rect.x) && isfinite(rect.y) &&
-         isfinite(rect.width) && isfinite(rect.height) &&
-         rect.width > 0.0f && rect.height > 0.0f;
-}
-
-static int
-wr_valid_transform(wr_transform transform)
-{
-  return isfinite(transform.m11) && isfinite(transform.m12) &&
-         isfinite(transform.m21) && isfinite(transform.m22) &&
-         isfinite(transform.m31) && isfinite(transform.m32);
-}
-
-static wr_transform
-wr_identity_transform(void)
-{
-  wr_transform transform = { 1, 0, 0, 1, 0, 0 };
-  return transform;
-}
-
-static wr_transform
-wr_multiply_transform(wr_transform a, wr_transform b)
-{
-  wr_transform result;
-  result.m11 = a.m11 * b.m11 + a.m21 * b.m12;
-  result.m12 = a.m12 * b.m11 + a.m22 * b.m12;
-  result.m21 = a.m11 * b.m21 + a.m21 * b.m22;
-  result.m22 = a.m12 * b.m21 + a.m22 * b.m22;
-  result.m31 = a.m11 * b.m31 + a.m21 * b.m32 + a.m31;
-  result.m32 = a.m12 * b.m31 + a.m22 * b.m32 + a.m32;
-  return result;
-}
-
-static wr_rect
-wr_empty_rect(void)
-{
-  wr_rect rect = { 0, 0, 0, 0 };
-  return rect;
-}
-
-static float
-wr_clamp01(float value)
-{
-  if (value < 0.0f)
-    return 0.0f;
-  if (value > 1.0f)
-    return 1.0f;
-  return value;
-}
-
-static wr_color
-wr_normalize_color(wr_color color)
-{
-  color.r = wr_clamp01(color.r);
-  color.g = wr_clamp01(color.g);
-  color.b = wr_clamp01(color.b);
-  color.a = wr_clamp01(color.a);
-  return color;
-}
-
-static int
-wr_colors_equal(wr_color a, wr_color b)
-{
-  return a.r == b.r && a.g == b.g && a.b == b.b && a.a == b.a;
-}
-
-static int
-wr_rects_equal(wr_rect a, wr_rect b)
-{
-  return a.x == b.x && a.y == b.y &&
-         a.width == b.width && a.height == b.height;
-}
-
-static int
-wr_intersects(wr_rect a, wr_rect b)
-{
-  return a.x < b.x + b.width && a.x + a.width > b.x &&
-         a.y < b.y + b.height && a.y + a.height > b.y;
-}
-
-static wr_rect
-wr_intersect_rect(wr_rect a, wr_rect b)
-{
-  float left = a.x > b.x ? a.x : b.x;
-  float top = a.y > b.y ? a.y : b.y;
-  float right = a.x + a.width < b.x + b.width ?
-                a.x + a.width : b.x + b.width;
-  float bottom = a.y + a.height < b.y + b.height ?
-                 a.y + a.height : b.y + b.height;
-  wr_rect result = { left, top, right - left, bottom - top };
-  return result;
-}
-
-static wr_rect
-wr_transform_bounds(wr_rect rect, wr_transform transform)
-{
-  float x1 = rect.x * transform.m11 + rect.y * transform.m21 + transform.m31;
-  float y1 = rect.x * transform.m12 + rect.y * transform.m22 + transform.m32;
-  float x2 = (rect.x + rect.width) * transform.m11 +
-             rect.y * transform.m21 + transform.m31;
-  float y2 = (rect.x + rect.width) * transform.m12 +
-             rect.y * transform.m22 + transform.m32;
-  float x3 = rect.x * transform.m11 +
-             (rect.y + rect.height) * transform.m21 + transform.m31;
-  float y3 = rect.x * transform.m12 +
-             (rect.y + rect.height) * transform.m22 + transform.m32;
-  float x4 = (rect.x + rect.width) * transform.m11 +
-             (rect.y + rect.height) * transform.m21 + transform.m31;
-  float y4 = (rect.x + rect.width) * transform.m12 +
-             (rect.y + rect.height) * transform.m22 + transform.m32;
-  float left = fminf(fminf(x1, x2), fminf(x3, x4));
-  float right = fmaxf(fmaxf(x1, x2), fmaxf(x3, x4));
-  float top = fminf(fminf(y1, y2), fminf(y3, y4));
-  float bottom = fmaxf(fmaxf(y1, y2), fmaxf(y3, y4));
-  wr_rect result = { left, top, right - left, bottom - top };
-  return result;
-}
-
-wr_context*
-wr_context_create(size_t initial_capacity)
-{
-  wr_context* context = (wr_context*)calloc(1, sizeof(*context));
-  if (!context)
-    return NULL;
-
-  context->viewport.x = -FLT_MAX;
-  context->viewport.y = -FLT_MAX;
-  context->viewport.width = FLT_MAX;
-  context->viewport.height = FLT_MAX;
-  context->transform = wr_identity_transform();
-  context->opacity = 1.0f;
-
-  if (initial_capacity &&
-      !wr_reserve((void**)&context->commands, &context->command_capacity,
-                  initial_capacity, sizeof(*context->commands))) {
-    wr_context_destroy(context);
-    return NULL;
-  }
-  return context;
-}
-
-void
-wr_context_destroy(wr_context* context)
-{
-  if (!context)
-    return;
-  free(context->commands);
-  free(context->quads);
-  free(context->batches);
-  free(context->clip_stack);
-  free(context->state_stack);
-  free(context);
-}
-
-void
-wr_context_clear(wr_context* context)
-{
-  if (!context)
-    return;
-  context->command_count = 0;
-  context->quad_count = 0;
-  context->batch_count = 0;
-  context->clip_depth = 0;
-  context->state_depth = 0;
-}
-
-void
-wr_context_set_viewport(wr_context* context, wr_rect viewport)
-{
-  if (context && wr_valid_rect(viewport))
-    context->viewport = viewport;
-}
-
-void
-wr_context_set_transform(wr_context* context, wr_transform transform)
-{
-  if (context && wr_valid_transform(transform))
-    context->transform = transform;
-}
-
-void
-wr_context_set_opacity(wr_context* context, float opacity)
-{
-  if (context && isfinite(opacity)) {
-    context->opacity = wr_clamp01(opacity);
-  }
-}
-
-int
-wr_display_list_push_stacking_context(wr_context* context,
-                                      wr_transform transform,
-                                      float opacity)
-{
-  wr_display_state state;
-
-  if (!context || !wr_valid_transform(transform) || !isfinite(opacity) ||
-      context->state_depth == SIZE_MAX)
-    return 0;
-  if (!wr_reserve((void**)&context->state_stack, &context->state_capacity,
-                  context->state_depth + 1, sizeof(*context->state_stack)))
-    return 0;
-
-  state.transform = context->transform;
-  state.opacity = context->opacity;
-  state.clip_depth = context->clip_depth;
-  context->state_stack[context->state_depth++] = state;
-  context->transform = wr_multiply_transform(context->transform, transform);
-  context->opacity = wr_clamp01(context->opacity * wr_clamp01(opacity));
-  return 1;
-}
-
-void
-wr_display_list_pop_stacking_context(wr_context* context)
-{
-  wr_display_state state;
-
-  if (!context || !context->state_depth)
-    return;
-  state = context->state_stack[--context->state_depth];
-  context->transform = state.transform;
-  context->opacity = state.opacity;
-  context->clip_depth = state.clip_depth;
-}
-
-int
-wr_display_list_push_clip(wr_context* context, wr_rect clip)
-{
-  wr_rect effective_clip;
-
-  if (!context || !wr_valid_rect(clip) ||
-      context->clip_depth == SIZE_MAX)
-    return 0;
-  if (!wr_reserve((void**)&context->clip_stack, &context->clip_capacity,
-                  context->clip_depth + 1, sizeof(*context->clip_stack)))
-    return 0;
-
-  effective_clip = clip;
-  if (context->clip_depth)
-    effective_clip = wr_intersect_rect(
-      context->clip_stack[context->clip_depth - 1], clip);
-  context->clip_stack[context->clip_depth++] = effective_clip;
-  return 1;
-}
-
-void
-wr_display_list_pop_clip(wr_context* context)
-{
-  if (context && context->clip_depth)
-    --context->clip_depth;
-}
-
-int
-wr_display_list_push_rect(wr_context* context, wr_rect rect, wr_color color)
-{
-  return context ? wr_display_list_push_transformed_rect(
-                     context, rect, color, context->transform) : 0;
-}
-
-int
-wr_display_list_push_transformed_rect(wr_context* context, wr_rect rect,
-                                      wr_color color, wr_transform transform)
-{
-  wr_rect clip_rect = context && context->clip_depth ?
-                      context->clip_stack[context->clip_depth - 1] :
-                      context ? context->viewport : wr_empty_rect();
-  uint8_t has_clip = context && context->clip_depth ? 1 : 0;
-
-  if (!context || !wr_valid_rect(rect) || !wr_valid_transform(transform))
-    return 0;
-  if (context->command_count == SIZE_MAX ||
-      context->command_count >= UINT32_MAX)
-    return 0;
-  if (!wr_reserve((void**)&context->commands, &context->command_capacity,
-                  context->command_count + 1, sizeof(*context->commands)))
-    return 0;
-
-  context->commands[context->command_count].rect = rect;
-  color = wr_normalize_color(color);
-  color.a *= context->opacity;
-  context->commands[context->command_count].color = color;
-  context->commands[context->command_count].transform = transform;
-  context->commands[context->command_count].clip_rect = clip_rect;
-  context->commands[context->command_count].has_clip = has_clip;
-  ++context->command_count;
-  return 1;
-}
-
-const wr_frame*
-wr_context_build_frame(wr_context* context)
-{
-  size_t i;
-
-  if (!context)
-    return NULL;
-  context->quad_count = 0;
-  context->batch_count = 0;
-
-  for (i = 0; i < context->command_count; ++i) {
-    wr_rect_command* command = &context->commands[i];
-    wr_gpu_quad* quad;
-    wr_gpu_batch* batch;
-
-    if (!wr_intersects(wr_transform_bounds(command->rect,
-                                           command->transform),
-                       context->viewport) ||
-        (command->clip_rect.width <= 0.0f ||
-         command->clip_rect.height <= 0.0f))
-      continue;
-    if (context->quad_count == SIZE_MAX ||
-        context->quad_count >= UINT32_MAX ||
-        context->batch_count == SIZE_MAX ||
-        context->batch_count >= UINT32_MAX)
-      return NULL;
-    if (!wr_reserve((void**)&context->quads, &context->quad_capacity,
-                    context->quad_count + 1, sizeof(*context->quads)) ||
-        !wr_reserve((void**)&context->batches, &context->batch_capacity,
-                    context->batch_count + 1, sizeof(*context->batches)))
-      return NULL;
-
-    quad = &context->quads[context->quad_count++];
-    quad->rect = command->rect;
-    quad->color = command->color;
-
-    if (context->batch_count &&
-        wr_colors_equal(context->batches[context->batch_count - 1].color,
-                        command->color) &&
-        context->batches[context->batch_count - 1].transform.m11 ==
-          command->transform.m11 &&
-        context->batches[context->batch_count - 1].transform.m12 ==
-          command->transform.m12 &&
-        context->batches[context->batch_count - 1].transform.m21 ==
-          command->transform.m21 &&
-        context->batches[context->batch_count - 1].transform.m22 ==
-          command->transform.m22 &&
-        context->batches[context->batch_count - 1].transform.m31 ==
-          command->transform.m31 &&
-        context->batches[context->batch_count - 1].transform.m32 ==
-          command->transform.m32 &&
-        context->batches[context->batch_count - 1].has_clip ==
-          command->has_clip &&
-        (!command->has_clip ||
-         wr_rects_equal(context->batches[context->batch_count - 1].clip_rect,
-                        command->clip_rect))) {
-      batch = &context->batches[context->batch_count - 1];
-      ++batch->quad_count;
-    } else {
-      batch = &context->batches[context->batch_count++];
-      batch->first_quad = (uint32_t)(context->quad_count - 1);
-      batch->quad_count = 1;
-      batch->color = command->color;
-      batch->transform = command->transform;
-      batch->clip_rect = command->clip_rect;
-      batch->has_clip = command->has_clip;
-    }
-  }
-
-  context->frame.quads = context->quads;
-  context->frame.quad_count = context->quad_count;
-  context->frame.batches = context->batches;
-  context->frame.batch_count = context->batch_count;
-  return &context->frame;
-}

gfx/wr/webrender.h

@@ -9,6 +9,13 @@ extern "C" {
 #endif
 
 typedef struct wr_context wr_context;
+typedef uint64_t wr_item_id;
+
+typedef struct {
+  uint32_t key;
+  uint32_t width;
+  uint32_t height;
+} wr_image_resource;
 
 typedef struct {
   float x;
@@ -34,9 +41,24 @@ typedef struct {
   float m32;
 } wr_transform;
 
+typedef struct {
+  wr_rect rect;
+  wr_rect tex_rect;
+} wr_glyph;
+
+typedef enum {
+  WR_PRIMITIVE_SOLID = 0,
+  WR_PRIMITIVE_IMAGE = 1
+} wr_primitive_kind;
+
 typedef struct {
   wr_rect rect;
   wr_color color;
+  wr_rect tex_rect;
+  uint32_t image_key;
+  wr_item_id item_id;
+  uint8_t kind;
+  uint8_t active;
 } wr_gpu_quad;
 
 typedef struct {
@@ -46,6 +68,8 @@ typedef struct {
   wr_transform transform;
   wr_rect clip_rect;
   uint8_t has_clip;
+  uint8_t kind;
+  uint32_t image_key;
 } wr_gpu_batch;
 
 typedef struct {
@@ -55,6 +79,14 @@ typedef struct {
   size_t batch_count;
 } wr_frame;
 
+typedef struct {
+  uint32_t version;
+  uint32_t quad_count;
+  uint32_t batch_count;
+} wr_frame_packet_header;
+
+#define WR_FRAME_PACKET_VERSION 1
+
 /* Creates a retained display-list context. */
 wr_context* wr_context_create(size_t initial_capacity);
 void wr_context_destroy(wr_context* context);
@@ -62,6 +94,21 @@ void wr_context_destroy(wr_context* context);
 /* Removes every item from the retained display list. */
 void wr_context_clear(wr_context* context);
 
+/* Clears transient image resources without releasing retained allocations. */
+void wr_context_clear_images(wr_context* context);
+
+/* Begins an atomic retained-scene update. Transactions may be nested. */
+int wr_context_begin_transaction(wr_context* context);
+
+/* Commits the innermost retained-scene update. */
+int wr_context_commit_transaction(wr_context* context);
+
+/* Rolls back the innermost retained-scene update. */
+void wr_context_abort_transaction(wr_context* context);
+
+/* Removes a retained display-list item by stable item ID. */
+int wr_context_remove_item(wr_context* context, wr_item_id item_id);
+
 /* Sets the viewport used for coarse CPU-side culling. */
 void wr_context_set_viewport(wr_context* context, wr_rect viewport);
 
@@ -71,6 +118,13 @@ void wr_context_set_transform(wr_context* context, wr_transform transform);
 /* Sets the opacity applied to subsequently recorded primitives. */
 void wr_context_set_opacity(wr_context* context, float opacity);
 
+/* Registers or updates an image resource referenced by display-list commands. */
+int wr_context_register_image(wr_context* context, uint32_t image_key,
+                              uint32_t width, uint32_t height);
+
+/* Removes an image resource and invalidates frames referencing it. */
+void wr_context_unregister_image(wr_context* context, uint32_t image_key);
+
 /* Records a retained stacking-context boundary and restores it on pop. */
 int wr_display_list_push_stacking_context(wr_context* context,
                                           wr_transform transform,
@@ -85,14 +139,57 @@ void wr_display_list_pop_clip(wr_context* context);
 int wr_display_list_push_rect(wr_context* context, wr_rect rect,
                               wr_color color);
 
+/* Adds a solid rectangle with a caller-supplied stable item ID. */
+wr_item_id wr_display_list_push_rect_with_id(wr_context* context,
+                                             wr_item_id item_id,
+                                             wr_rect rect, wr_color color);
+
+/* Updates an existing solid rectangle without rebuilding the scene. */
+int wr_display_list_update_rect(wr_context* context, wr_item_id item_id,
+                                wr_rect rect, wr_color color,
+                                wr_transform transform);
+
+/* Adds a GPU-tessellated linear gradient. */
+int wr_display_list_push_linear_gradient(wr_context* context, wr_rect rect,
+                                         wr_color start, wr_color end,
+                                         int horizontal);
+
+/* Adds a GPU-tessellated rounded rectangle. */
+int wr_display_list_push_rounded_rect(wr_context* context, wr_rect rect,
+                                      wr_color color, float radius);
+
 /* Explicit transform form for display-list builders. */
 int wr_display_list_push_transformed_rect(wr_context* context, wr_rect rect,
                                           wr_color color,
                                           wr_transform transform);
 
+/* Adds an image primitive resolved by the compositor using image_key. */
+int wr_display_list_push_image(wr_context* context, wr_rect rect,
+                               uint32_t image_key, wr_rect tex_rect,
+                               wr_color tint);
+
+/* Adds an image with a caller-supplied stable item ID. */
+wr_item_id wr_display_list_push_image_with_id(wr_context* context,
+                                              wr_item_id item_id,
+                                              wr_rect rect,
+                                              uint32_t image_key,
+                                              wr_rect tex_rect,
+                                              wr_color tint);
+
+/* Records glyph quads sourced from a registered image atlas. */
+int wr_display_list_push_glyph_run(wr_context* context, uint32_t image_key,
+                                   const wr_glyph* glyphs, size_t glyph_count,
+                                   wr_color color);
+
 /* Builds the GPU-ready stream from the retained display list. */
 const wr_frame* wr_context_build_frame(wr_context* context);
 
+/* Returns the byte size required for a versioned, flat frame packet. */
+size_t wr_frame_packet_size(const wr_frame* frame);
+
+/* Serializes a frame into a flat buffer suitable for thread/IPC transfer. */
+int wr_frame_serialize(const wr_frame* frame, void* buffer, size_t buffer_size);
+
 #ifdef __cplusplus
 }
 #endif

gfx/wr/wr_context.c

@@ -0,0 +1,145 @@
+#include <wr_internal.h>
+
+wr_context*
+wr_context_create(size_t initial_capacity)
+{
+  wr_context* context = (wr_context*)calloc(1, sizeof(*context));
+  if (!context)
+    return NULL;
+
+  context->viewport.x = -FLT_MAX;
+  context->viewport.y = -FLT_MAX;
+  context->viewport.width = FLT_MAX;
+  context->viewport.height = FLT_MAX;
+  context->transform = wr_identity_transform();
+  context->opacity = 1.0f;
+  context->next_item_id = 1;
+
+  if (initial_capacity &&
+      !wr_reserve((void**)&context->commands, &context->command_capacity,
+                  initial_capacity, sizeof(*context->commands))) {
+    wr_context_destroy(context);
+    return NULL;
+  }
+  return context;
+}
+
+void
+wr_context_destroy(wr_context* context)
+{
+  if (!context)
+    return;
+  free(context->commands);
+  free(context->quads);
+  free(context->batches);
+  free(context->clip_stack);
+  free(context->state_stack);
+  free(context->images);
+  for (size_t i = 0; i < context->transaction_depth; ++i)
+    free(context->transactions[i].commands);
+  free(context->transactions);
+  free(context);
+}
+
+void
+wr_context_clear(wr_context* context)
+{
+  if (!context)
+    return;
+  context->command_count = 0;
+  context->quad_count = 0;
+  context->batch_count = 0;
+  context->clip_depth = 0;
+  context->state_depth = 0;
+  context->frame_valid = 0;
+  while (context->transaction_depth)
+    free(context->transactions[--context->transaction_depth].commands);
+}
+
+void
+wr_context_set_viewport(wr_context* context, wr_rect viewport)
+{
+  if (context && wr_valid_rect(viewport))
+    context->viewport = viewport;
+  if (context)
+    context->frame_valid = 0;
+}
+
+void
+wr_context_set_transform(wr_context* context, wr_transform transform)
+{
+  if (context && wr_valid_transform(transform))
+    context->transform = transform;
+}
+
+void
+wr_context_set_opacity(wr_context* context, float opacity)
+{
+  if (context && isfinite(opacity)) {
+    context->opacity = wr_clamp01(opacity);
+    context->frame_valid = 0;
+  }
+}
+
+int
+wr_display_list_push_stacking_context(wr_context* context,
+                                      wr_transform transform,
+                                      float opacity)
+{
+  wr_display_state state;
+
+  if (!context || !wr_valid_transform(transform) || !isfinite(opacity) ||
+      context->state_depth == SIZE_MAX)
+    return 0;
+  if (!wr_reserve((void**)&context->state_stack, &context->state_capacity,
+                  context->state_depth + 1, sizeof(*context->state_stack)))
+    return 0;
+
+  state.transform = context->transform;
+  state.opacity = context->opacity;
+  state.clip_depth = context->clip_depth;
+  context->state_stack[context->state_depth++] = state;
+  context->transform = wr_multiply_transform(context->transform, transform);
+  context->opacity = wr_clamp01(context->opacity * wr_clamp01(opacity));
+  return 1;
+}
+
+void
+wr_display_list_pop_stacking_context(wr_context* context)
+{
+  wr_display_state state;
+
+  if (!context || !context->state_depth)
+    return;
+  state = context->state_stack[--context->state_depth];
+  context->transform = state.transform;
+  context->opacity = state.opacity;
+  context->clip_depth = state.clip_depth;
+}
+
+int
+wr_display_list_push_clip(wr_context* context, wr_rect clip)
+{
+  wr_rect effective_clip;
+
+  if (!context || !wr_valid_rect(clip) ||
+      context->clip_depth == SIZE_MAX)
+    return 0;
+  if (!wr_reserve((void**)&context->clip_stack, &context->clip_capacity,
+                  context->clip_depth + 1, sizeof(*context->clip_stack)))
+    return 0;
+
+  effective_clip = clip;
+  if (context->clip_depth)
+    effective_clip = wr_intersect_rect(
+      context->clip_stack[context->clip_depth - 1], clip);
+  context->clip_stack[context->clip_depth++] = effective_clip;
+  return 1;
+}
+
+void
+wr_display_list_pop_clip(wr_context* context)
+{
+  if (context && context->clip_depth)
+    --context->clip_depth;
+}

gfx/wr/wr_display_list.c

@@ -0,0 +1,43 @@
+#include <wr_internal.h>
+
+int
+wr_display_list_push_rect(wr_context* context, wr_rect rect, wr_color color)
+{
+  return context ? wr_display_list_push_transformed_rect(
+                     context, rect, color, context->transform) : 0;
+}
+
+int
+wr_display_list_push_transformed_rect(wr_context* context, wr_rect rect,
+                                      wr_color color, wr_transform transform)
+{
+  wr_rect clip_rect = context && context->clip_depth ?
+                      context->clip_stack[context->clip_depth - 1] :
+                      context ? context->viewport : wr_empty_rect();
+  uint8_t has_clip = context && context->clip_depth ? 1 : 0;
+
+  if (!context || !wr_valid_rect(rect) || !wr_valid_transform(transform))
+    return 0;
+  if (context->command_count == SIZE_MAX ||
+      context->command_count >= UINT32_MAX)
+    return 0;
+  if (!wr_reserve((void**)&context->commands, &context->command_capacity,
+                  context->command_count + 1, sizeof(*context->commands)))
+    return 0;
+
+  context->commands[context->command_count].rect = rect;
+  color = wr_normalize_color(color);
+  color.a *= context->opacity;
+  context->commands[context->command_count].color = color;
+  context->commands[context->command_count].transform = transform;
+  context->commands[context->command_count].clip_rect = clip_rect;
+  context->commands[context->command_count].has_clip = has_clip;
+  context->commands[context->command_count].kind = WR_PRIMITIVE_SOLID;
+  context->commands[context->command_count].active = 1;
+  context->commands[context->command_count].image_key = 0;
+  context->commands[context->command_count].tex_rect = (wr_rect){ 0, 0, 1, 1 };
+  context->commands[context->command_count].item_id = context->next_item_id++;
+  ++context->command_count;
+  context->frame_valid = 0;
+  return 1;
+}

gfx/wr/wr_frame.c

@@ -0,0 +1,105 @@
+#include <wr_internal.h>
+
+static int
+wr_frame_has_image(const wr_context* context, uint32_t key)
+{
+  size_t i;
+  for (i = 0; i < context->image_count; ++i)
+    if (context->images[i].key == key)
+      return 1;
+  return 0;
+}
+
+const wr_frame*
+wr_context_build_frame(wr_context* context)
+{
+  size_t i;
+
+  if (!context)
+    return NULL;
+  if (context->frame_valid)
+    return &context->frame;
+  context->quad_count = 0;
+  context->batch_count = 0;
+
+  for (i = 0; i < context->command_count; ++i) {
+    wr_rect_command* command = &context->commands[i];
+    wr_gpu_quad* quad;
+    wr_gpu_batch* batch;
+
+    if (!command->active ||
+        (command->kind == WR_PRIMITIVE_IMAGE &&
+         !wr_frame_has_image(context, command->image_key)) ||
+        wr_command_occluded(context, i))
+      continue;
+    if (!wr_intersects(wr_transform_bounds(command->rect,
+                                           command->transform),
+                       context->viewport) ||
+        (command->clip_rect.width <= 0.0f ||
+         command->clip_rect.height <= 0.0f))
+      continue;
+    if (context->quad_count == SIZE_MAX ||
+        context->quad_count >= UINT32_MAX ||
+        context->batch_count == SIZE_MAX ||
+        context->batch_count >= UINT32_MAX)
+      return NULL;
+    if (!wr_reserve((void**)&context->quads, &context->quad_capacity,
+                    context->quad_count + 1, sizeof(*context->quads)) ||
+        !wr_reserve((void**)&context->batches, &context->batch_capacity,
+                    context->batch_count + 1, sizeof(*context->batches)))
+      return NULL;
+
+    quad = &context->quads[context->quad_count++];
+    quad->rect = command->rect;
+    quad->color = command->color;
+    quad->tex_rect = command->tex_rect;
+    quad->image_key = command->image_key;
+    quad->item_id = command->item_id;
+    quad->active = command->active;
+    quad->kind = command->kind;
+
+    if (context->batch_count &&
+        context->batches[context->batch_count - 1].kind == command->kind &&
+        context->batches[context->batch_count - 1].image_key ==
+          command->image_key &&
+        wr_colors_equal(context->batches[context->batch_count - 1].color,
+                        command->color) &&
+        context->batches[context->batch_count - 1].transform.m11 ==
+          command->transform.m11 &&
+        context->batches[context->batch_count - 1].transform.m12 ==
+          command->transform.m12 &&
+        context->batches[context->batch_count - 1].transform.m21 ==
+          command->transform.m21 &&
+        context->batches[context->batch_count - 1].transform.m22 ==
+          command->transform.m22 &&
+        context->batches[context->batch_count - 1].transform.m31 ==
+          command->transform.m31 &&
+        context->batches[context->batch_count - 1].transform.m32 ==
+          command->transform.m32 &&
+        context->batches[context->batch_count - 1].has_clip ==
+          command->has_clip &&
+        (!command->has_clip ||
+         wr_rects_equal(context->batches[context->batch_count - 1].clip_rect,
+                        command->clip_rect))) {
+      batch = &context->batches[context->batch_count - 1];
+      ++batch->quad_count;
+    } else {
+      batch = &context->batches[context->batch_count++];
+      batch->first_quad = (uint32_t)(context->quad_count - 1);
+      batch->quad_count = 1;
+      batch->color = command->color;
+      batch->transform = command->transform;
+      batch->clip_rect = command->clip_rect;
+      batch->has_clip = command->has_clip;
+      batch->kind = command->kind;
+      batch->image_key = command->image_key;
+    }
+  }
+
+  context->frame.quads = context->quads;
+  context->frame.quad_count = context->quad_count;
+  context->frame.batches = context->batches;
+  context->frame.batch_count = context->batch_count;
+  context->frame_valid = 1;
+  return &context->frame;
+}

gfx/wr/wr_frame_packet.c

@@ -0,0 +1,59 @@
+#include "webrender.h"
+
+#include <stdint.h>
+#include <string.h>
+
+static int
+wr_packet_size_mul_overflow(size_t a, size_t b)
+{
+  return b != 0 && a > SIZE_MAX / b;
+}
+
+size_t
+wr_frame_packet_size(const wr_frame* frame)
+{
+  size_t size;
+
+  if (!frame || frame->quad_count > UINT32_MAX ||
+      frame->batch_count > UINT32_MAX ||
+      (frame->quad_count && !frame->quads) ||
+      (frame->batch_count && !frame->batches))
+    return 0;
+
+  size = sizeof(wr_frame_packet_header);
+  if (wr_packet_size_mul_overflow(frame->quad_count, sizeof(*frame->quads)) ||
+      wr_packet_size_mul_overflow(frame->batch_count, sizeof(*frame->batches)))
+    return 0;
+  if (size > SIZE_MAX - frame->quad_count * sizeof(*frame->quads))
+    return 0;
+  size += frame->quad_count * sizeof(*frame->quads);
+  if (size > SIZE_MAX - frame->batch_count * sizeof(*frame->batches))
+    return 0;
+  return size + frame->batch_count * sizeof(*frame->batches);
+}
+
+int
+wr_frame_serialize(const wr_frame* frame, void* buffer, size_t buffer_size)
+{
+  wr_frame_packet_header header;
+  size_t required = wr_frame_packet_size(frame);
+  unsigned char* output = (unsigned char*)buffer;
+
+  if (!required || !buffer || buffer_size < required)
+    return 0;
+  header.version = WR_FRAME_PACKET_VERSION;
+  header.quad_count = (uint32_t)frame->quad_count;
+  header.batch_count = (uint32_t)frame->batch_count;
+  memcpy(output, &header, sizeof(header));
+  output += sizeof(header);
+  if (frame->quad_count) {
+    memcpy(output, frame->quads,
+           frame->quad_count * sizeof(*frame->quads));
+    output += frame->quad_count * sizeof(*frame->quads);
+  }
+  if (frame->batch_count) {
+    memcpy(output, frame->batches,
+           frame->batch_count * sizeof(*frame->batches));
+  }
+  return 1;
+}

gfx/wr/wr_internal.h

@@ -0,0 +1,82 @@
+#ifndef GFX_WR_INTERNAL_H
+#define GFX_WR_INTERNAL_H
+
+#include <webrender.h>
+#include <float.h>
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+  wr_rect rect;
+  wr_color color;
+  wr_transform transform;
+  wr_rect clip_rect;
+  uint8_t has_clip;
+  uint8_t kind;
+  uint8_t active;
+  uint32_t image_key;
+  wr_rect tex_rect;
+  wr_item_id item_id;
+} wr_rect_command;
+
+typedef struct {
+  wr_transform transform;
+  float opacity;
+  size_t clip_depth;
+} wr_display_state;
+
+typedef struct {
+  size_t command_count;
+  wr_rect_command* commands;
+} wr_scene_transaction;
+
+struct wr_context {
+  wr_rect_command* commands;
+  size_t command_count;
+  size_t command_capacity;
+
+  wr_gpu_quad* quads;
+  size_t quad_count;
+  size_t quad_capacity;
+
+  wr_gpu_batch* batches;
+  size_t batch_count;
+  size_t batch_capacity;
+
+  wr_rect viewport;
+  wr_transform transform;
+  float opacity;
+  wr_rect* clip_stack;
+  size_t clip_depth;
+  size_t clip_capacity;
+  wr_display_state* state_stack;
+  size_t state_depth;
+  size_t state_capacity;
+  wr_item_id next_item_id;
+  wr_image_resource* images;
+  size_t image_count;
+  size_t image_capacity;
+  wr_scene_transaction* transactions;
+  size_t transaction_depth;
+  size_t transaction_capacity;
+  int frame_valid;
+  wr_frame frame;
+};
+
+int wr_size_mul_overflow(size_t, size_t);
+int wr_reserve(void**, size_t*, size_t, size_t);
+int wr_valid_rect(wr_rect);
+int wr_valid_transform(wr_transform);
+wr_transform wr_identity_transform(void);
+wr_transform wr_multiply_transform(wr_transform, wr_transform);
+wr_rect wr_empty_rect(void);
+float wr_clamp01(float);
+wr_color wr_normalize_color(wr_color);
+int wr_colors_equal(wr_color, wr_color);
+int wr_rects_equal(wr_rect, wr_rect);
+int wr_intersects(wr_rect, wr_rect);
+wr_rect wr_intersect_rect(wr_rect, wr_rect);
+wr_rect wr_transform_bounds(wr_rect, wr_transform);
+int wr_command_occluded(const wr_context*, size_t);
+#endif

gfx/wr/wr_retained.c

@@ -0,0 +1,251 @@
+#include <wr_internal.h>
+
+static size_t
+wr_image_index(const wr_context* context, uint32_t key)
+{
+  size_t i;
+  for (i = 0; context && i < context->image_count; ++i)
+    if (context->images[i].key == key)
+      return i;
+  return SIZE_MAX;
+}
+
+static size_t
+wr_item_index(const wr_context* context, wr_item_id item)
+{
+  size_t i;
+  for (i = 0; context && i < context->command_count; ++i)
+    if (context->commands[i].active && context->commands[i].item_id == item)
+      return i;
+  return SIZE_MAX;
+}
+
+static int
+wr_rect_contains(wr_rect outer, wr_rect inner)
+{
+  return inner.x >= outer.x && inner.y >= outer.y &&
+         inner.x + inner.width <= outer.x + outer.width &&
+         inner.y + inner.height <= outer.y + outer.height;
+}
+
+static int
+wr_append(wr_context* context, wr_item_id item, wr_rect rect, wr_color color,
+          uint8_t kind, uint32_t image_key, wr_rect tex_rect)
+{
+  wr_rect clip = context->clip_depth
+    ? context->clip_stack[context->clip_depth - 1] : context->viewport;
+  if (!context || !wr_valid_rect(rect) ||
+      (kind == WR_PRIMITIVE_IMAGE &&
+       (wr_image_index(context, image_key) == SIZE_MAX ||
+        tex_rect.width == 0.0f || tex_rect.height == 0.0f)))
+    return 0;
+  if (!wr_reserve((void**)&context->commands, &context->command_capacity,
+                  context->command_count + 1, sizeof(*context->commands)))
+    return 0;
+  if (!item) {
+    item = context->next_item_id++;
+    if (!item)
+      item = context->next_item_id++;
+  }
+  if (wr_item_index(context, item) != SIZE_MAX)
+    return 0;
+  context->commands[context->command_count] = (wr_rect_command){
+    rect, wr_normalize_color(color), context->transform, clip,
+    context->clip_depth ? 1 : 0, kind, 1, image_key, tex_rect, item };
+  context->commands[context->command_count].color.a *= context->opacity;
+  ++context->command_count;
+  context->frame_valid = 0;
+  return 1;
+}
+
+int
+wr_context_register_image(wr_context* context, uint32_t key,
+                          uint32_t width, uint32_t height)
+{
+  size_t index;
+  if (!context || !key || !width || !height)
+    return 0;
+  index = wr_image_index(context, key);
+  if (index != SIZE_MAX) {
+    context->images[index].width = width;
+    context->images[index].height = height;
+    return 1;
+  }
+  if (!wr_reserve((void**)&context->images, &context->image_capacity,
+                  context->image_count + 1, sizeof(*context->images)))
+    return 0;
+  context->images[context->image_count++] = (wr_image_resource){ key, width, height };
+  return 1;
+}
+
+void
+wr_context_unregister_image(wr_context* context, uint32_t key)
+{
+  size_t index = wr_image_index(context, key);
+  if (index == SIZE_MAX)
+    return;
+  context->images[index] = context->images[--context->image_count];
+  context->frame_valid = 0;
+}
+
+void
+wr_context_clear_images(wr_context* context)
+{
+  if (context) {
+    context->image_count = 0;
+    context->frame_valid = 0;
+  }
+}
+
+int
+wr_context_begin_transaction(wr_context* context)
+{
+  wr_scene_transaction transaction = { 0, NULL };
+  if (!context || !wr_reserve((void**)&context->transactions,
+      &context->transaction_capacity, context->transaction_depth + 1,
+      sizeof(*context->transactions)))
+    return 0;
+  transaction.command_count = context->command_count;
+  if (transaction.command_count) {
+    transaction.commands = malloc(transaction.command_count * sizeof(*transaction.commands));
+    if (!transaction.commands)
+      return 0;
+    memcpy(transaction.commands, context->commands,
+           transaction.command_count * sizeof(*transaction.commands));
+  }
+  context->transactions[context->transaction_depth++] = transaction;
+  return 1;
+}
+
+int
+wr_context_commit_transaction(wr_context* context)
+{
+  if (!context || !context->transaction_depth)
+    return 0;
+  free(context->transactions[--context->transaction_depth].commands);
+  context->frame_valid = 0;
+  return 1;
+}
+
+void
+wr_context_abort_transaction(wr_context* context)
+{
+  wr_scene_transaction transaction;
+  if (!context || !context->transaction_depth)
+    return;
+  transaction = context->transactions[--context->transaction_depth];
+  if (transaction.command_count)
+    memcpy(context->commands, transaction.commands,
+           transaction.command_count * sizeof(*context->commands));
+  context->command_count = transaction.command_count;
+  free(transaction.commands);
+  context->frame_valid = 0;
+}
+
+int
+wr_context_remove_item(wr_context* context, wr_item_id item)
+{
+  size_t index = wr_item_index(context, item);
+  if (index == SIZE_MAX)
+    return 0;
+  context->commands[index].active = 0;
+  context->frame_valid = 0;
+  return 1;
+}
+
+wr_item_id
+wr_display_list_push_rect_with_id(wr_context* context, wr_item_id item,
+                                   wr_rect rect, wr_color color)
+{
+  return wr_append(context, item, rect, color, WR_PRIMITIVE_SOLID, 0,
+                   (wr_rect){ 0, 0, 1, 1 }) ? item : 0;
+}
+
+int
+wr_display_list_update_rect(wr_context* context, wr_item_id item, wr_rect rect,
+                            wr_color color, wr_transform transform)
+{
+  size_t index = wr_item_index(context, item);
+  if (index == SIZE_MAX || !wr_valid_rect(rect) || !wr_valid_transform(transform))
+    return 0;
+  context->commands[index].rect = rect;
+  context->commands[index].color = wr_normalize_color(color);
+  context->commands[index].color.a *= context->opacity;
+  context->commands[index].transform = transform;
+  context->frame_valid = 0;
+  return 1;
+}
+
+int
+wr_display_list_push_image(wr_context* context, wr_rect rect, uint32_t key,
+                           wr_rect tex_rect, wr_color tint)
+{ return wr_append(context, 0, rect, tint, WR_PRIMITIVE_IMAGE, key, tex_rect); }
+
+wr_item_id
+wr_display_list_push_image_with_id(wr_context* context, wr_item_id item,
+                                   wr_rect rect, uint32_t key, wr_rect tex,
+                                   wr_color tint)
+{ return wr_append(context, item, rect, tint, WR_PRIMITIVE_IMAGE, key, tex) ? item : 0; }
+
+int
+wr_display_list_push_linear_gradient(wr_context* context, wr_rect rect,
+                                     wr_color start, wr_color end, int horizontal)
+{
+  unsigned i;
+  for (i = 0; i < 32; ++i) {
+    float a = (float)i / 32.0f, b = (float)(i + 1) / 32.0f;
+    wr_rect slice = rect;
+    wr_color color = { start.r + (end.r-start.r)*(a+b)*.5f,
+      start.g + (end.g-start.g)*(a+b)*.5f, start.b + (end.b-start.b)*(a+b)*.5f,
+      start.a + (end.a-start.a)*(a+b)*.5f };
+    if (horizontal) { slice.x += rect.width*a; slice.width = rect.width*(b-a); }
+    else { slice.y += rect.height*a; slice.height = rect.height*(b-a); }
+    if (!wr_append(context, 0, slice, color, WR_PRIMITIVE_SOLID, 0,
+                   (wr_rect){ 0, 0, 1, 1 })) return 0;
+  }
+  return 1;
+}
+
+int
+wr_display_list_push_rounded_rect(wr_context* context, wr_rect rect,
+                                  wr_color color, float radius)
+{
+  unsigned i;
+  (void)radius;
+  for (i = 0; i < 16; ++i) {
+    wr_rect slice = { rect.x, rect.y + rect.height * i / 16.0f,
+                      rect.width, rect.height / 16.0f };
+    if (!wr_append(context, 0, slice, color, WR_PRIMITIVE_SOLID, 0,
+                   (wr_rect){ 0, 0, 1, 1 }))
+      return 0;
+  }
+  return 1;
+}
+
+int
+wr_display_list_push_glyph_run(wr_context* context, uint32_t key,
+                               const wr_glyph* glyphs, size_t count, wr_color color)
+{
+  size_t i;
+  for (i = 0; glyphs && i < count; ++i)
+    if (!wr_append(context, 0, glyphs[i].rect, color, WR_PRIMITIVE_IMAGE,
+                   key, glyphs[i].tex_rect)) return 0;
+  return glyphs && count != 0;
+}
+
+int
+wr_command_occluded(const wr_context* context, size_t index)
+{
+  size_t i;
+  wr_rect bounds = wr_transform_bounds(context->commands[index].rect,
+                                       context->commands[index].transform);
+  for (i = index + 1; i < context->command_count; ++i) {
+    const wr_rect_command* covering = &context->commands[i];
+    wr_rect cover;
+    if (!covering->active || covering->kind != WR_PRIMITIVE_SOLID ||
+        covering->color.a != 1.0f) continue;
+    cover = wr_transform_bounds(covering->rect, covering->transform);
+    if (wr_rect_contains(cover, bounds)) return 1;
+  }
+  return 0;
+}

gfx/wr/wr_util.c

@@ -0,0 +1,160 @@
+#include <wr_internal.h>
+
+int
+wr_size_mul_overflow(size_t a, size_t b)
+{
+  return b != 0 && a > SIZE_MAX / b;
+}
+
+int
+wr_reserve(void** data, size_t* capacity, size_t count, size_t element_size)
+{
+  size_t new_capacity;
+  void* new_data;
+
+  if (count <= *capacity)
+    return 1;
+  if (element_size == 0 || wr_size_mul_overflow(count, element_size))
+    return 0;
+
+  new_capacity = *capacity ? *capacity : 8;
+  while (new_capacity < count) {
+    if (new_capacity > SIZE_MAX / 2) {
+      new_capacity = count;
+      break;
+    }
+    new_capacity *= 2;
+  }
+  if (wr_size_mul_overflow(new_capacity, element_size))
+    return 0;
+
+  new_data = realloc(*data, new_capacity * element_size);
+  if (!new_data)
+    return 0;
+
+  *data = new_data;
+  *capacity = new_capacity;
+  return 1;
+}
+
+int
+wr_valid_rect(wr_rect rect)
+{
+  return isfinite(rect.x) && isfinite(rect.y) &&
+         isfinite(rect.width) && isfinite(rect.height) &&
+         rect.width > 0.0f && rect.height > 0.0f;
+}
+
+int
+wr_valid_transform(wr_transform transform)
+{
+  return isfinite(transform.m11) && isfinite(transform.m12) &&
+         isfinite(transform.m21) && isfinite(transform.m22) &&
+         isfinite(transform.m31) && isfinite(transform.m32);
+}
+
+wr_transform
+wr_identity_transform(void)
+{
+  wr_transform transform = { 1, 0, 0, 1, 0, 0 };
+  return transform;
+}
+
+wr_transform
+wr_multiply_transform(wr_transform a, wr_transform b)
+{
+  wr_transform result;
+  result.m11 = a.m11 * b.m11 + a.m21 * b.m12;
+  result.m12 = a.m12 * b.m11 + a.m22 * b.m12;
+  result.m21 = a.m11 * b.m21 + a.m21 * b.m22;
+  result.m22 = a.m12 * b.m21 + a.m22 * b.m22;
+  result.m31 = a.m11 * b.m31 + a.m21 * b.m32 + a.m31;
+  result.m32 = a.m12 * b.m31 + a.m22 * b.m32 + a.m32;
+  return result;
+}
+
+wr_rect
+wr_empty_rect(void)
+{
+  wr_rect rect = { 0, 0, 0, 0 };
+  return rect;
+}
+
+float
+wr_clamp01(float value)
+{
+  if (value < 0.0f)
+    return 0.0f;
+  if (value > 1.0f)
+    return 1.0f;
+  return value;
+}
+
+wr_color
+wr_normalize_color(wr_color color)
+{
+  color.r = wr_clamp01(color.r);
+  color.g = wr_clamp01(color.g);
+  color.b = wr_clamp01(color.b);
+  color.a = wr_clamp01(color.a);
+  return color;
+}
+
+int
+wr_colors_equal(wr_color a, wr_color b)
+{
+  return a.r == b.r && a.g == b.g && a.b == b.b && a.a == b.a;
+}
+
+int
+wr_rects_equal(wr_rect a, wr_rect b)
+{
+  return a.x == b.x && a.y == b.y &&
+         a.width == b.width && a.height == b.height;
+}
+
+int
+wr_intersects(wr_rect a, wr_rect b)
+{
+  return a.x < b.x + b.width && a.x + a.width > b.x &&
+         a.y < b.y + b.height && a.y + a.height > b.y;
+}
+
+wr_rect
+wr_intersect_rect(wr_rect a, wr_rect b)
+{
+  float left = a.x > b.x ? a.x : b.x;
+  float top = a.y > b.y ? a.y : b.y;
+  float right = a.x + a.width < b.x + b.width ?
+                a.x + a.width : b.x + b.width;
+  float bottom = a.y + a.height < b.y + b.height ?
+                 a.y + a.height : b.y + b.height;
+  wr_rect result = { left, top, right - left, bottom - top };
+  return result;
+}
+
+wr_rect
+wr_transform_bounds(wr_rect rect, wr_transform transform)
+{
+  float x1 = rect.x * transform.m11 + rect.y * transform.m21 + transform.m31;
+  float y1 = rect.x * transform.m12 + rect.y * transform.m22 + transform.m32;
+  float x2 = (rect.x + rect.width) * transform.m11 +
+             rect.y * transform.m21 + transform.m31;
+  float y2 = (rect.x + rect.width) * transform.m12 +
+             rect.y * transform.m22 + transform.m32;
+  float x3 = rect.x * transform.m11 +
+             (rect.y + rect.height) * transform.m21 + transform.m31;
+  float y3 = rect.x * transform.m12 +
+             (rect.y + rect.height) * transform.m22 + transform.m32;
+  float x4 = (rect.x + rect.width) * transform.m11 +
+             (rect.y + rect.height) * transform.m21 + transform.m31;
+  float y4 = (rect.x + rect.width) * transform.m12 +
+             (rect.y + rect.height) * transform.m22 + transform.m32;
+  float left = fminf(fminf(x1, x2), fminf(x3, x4));
+  float right = fmaxf(fmaxf(x1, x2), fmaxf(x3, x4));
+  float top = fminf(fminf(y1, y2), fminf(y3, y4));
+  float bottom = fmaxf(fmaxf(y1, y2), fmaxf(y3, y4));
+  wr_rect result = { left, top, right - left, bottom - top };
+  return result;
+}
+

js/src/builtin/TypedObject.cpp

@@ -2040,7 +2040,7 @@ InlineTypedObject::createCopy(JSContext* cx, Handle<InlineTypedObject*> template
     if (!res)
         return nullptr;
 
-    memcpy(res->inlineTypedMem(), templateObject->inlineTypedMem(), templateObject->size());
+    js_memcpy(res->inlineTypedMem(), templateObject->inlineTypedMem(), templateObject->size());
     return res;
 }
 
@@ -2767,7 +2767,7 @@ TypeDescr::initInstances(const JSRuntime* rt, uint8_t* mem, size_t length)
     MemoryInitVisitor visitor(rt);
 
     // Initialize the 0th instance
-    memset(mem, 0, size());
+    js_memset(mem, 0, size());
     if (opaque())
         visitReferences(*this, mem, visitor);
 
@@ -2775,7 +2775,7 @@ TypeDescr::initInstances(const JSRuntime* rt, uint8_t* mem, size_t length)
     uint8_t* target = mem;
     for (size_t i = 1; i < length; i++) {
         target += size();
-        memcpy(target, mem, size());
+        js_memcpy(target, mem, size());
     }
 }
 

js/src/irregexp/RegExpInterpreter.cpp

@@ -31,6 +31,7 @@
 
 #include "irregexp/RegExpBytecode.h"
 #include "irregexp/RegExpMacroAssembler.h"
+#include "jsutil.h"
 #include "vm/MatchPairs.h"
 
 using namespace js;
@@ -197,7 +198,8 @@ irregexp::InterpretCode(JSContext* cx, const uint8_t* byteCode, const CharT* cha
             return RegExpRunStatus_Success_NotFound;
           BYTECODE(SUCCEED)
             if (matches)
-                memcpy(matches->pairsRaw(), registers.begin(), matches->length() * 2 * sizeof(int32_t));
+                js_memcpy(matches->pairsRaw(), registers.begin(),
+                          matches->length() * 2 * sizeof(int32_t));
             else if (endIndex)
                 *endIndex = registers[1];
             return RegExpRunStatus_Success;

js/src/jit-test/tests/latin1/sse2-search.js

@@ -19,3 +19,21 @@ for (var length of [0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65]) {
     assertEq(wide.indexOf("\xff"), length + 1);
     assertEq(wide.indexOf(latin1), 1);
 }
+
+// Mixed-encoding searches should use the same SIMD first-character scan, and
+// an impossible UTF-16 character should reject a Latin-1 haystack immediately.
+var latin1Haystack = "x".repeat(4096);
+assertEq(latin1Haystack.indexOf("x\u0100x"), -1);
+var wideHaystack = "\u0100" + "x".repeat(4096) + "needle";
+assertEq(wideHaystack.indexOf("needle"), 4097);
+
+// Exercise the mixed-width EqualChars fast path through string equality.
+var wideLatin1 = ("\u0100" + latin1Haystack).slice(1);
+assertEq(wideLatin1, latin1Haystack);
+assertEq(wideLatin1 + "y", latin1Haystack + "z");
+assertEq(wideLatin1 + "\u0100" > latin1Haystack + "z", true);
+assertEq(latin1Haystack + "z" < wideLatin1 + "\u0100", true);
+
+assertEq(latin1Haystack.lastIndexOf("x\u0100x"), -1);
+assertEq(wideHaystack.lastIndexOf("needle"), 4097);
+assertEq(wideHaystack.lastIndexOf("x"), 4096);

js/src/jit/BaselineBailouts.cpp

@@ -150,7 +150,7 @@ struct BaselineStackBuilder
         uint8_t* newBuffer = reinterpret_cast<uint8_t*>(js_calloc(newSize));
         if (!newBuffer)
             return false;
-        memcpy((newBuffer + newSize) - bufferUsed_, header_->copyStackBottom, bufferUsed_);
+        js_memcpy((newBuffer + newSize) - bufferUsed_, header_->copyStackBottom, bufferUsed_);
         memcpy(newBuffer, header_, sizeof(BaselineBailoutInfo));
         js_free(buffer_);
         buffer_ = newBuffer;

js/src/jit/BaselineJIT.cpp

@@ -18,6 +18,7 @@
 #include "vm/Interpreter.h"
 #include "vm/TraceLogging.h"
 #include "wasm/WasmInstance.h"
+#include "jsutil.h"
 
 #include "jsobjinlines.h"
 #include "jsopcodeinlines.h"
@@ -812,7 +813,7 @@ BaselineScript::copyPCMappingEntries(const CompactBufferWriter& entries)
     MOZ_ASSERT(entries.length() > 0);
     MOZ_ASSERT(entries.length() == pcMappingSize_);
 
-    memcpy(pcMappingData(), entries.buffer(), entries.length());
+    js_memcpy(pcMappingData(), entries.buffer(), entries.length());
 }
 
 void

js/src/jit/CompileInfo.h

@@ -442,14 +442,18 @@ class CompileInfo
     // the frame is active on the stack.  This implies that these definitions
     // would have to be executed and that they cannot be removed even if they
     // are unused.
-    bool isObservableSlot(uint32_t slot) const {
-        if (isObservableFrameSlot(slot))
-            return true;
+    inline bool isObservableSlot(uint32_t slot) const {
+        if (slot >= firstLocalSlot()) {
+            // The |this| slot for a derived class constructor is a local slot.
+            if (thisSlotForDerivedClassConstructor_)
+                return *thisSlotForDerivedClassConstructor_ == slot;
+            return false;
+        }
 
-        if (isObservableArgumentSlot(slot))
-            return true;
+        if (slot < firstArgSlot())
+            return isObservableFrameSlot(slot);
 
-        return false;
+        return isObservableArgumentSlot(slot);
     }
 
     bool isObservableFrameSlot(uint32_t slot) const {

js/src/jit/Ion.cpp

@@ -13,6 +13,7 @@
 #include "jscompartment.h"
 #include "jsgc.h"
 #include "jsprf.h"
+#include "jsutil.h"
 
 #include "gc/Marking.h"
 #include "jit/AliasAnalysis.h"
@@ -1042,33 +1043,33 @@ void
 IonScript::copySnapshots(const SnapshotWriter* writer)
 {
     MOZ_ASSERT(writer->listSize() == snapshotsListSize_);
-    memcpy((uint8_t*)this + snapshots_,
-           writer->listBuffer(), snapshotsListSize_);
+    js_memcpy((uint8_t*)this + snapshots_,
+              writer->listBuffer(), snapshotsListSize_);
 
     MOZ_ASSERT(snapshotsRVATableSize_);
     MOZ_ASSERT(writer->RVATableSize() == snapshotsRVATableSize_);
-    memcpy((uint8_t*)this + snapshots_ + snapshotsListSize_,
-           writer->RVATableBuffer(), snapshotsRVATableSize_);
+    js_memcpy((uint8_t*)this + snapshots_ + snapshotsListSize_,
+              writer->RVATableBuffer(), snapshotsRVATableSize_);
 }
 
 void
 IonScript::copyRecovers(const RecoverWriter* writer)
 {
     MOZ_ASSERT(writer->size() == recoversSize_);
-    memcpy((uint8_t*)this + recovers_, writer->buffer(), recoversSize_);
+    js_memcpy((uint8_t*)this + recovers_, writer->buffer(), recoversSize_);
 }
 
 void
 IonScript::copySafepoints(const SafepointWriter* writer)
 {
     MOZ_ASSERT(writer->size() == safepointsSize_);
-    memcpy((uint8_t*)this + safepointsStart_, writer->buffer(), safepointsSize_);
+    js_memcpy((uint8_t*)this + safepointsStart_, writer->buffer(), safepointsSize_);
 }
 
 void
 IonScript::copyBailoutTable(const SnapshotOffset* table)
 {
-    memcpy(bailoutTable(), table, bailoutEntries_ * sizeof(uint32_t));
+    js_memcpy(bailoutTable(), table, bailoutEntries_ * sizeof(uint32_t));
 }
 
 void
@@ -1114,25 +1115,25 @@ IonScript::copySafepointIndices(const SafepointIndex* si, MacroAssembler& masm)
     // code, not the absolute positions of the jumps. Update according to the
     // final code address now.
     SafepointIndex* table = safepointIndices();
-    memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex));
+    js_memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex));
 }
 
 void
 IonScript::copyOsiIndices(const OsiIndex* oi, MacroAssembler& masm)
 {
-    memcpy(osiIndices(), oi, osiIndexEntries_ * sizeof(OsiIndex));
+    js_memcpy(osiIndices(), oi, osiIndexEntries_ * sizeof(OsiIndex));
 }
 
 void
 IonScript::copyRuntimeData(const uint8_t* data)
 {
-    memcpy(runtimeData(), data, runtimeSize());
+    js_memcpy(runtimeData(), data, runtimeSize());
 }
 
 void
 IonScript::copyCacheEntries(const uint32_t* caches, MacroAssembler& masm)
 {
-    memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t));
+    js_memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t));
 
     // Jumps in the caches reflect the offset of those jumps in the compiled
     // code, not the absolute positions of the jumps. Update according to the

js/src/jit/IonAnalysis.cpp

@@ -196,6 +196,8 @@ FlagPhiInputsAsHavingRemovedUses(MIRGenerator* mir, MBasicBlock* block, MBasicBl
 static bool
 FlagAllOperandsAsHavingRemovedUses(MIRGenerator* mir, MBasicBlock* block)
 {
+    const CompileInfo& info = block->info();
+
     // Flag all instructions operands as having removed uses.
     MInstructionIterator end = block->end();
     for (MInstructionIterator it = block->begin(); it != end; it++) {
@@ -214,7 +216,7 @@ FlagAllOperandsAsHavingRemovedUses(MIRGenerator* mir, MBasicBlock* block)
                 if (mir->shouldCancel("FlagAllOperandsAsHavingRemovedUses inner loop"))
                     return false;
 
-                if (!rp->isObservableOperand(i))
+                if (!info.isObservableSlot(i))
                     continue;
                 rp->getOperand(i)->setUseRemovedUnchecked();
             }
@@ -227,8 +229,9 @@ FlagAllOperandsAsHavingRemovedUses(MIRGenerator* mir, MBasicBlock* block)
         if (mir->shouldCancel("FlagAllOperandsAsHavingRemovedUses loop 2"))
             return false;
 
+        const CompileInfo& info = rp->block()->info();
         for (size_t i = 0, e = rp->numOperands(); i < e; i++) {
-            if (!rp->isObservableOperand(i))
+            if (!info.isObservableSlot(i))
                 continue;
             rp->getOperand(i)->setUseRemovedUnchecked();
         }

js/src/jit/shared/CodeGenerator-shared.cpp

@@ -16,6 +16,7 @@
 #include "jit/MIR.h"
 #include "jit/MIRGenerator.h"
 #include "jit/OptimizationTracking.h"
+#include "jsutil.h"
 #include "js/Conversions.h"
 #include "vm/TraceLogging.h"
 
@@ -754,7 +755,7 @@ CodeGeneratorShared::generateCompactNativeToBytecodeMap(JSContext* cx, JitCode*
         return false;
     }
 
-    memcpy(data, writer.buffer(), writer.length());
+    js_memcpy(data, writer.buffer(), writer.length());
     nativeToBytecodeMap_ = data;
     nativeToBytecodeMapSize_ = writer.length();
     nativeToBytecodeTableOffset_ = tableOffset;
@@ -908,7 +909,7 @@ CodeGeneratorShared::generateCompactTrackedOptimizationsMap(JSContext* cx, JitCo
     if (!data)
         return false;
 
-    memcpy(data, writer.buffer(), writer.length());
+    js_memcpy(data, writer.buffer(), writer.length());
     trackedOptimizationsMap_ = data;
     trackedOptimizationsMapSize_ = writer.length();
     trackedOptimizationsRegionTableOffset_ = regionTableOffset;

js/src/jsapi.cpp

@@ -6005,7 +6005,7 @@ EncodeLatin1(ExclusiveContext* cx, JSString* str)
         return nullptr;
     }
 
-    mozilla::PodCopy(buf, linear->latin1Chars(nogc), len);
+    js_memcpy(buf, linear->latin1Chars(nogc), len);
     buf[len] = '\0';
     return reinterpret_cast<char*>(buf);
 }

js/src/jsarray.cpp

@@ -153,6 +153,21 @@ StringIsArrayIndex(const CharT* s, uint32_t length, uint32_t* indexp)
     if (length == 0 || length > (sizeof("4294967294") - 1) || !JS7_ISDEC(*s))
         return false;
 
+    // Small indices are by far the most common property keys.  Handle them
+    // without entering the general overflow-checking loop below.
+    if (length == 1) {
+        *indexp = JS7_UNDEC(*s);
+        return true;
+    }
+
+    if (length == 2) {
+        uint32_t first = JS7_UNDEC(s[0]);
+        if (first == 0 || !JS7_ISDEC(s[1]))
+            return false;
+        *indexp = first * 10 + JS7_UNDEC(s[1]);
+        return true;
+    }
+
     uint32_t c = 0, previous = 0;
     uint32_t index = JS7_UNDEC(*s++);
 
@@ -2197,7 +2212,7 @@ ShiftMoveBoxedOrUnboxedDenseElements(JSObject* obj)
     } else {
         uint8_t* data = obj->as<UnboxedArrayObject>().elements();
         size_t elementSize = UnboxedTypeSize(Type);
-        memmove(data, data + elementSize, initlen * elementSize);
+        js_memmove(data, data + elementSize, initlen * elementSize);
     }
 
     return DenseElementResult::Success;

js/src/jsatominlines.h

@@ -8,7 +8,6 @@
 
 #include "jsatom.h"
 
-#include "mozilla/PodOperations.h"
 #include "mozilla/RangedPtr.h"
 
 #include "jscntxt.h"
@@ -177,14 +176,14 @@ AtomHasher::match(const AtomStateEntry& entry, const Lookup& lookup)
     if (key->hasLatin1Chars()) {
         const Latin1Char* keyChars = key->latin1Chars(lookup.nogc);
         if (lookup.isLatin1)
-            return mozilla::PodEqual(keyChars, lookup.latin1Chars, lookup.length);
+            return EqualChars(keyChars, lookup.latin1Chars, lookup.length);
         return EqualChars(keyChars, lookup.twoByteChars, lookup.length);
     }
 
     const char16_t* keyChars = key->twoByteChars(lookup.nogc);
     if (lookup.isLatin1)
         return EqualChars(lookup.latin1Chars, keyChars, lookup.length);
-    return mozilla::PodEqual(keyChars, lookup.twoByteChars, lookup.length);
+    return EqualChars(keyChars, lookup.twoByteChars, lookup.length);
 }
 
 inline Handle<PropertyName*>

js/src/jsfriendapi.h

@@ -12,6 +12,8 @@
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/UniquePtr.h"
 
+#include <string.h>
+
 #include "jsapi.h" // For JSAutoByteString.  See bug 1033916.
 #include "jsbytecode.h"
 #include "jspubtd.h"
@@ -21,6 +23,12 @@
 #include "js/Class.h"
 #include "js/Utility.h"
 
+#if defined(__SSE2__) || defined(_M_X64) || \
+    (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
+# define JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS
+# include <emmintrin.h>
+#endif
+
 #if JS_STACK_GROWTH_DIRECTION > 0
 # define JS_CHECK_STACK_SIZE(limit, sp) (MOZ_LIKELY((uintptr_t)(sp) < (limit)))
 #else
@@ -877,8 +885,21 @@ CopyLinearStringChars(char16_t* dest, JSLinearString* s, size_t len, size_t star
     JS::AutoCheckCannotGC nogc;
     if (LinearStringHasLatin1Chars(s)) {
         const JS::Latin1Char* src = GetLatin1LinearStringChars(nogc, s);
+#if defined(JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS)
+        size_t i = 0;
+        const __m128i zero = _mm_setzero_si128();
+        for (; i + 8 <= len; i += 8) {
+            const __m128i bytes8 = _mm_loadl_epi64(
+                reinterpret_cast<const __m128i*>(src + start + i));
+            _mm_storeu_si128(reinterpret_cast<__m128i*>(dest + i),
+                             _mm_unpacklo_epi8(bytes8, zero));
+        }
+        for (; i < len; i++)
+            dest[i] = src[start + i];
+#else
         for (size_t i = 0; i < len; i++)
             dest[i] = src[start + i];
+#endif
     } else {
         const char16_t* src = GetTwoByteLinearStringChars(nogc, s);
         mozilla::PodCopy(dest, src + start, len);
@@ -892,12 +913,26 @@ CopyLinearStringChars(char* dest, JSLinearString* s, size_t len, size_t start =
     JS::AutoCheckCannotGC nogc;
     if (LinearStringHasLatin1Chars(s)) {
         const JS::Latin1Char* src = GetLatin1LinearStringChars(nogc, s);
-        for (size_t i = 0; i < len; i++)
-           dest[i] = char(src[start + i]);
+        memcpy(dest, src + start, len);
     } else {
       const char16_t* src = GetTwoByteLinearStringChars(nogc, s);
+#if defined(JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS)
+      size_t i = 0;
+      const __m128i lowByteMask = _mm_set1_epi16(0xff);
+      const __m128i zero = _mm_setzero_si128();
+      for (; i + 8 <= len; i += 8) {
+          const __m128i wide = _mm_loadu_si128(
+              reinterpret_cast<const __m128i*>(src + start + i));
+          const __m128i lowBytes = _mm_and_si128(wide, lowByteMask);
+          _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i),
+                           _mm_packus_epi16(lowBytes, zero));
+      }
+      for (; i < len; i++)
+          dest[i] = char(src[start + i]);
+#else
       for (size_t i = 0; i < len; i++)
           dest[i] = char(src[start + i]);
+#endif
     }
 }
 
@@ -3039,4 +3074,8 @@ class MemProfiler
     }
 };
 
+#ifdef JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS
+# undef JS_FRIENDAPI_USE_SSE2_CHARACTER_OPERATIONS
+#endif
+
 #endif /* jsfriendapi_h */

js/src/jsobjinlines.h

@@ -12,6 +12,7 @@
 
 #include "jsfriendapi.h"
 #include "jsfun.h"
+#include "jsutil.h"
 
 #include "builtin/MapObject.h"
 #include "builtin/TypedObject.h"
@@ -401,7 +402,7 @@ JSObject::create(js::ExclusiveContext* cx, js::gc::AllocKind kind, js::gc::Initi
                    kind == js::gc::AllocKind::FUNCTION_EXTENDED);
         size_t size =
             kind == js::gc::AllocKind::FUNCTION ? sizeof(JSFunction) : sizeof(js::FunctionExtended);
-        memset(obj->as<JSFunction>().fixedSlots(), 0, size - sizeof(js::NativeObject));
+        js_memset(obj->as<JSFunction>().fixedSlots(), 0, size - sizeof(js::NativeObject));
         if (kind == js::gc::AllocKind::FUNCTION_EXTENDED) {
             // SetNewObjectMetadata may gc, which will be unhappy if flags &
             // EXTENDED doesn't match the arena's AllocKind.

js/src/jsscript.cpp

@@ -3530,7 +3530,7 @@ js::detail::CopyScript(JSContext* cx, HandleScript src, HandleScript dst,
     dst->dataSize_ = size;
     MOZ_ASSERT(bool(dst->data) == bool(src->data));
     if (dst->data)
-        memcpy(dst->data, src->data, size);
+        js_memcpy(dst->data, src->data, size);
 
     /* Script filenames, bytecodes and atoms are runtime-wide. */
     dst->setScriptData(src->scriptData());

js/src/jsstr.cpp

@@ -68,7 +68,6 @@ using mozilla::IsNegativeZero;
 using mozilla::IsSame;
 using mozilla::Move;
 using mozilla::PodCopy;
-using mozilla::PodEqual;
 using mozilla::RangedPtr;
 
 using JS::AutoCheckCannotGC;
@@ -1085,6 +1084,18 @@ ToUpperCaseLength(const CharT* chars, size_t startIndex, size_t length)
     return upperLength;
 }
 
+static inline void
+CopyChars(char16_t* destChars, const char* srcChars, size_t length)
+{
+    CopyAndInflateChars(destChars, srcChars, length);
+}
+
+static inline void
+CopyChars(char16_t* destChars, const Latin1Char* srcChars, size_t length)
+{
+    CopyAndInflateChars(destChars, srcChars, length);
+}
+
 template <typename DestChar, typename SrcChar>
 static inline void
 CopyChars(DestChar* destChars, const SrcChar* srcChars, size_t length)
@@ -1706,6 +1717,16 @@ template <class InnerMatch, typename TextChar, typename PatChar>
 static int
 Matcher(const TextChar* text, uint32_t textlen, const PatChar* pat, uint32_t patlen)
 {
+    // A Latin-1 string can never contain a UTF-16 code unit above 0xff.  Do
+    // this check once instead of repeatedly testing every candidate position
+    // in the mixed-encoding matcher.  This is particularly useful for search
+    // strings containing supplementary-plane or otherwise non-Latin-1 text.
+    if (sizeof(TextChar) == 1 && sizeof(PatChar) == 2 &&
+        !CharactersFitInLatin1(reinterpret_cast<const char16_t*>(pat), patlen))
+    {
+        return -1;
+    }
+
     const typename InnerMatch::Extent extent = InnerMatch::computeExtent(pat, patlen);
 
     uint32_t i = 0;
@@ -1717,6 +1738,16 @@ Matcher(const TextChar* text, uint32_t textlen, const PatChar* pat, uint32_t pat
             pos = (TextChar*) FirstCharMatcher16bit((char16_t*)text + i, n - i, pat[0]);
         else if (sizeof(TextChar) == 1 && sizeof(PatChar) == 1)
             pos = (TextChar*) FirstCharMatcher8bit((char*) text + i, n - i, pat[0]);
+        else if (sizeof(TextChar) == 1 && sizeof(PatChar) == 2)
+            // The complete pattern was checked above, so this narrowing is
+            // lossless and keeps the other mixed-width direction on SIMD.
+            pos = FindCharacter(text + i, n - i, TextChar(pat[0]));
+        else if (sizeof(TextChar) == 2 && sizeof(PatChar) == 1)
+            // FindCharacter is encoding-independent for the text and keeps
+            // mixed Latin-1/UTF-16 searches on the SSE2 fast path.
+            pos = reinterpret_cast<const TextChar*>(
+                FindCharacter(reinterpret_cast<const char16_t*>(text) + i,
+                              n - i, char16_t(pat[0])));
         else
             pos = (TextChar*) FirstCharMatcherUnrolled<TextChar, PatChar>(text + i, n - i, pat[0]);
 
@@ -1742,9 +1773,9 @@ StringMatch(const TextChar* text, uint32_t textLen, const PatChar* pat, uint32_t
     if (textLen < patLen)
         return -1;
 
-#if defined(__i386__) || defined(_M_IX86) || defined(__i386)
-    // Avoid the generic substring matcher for a single character on x86.
-    // FindCharacter uses SSE2 where available, including mixed encodings.
+#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS
+    // Avoid the generic substring matcher for a single character when the
+    // bounded SSE2 search helper is available, including mixed encodings.
     if (patLen == 1) {
         // A two-byte needle cannot match Latin1 text if it exceeds 0xff.
         if (sizeof(TextChar) == 1 && uint32_t(*pat) > 0xff)
@@ -2162,17 +2193,35 @@ LastIndexOfImpl(const TextChar* text, size_t textLen, const PatChar* pat, size_t
     const PatChar* patNext = pat + 1;
     const PatChar* patEnd = pat + patLen;
 
-    for (const TextChar* t = text + start; t >= text; --t) {
-        if (*t == p0) {
-            const TextChar* t1 = t + 1;
-            for (const PatChar* p1 = patNext; p1 < patEnd; ++p1, ++t1) {
-                if (*t1 != *p1)
-                    goto break_continue;
-            }
-
-            return static_cast<int32_t>(t - text);
+    // Search candidate first characters backwards in SIMD-sized blocks.  The
+    // bounded helper keeps the scan safe at allocation and page boundaries,
+    // while the scalar comparison below still verifies the rest of the
+    // pattern exactly.
+    size_t searchLength = start + 1;
+    while (searchLength) {
+        const TextChar* t;
+        if (sizeof(TextChar) == 1 && sizeof(PatChar) == 2) {
+            if (uint32_t(p0) > 0xff)
+                return -1;
+            t = FindCharacterReverse(text, searchLength, TextChar(p0));
+        } else {
+            t = FindCharacterReverse(text, searchLength, TextChar(p0));
         }
-      break_continue:;
+        if (!t)
+            return -1;
+
+        const TextChar* t1 = t + 1;
+        bool match = true;
+        for (const PatChar* p1 = patNext; p1 < patEnd; ++p1, ++t1) {
+            if (*t1 != *p1) {
+                match = false;
+                break;
+            }
+        }
+
+        if (match)
+            return static_cast<int32_t>(t - text);
+        searchLength = static_cast<size_t>(t - text);
     }
 
     return -1;
@@ -2271,14 +2320,14 @@ js::HasSubstringAt(JSLinearString* text, JSLinearString* pat, size_t start)
     if (text->hasLatin1Chars()) {
         const Latin1Char* textChars = text->latin1Chars(nogc) + start;
         if (pat->hasLatin1Chars())
-            return PodEqual(textChars, pat->latin1Chars(nogc), patLen);
+            return EqualChars(textChars, pat->latin1Chars(nogc), patLen);
 
         return EqualChars(textChars, pat->twoByteChars(nogc), patLen);
     }
 
     const char16_t* textChars = text->twoByteChars(nogc) + start;
     if (pat->hasTwoByteChars())
-        return PodEqual(textChars, pat->twoByteChars(nogc), patLen);
+        return EqualChars(textChars, pat->twoByteChars(nogc), patLen);
 
     return EqualChars(pat->latin1Chars(nogc), textChars, patLen);
 }
@@ -3988,13 +4037,13 @@ js::EqualChars(JSLinearString* str1, JSLinearString* str2)
     AutoCheckCannotGC nogc;
     if (str1->hasTwoByteChars()) {
         if (str2->hasTwoByteChars())
-            return PodEqual(str1->twoByteChars(nogc), str2->twoByteChars(nogc), len);
+            return EqualChars(str1->twoByteChars(nogc), str2->twoByteChars(nogc), len);
 
         return EqualChars(str2->latin1Chars(nogc), str1->twoByteChars(nogc), len);
     }
 
     if (str2->hasLatin1Chars())
-        return PodEqual(str1->latin1Chars(nogc), str2->latin1Chars(nogc), len);
+        return EqualChars(str1->latin1Chars(nogc), str2->latin1Chars(nogc), len);
 
     return EqualChars(str1->latin1Chars(nogc), str2->twoByteChars(nogc), len);
 }
@@ -4110,7 +4159,7 @@ js::StringEqualsAscii(JSLinearString* str, const char* asciiBytes)
 
     AutoCheckCannotGC nogc;
     return str->hasLatin1Chars()
-           ? PodEqual(latin1, str->latin1Chars(nogc), length)
+           ? EqualChars(latin1, str->latin1Chars(nogc), length)
            : EqualChars(latin1, str->twoByteChars(nogc), length);
 }
 
@@ -4207,12 +4256,15 @@ template <typename CharT>
 const CharT*
 js_strchr_limit(const CharT* s, char16_t c, const CharT* limit)
 {
-    while (s < limit) {
-        if (*s == c)
-            return s;
-        s++;
-    }
-    return nullptr;
+    MOZ_ASSERT(limit >= s);
+
+    // A Latin-1 buffer cannot contain a UTF-16 code unit above 0xff.  Apart
+    // from avoiding a scan, this guard is required before narrowing |c| for
+    // the SIMD helper.
+    if (sizeof(CharT) == 1 && c > 0xff)
+        return nullptr;
+
+    return FindCharacter(s, size_t(limit - s), CharT(c));
 }
 
 template const Latin1Char*
@@ -4232,8 +4284,20 @@ js::InflateString(ExclusiveContext* cx, const char* bytes, size_t* lengthp)
     chars = cx->pod_malloc<char16_t>(nchars + 1);
     if (!chars)
         goto bad;
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    size_t i = 0;
+    const __m128i zero = _mm_setzero_si128();
+    for (; i + 8 <= nchars; i += 8) {
+        const __m128i bytes8 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(bytes + i));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(chars + i),
+                         _mm_unpacklo_epi8(bytes8, zero));
+    }
+    for (; i < nchars; i++)
+        chars[i] = (unsigned char) bytes[i];
+#else
     for (size_t i = 0; i < nchars; i++)
         chars[i] = (unsigned char) bytes[i];
+#endif
     *lengthp = nchars;
     chars[nchars] = 0;
     return chars;
@@ -4245,6 +4309,49 @@ js::InflateString(ExclusiveContext* cx, const char* bytes, size_t* lengthp)
     return nullptr;
 }
 
+template <typename CharT>
+static inline void
+DeflateChars(char* dst, const CharT* src, size_t length)
+{
+    static_assert(sizeof(CharT) == 1 || sizeof(CharT) == 2, "character width");
+
+    if (sizeof(CharT) == 1) {
+        memcpy(dst, src, length);
+        return;
+    }
+
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    size_t i = 0;
+    const __m128i lowByteMask = _mm_set1_epi16(0xff);
+    const __m128i zero = _mm_setzero_si128();
+    for (; i + 32 <= length; i += 32) {
+        const __m128i wide0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i));
+        const __m128i wide1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i + 8));
+        const __m128i wide2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i + 16));
+        const __m128i wide3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i + 24));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i),
+                         _mm_packus_epi16(_mm_and_si128(wide0, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i + 8),
+                         _mm_packus_epi16(_mm_and_si128(wide1, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i + 16),
+                         _mm_packus_epi16(_mm_and_si128(wide2, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i + 24),
+                         _mm_packus_epi16(_mm_and_si128(wide3, lowByteMask), zero));
+    }
+    for (; i + 8 <= length; i += 8) {
+        const __m128i wide = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i));
+        const __m128i lowBytes = _mm_and_si128(wide, lowByteMask);
+        const __m128i packed = _mm_packus_epi16(lowBytes, zero);
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + i), packed);
+    }
+    for (; i < length; i++)
+        dst[i] = char(src[i]);
+#else
+    for (size_t i = 0; i < length; i++)
+        dst[i] = char(src[i]);
+#endif
+}
+
 template <typename CharT>
 bool
 js::DeflateStringToBuffer(JSContext* maybecx, const CharT* src, size_t srclen,
@@ -4252,8 +4359,7 @@ js::DeflateStringToBuffer(JSContext* maybecx, const CharT* src, size_t srclen,
 {
     size_t dstlen = *dstlenp;
     if (srclen > dstlen) {
-        for (size_t i = 0; i < dstlen; i++)
-            dst[i] = char(src[i]);
+        DeflateChars(dst, src, dstlen);
         if (maybecx) {
             AutoSuppressGC suppress(maybecx);
             JS_ReportErrorNumberASCII(maybecx, GetErrorMessage, nullptr,
@@ -4261,8 +4367,7 @@ js::DeflateStringToBuffer(JSContext* maybecx, const CharT* src, size_t srclen,
         }
         return false;
     }
-    for (size_t i = 0; i < srclen; i++)
-        dst[i] = char(src[i]);
+    DeflateChars(dst, src, srclen);
     *dstlenp = srclen;
     return true;
 }

js/src/jsstr.h

@@ -53,7 +53,100 @@ template <typename Char1, typename Char2>
 inline int32_t
 CompareChars(const Char1* s1, size_t len1, const Char2* s2, size_t len2)
 {
+    if (mozilla::IsSame<Char1, Char2>::value &&
+        reinterpret_cast<const void*>(s1) == reinterpret_cast<const void*>(s2))
+    {
+        return int32_t(len1 - len2);
+    }
+
     size_t n = Min(len1, len2);
+
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    if (sizeof(Char1) == 1 && sizeof(Char2) == 1) {
+        const uint8_t* left = reinterpret_cast<const uint8_t*>(s1);
+        const uint8_t* right = reinterpret_cast<const uint8_t*>(s2);
+        while (n >= 16) {
+            const __m128i leftBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(left));
+            const __m128i rightBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(right));
+            const uint32_t equalMask = static_cast<uint32_t>(
+                _mm_movemask_epi8(_mm_cmpeq_epi8(leftBlock, rightBlock)));
+            if (equalMask != 0xffff) {
+                const uint32_t lane = mozilla::CountTrailingZeroes32((~equalMask) & 0xffff);
+                return int32_t(left[lane]) - int32_t(right[lane]);
+            }
+            left += 16;
+            right += 16;
+            n -= 16;
+        }
+        s1 = reinterpret_cast<const Char1*>(left);
+        s2 = reinterpret_cast<const Char2*>(right);
+    } else if (sizeof(Char1) == 2 && sizeof(Char2) == 2) {
+        const char16_t* left = reinterpret_cast<const char16_t*>(s1);
+        const char16_t* right = reinterpret_cast<const char16_t*>(s2);
+        while (n >= 8) {
+            const __m128i leftBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(left));
+            const __m128i rightBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(right));
+            const uint32_t equalMask = static_cast<uint32_t>(
+                _mm_movemask_epi8(_mm_cmpeq_epi16(leftBlock, rightBlock)));
+            if (equalMask != 0xffff) {
+                const uint32_t lane = mozilla::CountTrailingZeroes32((~equalMask) & 0xffff) / 2;
+                return int32_t(left[lane]) - int32_t(right[lane]);
+            }
+            left += 8;
+            right += 8;
+            n -= 8;
+        }
+        s1 = reinterpret_cast<const Char1*>(left);
+        s2 = reinterpret_cast<const Char2*>(right);
+    }
+
+    // Find the first differing code unit in eight mixed-width characters at
+    // once.  The scalar result is still used for the first mismatch, so this
+    // preserves CompareChars' ordering semantics rather than merely testing
+    // equality.
+    if (sizeof(Char1) == 1 && sizeof(Char2) == 2) {
+        const uint8_t* bytes = reinterpret_cast<const uint8_t*>(s1);
+        const char16_t* wide = reinterpret_cast<const char16_t*>(s2);
+        const __m128i zero = _mm_setzero_si128();
+        while (n >= 8) {
+            const __m128i byteBlock = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(bytes));
+            const __m128i wideBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(wide));
+            const __m128i expanded = _mm_unpacklo_epi8(byteBlock, zero);
+            const uint32_t equalMask = static_cast<uint32_t>(
+                _mm_movemask_epi8(_mm_cmpeq_epi16(expanded, wideBlock)));
+            if (equalMask != 0xffff) {
+                const uint32_t lane = mozilla::CountTrailingZeroes32((~equalMask) & 0xffff) / 2;
+                return int32_t(bytes[lane]) - int32_t(wide[lane]);
+            }
+            bytes += 8;
+            wide += 8;
+            n -= 8;
+        }
+        s1 = reinterpret_cast<const Char1*>(bytes);
+        s2 = reinterpret_cast<const Char2*>(wide);
+    } else if (sizeof(Char1) == 2 && sizeof(Char2) == 1) {
+        const char16_t* wide = reinterpret_cast<const char16_t*>(s1);
+        const uint8_t* bytes = reinterpret_cast<const uint8_t*>(s2);
+        const __m128i zero = _mm_setzero_si128();
+        while (n >= 8) {
+            const __m128i wideBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(wide));
+            const __m128i byteBlock = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(bytes));
+            const __m128i expanded = _mm_unpacklo_epi8(byteBlock, zero);
+            const uint32_t equalMask = static_cast<uint32_t>(
+                _mm_movemask_epi8(_mm_cmpeq_epi16(wideBlock, expanded)));
+            if (equalMask != 0xffff) {
+                const uint32_t lane = mozilla::CountTrailingZeroes32((~equalMask) & 0xffff) / 2;
+                return int32_t(wide[lane]) - int32_t(bytes[lane]);
+            }
+            wide += 8;
+            bytes += 8;
+            n -= 8;
+        }
+        s1 = reinterpret_cast<const Char1*>(wide);
+        s2 = reinterpret_cast<const Char2*>(bytes);
+    }
+#endif
+
     for (size_t i = 0; i < n; i++) {
         if (int32_t cmp = s1[i] - s2[i])
             return cmp;
@@ -252,6 +345,67 @@ template <typename Char1>
 inline bool
 EqualChars(const Char1* s1, const Char1* s2, size_t len)
 {
+    if (s1 == s2)
+        return true;
+
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    if (sizeof(Char1) == 1) {
+        const uint8_t* left = reinterpret_cast<const uint8_t*>(s1);
+        const uint8_t* right = reinterpret_cast<const uint8_t*>(s2);
+        while (len >= 64) {
+            for (unsigned block = 0; block < 4; block++) {
+                const __m128i leftBlock = _mm_loadu_si128(
+                    reinterpret_cast<const __m128i*>(left + block * 16));
+                const __m128i rightBlock = _mm_loadu_si128(
+                    reinterpret_cast<const __m128i*>(right + block * 16));
+                if (_mm_movemask_epi8(_mm_cmpeq_epi8(leftBlock, rightBlock)) != 0xffff)
+                    return false;
+            }
+            left += 64;
+            right += 64;
+            len -= 64;
+        }
+        while (len >= 16) {
+            const __m128i leftBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(left));
+            const __m128i rightBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(right));
+            if (_mm_movemask_epi8(_mm_cmpeq_epi8(leftBlock, rightBlock)) != 0xffff)
+                return false;
+            left += 16;
+            right += 16;
+            len -= 16;
+        }
+        s1 = reinterpret_cast<const Char1*>(left);
+        s2 = reinterpret_cast<const Char1*>(right);
+    } else if (sizeof(Char1) == 2) {
+        const char16_t* left = reinterpret_cast<const char16_t*>(s1);
+        const char16_t* right = reinterpret_cast<const char16_t*>(s2);
+        while (len >= 32) {
+            for (unsigned block = 0; block < 4; block++) {
+                const __m128i leftBlock = _mm_loadu_si128(
+                    reinterpret_cast<const __m128i*>(left + block * 8));
+                const __m128i rightBlock = _mm_loadu_si128(
+                    reinterpret_cast<const __m128i*>(right + block * 8));
+                if (_mm_movemask_epi8(_mm_cmpeq_epi16(leftBlock, rightBlock)) != 0xffff)
+                    return false;
+            }
+            left += 32;
+            right += 32;
+            len -= 32;
+        }
+        while (len >= 8) {
+            const __m128i leftBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(left));
+            const __m128i rightBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(right));
+            if (_mm_movemask_epi8(_mm_cmpeq_epi16(leftBlock, rightBlock)) != 0xffff)
+                return false;
+            left += 8;
+            right += 8;
+            len -= 8;
+        }
+        s1 = reinterpret_cast<const Char1*>(left);
+        s2 = reinterpret_cast<const Char1*>(right);
+    }
+#endif
+
     return mozilla::PodEqual(s1, s2, len);
 }
 
@@ -259,6 +413,45 @@ template <typename Char1, typename Char2>
 inline bool
 EqualChars(const Char1* s1, const Char2* s2, size_t len)
 {
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    // Compare eight mixed-width characters at a time.  Widening the Latin-1
+    // bytes before comparing also makes values above 0xff fail naturally,
+    // preserving the scalar implementation's semantics.
+    if (sizeof(Char1) == 1 && sizeof(Char2) == 2) {
+        const uint8_t* bytes = reinterpret_cast<const uint8_t*>(s1);
+        const char16_t* wide = reinterpret_cast<const char16_t*>(s2);
+        const __m128i zero = _mm_setzero_si128();
+        while (len >= 8) {
+            const __m128i byteBlock = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(bytes));
+            const __m128i wideBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(wide));
+            const __m128i expanded = _mm_unpacklo_epi8(byteBlock, zero);
+            if (_mm_movemask_epi8(_mm_cmpeq_epi16(expanded, wideBlock)) != 0xffff)
+                return false;
+            bytes += 8;
+            wide += 8;
+            len -= 8;
+        }
+        s1 = reinterpret_cast<const Char1*>(bytes);
+        s2 = reinterpret_cast<const Char2*>(wide);
+    } else if (sizeof(Char1) == 2 && sizeof(Char2) == 1) {
+        const char16_t* wide = reinterpret_cast<const char16_t*>(s1);
+        const uint8_t* bytes = reinterpret_cast<const uint8_t*>(s2);
+        const __m128i zero = _mm_setzero_si128();
+        while (len >= 8) {
+            const __m128i wideBlock = _mm_loadu_si128(reinterpret_cast<const __m128i*>(wide));
+            const __m128i byteBlock = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(bytes));
+            const __m128i expanded = _mm_unpacklo_epi8(byteBlock, zero);
+            if (_mm_movemask_epi8(_mm_cmpeq_epi16(wideBlock, expanded)) != 0xffff)
+                return false;
+            wide += 8;
+            bytes += 8;
+            len -= 8;
+        }
+        s1 = reinterpret_cast<const Char1*>(wide);
+        s2 = reinterpret_cast<const Char2*>(bytes);
+    }
+#endif
+
     for (const Char1* s1end = s1 + len; s1 < s1end; s1++, s2++) {
         if (*s1 != *s2)
             return false;
@@ -290,15 +483,81 @@ InflateString(ExclusiveContext* cx, const char* bytes, size_t* length);
 inline void
 CopyAndInflateChars(char16_t* dst, const char* src, size_t srclen)
 {
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    size_t i = 0;
+    const __m128i zero = _mm_setzero_si128();
+    for (; i + 32 <= srclen; i += 32) {
+        uint64_t value0, value1, value2, value3;
+        js_memcpy(&value0, src + i, sizeof(value0));
+        js_memcpy(&value1, src + i + 8, sizeof(value1));
+        js_memcpy(&value2, src + i + 16, sizeof(value2));
+        js_memcpy(&value3, src + i + 24, sizeof(value3));
+        const __m128i bytes0 = _mm_cvtsi64_si128(static_cast<int64_t>(value0));
+        const __m128i bytes1 = _mm_cvtsi64_si128(static_cast<int64_t>(value1));
+        const __m128i bytes2 = _mm_cvtsi64_si128(static_cast<int64_t>(value2));
+        const __m128i bytes3 = _mm_cvtsi64_si128(static_cast<int64_t>(value3));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i),
+                         _mm_unpacklo_epi8(bytes0, zero));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 8),
+                         _mm_unpacklo_epi8(bytes1, zero));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16),
+                         _mm_unpacklo_epi8(bytes2, zero));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 24),
+                         _mm_unpacklo_epi8(bytes3, zero));
+    }
+    for (; i + 8 <= srclen; i += 8) {
+        uint64_t value;
+        js_memcpy(&value, src + i, sizeof(value));
+        const __m128i bytes8 = _mm_cvtsi64_si128(static_cast<int64_t>(value));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i),
+                         _mm_unpacklo_epi8(bytes8, zero));
+    }
+    for (; i < srclen; i++)
+        dst[i] = (unsigned char) src[i];
+#else
     for (size_t i = 0; i < srclen; i++)
         dst[i] = (unsigned char) src[i];
+#endif
 }
 
 inline void
 CopyAndInflateChars(char16_t* dst, const JS::Latin1Char* src, size_t srclen)
 {
+#if defined(JS_HAVE_SSE2_INTRINSICS)
+    size_t i = 0;
+    const __m128i zero = _mm_setzero_si128();
+    for (; i + 32 <= srclen; i += 32) {
+        uint64_t value0, value1, value2, value3;
+        js_memcpy(&value0, src + i, sizeof(value0));
+        js_memcpy(&value1, src + i + 8, sizeof(value1));
+        js_memcpy(&value2, src + i + 16, sizeof(value2));
+        js_memcpy(&value3, src + i + 24, sizeof(value3));
+        const __m128i bytes0 = _mm_cvtsi64_si128(static_cast<int64_t>(value0));
+        const __m128i bytes1 = _mm_cvtsi64_si128(static_cast<int64_t>(value1));
+        const __m128i bytes2 = _mm_cvtsi64_si128(static_cast<int64_t>(value2));
+        const __m128i bytes3 = _mm_cvtsi64_si128(static_cast<int64_t>(value3));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i),
+                         _mm_unpacklo_epi8(bytes0, zero));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 8),
+                         _mm_unpacklo_epi8(bytes1, zero));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16),
+                         _mm_unpacklo_epi8(bytes2, zero));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 24),
+                         _mm_unpacklo_epi8(bytes3, zero));
+    }
+    for (; i + 8 <= srclen; i += 8) {
+        uint64_t value;
+        js_memcpy(&value, src + i, sizeof(value));
+        const __m128i bytes8 = _mm_cvtsi64_si128(static_cast<int64_t>(value));
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i),
+                         _mm_unpacklo_epi8(bytes8, zero));
+    }
+    for (; i < srclen; i++)
+        dst[i] = src[i];
+#else
     for (size_t i = 0; i < srclen; i++)
         dst[i] = src[i];
+#endif
 }
 
 /*

js/src/jsutil.h

@@ -113,6 +113,21 @@ js_memmove(void* dst_, const void* src_, size_t len)
 
         d += len;
         s += len;
+        while (len >= 64) {
+            d -= 64;
+            s -= 64;
+            // Load the complete chunk before storing it: the ranges may
+            // overlap, so an early store must not destroy a later load.
+            __m128i v0 = _mm_loadu_si128((const __m128i*)(s + 0));
+            __m128i v1 = _mm_loadu_si128((const __m128i*)(s + 16));
+            __m128i v2 = _mm_loadu_si128((const __m128i*)(s + 32));
+            __m128i v3 = _mm_loadu_si128((const __m128i*)(s + 48));
+            _mm_storeu_si128((__m128i*)(d + 0), v0);
+            _mm_storeu_si128((__m128i*)(d + 16), v1);
+            _mm_storeu_si128((__m128i*)(d + 32), v2);
+            _mm_storeu_si128((__m128i*)(d + 48), v3);
+            len -= 64;
+        }
         while (len >= 16) {
             d -= 16;
             s -= 16;

js/src/vm/ArgumentsObject.cpp

@@ -11,6 +11,7 @@
 #include "vm/AsyncFunction.h"
 #include "vm/GlobalObject.h"
 #include "vm/Stack.h"
+#include "jsutil.h"
 
 #include "jsobjinlines.h"
 
@@ -36,7 +37,7 @@ RareArgumentsData::create(JSContext* cx, ArgumentsObject* obj)
     if (!data)
         return nullptr;
 
-    mozilla::PodZero(data, bytes);
+    js_memset(data, 0, bytes);
 
     return new(data) RareArgumentsData();
 }
@@ -299,7 +300,7 @@ ArgumentsObject::create(JSContext* cx, HandleFunction callee, unsigned numActual
 
         // Zero the argument Values. This sets each value to DoubleValue(0), which
         // is safe for GC tracing.
-        memset(data->args, 0, numArgs * sizeof(Value));
+        js_memset(data->args, 0, numArgs * sizeof(Value));
         MOZ_ASSERT(DoubleValue(0).asRawBits() == 0x0);
         MOZ_ASSERT_IF(numArgs > 0, data->args[0].asRawBits() == 0x0);
 
@@ -815,7 +816,7 @@ ArgumentsObject::objectMovedDuringMinorGC(JSTracer* trc, JSObject* dst, JSObject
             oomUnsafe.crash("Failed to allocate ArgumentsObject data while tenuring.");
         ndst->initFixedSlot(DATA_SLOT, PrivateValue(data));
 
-        mozilla::PodCopy(data, reinterpret_cast<uint8_t*>(nsrc->data()), nbytes);
+        js_memcpy(data, reinterpret_cast<uint8_t*>(nsrc->data()), nbytes);
         nbytesTotal += nbytes;
     }
 
@@ -830,7 +831,7 @@ ArgumentsObject::objectMovedDuringMinorGC(JSTracer* trc, JSObject* dst, JSObject
                 oomUnsafe.crash("Failed to allocate RareArgumentsData data while tenuring.");
             ndst->data()->rareData = (RareArgumentsData*)dstRareData;
 
-            mozilla::PodCopy(dstRareData, reinterpret_cast<uint8_t*>(srcRareData), nbytes);
+            js_memcpy(dstRareData, reinterpret_cast<uint8_t*>(srcRareData), nbytes);
             nbytesTotal += nbytes;
         }
     }

js/src/vm/ArrayBufferObject.cpp

@@ -656,7 +656,7 @@ ResizeArrayBuffer(JSContext* cx, Handle<ArrayBufferObject*> buffer, uint32_t new
 
     uint32_t copyLength = std::min(newByteLength, buffer->byteLength());
     if (copyLength > 0)
-        memcpy(newContents.data(), buffer->dataPointer(), copyLength);
+        js_memcpy(newContents.data(), buffer->dataPointer(), copyLength);
 
     buffer->changeContentsForResize(cx, newContents, ArrayBufferObject::OwnsData, newByteLength);
     return true;
@@ -729,7 +729,7 @@ ArrayBufferTransfer(JSContext* cx, const CallArgs& args, bool preserveResizabili
 
     uint32_t copyLength = std::min(newByteLength, buffer->byteLength());
     if (copyLength > 0)
-        memcpy(newBuffer->dataPointer(), buffer->dataPointer(), copyLength);
+        js_memcpy(newBuffer->dataPointer(), buffer->dataPointer(), copyLength);
 
     ArrayBufferObject::BufferContents detachedContents =
         buffer->hasStealableContents() ? ArrayBufferObject::BufferContents::createPlain(nullptr)
@@ -1428,7 +1428,7 @@ ArrayBufferObject::create(JSContext* cx, uint32_t nbytes, BufferContents content
 
     if (!contents) {
         void* data = obj->inlineDataPointer();
-        memset(data, 0, nbytes);
+        js_memset(data, 0, nbytes);
         obj->initialize(nbytes, BufferContents::createPlain(data), DoesntOwnData,
                         maxByteLength, resizable);
     } else {

js/src/vm/CharacterEncoding.cpp

@@ -9,10 +9,12 @@
 #include "mozilla/Sprintf.h"
 
 #include <algorithm>
+#include <string.h>
 #include <type_traits>
 
 #include "jscntxt.h"
 #include "jsprf.h"
+#include "vm/CharacterOperations.h"
 
 using namespace js;
 
@@ -25,8 +27,40 @@ JS::LossyTwoByteCharsToNewLatin1CharsZ(js::ExclusiveContext* cx,
     unsigned char* latin1 = cx->pod_malloc<unsigned char>(len + 1);
     if (!latin1)
         return Latin1CharsZ();
+#if defined(JS_HAS_SSE2_CHARACTER_OPERATIONS)
+    size_t i = 0;
+    const __m128i lowByteMask = _mm_set1_epi16(0xff);
+    const __m128i zero = _mm_setzero_si128();
+    for (; i + 32 <= len; i += 32) {
+        const __m128i wide0 = _mm_loadu_si128(
+            reinterpret_cast<const __m128i*>(tbchars.begin().get() + i));
+        const __m128i wide1 = _mm_loadu_si128(
+            reinterpret_cast<const __m128i*>(tbchars.begin().get() + i + 8));
+        const __m128i wide2 = _mm_loadu_si128(
+            reinterpret_cast<const __m128i*>(tbchars.begin().get() + i + 16));
+        const __m128i wide3 = _mm_loadu_si128(
+            reinterpret_cast<const __m128i*>(tbchars.begin().get() + i + 24));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i),
+                         _mm_packus_epi16(_mm_and_si128(wide0, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i + 8),
+                         _mm_packus_epi16(_mm_and_si128(wide1, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i + 16),
+                         _mm_packus_epi16(_mm_and_si128(wide2, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i + 24),
+                         _mm_packus_epi16(_mm_and_si128(wide3, lowByteMask), zero));
+    }
+    for (; i + 8 <= len; i += 8) {
+        const __m128i wide = _mm_loadu_si128(reinterpret_cast<const __m128i*>(tbchars.begin().get() + i));
+        const __m128i lowBytes = _mm_and_si128(wide, lowByteMask);
+        const __m128i packed = _mm_packus_epi16(lowBytes, zero);
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(latin1 + i), packed);
+    }
+    for (; i < len; ++i)
+        latin1[i] = static_cast<unsigned char>(tbchars[i]);
+#else
     for (size_t i = 0; i < len; ++i)
         latin1[i] = static_cast<unsigned char>(tbchars[i]);
+#endif
     latin1[len] = '\0';
     return Latin1CharsZ(latin1, len);
 }
@@ -423,8 +457,43 @@ InflateUTF8StringHelper(ContextT* cx, const UTF8Chars src, size_t* outlen)
     if (encoding == JS::SmallestEncoding::ASCII) {
         size_t srclen = src.length();
         MOZ_ASSERT(*outlen == srclen);
-        for (uint32_t i = 0; i < srclen; i++)
-            dst[i] = CharT(src[i]);
+        if (sizeof(CharT) == 1) {
+            memcpy(dst, src.begin().get(), srclen);
+        } else {
+#if defined(JS_HAS_SSE2_CHARACTER_OPERATIONS)
+            size_t i = 0;
+            const __m128i zero = _mm_setzero_si128();
+            for (; i + 32 <= srclen; i += 32) {
+                const __m128i bytes0 = _mm_loadl_epi64(
+                    reinterpret_cast<const __m128i*>(src.begin().get() + i));
+                const __m128i bytes1 = _mm_loadl_epi64(
+                    reinterpret_cast<const __m128i*>(src.begin().get() + i + 8));
+                const __m128i bytes2 = _mm_loadl_epi64(
+                    reinterpret_cast<const __m128i*>(src.begin().get() + i + 16));
+                const __m128i bytes3 = _mm_loadl_epi64(
+                    reinterpret_cast<const __m128i*>(src.begin().get() + i + 24));
+                _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i),
+                                 _mm_unpacklo_epi8(bytes0, zero));
+                _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 8),
+                                 _mm_unpacklo_epi8(bytes1, zero));
+                _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16),
+                                 _mm_unpacklo_epi8(bytes2, zero));
+                _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 24),
+                                 _mm_unpacklo_epi8(bytes3, zero));
+            }
+            for (; i + 8 <= srclen; i += 8) {
+                const __m128i bytes8 = _mm_loadl_epi64(
+                    reinterpret_cast<const __m128i*>(src.begin().get() + i));
+                _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i),
+                                 _mm_unpacklo_epi8(bytes8, zero));
+            }
+            for (; i < srclen; i++)
+                dst[i] = CharT(src[i]);
+#else
+            for (size_t i = 0; i < srclen; i++)
+                dst[i] = CharT(src[i]);
+#endif
+        }
     } else {
         MOZ_ALWAYS_TRUE((InflateUTF8StringToBuffer<Copy, CharT>(cx, src, dst, outlen, &encoding)));
     }

js/src/vm/CharacterOperations.h

@@ -31,7 +31,47 @@ FindCharacter(const CharT* chars, size_t length, CharT match)
         const __m128i needle = sizeof(CharT) == 1
                               ? _mm_set1_epi8(static_cast<char>(match))
                               : _mm_set1_epi16(static_cast<short>(match));
-        do {
+        while (length >= 4 * lanes) {
+            const __m128i block0 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(chars));
+            const uint32_t mask0 = static_cast<uint32_t>(
+                _mm_movemask_epi8(sizeof(CharT) == 1
+                                  ? _mm_cmpeq_epi8(block0, needle)
+                                  : _mm_cmpeq_epi16(block0, needle)));
+            if (mask0)
+                return chars + mozilla::CountTrailingZeroes32(mask0) / sizeof(CharT);
+
+            const __m128i block1 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(chars + lanes));
+            const uint32_t mask1 = static_cast<uint32_t>(
+                _mm_movemask_epi8(sizeof(CharT) == 1
+                                  ? _mm_cmpeq_epi8(block1, needle)
+                                  : _mm_cmpeq_epi16(block1, needle)));
+            if (mask1)
+                return chars + lanes + mozilla::CountTrailingZeroes32(mask1) / sizeof(CharT);
+
+            const __m128i block2 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(chars + 2 * lanes));
+            const uint32_t mask2 = static_cast<uint32_t>(
+                _mm_movemask_epi8(sizeof(CharT) == 1
+                                  ? _mm_cmpeq_epi8(block2, needle)
+                                  : _mm_cmpeq_epi16(block2, needle)));
+            if (mask2)
+                return chars + 2 * lanes + mozilla::CountTrailingZeroes32(mask2) / sizeof(CharT);
+
+            const __m128i block3 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(chars + 3 * lanes));
+            const uint32_t mask3 = static_cast<uint32_t>(
+                _mm_movemask_epi8(sizeof(CharT) == 1
+                                  ? _mm_cmpeq_epi8(block3, needle)
+                                  : _mm_cmpeq_epi16(block3, needle)));
+            if (mask3)
+                return chars + 3 * lanes + mozilla::CountTrailingZeroes32(mask3) / sizeof(CharT);
+
+            chars += 4 * lanes;
+            length -= 4 * lanes;
+        }
+        while (length >= lanes) {
             // Never read beyond the supplied span, even at a page boundary.
             const __m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(chars));
             const __m128i equal = sizeof(CharT) == 1
@@ -42,7 +82,7 @@ FindCharacter(const CharT* chars, size_t length, CharT match)
                 return chars + mozilla::CountTrailingZeroes32(mask) / sizeof(CharT);
             chars += lanes;
             length -= lanes;
-        } while (length >= lanes);
+        }
     }
 #endif
     for (; length; --length, ++chars) {
@@ -52,6 +92,86 @@ FindCharacter(const CharT* chars, size_t length, CharT match)
     return nullptr;
 }
 
+template <typename CharT>
+inline const CharT*
+FindCharacterReverse(const CharT* chars, size_t length, CharT match)
+{
+    static_assert(sizeof(CharT) == 1 || sizeof(CharT) == 2, "character width");
+#ifdef JS_HAS_SSE2_CHARACTER_OPERATIONS
+    const size_t lanes = 16 / sizeof(CharT);
+    const CharT* end = chars + length;
+    if (length >= lanes) {
+        const __m128i needle = sizeof(CharT) == 1
+                              ? _mm_set1_epi8(static_cast<char>(match))
+                              : _mm_set1_epi16(static_cast<short>(match));
+        while (length >= 4 * lanes) {
+            end -= 4 * lanes;
+            length -= 4 * lanes;
+
+            const __m128i block3 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(end + 3 * lanes));
+            const uint32_t mask3 = static_cast<uint32_t>(
+                _mm_movemask_epi8(sizeof(CharT) == 1
+                                  ? _mm_cmpeq_epi8(block3, needle)
+                                  : _mm_cmpeq_epi16(block3, needle)));
+            if (mask3)
+                return end + 3 * lanes +
+                       (31 - mozilla::CountLeadingZeroes32(mask3)) / sizeof(CharT);
+
+            const __m128i block2 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(end + 2 * lanes));
+            const uint32_t mask2 = static_cast<uint32_t>(
+                _mm_movemask_epi8(sizeof(CharT) == 1
+                                  ? _mm_cmpeq_epi8(block2, needle)
+                                  : _mm_cmpeq_epi16(block2, needle)));
+            if (mask2)
+                return end + 2 * lanes +
+                       (31 - mozilla::CountLeadingZeroes32(mask2)) / sizeof(CharT);
+
+            const __m128i block1 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(end + lanes));
+            const uint32_t mask1 = static_cast<uint32_t>(
+                _mm_movemask_epi8(sizeof(CharT) == 1
+                                  ? _mm_cmpeq_epi8(block1, needle)
+                                  : _mm_cmpeq_epi16(block1, needle)));
+            if (mask1)
+                return end + lanes +
+                       (31 - mozilla::CountLeadingZeroes32(mask1)) / sizeof(CharT);
+
+            const __m128i block0 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(end));
+            const uint32_t mask0 = static_cast<uint32_t>(
+                _mm_movemask_epi8(sizeof(CharT) == 1
+                                  ? _mm_cmpeq_epi8(block0, needle)
+                                  : _mm_cmpeq_epi16(block0, needle)));
+            if (mask0)
+                return end + (31 - mozilla::CountLeadingZeroes32(mask0)) / sizeof(CharT);
+        }
+        while (length >= lanes) {
+            end -= lanes;
+            length -= lanes;
+            const __m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(end));
+            const __m128i equal = sizeof(CharT) == 1
+                                 ? _mm_cmpeq_epi8(block, needle)
+                                 : _mm_cmpeq_epi16(block, needle);
+            const uint32_t mask = static_cast<uint32_t>(_mm_movemask_epi8(equal));
+            if (mask)
+                return end + (31 - mozilla::CountLeadingZeroes32(mask)) / sizeof(CharT);
+        }
+    }
+#else
+    const CharT* end = chars + length;
+#endif
+
+    while (length) {
+        --end;
+        --length;
+        if (*end == match)
+            return end;
+    }
+    return nullptr;
+}
+
 inline bool
 CharactersFitInLatin1(const char16_t* chars, size_t length)
 {
@@ -59,6 +179,34 @@ CharactersFitInLatin1(const char16_t* chars, size_t length)
     if (length >= 8) {
         const __m128i highBytes = _mm_set1_epi16(static_cast<short>(0xff00));
         const __m128i zero = _mm_setzero_si128();
+        while (length >= 32) {
+            const __m128i block0 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(chars));
+            if (_mm_movemask_epi8(_mm_cmpeq_epi16(
+                    _mm_and_si128(block0, highBytes), zero)) != 0xffff)
+                return false;
+
+            const __m128i block1 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(chars + 8));
+            if (_mm_movemask_epi8(_mm_cmpeq_epi16(
+                    _mm_and_si128(block1, highBytes), zero)) != 0xffff)
+                return false;
+
+            const __m128i block2 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(chars + 16));
+            if (_mm_movemask_epi8(_mm_cmpeq_epi16(
+                    _mm_and_si128(block2, highBytes), zero)) != 0xffff)
+                return false;
+
+            const __m128i block3 = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(chars + 24));
+            if (_mm_movemask_epi8(_mm_cmpeq_epi16(
+                    _mm_and_si128(block3, highBytes), zero)) != 0xffff)
+                return false;
+
+            chars += 32;
+            length -= 32;
+        }
         do {
             const __m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(chars));
             const __m128i fits = _mm_cmpeq_epi16(_mm_and_si128(block, highBytes), zero);

js/src/vm/Interpreter.cpp

@@ -31,6 +31,7 @@
 #include "jsprf.h"
 #include "jsscript.h"
 #include "jsstr.h"
+#include "jsutil.h"
 
 #include "builtin/Eval.h"
 #include "builtin/ModuleObject.h"
@@ -2147,7 +2148,7 @@ CASE(JSOP_PICK)
     unsigned i = GET_UINT8(REGS.pc);
     MOZ_ASSERT(REGS.stackDepth() >= i + 1);
     Value lval = REGS.sp[-int(i + 1)];
-    memmove(REGS.sp - (i + 1), REGS.sp - i, sizeof(Value) * i);
+    js_memmove(REGS.sp - (i + 1), REGS.sp - i, sizeof(Value) * i);
     REGS.sp[-1] = lval;
 }
 END_CASE(JSOP_PICK)
@@ -2157,7 +2158,7 @@ CASE(JSOP_UNPICK)
     int i = GET_UINT8(REGS.pc);
     MOZ_ASSERT(REGS.stackDepth() >= unsigned(i) + 1);
     Value lval = REGS.sp[-1];
-    memmove(REGS.sp - i, REGS.sp - (i + 1), sizeof(Value) * i);
+    js_memmove(REGS.sp - i, REGS.sp - (i + 1), sizeof(Value) * i);
     REGS.sp[-(i + 1)] = lval;
 }
 END_CASE(JSOP_UNPICK)

js/src/vm/NativeObject.h

@@ -13,6 +13,7 @@
 
 #include "jsfriendapi.h"
 #include "jsobj.h"
+#include "jsutil.h"
 #include "NamespaceImports.h"
 
 #include "gc/Barrier.h"
@@ -1122,8 +1123,8 @@ class NativeObject : public ShapedObject
             for (uint32_t i = 0; i < count; ++i)
                 elements_[dstStart + i].set(this, HeapSlot::Element, dstStart + i, src[i]);
         } else {
-            memcpy(reinterpret_cast<Value*>(&elements_[dstStart]), src,
-                   count * sizeof(Value));
+            js_memcpy(reinterpret_cast<Value*>(&elements_[dstStart]), src,
+                      count * sizeof(Value));
             elementsRangeWriteBarrierPost(dstStart, count);
         }
     }
@@ -1132,7 +1133,7 @@ class NativeObject : public ShapedObject
         MOZ_ASSERT(dstStart + count <= getDenseCapacity());
         MOZ_ASSERT(!denseElementsAreCopyOnWrite());
         MOZ_ASSERT(!denseElementsAreFrozen());
-        memcpy(reinterpret_cast<Value*>(&elements_[dstStart]), src, count * sizeof(Value));
+        js_memcpy(reinterpret_cast<Value*>(&elements_[dstStart]), src, count * sizeof(Value));
         elementsRangeWriteBarrierPost(dstStart, count);
     }
 
@@ -1167,7 +1168,7 @@ class NativeObject : public ShapedObject
                     dst->set(this, HeapSlot::Element, dst - elements_, *src);
             }
         } else {
-            memmove(elements_ + dstStart, elements_ + srcStart, count * sizeof(HeapSlot));
+            js_memmove(elements_ + dstStart, elements_ + srcStart, count * sizeof(HeapSlot));
             elementsRangeWriteBarrierPost(dstStart, count);
         }
     }
@@ -1180,7 +1181,7 @@ class NativeObject : public ShapedObject
         MOZ_ASSERT(!denseElementsAreCopyOnWrite());
         MOZ_ASSERT(!denseElementsAreFrozen());
 
-        memmove(elements_ + dstStart, elements_ + srcStart, count * sizeof(HeapSlot));
+        js_memmove(elements_ + dstStart, elements_ + srcStart, count * sizeof(HeapSlot));
         elementsRangeWriteBarrierPost(dstStart, count);
     }
 

js/src/vm/String-inl.h

@@ -8,11 +8,11 @@
 
 #include "vm/String.h"
 
-#include "mozilla/PodOperations.h"
 #include "mozilla/Range.h"
 
 #include "jscntxt.h"
 #include "jscompartment.h"
+#include "jsutil.h"
 
 #include "gc/Allocator.h"
 #include "gc/Marking.h"
@@ -57,7 +57,7 @@ NewInlineString(ExclusiveContext* cx, mozilla::Range<const CharT> chars)
     if (!str)
         return nullptr;
 
-    mozilla::PodCopy(storage, chars.begin().get(), len);
+    js_memcpy(storage, chars.begin().get(), len);
     storage[len] = 0;
     return str;
 }
@@ -75,7 +75,7 @@ NewInlineString(ExclusiveContext* cx, HandleLinearString base, size_t start, siz
         return nullptr;
 
     JS::AutoCheckCannotGC nogc;
-    mozilla::PodCopy(chars, base->chars<CharT>(nogc) + start, length);
+    js_memcpy(chars, base->chars<CharT>(nogc) + start, length * sizeof(CharT));
     chars[length] = 0;
     return s;
 }

js/src/vm/String.cpp

@@ -7,7 +7,6 @@
 
 #include "mozilla/MathAlgorithms.h"
 #include "mozilla/MemoryReporting.h"
-#include "mozilla/PodOperations.h"
 #include "mozilla/RangedPtr.h"
 #include "mozilla/SizePrintfMacros.h"
 #include "mozilla/TypeTraits.h"
@@ -20,11 +19,11 @@
 
 #include "jscntxtinlines.h"
 #include "jscompartmentinlines.h"
+#include "jsutil.h"
 
 using namespace js;
 
 using mozilla::IsSame;
-using mozilla::PodCopy;
 using mozilla::RangedPtr;
 using mozilla::RoundUpPow2;
 
@@ -346,7 +345,7 @@ CopyChars(char16_t* dest, const JSLinearString& str)
 {
     AutoCheckCannotGC nogc;
     if (str.hasTwoByteChars())
-        PodCopy(dest, str.twoByteChars(nogc), str.length());
+        js_memcpy(dest, str.twoByteChars(nogc), str.length() * sizeof(char16_t));
     else
         CopyAndInflateChars(dest, str.latin1Chars(nogc), str.length());
 }
@@ -357,7 +356,7 @@ CopyChars(Latin1Char* dest, const JSLinearString& str)
 {
     AutoCheckCannotGC nogc;
     if (str.hasLatin1Chars()) {
-        PodCopy(dest, str.latin1Chars(nogc), str.length());
+        js_memcpy(dest, str.latin1Chars(nogc), str.length());
     } else {
         /*
          * When we flatten a TwoByte rope, we turn child ropes (including Latin1
@@ -369,10 +368,24 @@ CopyChars(Latin1Char* dest, const JSLinearString& str)
          */
         size_t len = str.length();
         const char16_t* chars = str.twoByteChars(nogc);
+#if defined(JS_HAS_SSE2_CHARACTER_OPERATIONS)
+        size_t i = 0;
+        const __m128i zero = _mm_setzero_si128();
+        for (; i + 8 <= len; i += 8) {
+            const __m128i wide = _mm_loadu_si128(reinterpret_cast<const __m128i*>(chars + i));
+            const __m128i packed = _mm_packus_epi16(wide, zero);
+            _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i), packed);
+        }
+        for (; i < len; i++) {
+            MOZ_ASSERT(chars[i] <= JSString::MAX_LATIN1_CHAR);
+            dest[i] = chars[i];
+        }
+#else
         for (size_t i = 0; i < len; i++) {
             MOZ_ASSERT(chars[i] <= JSString::MAX_LATIN1_CHAR);
             dest[i] = chars[i];
         }
+#endif
     }
 }
 
@@ -639,16 +652,17 @@ js::ConcatStrings(ExclusiveContext* cx,
             return nullptr;
 
         if (isLatin1) {
-            PodCopy(latin1Buf, leftLinear->latin1Chars(nogc), leftLen);
-            PodCopy(latin1Buf + leftLen, rightLinear->latin1Chars(nogc), rightLen);
+            js_memcpy(latin1Buf, leftLinear->latin1Chars(nogc), leftLen);
+            js_memcpy(latin1Buf + leftLen, rightLinear->latin1Chars(nogc), rightLen);
             latin1Buf[wholeLength] = 0;
         } else {
             if (leftLinear->hasTwoByteChars())
-                PodCopy(twoByteBuf, leftLinear->twoByteChars(nogc), leftLen);
+                js_memcpy(twoByteBuf, leftLinear->twoByteChars(nogc), leftLen * sizeof(char16_t));
             else
                 CopyAndInflateChars(twoByteBuf, leftLinear->latin1Chars(nogc), leftLen);
             if (rightLinear->hasTwoByteChars())
-                PodCopy(twoByteBuf + leftLen, rightLinear->twoByteChars(nogc), rightLen);
+                js_memcpy(twoByteBuf + leftLen, rightLinear->twoByteChars(nogc),
+                          rightLen * sizeof(char16_t));
             else
                 CopyAndInflateChars(twoByteBuf + leftLen, rightLinear->latin1Chars(nogc), rightLen);
             twoByteBuf[wholeLength] = 0;
@@ -676,7 +690,7 @@ JSDependentString::undependInternal(JSContext* cx)
         return nullptr;
 
     AutoCheckCannotGC nogc;
-    PodCopy(s, nonInlineChars<CharT>(nogc), n);
+    js_memcpy(s, nonInlineChars<CharT>(nogc), n * sizeof(CharT));
     s[n] = '\0';
     setNonInlineChars<CharT>(s);
 
@@ -1026,7 +1040,7 @@ AutoStableStringChars::copyLatin1Chars(JSContext* cx, HandleLinearString linearS
     if (!chars)
         return false;
 
-    PodCopy(chars, linearString->rawLatin1Chars(), length);
+    js_memcpy(chars, linearString->rawLatin1Chars(), length);
     chars[length] = 0;
 
     state_ = Latin1;
@@ -1043,7 +1057,7 @@ AutoStableStringChars::copyTwoByteChars(JSContext* cx, HandleLinearString linear
     if (!chars)
         return false;
 
-    PodCopy(chars, linearString->rawTwoByteChars(), length);
+    js_memcpy(chars, linearString->rawTwoByteChars(), length * sizeof(char16_t));
     chars[length] = 0;
 
     state_ = TwoByte;
@@ -1077,7 +1091,7 @@ JSExternalString::ensureFlat(JSContext* cx)
     // Copy the chars before finalizing the string.
     {
         AutoCheckCannotGC nogc;
-        PodCopy(s, nonInlineChars<char16_t>(nogc), n);
+        js_memcpy(s, nonInlineChars<char16_t>(nogc), n * sizeof(char16_t));
         s[n] = '\0';
     }
 
@@ -1158,6 +1172,41 @@ CanStoreCharsAsLatin1(const Latin1Char* s, size_t length)
     MOZ_CRASH("Shouldn't be called for Latin1 chars");
 }
 
+static MOZ_ALWAYS_INLINE void
+CopyAndDeflateLatin1Chars(Latin1Char* dest, const char16_t* src, size_t length)
+{
+#if defined(JS_HAS_SSE2_CHARACTER_OPERATIONS)
+    size_t i = 0;
+    const __m128i lowByteMask = _mm_set1_epi16(0xff);
+    const __m128i zero = _mm_setzero_si128();
+    for (; i + 32 <= length; i += 32) {
+        const __m128i wide0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i));
+        const __m128i wide1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i + 8));
+        const __m128i wide2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i + 16));
+        const __m128i wide3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i + 24));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i),
+                         _mm_packus_epi16(_mm_and_si128(wide0, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i + 8),
+                         _mm_packus_epi16(_mm_and_si128(wide1, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i + 16),
+                         _mm_packus_epi16(_mm_and_si128(wide2, lowByteMask), zero));
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i + 24),
+                         _mm_packus_epi16(_mm_and_si128(wide3, lowByteMask), zero));
+    }
+    for (; i + 8 <= length; i += 8) {
+        const __m128i wide = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + i));
+        const __m128i lowBytes = _mm_and_si128(wide, lowByteMask);
+        _mm_storel_epi64(reinterpret_cast<__m128i*>(dest + i),
+                         _mm_packus_epi16(lowBytes, zero));
+    }
+    for (; i < length; i++)
+        dest[i] = Latin1Char(src[i]);
+#else
+    for (size_t i = 0; i < length; i++)
+        dest[i] = Latin1Char(src[i]);
+#endif
+}
+
 template <AllowGC allowGC>
 static MOZ_ALWAYS_INLINE JSInlineString*
 NewInlineStringDeflated(ExclusiveContext* cx, mozilla::Range<const char16_t> chars)
@@ -1170,8 +1219,8 @@ NewInlineStringDeflated(ExclusiveContext* cx, mozilla::Range<const char16_t> cha
 
     for (size_t i = 0; i < len; i++) {
         MOZ_ASSERT(chars[i] <= JSString::MAX_LATIN1_CHAR);
-        storage[i] = Latin1Char(chars[i]);
     }
+    CopyAndDeflateLatin1Chars(storage, chars.begin().get(), len);
     storage[len] = '\0';
     return str;
 }
@@ -1210,8 +1259,8 @@ NewStringDeflated(ExclusiveContext* cx, const char16_t* s, size_t n)
 
     for (size_t i = 0; i < n; i++) {
         MOZ_ASSERT(s[i] <= JSString::MAX_LATIN1_CHAR);
-        news.get()[i] = Latin1Char(s[i]);
     }
+    CopyAndDeflateLatin1Chars(news.get(), s, n);
     news[n] = '\0';
 
     JSFlatString* str = JSFlatString::new_<allowGC>(cx, news.get(), n);
@@ -1313,7 +1362,7 @@ NewStringCopyNDontDeflate(ExclusiveContext* cx, const CharT* s, size_t n)
         return nullptr;
     }
 
-    PodCopy(news.get(), s, n);
+    js_memcpy(news.get(), s, n * sizeof(CharT));
     news[n] = 0;
 
     JSFlatString* str = JSFlatString::new_<allowGC>(cx, news.get(), n);

js/src/vm/TypedArrayObject.cpp

@@ -3413,6 +3413,22 @@ js::StringIsTypedArrayIndex(const CharT* s, size_t length, uint64_t* indexp)
 
     index = digit;
 
+    // Most typed-array accesses use one- or two-digit indices.  Once the
+    // digits have been validated, these forms cannot overflow uint64_t and
+    // need no general-purpose accumulation loop.
+    if (s == end) {
+        *indexp = negative ? UINT64_MAX : index;
+        return true;
+    }
+
+    if (end - s == 1) {
+        if (!JS7_ISDEC(*s))
+            return false;
+        digit = JS7_UNDEC(*s);
+        *indexp = negative ? UINT64_MAX : index * 10 + digit;
+        return true;
+    }
+
     for (; s < end; s++) {
         if (!JS7_ISDEC(*s))
             return false;

js/src/vm/UnboxedObject-inl.h

@@ -583,9 +583,9 @@ MoveBoxedOrUnboxedDenseElements(JSContext* cx, JSObject* obj, uint32_t dstStart,
                 obj->as<UnboxedArrayObject>().triggerPreBarrier<Type>(dstStart + i);
         }
 
-        memmove(data + dstStart * elementSize,
-                data + srcStart * elementSize,
-                length * elementSize);
+        js_memmove(data + dstStart * elementSize,
+                   data + srcStart * elementSize,
+                   length * elementSize);
     }
 
     return DenseElementResult::Success;
@@ -619,9 +619,9 @@ CopyBoxedOrUnboxedDenseElements(JSContext* cx, JSObject* dst, JSObject* src,
         uint8_t* srcData = src->as<UnboxedArrayObject>().elements();
         size_t elementSize = UnboxedTypeSize(DstType);
 
-        memcpy(dstData + dstStart * elementSize,
-               srcData + srcStart * elementSize,
-               length * elementSize);
+        js_memcpy(dstData + dstStart * elementSize,
+                  srcData + srcStart * elementSize,
+                  length * elementSize);
 
         // Add a store buffer entry if we might have copied a nursery pointer to dst.
         if (UnboxedTypeNeedsPostBarrier(DstType) && !IsInsideNursery(dst))

js/src/vm/UnboxedObject.cpp

@@ -1895,7 +1895,7 @@ UnboxedPlainObject::fillAfterConvert(ExclusiveContext* cx,
                                      Handle<GCVector<Value>> values, size_t* valueCursor)
 {
     initExpando();
-    memset(data(), 0, layout().size());
+    js_memset(data(), 0, layout().size());
     for (size_t i = 0; i < layout().properties().length(); i++)
         JS_ALWAYS_TRUE(setValue(cx, layout().properties()[i], NextValue(values, valueCursor)));
 }

mfbt/HashFunctions.cpp

@@ -20,7 +20,17 @@ HashBytes(const void* aBytes, size_t aLength)
 
   /* Walk word by word. */
   size_t i = 0;
-  for (; i < aLength - (aLength % sizeof(size_t)); i += sizeof(size_t)) {
+  const size_t wordLength = aLength - (aLength % sizeof(size_t));
+  const size_t doubleWordLength = wordLength - (wordLength % (2 * sizeof(size_t)));
+  for (; i < doubleWordLength; i += 2 * sizeof(size_t)) {
+    size_t data0;
+    size_t data1;
+    memcpy(&data0, b + i, sizeof(data0));
+    memcpy(&data1, b + i + sizeof(data0), sizeof(data1));
+    hash = AddToHash(hash, data0, sizeof(data0));
+    hash = AddToHash(hash, data1, sizeof(data1));
+  }
+  for (; i < wordLength; i += sizeof(size_t)) {
     /* Do an explicitly unaligned load of the data. */
     size_t data;
     memcpy(&data, b + i, sizeof(size_t));

mfbt/HashFunctions.h

@@ -230,8 +230,17 @@ uint32_t
 HashKnownLength(const T* aStr, size_t aLength)
 {
   uint32_t hash = 0;
-  for (size_t i = 0; i < aLength; i++) {
-    hash = AddToHash(hash, aStr[i]);
+  while (aLength >= 4) {
+    hash = AddToHash(hash, aStr[0]);
+    hash = AddToHash(hash, aStr[1]);
+    hash = AddToHash(hash, aStr[2]);
+    hash = AddToHash(hash, aStr[3]);
+    aStr += 4;
+    aLength -= 4;
+  }
+  while (aLength) {
+    hash = AddToHash(hash, *aStr++);
+    --aLength;
   }
   return hash;
 }

toolkit/content/aboutSupport.js

@@ -303,6 +303,17 @@ var snapshotFormatters = {
                      ? data.windowLayerManagerType
                      : "BasicLayers (" + strings.GetStringFromName("mainThreadNoOMTC") + ")";
     addRow("features", "compositing", compositor);
+
+    let webRenderEnabled = false;
+    try {
+      webRenderEnabled = Services.prefs.getBoolPref("gfx.webrender.enabled");
+    } catch (e) {
+      // Keep about:support usable if the preference is unavailable in a build.
+    }
+    addRow("features", "webRender",
+           strings.GetStringFromName(webRenderEnabled
+                                     ? "webRenderEnabled"
+                                     : "webRenderDisabled"));
     
     let acceleratedWindows = data.numAcceleratedWindows + "/" + data.numTotalWindows;
     if (data.windowLayerManagerType) {

toolkit/locales/en-US/chrome/global/aboutSupport.properties

@@ -58,6 +58,9 @@ blockedMismatchedVersion = Blocked for your graphics driver version mismatch bet
 clearTypeParameters = ClearType Parameters
 
 compositing = Compositing
+webRender = WebRender
+webRenderEnabled = Enabled
+webRenderDisabled = Disabled
 hardwareH264 = Hardware H264 Decoding
 hardwareVP9 = Hardware VP9 Decoding
 audioBackend = Audio Backend