59 GC.

59 GC.

js/src/gc/AllocKind.h

@@ -0,0 +1,215 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal definition of GC cell kinds.
+ */
+
+#ifndef gc_AllocKind_h
+#define gc_AllocKind_h
+
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/EnumeratedArray.h"
+#include "mozilla/EnumeratedRange.h"
+
+#include <stdint.h>
+
+#include "js/TraceKind.h"
+
+namespace js {
+namespace gc {
+
+// The GC allocation kinds.
+//
+// These are defined by macros which enumerate the different allocation kinds
+// and supply the following information:
+//
+//  - the corresponding AllocKind
+//  - their JS::TraceKind
+//  - their C++ base type
+//  - a C++ type of the correct size
+//  - whether they can be finalized on the background thread
+//  - whether they can be allocated in the nursery
+
+#define FOR_EACH_OBJECT_ALLOCKIND(D) \
+ /* AllocKind              TraceKind     TypeName           SizedType          BGFinal Nursery */ \
+    D(FUNCTION,            Object,       JSObject,          JSFunction,        true,   true)  \
+    D(FUNCTION_EXTENDED,   Object,       JSObject,          FunctionExtended,  true,   true)  \
+    D(OBJECT0,             Object,       JSObject,          JSObject_Slots0,   false,  false) \
+    D(OBJECT0_BACKGROUND,  Object,       JSObject,          JSObject_Slots0,   true,   true)  \
+    D(OBJECT2,             Object,       JSObject,          JSObject_Slots2,   false,  false) \
+    D(OBJECT2_BACKGROUND,  Object,       JSObject,          JSObject_Slots2,   true,   true)  \
+    D(OBJECT4,             Object,       JSObject,          JSObject_Slots4,   false,  false) \
+    D(OBJECT4_BACKGROUND,  Object,       JSObject,          JSObject_Slots4,   true,   true)  \
+    D(OBJECT8,             Object,       JSObject,          JSObject_Slots8,   false,  false) \
+    D(OBJECT8_BACKGROUND,  Object,       JSObject,          JSObject_Slots8,   true,   true)  \
+    D(OBJECT12,            Object,       JSObject,          JSObject_Slots12,  false,  false) \
+    D(OBJECT12_BACKGROUND, Object,       JSObject,          JSObject_Slots12,  true,   true)  \
+    D(OBJECT16,            Object,       JSObject,          JSObject_Slots16,  false,  false) \
+    D(OBJECT16_BACKGROUND, Object,       JSObject,          JSObject_Slots16,  true,   true)
+
+#define FOR_EACH_NONOBJECT_ALLOCKIND(D) \
+ /* AllocKind              TraceKind     TypeName           SizedType          BGFinal Nursery */ \
+    D(SCRIPT,              Script,       JSScript,          JSScript,          false,  false) \
+    D(LAZY_SCRIPT,         LazyScript,   js::LazyScript,    js::LazyScript,    true,   false) \
+    D(SHAPE,               Shape,        js::Shape,         js::Shape,         true,   false) \
+    D(ACCESSOR_SHAPE,      Shape,        js::AccessorShape, js::AccessorShape, true,   false) \
+    D(BASE_SHAPE,          BaseShape,    js::BaseShape,     js::BaseShape,     true,   false) \
+    D(OBJECT_GROUP,        ObjectGroup,  js::ObjectGroup,   js::ObjectGroup,   true,   false) \
+    D(FAT_INLINE_STRING,   String,       JSFatInlineString, JSFatInlineString, true,   false) \
+    D(STRING,              String,       JSString,          JSString,          true,   false) \
+    D(EXTERNAL_STRING,     String,       JSExternalString,  JSExternalString,  true,   false) \
+    D(FAT_INLINE_ATOM,     String,       js::FatInlineAtom, js::FatInlineAtom, true,   false) \
+    D(ATOM,                String,       js::NormalAtom,    js::NormalAtom,    true,   false) \
+    D(SYMBOL,              Symbol,       JS::Symbol,        JS::Symbol,        true,   false) \
+    D(JITCODE,             JitCode,      js::jit::JitCode,  js::jit::JitCode,  false,  false) \
+    D(SCOPE,               Scope,        js::Scope,         js::Scope,         true,   false) \
+    D(REGEXP_SHARED,       RegExpShared, js::RegExpShared,  js::RegExpShared,  true,   false)
+
+#define FOR_EACH_ALLOCKIND(D) \
+    FOR_EACH_OBJECT_ALLOCKIND(D) \
+    FOR_EACH_NONOBJECT_ALLOCKIND(D)
+
+enum class AllocKind : uint8_t {
+#define DEFINE_ALLOC_KIND(allocKind, _1, _2, _3, _4, _5) allocKind,
+
+    FOR_EACH_OBJECT_ALLOCKIND(DEFINE_ALLOC_KIND)
+
+    OBJECT_LIMIT,
+    OBJECT_LAST = OBJECT_LIMIT - 1,
+
+    FOR_EACH_NONOBJECT_ALLOCKIND(DEFINE_ALLOC_KIND)
+
+    LIMIT,
+    LAST = LIMIT - 1,
+
+    FIRST = 0,
+    OBJECT_FIRST = FUNCTION // Hardcoded to first object kind.
+
+#undef DEFINE_ALLOC_KIND
+};
+
+static_assert(int(AllocKind::FIRST) == 0,
+              "Various places depend on AllocKind starting at 0");
+static_assert(int(AllocKind::OBJECT_FIRST) == 0,
+              "OBJECT_FIRST must be defined as the first object kind");
+
+inline bool
+IsAllocKind(AllocKind kind)
+{
+    return kind >= AllocKind::FIRST && kind <= AllocKind::LIMIT;
+}
+
+inline bool
+IsValidAllocKind(AllocKind kind)
+{
+    return kind >= AllocKind::FIRST && kind <= AllocKind::LAST;
+}
+
+inline bool
+IsObjectAllocKind(AllocKind kind)
+{
+    return kind >= AllocKind::OBJECT_FIRST && kind <= AllocKind::OBJECT_LAST;
+}
+
+inline bool
+IsShapeAllocKind(AllocKind kind)
+{
+    return kind == AllocKind::SHAPE || kind == AllocKind::ACCESSOR_SHAPE;
+}
+
+// Returns a sequence for use in a range-based for loop,
+// to iterate over all alloc kinds.
+inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT))
+AllAllocKinds()
+{
+    return mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT);
+}
+
+// Returns a sequence for use in a range-based for loop,
+// to iterate over all object alloc kinds.
+inline decltype(mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT))
+ObjectAllocKinds()
+{
+    return mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT);
+}
+
+// Returns a sequence for use in a range-based for loop,
+// to iterate over alloc kinds from |first| to |limit|, exclusive.
+inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT))
+SomeAllocKinds(AllocKind first = AllocKind::FIRST, AllocKind limit = AllocKind::LIMIT)
+{
+    MOZ_ASSERT(IsAllocKind(first), "|first| is not a valid AllocKind!");
+    MOZ_ASSERT(IsAllocKind(limit), "|limit| is not a valid AllocKind!");
+    return mozilla::MakeEnumeratedRange(first, limit);
+}
+
+// AllAllocKindArray<ValueType> gives an enumerated array of ValueTypes,
+// with each index corresponding to a particular alloc kind.
+template<typename ValueType> using AllAllocKindArray =
+    mozilla::EnumeratedArray<AllocKind, AllocKind::LIMIT, ValueType>;
+
+// ObjectAllocKindArray<ValueType> gives an enumerated array of ValueTypes,
+// with each index corresponding to a particular object alloc kind.
+template<typename ValueType> using ObjectAllocKindArray =
+    mozilla::EnumeratedArray<AllocKind, AllocKind::OBJECT_LIMIT, ValueType>;
+
+static inline JS::TraceKind
+MapAllocToTraceKind(AllocKind kind)
+{
+    static const JS::TraceKind map[] = {
+#define EXPAND_ELEMENT(allocKind, traceKind, type, sizedType, bgFinal, nursery) \
+        JS::TraceKind::traceKind,
+FOR_EACH_ALLOCKIND(EXPAND_ELEMENT)
+#undef EXPAND_ELEMENT
+    };
+
+    static_assert(MOZ_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT),
+                  "AllocKind-to-TraceKind mapping must be in sync");
+    return map[size_t(kind)];
+}
+
+/*
+ * This must be an upper bound, but we do not need the least upper bound, so
+ * we just exclude non-background objects.
+ */
+static const size_t MAX_BACKGROUND_FINALIZE_KINDS =
+    size_t(AllocKind::LIMIT) - size_t(AllocKind::OBJECT_LIMIT) / 2;
+
+static inline bool
+IsNurseryAllocable(AllocKind kind)
+{
+    MOZ_ASSERT(IsValidAllocKind(kind));
+
+    static const bool map[] = {
+#define DEFINE_NURSERY_ALLOCABLE(_1, _2, _3, _4, _5, nursery) nursery,
+        FOR_EACH_ALLOCKIND(DEFINE_NURSERY_ALLOCABLE)
+#undef DEFINE_NURSERY_ALLOCABLE
+    };
+
+    JS_STATIC_ASSERT(JS_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT));
+    return map[size_t(kind)];
+}
+
+static inline bool
+IsBackgroundFinalized(AllocKind kind)
+{
+    MOZ_ASSERT(IsValidAllocKind(kind));
+
+    static const bool map[] = {
+#define DEFINE_BACKGROUND_FINALIZED(_1, _2, _3, _4, bgFinal, _5) bgFinal,
+        FOR_EACH_ALLOCKIND(DEFINE_BACKGROUND_FINALIZED)
+#undef DEFINE_BG_FINALIZE
+    };
+
+    JS_STATIC_ASSERT(JS_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT));
+    return map[size_t(kind)];
+}
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_AllocKind_h */

js/src/gc/Allocator.cpp

@@ -17,6 +17,7 @@
 
 #include "jsobjinlines.h"
 
+#include "gc/ArenaList-inl.h"
 #include "gc/Heap-inl.h"
 
 using namespace js;
@@ -147,9 +148,9 @@ js::Allocate(ExclusiveContext* cx)
     return GCRuntime::tryNewTenuredThing<T, allowGC>(cx, kind, thingSize);
 }
 
-#define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType) \
-    template type* js::Allocate<type, NoGC>(ExclusiveContext* cx);\
-    template type* js::Allocate<type, CanGC>(ExclusiveContext* cx);
+#define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType, bgFinal, nursery) \
+    template type* js::Allocate<type, NoGC>(JSContext* cx);\
+    template type* js::Allocate<type, CanGC>(JSContext* cx);
 FOR_EACH_NONOBJECT_ALLOCKIND(DECL_ALLOCATOR_INSTANCES)
 #undef DECL_ALLOCATOR_INSTANCES
 
@@ -295,7 +296,7 @@ GCRuntime::refillFreeListFromMainThread(JSContext* cx, AllocKind thingKind, size
     Zone *zone = cx->zone();
     MOZ_ASSERT(!cx->runtime()->isHeapBusy(), "allocating while under GC");
 
-    return cx->arenas()->allocateFromArena(zone, thingKind, CheckThresholds);
+    return cx->arenas()->allocateFromArena(zone, thingKind, ShouldCheckThresholds::CheckThresholds);
 }
 
 /* static */ TenuredCell*
@@ -306,7 +307,7 @@ GCRuntime::refillFreeListOffMainThread(ExclusiveContext* cx, AllocKind thingKind
     Zone* zone = cx->zone();
     MOZ_ASSERT(!zone->wasGCStarted());
 
-    return cx->arenas()->allocateFromArena(zone, thingKind, CheckThresholds);
+    return cx->arenas()->allocateFromArena(zone, thingKind, ShouldCheckThresholds::CheckThresholds);
 }
 
 /* static */ TenuredCell*
@@ -321,7 +322,7 @@ GCRuntime::refillFreeListInGC(Zone* zone, AllocKind thingKind)
     MOZ_ASSERT(rt->isHeapCollecting());
     MOZ_ASSERT_IF(!rt->isHeapMinorCollecting(), !rt->gc.isBackgroundSweeping());
 
-    return zone->arenas.allocateFromArena(zone, thingKind, DontCheckThresholds);
+    return zone->arenas.allocateFromArena(zone, thingKind, ShouldCheckThresholds::DontCheckThresholds);
 }
 
 TenuredCell*
@@ -413,14 +414,15 @@ GCRuntime::allocateArena(Chunk* chunk, Zone* zone, AllocKind thingKind,
     MOZ_ASSERT(chunk->hasAvailableArenas());
 
     // Fail the allocation if we are over our heap size limits.
-    if (checkThresholds && usage.gcBytes() >= tunables.gcMaxBytes())
+    if ((checkThresholds != ShouldCheckThresholds::DontCheckThresholds) &&
+        (usage.gcBytes() >= tunables.gcMaxBytes()))
         return nullptr;
 
     Arena* arena = chunk->allocateArena(rt, zone, thingKind, lock);
     zone->usage.addGCArena();
 
     // Trigger an incremental slice if needed.
-    if (checkThresholds)
+    if (checkThresholds != ShouldCheckThresholds::DontCheckThresholds)
         maybeAllocTriggerZoneGC(zone, lock, ArenaSize);
 
     return arena;

js/src/gc/Allocator.h

@@ -1,4 +1,5 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@@ -10,6 +11,7 @@
 #include "js/RootingAPI.h"
 
 namespace js {
+
 struct Class;
 
 // Allocate a new GC thing. After a successful allocation the caller must

js/src/gc/ArenaList-inl.h

@@ -0,0 +1,353 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_ArenaList_inl_h
+#define gc_ArenaList_inl_h
+
+#include "gc/ArenaList.h"
+
+#include "gc/Heap.h"
+
+void
+js::gc::SortedArenaListSegment::append(Arena* arena)
+{
+    MOZ_ASSERT(arena);
+    MOZ_ASSERT_IF(head, head->getAllocKind() == arena->getAllocKind());
+    *tailp = arena;
+    tailp = &arena->next;
+}
+
+inline
+js::gc::ArenaList::ArenaList()
+{
+    clear();
+}
+
+void
+js::gc::ArenaList::copy(const ArenaList& other)
+{
+    other.check();
+    head_ = other.head_;
+    cursorp_ = other.isCursorAtHead() ? &head_ : other.cursorp_;
+    check();
+}
+
+inline
+js::gc::ArenaList::ArenaList(const ArenaList& other)
+{
+    copy(other);
+}
+
+js::gc::ArenaList&
+js::gc::ArenaList::operator=(const ArenaList& other)
+{
+    copy(other);
+    return *this;
+}
+
+inline
+js::gc::ArenaList::ArenaList(const SortedArenaListSegment& segment)
+{
+    head_ = segment.head;
+    cursorp_ = segment.isEmpty() ? &head_ : segment.tailp;
+    check();
+}
+
+// This does checking just of |head_| and |cursorp_|.
+void
+js::gc::ArenaList::check() const
+{
+#ifdef DEBUG
+    // If the list is empty, it must have this form.
+    MOZ_ASSERT_IF(!head_, cursorp_ == &head_);
+
+    // If there's an arena following the cursor, it must not be full.
+    Arena* cursor = *cursorp_;
+    MOZ_ASSERT_IF(cursor, cursor->hasFreeThings());
+#endif
+}
+
+void
+js::gc::ArenaList::clear()
+{
+    head_ = nullptr;
+    cursorp_ = &head_;
+    check();
+}
+
+js::gc::ArenaList
+js::gc::ArenaList::copyAndClear()
+{
+    ArenaList result = *this;
+    clear();
+    return result;
+}
+
+bool
+js::gc::ArenaList::isEmpty() const
+{
+    check();
+    return !head_;
+}
+
+js::gc::Arena*
+js::gc::ArenaList::head() const
+{
+    check();
+    return head_;
+}
+
+bool
+js::gc::ArenaList::isCursorAtHead() const
+{
+    check();
+    return cursorp_ == &head_;
+}
+
+bool
+js::gc::ArenaList::isCursorAtEnd() const
+{
+    check();
+    return !*cursorp_;
+}
+
+void
+js::gc::ArenaList::moveCursorToEnd()
+{
+    while (!isCursorAtEnd())
+        cursorp_ = &(*cursorp_)->next;
+}
+
+js::gc::Arena*
+js::gc::ArenaList::arenaAfterCursor() const
+{
+    check();
+    return *cursorp_;
+}
+
+js::gc::Arena*
+js::gc::ArenaList::takeNextArena()
+{
+    check();
+    Arena* arena = *cursorp_;
+    if (!arena)
+        return nullptr;
+    cursorp_ = &arena->next;
+    check();
+    return arena;
+}
+
+void
+js::gc::ArenaList::insertAtCursor(Arena* a)
+{
+    check();
+    a->next = *cursorp_;
+    *cursorp_ = a;
+    // At this point, the cursor is sitting before |a|. Move it after |a|
+    // if necessary.
+    if (!a->hasFreeThings())
+        cursorp_ = &a->next;
+    check();
+}
+
+void
+js::gc::ArenaList::insertBeforeCursor(Arena* a)
+{
+    check();
+    a->next = *cursorp_;
+    *cursorp_ = a;
+    cursorp_ = &a->next;
+    check();
+}
+
+js::gc::ArenaList&
+js::gc::ArenaList::insertListWithCursorAtEnd(const ArenaList& other)
+{
+    check();
+    other.check();
+    MOZ_ASSERT(other.isCursorAtEnd());
+    if (other.isCursorAtHead())
+        return *this;
+    // Insert the full arenas of |other| after those of |this|.
+    *other.cursorp_ = *cursorp_;
+    *cursorp_ = other.head_;
+    cursorp_ = other.cursorp_;
+    check();
+    return *this;
+}
+
+js::gc::SortedArenaList::SortedArenaList(size_t thingsPerArena)
+{
+    reset(thingsPerArena);
+}
+
+void
+js::gc::SortedArenaList::setThingsPerArena(size_t thingsPerArena)
+{
+    MOZ_ASSERT(thingsPerArena && thingsPerArena <= MaxThingsPerArena);
+    thingsPerArena_ = thingsPerArena;
+}
+
+void
+js::gc::SortedArenaList::reset(size_t thingsPerArena)
+{
+    setThingsPerArena(thingsPerArena);
+    // Initialize the segments.
+    for (size_t i = 0; i <= thingsPerArena; ++i)
+        segments[i].clear();
+}
+
+void
+js::gc::SortedArenaList::insertAt(Arena* arena, size_t nfree)
+{
+    MOZ_ASSERT(nfree <= thingsPerArena_);
+    segments[nfree].append(arena);
+}
+
+void
+js::gc::SortedArenaList::extractEmpty(Arena** empty)
+{
+    SortedArenaListSegment& segment = segments[thingsPerArena_];
+    if (segment.head) {
+        *segment.tailp = *empty;
+        *empty = segment.head;
+        segment.clear();
+    }
+}
+
+js::gc::ArenaList
+js::gc::SortedArenaList::toArenaList()
+{
+    // Link the non-empty segment tails up to the non-empty segment heads.
+    size_t tailIndex = 0;
+    for (size_t headIndex = 1; headIndex <= thingsPerArena_; ++headIndex) {
+        if (headAt(headIndex)) {
+            segments[tailIndex].linkTo(headAt(headIndex));
+            tailIndex = headIndex;
+        }
+    }
+    // Point the tail of the final non-empty segment at null. Note that if
+    // the list is empty, this will just set segments[0].head to null.
+    segments[tailIndex].linkTo(nullptr);
+    // Create an ArenaList with head and cursor set to the head and tail of
+    // the first segment (if that segment is empty, only the head is used).
+    return ArenaList(segments[0]);
+}
+
+js::gc::Arena*
+js::gc::ArenaLists::getFirstArena(AllocKind thingKind) const
+{
+    return arenaLists(thingKind).head();
+}
+
+js::gc::Arena*
+js::gc::ArenaLists::getFirstArenaToSweep(AllocKind thingKind) const
+{
+    return arenaListsToSweep(thingKind);
+}
+
+js::gc::Arena*
+js::gc::ArenaLists::getFirstSweptArena(AllocKind thingKind) const
+{
+    if (thingKind != incrementalSweptArenaKind.ref())
+        return nullptr;
+    return incrementalSweptArenas.ref().head();
+}
+
+js::gc::Arena*
+js::gc::ArenaLists::getArenaAfterCursor(AllocKind thingKind) const
+{
+    return arenaLists(thingKind).arenaAfterCursor();
+}
+
+bool
+js::gc::ArenaLists::arenaListsAreEmpty() const
+{
+    for (auto i : AllAllocKinds()) {
+        /*
+         * The arena cannot be empty if the background finalization is not yet
+         * done.
+         */
+        if (backgroundFinalizeState(i) != BFS_DONE)
+            return false;
+        if (!arenaLists(i).isEmpty())
+            return false;
+    }
+    return true;
+}
+
+void
+js::gc::ArenaLists::unmarkAll()
+{
+    for (auto i : AllAllocKinds()) {
+        /* The background finalization must have stopped at this point. */
+        MOZ_ASSERT(backgroundFinalizeState(i) == BFS_DONE);
+        for (Arena* arena = arenaLists(i).head(); arena; arena = arena->next)
+            arena->unmarkAll();
+    }
+}
+
+bool
+js::gc::ArenaLists::doneBackgroundFinalize(AllocKind kind) const
+{
+    return backgroundFinalizeState(kind) == BFS_DONE;
+}
+
+bool
+js::gc::ArenaLists::needBackgroundFinalizeWait(AllocKind kind) const
+{
+    return backgroundFinalizeState(kind) != BFS_DONE;
+}
+
+void
+js::gc::ArenaLists::purge()
+{
+    for (auto i : AllAllocKinds())
+        freeLists(i) = &placeholder;
+}
+
+bool
+js::gc::ArenaLists::arenaIsInUse(Arena* arena, AllocKind kind) const
+{
+    MOZ_ASSERT(arena);
+    return arena == freeLists(kind)->getArenaUnchecked();
+}
+
+MOZ_ALWAYS_INLINE js::gc::TenuredCell*
+js::gc::ArenaLists::allocateFromFreeList(AllocKind thingKind, size_t thingSize)
+{
+    return freeLists(thingKind)->allocate(thingSize);
+}
+
+void
+js::gc::ArenaLists::checkEmptyFreeLists()
+{
+#ifdef DEBUG
+    for (auto i : AllAllocKinds())
+        checkEmptyFreeList(i);
+#endif
+}
+
+bool
+js::gc::ArenaLists::checkEmptyArenaLists()
+{
+    bool empty = true;
+#ifdef DEBUG
+    for (auto i : AllAllocKinds()) {
+        if (!checkEmptyArenaList(i))
+            empty = false;
+    }
+#endif
+    return empty;
+}
+
+void
+js::gc::ArenaLists::checkEmptyFreeList(AllocKind kind)
+{
+    MOZ_ASSERT(freeLists(kind)->isEmpty());
+}
+
+#endif // gc_ArenaList_inl_h

js/src/gc/ArenaList.h

@@ -0,0 +1,358 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal definitions of ArenaList and associated heap data structures.
+ */
+
+#ifndef gc_ArenaList_h
+#define gc_ArenaList_h
+
+#include "gc/AllocKind.h"
+#include "js/SliceBudget.h"
+#include "threading/ProtectedData.h"
+
+namespace JS {
+
+struct Zone;
+
+} /* namespace JS */
+
+namespace js {
+
+class FreeOp;
+class Nursery;
+class TenuringTracer;
+
+namespace gcstats {
+struct Statistics;
+}
+
+namespace gc {
+
+class Arena;
+struct FinalizePhase;
+class FreeSpan;
+class TenuredCell;
+
+/*
+ * A single segment of a SortedArenaList. Each segment has a head and a tail,
+ * which track the start and end of a segment for O(1) append and concatenation.
+ */
+struct SortedArenaListSegment
+{
+    Arena* head;
+    Arena** tailp;
+
+    void clear() {
+        head = nullptr;
+        tailp = &head;
+    }
+
+    bool isEmpty() const {
+        return tailp == &head;
+    }
+
+    // Appends |arena| to this segment.
+    inline void append(Arena* arena);
+
+    // Points the tail of this segment at |arena|, which may be null. Note
+    // that this does not change the tail itself, but merely which arena
+    // follows it. This essentially turns the tail into a cursor (see also the
+    // description of ArenaList), but from the perspective of a SortedArenaList
+    // this makes no difference.
+    void linkTo(Arena* arena) {
+        *tailp = arena;
+    }
+};
+
+/*
+ * Arena lists have a head and a cursor. The cursor conceptually lies on arena
+ * boundaries, i.e. before the first arena, between two arenas, or after the
+ * last arena.
+ *
+ * Arenas are usually sorted in order of increasing free space, with the cursor
+ * following the Arena currently being allocated from. This ordering should not
+ * be treated as an invariant, however, as the free lists may be cleared,
+ * leaving arenas previously used for allocation partially full. Sorting order
+ * is restored during sweeping.
+
+ * Arenas following the cursor should not be full.
+ */
+class ArenaList {
+    // The cursor is implemented via an indirect pointer, |cursorp_|, to allow
+    // for efficient list insertion at the cursor point and other list
+    // manipulations.
+    //
+    // - If the list is empty: |head| is null, |cursorp_| points to |head|, and
+    //   therefore |*cursorp_| is null.
+    //
+    // - If the list is not empty: |head| is non-null, and...
+    //
+    //   - If the cursor is at the start of the list: |cursorp_| points to
+    //     |head|, and therefore |*cursorp_| points to the first arena.
+    //
+    //   - If cursor is at the end of the list: |cursorp_| points to the |next|
+    //     field of the last arena, and therefore |*cursorp_| is null.
+    //
+    //   - If the cursor is at neither the start nor the end of the list:
+    //     |cursorp_| points to the |next| field of the arena preceding the
+    //     cursor, and therefore |*cursorp_| points to the arena following the
+    //     cursor.
+    //
+    // |cursorp_| is never null.
+    //
+    Arena* head_;
+    Arena** cursorp_;
+
+    inline void copy(const ArenaList& other);
+
+  public:
+    inline ArenaList();
+    inline ArenaList(const ArenaList& other);
+
+    inline ArenaList& operator=(const ArenaList& other);
+
+    inline explicit ArenaList(const SortedArenaListSegment& segment);
+
+    inline void check() const;
+
+    inline void clear();
+    inline ArenaList copyAndClear();
+    inline bool isEmpty() const;
+
+    // This returns nullptr if the list is empty.
+    inline Arena* head() const;
+
+    inline bool isCursorAtHead() const;
+    inline bool isCursorAtEnd() const;
+
+    inline void moveCursorToEnd();
+
+    // This can return nullptr.
+    inline Arena* arenaAfterCursor() const;
+
+    // This returns the arena after the cursor and moves the cursor past it.
+    inline Arena* takeNextArena();
+
+    // This does two things.
+    // - Inserts |a| at the cursor.
+    // - Leaves the cursor sitting just before |a|, if |a| is not full, or just
+    //   after |a|, if |a| is full.
+    inline void insertAtCursor(Arena* a);
+
+    // Inserts |a| at the cursor, then moves the cursor past it.
+    inline void insertBeforeCursor(Arena* a);
+
+    // This inserts |other|, which must be full, at the cursor of |this|.
+    inline ArenaList& insertListWithCursorAtEnd(const ArenaList& other);
+
+    Arena* removeRemainingArenas(Arena** arenap);
+    Arena** pickArenasToRelocate(size_t& arenaTotalOut, size_t& relocTotalOut);
+    Arena* relocateArenas(Arena* toRelocate, Arena* relocated,
+                          js::SliceBudget& sliceBudget, gcstats::Statistics& stats);
+};
+
+/*
+ * A class that holds arenas in sorted order by appending arenas to specific
+ * segments. Each segment has a head and a tail, which can be linked up to
+ * other segments to create a contiguous ArenaList.
+ */
+class SortedArenaList
+{
+  public:
+    // The minimum size, in bytes, of a GC thing.
+    static const size_t MinThingSize = 16;
+
+    static_assert(ArenaSize <= 4096, "When increasing the Arena size, please consider how"\
+                                     " this will affect the size of a SortedArenaList.");
+
+    static_assert(MinThingSize >= 16, "When decreasing the minimum thing size, please consider"\
+                                      " how this will affect the size of a SortedArenaList.");
+
+  private:
+    // The maximum number of GC things that an arena can hold.
+    static const size_t MaxThingsPerArena = (ArenaSize - ArenaHeaderSize) / MinThingSize;
+
+    size_t thingsPerArena_;
+    SortedArenaListSegment segments[MaxThingsPerArena + 1];
+
+    // Convenience functions to get the nth head and tail.
+    Arena* headAt(size_t n) { return segments[n].head; }
+    Arena** tailAt(size_t n) { return segments[n].tailp; }
+
+  public:
+    inline explicit SortedArenaList(size_t thingsPerArena = MaxThingsPerArena);
+
+    inline void setThingsPerArena(size_t thingsPerArena);
+
+    // Resets the first |thingsPerArena| segments of this list for further use.
+    inline void reset(size_t thingsPerArena = MaxThingsPerArena);
+
+    // Inserts an arena, which has room for |nfree| more things, in its segment.
+    inline void insertAt(Arena* arena, size_t nfree);
+
+    // Remove all empty arenas, inserting them as a linked list.
+    inline void extractEmpty(Arena** empty);
+
+    // Links up the tail of each non-empty segment to the head of the next
+    // non-empty segment, creating a contiguous list that is returned as an
+    // ArenaList. This is not a destructive operation: neither the head nor tail
+    // of any segment is modified. However, note that the Arenas in the
+    // resulting ArenaList should be treated as read-only unless the
+    // SortedArenaList is no longer needed: inserting or removing arenas would
+    // invalidate the SortedArenaList.
+    inline ArenaList toArenaList();
+};
+
+enum class ShouldCheckThresholds
+{
+    DontCheckThresholds = 0,
+    CheckThresholds = 1
+};
+
+class ArenaLists
+{
+    JSRuntime* const runtime_;
+
+    /*
+     * For each arena kind its free list is represented as the first span with
+     * free things. Initially all the spans are initialized as empty. After we
+     * find a new arena with available things we move its first free span into
+     * the list and set the arena as fully allocated. way we do not need to
+     * update the arena after the initial allocation. When starting the
+     * GC we only move the head of the of the list of spans back to the arena
+     * only for the arena that was not fully allocated.
+     */
+    ZoneGroupData<AllAllocKindArray<FreeSpan*>> freeLists_;
+    FreeSpan*& freeLists(AllocKind i) { return freeLists_.ref()[i]; }
+    FreeSpan* freeLists(AllocKind i) const { return freeLists_.ref()[i]; }
+
+    // Because the JITs can allocate from the free lists, they cannot be null.
+    // We use a placeholder FreeSpan that is empty (and wihout an associated
+    // Arena) so the JITs can fall back gracefully.
+    static FreeSpan placeholder;
+
+    ZoneGroupOrGCTaskData<AllAllocKindArray<ArenaList>> arenaLists_;
+    ArenaList& arenaLists(AllocKind i) { return arenaLists_.ref()[i]; }
+    const ArenaList& arenaLists(AllocKind i) const { return arenaLists_.ref()[i]; }
+
+    enum BackgroundFinalizeStateEnum { BFS_DONE, BFS_RUN };
+
+    typedef mozilla::Atomic<BackgroundFinalizeStateEnum, mozilla::SequentiallyConsistent>
+        BackgroundFinalizeState;
+
+    /* The current background finalization state, accessed atomically. */
+    UnprotectedData<AllAllocKindArray<BackgroundFinalizeState>> backgroundFinalizeState_;
+    BackgroundFinalizeState& backgroundFinalizeState(AllocKind i) { return backgroundFinalizeState_.ref()[i]; }
+    const BackgroundFinalizeState& backgroundFinalizeState(AllocKind i) const { return backgroundFinalizeState_.ref()[i]; }
+
+    /* For each arena kind, a list of arenas remaining to be swept. */
+    ActiveThreadOrGCTaskData<AllAllocKindArray<Arena*>> arenaListsToSweep_;
+    Arena*& arenaListsToSweep(AllocKind i) { return arenaListsToSweep_.ref()[i]; }
+    Arena* arenaListsToSweep(AllocKind i) const { return arenaListsToSweep_.ref()[i]; }
+
+    /* During incremental sweeping, a list of the arenas already swept. */
+    ZoneGroupOrGCTaskData<AllocKind> incrementalSweptArenaKind;
+    ZoneGroupOrGCTaskData<ArenaList> incrementalSweptArenas;
+
+    // Arena lists which have yet to be swept, but need additional foreground
+    // processing before they are swept.
+    ZoneGroupData<Arena*> gcShapeArenasToUpdate;
+    ZoneGroupData<Arena*> gcAccessorShapeArenasToUpdate;
+    ZoneGroupData<Arena*> gcScriptArenasToUpdate;
+    ZoneGroupData<Arena*> gcObjectGroupArenasToUpdate;
+
+    // While sweeping type information, these lists save the arenas for the
+    // objects which have already been finalized in the foreground (which must
+    // happen at the beginning of the GC), so that type sweeping can determine
+    // which of the object pointers are marked.
+    ZoneGroupData<ObjectAllocKindArray<ArenaList>> savedObjectArenas_;
+    ArenaList& savedObjectArenas(AllocKind i) { return savedObjectArenas_.ref()[i]; }
+    ZoneGroupData<Arena*> savedEmptyObjectArenas;
+
+  public:
+    explicit ArenaLists(JSRuntime* rt, ZoneGroup* group);
+    ~ArenaLists();
+
+    const void* addressOfFreeList(AllocKind thingKind) const {
+        return reinterpret_cast<const void*>(&freeLists_.refNoCheck()[thingKind]);
+    }
+
+    inline Arena* getFirstArena(AllocKind thingKind) const;
+    inline Arena* getFirstArenaToSweep(AllocKind thingKind) const;
+    inline Arena* getFirstSweptArena(AllocKind thingKind) const;
+    inline Arena* getArenaAfterCursor(AllocKind thingKind) const;
+
+    inline bool arenaListsAreEmpty() const;
+
+    inline void unmarkAll();
+
+    inline bool doneBackgroundFinalize(AllocKind kind) const;
+    inline bool needBackgroundFinalizeWait(AllocKind kind) const;
+
+    /* Clear the free lists so we won't try to allocate from swept arenas. */
+    inline void purge();
+
+    inline void prepareForIncrementalGC();
+
+    /* Check if this arena is in use. */
+    inline bool arenaIsInUse(Arena* arena, AllocKind kind) const;
+
+    MOZ_ALWAYS_INLINE TenuredCell* allocateFromFreeList(AllocKind thingKind, size_t thingSize);
+
+    /* Moves all arenas from |fromArenaLists| into |this|. */
+    void adoptArenas(JSRuntime* runtime, ArenaLists* fromArenaLists, bool targetZoneIsCollecting);
+
+    /* True if the Arena in question is found in this ArenaLists */
+    bool containsArena(JSRuntime* runtime, Arena* arena);
+
+    inline void checkEmptyFreeLists();
+    inline bool checkEmptyArenaLists();
+    inline void checkEmptyFreeList(AllocKind kind);
+
+    bool checkEmptyArenaList(AllocKind kind);
+
+    bool relocateArenas(JS::Zone* zone, Arena*& relocatedListOut, JS::gcreason::Reason reason,
+                        js::SliceBudget& sliceBudget, gcstats::Statistics& stats);
+
+    void queueForegroundObjectsForSweep(FreeOp* fop);
+    void queueForegroundThingsForSweep(FreeOp* fop);
+
+    void mergeForegroundSweptObjectArenas();
+
+    bool foregroundFinalize(FreeOp* fop, AllocKind thingKind, js::SliceBudget& sliceBudget,
+                            SortedArenaList& sweepList);
+    static void backgroundFinalize(FreeOp* fop, Arena* listHead, Arena** empty);
+
+    // When finalizing arenas, whether to keep empty arenas on the list or
+    // release them immediately.
+    enum KeepArenasEnum {
+        RELEASE_ARENAS,
+        KEEP_ARENAS
+    };
+
+  private:
+    inline void queueForForegroundSweep(FreeOp* fop, const FinalizePhase& phase);
+    inline void queueForBackgroundSweep(FreeOp* fop, const FinalizePhase& phase);
+    inline void queueForForegroundSweep(FreeOp* fop, AllocKind thingKind);
+    inline void queueForBackgroundSweep(FreeOp* fop, AllocKind thingKind);
+    inline void mergeSweptArenas(AllocKind thingKind);
+
+    TenuredCell* allocateFromArena(JS::Zone* zone, AllocKind thingKind,
+                                   ShouldCheckThresholds checkThresholds);
+    inline TenuredCell* allocateFromArenaInner(JS::Zone* zone, Arena* arena, AllocKind kind);
+
+    friend class GCRuntime;
+    friend class js::Nursery;
+    friend class js::TenuringTracer;
+};
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_ArenaList_h */
+

js/src/gc/AtomMarking.h

@@ -0,0 +1,86 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_AtomMarking_h
+#define gc_AtomMarking_h
+
+#include "NamespaceImports.h"
+#include "ds/Bitmap.h"
+#include "threading/ProtectedData.h"
+#include "vm/Symbol.h"
+
+namespace js {
+namespace gc {
+
+class Arena;
+
+// This class manages state used for marking atoms during GCs.
+// See AtomMarking.cpp for details.
+class AtomMarkingRuntime
+{
+    // Unused arena atom bitmap indexes. Protected by the GC lock.
+    js::ExclusiveAccessLockOrGCTaskData<Vector<size_t, 0, SystemAllocPolicy>> freeArenaIndexes;
+
+    void markChildren(JSContext* cx, JSAtom*) {}
+
+    void markChildren(JSContext* cx, JS::Symbol* symbol) {
+        if (JSAtom* description = symbol->description())
+            markAtom(cx, description);
+    }
+
+  public:
+    // The extent of all allocated and free words in atom mark bitmaps.
+    // This monotonically increases and may be read from without locking.
+    mozilla::Atomic<size_t> allocatedWords;
+
+    AtomMarkingRuntime()
+      : allocatedWords(0)
+    {}
+
+    // Mark an arena as holding things in the atoms zone.
+    void registerArena(Arena* arena);
+
+    // Mark an arena as no longer holding things in the atoms zone.
+    void unregisterArena(Arena* arena);
+
+    // Fill |bitmap| with an atom marking bitmap based on the things that are
+    // currently marked in the chunks used by atoms zone arenas. This returns
+    // false on an allocation failure (but does not report an exception).
+    bool computeBitmapFromChunkMarkBits(JSRuntime* runtime, DenseBitmap& bitmap);
+
+    // Update the atom marking bitmap in |zone| according to another
+    // overapproximation of the reachable atoms in |bitmap|.
+    void updateZoneBitmap(Zone* zone, const DenseBitmap& bitmap);
+
+    // Set any bits in the chunk mark bitmaps for atoms which are marked in any
+    // zone in the runtime.
+    void updateChunkMarkBits(JSRuntime* runtime);
+
+    // Mark an atom or id as being newly reachable by the context's zone.
+    template <typename T> void markAtom(JSContext* cx, T* thing);
+
+    // Version of markAtom that's always inlined, for performance-sensitive
+    // callers.
+    template <typename T> MOZ_ALWAYS_INLINE void inlinedMarkAtom(JSContext* cx, T* thing);
+
+    void markId(JSContext* cx, jsid id);
+    void markAtomValue(JSContext* cx, const Value& value);
+
+    // Mark all atoms in |source| as being reachable within |target|.
+    void adoptMarkedAtoms(Zone* target, Zone* source);
+
+#ifdef DEBUG
+    // Return whether |thing/id| is in the atom marking bitmap for |zone|.
+    template <typename T> bool atomIsMarked(Zone* zone, T* thing);
+    bool idIsMarked(Zone* zone, jsid id);
+    bool valueIsMarked(Zone* zone, const Value& value);
+#endif
+};
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_AtomMarking_h

js/src/gc/Barrier.h

@@ -9,7 +9,7 @@
 
 #include "NamespaceImports.h"
 
-#include "gc/Heap.h"
+#include "gc/Cell.h"
 #include "gc/StoreBuffer.h"
 #include "js/HeapAPI.h"
 #include "js/Id.h"

js/src/gc/Cell.h

@@ -0,0 +1,413 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Cell_h
+#define gc_Cell_h
+
+#include "gc/GCEnum.h"
+#include "gc/Heap.h"
+#include "js/GCAnnotations.h"
+
+namespace JS {
+
+namespace shadow {
+struct Zone;
+} /* namespace shadow */
+
+enum class TraceKind;
+struct Zone;
+} /* namespace JS */
+
+namespace js {
+
+class GenericPrinter;
+
+extern bool
+RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone);
+
+#ifdef DEBUG
+
+// Barriers can't be triggered during backend Ion compilation, which may run on
+// a helper thread.
+extern bool
+CurrentThreadIsIonCompiling();
+#endif
+
+extern void
+TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name);
+
+namespace gc {
+
+class Arena;
+enum class AllocKind : uint8_t;
+struct Chunk;
+class TenuredCell;
+
+// A GC cell is the base class for all GC things.
+struct Cell
+{
+  public:
+    MOZ_ALWAYS_INLINE bool isTenured() const { return !IsInsideNursery(this); }
+    MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const;
+    MOZ_ALWAYS_INLINE TenuredCell& asTenured();
+
+    MOZ_ALWAYS_INLINE bool isMarkedAny() const;
+    MOZ_ALWAYS_INLINE bool isMarkedBlack() const;
+    MOZ_ALWAYS_INLINE bool isMarkedGray() const;
+
+    inline JSRuntime* runtimeFromActiveCooperatingThread() const;
+
+    // Note: Unrestricted access to the runtime of a GC thing from an arbitrary
+    // thread can easily lead to races. Use this method very carefully.
+    inline JSRuntime* runtimeFromAnyThread() const;
+
+    // May be overridden by GC thing kinds that have a compartment pointer.
+    inline JSCompartment* maybeCompartment() const { return nullptr; }
+
+    // The StoreBuffer used to record incoming pointers from the tenured heap.
+    // This will return nullptr for a tenured cell.
+    inline StoreBuffer* storeBuffer() const;
+
+    inline JS::TraceKind getTraceKind() const;
+
+    static MOZ_ALWAYS_INLINE bool needWriteBarrierPre(JS::Zone* zone);
+
+#ifdef DEBUG
+    inline bool isAligned() const;
+    void dump(GenericPrinter& out) const;
+    void dump() const;
+#endif
+
+  protected:
+    uintptr_t address() const;
+    inline Chunk* chunk() const;
+} JS_HAZ_GC_THING;
+
+// A GC TenuredCell gets behaviors that are valid for things in the Tenured
+// heap, such as access to the arena and mark bits.
+class TenuredCell : public Cell
+{
+  public:
+    // Construct a TenuredCell from a void*, making various sanity assertions.
+    static MOZ_ALWAYS_INLINE TenuredCell* fromPointer(void* ptr);
+    static MOZ_ALWAYS_INLINE const TenuredCell* fromPointer(const void* ptr);
+
+    // Mark bit management.
+    MOZ_ALWAYS_INLINE bool isMarkedAny() const;
+    MOZ_ALWAYS_INLINE bool isMarkedBlack() const;
+    MOZ_ALWAYS_INLINE bool isMarkedGray() const;
+
+    // The return value indicates if the cell went from unmarked to marked.
+    MOZ_ALWAYS_INLINE bool markIfUnmarked(MarkColor color = MarkColor::Black) const;
+    MOZ_ALWAYS_INLINE void markBlack() const;
+    MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src);
+
+    // Access to the arena.
+    inline Arena* arena() const;
+    inline AllocKind getAllocKind() const;
+    inline JS::TraceKind getTraceKind() const;
+    inline JS::Zone* zone() const;
+    inline JS::Zone* zoneFromAnyThread() const;
+    inline bool isInsideZone(JS::Zone* zone) const;
+
+    MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZone() const {
+        return JS::shadow::Zone::asShadowZone(zone());
+    }
+    MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZoneFromAnyThread() const {
+        return JS::shadow::Zone::asShadowZone(zoneFromAnyThread());
+    }
+
+    static MOZ_ALWAYS_INLINE void readBarrier(TenuredCell* thing);
+    static MOZ_ALWAYS_INLINE void writeBarrierPre(TenuredCell* thing);
+
+    static void MOZ_ALWAYS_INLINE writeBarrierPost(void* cellp, TenuredCell* prior,
+                                                   TenuredCell* next);
+
+    // Default implementation for kinds that don't require fixup.
+    void fixupAfterMovingGC() {}
+
+#ifdef DEBUG
+    inline bool isAligned() const;
+#endif
+};
+
+MOZ_ALWAYS_INLINE const TenuredCell&
+Cell::asTenured() const
+{
+    MOZ_ASSERT(isTenured());
+    return *static_cast<const TenuredCell*>(this);
+}
+
+MOZ_ALWAYS_INLINE TenuredCell&
+Cell::asTenured()
+{
+    MOZ_ASSERT(isTenured());
+    return *static_cast<TenuredCell*>(this);
+}
+
+MOZ_ALWAYS_INLINE bool
+Cell::isMarkedAny() const
+{
+    return !isTenured() || asTenured().isMarkedAny();
+}
+
+MOZ_ALWAYS_INLINE bool
+Cell::isMarkedBlack() const
+{
+    return !isTenured() || asTenured().isMarkedBlack();
+}
+
+MOZ_ALWAYS_INLINE bool
+Cell::isMarkedGray() const
+{
+    return isTenured() && asTenured().isMarkedGray();
+}
+
+inline JSRuntime*
+Cell::runtimeFromActiveCooperatingThread() const
+{
+    JSRuntime* rt = chunk()->trailer.runtime;
+    MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+    return rt;
+}
+
+inline JSRuntime*
+Cell::runtimeFromAnyThread() const
+{
+    return chunk()->trailer.runtime;
+}
+
+inline uintptr_t
+Cell::address() const
+{
+    uintptr_t addr = uintptr_t(this);
+    MOZ_ASSERT(addr % CellAlignBytes == 0);
+    MOZ_ASSERT(Chunk::withinValidRange(addr));
+    return addr;
+}
+
+Chunk*
+Cell::chunk() const
+{
+    uintptr_t addr = uintptr_t(this);
+    MOZ_ASSERT(addr % CellAlignBytes == 0);
+    addr &= ~ChunkMask;
+    return reinterpret_cast<Chunk*>(addr);
+}
+
+inline StoreBuffer*
+Cell::storeBuffer() const
+{
+    return chunk()->trailer.storeBuffer;
+}
+
+inline JS::TraceKind
+Cell::getTraceKind() const
+{
+    return isTenured() ? asTenured().getTraceKind() : JS::TraceKind::Object;
+}
+
+/* static */ MOZ_ALWAYS_INLINE bool
+Cell::needWriteBarrierPre(JS::Zone* zone) {
+    return JS::shadow::Zone::asShadowZone(zone)->needsIncrementalBarrier();
+}
+
+/* static */ MOZ_ALWAYS_INLINE TenuredCell*
+TenuredCell::fromPointer(void* ptr)
+{
+    MOZ_ASSERT(static_cast<TenuredCell*>(ptr)->isTenured());
+    return static_cast<TenuredCell*>(ptr);
+}
+
+/* static */ MOZ_ALWAYS_INLINE const TenuredCell*
+TenuredCell::fromPointer(const void* ptr)
+{
+    MOZ_ASSERT(static_cast<const TenuredCell*>(ptr)->isTenured());
+    return static_cast<const TenuredCell*>(ptr);
+}
+
+bool
+TenuredCell::isMarkedAny() const
+{
+    MOZ_ASSERT(arena()->allocated());
+    return chunk()->bitmap.isMarkedAny(this);
+}
+
+bool
+TenuredCell::isMarkedBlack() const
+{
+    MOZ_ASSERT(arena()->allocated());
+    return chunk()->bitmap.isMarkedBlack(this);
+}
+
+bool
+TenuredCell::isMarkedGray() const
+{
+    MOZ_ASSERT(arena()->allocated());
+    return chunk()->bitmap.isMarkedGray(this);
+}
+
+bool
+TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const
+{
+    return chunk()->bitmap.markIfUnmarked(this, color);
+}
+
+void
+TenuredCell::markBlack() const
+{
+    chunk()->bitmap.markBlack(this);
+}
+
+void
+TenuredCell::copyMarkBitsFrom(const TenuredCell* src)
+{
+    ChunkBitmap& bitmap = chunk()->bitmap;
+    bitmap.copyMarkBit(this, src, ColorBit::BlackBit);
+    bitmap.copyMarkBit(this, src, ColorBit::GrayOrBlackBit);
+}
+
+inline Arena*
+TenuredCell::arena() const
+{
+    MOZ_ASSERT(isTenured());
+    uintptr_t addr = address();
+    addr &= ~ArenaMask;
+    return reinterpret_cast<Arena*>(addr);
+}
+
+AllocKind
+TenuredCell::getAllocKind() const
+{
+    return arena()->getAllocKind();
+}
+
+JS::TraceKind
+TenuredCell::getTraceKind() const
+{
+    return MapAllocToTraceKind(getAllocKind());
+}
+
+JS::Zone*
+TenuredCell::zone() const
+{
+    JS::Zone* zone = arena()->zone;
+    MOZ_ASSERT(CurrentThreadCanAccessZone(zone));
+    return zone;
+}
+
+JS::Zone*
+TenuredCell::zoneFromAnyThread() const
+{
+    return arena()->zone;
+}
+
+bool
+TenuredCell::isInsideZone(JS::Zone* zone) const
+{
+    return zone == arena()->zone;
+}
+
+/* static */ MOZ_ALWAYS_INLINE void
+TenuredCell::readBarrier(TenuredCell* thing)
+{
+    MOZ_ASSERT(!CurrentThreadIsIonCompiling());
+    MOZ_ASSERT(thing);
+    MOZ_ASSERT(CurrentThreadCanAccessZone(thing->zoneFromAnyThread()));
+
+    // It would be good if barriers were never triggered during collection, but
+    // at the moment this can happen e.g. when rekeying tables containing
+    // read-barriered GC things after a moving GC.
+    //
+    // TODO: Fix this and assert we're not collecting if we're on the active
+    // thread.
+
+    JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread();
+    if (shadowZone->needsIncrementalBarrier()) {
+        // Barriers are only enabled on the active thread and are disabled while collecting.
+        MOZ_ASSERT(!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone));
+        Cell* tmp = thing;
+        TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "read barrier");
+        MOZ_ASSERT(tmp == thing);
+    }
+
+    if (thing->isMarkedGray()) {
+        // There shouldn't be anything marked grey unless we're on the active thread.
+        MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread()));
+        if (!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone))
+            JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(thing, thing->getTraceKind()));
+    }
+}
+
+void
+AssertSafeToSkipBarrier(TenuredCell* thing);
+
+/* static */ MOZ_ALWAYS_INLINE void
+TenuredCell::writeBarrierPre(TenuredCell* thing)
+{
+    MOZ_ASSERT(!CurrentThreadIsIonCompiling());
+    if (!thing)
+        return;
+
+#ifdef JS_GC_ZEAL
+    // When verifying pre barriers we need to switch on all barriers, even
+    // those on the Atoms Zone. Normally, we never enter a parse task when
+    // collecting in the atoms zone, so will filter out atoms below.
+    // Unfortuantely, If we try that when verifying pre-barriers, we'd never be
+    // able to handle off thread parse tasks at all as we switch on the verifier any
+    // time we're not doing GC. This would cause us to deadlock, as off thread parsing
+    // is meant to resume after GC work completes. Instead we filter out any
+    // off thread barriers that reach us and assert that they would normally not be
+    // possible.
+    if (!CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())) {
+        AssertSafeToSkipBarrier(thing);
+        return;
+    }
+#endif
+
+    JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread();
+    if (shadowZone->needsIncrementalBarrier()) {
+        MOZ_ASSERT(!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone));
+        Cell* tmp = thing;
+        TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "pre barrier");
+        MOZ_ASSERT(tmp == thing);
+    }
+}
+
+static MOZ_ALWAYS_INLINE void
+AssertValidToSkipBarrier(TenuredCell* thing)
+{
+    MOZ_ASSERT(!IsInsideNursery(thing));
+    MOZ_ASSERT_IF(thing, MapAllocToTraceKind(thing->getAllocKind()) != JS::TraceKind::Object);
+}
+
+/* static */ MOZ_ALWAYS_INLINE void
+TenuredCell::writeBarrierPost(void* cellp, TenuredCell* prior, TenuredCell* next)
+{
+    AssertValidToSkipBarrier(next);
+}
+
+#ifdef DEBUG
+bool
+Cell::isAligned() const
+{
+    if (!isTenured())
+        return true;
+    return asTenured().isAligned();
+}
+
+bool
+TenuredCell::isAligned() const
+{
+    return Arena::isAligned(address(), arena()->getThingSize());
+}
+
+#endif
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_Cell_h */

js/src/gc/GCEnum.h

@@ -0,0 +1,88 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal enum definitions.
+ */
+
+#ifndef gc_GCEnum_h
+#define gc_GCEnum_h
+
+#include <stdint.h>
+
+namespace js {
+namespace gc {
+
+// Mark colors to pass to markIfUnmarked.
+enum class MarkColor : uint32_t
+{
+    Black = 0,
+    Gray
+};
+
+// The phases of an incremental GC.
+#define GCSTATES(D) \
+    D(NotActive) \
+    D(MarkRoots) \
+    D(Mark) \
+    D(Sweep) \
+    D(Finalize) \
+    D(Compact) \
+    D(Decommit)
+enum class State {
+#define MAKE_STATE(name) name,
+    GCSTATES(MAKE_STATE)
+#undef MAKE_STATE
+};
+
+// Reasons we reset an ongoing incremental GC or perform a non-incremental GC.
+#define GC_ABORT_REASONS(D) \
+    D(None) \
+    D(NonIncrementalRequested) \
+    D(AbortRequested) \
+    D(Unused1) \
+    D(IncrementalDisabled) \
+    D(ModeChange) \
+    D(MallocBytesTrigger) \
+    D(GCBytesTrigger) \
+    D(ZoneChange) \
+    D(CompartmentRevived)
+enum class AbortReason {
+#define MAKE_REASON(name) name,
+    GC_ABORT_REASONS(MAKE_REASON)
+#undef MAKE_REASON
+};
+
+#define JS_FOR_EACH_ZEAL_MODE(D)       \
+    D(RootsChange, 1)                  \
+    D(Alloc, 2)                        \
+    D(FrameGC, 3)                      \
+    D(VerifierPre, 4)                  \
+    D(FrameVerifierPre, 5)             \
+    D(GenerationalGC, 7)               \
+    D(IncrementalRootsThenFinish, 8)   \
+    D(IncrementalMarkAllThenFinish, 9) \
+    D(IncrementalMultipleSlices, 10)   \
+    D(IncrementalMarkingValidator, 11) \
+    D(ElementsBarrier, 12)             \
+    D(CheckHashTablesOnMinorGC, 13)    \
+    D(Compact, 14)                     \
+    D(CheckHeapAfterGC, 15)            \
+    D(CheckNursery, 16)                \
+    D(IncrementalSweepThenFinish, 17)  \
+    D(CheckGrayMarking, 18)
+
+enum class ZealMode {
+#define ZEAL_MODE(name, value) name = value,
+    JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE)
+#undef ZEAL_MODE
+    Limit = 18
+};
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif /* gc_GCEnum_h */

js/src/gc/GCHelperState.h

@@ -0,0 +1,104 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCHelperState_h
+#define gc_GCHelperState_h
+
+#include "threading/ConditionVariable.h"
+#include "threading/ProtectedData.h"
+
+struct JSRuntime;
+
+namespace js {
+class AutoLockHelperThreadState;
+
+namespace gc {
+class ArenaLists;
+} /* namespace gc */
+
+/*
+ * Helper state for use when JS helper threads sweep and allocate GC thing kinds
+ * that can be swept and allocated off thread.
+ *
+ * In non-threadsafe builds, all actual sweeping and allocation is performed
+ * on the active thread, but GCHelperState encapsulates this from clients as
+ * much as possible.
+ */
+class GCHelperState
+{
+    enum State {
+        IDLE,
+        SWEEPING
+    };
+
+    // Associated runtime.
+    JSRuntime* const rt;
+
+    // Condvar for notifying the active thread when work has finished. This is
+    // associated with the runtime's GC lock --- the worker thread state
+    // condvars can't be used here due to lock ordering issues.
+    ConditionVariable done;
+
+    // Activity for the helper to do, protected by the GC lock.
+    ActiveThreadOrGCTaskData<State> state_;
+
+    // Whether work is being performed on some thread.
+    GCLockData<bool> hasThread;
+
+    void startBackgroundThread(State newState, const AutoLockGC& lock,
+                               const AutoLockHelperThreadState& helperLock);
+    void waitForBackgroundThread(js::AutoLockGC& lock);
+
+    State state(const AutoLockGC&);
+    void setState(State state, const AutoLockGC&);
+
+    friend class js::gc::ArenaLists;
+
+    static void freeElementsAndArray(void** array, void** end) {
+        MOZ_ASSERT(array <= end);
+        for (void** p = array; p != end; ++p)
+            js_free(*p);
+        js_free(array);
+    }
+
+    void doSweep(AutoLockGC& lock);
+
+  public:
+    explicit GCHelperState(JSRuntime* rt)
+      : rt(rt),
+        done(),
+        state_(IDLE)
+    { }
+
+    JSRuntime* runtime() { return rt; }
+
+    void finish();
+
+    void work();
+
+    void maybeStartBackgroundSweep(const AutoLockGC& lock,
+                                   const AutoLockHelperThreadState& helperLock);
+    void startBackgroundShrink(const AutoLockGC& lock);
+
+    /* Must be called without the GC lock taken. */
+    void waitBackgroundSweepEnd();
+
+#ifdef DEBUG
+    bool onBackgroundThread();
+#endif
+
+    /*
+     * Outside the GC lock may give true answer when in fact the sweeping has
+     * been done.
+     */
+    bool isBackgroundSweeping() const {
+        return state_ == SWEEPING;
+    }
+};
+
+} /* namespace js */
+
+#endif /* gc_GCHelperState_h */

js/src/gc/GCInternals.h

@@ -1,8 +1,13 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+/*
+ * GC-internal definitions.
+ */
+
 #ifndef gc_GCInternals_h
 #define gc_GCInternals_h
 
@@ -11,6 +16,7 @@
 
 #include "jscntxt.h"
 
+#include "gc/RelocationOverlay.h"
 #include "gc/Zone.h"
 #include "vm/HelperThreads.h"
 #include "vm/Runtime.h"
@@ -129,6 +135,92 @@ struct TenureCountCache
     }
 };
 
+struct MOZ_RAII AutoAssertNoNurseryAlloc
+{
+#ifdef DEBUG
+    AutoAssertNoNurseryAlloc();
+    ~AutoAssertNoNurseryAlloc();
+#else
+    AutoAssertNoNurseryAlloc() {}
+#endif
+};
+
+/*
+ * There are a couple of classes here that serve mostly as "tokens" indicating
+ * that a condition holds. Some functions force the caller to possess such a
+ * token because they would misbehave if the condition were false, and it is
+ * far more clear to make the condition visible at the point where it can be
+ * affected rather than just crashing in an assertion down in the place where
+ * it is relied upon.
+ */
+
+/*
+ * A class that serves as a token that the nursery in the current thread's zone
+ * group is empty.
+ */
+class MOZ_RAII AutoAssertEmptyNursery
+{
+  protected:
+    JSContext* cx;
+
+    mozilla::Maybe<AutoAssertNoNurseryAlloc> noAlloc;
+
+    // Check that the nursery is empty.
+    void checkCondition(JSContext* cx);
+
+    // For subclasses that need to empty the nursery in their constructors.
+    AutoAssertEmptyNursery() : cx(nullptr) {
+    }
+
+  public:
+    explicit AutoAssertEmptyNursery(JSContext* cx) : cx(nullptr) {
+        checkCondition(cx);
+    }
+
+    AutoAssertEmptyNursery(const AutoAssertEmptyNursery& other) : AutoAssertEmptyNursery(other.cx)
+    {
+    }
+};
+
+/*
+ * Evict the nursery upon construction. Serves as a token indicating that the
+ * nursery is empty. (See AutoAssertEmptyNursery, above.)
+ *
+ * Note that this is very improper subclass of AutoAssertHeapBusy, in that the
+ * heap is *not* busy within the scope of an AutoEmptyNursery. I will most
+ * likely fix this by removing AutoAssertHeapBusy, but that is currently
+ * waiting on jonco's review.
+ */
+class MOZ_RAII AutoEmptyNursery : public AutoAssertEmptyNursery
+{
+  public:
+    explicit AutoEmptyNursery(JSContext* cx);
+};
+
+extern void
+DelayCrossCompartmentGrayMarking(JSObject* src);
+
+inline bool
+IsOOMReason(JS::gcreason::Reason reason)
+{
+    return reason == JS::gcreason::LAST_DITCH ||
+           reason == JS::gcreason::MEM_PRESSURE;
+}
+
+inline void
+RelocationOverlay::forwardTo(Cell* cell)
+{
+    MOZ_ASSERT(!isForwarded());
+    // The location of magic_ is important because it must never be valid to see
+    // the value Relocated there in a GC thing that has not been moved.
+    static_assert(offsetof(RelocationOverlay, magic_) == offsetof(JSObject, group_) &&
+                  offsetof(RelocationOverlay, magic_) == offsetof(js::Shape, base_) &&
+                  offsetof(RelocationOverlay, magic_) == offsetof(JSString, d.u1.flags),
+                  "RelocationOverlay::magic_ is in the wrong location");
+    magic_ = Relocated;
+    newLocation_ = cell;
+}
+
 } /* namespace gc */
 } /* namespace js */
 

js/src/gc/GCMarker.h

@@ -0,0 +1,391 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCMarker_h
+#define gc_GCMarker_h
+
+#include "ds/OrderedHashTable.h"
+#include "js/SliceBudget.h"
+#include "js/TracingAPI.h"
+
+namespace JS {
+class Symbol;
+}
+
+namespace js {
+
+class WeakMapBase;
+
+static const size_t NON_INCREMENTAL_MARK_STACK_BASE_CAPACITY = 4096;
+static const size_t INCREMENTAL_MARK_STACK_BASE_CAPACITY = 32768;
+
+namespace gc {
+
+struct Cell;
+
+struct WeakKeyTableHashPolicy {
+    typedef JS::GCCellPtr Lookup;
+    static HashNumber hash(const Lookup& v, const mozilla::HashCodeScrambler&) {
+        return mozilla::HashGeneric(v.asCell());
+    }
+    static bool match(const JS::GCCellPtr& k, const Lookup& l) { return k == l; }
+    static bool isEmpty(const JS::GCCellPtr& v) { return !v; }
+    static void makeEmpty(JS::GCCellPtr* vp) { *vp = nullptr; }
+};
+
+struct WeakMarkable {
+    WeakMapBase* weakmap;
+    JS::GCCellPtr key;
+
+    WeakMarkable(WeakMapBase* weakmapArg, JS::GCCellPtr keyArg)
+      : weakmap(weakmapArg), key(keyArg) {}
+};
+
+using WeakEntryVector = Vector<WeakMarkable, 2, js::SystemAllocPolicy>;
+
+using WeakKeyTable = OrderedHashMap<JS::GCCellPtr,
+                                    WeakEntryVector,
+                                    WeakKeyTableHashPolicy,
+                                    js::SystemAllocPolicy>;
+
+/*
+ * When the native stack is low, the GC does not call js::TraceChildren to mark
+ * the reachable "children" of the thing. Rather the thing is put aside and
+ * js::TraceChildren is called later with more space on the C stack.
+ *
+ * To implement such delayed marking of the children with minimal overhead for
+ * the normal case of sufficient native stack, the code adds a field per arena.
+ * The field markingDelay->link links all arenas with delayed things into a
+ * stack list with the pointer to stack top in GCMarker::unmarkedArenaStackTop.
+ * GCMarker::delayMarkingChildren adds arenas to the stack as necessary while
+ * markDelayedChildren pops the arenas from the stack until it empties.
+ */
+class MarkStack
+{
+  public:
+    /*
+     * We use a common mark stack to mark GC things of different types and use
+     * the explicit tags to distinguish them when it cannot be deduced from
+     * the context of push or pop operation.
+     */
+    enum Tag {
+        ValueArrayTag,
+        ObjectTag,
+        GroupTag,
+        SavedValueArrayTag,
+        JitCodeTag,
+        ScriptTag,
+        TempRopeTag,
+
+        LastTag = TempRopeTag
+    };
+
+    static const uintptr_t TagMask = 7;
+    static_assert(TagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags.");
+    static_assert(TagMask <= gc::CellAlignMask, "The tag mask must be embeddable in a Cell*.");
+
+    class TaggedPtr
+    {
+        uintptr_t bits;
+
+        Cell* ptr() const;
+
+      public:
+        TaggedPtr(Tag tag, Cell* ptr);
+        Tag tag() const;
+        template <typename T> T* as() const;
+
+        JSObject* asValueArrayObject() const;
+        JSObject* asSavedValueArrayObject() const;
+        JSRope* asTempRope() const;
+    };
+
+    struct ValueArray
+    {
+        ValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end);
+
+        HeapSlot* end;
+        HeapSlot* start;
+        TaggedPtr ptr;
+    };
+
+    struct SavedValueArray
+    {
+        SavedValueArray(JSObject* obj, size_t index, HeapSlot::Kind kind);
+
+        uintptr_t kind;
+        uintptr_t index;
+        TaggedPtr ptr;
+    };
+
+    explicit MarkStack(size_t maxCapacity = DefaultCapacity);
+    ~MarkStack();
+
+    static const size_t DefaultCapacity = SIZE_MAX;
+
+    size_t capacity() { return end_ - stack_; }
+
+    size_t position() const {
+        auto result = tos_ - stack_;
+        MOZ_ASSERT(result >= 0);
+        return size_t(result);
+    }
+
+    void setStack(TaggedPtr* stack, size_t tosIndex, size_t capacity);
+
+    MOZ_MUST_USE bool init(JSGCMode gcMode);
+
+    void setBaseCapacity(JSGCMode mode);
+    size_t maxCapacity() const { return maxCapacity_; }
+    void setMaxCapacity(size_t maxCapacity);
+
+    template <typename T>
+    MOZ_MUST_USE bool push(T* ptr);
+
+    MOZ_MUST_USE bool push(JSObject* obj, HeapSlot* start, HeapSlot* end);
+    MOZ_MUST_USE bool push(const ValueArray& array);
+    MOZ_MUST_USE bool push(const SavedValueArray& array);
+
+    // GCMarker::eagerlyMarkChildren uses unused marking stack as temporary
+    // storage to hold rope pointers.
+    MOZ_MUST_USE bool pushTempRope(JSRope* ptr);
+
+    bool isEmpty() const {
+        return tos_ == stack_;
+    }
+
+    Tag peekTag() const;
+    TaggedPtr popPtr();
+    ValueArray popValueArray();
+    SavedValueArray popSavedValueArray();
+
+    void reset();
+
+    void setGCMode(JSGCMode gcMode);
+
+    size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
+
+  private:
+    MOZ_MUST_USE bool ensureSpace(size_t count);
+
+    /* Grow the stack, ensuring there is space for at least count elements. */
+    MOZ_MUST_USE bool enlarge(size_t count);
+
+    const TaggedPtr& peekPtr() const;
+    MOZ_MUST_USE bool pushTaggedPtr(Tag tag, Cell* ptr);
+
+    ActiveThreadData<TaggedPtr*> stack_;
+    ActiveThreadData<TaggedPtr*> tos_;
+    ActiveThreadData<TaggedPtr*> end_;
+
+    // The capacity we start with and reset() to.
+    ActiveThreadData<size_t> baseCapacity_;
+    ActiveThreadData<size_t> maxCapacity_;
+
+#ifdef DEBUG
+    mutable size_t iteratorCount_;
+#endif
+
+    friend class MarkStackIter;
+};
+
+class MarkStackIter
+{
+    const MarkStack& stack_;
+    MarkStack::TaggedPtr* pos_;
+
+  public:
+    explicit MarkStackIter(const MarkStack& stack);
+    ~MarkStackIter();
+
+    bool done() const;
+    MarkStack::Tag peekTag() const;
+    MarkStack::TaggedPtr peekPtr() const;
+    MarkStack::ValueArray peekValueArray() const;
+    void next();
+    void nextPtr();
+    void nextArray();
+
+    // Mutate the current ValueArray to a SavedValueArray.
+    void saveValueArray(NativeObject* obj, uintptr_t index, HeapSlot::Kind kind);
+
+  private:
+    size_t position() const;
+};
+
+} /* namespace gc */
+
+class GCMarker : public JSTracer
+{
+  public:
+    explicit GCMarker(JSRuntime* rt);
+    MOZ_MUST_USE bool init(JSGCMode gcMode);
+
+    void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); }
+    size_t maxCapacity() const { return stack.maxCapacity(); }
+
+    void start();
+    void stop();
+    void reset();
+
+    // Mark the given GC thing and traverse its children at some point.
+    template <typename T> void traverse(T thing);
+
+    // Calls traverse on target after making additional assertions.
+    template <typename S, typename T> void traverseEdge(S source, T* target);
+    template <typename S, typename T> void traverseEdge(S source, const T& target);
+
+    // Notes a weak graph edge for later sweeping.
+    template <typename T> void noteWeakEdge(T* edge);
+
+    /*
+     * Care must be taken changing the mark color from gray to black. The cycle
+     * collector depends on the invariant that there are no black to gray edges
+     * in the GC heap. This invariant lets the CC not trace through black
+     * objects. If this invariant is violated, the cycle collector may free
+     * objects that are still reachable.
+     */
+    void setMarkColorGray() {
+        MOZ_ASSERT(isDrained());
+        MOZ_ASSERT(color == gc::MarkColor::Black);
+        color = gc::MarkColor::Gray;
+    }
+    void setMarkColorBlack() {
+        MOZ_ASSERT(isDrained());
+        MOZ_ASSERT(color == gc::MarkColor::Gray);
+        color = gc::MarkColor::Black;
+    }
+    gc::MarkColor markColor() const { return color; }
+
+    void enterWeakMarkingMode();
+    void leaveWeakMarkingMode();
+    void abortLinearWeakMarking() {
+        leaveWeakMarkingMode();
+        linearWeakMarkingDisabled_ = true;
+    }
+
+    void delayMarkingArena(gc::Arena* arena);
+    void delayMarkingChildren(const void* thing);
+    void markDelayedChildren(gc::Arena* arena);
+    MOZ_MUST_USE bool markDelayedChildren(SliceBudget& budget);
+    bool hasDelayedChildren() const {
+        return !!unmarkedArenaStackTop;
+    }
+
+    bool isDrained() {
+        return isMarkStackEmpty() && !unmarkedArenaStackTop;
+    }
+
+    MOZ_MUST_USE bool drainMarkStack(SliceBudget& budget);
+
+    void setGCMode(JSGCMode mode) { stack.setGCMode(mode); }
+
+    size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
+
+#ifdef DEBUG
+
+    bool shouldCheckCompartments() { return strictCompartmentChecking; }
+
+    JS::Zone* stackContainsCrossZonePointerTo(const gc::Cell* cell) const;
+
+#endif
+
+    void markEphemeronValues(gc::Cell* markedCell, gc::WeakEntryVector& entry);
+
+    static GCMarker* fromTracer(JSTracer* trc) {
+        MOZ_ASSERT(trc->isMarkingTracer());
+        return static_cast<GCMarker*>(trc);
+    }
+
+  private:
+#ifdef DEBUG
+    void checkZone(void* p);
+#else
+    void checkZone(void* p) {}
+#endif
+
+    // Push an object onto the stack for later tracing and assert that it has
+    // already been marked.
+    inline void repush(JSObject* obj);
+
+    template <typename T> void markAndTraceChildren(T* thing);
+    template <typename T> void markAndPush(T* thing);
+    template <typename T> void markAndScan(T* thing);
+    template <typename T> void markImplicitEdgesHelper(T oldThing);
+    template <typename T> void markImplicitEdges(T* oldThing);
+    void eagerlyMarkChildren(JSLinearString* str);
+    void eagerlyMarkChildren(JSRope* rope);
+    void eagerlyMarkChildren(JSString* str);
+    void eagerlyMarkChildren(LazyScript *thing);
+    void eagerlyMarkChildren(Shape* shape);
+    void eagerlyMarkChildren(Scope* scope);
+    void lazilyMarkChildren(ObjectGroup* group);
+
+    // We may not have concrete types yet, so this has to be outside the header.
+    template <typename T>
+    void dispatchToTraceChildren(T* thing);
+
+    // Mark the given GC thing, but do not trace its children. Return true
+    // if the thing became marked.
+    template <typename T>
+    MOZ_MUST_USE bool mark(T* thing);
+
+    template <typename T>
+    inline void pushTaggedPtr(T* ptr);
+
+    inline void pushValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end);
+
+    bool isMarkStackEmpty() {
+        return stack.isEmpty();
+    }
+
+    MOZ_MUST_USE bool restoreValueArray(const gc::MarkStack::SavedValueArray& array,
+                                        HeapSlot** vpp, HeapSlot** endp);
+    void saveValueRanges();
+    inline void processMarkStackTop(SliceBudget& budget);
+
+    /* The mark stack. Pointers in this stack are "gray" in the GC sense. */
+    gc::MarkStack stack;
+
+    /* The color is only applied to objects and functions. */
+    ActiveThreadData<gc::MarkColor> color;
+
+    /* Pointer to the top of the stack of arenas we are delaying marking on. */
+    ActiveThreadData<js::gc::Arena*> unmarkedArenaStackTop;
+
+    /*
+     * If the weakKeys table OOMs, disable the linear algorithm and fall back
+     * to iterating until the next GC.
+     */
+    ActiveThreadData<bool> linearWeakMarkingDisabled_;
+
+#ifdef DEBUG
+    /* Count of arenas that are currently in the stack. */
+    ActiveThreadData<size_t> markLaterArenas;
+
+    /* Assert that start and stop are called with correct ordering. */
+    ActiveThreadData<bool> started;
+
+    /*
+     * If this is true, all marked objects must belong to a compartment being
+     * GCed. This is used to look for compartment bugs.
+     */
+    ActiveThreadData<bool> strictCompartmentChecking;
+#endif // DEBUG
+};
+
+} /* namespace js */
+
+// Exported for Tracer.cpp
+inline bool ThingIsPermanentAtomOrWellKnownSymbol(js::gc::Cell* thing) { return false; }
+bool ThingIsPermanentAtomOrWellKnownSymbol(JSString*);
+bool ThingIsPermanentAtomOrWellKnownSymbol(JSFlatString*);
+bool ThingIsPermanentAtomOrWellKnownSymbol(JSLinearString*);
+bool ThingIsPermanentAtomOrWellKnownSymbol(JSAtom*);
+bool ThingIsPermanentAtomOrWellKnownSymbol(js::PropertyName*);
+bool ThingIsPermanentAtomOrWellKnownSymbol(JS::Symbol*);
+
+#endif /* gc_GCMarker_h */

js/src/gc/GCParallelTask.h

@@ -0,0 +1,91 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_GCParallelTask_h
+#define gc_GCParallelTask_h
+
+#include "threading/ProtectedData.h"
+
+struct JSRuntime;
+
+namespace js {
+
+// A generic task used to dispatch work to the helper thread system.
+// Users should derive from GCParallelTask add what data they need and
+// override |run|.
+class GCParallelTask
+{
+    JSRuntime* const runtime_;
+
+    // The state of the parallel computation.
+    enum TaskState {
+        NotStarted,
+        Dispatched,
+        Finished,
+    };
+    UnprotectedData<TaskState> state;
+
+    // Amount of time this task took to execute.
+    ActiveThreadOrGCTaskData<mozilla::TimeDuration> duration_;
+
+    explicit GCParallelTask(const GCParallelTask&) = delete;
+
+  protected:
+    // A flag to signal a request for early completion of the off-thread task.
+    mozilla::Atomic<bool> cancel_;
+
+    virtual void run() = 0;
+
+  public:
+    explicit GCParallelTask(JSRuntime* runtime) : runtime_(runtime), state(NotStarted), duration_(nullptr) {}
+    GCParallelTask(GCParallelTask&& other)
+      : runtime_(other.runtime_),
+        state(other.state),
+        duration_(nullptr),
+        cancel_(false)
+    {}
+
+    // Derived classes must override this to ensure that join() gets called
+    // before members get destructed.
+    virtual ~GCParallelTask();
+
+    JSRuntime* runtime() { return runtime_; }
+
+    // Time spent in the most recent invocation of this task.
+    mozilla::TimeDuration duration() const { return duration_; }
+
+    // The simple interface to a parallel task works exactly like pthreads.
+    bool start();
+    void join();
+
+    // If multiple tasks are to be started or joined at once, it is more
+    // efficient to take the helper thread lock once and use these methods.
+    bool startWithLockHeld(AutoLockHelperThreadState& locked);
+    void joinWithLockHeld(AutoLockHelperThreadState& locked);
+
+    // Instead of dispatching to a helper, run the task on the current thread.
+    void runFromActiveCooperatingThread(JSRuntime* rt);
+
+    // Dispatch a cancelation request.
+    enum CancelMode { CancelNoWait, CancelAndWait};
+    void cancel(CancelMode mode = CancelNoWait) {
+        cancel_ = true;
+        if (mode == CancelAndWait)
+            join();
+    }
+
+    // Check if a task is actively running.
+    bool isRunningWithLockHeld(const AutoLockHelperThreadState& locked) const;
+    bool isRunning() const;
+
+    // This should be friended to HelperThread, but cannot be because it
+    // would introduce several circular dependencies.
+  public:
+    void runFromHelperThread(AutoLockHelperThreadState& locked);
+};
+
+} /* namespace js */
+#endif /* gc_GCParallelTask_h */

js/src/gc/GCRuntime.h

@@ -10,14 +10,16 @@
 #include "mozilla/Atomics.h"
 #include "mozilla/EnumSet.h"
 
-#include "jsfriendapi.h"
-#include "jsgc.h"
+#include "jsatom.h"
 
-#include "gc/Heap.h"
+#include "gc/ArenaList.h"
+#include "gc/AtomMarking.h"
+#include "gc/GCHelperState.h"
+#include "gc/GCMarker.h"
+#include "gc/GCParallelTask.h"
 #include "gc/Nursery.h"
 #include "gc/Statistics.h"
 #include "gc/StoreBuffer.h"
-#include "gc/Tracer.h"
 #include "js/GCAnnotations.h"
 
 namespace js {
@@ -38,6 +40,7 @@ class AutoTraceSession;
 class MarkingValidator;
 class AutoTraceSession;
 struct MovingTracer;
+enum class ShouldCheckThresholds;
 class SweepGroupsIter;
 class WeakCacheSweepIterator;
 
@@ -703,6 +706,34 @@ class MemoryCounter
     void reset();
 };
 
+// A singly linked list of zones.
+class ZoneList
+{
+    static Zone * const End;
+
+    Zone* head;
+    Zone* tail;
+
+  public:
+    ZoneList();
+    ~ZoneList();
+
+    bool isEmpty() const;
+    Zone* front() const;
+
+    void append(Zone* zone);
+    void transferFrom(ZoneList& other);
+    void removeFront();
+    void clear();
+
+  private:
+    explicit ZoneList(Zone* singleZone);
+    void check() const;
+
+    ZoneList(const ZoneList& other) = delete;
+    ZoneList& operator=(const ZoneList& other) = delete;
+};
+
 class GCRuntime
 {
   public:
@@ -1182,11 +1213,12 @@ class GCRuntime
     MemProfiler mMemProfiler;
 
   private:
-    // When empty, chunks reside in the emptyChunks pool and are re-used as
-    // needed or eventually expired if not re-used. The emptyChunks pool gets
-    // refilled from the background allocation task heuristically so that empty
-    // chunks should always available for immediate allocation without syscalls.
-    ChunkPool             emptyChunks_;
+    // When chunks are empty, they reside in the emptyChunks pool and are
+    // re-used as needed or eventually expired if not re-used. The emptyChunks
+    // pool gets refilled from the background allocation task heuristically so
+    // that empty chunks should always be available for immediate allocation
+    // without syscalls.
+    GCLockData<ChunkPool> emptyChunks_;
 
     // Chunks which have had some, but not all, of their arenas allocated live
     // in the available chunk lists. When all available arenas in a chunk have
@@ -1477,7 +1509,8 @@ class GCRuntime
 
     BackgroundAllocTask allocTask;
     BackgroundDecommitTask decommitTask;
-    GCHelperState helperState;
+
+    js::GCHelperState helperState;
 
     /*
      * During incremental sweeping, this field temporarily holds the arenas of

js/src/gc/Heap.h

@@ -11,8 +11,6 @@
 #include "mozilla/Atomics.h"
 #include "mozilla/Attributes.h"
 #include "mozilla/DebugOnly.h"
-#include "mozilla/EnumeratedArray.h"
-#include "mozilla/EnumeratedRange.h"
 #include "mozilla/PodOperations.h"
 
 #include <stddef.h>
@@ -24,12 +22,12 @@
 #include "jsutil.h"
 
 #include "ds/BitArray.h"
+#include "gc/AllocKind.h"
+#include "gc/GCEnum.h"
 #include "gc/Memory.h"
-#include "js/GCAPI.h"
 #include "js/HeapAPI.h"
 #include "js/RootingAPI.h"
 #include "js/TracingAPI.h"
-#include "js/TraceKind.h"
 
 #include "vm/Printer.h"
 
@@ -47,31 +45,13 @@ class AutoLockGC;
 class AutoLockGCBgAlloc;
 class FreeOp;
 
-extern bool
-RuntimeFromMainThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone);
-
-#ifdef DEBUG
-
-// Barriers can't be triggered during backend Ion compilation, which may run on
-// a helper thread.
-extern bool
-CurrentThreadIsIonCompiling();
-#endif
-
-// The return value indicates if anything was unmarked.
-extern bool
-UnmarkGrayCellRecursively(gc::Cell* cell, JS::TraceKind kind);
-
-
-extern void
-TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name);
-
 namespace gc {
 
 class Arena;
 class ArenaCellSet;
 class ArenaList;
 class SortedArenaList;
+class TenuredCell;
 struct Chunk;
 
 /*
@@ -79,297 +59,21 @@ struct Chunk;
  * estimated lifetime or lifetime requirements of objects allocated from that
  * site.
  */
-enum InitialHeap {
+enum InitialHeap : uint8_t {
     DefaultHeap,
     TenuredHeap
 };
 
-/* The GC allocation kinds. */
-// FIXME: uint8_t would make more sense for the underlying type, but causes
-// miscompilations in GCC (fixed in 4.8.5 and 4.9.3). See also bug 1143966.
-enum class AllocKind {
-    FIRST,
-    OBJECT_FIRST = FIRST,
-    FUNCTION = FIRST,
-    FUNCTION_EXTENDED,
-    OBJECT0,
-    OBJECT0_BACKGROUND,
-    OBJECT2,
-    OBJECT2_BACKGROUND,
-    OBJECT4,
-    OBJECT4_BACKGROUND,
-    OBJECT8,
-    OBJECT8_BACKGROUND,
-    OBJECT12,
-    OBJECT12_BACKGROUND,
-    OBJECT16,
-    OBJECT16_BACKGROUND,
-    OBJECT_LIMIT,
-    OBJECT_LAST = OBJECT_LIMIT - 1,
-    SCRIPT,
-    LAZY_SCRIPT,
-    SHAPE,
-    ACCESSOR_SHAPE,
-    BASE_SHAPE,
-    OBJECT_GROUP,
-    FAT_INLINE_STRING,
-    STRING,
-    EXTERNAL_STRING,
-    FAT_INLINE_ATOM,
-    ATOM,
-    SYMBOL,
-    BIGINT,
-    JITCODE,
-    SCOPE,
-    REGEXP_SHARED,
-    LIMIT,
-    LAST = LIMIT - 1
-};
-
-// Macro to enumerate the different allocation kinds supplying information about
-// the trace kind, C++ type and allocation size.
-#define FOR_EACH_OBJECT_ALLOCKIND(D) \
- /* AllocKind              TraceKind      TypeName           SizedType */ \
-    D(FUNCTION,            Object,        JSObject,          JSFunction) \
-    D(FUNCTION_EXTENDED,   Object,        JSObject,          FunctionExtended) \
-    D(OBJECT0,             Object,        JSObject,          JSObject_Slots0) \
-    D(OBJECT0_BACKGROUND,  Object,        JSObject,          JSObject_Slots0) \
-    D(OBJECT2,             Object,        JSObject,          JSObject_Slots2) \
-    D(OBJECT2_BACKGROUND,  Object,        JSObject,          JSObject_Slots2) \
-    D(OBJECT4,             Object,        JSObject,          JSObject_Slots4) \
-    D(OBJECT4_BACKGROUND,  Object,        JSObject,          JSObject_Slots4) \
-    D(OBJECT8,             Object,        JSObject,          JSObject_Slots8) \
-    D(OBJECT8_BACKGROUND,  Object,        JSObject,          JSObject_Slots8) \
-    D(OBJECT12,            Object,        JSObject,          JSObject_Slots12) \
-    D(OBJECT12_BACKGROUND, Object,        JSObject,          JSObject_Slots12) \
-    D(OBJECT16,            Object,        JSObject,          JSObject_Slots16) \
-    D(OBJECT16_BACKGROUND, Object,        JSObject,          JSObject_Slots16)
-
-#define FOR_EACH_NONOBJECT_ALLOCKIND(D) \
- /* AllocKind              TraceKind      TypeName           SizedType */ \
-    D(SCRIPT,              Script,        JSScript,          JSScript) \
-    D(LAZY_SCRIPT,         LazyScript,    js::LazyScript,    js::LazyScript) \
-    D(SHAPE,               Shape,         js::Shape,         js::Shape) \
-    D(ACCESSOR_SHAPE,      Shape,         js::AccessorShape, js::AccessorShape) \
-    D(BASE_SHAPE,          BaseShape,     js::BaseShape,     js::BaseShape) \
-    D(OBJECT_GROUP,        ObjectGroup,   js::ObjectGroup,   js::ObjectGroup) \
-    D(FAT_INLINE_STRING,   String,        JSFatInlineString, JSFatInlineString) \
-    D(STRING,              String,        JSString,          JSString) \
-    D(EXTERNAL_STRING,     String,        JSExternalString,  JSExternalString) \
-    D(FAT_INLINE_ATOM,     String,        js::FatInlineAtom, js::FatInlineAtom) \
-    D(ATOM,                String,        js::NormalAtom,    js::NormalAtom) \
-    D(SYMBOL,              Symbol,        JS::Symbol,        JS::Symbol) \
-    D(BIGINT,              BigInt,        JS::BigInt,        JS::BigInt) \
-    D(JITCODE,             JitCode,       js::jit::JitCode,  js::jit::JitCode) \
-    D(SCOPE,               Scope,         js::Scope,         js::Scope) \
-    D(REGEXP_SHARED,       RegExpShared,  js::RegExpShared,  js::RegExpShared)
-
-#define FOR_EACH_ALLOCKIND(D) \
-    FOR_EACH_OBJECT_ALLOCKIND(D) \
-    FOR_EACH_NONOBJECT_ALLOCKIND(D)
-
-static_assert(int(AllocKind::FIRST) == 0, "Various places depend on AllocKind starting at 0, "
-                                          "please audit them carefully!");
-static_assert(int(AllocKind::OBJECT_FIRST) == 0, "Various places depend on AllocKind::OBJECT_FIRST "
-                                                 "being 0, please audit them carefully!");
-
-inline bool
-IsAllocKind(AllocKind kind)
-{
-    return kind >= AllocKind::FIRST && kind <= AllocKind::LIMIT;
-}
-
-inline bool
-IsValidAllocKind(AllocKind kind)
-{
-    return kind >= AllocKind::FIRST && kind <= AllocKind::LAST;
-}
-
-inline bool
-IsObjectAllocKind(AllocKind kind)
-{
-    return kind >= AllocKind::OBJECT_FIRST && kind <= AllocKind::OBJECT_LAST;
-}
-
-inline bool
-IsShapeAllocKind(AllocKind kind)
-{
-    return kind == AllocKind::SHAPE || kind == AllocKind::ACCESSOR_SHAPE;
-}
-
-// Returns a sequence for use in a range-based for loop,
-// to iterate over all alloc kinds.
-inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT))
-AllAllocKinds()
-{
-    return mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT);
-}
-
-// Returns a sequence for use in a range-based for loop,
-// to iterate over all object alloc kinds.
-inline decltype(mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT))
-ObjectAllocKinds()
-{
-    return mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT);
-}
-
-// Returns a sequence for use in a range-based for loop,
-// to iterate over alloc kinds from |first| to |limit|, exclusive.
-inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT))
-SomeAllocKinds(AllocKind first = AllocKind::FIRST, AllocKind limit = AllocKind::LIMIT)
-{
-    MOZ_ASSERT(IsAllocKind(first), "|first| is not a valid AllocKind!");
-    MOZ_ASSERT(IsAllocKind(limit), "|limit| is not a valid AllocKind!");
-    return mozilla::MakeEnumeratedRange(first, limit);
-}
-
-// AllAllocKindArray<ValueType> gives an enumerated array of ValueTypes,
-// with each index corresponding to a particular alloc kind.
-template<typename ValueType> using AllAllocKindArray =
-    mozilla::EnumeratedArray<AllocKind, AllocKind::LIMIT, ValueType>;
-
-// ObjectAllocKindArray<ValueType> gives an enumerated array of ValueTypes,
-// with each index corresponding to a particular object alloc kind.
-template<typename ValueType> using ObjectAllocKindArray =
-    mozilla::EnumeratedArray<AllocKind, AllocKind::OBJECT_LIMIT, ValueType>;
-
-static inline JS::TraceKind
-MapAllocToTraceKind(AllocKind kind)
-{
-    static const JS::TraceKind map[] = {
-#define EXPAND_ELEMENT(allocKind, traceKind, type, sizedType) \
-        JS::TraceKind::traceKind,
-FOR_EACH_ALLOCKIND(EXPAND_ELEMENT)
-#undef EXPAND_ELEMENT
-    };
-
-    static_assert(MOZ_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT),
-                  "AllocKind-to-TraceKind mapping must be in sync");
-    return map[size_t(kind)];
-}
+/* Cells are aligned to CellAlignShift, so the largest tagged null pointer is: */
+const uintptr_t LargestTaggedNullCellPointer = (1 << CellAlignShift) - 1;
 
 /*
- * This must be an upper bound, but we do not need the least upper bound, so
- * we just exclude non-background objects.
+ * The minimum cell size ends up as twice the cell alignment because the mark
+ * bitmap contains one bit per CellBytesPerMarkBit bytes (which is equal to
+ * CellAlignBytes) and we need two mark bits per cell.
  */
-static const size_t MAX_BACKGROUND_FINALIZE_KINDS =
-    size_t(AllocKind::LIMIT) - size_t(AllocKind::OBJECT_LIMIT) / 2;
-
-/* Mark colors to pass to markIfUnmarked. */
-enum class MarkColor : uint32_t
-{
-    Black = 0,
-    Gray
-};
-
-class TenuredCell;
-
-// A GC cell is the base class for all GC things.
-struct Cell
-{
-  public:
-    MOZ_ALWAYS_INLINE bool isTenured() const { return !IsInsideNursery(this); }
-    MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const;
-    MOZ_ALWAYS_INLINE TenuredCell& asTenured();
-
-    MOZ_ALWAYS_INLINE bool isMarkedAny() const;
-    MOZ_ALWAYS_INLINE bool isMarkedBlack() const;
-    MOZ_ALWAYS_INLINE bool isMarkedGray() const;
-
-    inline JSRuntime* runtimeFromActiveCooperatingThread() const;
-
-    // Note: Unrestricted access to the runtime of a GC thing from an arbitrary
-    // thread can easily lead to races. Use this method very carefully.
-    inline JSRuntime* runtimeFromAnyThread() const;
-    inline JS::shadow::Runtime* shadowRuntimeFromAnyThread() const;
-
-    // May be overridden by GC thing kinds that have a compartment pointer.
-    inline JSCompartment* maybeCompartment() const { return nullptr; }
-
-    inline StoreBuffer* storeBuffer() const;
-
-    inline JS::TraceKind getTraceKind() const;
-
-    static MOZ_ALWAYS_INLINE bool needWriteBarrierPre(JS::Zone* zone);
-
-    template <class T>
-    inline bool is() const {
-        return getTraceKind() == JS::MapTypeToTraceKind<T>::kind;
-    }
-
-    template<class T>
-    inline T* as() {
-        MOZ_ASSERT(this->is<T>());
-        return static_cast<T*>(this);
-    }
-
-    template <class T>
-    inline const T* as() const {
-        MOZ_ASSERT(this->is<T>());
-        return static_cast<const T*>(this);
-    }
-
-#ifdef DEBUG
-    inline bool isAligned() const;
-    void dump(FILE* fp) const;
-    void dump() const;
-#endif
-
-  protected:
-    inline uintptr_t address() const;
-    inline Chunk* chunk() const;
-} JS_HAZ_GC_THING;
-
-// A GC TenuredCell gets behaviors that are valid for things in the Tenured
-// heap, such as access to the arena and mark bits.
-class TenuredCell : public Cell
-{
-  public:
-    // Construct a TenuredCell from a void*, making various sanity assertions.
-    static MOZ_ALWAYS_INLINE TenuredCell* fromPointer(void* ptr);
-    static MOZ_ALWAYS_INLINE const TenuredCell* fromPointer(const void* ptr);
-
-    // Mark bit management.
-    MOZ_ALWAYS_INLINE bool isMarkedAny() const;
-    MOZ_ALWAYS_INLINE bool isMarkedBlack() const;
-    MOZ_ALWAYS_INLINE bool isMarkedGray() const;
-
-    // The return value indicates if the cell went from unmarked to marked.
-    MOZ_ALWAYS_INLINE bool markIfUnmarked(MarkColor color = MarkColor::Black) const;
-    MOZ_ALWAYS_INLINE void markBlack() const;
-    MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src);
-
-    // Access to the arena.
-    inline Arena* arena() const;
-    inline AllocKind getAllocKind() const;
-    inline JS::TraceKind getTraceKind() const;
-    inline JS::Zone* zone() const;
-    inline JS::Zone* zoneFromAnyThread() const;
-    inline bool isInsideZone(JS::Zone* zone) const;
-
-    MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZone() const {
-        return JS::shadow::Zone::asShadowZone(zone());
-    }
-    MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZoneFromAnyThread() const {
-        return JS::shadow::Zone::asShadowZone(zoneFromAnyThread());
-    }
-
-    static MOZ_ALWAYS_INLINE void readBarrier(TenuredCell* thing);
-    static MOZ_ALWAYS_INLINE void writeBarrierPre(TenuredCell* thing);
-
-    static MOZ_ALWAYS_INLINE void writeBarrierPost(void* cellp, TenuredCell* prior,
-                                                   TenuredCell* next);
-
-    // Default implementation for kinds that don't require fixup.
-    void fixupAfterMovingGC() {}
-
-#ifdef DEBUG
-    inline bool isAligned() const;
-#endif
-};
-
-/* Cells are aligned to CellShift, so the largest tagged null pointer is: */
-const uintptr_t LargestTaggedNullCellPointer = (1 << CellShift) - 1;
+const size_t MarkBitsPerCell = 2;
+const size_t MinCellSize = CellBytesPerMarkBit * MarkBitsPerCell;
 
 constexpr size_t
 DivideAndRoundUp(size_t numerator, size_t divisor) {
@@ -380,12 +84,11 @@ const size_t ArenaCellCount = ArenaSize / CellSize;
 static_assert(ArenaSize % CellSize == 0, "Arena size must be a multiple of cell size");
 
 /*
- * The mark bitmap has one bit per each GC cell. For multi-cell GC things this
- * wastes space but allows to avoid expensive devisions by thing's size when
- * accessing the bitmap. In addition this allows to use some bits for colored
- * marking during the cycle GC.
+ * The mark bitmap has one bit per each possible cell start position. This
+ * wastes some space for larger GC things but allows us to avoid division by the
+ * cell's size when accessing the bitmap.
  */
-const size_t ArenaBitmapBits = ArenaCellCount;
+const size_t ArenaBitmapBits = ArenaSize / CellBytesPerMarkBit;
 const size_t ArenaBitmapBytes = DivideAndRoundUp(ArenaBitmapBits, 8);
 const size_t ArenaBitmapWords = DivideAndRoundUp(ArenaBitmapBits, JS_BITS_PER_WORD);
 
@@ -478,7 +181,6 @@ class FreeSpan
         }
         checkSpan(arena);
         JS_EXTRA_POISON(reinterpret_cast<void*>(thing), JS_ALLOCATED_TENURED_PATTERN, thingSize);
-        MemProfiler::SampleTenured(reinterpret_cast<void*>(thing), thingSize);
         return reinterpret_cast<TenuredCell*>(thing);
     }
 
@@ -899,15 +601,6 @@ static_assert(ArenasPerChunk == 62, "Do not accidentally change our heap's densi
 static_assert(ArenasPerChunk == 252, "Do not accidentally change our heap's density.");
 #endif
 
-static inline void
-AssertValidColorBit(const TenuredCell* thing, ColorBit colorBit)
-{
-#ifdef DEBUG
-    Arena* arena = thing->arena();
-    MOZ_ASSERT(unsigned(colorBit) < arena->getThingSize() / CellBytesPerMarkBit);
-#endif
-}
-
 /* A chunk bitmap contains enough mark bits for all the cells in a chunk. */
 struct ChunkBitmap
 {
@@ -919,11 +612,11 @@ struct ChunkBitmap
     MOZ_ALWAYS_INLINE void getMarkWordAndMask(const TenuredCell* cell, ColorBit colorBit,
                                               uintptr_t** wordp, uintptr_t* maskp)
     {
+        MOZ_ASSERT(size_t(colorBit) < MarkBitsPerCell);
         detail::GetGCThingMarkWordAndMask(uintptr_t(cell), colorBit, wordp, maskp);
     }
 
     MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool markBit(const TenuredCell* cell, ColorBit colorBit) {
-        AssertValidColorBit(cell, colorBit);
         uintptr_t* word, mask;
         getMarkWordAndMask(cell, colorBit, &word, &mask);
         return *word & mask;
@@ -1172,94 +865,6 @@ Arena::chunk() const
     return Chunk::fromAddress(address());
 }
 
-MOZ_ALWAYS_INLINE const TenuredCell&
-Cell::asTenured() const
-{
-    MOZ_ASSERT(isTenured());
-    return *static_cast<const TenuredCell*>(this);
-}
-
-MOZ_ALWAYS_INLINE TenuredCell&
-Cell::asTenured()
-{
-    MOZ_ASSERT(isTenured());
-    return *static_cast<TenuredCell*>(this);
-}
-
-MOZ_ALWAYS_INLINE bool
-Cell::isMarkedAny() const
-{
-    return !isTenured() || asTenured().isMarkedAny();
-}
-
-MOZ_ALWAYS_INLINE bool
-Cell::isMarkedBlack() const
-{
-    return !isTenured() || asTenured().isMarkedBlack();
-}
-
-MOZ_ALWAYS_INLINE bool
-Cell::isMarkedGray() const
-{
-    return isTenured() && asTenured().isMarkedGray();
-}
-
-inline JSRuntime*
-Cell::runtimeFromMainThread() const
-{
-    JSRuntime* rt = chunk()->trailer.runtime;
-    MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
-    return rt;
-}
-
-inline JS::shadow::Runtime*
-Cell::shadowRuntimeFromMainThread() const
-{
-    return reinterpret_cast<JS::shadow::Runtime*>(runtimeFromMainThread());
-}
-
-inline JSRuntime*
-Cell::runtimeFromAnyThread() const
-{
-    return chunk()->trailer.runtime;
-}
-
-inline JS::shadow::Runtime*
-Cell::shadowRuntimeFromAnyThread() const
-{
-    return reinterpret_cast<JS::shadow::Runtime*>(runtimeFromAnyThread());
-}
-
-inline uintptr_t
-Cell::address() const
-{
-    uintptr_t addr = uintptr_t(this);
-    MOZ_ASSERT(addr % CellSize == 0);
-    MOZ_ASSERT(Chunk::withinValidRange(addr));
-    return addr;
-}
-
-Chunk*
-Cell::chunk() const
-{
-    uintptr_t addr = uintptr_t(this);
-    MOZ_ASSERT(addr % CellSize == 0);
-    addr &= ~ChunkMask;
-    return reinterpret_cast<Chunk*>(addr);
-}
-
-inline StoreBuffer*
-Cell::storeBuffer() const
-{
-    return chunk()->trailer.storeBuffer;
-}
-
-inline JS::TraceKind
-Cell::getTraceKind() const
-{
-    return isTenured() ? asTenured().getTraceKind() : JS::TraceKind::Object;
-}
-
 inline bool
 InFreeList(Arena* arena, void* thing)
 {
@@ -1268,187 +873,6 @@ InFreeList(Arena* arena, void* thing)
     return arena->inFreeList(addr);
 }
 
-/* static */ MOZ_ALWAYS_INLINE bool
-Cell::needWriteBarrierPre(JS::Zone* zone) {
-    return JS::shadow::Zone::asShadowZone(zone)->needsIncrementalBarrier();
-}
-
-/* static */ MOZ_ALWAYS_INLINE TenuredCell*
-TenuredCell::fromPointer(void* ptr)
-{
-    MOZ_ASSERT(static_cast<TenuredCell*>(ptr)->isTenured());
-    return static_cast<TenuredCell*>(ptr);
-}
-
-/* static */ MOZ_ALWAYS_INLINE const TenuredCell*
-TenuredCell::fromPointer(const void* ptr)
-{
-    MOZ_ASSERT(static_cast<const TenuredCell*>(ptr)->isTenured());
-    return static_cast<const TenuredCell*>(ptr);
-}
-
-bool
-TenuredCell::isMarkedAny() const
-{
-    MOZ_ASSERT(arena()->allocated());
-    return chunk()->bitmap.isMarkedAny(this);
-}
-
-bool
-TenuredCell::isMarkedBlack() const
-{
-    MOZ_ASSERT(arena()->allocated());
-    return chunk()->bitmap.isMarkedBlack(this);
-}
-
-bool
-TenuredCell::isMarkedGray() const
-{
-    MOZ_ASSERT(arena()->allocated());
-    return chunk()->bitmap.isMarkedGray(this);
-}
-
-bool
-TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const
-{
-    return chunk()->bitmap.markIfUnmarked(this, color);
-}
-
-void
-TenuredCell::unmark(uint32_t color) const
-{
-    MOZ_ASSERT(color != BLACK);
-    AssertValidColor(this, color);
-    chunk()->bitmap.unmark(this, color);
-}
-
-void
-TenuredCell::copyMarkBitsFrom(const TenuredCell* src)
-{
-    ChunkBitmap& bitmap = chunk()->bitmap;
-    bitmap.copyMarkBit(this, src, BLACK);
-    bitmap.copyMarkBit(this, src, GRAY);
-}
-
-inline Arena*
-TenuredCell::arena() const
-{
-    MOZ_ASSERT(isTenured());
-    uintptr_t addr = address();
-    addr &= ~ArenaMask;
-    return reinterpret_cast<Arena*>(addr);
-}
-
-AllocKind
-TenuredCell::getAllocKind() const
-{
-    return arena()->getAllocKind();
-}
-
-JS::TraceKind
-TenuredCell::getTraceKind() const
-{
-    return MapAllocToTraceKind(getAllocKind());
-}
-
-JS::Zone*
-TenuredCell::zone() const
-{
-    JS::Zone* zone = arena()->zone;
-    MOZ_ASSERT(CurrentThreadCanAccessZone(zone));
-    return zone;
-}
-
-JS::Zone*
-TenuredCell::zoneFromAnyThread() const
-{
-    return arena()->zone;
-}
-
-bool
-TenuredCell::isInsideZone(JS::Zone* zone) const
-{
-    return zone == arena()->zone;
-}
-
-/* static */ MOZ_ALWAYS_INLINE void
-TenuredCell::readBarrier(TenuredCell* thing)
-{
-    MOZ_ASSERT(!CurrentThreadIsIonCompiling());
-    MOZ_ASSERT(thing);
-
-    // It would be good if barriers were never triggered during collection, but
-    // at the moment this can happen e.g. when rekeying tables containing
-    // read-barriered GC things after a moving GC.
-    //
-    // TODO: Fix this and assert we're not collecting if we're on the main
-    // thread.
-
-    JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread();
-    if (shadowZone->needsIncrementalBarrier()) {
-        // Barriers are only enabled on the main thread and are disabled while collecting.
-        MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone));
-        Cell* tmp = thing;
-        TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "read barrier");
-        MOZ_ASSERT(tmp == thing);
-    }
-
-    if (thing->isMarkedGray()) {
-        // There shouldn't be anything marked grey unless we're on the active thread.
-        MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread()));
-        if (!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone))
-            UnmarkGrayCellRecursively(thing, thing->getTraceKind());
-    }
-}
-
-void
-AssertSafeToSkipBarrier(TenuredCell* thing);
-
-/* static */ MOZ_ALWAYS_INLINE void
-TenuredCell::writeBarrierPre(TenuredCell* thing)
-{
-    MOZ_ASSERT(!CurrentThreadIsIonCompiling());
-    if (!thing)
-        return;
-
-    JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread();
-    if (shadowZone->needsIncrementalBarrier()) {
-        MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone));
-        Cell* tmp = thing;
-        TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "pre barrier");
-        MOZ_ASSERT(tmp == thing);
-    }
-}
-
-static MOZ_ALWAYS_INLINE void
-AssertValidToSkipBarrier(TenuredCell* thing)
-{
-    MOZ_ASSERT(!IsInsideNursery(thing));
-    MOZ_ASSERT_IF(thing, MapAllocToTraceKind(thing->getAllocKind()) != JS::TraceKind::Object);
-}
-
-/* static */ MOZ_ALWAYS_INLINE void
-TenuredCell::writeBarrierPost(void* cellp, TenuredCell* prior, TenuredCell* next)
-{
-    AssertValidToSkipBarrier(next);
-}
-
-#ifdef DEBUG
-bool
-Cell::isAligned() const
-{
-    if (!isTenured())
-        return true;
-    return asTenured().isAligned();
-}
-
-bool
-TenuredCell::isAligned() const
-{
-    return Arena::isAligned(address(), arena()->getThingSize());
-}
-#endif
-
 static const int32_t ChunkLocationOffsetFromLastByte =
     int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask);
 

js/src/gc/Iteration-inl.h

@@ -0,0 +1,121 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal iterators for various data structures.
+ */
+
+#ifndef gc_Iteration_inl_h
+#define gc_Iteration_inl_h
+
+#include "jsgcinlines.h"
+
+namespace js {
+namespace gc {
+
+class ArenaCellIterUnderGC : public ArenaCellIterImpl
+{
+  public:
+    explicit ArenaCellIterUnderGC(Arena* arena)
+      : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier)
+    {
+        MOZ_ASSERT(CurrentThreadIsPerformingGC());
+    }
+};
+
+class ArenaCellIterUnderFinalize : public ArenaCellIterImpl
+{
+  public:
+    explicit ArenaCellIterUnderFinalize(Arena* arena)
+      : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier)
+    {
+        MOZ_ASSERT(CurrentThreadIsGCSweeping());
+    }
+};
+
+class ArenaCellIterUnbarriered : public ArenaCellIterImpl
+{
+  public:
+    explicit ArenaCellIterUnbarriered(Arena* arena)
+      : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier)
+    {}
+};
+
+class GrayObjectIter : public ZoneCellIter<js::gc::TenuredCell> {
+  public:
+    explicit GrayObjectIter(JS::Zone* zone, AllocKind kind) : ZoneCellIter<js::gc::TenuredCell>() {
+        initForTenuredIteration(zone, kind);
+    }
+
+    JSObject* get() const { return ZoneCellIter<js::gc::TenuredCell>::get<JSObject>(); }
+    operator JSObject*() const { return get(); }
+    JSObject* operator ->() const { return get(); }
+};
+
+class GCZonesIter
+{
+  private:
+    ZonesIter zone;
+
+  public:
+    explicit GCZonesIter(JSRuntime* rt, ZoneSelector selector = WithAtoms) : zone(rt, selector) {
+        MOZ_ASSERT(JS::CurrentThreadIsHeapBusy());
+        if (!zone->isCollectingFromAnyThread())
+            next();
+    }
+
+    bool done() const { return zone.done(); }
+
+    void next() {
+        MOZ_ASSERT(!done());
+        do {
+            zone.next();
+        } while (!zone.done() && !zone->isCollectingFromAnyThread());
+    }
+
+    JS::Zone* get() const {
+        MOZ_ASSERT(!done());
+        return zone;
+    }
+
+    operator JS::Zone*() const { return get(); }
+    JS::Zone* operator->() const { return get(); }
+};
+
+typedef CompartmentsIterT<GCZonesIter> GCCompartmentsIter;
+
+/* Iterates over all zones in the current sweep group. */
+class SweepGroupZonesIter {
+    JS::Zone* current;
+
+  public:
+    explicit SweepGroupZonesIter(JSRuntime* rt) {
+        MOZ_ASSERT(CurrentThreadIsPerformingGC());
+        current = rt->gc.getCurrentSweepGroup();
+    }
+
+    bool done() const { return !current; }
+
+    void next() {
+        MOZ_ASSERT(!done());
+        current = current->nextNodeInGroup();
+    }
+
+    JS::Zone* get() const {
+        MOZ_ASSERT(!done());
+        return current;
+    }
+
+    operator JS::Zone*() const { return get(); }
+    JS::Zone* operator->() const { return get(); }
+};
+
+typedef CompartmentsIterT<SweepGroupZonesIter> SweepGroupCompartmentsIter;
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_Iteration_h

js/src/gc/Iteration.cpp

@@ -4,10 +4,11 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include "gc/Iteration-inl.h"
+
 #include "mozilla/DebugOnly.h"
 
 #include "jscompartment.h"
-#include "jsgc.h"
 
 #include "gc/GCInternals.h"
 #include "js/HashTable.h"

js/src/gc/Marking-inl.h

@@ -0,0 +1,128 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef gc_Marking_inl_h
+#define gc_Marking_inl_h
+
+#include "gc/Marking.h"
+
+#include "gc/RelocationOverlay.h"
+
+namespace js {
+namespace gc {
+
+template <typename T>
+struct MightBeForwarded
+{
+    static_assert(mozilla::IsBaseOf<Cell, T>::value,
+                  "T must derive from Cell");
+    static_assert(!mozilla::IsSame<Cell, T>::value && !mozilla::IsSame<TenuredCell, T>::value,
+                  "T must not be Cell or TenuredCell");
+
+    static const bool value = mozilla::IsBaseOf<JSObject, T>::value ||
+                              mozilla::IsBaseOf<Shape, T>::value ||
+                              mozilla::IsBaseOf<BaseShape, T>::value ||
+                              mozilla::IsBaseOf<JSString, T>::value ||
+                              mozilla::IsBaseOf<JSScript, T>::value ||
+                              mozilla::IsBaseOf<js::LazyScript, T>::value ||
+                              mozilla::IsBaseOf<js::Scope, T>::value ||
+                              mozilla::IsBaseOf<js::RegExpShared, T>::value;
+};
+
+template <typename T>
+inline bool
+IsForwarded(T* t)
+{
+    RelocationOverlay* overlay = RelocationOverlay::fromCell(t);
+    if (!MightBeForwarded<T>::value) {
+        MOZ_ASSERT(!overlay->isForwarded());
+        return false;
+    }
+
+    return overlay->isForwarded();
+}
+
+struct IsForwardedFunctor : public BoolDefaultAdaptor<Value, false> {
+    template <typename T> bool operator()(T* t) { return IsForwarded(t); }
+};
+
+inline bool
+IsForwarded(const JS::Value& value)
+{
+    return DispatchTyped(IsForwardedFunctor(), value);
+}
+
+template <typename T>
+inline T*
+Forwarded(T* t)
+{
+    RelocationOverlay* overlay = RelocationOverlay::fromCell(t);
+    MOZ_ASSERT(overlay->isForwarded());
+    return reinterpret_cast<T*>(overlay->forwardingAddress());
+}
+
+struct ForwardedFunctor : public IdentityDefaultAdaptor<Value> {
+    template <typename T> inline Value operator()(T* t) {
+        return js::gc::RewrapTaggedPointer<Value, T>::wrap(Forwarded(t));
+    }
+};
+
+inline Value
+Forwarded(const JS::Value& value)
+{
+    return DispatchTyped(ForwardedFunctor(), value);
+}
+
+template <typename T>
+inline T
+MaybeForwarded(T t)
+{
+    if (IsForwarded(t))
+        t = Forwarded(t);
+    MakeAccessibleAfterMovingGC(t);
+    return t;
+}
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+
+template <typename T>
+inline bool
+IsGCThingValidAfterMovingGC(T* t)
+{
+    return !IsInsideNursery(t) && !RelocationOverlay::isCellForwarded(t);
+}
+
+template <typename T>
+inline void
+CheckGCThingAfterMovingGC(T* t)
+{
+    if (t)
+        MOZ_RELEASE_ASSERT(IsGCThingValidAfterMovingGC(t));
+}
+
+template <typename T>
+inline void
+CheckGCThingAfterMovingGC(const ReadBarriered<T*>& t)
+{
+    CheckGCThingAfterMovingGC(t.unbarrieredGet());
+}
+
+struct CheckValueAfterMovingGCFunctor : public VoidDefaultAdaptor<Value> {
+    template <typename T> void operator()(T* t) { CheckGCThingAfterMovingGC(t); }
+};
+
+inline void
+CheckValueAfterMovingGC(const JS::Value& value)
+{
+    DispatchTyped(CheckValueAfterMovingGCFunctor(), value);
+}
+
+#endif // JSGC_HASH_TABLE_CHECKS
+
+} /* namespace gc */
+} /* namespace js */
+
+#endif // gc_Marking_inl_h

js/src/gc/Marking.cpp

@@ -4,7 +4,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#include "gc/Marking.h"
+#include "gc/Marking-inl.h"
 
 #include "mozilla/DebugOnly.h"
 #include "mozilla/IntegerRange.h"
@@ -12,7 +12,6 @@
 #include "mozilla/ScopeExit.h"
 #include "mozilla/TypeTraits.h"
 
-#include "jsgc.h"
 #include "jsprf.h"
 
 #include "builtin/ModuleObject.h"
@@ -36,8 +35,8 @@
 
 #include "jscompartmentinlines.h"
 #include "jsgcinlines.h"
-#include "jsobjinlines.h"
 
+#include "gc/Iteration-inl.h"
 #include "gc/Nursery-inl.h"
 #include "vm/NativeObject-inl.h"
 #include "vm/String-inl.h"
@@ -975,15 +974,25 @@ js::GCMarker::traverseEdge(S source, const T& thing)
     DispatchTyped(TraverseEdgeFunctor<T, S>(), thing, this, source);
 }
 
+namespace {
+
+template <typename T> struct ParticipatesInCC {};
+#define EXPAND_PARTICIPATES_IN_CC(_, type, addToCCKind) \
+    template <> struct ParticipatesInCC<type> { static const bool value = addToCCKind; };
+JS_FOR_EACH_TRACEKIND(EXPAND_PARTICIPATES_IN_CC)
+#undef EXPAND_PARTICIPATES_IN_CC
+
+} // namespace
+
 template <typename T>
 bool
 js::GCMarker::mark(T* thing)
 {
     AssertShouldMarkInZone(thing);
-    MOZ_ASSERT(!IsInsideNursery(gc::TenuredCell::fromPointer(thing)));
-    return gc::ParticipatesInCC<T>::value
-           ? gc::TenuredCell::fromPointer(thing)->markIfUnmarked(markColor())
-           : gc::TenuredCell::fromPointer(thing)->markIfUnmarked(gc::MarkColor::Black);
+    MOZ_ASSERT(!IsInsideNursery(TenuredCell::fromPointer(thing)));
+    return ParticipatesInCC<T>::value
+           ? TenuredCell::fromPointer(thing)->markIfUnmarked(markColor())
+           : TenuredCell::fromPointer(thing)->markIfUnmarked(MarkColor::Black);
 }
 
 
@@ -2523,6 +2532,12 @@ js::TenuringTracer::traceSlots(Value* vp, Value* end)
         traverse(vp);
 }
 
+inline void
+js::TenuringTracer::traceSlots(JS::Value* vp, uint32_t nslots)
+{
+    traceSlots(vp, vp + nslots);
+}
+
 #ifdef DEBUG
 static inline ptrdiff_t
 OffsetToChunkEnd(void* p)

js/src/gc/Marking.h

@@ -4,408 +4,50 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+/*
+ * Marking and sweeping APIs for use by implementations of different GC cell
+ * kinds.
+ */
+
 #ifndef gc_Marking_h
 #define gc_Marking_h
 
-#include "mozilla/HashFunctions.h"
-#include "mozilla/Move.h"
-
-#include "jsfriendapi.h"
-
-#include "ds/OrderedHashTable.h"
-#include "gc/Heap.h"
-#include "gc/Tracer.h"
-#include "js/GCAPI.h"
-#include "js/HeapAPI.h"
-#include "js/SliceBudget.h"
-#include "js/TracingAPI.h"
 #include "vm/TaggedProto.h"
 
 class JSLinearString;
 class JSRope;
+struct JSRuntime;
+class JSTracer;
+
 namespace js {
 class BaseShape;
 class GCMarker;
 class LazyScript;
 class NativeObject;
 class ObjectGroup;
+class Shape;
 class WeakMapBase;
-namespace gc {
-class Arena;
-} // namespace gc
+
 namespace jit {
 class JitCode;
 } // namespace jit
 
-static const size_t NON_INCREMENTAL_MARK_STACK_BASE_CAPACITY = 4096;
-static const size_t INCREMENTAL_MARK_STACK_BASE_CAPACITY = 32768;
-
-/*
- * When the native stack is low, the GC does not call js::TraceChildren to mark
- * the reachable "children" of the thing. Rather the thing is put aside and
- * js::TraceChildren is called later with more space on the C stack.
- *
- * To implement such delayed marking of the children with minimal overhead for
- * the normal case of sufficient native stack, the code adds a field per arena.
- * The field markingDelay->link links all arenas with delayed things into a
- * stack list with the pointer to stack top in GCMarker::unmarkedArenaStackTop.
- * GCMarker::delayMarkingChildren adds arenas to the stack as necessary while
- * markDelayedChildren pops the arenas from the stack until it empties.
- */
-class MarkStack
-{
-    friend class GCMarker;
-
-    uintptr_t* stack_;
-    uintptr_t* tos_;
-    uintptr_t* end_;
-
-    // The capacity we start with and reset() to.
-    size_t baseCapacity_;
-    size_t maxCapacity_;
-
-  public:
-    explicit MarkStack(size_t maxCapacity)
-      : stack_(nullptr),
-        tos_(nullptr),
-        end_(nullptr),
-        baseCapacity_(0),
-        maxCapacity_(maxCapacity)
-    {}
-
-        LastTag = TempRopeTag
-    };
-
-    static const uintptr_t TagMask = 7;
-    static_assert(TagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags.");
-    static_assert(TagMask <= gc::CellAlignMask, "The tag mask must be embeddable in a Cell*.");
-
-    class TaggedPtr
-    {
-        uintptr_t bits;
-
-        Cell* ptr() const;
-
-      public:
-        TaggedPtr(Tag tag, Cell* ptr);
-        Tag tag() const;
-        template <typename T> T* as() const;
-        JSObject* asValueArrayObject() const;
-        JSObject* asSavedValueArrayObject() const;
-        JSRope* asTempRope() const;
-    };
-
-    struct ValueArray
-    {
-        ValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end);
-
-        HeapSlot* end;
-        HeapSlot* start;
-        TaggedPtr ptr;
-    };
-
-    struct SavedValueArray
-    {
-        SavedValueArray(JSObject* obj, size_t index, HeapSlot::Kind kind);
-
-        uintptr_t kind;
-        uintptr_t index;
-        TaggedPtr ptr;
-    };
-
-    explicit MarkStack(size_t maxCapacity = DefaultCapacity);
-    ~MarkStack();
-
-    static const size_t DefaultCapacity = SIZE_MAX;
-
-    size_t capacity() { return end_ - stack_; }
-
-    ptrdiff_t position() const { return tos_ - stack_; }
-
-    void setStack(uintptr_t* stack, size_t tosIndex, size_t capacity) {
-        stack_ = stack;
-        tos_ = stack + tosIndex;
-        end_ = stack + capacity;
-    }
-
-    MOZ_MUST_USE bool init(JSGCMode gcMode);
-
-    void setBaseCapacity(JSGCMode mode);
-    size_t maxCapacity() const { return maxCapacity_; }
-    void setMaxCapacity(size_t maxCapacity);
-
-    MOZ_MUST_USE bool push(uintptr_t item) {
-        if (tos_ == end_) {
-            if (!enlarge(1))
-                return false;
-        }
-        MOZ_ASSERT(tos_ < end_);
-        *tos_++ = item;
-        return true;
-    }
-
-    MOZ_MUST_USE bool push(uintptr_t item1, uintptr_t item2, uintptr_t item3) {
-        uintptr_t* nextTos = tos_ + 3;
-        if (nextTos > end_) {
-            if (!enlarge(3))
-                return false;
-            nextTos = tos_ + 3;
-        }
-        MOZ_ASSERT(nextTos <= end_);
-        tos_[0] = item1;
-        tos_[1] = item2;
-        tos_[2] = item3;
-        tos_ = nextTos;
-        return true;
-    }
-
-    bool isEmpty() const {
-        return tos_ == stack_;
-    }
-
-    uintptr_t pop() {
-        MOZ_ASSERT(!isEmpty());
-        return *--tos_;
-    }
-
-    void reset();
-
-    /* Grow the stack, ensuring there is space for at least count elements. */
-    MOZ_MUST_USE bool enlarge(unsigned count);
-
-    void setGCMode(JSGCMode gcMode);
-
-    size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
-};
-
-namespace gc {
-
-struct WeakKeyTableHashPolicy {
-    typedef JS::GCCellPtr Lookup;
-    static HashNumber hash(const Lookup& v, const mozilla::HashCodeScrambler&) {
-        return mozilla::HashGeneric(v.asCell());
-    }
-    static bool match(const JS::GCCellPtr& k, const Lookup& l) { return k == l; }
-    static bool isEmpty(const JS::GCCellPtr& v) { return !v; }
-    static void makeEmpty(JS::GCCellPtr* vp) { *vp = nullptr; }
-};
-
-struct WeakMarkable {
-    WeakMapBase* weakmap;
-    JS::GCCellPtr key;
-
-    WeakMarkable(WeakMapBase* weakmapArg, JS::GCCellPtr keyArg)
-      : weakmap(weakmapArg), key(keyArg) {}
-};
-
-using WeakEntryVector = Vector<WeakMarkable, 2, js::SystemAllocPolicy>;
-
-using WeakKeyTable = OrderedHashMap<JS::GCCellPtr,
-                                    WeakEntryVector,
-                                    WeakKeyTableHashPolicy,
-                                    js::SystemAllocPolicy>;
-
-} /* namespace gc */
-
-class GCMarker : public JSTracer
-{
-  public:
-    explicit GCMarker(JSRuntime* rt);
-    MOZ_MUST_USE bool init(JSGCMode gcMode);
-
-    void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); }
-    size_t maxCapacity() const { return stack.maxCapacity(); }
-
-    void start();
-    void stop();
-    void reset();
-
-    // Mark the given GC thing and traverse its children at some point.
-    template <typename T> void traverse(T thing);
-
-    // Calls traverse on target after making additional assertions.
-    template <typename S, typename T> void traverseEdge(S source, T* target);
-    template <typename S, typename T> void traverseEdge(S source, const T& target);
-
-    // Notes a weak graph edge for later sweeping.
-    template <typename T> void noteWeakEdge(T* edge);
-
-    /*
-     * Care must be taken changing the mark color from gray to black. The cycle
-     * collector depends on the invariant that there are no black to gray edges
-     * in the GC heap. This invariant lets the CC not trace through black
-     * objects. If this invariant is violated, the cycle collector may free
-     * objects that are still reachable.
-     */
-    void setMarkColorGray() {
-        MOZ_ASSERT(isDrained());
-        MOZ_ASSERT(color == gc::MarkColor::Black);
-        color = gc::MarkColor::Gray;
-    }
-    void setMarkColorBlack() {
-        MOZ_ASSERT(isDrained());
-        MOZ_ASSERT(color == gc::MarkColor::Gray);
-        color = gc::MarkColor::Black;
-    }
-    gc::MarkColor markColor() const { return color; }
-
-    void enterWeakMarkingMode();
-    void leaveWeakMarkingMode();
-    void abortLinearWeakMarking() {
-        leaveWeakMarkingMode();
-        linearWeakMarkingDisabled_ = true;
-    }
-
-    void delayMarkingArena(gc::Arena* arena);
-    void delayMarkingChildren(const void* thing);
-    void markDelayedChildren(gc::Arena* arena);
-    MOZ_MUST_USE bool markDelayedChildren(SliceBudget& budget);
-    bool hasDelayedChildren() const {
-        return !!unmarkedArenaStackTop;
-    }
-
-    bool isDrained() {
-        return isMarkStackEmpty() && !unmarkedArenaStackTop;
-    }
-
-    MOZ_MUST_USE bool drainMarkStack(SliceBudget& budget);
-
-    void setGCMode(JSGCMode mode) { stack.setGCMode(mode); }
-
-    size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
-
-#ifdef DEBUG
-    bool shouldCheckCompartments() { return strictCompartmentChecking; }
-#endif
-
-    void markEphemeronValues(gc::Cell* markedCell, gc::WeakEntryVector& entry);
-
-  private:
-#ifdef DEBUG
-    void checkZone(void* p);
-#else
-    void checkZone(void* p) {}
-#endif
-
-    /*
-     * We use a common mark stack to mark GC things of different types and use
-     * the explicit tags to distinguish them when it cannot be deduced from
-     * the context of push or pop operation.
-     */
-    enum StackTag {
-        ValueArrayTag,
-        ObjectTag,
-        GroupTag,
-        SavedValueArrayTag,
-        JitCodeTag,
-        ScriptTag,
-        LastTag = JitCodeTag
-    };
-
-    static const uintptr_t StackTagMask = 7;
-    static_assert(StackTagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags.");
-    static_assert(StackTagMask <= gc::CellMask, "The tag mask must be embeddable in a Cell*.");
-
-    // Push an object onto the stack for later tracing and assert that it has
-    // already been marked.
-    void repush(JSObject* obj) {
-        MOZ_ASSERT(gc::TenuredCell::fromPointer(obj)->isMarked(markColor()));
-        pushTaggedPtr(ObjectTag, obj);
-    }
-
-    template <typename T> void markAndTraceChildren(T* thing);
-    template <typename T> void markAndPush(StackTag tag, T* thing);
-    template <typename T> void markAndScan(T* thing);
-    template <typename T> void markImplicitEdgesHelper(T oldThing);
-    template <typename T> void markImplicitEdges(T* oldThing);
-    void eagerlyMarkChildren(JSLinearString* str);
-    void eagerlyMarkChildren(JSRope* rope);
-    void eagerlyMarkChildren(JSString* str);
-    void eagerlyMarkChildren(LazyScript *thing);
-    void eagerlyMarkChildren(Shape* shape);
-    void eagerlyMarkChildren(Scope* scope);
-    void lazilyMarkChildren(ObjectGroup* group);
-
-    // We may not have concrete types yet, so this has to be outside the header.
-    template <typename T>
-    void dispatchToTraceChildren(T* thing);
-
-    // Mark the given GC thing, but do not trace its children. Return true
-    // if the thing became marked.
-    template <typename T>
-    MOZ_MUST_USE bool mark(T* thing);
-
-    void pushTaggedPtr(StackTag tag, void* ptr) {
-        checkZone(ptr);
-        uintptr_t addr = reinterpret_cast<uintptr_t>(ptr);
-        MOZ_ASSERT(!(addr & StackTagMask));
-        if (!stack.push(addr | uintptr_t(tag)))
-            delayMarkingChildren(ptr);
-    }
-
-    void pushValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end) {
-        checkZone(obj);
-
-        MOZ_ASSERT(start <= end);
-        uintptr_t tagged = reinterpret_cast<uintptr_t>(obj) | GCMarker::ValueArrayTag;
-        uintptr_t startAddr = reinterpret_cast<uintptr_t>(start);
-        uintptr_t endAddr = reinterpret_cast<uintptr_t>(end);
-
-        /*
-         * Push in the reverse order so obj will be on top. If we cannot push
-         * the array, we trigger delay marking for the whole object.
-         */
-        if (!stack.push(endAddr, startAddr, tagged))
-            delayMarkingChildren(obj);
-    }
-
-    bool isMarkStackEmpty() {
-        return stack.isEmpty();
-    }
-
-    MOZ_MUST_USE bool restoreValueArray(JSObject* obj, void** vpp, void** endp);
-    void saveValueRanges();
-    inline void processMarkStackTop(SliceBudget& budget);
-
-    /* The mark stack. Pointers in this stack are "gray" in the GC sense. */
-    MarkStack stack;
-
-    /* The color is only applied to objects and functions. */
-    ActiveThreadData<gc::MarkColor> color;
-
-    /* Pointer to the top of the stack of arenas we are delaying marking on. */
-    js::gc::Arena* unmarkedArenaStackTop;
-
-    /*
-     * If the weakKeys table OOMs, disable the linear algorithm and fall back
-     * to iterating until the next GC.
-     */
-    bool linearWeakMarkingDisabled_;
-
-#ifdef DEBUG
-    /* Count of arenas that are currently in the stack. */
-    size_t markLaterArenas;
-
-    /* Assert that start and stop are called with correct ordering. */
-    bool started;
-
-    /*
-     * If this is true, all marked objects must belong to a compartment being
-     * GCed. This is used to look for compartment bugs.
-     */
-    bool strictCompartmentChecking;
-#endif // DEBUG
-};
-
 #ifdef DEBUG
 // Return true if this trace is happening on behalf of gray buffering during
 // the marking phase of incremental GC.
 bool
 IsBufferGrayRootsTracer(JSTracer* trc);
+
+bool
+IsUnmarkGrayTracer(JSTracer* trc);
 #endif
 
 namespace gc {
 
+class Arena;
+struct Cell;
+class TenuredCell;
+
 /*** Special Cases ***/
 
 void
@@ -478,7 +120,7 @@ struct IsPrivateGCThingInValue
   : public mozilla::EnableIf<mozilla::IsBaseOf<Cell, T>::value &&
                              !mozilla::IsBaseOf<JSObject, T>::value &&
                              !mozilla::IsBaseOf<JSString, T>::value &&
-                             !mozilla::IsBaseOf<JS::Symbol, T>::value &&
+                              !mozilla::IsBaseOf<JS::Symbol, T>::value &&
                              !mozilla::IsBaseOf<JS::BigInt, T>::value, T>
 {
     static_assert(!mozilla::IsSame<Cell, T>::value && !mozilla::IsSame<TenuredCell, T>::value,
@@ -507,6 +149,56 @@ template<typename T>
 void
 CheckTracedThing(JSTracer* trc, T thing);
 
+namespace gc {
+
+// Functions for checking and updating GC thing pointers that might have been
+// moved by compacting GC. Overloads are also provided that work with Values.
+//
+// IsForwarded    - check whether a pointer refers to an GC thing that has been
+//                  moved.
+//
+// Forwarded      - return a pointer to the new location of a GC thing given a
+//                  pointer to old location.
+//
+// MaybeForwarded - used before dereferencing a pointer that may refer to a
+//                  moved GC thing without updating it. For JSObjects this will
+//                  also update the object's shape pointer if it has been moved
+//                  to allow slots to be accessed.
+
+template <typename T>
+inline bool IsForwarded(T* t);
+inline bool IsForwarded(const JS::Value& value);
+
+template <typename T>
+inline T* Forwarded(T* t);
+
+inline Value Forwarded(const JS::Value& value);
+
+template <typename T>
+inline T MaybeForwarded(T t);
+
+inline void
+MakeAccessibleAfterMovingGC(void* anyp) {}
+
+inline void
+MakeAccessibleAfterMovingGC(JSObject* obj); // Defined in jsobjinlines.h.
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+
+template <typename T>
+inline bool IsGCThingValidAfterMovingGC(T* t);
+
+template <typename T>
+inline void CheckGCThingAfterMovingGC(T* t);
+
+template <typename T>
+inline void CheckGCThingAfterMovingGC(const ReadBarriered<T*>& t);
+
+inline void CheckValueAfterMovingGC(const JS::Value& value);
+
+#endif // JSGC_HASH_TABLE_CHECKS
+
+} /* namespace gc */
 } /* namespace js */
 
 #endif /* gc_Marking_h */

js/src/gc/Nursery-inl.h

@@ -13,9 +13,18 @@
 #include "jscntxt.h"
 
 #include "gc/Heap.h"
+#include "gc/RelocationOverlay.h"
 #include "gc/Zone.h"
 #include "js/TracingAPI.h"
 #include "vm/Runtime.h"
+#include "vm/SharedMem.h"
+
+template<typename T>
+bool
+js::Nursery::isInside(const SharedMem<T>& p) const
+{
+    return isInside(p.unwrap(/*safe - used for value in comparison above*/));
+}
 
 MOZ_ALWAYS_INLINE bool
 js::Nursery::getForwardedPointer(JSObject** ref) const

js/src/gc/Nursery.cpp

@@ -13,8 +13,6 @@
 #include "mozilla/Unused.h"
 
 #include "jscompartment.h"
-#include "jsfriendapi.h"
-#include "jsgc.h"
 #include "jsutil.h"
 
 #include "gc/GCInternals.h"
@@ -29,8 +27,7 @@
 #include "vm/TypedArrayObject.h"
 #include "vm/TypeInference.h"
 
-#include "jsobjinlines.h"
-
+#include "gc/Marking-inl.h"
 #include "vm/NativeObject-inl.h"
 
 using namespace js;
@@ -74,27 +71,43 @@ struct js::Nursery::SweepAction
 #endif
 };
 
+namespace js {
+struct NurseryChunk {
+    char data[Nursery::NurseryChunkUsableSize];
+    gc::ChunkTrailer trailer;
+    static NurseryChunk* fromChunk(gc::Chunk* chunk);
+    void init(JSRuntime* rt);
+    void poisonAndInit(JSRuntime* rt, uint8_t poison);
+    uintptr_t start() const { return uintptr_t(&data); }
+    uintptr_t end() const { return uintptr_t(&trailer); }
+    gc::Chunk* toChunk(JSRuntime* rt);
+};
+static_assert(sizeof(js::NurseryChunk) == gc::ChunkSize,
+              "Nursery chunk size must match gc::Chunk size.");
+
+} /* namespace js */
+
 inline void
-js::Nursery::NurseryChunk::poisonAndInit(JSRuntime* rt, uint8_t poison)
+js::NurseryChunk::poisonAndInit(JSRuntime* rt, uint8_t poison)
 {
     JS_POISON(this, poison, ChunkSize);
     init(rt);
 }
 
 inline void
-js::Nursery::NurseryChunk::init(JSRuntime* rt)
+js::NurseryChunk::init(JSRuntime* rt)
 {
     new (&trailer) gc::ChunkTrailer(rt, &rt->gc.storeBuffer);
 }
 
-/* static */ inline js::Nursery::NurseryChunk*
-js::Nursery::NurseryChunk::fromChunk(Chunk* chunk)
+/* static */ inline js::NurseryChunk*
+js::NurseryChunk::fromChunk(Chunk* chunk)
 {
     return reinterpret_cast<NurseryChunk*>(chunk);
 }
 
 inline Chunk*
-js::Nursery::NurseryChunk::toChunk(JSRuntime* rt)
+js::NurseryChunk::toChunk(JSRuntime* rt)
 {
     auto chunk = reinterpret_cast<Chunk*>(this);
     chunk->init(rt);
@@ -108,7 +121,9 @@ js::Nursery::Nursery(JSRuntime* rt)
   , currentStartPosition_(0)
   , currentEnd_(0)
   , currentChunk_(0)
-  , maxNurseryChunks_(0)
+  , maxChunkCount_(0)
+  , chunkCountLimit_(0)
+  , timeInChunkAlloc_(0)
   , previousPromotionRate_(0)
   , profileThreshold_(0)
   , enableProfiling_(false)
@@ -129,15 +144,18 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock)
         return false;
 
     /* maxNurseryBytes parameter is rounded down to a multiple of chunk size. */
-    maxNurseryChunks_ = maxNurseryBytes >> ChunkShift;
+    chunkCountLimit_ = maxNurseryBytes >> ChunkShift;
 
-    /* If no chunks are specified then the nursery is permenantly disabled. */
-    if (maxNurseryChunks_ == 0)
+    /* If no chunks are specified then the nursery is permanently disabled. */
+    if (chunkCountLimit_ == 0)
         return true;
 
-    updateNumChunksLocked(1, lock);
-    if (numChunks() == 0)
+    maxChunkCount_ = 1;
+    if (!allocateNextChunk(0, lock)) {
+        maxChunkCount_ = 0;
         return false;
+    }
+    /* After this point the Nursery has been enabled */
 
     setCurrentChunk(0);
     setStartPosition();
@@ -184,12 +202,18 @@ void
 js::Nursery::enable()
 {
     MOZ_ASSERT(isEmpty());
-    if (isEnabled())
+    MOZ_ASSERT(!runtime()->gc.isVerifyPreBarriersEnabled());
+    if (isEnabled() || !chunkCountLimit())
         return;
 
-    updateNumChunks(1);
-    if (numChunks() == 0)
-        return;
+    {
+        AutoLockGCBgAlloc lock(runtime());
+        maxChunkCount_ = 1;
+        if (!allocateNextChunk(0, lock)) {
+            maxChunkCount_ = 0;
+            return;
+        }
+    }
 
     setCurrentChunk(0);
     setStartPosition();
@@ -203,8 +227,12 @@ js::Nursery::disable()
     MOZ_ASSERT(isEmpty());
     if (!isEnabled())
         return;
-    updateNumChunks(0);
+
+    freeChunksFrom(0);
+    maxChunkCount_ = 0;
+
     currentEnd_ = 0;
+
     runtime()->gc.storeBuffer().disable();
 }
 
@@ -219,6 +247,23 @@ js::Nursery::isEmpty() const
     return position() == currentStartPosition_;
 }
 
+#ifdef JS_GC_ZEAL
+void
+js::Nursery::enterZealMode() {
+    if (isEnabled())
+        maxChunkCount_ = chunkCountLimit();
+}
+
+void
+js::Nursery::leaveZealMode() {
+    if (isEnabled()) {
+        MOZ_ASSERT(isEmpty());
+        setCurrentChunk(0);
+        setStartPosition();
+    }
+}
+#endif // JS_GC_ZEAL
+
 JSObject*
 js::Nursery::allocateObject(JSContext* cx, size_t size, size_t numDynamic, const js::Class* clasp)
 {
@@ -265,9 +310,23 @@ js::Nursery::allocate(size_t size)
     MOZ_ASSERT(size % gc::CellSize == 0);
 
     if (currentEnd() < position() + size) {
-        if (currentChunk_ + 1 == numChunks())
+        unsigned chunkno = currentChunk_ + 1;
+        MOZ_ASSERT(chunkno <= chunkCountLimit());
+        MOZ_ASSERT(chunkno <= maxChunkCount());
+        MOZ_ASSERT(chunkno <= allocatedChunkCount());
+        if (chunkno == maxChunkCount())
             return nullptr;
-        setCurrentChunk(currentChunk_ + 1);
+        if (MOZ_UNLIKELY(chunkno == allocatedChunkCount())) {
+            mozilla::TimeStamp start = TimeStamp::Now();
+            {
+                AutoLockGCBgAlloc lock(runtime());
+                if (!allocateNextChunk(chunkno, lock))
+                    return nullptr;
+            }
+            timeInChunkAlloc_ += TimeStamp::Now() - start;
+            MOZ_ASSERT(chunkno < allocatedChunkCount());
+        }
+        setCurrentChunk(chunkno);
     }
 
     void* thing = (void*)position();
@@ -424,15 +483,24 @@ js::Nursery::calcPromotionRate(bool *validForTenuring) const {
     float used = float(previousGC.nurseryUsedBytes);
     float capacity = float(previousGC.nurseryCapacity);
     float tenured = float(previousGC.tenuredBytes);
+    float rate;
 
-    if (validForTenuring) {
-        /*
-         * We can only use promotion rates if they're likely to be valid,
-         * they're only valid if the nursury was at least 90% full.
-         */
-        *validForTenuring = used > capacity * 0.9f;
+    if (previousGC.nurseryUsedBytes > 0) {
+        if (validForTenuring) {
+            /*
+             * We can only use promotion rates if they're likely to be valid,
+             * they're only valid if the nursury was at least 90% full.
+             */
+            *validForTenuring = used > capacity * 0.9f;
+        }
+        rate = tenured / used;
+    } else {
+        if (validForTenuring)
+            *validForTenuring = false;
+        rate = 0.0f;
     }
-    return tenured / used;
+
+    return rate;
 }
 
 void
@@ -451,20 +519,28 @@ js::Nursery::renderProfileJSON(JSONPrinter& json) const
         // requested. (And as a public API, this function should not crash in
         // such a case.)
         json.beginObject();
-        json.property("status", "no collection");
+        json.property("status", "nursery empty");
         json.endObject();
         return;
     }
 
     json.beginObject();
 
+    json.property("status", "complete");
+
     json.property("reason", JS::gcreason::ExplainReason(previousGC.reason));
     json.property("bytes_tenured", previousGC.tenuredBytes);
-    json.floatProperty("promotion_rate", calcPromotionRate(nullptr), 0);
-    json.property("nursery_bytes", previousGC.nurseryUsedBytes);
-    json.property("new_nursery_bytes", numChunks() * ChunkSize);
+    json.property("bytes_used", previousGC.nurseryUsedBytes);
+    json.property("cur_capacity", previousGC.nurseryCapacity);
+    const size_t newCapacity = spaceToEnd(maxChunkCount());
+    if (newCapacity != previousGC.nurseryCapacity)
+        json.property("new_capacity", newCapacity);
+    if (previousGC.nurseryLazyCapacity != previousGC.nurseryCapacity)
+        json.property("lazy_capacity", previousGC.nurseryLazyCapacity);
+    if (!timeInChunkAlloc_.IsZero())
+        json.property("chunk_alloc_us", timeInChunkAlloc_, json.MICROSECONDS);
 
-    json.beginObjectProperty("timings");
+    json.beginObjectProperty("phase_times");
 
 #define EXTRACT_NAME(name, text) #name,
     static const char* names[] = {
@@ -566,14 +642,13 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason)
         doCollection(reason, tenureCounts);
     } else {
         previousGC.nurseryUsedBytes = 0;
-        previousGC.nurseryCapacity = spaceToEnd();
+        previousGC.nurseryCapacity = spaceToEnd(maxChunkCount());
+        previousGC.nurseryLazyCapacity = spaceToEnd(allocatedChunkCount());
         previousGC.tenuredBytes = 0;
     }
 
     // Resize the nursery.
-    startProfile(ProfileKey::Resize);
     maybeResizeNursery(reason);
-    endProfile(ProfileKey::Resize);
 
     // If we are promoting the nursery, or exhausted the store buffer with
     // pointers to nursery things, which will force a collection well before
@@ -589,7 +664,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason)
             for (auto& entry : tenureCounts.entries) {
                 if (entry.count >= 3000) {
                     ObjectGroup* group = entry.group;
-                    if (group->canPreTenure()) {
+                    if (group->canPreTenure() && group->zone()->group()->canEnterWithoutYielding(cx)) {
                         AutoCompartment ac(cx, group);
                         group->setShouldPreTenure(cx);
                         pretenureCount++;
@@ -608,7 +683,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason)
     // Disable the nursery if the user changed the configuration setting.  The
     // nursery can only be re-enabled by resetting the configurationa and
     // restarting firefox.
-    if (maxNurseryChunks_ == 0)
+    if (chunkCountLimit_ == 0)
         disable();
 
     endProfile(ProfileKey::Total);
@@ -618,6 +693,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason)
 
     rt->gc.stats.endNurseryCollection(reason);
     TraceMinorGCEnd();
+    timeInChunkAlloc_ = mozilla::TimeDuration();
 
     if (enableProfiling_ && totalTime >= profileThreshold_) {
         static int printedHeader = 0;
@@ -629,8 +705,8 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason)
         fprintf(stderr, "MinorGC: %20s %5.1f%% %4u        ",
                 JS::gcreason::ExplainReason(reason),
                 promotionRate * 100,
-                numChunks());
-        printProfileTimes(profileTimes_);
+                maxChunkCount());
+        printProfileDurations(profileDurations_);
 
         if (reportTenurings_) {
             for (auto& entry : tenureCounts.entries) {
@@ -652,7 +728,7 @@ js::Nursery::doCollection(JS::gcreason::Reason reason,
     AutoDisableProxyCheck disableStrictProxyChecking(rt);
     mozilla::DebugOnly<AutoEnterOOMUnsafeRegion> oomUnsafeRegion;
 
-    const size_t initialNurseryCapacity = spaceToEnd();
+    const size_t initialNurseryCapacity = spaceToEnd(maxChunkCount());
     const size_t initialNurseryUsedBytes = initialNurseryCapacity - freeSpace();
 
     // Move objects pointed to by roots from the nursery to the major heap.
@@ -751,6 +827,7 @@ js::Nursery::doCollection(JS::gcreason::Reason reason,
 
     previousGC.reason = reason;
     previousGC.nurseryCapacity = initialNurseryCapacity;
+    previousGC.nurseryLazyCapacity = spaceToEnd(allocatedChunkCount());
     previousGC.nurseryUsedBytes = initialNurseryUsedBytes;
     previousGC.tenuredBytes = mover.tenuredSize;
 }
@@ -832,12 +909,18 @@ js::Nursery::clear()
 {
 #ifdef JS_GC_ZEAL
     /* Poison the nursery contents so touching a freed object will crash. */
-    for (unsigned i = 0; i < numChunks(); i++)
+    for (unsigned i = 0; i < allocatedChunkCount(); i++)
         chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN);
 
+    if (runtime()->hasZealMode(ZealMode::GenerationalGC)) {
+        /* Only reset the alloc point when we are close to the end. */
+        if (currentChunk_ + 1 == maxChunkCount())
+            setCurrentChunk(0);
+    } else
+#endif
     {
 #ifdef JS_CRASH_DIAGNOSTICS
-        for (unsigned i = 0; i < numChunks(); ++i)
+        for (unsigned i = 0; i < allocatedChunkCount(); ++i)
             chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN);
 #endif
         setCurrentChunk(0);
@@ -849,9 +932,9 @@ js::Nursery::clear()
 }
 
 size_t
-js::Nursery::spaceToEnd() const
+js::Nursery::spaceToEnd(unsigned chunkCount) const
 {
-    unsigned lastChunk = numChunks() - 1;
+    unsigned lastChunk = chunkCount - 1;
 
     MOZ_ASSERT(lastChunk >= currentStartChunk_);
     MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <= NurseryChunkUsableSize);
@@ -859,7 +942,7 @@ js::Nursery::spaceToEnd() const
     size_t bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) +
                    ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize);
 
-    MOZ_ASSERT(bytes <= numChunks() * NurseryChunkUsableSize);
+    MOZ_ASSERT(bytes <= maxChunkCount() * NurseryChunkUsableSize);
 
     return bytes;
 }
@@ -867,14 +950,40 @@ js::Nursery::spaceToEnd() const
 MOZ_ALWAYS_INLINE void
 js::Nursery::setCurrentChunk(unsigned chunkno)
 {
-    MOZ_ASSERT(chunkno < maxChunks());
-    MOZ_ASSERT(chunkno < numChunks());
+    MOZ_ASSERT(chunkno < chunkCountLimit());
+    MOZ_ASSERT(chunkno < allocatedChunkCount());
     currentChunk_ = chunkno;
     position_ = chunk(chunkno).start();
     currentEnd_ = chunk(chunkno).end();
     chunk(chunkno).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN);
 }
 
+bool
+js::Nursery::allocateNextChunk(const unsigned chunkno,
+    AutoLockGCBgAlloc& lock)
+{
+    const unsigned priorCount = allocatedChunkCount();
+    const unsigned newCount = priorCount + 1;
+
+    MOZ_ASSERT((chunkno == currentChunk_ + 1) || (chunkno == 0 && allocatedChunkCount() == 0));
+    MOZ_ASSERT(chunkno == allocatedChunkCount());
+    MOZ_ASSERT(chunkno < chunkCountLimit());
+    MOZ_ASSERT(chunkno < maxChunkCount());
+
+    if (!chunks_.resize(newCount))
+        return false;
+
+    Chunk* newChunk;
+    newChunk = runtime()->gc.getOrAllocChunk(lock);
+    if (!newChunk) {
+        chunks_.shrinkTo(priorCount);
+        return false;
+    }
+
+    chunks_[chunkno] = NurseryChunk::fromChunk(newChunk);
+    return true;
+}
+
 MOZ_ALWAYS_INLINE void
 js::Nursery::setStartPosition()
 {
@@ -911,23 +1020,25 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason)
         float(previousGC.tenuredBytes) / float(previousGC.nurseryCapacity);
 
     newMaxNurseryChunks = runtime()->gc.tunables.gcMaxNurseryBytes() >> ChunkShift;
-    if (newMaxNurseryChunks != maxNurseryChunks_) {
-        maxNurseryChunks_ = newMaxNurseryChunks;
+    if (newMaxNurseryChunks != chunkCountLimit_) {
+        chunkCountLimit_ = newMaxNurseryChunks;
         /* The configured maximum nursery size is changing */
-        const int extraChunks = numChunks() - newMaxNurseryChunks;
-        if (extraChunks > 0) {
+        if (maxChunkCount() > newMaxNurseryChunks) {
             /* We need to shrink the nursery */
-            shrinkAllocableSpace(extraChunks);
+            shrinkAllocableSpace(newMaxNurseryChunks);
 
             previousPromotionRate_ = promotionRate;
             return;
         }
     }
 
-    if (promotionRate > GrowThreshold)
+    if (promotionRate > GrowThreshold) {
+        // The GC nursery is an optimization and so if we fail to allocate
+        // nursery chunks we do not report an error.
         growAllocableSpace();
-    else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold)
-        shrinkAllocableSpace(1);
+    } else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) {
+        shrinkAllocableSpace(maxChunkCount() - 1);
+    }
 
     previousPromotionRate_ = promotionRate;
 }
@@ -935,66 +1046,46 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason)
 void
 js::Nursery::growAllocableSpace()
 {
-    updateNumChunks(Min(numChunks() * 2, maxNurseryChunks_));
+    maxChunkCount_ = Min(maxChunkCount() * 2, chunkCountLimit());
 }
 
 void
-js::Nursery::shrinkAllocableSpace(unsigned removeNumChunks)
+js::Nursery::freeChunksFrom(unsigned firstFreeChunk)
+{
+    MOZ_ASSERT(firstFreeChunk < chunks_.length());
+    {
+        AutoLockGC lock(runtime());
+        for (unsigned i = firstFreeChunk; i < chunks_.length(); i++)
+            runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock);
+    }
+    chunks_.shrinkTo(firstFreeChunk);
+}
+
+void
+js::Nursery::shrinkAllocableSpace(unsigned newCount)
 {
 #ifdef JS_GC_ZEAL
     if (runtime()->hasZealMode(ZealMode::GenerationalGC))
         return;
 #endif
-    updateNumChunks(Max(numChunks() - removeNumChunks, 1u));
+
+    // Don't shrink the nursery to zero (use Nursery::disable() instead) and
+    // don't attempt to shrink it to the same size.
+    if ((newCount == 0) || (newCount == maxChunkCount()))
+        return;
+
+    MOZ_ASSERT(newCount < maxChunkCount());
+
+    if (newCount < allocatedChunkCount())
+        freeChunksFrom(newCount);
+
+    maxChunkCount_ = newCount;
 }
 
 void
 js::Nursery::minimizeAllocableSpace()
 {
-    updateNumChunks(1);
-}
-
-void
-js::Nursery::updateNumChunks(unsigned newCount)
-{
-    if (numChunks() != newCount) {
-        AutoLockGCBgAlloc lock(runtime());
-        updateNumChunksLocked(newCount, lock);
-    }
-}
-
-void
-js::Nursery::updateNumChunksLocked(unsigned newCount,
-                                   AutoLockGCBgAlloc& lock)
-{
-    // The GC nursery is an optimization and so if we fail to allocate nursery
-    // chunks we do not report an error.
-
-    unsigned priorCount = numChunks();
-    MOZ_ASSERT(priorCount != newCount);
-
-    if (newCount < priorCount) {
-        // Shrink the nursery and free unused chunks.
-        for (unsigned i = newCount; i < priorCount; i++)
-            runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock);
-        chunks_.shrinkTo(newCount);
-        return;
-    }
-
-    // Grow the nursery and allocate new chunks.
-    if (!chunks_.resize(newCount))
-        return;
-
-    for (unsigned i = priorCount; i < newCount; i++) {
-        auto newChunk = runtime()->gc.getOrAllocChunk(lock);
-        if (!newChunk) {
-            chunks_.shrinkTo(i);
-            return;
-        }
-
-        chunks_[i] = NurseryChunk::fromChunk(newChunk);
-        chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN);
-    }
+    shrinkAllocableSpace(1);
 }
 
 void
@@ -1033,6 +1124,13 @@ js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj)
     return dictionaryModeObjects_.append(obj);
 }
 
+uintptr_t
+js::Nursery::currentEnd() const
+{
+    MOZ_ASSERT(currentEnd_ == chunk(currentChunk_).end());
+    return currentEnd_;
+}
+
 void
 js::Nursery::sweepDictionaryModeObjects()
 {

js/src/gc/Nursery.h

@@ -10,19 +10,10 @@
 
 #include "mozilla/EnumeratedArray.h"
 
-#include "jsalloc.h"
-#include "jspubtd.h"
-
-#include "ds/BitArray.h"
-#include "gc/Heap.h"
-#include "gc/Memory.h"
 #include "js/Class.h"
-#include "js/GCAPI.h"
-#include "js/HashTable.h"
 #include "js/HeapAPI.h"
-#include "js/Value.h"
+#include "js/TracingAPI.h"
 #include "js/Vector.h"
-#include "vm/SharedMem.h"
 
 #define FOR_EACH_NURSERY_PROFILE_TIME(_)                                      \
    /* Key                       Header text */                                \
@@ -44,19 +35,22 @@
     _(FreeMallocedBuffers,      "frSlts")                                     \
     _(ClearStoreBuffer,         "clrSB")                                      \
     _(ClearNursery,             "clear")                                      \
-    _(Resize,                   "resize")                                     \
     _(Pretenure,                "pretnr")
 
+template<typename T> class SharedMem;
+
 namespace JS {
 struct Zone;
 } // namespace JS
 
 namespace js {
 
+class AutoLockGCBgAlloc;
 class ObjectElements;
 class PlainObject;
 class NativeObject;
 class Nursery;
+struct NurseryChunk;
 class HeapSlot;
 
 namespace gc {
@@ -65,6 +59,8 @@ struct Cell;
 class MinorCollectionTracer;
 class RelocationOverlay;
 struct TenureCountCache;
+enum class AllocKind : uint8_t;
+class TenuredCell;
 } /* namespace gc */
 
 namespace jit {
@@ -97,7 +93,7 @@ class TenuringTracer : public JSTracer
     // The store buffers need to be able to call these directly.
     void traceObject(JSObject* src);
     void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length);
-    void traceSlots(JS::Value* vp, uint32_t nslots) { traceSlots(vp, vp + nslots); }
+    void traceSlots(JS::Value* vp, uint32_t nslots);
 
   private:
     Nursery& nursery() { return nursery_; }
@@ -139,15 +135,22 @@ class Nursery
 
     [[nodiscard]] bool init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock);
 
-    unsigned maxChunks() const { return maxNurseryChunks_; }
-    unsigned numChunks() const { return chunks_.length(); }
+    unsigned chunkCountLimit() const { return chunkCountLimit_; }
 
-    bool exists() const { return maxChunks() != 0; }
-    size_t nurserySize() const { return maxChunks() << ChunkShift; }
+    // Number of allocated (ready to use) chunks.
+    unsigned allocatedChunkCount() const { return chunks_.length(); }
+
+    // Total number of chunks and the capacity of the nursery. Chunks will be
+    // lazilly allocated and added to the chunks array up to this limit, after
+    // that the nursery must be collected, this limit may be raised during
+    // collection.
+    unsigned maxChunkCount() const { return maxChunkCount_; }
+
+    bool exists() const { return chunkCountLimit() != 0; }
 
     void enable();
     void disable();
-    bool isEnabled() const { return numChunks() != 0; }
+    bool isEnabled() const { return maxChunkCount() != 0; }
 
     /* Return true if no allocations have been made since the last collection. */
     bool isEmpty() const;
@@ -159,15 +162,14 @@ class Nursery
     MOZ_ALWAYS_INLINE bool isInside(gc::Cell* cellp) const = delete;
     MOZ_ALWAYS_INLINE bool isInside(const void* p) const {
         for (auto chunk : chunks_) {
-            if (uintptr_t(p) - chunk->start() < gc::ChunkSize)
+            if (uintptr_t(p) - uintptr_t(chunk) < gc::ChunkSize)
                 return true;
         }
         return false;
     }
+
     template<typename T>
-    bool isInside(const SharedMem<T>& p) const {
-        return isInside(p.unwrap(/*safe - used for value in comparison above*/));
-    }
+    inline bool isInside(const SharedMem<T>& p) const;
 
     /*
      * Allocate and return a pointer to a new GC object with its |slots|
@@ -238,7 +240,7 @@ class Nursery
     MOZ_MUST_USE bool queueDictionaryModeObjectToSweep(NativeObject* obj);
 
     size_t sizeOfHeapCommitted() const {
-        return numChunks() * gc::ChunkSize;
+        return allocatedChunkCount() * gc::ChunkSize;
     }
     size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const {
         size_t total = 0;
@@ -249,13 +251,16 @@ class Nursery
     }
 
     // The number of bytes from the start position to the end of the nursery.
-    size_t spaceToEnd() const;
+    // pass maxChunkCount(), allocatedChunkCount() or chunkCountLimit()
+    // to calculate the nursery size, current lazy-allocated size or nursery
+    // limit respectively.
+    size_t spaceToEnd(unsigned chunkCount) const;
 
     // Free space remaining, not counting chunk trailers.
     MOZ_ALWAYS_INLINE size_t freeSpace() const {
         MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize);
         return (currentEnd_ - position_) +
-               (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize;
+               (maxChunkCount() - currentChunk_ - 1) * NurseryChunkUsableSize;
     }
 
     /* Print total profile times on shutdown. */
@@ -270,24 +275,19 @@ class Nursery
     JS::gcreason::Reason minorGCTriggerReason() const { return minorGCTriggerReason_; }
     void clearMinorGCRequest() { minorGCTriggerReason_ = JS::gcreason::NO_REASON; }
 
+    bool needIdleTimeCollection() const {
+        return minorGCRequested() ||
+               (freeSpace() < kIdleTimeCollectionThreshold);
+    }
+
     bool enableProfiling() const { return enableProfiling_; }
 
   private:
     /* The amount of space in the mapped nursery available to allocations. */
-    static const size_t NurseryChunkUsableSize = gc::ChunkSize - sizeof(gc::ChunkTrailer);
+    static const size_t NurseryChunkUsableSize = gc::ChunkSize - gc::ChunkTrailerSize;
 
-    struct NurseryChunk {
-        char data[NurseryChunkUsableSize];
-        gc::ChunkTrailer trailer;
-        static NurseryChunk* fromChunk(gc::Chunk* chunk);
-        void init(JSRuntime* rt);
-        void poisonAndInit(JSRuntime* rt, uint8_t poison);
-        uintptr_t start() const { return uintptr_t(&data); }
-        uintptr_t end() const { return uintptr_t(&trailer); }
-        gc::Chunk* toChunk(JSRuntime* rt);
-    };
-    static_assert(sizeof(NurseryChunk) == gc::ChunkSize,
-                  "Nursery chunk size must match gc::Chunk size.");
+    /* Attemp to run a minor GC in the idle time if the free space falls below this threshold. */
+    static constexpr size_t kIdleTimeCollectionThreshold = NurseryChunkUsableSize / 4;
 
     /*
      * The start and end pointers are stored under the runtime so that we can
@@ -312,8 +312,20 @@ class Nursery
     /* The index of the chunk that is currently being allocated from. */
     unsigned currentChunk_;
 
-    /* Maximum number of chunks to allocate for the nursery. */
-    unsigned maxNurseryChunks_;
+    /*
+     * The nursery may grow the chunks_ vector up to this size without a
+     * collection.  This allows the nursery to grow lazilly.  This limit may
+     * change during maybeResizeNursery() each collection.
+     */
+    unsigned maxChunkCount_;
+
+    /*
+     * This limit is fixed by configuration.  It represents the maximum size
+     * the nursery is permitted to tune itself to in maybeResizeNursery();
+     */
+    unsigned chunkCountLimit_;
+
+    mozilla::TimeDuration timeInChunkAlloc_;
 
     /* Promotion rate for the previous minor collection. */
     float previousPromotionRate_;
@@ -350,6 +362,7 @@ class Nursery
     struct {
         JS::gcreason::Reason reason;
         size_t nurseryCapacity;
+        size_t nurseryLazyCapacity;
         size_t nurseryUsedBytes;
         size_t tenuredBytes;
     } previousGC;
@@ -408,7 +421,10 @@ class Nursery
     using NativeObjectVector = Vector<NativeObject*, 0, SystemAllocPolicy>;
     NativeObjectVector dictionaryModeObjects_;
 
-    NurseryChunk* allocChunk();
+#ifdef JS_GC_ZEAL
+    struct Canary;
+    Canary* lastCanary_;
+#endif
 
     NurseryChunk& chunk(unsigned index) const {
         return *chunks_[index];
@@ -417,24 +433,14 @@ class Nursery
     void setCurrentChunk(unsigned chunkno);
     void setStartPosition();
 
-    void updateNumChunks(unsigned newCount);
-    void updateNumChunksLocked(unsigned newCount,
-                               AutoLockGCBgAlloc& lock);
+    /*
+     * Allocate the next chunk, or the first chunk for initialization.
+     * Callers will probably want to call setCurrentChunk(0) next.
+     */
+    [[nodiscard]] bool allocateNextChunk(unsigned chunkno,
+        AutoLockGCBgAlloc& lock);
 
-    MOZ_ALWAYS_INLINE uintptr_t allocationEnd() const {
-        MOZ_ASSERT(numChunks() > 0);
-        return chunks_.back()->end();
-    }
-
-    MOZ_ALWAYS_INLINE uintptr_t currentEnd() const {
-        MOZ_ASSERT(runtime_);
-        MOZ_ASSERT(currentEnd_ == chunk(currentChunk_).end());
-        return currentEnd_;
-    }
-    void* addressOfCurrentEnd() const {
-        MOZ_ASSERT(runtime_);
-        return (void*)&currentEnd_;
-    }
+    MOZ_ALWAYS_INLINE uintptr_t currentEnd() const;
 
     uintptr_t position() const { return position_; }
     void* addressOfPosition() const { return (void*)&position_; }
@@ -487,9 +493,13 @@ class Nursery
     /* Change the allocable space provided by the nursery. */
     void maybeResizeNursery(JS::gcreason::Reason reason);
     void growAllocableSpace();
-    void shrinkAllocableSpace(unsigned removeNumChunks);
+    void shrinkAllocableSpace(unsigned newCount);
     void minimizeAllocableSpace();
 
+    // Free the chunks starting at firstFreeChunk until the end of the chunks
+    // vector. Shrinks the vector but does not update maxChunkCount().
+    void freeChunksFrom(unsigned firstFreeChunk);
+
     /* Profile recording and printing. */
     void startProfile(ProfileKey key);
     void endProfile(ProfileKey key);
@@ -498,6 +508,7 @@ class Nursery
     friend class TenuringTracer;
     friend class gc::MinorCollectionTracer;
     friend class jit::MacroAssembler;
+    friend struct NurseryChunk;
 };
 
 } /* namespace js */

js/src/gc/ObjectKind-inl.h

@@ -0,0 +1,174 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal helper functions for getting the AllocKind used to allocate a
+ * JSObject and related information.
+ */
+
+#ifndef gc_ObjectKind_inl_h
+#define gc_ObjectKind_inl_h
+
+#include "vm/NativeObject.h"
+
+namespace js {
+namespace gc {
+
+static inline bool
+CanBeFinalizedInBackground(AllocKind kind, const Class* clasp)
+{
+    MOZ_ASSERT(IsObjectAllocKind(kind));
+    /* If the class has no finalizer or a finalizer that is safe to call on
+     * a different thread, we change the alloc kind. For example,
+     * AllocKind::OBJECT0 calls the finalizer on the active thread,
+     * AllocKind::OBJECT0_BACKGROUND calls the finalizer on the gcHelperThread.
+     * IsBackgroundFinalized is called to prevent recursively incrementing
+     * the alloc kind; kind may already be a background finalize kind.
+     */
+    return (!IsBackgroundFinalized(kind) &&
+            (!clasp->hasFinalize() || (clasp->flags & JSCLASS_BACKGROUND_FINALIZE)));
+}
+
+static inline AllocKind
+GetBackgroundAllocKind(AllocKind kind)
+{
+    MOZ_ASSERT(!IsBackgroundFinalized(kind));
+    MOZ_ASSERT(IsObjectAllocKind(kind));
+    return AllocKind(size_t(kind) + 1);
+}
+
+/* Capacity for slotsToThingKind */
+const size_t SLOTS_TO_THING_KIND_LIMIT = 17;
+
+extern const AllocKind slotsToThingKind[];
+
+/* Get the best kind to use when making an object with the given slot count. */
+static inline AllocKind
+GetGCObjectKind(size_t numSlots)
+{
+    if (numSlots >= SLOTS_TO_THING_KIND_LIMIT)
+        return AllocKind::OBJECT16;
+    return slotsToThingKind[numSlots];
+}
+
+static inline AllocKind
+GetGCObjectKind(const Class* clasp)
+{
+    if (clasp == FunctionClassPtr)
+        return AllocKind::FUNCTION;
+
+    MOZ_ASSERT(!clasp->isProxy(), "Proxies should use GetProxyGCObjectKind");
+
+    uint32_t nslots = JSCLASS_RESERVED_SLOTS(clasp);
+    if (clasp->flags & JSCLASS_HAS_PRIVATE)
+        nslots++;
+    return GetGCObjectKind(nslots);
+}
+
+/* As for GetGCObjectKind, but for dense array allocation. */
+static inline AllocKind
+GetGCArrayKind(size_t numElements)
+{
+    /*
+     * Dense arrays can use their fixed slots to hold their elements array
+     * (less two Values worth of ObjectElements header), but if more than the
+     * maximum number of fixed slots is needed then the fixed slots will be
+     * unused.
+     */
+    JS_STATIC_ASSERT(ObjectElements::VALUES_PER_HEADER == 2);
+    if (numElements > NativeObject::MAX_DENSE_ELEMENTS_COUNT ||
+        numElements + ObjectElements::VALUES_PER_HEADER >= SLOTS_TO_THING_KIND_LIMIT)
+    {
+        return AllocKind::OBJECT2;
+    }
+    return slotsToThingKind[numElements + ObjectElements::VALUES_PER_HEADER];
+}
+
+static inline AllocKind
+GetGCObjectFixedSlotsKind(size_t numFixedSlots)
+{
+    MOZ_ASSERT(numFixedSlots < SLOTS_TO_THING_KIND_LIMIT);
+    return slotsToThingKind[numFixedSlots];
+}
+
+// Get the best kind to use when allocating an object that needs a specific
+// number of bytes.
+static inline AllocKind
+GetGCObjectKindForBytes(size_t nbytes)
+{
+    MOZ_ASSERT(nbytes <= JSObject::MAX_BYTE_SIZE);
+
+    if (nbytes <= sizeof(NativeObject))
+        return AllocKind::OBJECT0;
+    nbytes -= sizeof(NativeObject);
+
+    size_t dataSlots = AlignBytes(nbytes, sizeof(Value)) / sizeof(Value);
+    MOZ_ASSERT(nbytes <= dataSlots * sizeof(Value));
+    return GetGCObjectKind(dataSlots);
+}
+
+/* Get the number of fixed slots and initial capacity associated with a kind. */
+static inline size_t
+GetGCKindSlots(AllocKind thingKind)
+{
+    /* Using a switch in hopes that thingKind will usually be a compile-time constant. */
+    switch (thingKind) {
+      case AllocKind::FUNCTION:
+      case AllocKind::OBJECT0:
+      case AllocKind::OBJECT0_BACKGROUND:
+        return 0;
+      case AllocKind::FUNCTION_EXTENDED:
+      case AllocKind::OBJECT2:
+      case AllocKind::OBJECT2_BACKGROUND:
+        return 2;
+      case AllocKind::OBJECT4:
+      case AllocKind::OBJECT4_BACKGROUND:
+        return 4;
+      case AllocKind::OBJECT8:
+      case AllocKind::OBJECT8_BACKGROUND:
+        return 8;
+      case AllocKind::OBJECT12:
+      case AllocKind::OBJECT12_BACKGROUND:
+        return 12;
+      case AllocKind::OBJECT16:
+      case AllocKind::OBJECT16_BACKGROUND:
+        return 16;
+      default:
+        MOZ_CRASH("Bad object alloc kind");
+    }
+}
+
+static inline size_t
+GetGCKindSlots(AllocKind thingKind, const Class* clasp)
+{
+    size_t nslots = GetGCKindSlots(thingKind);
+
+    /* An object's private data uses the space taken by its last fixed slot. */
+    if (clasp->flags & JSCLASS_HAS_PRIVATE) {
+        MOZ_ASSERT(nslots > 0);
+        nslots--;
+    }
+
+    /*
+     * Functions have a larger alloc kind than AllocKind::OBJECT to reserve
+     * space for the extra fields in JSFunction, but have no fixed slots.
+     */
+    if (clasp == FunctionClassPtr)
+        nslots = 0;
+
+    return nslots;
+}
+
+static inline size_t
+GetGCKindBytes(AllocKind thingKind)
+{
+    return sizeof(JSObject_Slots0) + GetGCKindSlots(thingKind) * sizeof(Value);
+}
+
+} // namespace gc
+} // namespace js
+
+#endif // gc_ObjectKind_inl_h

js/src/gc/RelocationOverlay.h

@@ -0,0 +1,75 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * GC-internal definition of relocation overlay used while moving cells.
+ */
+
+#ifndef gc_RelocationOverlay_h
+#define gc_RelocationOverlay_h
+
+#include "mozilla/Assertions.h"
+
+#include <stdint.h>
+
+namespace js {
+namespace gc {
+
+struct Cell;
+
+/*
+ * This structure overlays a Cell that has been moved and provides a way to find
+ * its new location. It's used during generational and compacting GC.
+ */
+class RelocationOverlay
+{
+    /* The low bit is set so this should never equal a normal pointer. */
+    static const uintptr_t Relocated = uintptr_t(0xbad0bad1);
+
+    /* Set to Relocated when moved. */
+    uintptr_t magic_;
+
+    /* The location |this| was moved to. */
+    Cell* newLocation_;
+
+    /* A list entry to track all relocated things. */
+    RelocationOverlay* next_;
+
+  public:
+    static RelocationOverlay* fromCell(Cell* cell) {
+        return reinterpret_cast<RelocationOverlay*>(cell);
+    }
+
+    bool isForwarded() const {
+        return magic_ == Relocated;
+    }
+
+    Cell* forwardingAddress() const {
+        MOZ_ASSERT(isForwarded());
+        return newLocation_;
+    }
+
+    void forwardTo(Cell* cell);
+
+    RelocationOverlay*& nextRef() {
+        MOZ_ASSERT(isForwarded());
+        return next_;
+    }
+
+    RelocationOverlay* next() const {
+        MOZ_ASSERT(isForwarded());
+        return next_;
+    }
+
+    static bool isCellForwarded(Cell* cell) {
+        return fromCell(cell)->isForwarded();
+    }
+};
+
+} // namespace gc
+} // namespace js
+
+#endif /* gc_RelocationOverlay_h */

js/src/gc/RootMarking.cpp

@@ -10,7 +10,6 @@
 #endif
 
 #include "jscntxt.h"
-#include "jsgc.h"
 #include "jsprf.h"
 #include "jstypes.h"
 
@@ -26,6 +25,9 @@
 #include "jsgcinlines.h"
 #include "jsobjinlines.h"
 
+#include "gc/Iteration-inl.h"
+#include "gc/Nursery-inl.h"
+
 using namespace js;
 using namespace js::gc;
 

js/src/gc/StoreBuffer-inl.h

@@ -8,6 +8,7 @@
 
 #include "gc/StoreBuffer.h"
 
+#include "gc/Cell.h"
 #include "gc/Heap.h"
 
 namespace js {

js/src/gc/StoreBuffer.cpp

@@ -7,8 +7,6 @@
 
 #include "mozilla/Assertions.h"
 
-#include "jscompartment.h"
-
 #include "gc/Statistics.h"
 #include "vm/ArgumentsObject.h"
 #include "vm/Runtime.h"

js/src/gc/StoreBuffer.h

@@ -14,6 +14,7 @@
 
 #include "jsalloc.h"
 
+#include "ds/BitArray.h"
 #include "ds/LifoAlloc.h"
 #include "gc/Nursery.h"
 #include "js/MemoryMetrics.h"
@@ -21,6 +22,7 @@
 namespace js {
 namespace gc {
 
+class Arena;
 class ArenaCellSet;
 
 /*

js/src/gc/Tracer.cpp

@@ -8,9 +8,7 @@
 #include "mozilla/DebugOnly.h"
 #include "mozilla/SizePrintfMacros.h"
 
-#include "jsapi.h"
 #include "jsfun.h"
-#include "jsgc.h"
 #include "jsprf.h"
 #include "jsscript.h"
 #include "jsutil.h"
@@ -438,3 +436,10 @@ JS_GetTraceThingInfo(char* buf, size_t bufsize, JSTracer* trc, void* thing,
 JS::CallbackTracer::CallbackTracer(JSContext* cx, WeakMapTraceKind weakTraceKind)
   : CallbackTracer(cx->runtime(), weakTraceKind)
 {}
+
+uint32_t
+JSTracer::gcNumberForMarking() const
+{
+    MOZ_ASSERT(isMarkingTracer());
+    return runtime()->gc.gcNumber();
+}

js/src/gc/Verifier.cpp

@@ -11,17 +11,17 @@
 #include "mozilla/Sprintf.h"
 
 #include "jscntxt.h"
-#include "jsgc.h"
 #include "jsprf.h"
 
 #include "gc/GCInternals.h"
 #include "gc/Zone.h"
-#include "js/GCAPI.h"
 #include "js/HashTable.h"
 
 #include "jscntxtinlines.h"
 #include "jsgcinlines.h"
 
+#include "gc/Marking-inl.h"
+
 using namespace js;
 using namespace js::gc;
 
@@ -564,14 +564,28 @@ CheckHeapTracer::check(AutoLockForExclusiveAccess& lock)
     return !oom;
 }
 
+static const char*
+GetCellColorName(Cell* cell)
+{
+    if (cell->isMarkedBlack())
+        return "black";
+    if (cell->isMarkedGray())
+        return "gray";
+    return "white";
+}
+
 void
 HeapCheckTracerBase::dumpCellInfo(Cell* cell)
 {
     auto kind = cell->getTraceKind();
-    fprintf(stderr, "%s", GCTraceKindToAscii(kind));
-    if (kind == JS::TraceKind::Object)
-        fprintf(stderr, " %s", static_cast<JSObject*>(cell)->getClass()->name);
+    JSObject* obj = kind == JS::TraceKind::Object ? static_cast<JSObject*>(cell) : nullptr;
+
+    fprintf(stderr, "%s %s", GetCellColorName(cell), GCTraceKindToAscii(kind));
+    if (obj)
+        fprintf(stderr, " %s", obj->getClass()->name);
     fprintf(stderr, " %p", cell);
+    if (obj)
+        fprintf(stderr, " (compartment %p)", obj->compartment());
 }
 
 void
@@ -629,10 +643,8 @@ CheckHeapTracer::check(AutoLockForExclusiveAccess& lock)
     if (!traceHeap(lock))
         return;
 
-    if (failures) {
-        fprintf(stderr, "Heap check: %" PRIuSIZE " failure(s) out of %" PRIu32 " pointers checked\n",
-                failures, visited.count());
-    }
+    if (failures)
+        fprintf(stderr, "Heap check: %zu failure(s)\n", failures);
     MOZ_RELEASE_ASSERT(failures == 0);
 }
 
@@ -647,7 +659,7 @@ js::gc::CheckHeapAfterGC(JSRuntime* rt)
 
 #endif /* JSGC_HASH_TABLE_CHECKS */
 
-#ifdef DEBUG
+#if defined(JS_GC_ZEAL) || defined(DEBUG)
 
 class CheckGrayMarkingTracer final : public HeapCheckTracerBase
 {
@@ -675,10 +687,18 @@ CheckGrayMarkingTracer::checkCell(Cell* cell)
 
     if (parent->isMarkedBlack() && cell->isMarkedGray()) {
         failures++;
+
         fprintf(stderr, "Found black to gray edge to ");
         dumpCellInfo(cell);
         fprintf(stderr, "\n");
         dumpCellPath();
+
+#ifdef DEBUG
+        if (cell->getTraceKind() == JS::TraceKind::Object) {
+            fprintf(stderr, "\n");
+            DumpObject(static_cast<JSObject*>(cell), stderr);
+        }
+#endif
     }
 }
 
@@ -709,4 +729,4 @@ js::CheckGrayMarkingState(JSContext* cx)
     return tracer.check(session.lock);
 }
 
-#endif // DEBUG
+#endif // defined(JS_GC_ZEAL) || defined(DEBUG)

js/src/gc/Zone.cpp

@@ -6,8 +6,6 @@
 
 #include "gc/Zone.h"
 
-#include "jsgc.h"
-
 #include "gc/Policy.h"
 #include "jit/BaselineJIT.h"
 #include "jit/Ion.h"
@@ -17,6 +15,7 @@
 
 #include "jscompartmentinlines.h"
 #include "jsgcinlines.h"
+#include "gc/Marking-inl.h"
 
 using namespace js;
 using namespace js::gc;

js/src/gc/Zone.h

@@ -28,6 +28,9 @@ class JitZone;
 
 namespace gc {
 
+class GCSchedulingState;
+class GCSchedulingTunables;
+
 // This class encapsulates the data that determines when we need to do a zone GC.
 class ZoneHeapThreshold
 {
@@ -196,7 +199,7 @@ struct Zone : public JS::shadow::Zone,
     bool isPreservingCode() const { return gcPreserveCode_; }
 
     bool canCollect();
-	
+
     void notifyObservingDebuggers();
 
     void setGCState(GCState state) {

js/src/gc/ZoneGroup.cpp

@@ -94,6 +94,12 @@ ZoneGroup::leave()
         ownerContext_ = CooperatingContext(nullptr);
 }
 
+bool
+ZoneGroup::canEnterWithoutYielding(JSContext* cx)
+{
+    return ownerContext().context() == cx || ownerContext().context() == nullptr;
+}
+
 bool
 ZoneGroup::ownedByCurrentThread()
 {

js/src/gc/ZoneGroup.h

@@ -7,8 +7,6 @@
 #ifndef gc_ZoneGroup_h
 #define gc_ZoneGroup_h
 
-#include "jsgc.h"
-
 #include "gc/Statistics.h"
 #include "vm/Caches.h"
 #include "vm/Stack.h"
@@ -51,6 +49,7 @@ class ZoneGroup
 
     void enter(JSContext* cx);
     void leave();
+    bool canEnterWithoutYielding(JSContext* cx);
     bool ownedByCurrentThread();
 
     // All zones in the group.