diff --git a/js/src/gc/AllocKind.h b/js/src/gc/AllocKind.h new file mode 100644 index 0000000000..3c0416edac --- /dev/null +++ b/js/src/gc/AllocKind.h @@ -0,0 +1,215 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal definition of GC cell kinds. + */ + +#ifndef gc_AllocKind_h +#define gc_AllocKind_h + +#include "mozilla/ArrayUtils.h" +#include "mozilla/EnumeratedArray.h" +#include "mozilla/EnumeratedRange.h" + +#include + +#include "js/TraceKind.h" + +namespace js { +namespace gc { + +// The GC allocation kinds. +// +// These are defined by macros which enumerate the different allocation kinds +// and supply the following information: +// +// - the corresponding AllocKind +// - their JS::TraceKind +// - their C++ base type +// - a C++ type of the correct size +// - whether they can be finalized on the background thread +// - whether they can be allocated in the nursery + +#define FOR_EACH_OBJECT_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType BGFinal Nursery */ \ + D(FUNCTION, Object, JSObject, JSFunction, true, true) \ + D(FUNCTION_EXTENDED, Object, JSObject, FunctionExtended, true, true) \ + D(OBJECT0, Object, JSObject, JSObject_Slots0, false, false) \ + D(OBJECT0_BACKGROUND, Object, JSObject, JSObject_Slots0, true, true) \ + D(OBJECT2, Object, JSObject, JSObject_Slots2, false, false) \ + D(OBJECT2_BACKGROUND, Object, JSObject, JSObject_Slots2, true, true) \ + D(OBJECT4, Object, JSObject, JSObject_Slots4, false, false) \ + D(OBJECT4_BACKGROUND, Object, JSObject, JSObject_Slots4, true, true) \ + D(OBJECT8, Object, JSObject, JSObject_Slots8, false, false) \ + D(OBJECT8_BACKGROUND, Object, JSObject, JSObject_Slots8, true, true) \ + D(OBJECT12, Object, JSObject, JSObject_Slots12, false, false) \ + D(OBJECT12_BACKGROUND, Object, JSObject, JSObject_Slots12, true, true) \ + D(OBJECT16, Object, JSObject, JSObject_Slots16, false, false) \ + D(OBJECT16_BACKGROUND, Object, JSObject, JSObject_Slots16, true, true) + +#define FOR_EACH_NONOBJECT_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType BGFinal Nursery */ \ + D(SCRIPT, Script, JSScript, JSScript, false, false) \ + D(LAZY_SCRIPT, LazyScript, js::LazyScript, js::LazyScript, true, false) \ + D(SHAPE, Shape, js::Shape, js::Shape, true, false) \ + D(ACCESSOR_SHAPE, Shape, js::AccessorShape, js::AccessorShape, true, false) \ + D(BASE_SHAPE, BaseShape, js::BaseShape, js::BaseShape, true, false) \ + D(OBJECT_GROUP, ObjectGroup, js::ObjectGroup, js::ObjectGroup, true, false) \ + D(FAT_INLINE_STRING, String, JSFatInlineString, JSFatInlineString, true, false) \ + D(STRING, String, JSString, JSString, true, false) \ + D(EXTERNAL_STRING, String, JSExternalString, JSExternalString, true, false) \ + D(FAT_INLINE_ATOM, String, js::FatInlineAtom, js::FatInlineAtom, true, false) \ + D(ATOM, String, js::NormalAtom, js::NormalAtom, true, false) \ + D(SYMBOL, Symbol, JS::Symbol, JS::Symbol, true, false) \ + D(JITCODE, JitCode, js::jit::JitCode, js::jit::JitCode, false, false) \ + D(SCOPE, Scope, js::Scope, js::Scope, true, false) \ + D(REGEXP_SHARED, RegExpShared, js::RegExpShared, js::RegExpShared, true, false) + +#define FOR_EACH_ALLOCKIND(D) \ + FOR_EACH_OBJECT_ALLOCKIND(D) \ + FOR_EACH_NONOBJECT_ALLOCKIND(D) + +enum class AllocKind : uint8_t { +#define DEFINE_ALLOC_KIND(allocKind, _1, _2, _3, _4, _5) allocKind, + + FOR_EACH_OBJECT_ALLOCKIND(DEFINE_ALLOC_KIND) + + OBJECT_LIMIT, + OBJECT_LAST = OBJECT_LIMIT - 1, + + FOR_EACH_NONOBJECT_ALLOCKIND(DEFINE_ALLOC_KIND) + + LIMIT, + LAST = LIMIT - 1, + + FIRST = 0, + OBJECT_FIRST = FUNCTION // Hardcoded to first object kind. + +#undef DEFINE_ALLOC_KIND +}; + +static_assert(int(AllocKind::FIRST) == 0, + "Various places depend on AllocKind starting at 0"); +static_assert(int(AllocKind::OBJECT_FIRST) == 0, + "OBJECT_FIRST must be defined as the first object kind"); + +inline bool +IsAllocKind(AllocKind kind) +{ + return kind >= AllocKind::FIRST && kind <= AllocKind::LIMIT; +} + +inline bool +IsValidAllocKind(AllocKind kind) +{ + return kind >= AllocKind::FIRST && kind <= AllocKind::LAST; +} + +inline bool +IsObjectAllocKind(AllocKind kind) +{ + return kind >= AllocKind::OBJECT_FIRST && kind <= AllocKind::OBJECT_LAST; +} + +inline bool +IsShapeAllocKind(AllocKind kind) +{ + return kind == AllocKind::SHAPE || kind == AllocKind::ACCESSOR_SHAPE; +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over all alloc kinds. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT)) +AllAllocKinds() +{ + return mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT); +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over all object alloc kinds. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT)) +ObjectAllocKinds() +{ + return mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT); +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over alloc kinds from |first| to |limit|, exclusive. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT)) +SomeAllocKinds(AllocKind first = AllocKind::FIRST, AllocKind limit = AllocKind::LIMIT) +{ + MOZ_ASSERT(IsAllocKind(first), "|first| is not a valid AllocKind!"); + MOZ_ASSERT(IsAllocKind(limit), "|limit| is not a valid AllocKind!"); + return mozilla::MakeEnumeratedRange(first, limit); +} + +// AllAllocKindArray gives an enumerated array of ValueTypes, +// with each index corresponding to a particular alloc kind. +template using AllAllocKindArray = + mozilla::EnumeratedArray; + +// ObjectAllocKindArray gives an enumerated array of ValueTypes, +// with each index corresponding to a particular object alloc kind. +template using ObjectAllocKindArray = + mozilla::EnumeratedArray; + +static inline JS::TraceKind +MapAllocToTraceKind(AllocKind kind) +{ + static const JS::TraceKind map[] = { +#define EXPAND_ELEMENT(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ + JS::TraceKind::traceKind, +FOR_EACH_ALLOCKIND(EXPAND_ELEMENT) +#undef EXPAND_ELEMENT + }; + + static_assert(MOZ_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT), + "AllocKind-to-TraceKind mapping must be in sync"); + return map[size_t(kind)]; +} + +/* + * This must be an upper bound, but we do not need the least upper bound, so + * we just exclude non-background objects. + */ +static const size_t MAX_BACKGROUND_FINALIZE_KINDS = + size_t(AllocKind::LIMIT) - size_t(AllocKind::OBJECT_LIMIT) / 2; + +static inline bool +IsNurseryAllocable(AllocKind kind) +{ + MOZ_ASSERT(IsValidAllocKind(kind)); + + static const bool map[] = { +#define DEFINE_NURSERY_ALLOCABLE(_1, _2, _3, _4, _5, nursery) nursery, + FOR_EACH_ALLOCKIND(DEFINE_NURSERY_ALLOCABLE) +#undef DEFINE_NURSERY_ALLOCABLE + }; + + JS_STATIC_ASSERT(JS_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT)); + return map[size_t(kind)]; +} + +static inline bool +IsBackgroundFinalized(AllocKind kind) +{ + MOZ_ASSERT(IsValidAllocKind(kind)); + + static const bool map[] = { +#define DEFINE_BACKGROUND_FINALIZED(_1, _2, _3, _4, bgFinal, _5) bgFinal, + FOR_EACH_ALLOCKIND(DEFINE_BACKGROUND_FINALIZED) +#undef DEFINE_BG_FINALIZE + }; + + JS_STATIC_ASSERT(JS_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT)); + return map[size_t(kind)]; +} + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_AllocKind_h */ diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp index 2e1a4e7deb..e8780722a5 100644 --- a/js/src/gc/Allocator.cpp +++ b/js/src/gc/Allocator.cpp @@ -17,6 +17,7 @@ #include "jsobjinlines.h" +#include "gc/ArenaList-inl.h" #include "gc/Heap-inl.h" using namespace js; @@ -147,9 +148,9 @@ js::Allocate(ExclusiveContext* cx) return GCRuntime::tryNewTenuredThing(cx, kind, thingSize); } -#define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType) \ - template type* js::Allocate(ExclusiveContext* cx);\ - template type* js::Allocate(ExclusiveContext* cx); +#define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ + template type* js::Allocate(JSContext* cx);\ + template type* js::Allocate(JSContext* cx); FOR_EACH_NONOBJECT_ALLOCKIND(DECL_ALLOCATOR_INSTANCES) #undef DECL_ALLOCATOR_INSTANCES @@ -295,7 +296,7 @@ GCRuntime::refillFreeListFromMainThread(JSContext* cx, AllocKind thingKind, size Zone *zone = cx->zone(); MOZ_ASSERT(!cx->runtime()->isHeapBusy(), "allocating while under GC"); - return cx->arenas()->allocateFromArena(zone, thingKind, CheckThresholds); + return cx->arenas()->allocateFromArena(zone, thingKind, ShouldCheckThresholds::CheckThresholds); } /* static */ TenuredCell* @@ -306,7 +307,7 @@ GCRuntime::refillFreeListOffMainThread(ExclusiveContext* cx, AllocKind thingKind Zone* zone = cx->zone(); MOZ_ASSERT(!zone->wasGCStarted()); - return cx->arenas()->allocateFromArena(zone, thingKind, CheckThresholds); + return cx->arenas()->allocateFromArena(zone, thingKind, ShouldCheckThresholds::CheckThresholds); } /* static */ TenuredCell* @@ -321,7 +322,7 @@ GCRuntime::refillFreeListInGC(Zone* zone, AllocKind thingKind) MOZ_ASSERT(rt->isHeapCollecting()); MOZ_ASSERT_IF(!rt->isHeapMinorCollecting(), !rt->gc.isBackgroundSweeping()); - return zone->arenas.allocateFromArena(zone, thingKind, DontCheckThresholds); + return zone->arenas.allocateFromArena(zone, thingKind, ShouldCheckThresholds::DontCheckThresholds); } TenuredCell* @@ -413,14 +414,15 @@ GCRuntime::allocateArena(Chunk* chunk, Zone* zone, AllocKind thingKind, MOZ_ASSERT(chunk->hasAvailableArenas()); // Fail the allocation if we are over our heap size limits. - if (checkThresholds && usage.gcBytes() >= tunables.gcMaxBytes()) + if ((checkThresholds != ShouldCheckThresholds::DontCheckThresholds) && + (usage.gcBytes() >= tunables.gcMaxBytes())) return nullptr; Arena* arena = chunk->allocateArena(rt, zone, thingKind, lock); zone->usage.addGCArena(); // Trigger an incremental slice if needed. - if (checkThresholds) + if (checkThresholds != ShouldCheckThresholds::DontCheckThresholds) maybeAllocTriggerZoneGC(zone, lock, ArenaSize); return arena; diff --git a/js/src/gc/Allocator.h b/js/src/gc/Allocator.h index d43c744dca..3f7cdf00a7 100644 --- a/js/src/gc/Allocator.h +++ b/js/src/gc/Allocator.h @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -10,6 +11,7 @@ #include "js/RootingAPI.h" namespace js { + struct Class; // Allocate a new GC thing. After a successful allocation the caller must diff --git a/js/src/gc/ArenaList-inl.h b/js/src/gc/ArenaList-inl.h new file mode 100644 index 0000000000..8097f48c52 --- /dev/null +++ b/js/src/gc/ArenaList-inl.h @@ -0,0 +1,353 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_ArenaList_inl_h +#define gc_ArenaList_inl_h + +#include "gc/ArenaList.h" + +#include "gc/Heap.h" + +void +js::gc::SortedArenaListSegment::append(Arena* arena) +{ + MOZ_ASSERT(arena); + MOZ_ASSERT_IF(head, head->getAllocKind() == arena->getAllocKind()); + *tailp = arena; + tailp = &arena->next; +} + +inline +js::gc::ArenaList::ArenaList() +{ + clear(); +} + +void +js::gc::ArenaList::copy(const ArenaList& other) +{ + other.check(); + head_ = other.head_; + cursorp_ = other.isCursorAtHead() ? &head_ : other.cursorp_; + check(); +} + +inline +js::gc::ArenaList::ArenaList(const ArenaList& other) +{ + copy(other); +} + +js::gc::ArenaList& +js::gc::ArenaList::operator=(const ArenaList& other) +{ + copy(other); + return *this; +} + +inline +js::gc::ArenaList::ArenaList(const SortedArenaListSegment& segment) +{ + head_ = segment.head; + cursorp_ = segment.isEmpty() ? &head_ : segment.tailp; + check(); +} + +// This does checking just of |head_| and |cursorp_|. +void +js::gc::ArenaList::check() const +{ +#ifdef DEBUG + // If the list is empty, it must have this form. + MOZ_ASSERT_IF(!head_, cursorp_ == &head_); + + // If there's an arena following the cursor, it must not be full. + Arena* cursor = *cursorp_; + MOZ_ASSERT_IF(cursor, cursor->hasFreeThings()); +#endif +} + +void +js::gc::ArenaList::clear() +{ + head_ = nullptr; + cursorp_ = &head_; + check(); +} + +js::gc::ArenaList +js::gc::ArenaList::copyAndClear() +{ + ArenaList result = *this; + clear(); + return result; +} + +bool +js::gc::ArenaList::isEmpty() const +{ + check(); + return !head_; +} + +js::gc::Arena* +js::gc::ArenaList::head() const +{ + check(); + return head_; +} + +bool +js::gc::ArenaList::isCursorAtHead() const +{ + check(); + return cursorp_ == &head_; +} + +bool +js::gc::ArenaList::isCursorAtEnd() const +{ + check(); + return !*cursorp_; +} + +void +js::gc::ArenaList::moveCursorToEnd() +{ + while (!isCursorAtEnd()) + cursorp_ = &(*cursorp_)->next; +} + +js::gc::Arena* +js::gc::ArenaList::arenaAfterCursor() const +{ + check(); + return *cursorp_; +} + +js::gc::Arena* +js::gc::ArenaList::takeNextArena() +{ + check(); + Arena* arena = *cursorp_; + if (!arena) + return nullptr; + cursorp_ = &arena->next; + check(); + return arena; +} + +void +js::gc::ArenaList::insertAtCursor(Arena* a) +{ + check(); + a->next = *cursorp_; + *cursorp_ = a; + // At this point, the cursor is sitting before |a|. Move it after |a| + // if necessary. + if (!a->hasFreeThings()) + cursorp_ = &a->next; + check(); +} + +void +js::gc::ArenaList::insertBeforeCursor(Arena* a) +{ + check(); + a->next = *cursorp_; + *cursorp_ = a; + cursorp_ = &a->next; + check(); +} + +js::gc::ArenaList& +js::gc::ArenaList::insertListWithCursorAtEnd(const ArenaList& other) +{ + check(); + other.check(); + MOZ_ASSERT(other.isCursorAtEnd()); + if (other.isCursorAtHead()) + return *this; + // Insert the full arenas of |other| after those of |this|. + *other.cursorp_ = *cursorp_; + *cursorp_ = other.head_; + cursorp_ = other.cursorp_; + check(); + return *this; +} + +js::gc::SortedArenaList::SortedArenaList(size_t thingsPerArena) +{ + reset(thingsPerArena); +} + +void +js::gc::SortedArenaList::setThingsPerArena(size_t thingsPerArena) +{ + MOZ_ASSERT(thingsPerArena && thingsPerArena <= MaxThingsPerArena); + thingsPerArena_ = thingsPerArena; +} + +void +js::gc::SortedArenaList::reset(size_t thingsPerArena) +{ + setThingsPerArena(thingsPerArena); + // Initialize the segments. + for (size_t i = 0; i <= thingsPerArena; ++i) + segments[i].clear(); +} + +void +js::gc::SortedArenaList::insertAt(Arena* arena, size_t nfree) +{ + MOZ_ASSERT(nfree <= thingsPerArena_); + segments[nfree].append(arena); +} + +void +js::gc::SortedArenaList::extractEmpty(Arena** empty) +{ + SortedArenaListSegment& segment = segments[thingsPerArena_]; + if (segment.head) { + *segment.tailp = *empty; + *empty = segment.head; + segment.clear(); + } +} + +js::gc::ArenaList +js::gc::SortedArenaList::toArenaList() +{ + // Link the non-empty segment tails up to the non-empty segment heads. + size_t tailIndex = 0; + for (size_t headIndex = 1; headIndex <= thingsPerArena_; ++headIndex) { + if (headAt(headIndex)) { + segments[tailIndex].linkTo(headAt(headIndex)); + tailIndex = headIndex; + } + } + // Point the tail of the final non-empty segment at null. Note that if + // the list is empty, this will just set segments[0].head to null. + segments[tailIndex].linkTo(nullptr); + // Create an ArenaList with head and cursor set to the head and tail of + // the first segment (if that segment is empty, only the head is used). + return ArenaList(segments[0]); +} + +js::gc::Arena* +js::gc::ArenaLists::getFirstArena(AllocKind thingKind) const +{ + return arenaLists(thingKind).head(); +} + +js::gc::Arena* +js::gc::ArenaLists::getFirstArenaToSweep(AllocKind thingKind) const +{ + return arenaListsToSweep(thingKind); +} + +js::gc::Arena* +js::gc::ArenaLists::getFirstSweptArena(AllocKind thingKind) const +{ + if (thingKind != incrementalSweptArenaKind.ref()) + return nullptr; + return incrementalSweptArenas.ref().head(); +} + +js::gc::Arena* +js::gc::ArenaLists::getArenaAfterCursor(AllocKind thingKind) const +{ + return arenaLists(thingKind).arenaAfterCursor(); +} + +bool +js::gc::ArenaLists::arenaListsAreEmpty() const +{ + for (auto i : AllAllocKinds()) { + /* + * The arena cannot be empty if the background finalization is not yet + * done. + */ + if (backgroundFinalizeState(i) != BFS_DONE) + return false; + if (!arenaLists(i).isEmpty()) + return false; + } + return true; +} + +void +js::gc::ArenaLists::unmarkAll() +{ + for (auto i : AllAllocKinds()) { + /* The background finalization must have stopped at this point. */ + MOZ_ASSERT(backgroundFinalizeState(i) == BFS_DONE); + for (Arena* arena = arenaLists(i).head(); arena; arena = arena->next) + arena->unmarkAll(); + } +} + +bool +js::gc::ArenaLists::doneBackgroundFinalize(AllocKind kind) const +{ + return backgroundFinalizeState(kind) == BFS_DONE; +} + +bool +js::gc::ArenaLists::needBackgroundFinalizeWait(AllocKind kind) const +{ + return backgroundFinalizeState(kind) != BFS_DONE; +} + +void +js::gc::ArenaLists::purge() +{ + for (auto i : AllAllocKinds()) + freeLists(i) = &placeholder; +} + +bool +js::gc::ArenaLists::arenaIsInUse(Arena* arena, AllocKind kind) const +{ + MOZ_ASSERT(arena); + return arena == freeLists(kind)->getArenaUnchecked(); +} + +MOZ_ALWAYS_INLINE js::gc::TenuredCell* +js::gc::ArenaLists::allocateFromFreeList(AllocKind thingKind, size_t thingSize) +{ + return freeLists(thingKind)->allocate(thingSize); +} + +void +js::gc::ArenaLists::checkEmptyFreeLists() +{ +#ifdef DEBUG + for (auto i : AllAllocKinds()) + checkEmptyFreeList(i); +#endif +} + +bool +js::gc::ArenaLists::checkEmptyArenaLists() +{ + bool empty = true; +#ifdef DEBUG + for (auto i : AllAllocKinds()) { + if (!checkEmptyArenaList(i)) + empty = false; + } +#endif + return empty; +} + +void +js::gc::ArenaLists::checkEmptyFreeList(AllocKind kind) +{ + MOZ_ASSERT(freeLists(kind)->isEmpty()); +} + +#endif // gc_ArenaList_inl_h diff --git a/js/src/gc/ArenaList.h b/js/src/gc/ArenaList.h new file mode 100644 index 0000000000..caad7724b5 --- /dev/null +++ b/js/src/gc/ArenaList.h @@ -0,0 +1,358 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal definitions of ArenaList and associated heap data structures. + */ + +#ifndef gc_ArenaList_h +#define gc_ArenaList_h + +#include "gc/AllocKind.h" +#include "js/SliceBudget.h" +#include "threading/ProtectedData.h" + +namespace JS { + +struct Zone; + +} /* namespace JS */ + +namespace js { + +class FreeOp; +class Nursery; +class TenuringTracer; + +namespace gcstats { +struct Statistics; +} + +namespace gc { + +class Arena; +struct FinalizePhase; +class FreeSpan; +class TenuredCell; + +/* + * A single segment of a SortedArenaList. Each segment has a head and a tail, + * which track the start and end of a segment for O(1) append and concatenation. + */ +struct SortedArenaListSegment +{ + Arena* head; + Arena** tailp; + + void clear() { + head = nullptr; + tailp = &head; + } + + bool isEmpty() const { + return tailp == &head; + } + + // Appends |arena| to this segment. + inline void append(Arena* arena); + + // Points the tail of this segment at |arena|, which may be null. Note + // that this does not change the tail itself, but merely which arena + // follows it. This essentially turns the tail into a cursor (see also the + // description of ArenaList), but from the perspective of a SortedArenaList + // this makes no difference. + void linkTo(Arena* arena) { + *tailp = arena; + } +}; + +/* + * Arena lists have a head and a cursor. The cursor conceptually lies on arena + * boundaries, i.e. before the first arena, between two arenas, or after the + * last arena. + * + * Arenas are usually sorted in order of increasing free space, with the cursor + * following the Arena currently being allocated from. This ordering should not + * be treated as an invariant, however, as the free lists may be cleared, + * leaving arenas previously used for allocation partially full. Sorting order + * is restored during sweeping. + + * Arenas following the cursor should not be full. + */ +class ArenaList { + // The cursor is implemented via an indirect pointer, |cursorp_|, to allow + // for efficient list insertion at the cursor point and other list + // manipulations. + // + // - If the list is empty: |head| is null, |cursorp_| points to |head|, and + // therefore |*cursorp_| is null. + // + // - If the list is not empty: |head| is non-null, and... + // + // - If the cursor is at the start of the list: |cursorp_| points to + // |head|, and therefore |*cursorp_| points to the first arena. + // + // - If cursor is at the end of the list: |cursorp_| points to the |next| + // field of the last arena, and therefore |*cursorp_| is null. + // + // - If the cursor is at neither the start nor the end of the list: + // |cursorp_| points to the |next| field of the arena preceding the + // cursor, and therefore |*cursorp_| points to the arena following the + // cursor. + // + // |cursorp_| is never null. + // + Arena* head_; + Arena** cursorp_; + + inline void copy(const ArenaList& other); + + public: + inline ArenaList(); + inline ArenaList(const ArenaList& other); + + inline ArenaList& operator=(const ArenaList& other); + + inline explicit ArenaList(const SortedArenaListSegment& segment); + + inline void check() const; + + inline void clear(); + inline ArenaList copyAndClear(); + inline bool isEmpty() const; + + // This returns nullptr if the list is empty. + inline Arena* head() const; + + inline bool isCursorAtHead() const; + inline bool isCursorAtEnd() const; + + inline void moveCursorToEnd(); + + // This can return nullptr. + inline Arena* arenaAfterCursor() const; + + // This returns the arena after the cursor and moves the cursor past it. + inline Arena* takeNextArena(); + + // This does two things. + // - Inserts |a| at the cursor. + // - Leaves the cursor sitting just before |a|, if |a| is not full, or just + // after |a|, if |a| is full. + inline void insertAtCursor(Arena* a); + + // Inserts |a| at the cursor, then moves the cursor past it. + inline void insertBeforeCursor(Arena* a); + + // This inserts |other|, which must be full, at the cursor of |this|. + inline ArenaList& insertListWithCursorAtEnd(const ArenaList& other); + + Arena* removeRemainingArenas(Arena** arenap); + Arena** pickArenasToRelocate(size_t& arenaTotalOut, size_t& relocTotalOut); + Arena* relocateArenas(Arena* toRelocate, Arena* relocated, + js::SliceBudget& sliceBudget, gcstats::Statistics& stats); +}; + +/* + * A class that holds arenas in sorted order by appending arenas to specific + * segments. Each segment has a head and a tail, which can be linked up to + * other segments to create a contiguous ArenaList. + */ +class SortedArenaList +{ + public: + // The minimum size, in bytes, of a GC thing. + static const size_t MinThingSize = 16; + + static_assert(ArenaSize <= 4096, "When increasing the Arena size, please consider how"\ + " this will affect the size of a SortedArenaList."); + + static_assert(MinThingSize >= 16, "When decreasing the minimum thing size, please consider"\ + " how this will affect the size of a SortedArenaList."); + + private: + // The maximum number of GC things that an arena can hold. + static const size_t MaxThingsPerArena = (ArenaSize - ArenaHeaderSize) / MinThingSize; + + size_t thingsPerArena_; + SortedArenaListSegment segments[MaxThingsPerArena + 1]; + + // Convenience functions to get the nth head and tail. + Arena* headAt(size_t n) { return segments[n].head; } + Arena** tailAt(size_t n) { return segments[n].tailp; } + + public: + inline explicit SortedArenaList(size_t thingsPerArena = MaxThingsPerArena); + + inline void setThingsPerArena(size_t thingsPerArena); + + // Resets the first |thingsPerArena| segments of this list for further use. + inline void reset(size_t thingsPerArena = MaxThingsPerArena); + + // Inserts an arena, which has room for |nfree| more things, in its segment. + inline void insertAt(Arena* arena, size_t nfree); + + // Remove all empty arenas, inserting them as a linked list. + inline void extractEmpty(Arena** empty); + + // Links up the tail of each non-empty segment to the head of the next + // non-empty segment, creating a contiguous list that is returned as an + // ArenaList. This is not a destructive operation: neither the head nor tail + // of any segment is modified. However, note that the Arenas in the + // resulting ArenaList should be treated as read-only unless the + // SortedArenaList is no longer needed: inserting or removing arenas would + // invalidate the SortedArenaList. + inline ArenaList toArenaList(); +}; + +enum class ShouldCheckThresholds +{ + DontCheckThresholds = 0, + CheckThresholds = 1 +}; + +class ArenaLists +{ + JSRuntime* const runtime_; + + /* + * For each arena kind its free list is represented as the first span with + * free things. Initially all the spans are initialized as empty. After we + * find a new arena with available things we move its first free span into + * the list and set the arena as fully allocated. way we do not need to + * update the arena after the initial allocation. When starting the + * GC we only move the head of the of the list of spans back to the arena + * only for the arena that was not fully allocated. + */ + ZoneGroupData> freeLists_; + FreeSpan*& freeLists(AllocKind i) { return freeLists_.ref()[i]; } + FreeSpan* freeLists(AllocKind i) const { return freeLists_.ref()[i]; } + + // Because the JITs can allocate from the free lists, they cannot be null. + // We use a placeholder FreeSpan that is empty (and wihout an associated + // Arena) so the JITs can fall back gracefully. + static FreeSpan placeholder; + + ZoneGroupOrGCTaskData> arenaLists_; + ArenaList& arenaLists(AllocKind i) { return arenaLists_.ref()[i]; } + const ArenaList& arenaLists(AllocKind i) const { return arenaLists_.ref()[i]; } + + enum BackgroundFinalizeStateEnum { BFS_DONE, BFS_RUN }; + + typedef mozilla::Atomic + BackgroundFinalizeState; + + /* The current background finalization state, accessed atomically. */ + UnprotectedData> backgroundFinalizeState_; + BackgroundFinalizeState& backgroundFinalizeState(AllocKind i) { return backgroundFinalizeState_.ref()[i]; } + const BackgroundFinalizeState& backgroundFinalizeState(AllocKind i) const { return backgroundFinalizeState_.ref()[i]; } + + /* For each arena kind, a list of arenas remaining to be swept. */ + ActiveThreadOrGCTaskData> arenaListsToSweep_; + Arena*& arenaListsToSweep(AllocKind i) { return arenaListsToSweep_.ref()[i]; } + Arena* arenaListsToSweep(AllocKind i) const { return arenaListsToSweep_.ref()[i]; } + + /* During incremental sweeping, a list of the arenas already swept. */ + ZoneGroupOrGCTaskData incrementalSweptArenaKind; + ZoneGroupOrGCTaskData incrementalSweptArenas; + + // Arena lists which have yet to be swept, but need additional foreground + // processing before they are swept. + ZoneGroupData gcShapeArenasToUpdate; + ZoneGroupData gcAccessorShapeArenasToUpdate; + ZoneGroupData gcScriptArenasToUpdate; + ZoneGroupData gcObjectGroupArenasToUpdate; + + // While sweeping type information, these lists save the arenas for the + // objects which have already been finalized in the foreground (which must + // happen at the beginning of the GC), so that type sweeping can determine + // which of the object pointers are marked. + ZoneGroupData> savedObjectArenas_; + ArenaList& savedObjectArenas(AllocKind i) { return savedObjectArenas_.ref()[i]; } + ZoneGroupData savedEmptyObjectArenas; + + public: + explicit ArenaLists(JSRuntime* rt, ZoneGroup* group); + ~ArenaLists(); + + const void* addressOfFreeList(AllocKind thingKind) const { + return reinterpret_cast(&freeLists_.refNoCheck()[thingKind]); + } + + inline Arena* getFirstArena(AllocKind thingKind) const; + inline Arena* getFirstArenaToSweep(AllocKind thingKind) const; + inline Arena* getFirstSweptArena(AllocKind thingKind) const; + inline Arena* getArenaAfterCursor(AllocKind thingKind) const; + + inline bool arenaListsAreEmpty() const; + + inline void unmarkAll(); + + inline bool doneBackgroundFinalize(AllocKind kind) const; + inline bool needBackgroundFinalizeWait(AllocKind kind) const; + + /* Clear the free lists so we won't try to allocate from swept arenas. */ + inline void purge(); + + inline void prepareForIncrementalGC(); + + /* Check if this arena is in use. */ + inline bool arenaIsInUse(Arena* arena, AllocKind kind) const; + + MOZ_ALWAYS_INLINE TenuredCell* allocateFromFreeList(AllocKind thingKind, size_t thingSize); + + /* Moves all arenas from |fromArenaLists| into |this|. */ + void adoptArenas(JSRuntime* runtime, ArenaLists* fromArenaLists, bool targetZoneIsCollecting); + + /* True if the Arena in question is found in this ArenaLists */ + bool containsArena(JSRuntime* runtime, Arena* arena); + + inline void checkEmptyFreeLists(); + inline bool checkEmptyArenaLists(); + inline void checkEmptyFreeList(AllocKind kind); + + bool checkEmptyArenaList(AllocKind kind); + + bool relocateArenas(JS::Zone* zone, Arena*& relocatedListOut, JS::gcreason::Reason reason, + js::SliceBudget& sliceBudget, gcstats::Statistics& stats); + + void queueForegroundObjectsForSweep(FreeOp* fop); + void queueForegroundThingsForSweep(FreeOp* fop); + + void mergeForegroundSweptObjectArenas(); + + bool foregroundFinalize(FreeOp* fop, AllocKind thingKind, js::SliceBudget& sliceBudget, + SortedArenaList& sweepList); + static void backgroundFinalize(FreeOp* fop, Arena* listHead, Arena** empty); + + // When finalizing arenas, whether to keep empty arenas on the list or + // release them immediately. + enum KeepArenasEnum { + RELEASE_ARENAS, + KEEP_ARENAS + }; + + private: + inline void queueForForegroundSweep(FreeOp* fop, const FinalizePhase& phase); + inline void queueForBackgroundSweep(FreeOp* fop, const FinalizePhase& phase); + inline void queueForForegroundSweep(FreeOp* fop, AllocKind thingKind); + inline void queueForBackgroundSweep(FreeOp* fop, AllocKind thingKind); + inline void mergeSweptArenas(AllocKind thingKind); + + TenuredCell* allocateFromArena(JS::Zone* zone, AllocKind thingKind, + ShouldCheckThresholds checkThresholds); + inline TenuredCell* allocateFromArenaInner(JS::Zone* zone, Arena* arena, AllocKind kind); + + friend class GCRuntime; + friend class js::Nursery; + friend class js::TenuringTracer; +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_ArenaList_h */ + diff --git a/js/src/gc/AtomMarking.h b/js/src/gc/AtomMarking.h new file mode 100644 index 0000000000..44b3ce6024 --- /dev/null +++ b/js/src/gc/AtomMarking.h @@ -0,0 +1,86 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_AtomMarking_h +#define gc_AtomMarking_h + +#include "NamespaceImports.h" +#include "ds/Bitmap.h" +#include "threading/ProtectedData.h" +#include "vm/Symbol.h" + +namespace js { +namespace gc { + +class Arena; + +// This class manages state used for marking atoms during GCs. +// See AtomMarking.cpp for details. +class AtomMarkingRuntime +{ + // Unused arena atom bitmap indexes. Protected by the GC lock. + js::ExclusiveAccessLockOrGCTaskData> freeArenaIndexes; + + void markChildren(JSContext* cx, JSAtom*) {} + + void markChildren(JSContext* cx, JS::Symbol* symbol) { + if (JSAtom* description = symbol->description()) + markAtom(cx, description); + } + + public: + // The extent of all allocated and free words in atom mark bitmaps. + // This monotonically increases and may be read from without locking. + mozilla::Atomic allocatedWords; + + AtomMarkingRuntime() + : allocatedWords(0) + {} + + // Mark an arena as holding things in the atoms zone. + void registerArena(Arena* arena); + + // Mark an arena as no longer holding things in the atoms zone. + void unregisterArena(Arena* arena); + + // Fill |bitmap| with an atom marking bitmap based on the things that are + // currently marked in the chunks used by atoms zone arenas. This returns + // false on an allocation failure (but does not report an exception). + bool computeBitmapFromChunkMarkBits(JSRuntime* runtime, DenseBitmap& bitmap); + + // Update the atom marking bitmap in |zone| according to another + // overapproximation of the reachable atoms in |bitmap|. + void updateZoneBitmap(Zone* zone, const DenseBitmap& bitmap); + + // Set any bits in the chunk mark bitmaps for atoms which are marked in any + // zone in the runtime. + void updateChunkMarkBits(JSRuntime* runtime); + + // Mark an atom or id as being newly reachable by the context's zone. + template void markAtom(JSContext* cx, T* thing); + + // Version of markAtom that's always inlined, for performance-sensitive + // callers. + template MOZ_ALWAYS_INLINE void inlinedMarkAtom(JSContext* cx, T* thing); + + void markId(JSContext* cx, jsid id); + void markAtomValue(JSContext* cx, const Value& value); + + // Mark all atoms in |source| as being reachable within |target|. + void adoptMarkedAtoms(Zone* target, Zone* source); + +#ifdef DEBUG + // Return whether |thing/id| is in the atom marking bitmap for |zone|. + template bool atomIsMarked(Zone* zone, T* thing); + bool idIsMarked(Zone* zone, jsid id); + bool valueIsMarked(Zone* zone, const Value& value); +#endif +}; + +} // namespace gc +} // namespace js + +#endif // gc_AtomMarking_h diff --git a/js/src/gc/Barrier.h b/js/src/gc/Barrier.h index b0c4917af7..3c2f376ef5 100644 --- a/js/src/gc/Barrier.h +++ b/js/src/gc/Barrier.h @@ -9,7 +9,7 @@ #include "NamespaceImports.h" -#include "gc/Heap.h" +#include "gc/Cell.h" #include "gc/StoreBuffer.h" #include "js/HeapAPI.h" #include "js/Id.h" diff --git a/js/src/gc/Cell.h b/js/src/gc/Cell.h new file mode 100644 index 0000000000..57e5fe16aa --- /dev/null +++ b/js/src/gc/Cell.h @@ -0,0 +1,413 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Cell_h +#define gc_Cell_h + +#include "gc/GCEnum.h" +#include "gc/Heap.h" +#include "js/GCAnnotations.h" + +namespace JS { + +namespace shadow { +struct Zone; +} /* namespace shadow */ + +enum class TraceKind; +struct Zone; +} /* namespace JS */ + +namespace js { + +class GenericPrinter; + +extern bool +RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone); + +#ifdef DEBUG + +// Barriers can't be triggered during backend Ion compilation, which may run on +// a helper thread. +extern bool +CurrentThreadIsIonCompiling(); +#endif + +extern void +TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name); + +namespace gc { + +class Arena; +enum class AllocKind : uint8_t; +struct Chunk; +class TenuredCell; + +// A GC cell is the base class for all GC things. +struct Cell +{ + public: + MOZ_ALWAYS_INLINE bool isTenured() const { return !IsInsideNursery(this); } + MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const; + MOZ_ALWAYS_INLINE TenuredCell& asTenured(); + + MOZ_ALWAYS_INLINE bool isMarkedAny() const; + MOZ_ALWAYS_INLINE bool isMarkedBlack() const; + MOZ_ALWAYS_INLINE bool isMarkedGray() const; + + inline JSRuntime* runtimeFromActiveCooperatingThread() const; + + // Note: Unrestricted access to the runtime of a GC thing from an arbitrary + // thread can easily lead to races. Use this method very carefully. + inline JSRuntime* runtimeFromAnyThread() const; + + // May be overridden by GC thing kinds that have a compartment pointer. + inline JSCompartment* maybeCompartment() const { return nullptr; } + + // The StoreBuffer used to record incoming pointers from the tenured heap. + // This will return nullptr for a tenured cell. + inline StoreBuffer* storeBuffer() const; + + inline JS::TraceKind getTraceKind() const; + + static MOZ_ALWAYS_INLINE bool needWriteBarrierPre(JS::Zone* zone); + +#ifdef DEBUG + inline bool isAligned() const; + void dump(GenericPrinter& out) const; + void dump() const; +#endif + + protected: + uintptr_t address() const; + inline Chunk* chunk() const; +} JS_HAZ_GC_THING; + +// A GC TenuredCell gets behaviors that are valid for things in the Tenured +// heap, such as access to the arena and mark bits. +class TenuredCell : public Cell +{ + public: + // Construct a TenuredCell from a void*, making various sanity assertions. + static MOZ_ALWAYS_INLINE TenuredCell* fromPointer(void* ptr); + static MOZ_ALWAYS_INLINE const TenuredCell* fromPointer(const void* ptr); + + // Mark bit management. + MOZ_ALWAYS_INLINE bool isMarkedAny() const; + MOZ_ALWAYS_INLINE bool isMarkedBlack() const; + MOZ_ALWAYS_INLINE bool isMarkedGray() const; + + // The return value indicates if the cell went from unmarked to marked. + MOZ_ALWAYS_INLINE bool markIfUnmarked(MarkColor color = MarkColor::Black) const; + MOZ_ALWAYS_INLINE void markBlack() const; + MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src); + + // Access to the arena. + inline Arena* arena() const; + inline AllocKind getAllocKind() const; + inline JS::TraceKind getTraceKind() const; + inline JS::Zone* zone() const; + inline JS::Zone* zoneFromAnyThread() const; + inline bool isInsideZone(JS::Zone* zone) const; + + MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZone() const { + return JS::shadow::Zone::asShadowZone(zone()); + } + MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZoneFromAnyThread() const { + return JS::shadow::Zone::asShadowZone(zoneFromAnyThread()); + } + + static MOZ_ALWAYS_INLINE void readBarrier(TenuredCell* thing); + static MOZ_ALWAYS_INLINE void writeBarrierPre(TenuredCell* thing); + + static void MOZ_ALWAYS_INLINE writeBarrierPost(void* cellp, TenuredCell* prior, + TenuredCell* next); + + // Default implementation for kinds that don't require fixup. + void fixupAfterMovingGC() {} + +#ifdef DEBUG + inline bool isAligned() const; +#endif +}; + +MOZ_ALWAYS_INLINE const TenuredCell& +Cell::asTenured() const +{ + MOZ_ASSERT(isTenured()); + return *static_cast(this); +} + +MOZ_ALWAYS_INLINE TenuredCell& +Cell::asTenured() +{ + MOZ_ASSERT(isTenured()); + return *static_cast(this); +} + +MOZ_ALWAYS_INLINE bool +Cell::isMarkedAny() const +{ + return !isTenured() || asTenured().isMarkedAny(); +} + +MOZ_ALWAYS_INLINE bool +Cell::isMarkedBlack() const +{ + return !isTenured() || asTenured().isMarkedBlack(); +} + +MOZ_ALWAYS_INLINE bool +Cell::isMarkedGray() const +{ + return isTenured() && asTenured().isMarkedGray(); +} + +inline JSRuntime* +Cell::runtimeFromActiveCooperatingThread() const +{ + JSRuntime* rt = chunk()->trailer.runtime; + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + return rt; +} + +inline JSRuntime* +Cell::runtimeFromAnyThread() const +{ + return chunk()->trailer.runtime; +} + +inline uintptr_t +Cell::address() const +{ + uintptr_t addr = uintptr_t(this); + MOZ_ASSERT(addr % CellAlignBytes == 0); + MOZ_ASSERT(Chunk::withinValidRange(addr)); + return addr; +} + +Chunk* +Cell::chunk() const +{ + uintptr_t addr = uintptr_t(this); + MOZ_ASSERT(addr % CellAlignBytes == 0); + addr &= ~ChunkMask; + return reinterpret_cast(addr); +} + +inline StoreBuffer* +Cell::storeBuffer() const +{ + return chunk()->trailer.storeBuffer; +} + +inline JS::TraceKind +Cell::getTraceKind() const +{ + return isTenured() ? asTenured().getTraceKind() : JS::TraceKind::Object; +} + +/* static */ MOZ_ALWAYS_INLINE bool +Cell::needWriteBarrierPre(JS::Zone* zone) { + return JS::shadow::Zone::asShadowZone(zone)->needsIncrementalBarrier(); +} + +/* static */ MOZ_ALWAYS_INLINE TenuredCell* +TenuredCell::fromPointer(void* ptr) +{ + MOZ_ASSERT(static_cast(ptr)->isTenured()); + return static_cast(ptr); +} + +/* static */ MOZ_ALWAYS_INLINE const TenuredCell* +TenuredCell::fromPointer(const void* ptr) +{ + MOZ_ASSERT(static_cast(ptr)->isTenured()); + return static_cast(ptr); +} + +bool +TenuredCell::isMarkedAny() const +{ + MOZ_ASSERT(arena()->allocated()); + return chunk()->bitmap.isMarkedAny(this); +} + +bool +TenuredCell::isMarkedBlack() const +{ + MOZ_ASSERT(arena()->allocated()); + return chunk()->bitmap.isMarkedBlack(this); +} + +bool +TenuredCell::isMarkedGray() const +{ + MOZ_ASSERT(arena()->allocated()); + return chunk()->bitmap.isMarkedGray(this); +} + +bool +TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const +{ + return chunk()->bitmap.markIfUnmarked(this, color); +} + +void +TenuredCell::markBlack() const +{ + chunk()->bitmap.markBlack(this); +} + +void +TenuredCell::copyMarkBitsFrom(const TenuredCell* src) +{ + ChunkBitmap& bitmap = chunk()->bitmap; + bitmap.copyMarkBit(this, src, ColorBit::BlackBit); + bitmap.copyMarkBit(this, src, ColorBit::GrayOrBlackBit); +} + +inline Arena* +TenuredCell::arena() const +{ + MOZ_ASSERT(isTenured()); + uintptr_t addr = address(); + addr &= ~ArenaMask; + return reinterpret_cast(addr); +} + +AllocKind +TenuredCell::getAllocKind() const +{ + return arena()->getAllocKind(); +} + +JS::TraceKind +TenuredCell::getTraceKind() const +{ + return MapAllocToTraceKind(getAllocKind()); +} + +JS::Zone* +TenuredCell::zone() const +{ + JS::Zone* zone = arena()->zone; + MOZ_ASSERT(CurrentThreadCanAccessZone(zone)); + return zone; +} + +JS::Zone* +TenuredCell::zoneFromAnyThread() const +{ + return arena()->zone; +} + +bool +TenuredCell::isInsideZone(JS::Zone* zone) const +{ + return zone == arena()->zone; +} + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::readBarrier(TenuredCell* thing) +{ + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + MOZ_ASSERT(thing); + MOZ_ASSERT(CurrentThreadCanAccessZone(thing->zoneFromAnyThread())); + + // It would be good if barriers were never triggered during collection, but + // at the moment this can happen e.g. when rekeying tables containing + // read-barriered GC things after a moving GC. + // + // TODO: Fix this and assert we're not collecting if we're on the active + // thread. + + JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); + if (shadowZone->needsIncrementalBarrier()) { + // Barriers are only enabled on the active thread and are disabled while collecting. + MOZ_ASSERT(!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone)); + Cell* tmp = thing; + TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "read barrier"); + MOZ_ASSERT(tmp == thing); + } + + if (thing->isMarkedGray()) { + // There shouldn't be anything marked grey unless we're on the active thread. + MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())); + if (!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone)) + JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(thing, thing->getTraceKind())); + } +} + +void +AssertSafeToSkipBarrier(TenuredCell* thing); + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::writeBarrierPre(TenuredCell* thing) +{ + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + if (!thing) + return; + +#ifdef JS_GC_ZEAL + // When verifying pre barriers we need to switch on all barriers, even + // those on the Atoms Zone. Normally, we never enter a parse task when + // collecting in the atoms zone, so will filter out atoms below. + // Unfortuantely, If we try that when verifying pre-barriers, we'd never be + // able to handle off thread parse tasks at all as we switch on the verifier any + // time we're not doing GC. This would cause us to deadlock, as off thread parsing + // is meant to resume after GC work completes. Instead we filter out any + // off thread barriers that reach us and assert that they would normally not be + // possible. + if (!CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())) { + AssertSafeToSkipBarrier(thing); + return; + } +#endif + + JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); + if (shadowZone->needsIncrementalBarrier()) { + MOZ_ASSERT(!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone)); + Cell* tmp = thing; + TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "pre barrier"); + MOZ_ASSERT(tmp == thing); + } +} + +static MOZ_ALWAYS_INLINE void +AssertValidToSkipBarrier(TenuredCell* thing) +{ + MOZ_ASSERT(!IsInsideNursery(thing)); + MOZ_ASSERT_IF(thing, MapAllocToTraceKind(thing->getAllocKind()) != JS::TraceKind::Object); +} + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::writeBarrierPost(void* cellp, TenuredCell* prior, TenuredCell* next) +{ + AssertValidToSkipBarrier(next); +} + +#ifdef DEBUG +bool +Cell::isAligned() const +{ + if (!isTenured()) + return true; + return asTenured().isAligned(); +} + +bool +TenuredCell::isAligned() const +{ + return Arena::isAligned(address(), arena()->getThingSize()); +} + +#endif + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_Cell_h */ diff --git a/js/src/gc/GCEnum.h b/js/src/gc/GCEnum.h new file mode 100644 index 0000000000..32e66bd230 --- /dev/null +++ b/js/src/gc/GCEnum.h @@ -0,0 +1,88 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal enum definitions. + */ + +#ifndef gc_GCEnum_h +#define gc_GCEnum_h + +#include + +namespace js { +namespace gc { + +// Mark colors to pass to markIfUnmarked. +enum class MarkColor : uint32_t +{ + Black = 0, + Gray +}; + +// The phases of an incremental GC. +#define GCSTATES(D) \ + D(NotActive) \ + D(MarkRoots) \ + D(Mark) \ + D(Sweep) \ + D(Finalize) \ + D(Compact) \ + D(Decommit) +enum class State { +#define MAKE_STATE(name) name, + GCSTATES(MAKE_STATE) +#undef MAKE_STATE +}; + +// Reasons we reset an ongoing incremental GC or perform a non-incremental GC. +#define GC_ABORT_REASONS(D) \ + D(None) \ + D(NonIncrementalRequested) \ + D(AbortRequested) \ + D(Unused1) \ + D(IncrementalDisabled) \ + D(ModeChange) \ + D(MallocBytesTrigger) \ + D(GCBytesTrigger) \ + D(ZoneChange) \ + D(CompartmentRevived) +enum class AbortReason { +#define MAKE_REASON(name) name, + GC_ABORT_REASONS(MAKE_REASON) +#undef MAKE_REASON +}; + +#define JS_FOR_EACH_ZEAL_MODE(D) \ + D(RootsChange, 1) \ + D(Alloc, 2) \ + D(FrameGC, 3) \ + D(VerifierPre, 4) \ + D(FrameVerifierPre, 5) \ + D(GenerationalGC, 7) \ + D(IncrementalRootsThenFinish, 8) \ + D(IncrementalMarkAllThenFinish, 9) \ + D(IncrementalMultipleSlices, 10) \ + D(IncrementalMarkingValidator, 11) \ + D(ElementsBarrier, 12) \ + D(CheckHashTablesOnMinorGC, 13) \ + D(Compact, 14) \ + D(CheckHeapAfterGC, 15) \ + D(CheckNursery, 16) \ + D(IncrementalSweepThenFinish, 17) \ + D(CheckGrayMarking, 18) + +enum class ZealMode { +#define ZEAL_MODE(name, value) name = value, + JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE) +#undef ZEAL_MODE + Limit = 18 +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_GCEnum_h */ diff --git a/js/src/gc/GCHelperState.h b/js/src/gc/GCHelperState.h new file mode 100644 index 0000000000..91eae29c4e --- /dev/null +++ b/js/src/gc/GCHelperState.h @@ -0,0 +1,104 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCHelperState_h +#define gc_GCHelperState_h + +#include "threading/ConditionVariable.h" +#include "threading/ProtectedData.h" + +struct JSRuntime; + +namespace js { +class AutoLockHelperThreadState; + +namespace gc { +class ArenaLists; +} /* namespace gc */ + +/* + * Helper state for use when JS helper threads sweep and allocate GC thing kinds + * that can be swept and allocated off thread. + * + * In non-threadsafe builds, all actual sweeping and allocation is performed + * on the active thread, but GCHelperState encapsulates this from clients as + * much as possible. + */ +class GCHelperState +{ + enum State { + IDLE, + SWEEPING + }; + + // Associated runtime. + JSRuntime* const rt; + + // Condvar for notifying the active thread when work has finished. This is + // associated with the runtime's GC lock --- the worker thread state + // condvars can't be used here due to lock ordering issues. + ConditionVariable done; + + // Activity for the helper to do, protected by the GC lock. + ActiveThreadOrGCTaskData state_; + + // Whether work is being performed on some thread. + GCLockData hasThread; + + void startBackgroundThread(State newState, const AutoLockGC& lock, + const AutoLockHelperThreadState& helperLock); + void waitForBackgroundThread(js::AutoLockGC& lock); + + State state(const AutoLockGC&); + void setState(State state, const AutoLockGC&); + + friend class js::gc::ArenaLists; + + static void freeElementsAndArray(void** array, void** end) { + MOZ_ASSERT(array <= end); + for (void** p = array; p != end; ++p) + js_free(*p); + js_free(array); + } + + void doSweep(AutoLockGC& lock); + + public: + explicit GCHelperState(JSRuntime* rt) + : rt(rt), + done(), + state_(IDLE) + { } + + JSRuntime* runtime() { return rt; } + + void finish(); + + void work(); + + void maybeStartBackgroundSweep(const AutoLockGC& lock, + const AutoLockHelperThreadState& helperLock); + void startBackgroundShrink(const AutoLockGC& lock); + + /* Must be called without the GC lock taken. */ + void waitBackgroundSweepEnd(); + +#ifdef DEBUG + bool onBackgroundThread(); +#endif + + /* + * Outside the GC lock may give true answer when in fact the sweeping has + * been done. + */ + bool isBackgroundSweeping() const { + return state_ == SWEEPING; + } +}; + +} /* namespace js */ + +#endif /* gc_GCHelperState_h */ diff --git a/js/src/gc/GCInternals.h b/js/src/gc/GCInternals.h index c0e980a7c1..f93a0bea8a 100644 --- a/js/src/gc/GCInternals.h +++ b/js/src/gc/GCInternals.h @@ -1,8 +1,13 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * GC-internal definitions. + */ + #ifndef gc_GCInternals_h #define gc_GCInternals_h @@ -11,6 +16,7 @@ #include "jscntxt.h" +#include "gc/RelocationOverlay.h" #include "gc/Zone.h" #include "vm/HelperThreads.h" #include "vm/Runtime.h" @@ -129,6 +135,92 @@ struct TenureCountCache } }; +struct MOZ_RAII AutoAssertNoNurseryAlloc +{ +#ifdef DEBUG + AutoAssertNoNurseryAlloc(); + ~AutoAssertNoNurseryAlloc(); +#else + AutoAssertNoNurseryAlloc() {} +#endif +}; + +/* + * There are a couple of classes here that serve mostly as "tokens" indicating + * that a condition holds. Some functions force the caller to possess such a + * token because they would misbehave if the condition were false, and it is + * far more clear to make the condition visible at the point where it can be + * affected rather than just crashing in an assertion down in the place where + * it is relied upon. + */ + +/* + * A class that serves as a token that the nursery in the current thread's zone + * group is empty. + */ +class MOZ_RAII AutoAssertEmptyNursery +{ + protected: + JSContext* cx; + + mozilla::Maybe noAlloc; + + // Check that the nursery is empty. + void checkCondition(JSContext* cx); + + // For subclasses that need to empty the nursery in their constructors. + AutoAssertEmptyNursery() : cx(nullptr) { + } + + public: + explicit AutoAssertEmptyNursery(JSContext* cx) : cx(nullptr) { + checkCondition(cx); + } + + AutoAssertEmptyNursery(const AutoAssertEmptyNursery& other) : AutoAssertEmptyNursery(other.cx) + { + } +}; + +/* + * Evict the nursery upon construction. Serves as a token indicating that the + * nursery is empty. (See AutoAssertEmptyNursery, above.) + * + * Note that this is very improper subclass of AutoAssertHeapBusy, in that the + * heap is *not* busy within the scope of an AutoEmptyNursery. I will most + * likely fix this by removing AutoAssertHeapBusy, but that is currently + * waiting on jonco's review. + */ +class MOZ_RAII AutoEmptyNursery : public AutoAssertEmptyNursery +{ + public: + explicit AutoEmptyNursery(JSContext* cx); +}; + +extern void +DelayCrossCompartmentGrayMarking(JSObject* src); + +inline bool +IsOOMReason(JS::gcreason::Reason reason) +{ + return reason == JS::gcreason::LAST_DITCH || + reason == JS::gcreason::MEM_PRESSURE; +} + +inline void +RelocationOverlay::forwardTo(Cell* cell) +{ + MOZ_ASSERT(!isForwarded()); + // The location of magic_ is important because it must never be valid to see + // the value Relocated there in a GC thing that has not been moved. + static_assert(offsetof(RelocationOverlay, magic_) == offsetof(JSObject, group_) && + offsetof(RelocationOverlay, magic_) == offsetof(js::Shape, base_) && + offsetof(RelocationOverlay, magic_) == offsetof(JSString, d.u1.flags), + "RelocationOverlay::magic_ is in the wrong location"); + magic_ = Relocated; + newLocation_ = cell; +} + } /* namespace gc */ } /* namespace js */ diff --git a/js/src/gc/GCMarker.h b/js/src/gc/GCMarker.h new file mode 100644 index 0000000000..f3222a6d1e --- /dev/null +++ b/js/src/gc/GCMarker.h @@ -0,0 +1,391 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCMarker_h +#define gc_GCMarker_h + +#include "ds/OrderedHashTable.h" +#include "js/SliceBudget.h" +#include "js/TracingAPI.h" + +namespace JS { +class Symbol; +} + +namespace js { + +class WeakMapBase; + +static const size_t NON_INCREMENTAL_MARK_STACK_BASE_CAPACITY = 4096; +static const size_t INCREMENTAL_MARK_STACK_BASE_CAPACITY = 32768; + +namespace gc { + +struct Cell; + +struct WeakKeyTableHashPolicy { + typedef JS::GCCellPtr Lookup; + static HashNumber hash(const Lookup& v, const mozilla::HashCodeScrambler&) { + return mozilla::HashGeneric(v.asCell()); + } + static bool match(const JS::GCCellPtr& k, const Lookup& l) { return k == l; } + static bool isEmpty(const JS::GCCellPtr& v) { return !v; } + static void makeEmpty(JS::GCCellPtr* vp) { *vp = nullptr; } +}; + +struct WeakMarkable { + WeakMapBase* weakmap; + JS::GCCellPtr key; + + WeakMarkable(WeakMapBase* weakmapArg, JS::GCCellPtr keyArg) + : weakmap(weakmapArg), key(keyArg) {} +}; + +using WeakEntryVector = Vector; + +using WeakKeyTable = OrderedHashMap; + +/* + * When the native stack is low, the GC does not call js::TraceChildren to mark + * the reachable "children" of the thing. Rather the thing is put aside and + * js::TraceChildren is called later with more space on the C stack. + * + * To implement such delayed marking of the children with minimal overhead for + * the normal case of sufficient native stack, the code adds a field per arena. + * The field markingDelay->link links all arenas with delayed things into a + * stack list with the pointer to stack top in GCMarker::unmarkedArenaStackTop. + * GCMarker::delayMarkingChildren adds arenas to the stack as necessary while + * markDelayedChildren pops the arenas from the stack until it empties. + */ +class MarkStack +{ + public: + /* + * We use a common mark stack to mark GC things of different types and use + * the explicit tags to distinguish them when it cannot be deduced from + * the context of push or pop operation. + */ + enum Tag { + ValueArrayTag, + ObjectTag, + GroupTag, + SavedValueArrayTag, + JitCodeTag, + ScriptTag, + TempRopeTag, + + LastTag = TempRopeTag + }; + + static const uintptr_t TagMask = 7; + static_assert(TagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags."); + static_assert(TagMask <= gc::CellAlignMask, "The tag mask must be embeddable in a Cell*."); + + class TaggedPtr + { + uintptr_t bits; + + Cell* ptr() const; + + public: + TaggedPtr(Tag tag, Cell* ptr); + Tag tag() const; + template T* as() const; + + JSObject* asValueArrayObject() const; + JSObject* asSavedValueArrayObject() const; + JSRope* asTempRope() const; + }; + + struct ValueArray + { + ValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end); + + HeapSlot* end; + HeapSlot* start; + TaggedPtr ptr; + }; + + struct SavedValueArray + { + SavedValueArray(JSObject* obj, size_t index, HeapSlot::Kind kind); + + uintptr_t kind; + uintptr_t index; + TaggedPtr ptr; + }; + + explicit MarkStack(size_t maxCapacity = DefaultCapacity); + ~MarkStack(); + + static const size_t DefaultCapacity = SIZE_MAX; + + size_t capacity() { return end_ - stack_; } + + size_t position() const { + auto result = tos_ - stack_; + MOZ_ASSERT(result >= 0); + return size_t(result); + } + + void setStack(TaggedPtr* stack, size_t tosIndex, size_t capacity); + + MOZ_MUST_USE bool init(JSGCMode gcMode); + + void setBaseCapacity(JSGCMode mode); + size_t maxCapacity() const { return maxCapacity_; } + void setMaxCapacity(size_t maxCapacity); + + template + MOZ_MUST_USE bool push(T* ptr); + + MOZ_MUST_USE bool push(JSObject* obj, HeapSlot* start, HeapSlot* end); + MOZ_MUST_USE bool push(const ValueArray& array); + MOZ_MUST_USE bool push(const SavedValueArray& array); + + // GCMarker::eagerlyMarkChildren uses unused marking stack as temporary + // storage to hold rope pointers. + MOZ_MUST_USE bool pushTempRope(JSRope* ptr); + + bool isEmpty() const { + return tos_ == stack_; + } + + Tag peekTag() const; + TaggedPtr popPtr(); + ValueArray popValueArray(); + SavedValueArray popSavedValueArray(); + + void reset(); + + void setGCMode(JSGCMode gcMode); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + + private: + MOZ_MUST_USE bool ensureSpace(size_t count); + + /* Grow the stack, ensuring there is space for at least count elements. */ + MOZ_MUST_USE bool enlarge(size_t count); + + const TaggedPtr& peekPtr() const; + MOZ_MUST_USE bool pushTaggedPtr(Tag tag, Cell* ptr); + + ActiveThreadData stack_; + ActiveThreadData tos_; + ActiveThreadData end_; + + // The capacity we start with and reset() to. + ActiveThreadData baseCapacity_; + ActiveThreadData maxCapacity_; + +#ifdef DEBUG + mutable size_t iteratorCount_; +#endif + + friend class MarkStackIter; +}; + +class MarkStackIter +{ + const MarkStack& stack_; + MarkStack::TaggedPtr* pos_; + + public: + explicit MarkStackIter(const MarkStack& stack); + ~MarkStackIter(); + + bool done() const; + MarkStack::Tag peekTag() const; + MarkStack::TaggedPtr peekPtr() const; + MarkStack::ValueArray peekValueArray() const; + void next(); + void nextPtr(); + void nextArray(); + + // Mutate the current ValueArray to a SavedValueArray. + void saveValueArray(NativeObject* obj, uintptr_t index, HeapSlot::Kind kind); + + private: + size_t position() const; +}; + +} /* namespace gc */ + +class GCMarker : public JSTracer +{ + public: + explicit GCMarker(JSRuntime* rt); + MOZ_MUST_USE bool init(JSGCMode gcMode); + + void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); } + size_t maxCapacity() const { return stack.maxCapacity(); } + + void start(); + void stop(); + void reset(); + + // Mark the given GC thing and traverse its children at some point. + template void traverse(T thing); + + // Calls traverse on target after making additional assertions. + template void traverseEdge(S source, T* target); + template void traverseEdge(S source, const T& target); + + // Notes a weak graph edge for later sweeping. + template void noteWeakEdge(T* edge); + + /* + * Care must be taken changing the mark color from gray to black. The cycle + * collector depends on the invariant that there are no black to gray edges + * in the GC heap. This invariant lets the CC not trace through black + * objects. If this invariant is violated, the cycle collector may free + * objects that are still reachable. + */ + void setMarkColorGray() { + MOZ_ASSERT(isDrained()); + MOZ_ASSERT(color == gc::MarkColor::Black); + color = gc::MarkColor::Gray; + } + void setMarkColorBlack() { + MOZ_ASSERT(isDrained()); + MOZ_ASSERT(color == gc::MarkColor::Gray); + color = gc::MarkColor::Black; + } + gc::MarkColor markColor() const { return color; } + + void enterWeakMarkingMode(); + void leaveWeakMarkingMode(); + void abortLinearWeakMarking() { + leaveWeakMarkingMode(); + linearWeakMarkingDisabled_ = true; + } + + void delayMarkingArena(gc::Arena* arena); + void delayMarkingChildren(const void* thing); + void markDelayedChildren(gc::Arena* arena); + MOZ_MUST_USE bool markDelayedChildren(SliceBudget& budget); + bool hasDelayedChildren() const { + return !!unmarkedArenaStackTop; + } + + bool isDrained() { + return isMarkStackEmpty() && !unmarkedArenaStackTop; + } + + MOZ_MUST_USE bool drainMarkStack(SliceBudget& budget); + + void setGCMode(JSGCMode mode) { stack.setGCMode(mode); } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + +#ifdef DEBUG + + bool shouldCheckCompartments() { return strictCompartmentChecking; } + + JS::Zone* stackContainsCrossZonePointerTo(const gc::Cell* cell) const; + +#endif + + void markEphemeronValues(gc::Cell* markedCell, gc::WeakEntryVector& entry); + + static GCMarker* fromTracer(JSTracer* trc) { + MOZ_ASSERT(trc->isMarkingTracer()); + return static_cast(trc); + } + + private: +#ifdef DEBUG + void checkZone(void* p); +#else + void checkZone(void* p) {} +#endif + + // Push an object onto the stack for later tracing and assert that it has + // already been marked. + inline void repush(JSObject* obj); + + template void markAndTraceChildren(T* thing); + template void markAndPush(T* thing); + template void markAndScan(T* thing); + template void markImplicitEdgesHelper(T oldThing); + template void markImplicitEdges(T* oldThing); + void eagerlyMarkChildren(JSLinearString* str); + void eagerlyMarkChildren(JSRope* rope); + void eagerlyMarkChildren(JSString* str); + void eagerlyMarkChildren(LazyScript *thing); + void eagerlyMarkChildren(Shape* shape); + void eagerlyMarkChildren(Scope* scope); + void lazilyMarkChildren(ObjectGroup* group); + + // We may not have concrete types yet, so this has to be outside the header. + template + void dispatchToTraceChildren(T* thing); + + // Mark the given GC thing, but do not trace its children. Return true + // if the thing became marked. + template + MOZ_MUST_USE bool mark(T* thing); + + template + inline void pushTaggedPtr(T* ptr); + + inline void pushValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end); + + bool isMarkStackEmpty() { + return stack.isEmpty(); + } + + MOZ_MUST_USE bool restoreValueArray(const gc::MarkStack::SavedValueArray& array, + HeapSlot** vpp, HeapSlot** endp); + void saveValueRanges(); + inline void processMarkStackTop(SliceBudget& budget); + + /* The mark stack. Pointers in this stack are "gray" in the GC sense. */ + gc::MarkStack stack; + + /* The color is only applied to objects and functions. */ + ActiveThreadData color; + + /* Pointer to the top of the stack of arenas we are delaying marking on. */ + ActiveThreadData unmarkedArenaStackTop; + + /* + * If the weakKeys table OOMs, disable the linear algorithm and fall back + * to iterating until the next GC. + */ + ActiveThreadData linearWeakMarkingDisabled_; + +#ifdef DEBUG + /* Count of arenas that are currently in the stack. */ + ActiveThreadData markLaterArenas; + + /* Assert that start and stop are called with correct ordering. */ + ActiveThreadData started; + + /* + * If this is true, all marked objects must belong to a compartment being + * GCed. This is used to look for compartment bugs. + */ + ActiveThreadData strictCompartmentChecking; +#endif // DEBUG +}; + +} /* namespace js */ + +// Exported for Tracer.cpp +inline bool ThingIsPermanentAtomOrWellKnownSymbol(js::gc::Cell* thing) { return false; } +bool ThingIsPermanentAtomOrWellKnownSymbol(JSString*); +bool ThingIsPermanentAtomOrWellKnownSymbol(JSFlatString*); +bool ThingIsPermanentAtomOrWellKnownSymbol(JSLinearString*); +bool ThingIsPermanentAtomOrWellKnownSymbol(JSAtom*); +bool ThingIsPermanentAtomOrWellKnownSymbol(js::PropertyName*); +bool ThingIsPermanentAtomOrWellKnownSymbol(JS::Symbol*); + +#endif /* gc_GCMarker_h */ diff --git a/js/src/gc/GCParallelTask.h b/js/src/gc/GCParallelTask.h new file mode 100644 index 0000000000..56987ee266 --- /dev/null +++ b/js/src/gc/GCParallelTask.h @@ -0,0 +1,91 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCParallelTask_h +#define gc_GCParallelTask_h + +#include "threading/ProtectedData.h" + +struct JSRuntime; + +namespace js { + +// A generic task used to dispatch work to the helper thread system. +// Users should derive from GCParallelTask add what data they need and +// override |run|. +class GCParallelTask +{ + JSRuntime* const runtime_; + + // The state of the parallel computation. + enum TaskState { + NotStarted, + Dispatched, + Finished, + }; + UnprotectedData state; + + // Amount of time this task took to execute. + ActiveThreadOrGCTaskData duration_; + + explicit GCParallelTask(const GCParallelTask&) = delete; + + protected: + // A flag to signal a request for early completion of the off-thread task. + mozilla::Atomic cancel_; + + virtual void run() = 0; + + public: + explicit GCParallelTask(JSRuntime* runtime) : runtime_(runtime), state(NotStarted), duration_(nullptr) {} + GCParallelTask(GCParallelTask&& other) + : runtime_(other.runtime_), + state(other.state), + duration_(nullptr), + cancel_(false) + {} + + // Derived classes must override this to ensure that join() gets called + // before members get destructed. + virtual ~GCParallelTask(); + + JSRuntime* runtime() { return runtime_; } + + // Time spent in the most recent invocation of this task. + mozilla::TimeDuration duration() const { return duration_; } + + // The simple interface to a parallel task works exactly like pthreads. + bool start(); + void join(); + + // If multiple tasks are to be started or joined at once, it is more + // efficient to take the helper thread lock once and use these methods. + bool startWithLockHeld(AutoLockHelperThreadState& locked); + void joinWithLockHeld(AutoLockHelperThreadState& locked); + + // Instead of dispatching to a helper, run the task on the current thread. + void runFromActiveCooperatingThread(JSRuntime* rt); + + // Dispatch a cancelation request. + enum CancelMode { CancelNoWait, CancelAndWait}; + void cancel(CancelMode mode = CancelNoWait) { + cancel_ = true; + if (mode == CancelAndWait) + join(); + } + + // Check if a task is actively running. + bool isRunningWithLockHeld(const AutoLockHelperThreadState& locked) const; + bool isRunning() const; + + // This should be friended to HelperThread, but cannot be because it + // would introduce several circular dependencies. + public: + void runFromHelperThread(AutoLockHelperThreadState& locked); +}; + +} /* namespace js */ +#endif /* gc_GCParallelTask_h */ diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index 473efb94fb..9e1b43818a 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -10,14 +10,16 @@ #include "mozilla/Atomics.h" #include "mozilla/EnumSet.h" -#include "jsfriendapi.h" -#include "jsgc.h" +#include "jsatom.h" -#include "gc/Heap.h" +#include "gc/ArenaList.h" +#include "gc/AtomMarking.h" +#include "gc/GCHelperState.h" +#include "gc/GCMarker.h" +#include "gc/GCParallelTask.h" #include "gc/Nursery.h" #include "gc/Statistics.h" #include "gc/StoreBuffer.h" -#include "gc/Tracer.h" #include "js/GCAnnotations.h" namespace js { @@ -38,6 +40,7 @@ class AutoTraceSession; class MarkingValidator; class AutoTraceSession; struct MovingTracer; +enum class ShouldCheckThresholds; class SweepGroupsIter; class WeakCacheSweepIterator; @@ -703,6 +706,34 @@ class MemoryCounter void reset(); }; +// A singly linked list of zones. +class ZoneList +{ + static Zone * const End; + + Zone* head; + Zone* tail; + + public: + ZoneList(); + ~ZoneList(); + + bool isEmpty() const; + Zone* front() const; + + void append(Zone* zone); + void transferFrom(ZoneList& other); + void removeFront(); + void clear(); + + private: + explicit ZoneList(Zone* singleZone); + void check() const; + + ZoneList(const ZoneList& other) = delete; + ZoneList& operator=(const ZoneList& other) = delete; +}; + class GCRuntime { public: @@ -1182,11 +1213,12 @@ class GCRuntime MemProfiler mMemProfiler; private: - // When empty, chunks reside in the emptyChunks pool and are re-used as - // needed or eventually expired if not re-used. The emptyChunks pool gets - // refilled from the background allocation task heuristically so that empty - // chunks should always available for immediate allocation without syscalls. - ChunkPool emptyChunks_; + // When chunks are empty, they reside in the emptyChunks pool and are + // re-used as needed or eventually expired if not re-used. The emptyChunks + // pool gets refilled from the background allocation task heuristically so + // that empty chunks should always be available for immediate allocation + // without syscalls. + GCLockData emptyChunks_; // Chunks which have had some, but not all, of their arenas allocated live // in the available chunk lists. When all available arenas in a chunk have @@ -1477,7 +1509,8 @@ class GCRuntime BackgroundAllocTask allocTask; BackgroundDecommitTask decommitTask; - GCHelperState helperState; + + js::GCHelperState helperState; /* * During incremental sweeping, this field temporarily holds the arenas of diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h index 8aa508c978..84eda138e8 100644 --- a/js/src/gc/Heap.h +++ b/js/src/gc/Heap.h @@ -11,8 +11,6 @@ #include "mozilla/Atomics.h" #include "mozilla/Attributes.h" #include "mozilla/DebugOnly.h" -#include "mozilla/EnumeratedArray.h" -#include "mozilla/EnumeratedRange.h" #include "mozilla/PodOperations.h" #include @@ -24,12 +22,12 @@ #include "jsutil.h" #include "ds/BitArray.h" +#include "gc/AllocKind.h" +#include "gc/GCEnum.h" #include "gc/Memory.h" -#include "js/GCAPI.h" #include "js/HeapAPI.h" #include "js/RootingAPI.h" #include "js/TracingAPI.h" -#include "js/TraceKind.h" #include "vm/Printer.h" @@ -47,31 +45,13 @@ class AutoLockGC; class AutoLockGCBgAlloc; class FreeOp; -extern bool -RuntimeFromMainThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone); - -#ifdef DEBUG - -// Barriers can't be triggered during backend Ion compilation, which may run on -// a helper thread. -extern bool -CurrentThreadIsIonCompiling(); -#endif - -// The return value indicates if anything was unmarked. -extern bool -UnmarkGrayCellRecursively(gc::Cell* cell, JS::TraceKind kind); - - -extern void -TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name); - namespace gc { class Arena; class ArenaCellSet; class ArenaList; class SortedArenaList; +class TenuredCell; struct Chunk; /* @@ -79,297 +59,21 @@ struct Chunk; * estimated lifetime or lifetime requirements of objects allocated from that * site. */ -enum InitialHeap { +enum InitialHeap : uint8_t { DefaultHeap, TenuredHeap }; -/* The GC allocation kinds. */ -// FIXME: uint8_t would make more sense for the underlying type, but causes -// miscompilations in GCC (fixed in 4.8.5 and 4.9.3). See also bug 1143966. -enum class AllocKind { - FIRST, - OBJECT_FIRST = FIRST, - FUNCTION = FIRST, - FUNCTION_EXTENDED, - OBJECT0, - OBJECT0_BACKGROUND, - OBJECT2, - OBJECT2_BACKGROUND, - OBJECT4, - OBJECT4_BACKGROUND, - OBJECT8, - OBJECT8_BACKGROUND, - OBJECT12, - OBJECT12_BACKGROUND, - OBJECT16, - OBJECT16_BACKGROUND, - OBJECT_LIMIT, - OBJECT_LAST = OBJECT_LIMIT - 1, - SCRIPT, - LAZY_SCRIPT, - SHAPE, - ACCESSOR_SHAPE, - BASE_SHAPE, - OBJECT_GROUP, - FAT_INLINE_STRING, - STRING, - EXTERNAL_STRING, - FAT_INLINE_ATOM, - ATOM, - SYMBOL, - BIGINT, - JITCODE, - SCOPE, - REGEXP_SHARED, - LIMIT, - LAST = LIMIT - 1 -}; - -// Macro to enumerate the different allocation kinds supplying information about -// the trace kind, C++ type and allocation size. -#define FOR_EACH_OBJECT_ALLOCKIND(D) \ - /* AllocKind TraceKind TypeName SizedType */ \ - D(FUNCTION, Object, JSObject, JSFunction) \ - D(FUNCTION_EXTENDED, Object, JSObject, FunctionExtended) \ - D(OBJECT0, Object, JSObject, JSObject_Slots0) \ - D(OBJECT0_BACKGROUND, Object, JSObject, JSObject_Slots0) \ - D(OBJECT2, Object, JSObject, JSObject_Slots2) \ - D(OBJECT2_BACKGROUND, Object, JSObject, JSObject_Slots2) \ - D(OBJECT4, Object, JSObject, JSObject_Slots4) \ - D(OBJECT4_BACKGROUND, Object, JSObject, JSObject_Slots4) \ - D(OBJECT8, Object, JSObject, JSObject_Slots8) \ - D(OBJECT8_BACKGROUND, Object, JSObject, JSObject_Slots8) \ - D(OBJECT12, Object, JSObject, JSObject_Slots12) \ - D(OBJECT12_BACKGROUND, Object, JSObject, JSObject_Slots12) \ - D(OBJECT16, Object, JSObject, JSObject_Slots16) \ - D(OBJECT16_BACKGROUND, Object, JSObject, JSObject_Slots16) - -#define FOR_EACH_NONOBJECT_ALLOCKIND(D) \ - /* AllocKind TraceKind TypeName SizedType */ \ - D(SCRIPT, Script, JSScript, JSScript) \ - D(LAZY_SCRIPT, LazyScript, js::LazyScript, js::LazyScript) \ - D(SHAPE, Shape, js::Shape, js::Shape) \ - D(ACCESSOR_SHAPE, Shape, js::AccessorShape, js::AccessorShape) \ - D(BASE_SHAPE, BaseShape, js::BaseShape, js::BaseShape) \ - D(OBJECT_GROUP, ObjectGroup, js::ObjectGroup, js::ObjectGroup) \ - D(FAT_INLINE_STRING, String, JSFatInlineString, JSFatInlineString) \ - D(STRING, String, JSString, JSString) \ - D(EXTERNAL_STRING, String, JSExternalString, JSExternalString) \ - D(FAT_INLINE_ATOM, String, js::FatInlineAtom, js::FatInlineAtom) \ - D(ATOM, String, js::NormalAtom, js::NormalAtom) \ - D(SYMBOL, Symbol, JS::Symbol, JS::Symbol) \ - D(BIGINT, BigInt, JS::BigInt, JS::BigInt) \ - D(JITCODE, JitCode, js::jit::JitCode, js::jit::JitCode) \ - D(SCOPE, Scope, js::Scope, js::Scope) \ - D(REGEXP_SHARED, RegExpShared, js::RegExpShared, js::RegExpShared) - -#define FOR_EACH_ALLOCKIND(D) \ - FOR_EACH_OBJECT_ALLOCKIND(D) \ - FOR_EACH_NONOBJECT_ALLOCKIND(D) - -static_assert(int(AllocKind::FIRST) == 0, "Various places depend on AllocKind starting at 0, " - "please audit them carefully!"); -static_assert(int(AllocKind::OBJECT_FIRST) == 0, "Various places depend on AllocKind::OBJECT_FIRST " - "being 0, please audit them carefully!"); - -inline bool -IsAllocKind(AllocKind kind) -{ - return kind >= AllocKind::FIRST && kind <= AllocKind::LIMIT; -} - -inline bool -IsValidAllocKind(AllocKind kind) -{ - return kind >= AllocKind::FIRST && kind <= AllocKind::LAST; -} - -inline bool -IsObjectAllocKind(AllocKind kind) -{ - return kind >= AllocKind::OBJECT_FIRST && kind <= AllocKind::OBJECT_LAST; -} - -inline bool -IsShapeAllocKind(AllocKind kind) -{ - return kind == AllocKind::SHAPE || kind == AllocKind::ACCESSOR_SHAPE; -} - -// Returns a sequence for use in a range-based for loop, -// to iterate over all alloc kinds. -inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT)) -AllAllocKinds() -{ - return mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT); -} - -// Returns a sequence for use in a range-based for loop, -// to iterate over all object alloc kinds. -inline decltype(mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT)) -ObjectAllocKinds() -{ - return mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT); -} - -// Returns a sequence for use in a range-based for loop, -// to iterate over alloc kinds from |first| to |limit|, exclusive. -inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT)) -SomeAllocKinds(AllocKind first = AllocKind::FIRST, AllocKind limit = AllocKind::LIMIT) -{ - MOZ_ASSERT(IsAllocKind(first), "|first| is not a valid AllocKind!"); - MOZ_ASSERT(IsAllocKind(limit), "|limit| is not a valid AllocKind!"); - return mozilla::MakeEnumeratedRange(first, limit); -} - -// AllAllocKindArray gives an enumerated array of ValueTypes, -// with each index corresponding to a particular alloc kind. -template using AllAllocKindArray = - mozilla::EnumeratedArray; - -// ObjectAllocKindArray gives an enumerated array of ValueTypes, -// with each index corresponding to a particular object alloc kind. -template using ObjectAllocKindArray = - mozilla::EnumeratedArray; - -static inline JS::TraceKind -MapAllocToTraceKind(AllocKind kind) -{ - static const JS::TraceKind map[] = { -#define EXPAND_ELEMENT(allocKind, traceKind, type, sizedType) \ - JS::TraceKind::traceKind, -FOR_EACH_ALLOCKIND(EXPAND_ELEMENT) -#undef EXPAND_ELEMENT - }; - - static_assert(MOZ_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT), - "AllocKind-to-TraceKind mapping must be in sync"); - return map[size_t(kind)]; -} +/* Cells are aligned to CellAlignShift, so the largest tagged null pointer is: */ +const uintptr_t LargestTaggedNullCellPointer = (1 << CellAlignShift) - 1; /* - * This must be an upper bound, but we do not need the least upper bound, so - * we just exclude non-background objects. + * The minimum cell size ends up as twice the cell alignment because the mark + * bitmap contains one bit per CellBytesPerMarkBit bytes (which is equal to + * CellAlignBytes) and we need two mark bits per cell. */ -static const size_t MAX_BACKGROUND_FINALIZE_KINDS = - size_t(AllocKind::LIMIT) - size_t(AllocKind::OBJECT_LIMIT) / 2; - -/* Mark colors to pass to markIfUnmarked. */ -enum class MarkColor : uint32_t -{ - Black = 0, - Gray -}; - -class TenuredCell; - -// A GC cell is the base class for all GC things. -struct Cell -{ - public: - MOZ_ALWAYS_INLINE bool isTenured() const { return !IsInsideNursery(this); } - MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const; - MOZ_ALWAYS_INLINE TenuredCell& asTenured(); - - MOZ_ALWAYS_INLINE bool isMarkedAny() const; - MOZ_ALWAYS_INLINE bool isMarkedBlack() const; - MOZ_ALWAYS_INLINE bool isMarkedGray() const; - - inline JSRuntime* runtimeFromActiveCooperatingThread() const; - - // Note: Unrestricted access to the runtime of a GC thing from an arbitrary - // thread can easily lead to races. Use this method very carefully. - inline JSRuntime* runtimeFromAnyThread() const; - inline JS::shadow::Runtime* shadowRuntimeFromAnyThread() const; - - // May be overridden by GC thing kinds that have a compartment pointer. - inline JSCompartment* maybeCompartment() const { return nullptr; } - - inline StoreBuffer* storeBuffer() const; - - inline JS::TraceKind getTraceKind() const; - - static MOZ_ALWAYS_INLINE bool needWriteBarrierPre(JS::Zone* zone); - - template - inline bool is() const { - return getTraceKind() == JS::MapTypeToTraceKind::kind; - } - - template - inline T* as() { - MOZ_ASSERT(this->is()); - return static_cast(this); - } - - template - inline const T* as() const { - MOZ_ASSERT(this->is()); - return static_cast(this); - } - -#ifdef DEBUG - inline bool isAligned() const; - void dump(FILE* fp) const; - void dump() const; -#endif - - protected: - inline uintptr_t address() const; - inline Chunk* chunk() const; -} JS_HAZ_GC_THING; - -// A GC TenuredCell gets behaviors that are valid for things in the Tenured -// heap, such as access to the arena and mark bits. -class TenuredCell : public Cell -{ - public: - // Construct a TenuredCell from a void*, making various sanity assertions. - static MOZ_ALWAYS_INLINE TenuredCell* fromPointer(void* ptr); - static MOZ_ALWAYS_INLINE const TenuredCell* fromPointer(const void* ptr); - - // Mark bit management. - MOZ_ALWAYS_INLINE bool isMarkedAny() const; - MOZ_ALWAYS_INLINE bool isMarkedBlack() const; - MOZ_ALWAYS_INLINE bool isMarkedGray() const; - - // The return value indicates if the cell went from unmarked to marked. - MOZ_ALWAYS_INLINE bool markIfUnmarked(MarkColor color = MarkColor::Black) const; - MOZ_ALWAYS_INLINE void markBlack() const; - MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src); - - // Access to the arena. - inline Arena* arena() const; - inline AllocKind getAllocKind() const; - inline JS::TraceKind getTraceKind() const; - inline JS::Zone* zone() const; - inline JS::Zone* zoneFromAnyThread() const; - inline bool isInsideZone(JS::Zone* zone) const; - - MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZone() const { - return JS::shadow::Zone::asShadowZone(zone()); - } - MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZoneFromAnyThread() const { - return JS::shadow::Zone::asShadowZone(zoneFromAnyThread()); - } - - static MOZ_ALWAYS_INLINE void readBarrier(TenuredCell* thing); - static MOZ_ALWAYS_INLINE void writeBarrierPre(TenuredCell* thing); - - static MOZ_ALWAYS_INLINE void writeBarrierPost(void* cellp, TenuredCell* prior, - TenuredCell* next); - - // Default implementation for kinds that don't require fixup. - void fixupAfterMovingGC() {} - -#ifdef DEBUG - inline bool isAligned() const; -#endif -}; - -/* Cells are aligned to CellShift, so the largest tagged null pointer is: */ -const uintptr_t LargestTaggedNullCellPointer = (1 << CellShift) - 1; +const size_t MarkBitsPerCell = 2; +const size_t MinCellSize = CellBytesPerMarkBit * MarkBitsPerCell; constexpr size_t DivideAndRoundUp(size_t numerator, size_t divisor) { @@ -380,12 +84,11 @@ const size_t ArenaCellCount = ArenaSize / CellSize; static_assert(ArenaSize % CellSize == 0, "Arena size must be a multiple of cell size"); /* - * The mark bitmap has one bit per each GC cell. For multi-cell GC things this - * wastes space but allows to avoid expensive devisions by thing's size when - * accessing the bitmap. In addition this allows to use some bits for colored - * marking during the cycle GC. + * The mark bitmap has one bit per each possible cell start position. This + * wastes some space for larger GC things but allows us to avoid division by the + * cell's size when accessing the bitmap. */ -const size_t ArenaBitmapBits = ArenaCellCount; +const size_t ArenaBitmapBits = ArenaSize / CellBytesPerMarkBit; const size_t ArenaBitmapBytes = DivideAndRoundUp(ArenaBitmapBits, 8); const size_t ArenaBitmapWords = DivideAndRoundUp(ArenaBitmapBits, JS_BITS_PER_WORD); @@ -478,7 +181,6 @@ class FreeSpan } checkSpan(arena); JS_EXTRA_POISON(reinterpret_cast(thing), JS_ALLOCATED_TENURED_PATTERN, thingSize); - MemProfiler::SampleTenured(reinterpret_cast(thing), thingSize); return reinterpret_cast(thing); } @@ -899,15 +601,6 @@ static_assert(ArenasPerChunk == 62, "Do not accidentally change our heap's densi static_assert(ArenasPerChunk == 252, "Do not accidentally change our heap's density."); #endif -static inline void -AssertValidColorBit(const TenuredCell* thing, ColorBit colorBit) -{ -#ifdef DEBUG - Arena* arena = thing->arena(); - MOZ_ASSERT(unsigned(colorBit) < arena->getThingSize() / CellBytesPerMarkBit); -#endif -} - /* A chunk bitmap contains enough mark bits for all the cells in a chunk. */ struct ChunkBitmap { @@ -919,11 +612,11 @@ struct ChunkBitmap MOZ_ALWAYS_INLINE void getMarkWordAndMask(const TenuredCell* cell, ColorBit colorBit, uintptr_t** wordp, uintptr_t* maskp) { + MOZ_ASSERT(size_t(colorBit) < MarkBitsPerCell); detail::GetGCThingMarkWordAndMask(uintptr_t(cell), colorBit, wordp, maskp); } MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool markBit(const TenuredCell* cell, ColorBit colorBit) { - AssertValidColorBit(cell, colorBit); uintptr_t* word, mask; getMarkWordAndMask(cell, colorBit, &word, &mask); return *word & mask; @@ -1172,94 +865,6 @@ Arena::chunk() const return Chunk::fromAddress(address()); } -MOZ_ALWAYS_INLINE const TenuredCell& -Cell::asTenured() const -{ - MOZ_ASSERT(isTenured()); - return *static_cast(this); -} - -MOZ_ALWAYS_INLINE TenuredCell& -Cell::asTenured() -{ - MOZ_ASSERT(isTenured()); - return *static_cast(this); -} - -MOZ_ALWAYS_INLINE bool -Cell::isMarkedAny() const -{ - return !isTenured() || asTenured().isMarkedAny(); -} - -MOZ_ALWAYS_INLINE bool -Cell::isMarkedBlack() const -{ - return !isTenured() || asTenured().isMarkedBlack(); -} - -MOZ_ALWAYS_INLINE bool -Cell::isMarkedGray() const -{ - return isTenured() && asTenured().isMarkedGray(); -} - -inline JSRuntime* -Cell::runtimeFromMainThread() const -{ - JSRuntime* rt = chunk()->trailer.runtime; - MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); - return rt; -} - -inline JS::shadow::Runtime* -Cell::shadowRuntimeFromMainThread() const -{ - return reinterpret_cast(runtimeFromMainThread()); -} - -inline JSRuntime* -Cell::runtimeFromAnyThread() const -{ - return chunk()->trailer.runtime; -} - -inline JS::shadow::Runtime* -Cell::shadowRuntimeFromAnyThread() const -{ - return reinterpret_cast(runtimeFromAnyThread()); -} - -inline uintptr_t -Cell::address() const -{ - uintptr_t addr = uintptr_t(this); - MOZ_ASSERT(addr % CellSize == 0); - MOZ_ASSERT(Chunk::withinValidRange(addr)); - return addr; -} - -Chunk* -Cell::chunk() const -{ - uintptr_t addr = uintptr_t(this); - MOZ_ASSERT(addr % CellSize == 0); - addr &= ~ChunkMask; - return reinterpret_cast(addr); -} - -inline StoreBuffer* -Cell::storeBuffer() const -{ - return chunk()->trailer.storeBuffer; -} - -inline JS::TraceKind -Cell::getTraceKind() const -{ - return isTenured() ? asTenured().getTraceKind() : JS::TraceKind::Object; -} - inline bool InFreeList(Arena* arena, void* thing) { @@ -1268,187 +873,6 @@ InFreeList(Arena* arena, void* thing) return arena->inFreeList(addr); } -/* static */ MOZ_ALWAYS_INLINE bool -Cell::needWriteBarrierPre(JS::Zone* zone) { - return JS::shadow::Zone::asShadowZone(zone)->needsIncrementalBarrier(); -} - -/* static */ MOZ_ALWAYS_INLINE TenuredCell* -TenuredCell::fromPointer(void* ptr) -{ - MOZ_ASSERT(static_cast(ptr)->isTenured()); - return static_cast(ptr); -} - -/* static */ MOZ_ALWAYS_INLINE const TenuredCell* -TenuredCell::fromPointer(const void* ptr) -{ - MOZ_ASSERT(static_cast(ptr)->isTenured()); - return static_cast(ptr); -} - -bool -TenuredCell::isMarkedAny() const -{ - MOZ_ASSERT(arena()->allocated()); - return chunk()->bitmap.isMarkedAny(this); -} - -bool -TenuredCell::isMarkedBlack() const -{ - MOZ_ASSERT(arena()->allocated()); - return chunk()->bitmap.isMarkedBlack(this); -} - -bool -TenuredCell::isMarkedGray() const -{ - MOZ_ASSERT(arena()->allocated()); - return chunk()->bitmap.isMarkedGray(this); -} - -bool -TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const -{ - return chunk()->bitmap.markIfUnmarked(this, color); -} - -void -TenuredCell::unmark(uint32_t color) const -{ - MOZ_ASSERT(color != BLACK); - AssertValidColor(this, color); - chunk()->bitmap.unmark(this, color); -} - -void -TenuredCell::copyMarkBitsFrom(const TenuredCell* src) -{ - ChunkBitmap& bitmap = chunk()->bitmap; - bitmap.copyMarkBit(this, src, BLACK); - bitmap.copyMarkBit(this, src, GRAY); -} - -inline Arena* -TenuredCell::arena() const -{ - MOZ_ASSERT(isTenured()); - uintptr_t addr = address(); - addr &= ~ArenaMask; - return reinterpret_cast(addr); -} - -AllocKind -TenuredCell::getAllocKind() const -{ - return arena()->getAllocKind(); -} - -JS::TraceKind -TenuredCell::getTraceKind() const -{ - return MapAllocToTraceKind(getAllocKind()); -} - -JS::Zone* -TenuredCell::zone() const -{ - JS::Zone* zone = arena()->zone; - MOZ_ASSERT(CurrentThreadCanAccessZone(zone)); - return zone; -} - -JS::Zone* -TenuredCell::zoneFromAnyThread() const -{ - return arena()->zone; -} - -bool -TenuredCell::isInsideZone(JS::Zone* zone) const -{ - return zone == arena()->zone; -} - -/* static */ MOZ_ALWAYS_INLINE void -TenuredCell::readBarrier(TenuredCell* thing) -{ - MOZ_ASSERT(!CurrentThreadIsIonCompiling()); - MOZ_ASSERT(thing); - - // It would be good if barriers were never triggered during collection, but - // at the moment this can happen e.g. when rekeying tables containing - // read-barriered GC things after a moving GC. - // - // TODO: Fix this and assert we're not collecting if we're on the main - // thread. - - JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); - if (shadowZone->needsIncrementalBarrier()) { - // Barriers are only enabled on the main thread and are disabled while collecting. - MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)); - Cell* tmp = thing; - TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "read barrier"); - MOZ_ASSERT(tmp == thing); - } - - if (thing->isMarkedGray()) { - // There shouldn't be anything marked grey unless we're on the active thread. - MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())); - if (!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone)) - UnmarkGrayCellRecursively(thing, thing->getTraceKind()); - } -} - -void -AssertSafeToSkipBarrier(TenuredCell* thing); - -/* static */ MOZ_ALWAYS_INLINE void -TenuredCell::writeBarrierPre(TenuredCell* thing) -{ - MOZ_ASSERT(!CurrentThreadIsIonCompiling()); - if (!thing) - return; - - JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); - if (shadowZone->needsIncrementalBarrier()) { - MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)); - Cell* tmp = thing; - TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "pre barrier"); - MOZ_ASSERT(tmp == thing); - } -} - -static MOZ_ALWAYS_INLINE void -AssertValidToSkipBarrier(TenuredCell* thing) -{ - MOZ_ASSERT(!IsInsideNursery(thing)); - MOZ_ASSERT_IF(thing, MapAllocToTraceKind(thing->getAllocKind()) != JS::TraceKind::Object); -} - -/* static */ MOZ_ALWAYS_INLINE void -TenuredCell::writeBarrierPost(void* cellp, TenuredCell* prior, TenuredCell* next) -{ - AssertValidToSkipBarrier(next); -} - -#ifdef DEBUG -bool -Cell::isAligned() const -{ - if (!isTenured()) - return true; - return asTenured().isAligned(); -} - -bool -TenuredCell::isAligned() const -{ - return Arena::isAligned(address(), arena()->getThingSize()); -} -#endif - static const int32_t ChunkLocationOffsetFromLastByte = int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask); diff --git a/js/src/gc/Iteration-inl.h b/js/src/gc/Iteration-inl.h new file mode 100644 index 0000000000..7f52acecb1 --- /dev/null +++ b/js/src/gc/Iteration-inl.h @@ -0,0 +1,121 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal iterators for various data structures. + */ + +#ifndef gc_Iteration_inl_h +#define gc_Iteration_inl_h + +#include "jsgcinlines.h" + +namespace js { +namespace gc { + +class ArenaCellIterUnderGC : public ArenaCellIterImpl +{ + public: + explicit ArenaCellIterUnderGC(Arena* arena) + : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier) + { + MOZ_ASSERT(CurrentThreadIsPerformingGC()); + } +}; + +class ArenaCellIterUnderFinalize : public ArenaCellIterImpl +{ + public: + explicit ArenaCellIterUnderFinalize(Arena* arena) + : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier) + { + MOZ_ASSERT(CurrentThreadIsGCSweeping()); + } +}; + +class ArenaCellIterUnbarriered : public ArenaCellIterImpl +{ + public: + explicit ArenaCellIterUnbarriered(Arena* arena) + : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier) + {} +}; + +class GrayObjectIter : public ZoneCellIter { + public: + explicit GrayObjectIter(JS::Zone* zone, AllocKind kind) : ZoneCellIter() { + initForTenuredIteration(zone, kind); + } + + JSObject* get() const { return ZoneCellIter::get(); } + operator JSObject*() const { return get(); } + JSObject* operator ->() const { return get(); } +}; + +class GCZonesIter +{ + private: + ZonesIter zone; + + public: + explicit GCZonesIter(JSRuntime* rt, ZoneSelector selector = WithAtoms) : zone(rt, selector) { + MOZ_ASSERT(JS::CurrentThreadIsHeapBusy()); + if (!zone->isCollectingFromAnyThread()) + next(); + } + + bool done() const { return zone.done(); } + + void next() { + MOZ_ASSERT(!done()); + do { + zone.next(); + } while (!zone.done() && !zone->isCollectingFromAnyThread()); + } + + JS::Zone* get() const { + MOZ_ASSERT(!done()); + return zone; + } + + operator JS::Zone*() const { return get(); } + JS::Zone* operator->() const { return get(); } +}; + +typedef CompartmentsIterT GCCompartmentsIter; + +/* Iterates over all zones in the current sweep group. */ +class SweepGroupZonesIter { + JS::Zone* current; + + public: + explicit SweepGroupZonesIter(JSRuntime* rt) { + MOZ_ASSERT(CurrentThreadIsPerformingGC()); + current = rt->gc.getCurrentSweepGroup(); + } + + bool done() const { return !current; } + + void next() { + MOZ_ASSERT(!done()); + current = current->nextNodeInGroup(); + } + + JS::Zone* get() const { + MOZ_ASSERT(!done()); + return current; + } + + operator JS::Zone*() const { return get(); } + JS::Zone* operator->() const { return get(); } +}; + +typedef CompartmentsIterT SweepGroupCompartmentsIter; + +} // namespace gc +} // namespace js + +#endif // gc_Iteration_h diff --git a/js/src/gc/Iteration.cpp b/js/src/gc/Iteration.cpp index c011e184b0..10fbf2ddde 100644 --- a/js/src/gc/Iteration.cpp +++ b/js/src/gc/Iteration.cpp @@ -4,10 +4,11 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "gc/Iteration-inl.h" + #include "mozilla/DebugOnly.h" #include "jscompartment.h" -#include "jsgc.h" #include "gc/GCInternals.h" #include "js/HashTable.h" diff --git a/js/src/gc/Marking-inl.h b/js/src/gc/Marking-inl.h new file mode 100644 index 0000000000..d1148d2638 --- /dev/null +++ b/js/src/gc/Marking-inl.h @@ -0,0 +1,128 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Marking_inl_h +#define gc_Marking_inl_h + +#include "gc/Marking.h" + +#include "gc/RelocationOverlay.h" + +namespace js { +namespace gc { + +template +struct MightBeForwarded +{ + static_assert(mozilla::IsBaseOf::value, + "T must derive from Cell"); + static_assert(!mozilla::IsSame::value && !mozilla::IsSame::value, + "T must not be Cell or TenuredCell"); + + static const bool value = mozilla::IsBaseOf::value || + mozilla::IsBaseOf::value || + mozilla::IsBaseOf::value || + mozilla::IsBaseOf::value || + mozilla::IsBaseOf::value || + mozilla::IsBaseOf::value || + mozilla::IsBaseOf::value || + mozilla::IsBaseOf::value; +}; + +template +inline bool +IsForwarded(T* t) +{ + RelocationOverlay* overlay = RelocationOverlay::fromCell(t); + if (!MightBeForwarded::value) { + MOZ_ASSERT(!overlay->isForwarded()); + return false; + } + + return overlay->isForwarded(); +} + +struct IsForwardedFunctor : public BoolDefaultAdaptor { + template bool operator()(T* t) { return IsForwarded(t); } +}; + +inline bool +IsForwarded(const JS::Value& value) +{ + return DispatchTyped(IsForwardedFunctor(), value); +} + +template +inline T* +Forwarded(T* t) +{ + RelocationOverlay* overlay = RelocationOverlay::fromCell(t); + MOZ_ASSERT(overlay->isForwarded()); + return reinterpret_cast(overlay->forwardingAddress()); +} + +struct ForwardedFunctor : public IdentityDefaultAdaptor { + template inline Value operator()(T* t) { + return js::gc::RewrapTaggedPointer::wrap(Forwarded(t)); + } +}; + +inline Value +Forwarded(const JS::Value& value) +{ + return DispatchTyped(ForwardedFunctor(), value); +} + +template +inline T +MaybeForwarded(T t) +{ + if (IsForwarded(t)) + t = Forwarded(t); + MakeAccessibleAfterMovingGC(t); + return t; +} + +#ifdef JSGC_HASH_TABLE_CHECKS + +template +inline bool +IsGCThingValidAfterMovingGC(T* t) +{ + return !IsInsideNursery(t) && !RelocationOverlay::isCellForwarded(t); +} + +template +inline void +CheckGCThingAfterMovingGC(T* t) +{ + if (t) + MOZ_RELEASE_ASSERT(IsGCThingValidAfterMovingGC(t)); +} + +template +inline void +CheckGCThingAfterMovingGC(const ReadBarriered& t) +{ + CheckGCThingAfterMovingGC(t.unbarrieredGet()); +} + +struct CheckValueAfterMovingGCFunctor : public VoidDefaultAdaptor { + template void operator()(T* t) { CheckGCThingAfterMovingGC(t); } +}; + +inline void +CheckValueAfterMovingGC(const JS::Value& value) +{ + DispatchTyped(CheckValueAfterMovingGCFunctor(), value); +} + +#endif // JSGC_HASH_TABLE_CHECKS + +} /* namespace gc */ +} /* namespace js */ + +#endif // gc_Marking_inl_h diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp index cd82ff9a8e..cb857c4097 100644 --- a/js/src/gc/Marking.cpp +++ b/js/src/gc/Marking.cpp @@ -4,7 +4,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "gc/Marking.h" +#include "gc/Marking-inl.h" #include "mozilla/DebugOnly.h" #include "mozilla/IntegerRange.h" @@ -12,7 +12,6 @@ #include "mozilla/ScopeExit.h" #include "mozilla/TypeTraits.h" -#include "jsgc.h" #include "jsprf.h" #include "builtin/ModuleObject.h" @@ -36,8 +35,8 @@ #include "jscompartmentinlines.h" #include "jsgcinlines.h" -#include "jsobjinlines.h" +#include "gc/Iteration-inl.h" #include "gc/Nursery-inl.h" #include "vm/NativeObject-inl.h" #include "vm/String-inl.h" @@ -975,15 +974,25 @@ js::GCMarker::traverseEdge(S source, const T& thing) DispatchTyped(TraverseEdgeFunctor(), thing, this, source); } +namespace { + +template struct ParticipatesInCC {}; +#define EXPAND_PARTICIPATES_IN_CC(_, type, addToCCKind) \ + template <> struct ParticipatesInCC { static const bool value = addToCCKind; }; +JS_FOR_EACH_TRACEKIND(EXPAND_PARTICIPATES_IN_CC) +#undef EXPAND_PARTICIPATES_IN_CC + +} // namespace + template bool js::GCMarker::mark(T* thing) { AssertShouldMarkInZone(thing); - MOZ_ASSERT(!IsInsideNursery(gc::TenuredCell::fromPointer(thing))); - return gc::ParticipatesInCC::value - ? gc::TenuredCell::fromPointer(thing)->markIfUnmarked(markColor()) - : gc::TenuredCell::fromPointer(thing)->markIfUnmarked(gc::MarkColor::Black); + MOZ_ASSERT(!IsInsideNursery(TenuredCell::fromPointer(thing))); + return ParticipatesInCC::value + ? TenuredCell::fromPointer(thing)->markIfUnmarked(markColor()) + : TenuredCell::fromPointer(thing)->markIfUnmarked(MarkColor::Black); } @@ -2523,6 +2532,12 @@ js::TenuringTracer::traceSlots(Value* vp, Value* end) traverse(vp); } +inline void +js::TenuringTracer::traceSlots(JS::Value* vp, uint32_t nslots) +{ + traceSlots(vp, vp + nslots); +} + #ifdef DEBUG static inline ptrdiff_t OffsetToChunkEnd(void* p) diff --git a/js/src/gc/Marking.h b/js/src/gc/Marking.h index 8e33520e08..b6b37e8899 100644 --- a/js/src/gc/Marking.h +++ b/js/src/gc/Marking.h @@ -4,408 +4,50 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Marking and sweeping APIs for use by implementations of different GC cell + * kinds. + */ + #ifndef gc_Marking_h #define gc_Marking_h -#include "mozilla/HashFunctions.h" -#include "mozilla/Move.h" - -#include "jsfriendapi.h" - -#include "ds/OrderedHashTable.h" -#include "gc/Heap.h" -#include "gc/Tracer.h" -#include "js/GCAPI.h" -#include "js/HeapAPI.h" -#include "js/SliceBudget.h" -#include "js/TracingAPI.h" #include "vm/TaggedProto.h" class JSLinearString; class JSRope; +struct JSRuntime; +class JSTracer; + namespace js { class BaseShape; class GCMarker; class LazyScript; class NativeObject; class ObjectGroup; +class Shape; class WeakMapBase; -namespace gc { -class Arena; -} // namespace gc + namespace jit { class JitCode; } // namespace jit -static const size_t NON_INCREMENTAL_MARK_STACK_BASE_CAPACITY = 4096; -static const size_t INCREMENTAL_MARK_STACK_BASE_CAPACITY = 32768; - -/* - * When the native stack is low, the GC does not call js::TraceChildren to mark - * the reachable "children" of the thing. Rather the thing is put aside and - * js::TraceChildren is called later with more space on the C stack. - * - * To implement such delayed marking of the children with minimal overhead for - * the normal case of sufficient native stack, the code adds a field per arena. - * The field markingDelay->link links all arenas with delayed things into a - * stack list with the pointer to stack top in GCMarker::unmarkedArenaStackTop. - * GCMarker::delayMarkingChildren adds arenas to the stack as necessary while - * markDelayedChildren pops the arenas from the stack until it empties. - */ -class MarkStack -{ - friend class GCMarker; - - uintptr_t* stack_; - uintptr_t* tos_; - uintptr_t* end_; - - // The capacity we start with and reset() to. - size_t baseCapacity_; - size_t maxCapacity_; - - public: - explicit MarkStack(size_t maxCapacity) - : stack_(nullptr), - tos_(nullptr), - end_(nullptr), - baseCapacity_(0), - maxCapacity_(maxCapacity) - {} - - LastTag = TempRopeTag - }; - - static const uintptr_t TagMask = 7; - static_assert(TagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags."); - static_assert(TagMask <= gc::CellAlignMask, "The tag mask must be embeddable in a Cell*."); - - class TaggedPtr - { - uintptr_t bits; - - Cell* ptr() const; - - public: - TaggedPtr(Tag tag, Cell* ptr); - Tag tag() const; - template T* as() const; - JSObject* asValueArrayObject() const; - JSObject* asSavedValueArrayObject() const; - JSRope* asTempRope() const; - }; - - struct ValueArray - { - ValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end); - - HeapSlot* end; - HeapSlot* start; - TaggedPtr ptr; - }; - - struct SavedValueArray - { - SavedValueArray(JSObject* obj, size_t index, HeapSlot::Kind kind); - - uintptr_t kind; - uintptr_t index; - TaggedPtr ptr; - }; - - explicit MarkStack(size_t maxCapacity = DefaultCapacity); - ~MarkStack(); - - static const size_t DefaultCapacity = SIZE_MAX; - - size_t capacity() { return end_ - stack_; } - - ptrdiff_t position() const { return tos_ - stack_; } - - void setStack(uintptr_t* stack, size_t tosIndex, size_t capacity) { - stack_ = stack; - tos_ = stack + tosIndex; - end_ = stack + capacity; - } - - MOZ_MUST_USE bool init(JSGCMode gcMode); - - void setBaseCapacity(JSGCMode mode); - size_t maxCapacity() const { return maxCapacity_; } - void setMaxCapacity(size_t maxCapacity); - - MOZ_MUST_USE bool push(uintptr_t item) { - if (tos_ == end_) { - if (!enlarge(1)) - return false; - } - MOZ_ASSERT(tos_ < end_); - *tos_++ = item; - return true; - } - - MOZ_MUST_USE bool push(uintptr_t item1, uintptr_t item2, uintptr_t item3) { - uintptr_t* nextTos = tos_ + 3; - if (nextTos > end_) { - if (!enlarge(3)) - return false; - nextTos = tos_ + 3; - } - MOZ_ASSERT(nextTos <= end_); - tos_[0] = item1; - tos_[1] = item2; - tos_[2] = item3; - tos_ = nextTos; - return true; - } - - bool isEmpty() const { - return tos_ == stack_; - } - - uintptr_t pop() { - MOZ_ASSERT(!isEmpty()); - return *--tos_; - } - - void reset(); - - /* Grow the stack, ensuring there is space for at least count elements. */ - MOZ_MUST_USE bool enlarge(unsigned count); - - void setGCMode(JSGCMode gcMode); - - size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; -}; - -namespace gc { - -struct WeakKeyTableHashPolicy { - typedef JS::GCCellPtr Lookup; - static HashNumber hash(const Lookup& v, const mozilla::HashCodeScrambler&) { - return mozilla::HashGeneric(v.asCell()); - } - static bool match(const JS::GCCellPtr& k, const Lookup& l) { return k == l; } - static bool isEmpty(const JS::GCCellPtr& v) { return !v; } - static void makeEmpty(JS::GCCellPtr* vp) { *vp = nullptr; } -}; - -struct WeakMarkable { - WeakMapBase* weakmap; - JS::GCCellPtr key; - - WeakMarkable(WeakMapBase* weakmapArg, JS::GCCellPtr keyArg) - : weakmap(weakmapArg), key(keyArg) {} -}; - -using WeakEntryVector = Vector; - -using WeakKeyTable = OrderedHashMap; - -} /* namespace gc */ - -class GCMarker : public JSTracer -{ - public: - explicit GCMarker(JSRuntime* rt); - MOZ_MUST_USE bool init(JSGCMode gcMode); - - void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); } - size_t maxCapacity() const { return stack.maxCapacity(); } - - void start(); - void stop(); - void reset(); - - // Mark the given GC thing and traverse its children at some point. - template void traverse(T thing); - - // Calls traverse on target after making additional assertions. - template void traverseEdge(S source, T* target); - template void traverseEdge(S source, const T& target); - - // Notes a weak graph edge for later sweeping. - template void noteWeakEdge(T* edge); - - /* - * Care must be taken changing the mark color from gray to black. The cycle - * collector depends on the invariant that there are no black to gray edges - * in the GC heap. This invariant lets the CC not trace through black - * objects. If this invariant is violated, the cycle collector may free - * objects that are still reachable. - */ - void setMarkColorGray() { - MOZ_ASSERT(isDrained()); - MOZ_ASSERT(color == gc::MarkColor::Black); - color = gc::MarkColor::Gray; - } - void setMarkColorBlack() { - MOZ_ASSERT(isDrained()); - MOZ_ASSERT(color == gc::MarkColor::Gray); - color = gc::MarkColor::Black; - } - gc::MarkColor markColor() const { return color; } - - void enterWeakMarkingMode(); - void leaveWeakMarkingMode(); - void abortLinearWeakMarking() { - leaveWeakMarkingMode(); - linearWeakMarkingDisabled_ = true; - } - - void delayMarkingArena(gc::Arena* arena); - void delayMarkingChildren(const void* thing); - void markDelayedChildren(gc::Arena* arena); - MOZ_MUST_USE bool markDelayedChildren(SliceBudget& budget); - bool hasDelayedChildren() const { - return !!unmarkedArenaStackTop; - } - - bool isDrained() { - return isMarkStackEmpty() && !unmarkedArenaStackTop; - } - - MOZ_MUST_USE bool drainMarkStack(SliceBudget& budget); - - void setGCMode(JSGCMode mode) { stack.setGCMode(mode); } - - size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; - -#ifdef DEBUG - bool shouldCheckCompartments() { return strictCompartmentChecking; } -#endif - - void markEphemeronValues(gc::Cell* markedCell, gc::WeakEntryVector& entry); - - private: -#ifdef DEBUG - void checkZone(void* p); -#else - void checkZone(void* p) {} -#endif - - /* - * We use a common mark stack to mark GC things of different types and use - * the explicit tags to distinguish them when it cannot be deduced from - * the context of push or pop operation. - */ - enum StackTag { - ValueArrayTag, - ObjectTag, - GroupTag, - SavedValueArrayTag, - JitCodeTag, - ScriptTag, - LastTag = JitCodeTag - }; - - static const uintptr_t StackTagMask = 7; - static_assert(StackTagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags."); - static_assert(StackTagMask <= gc::CellMask, "The tag mask must be embeddable in a Cell*."); - - // Push an object onto the stack for later tracing and assert that it has - // already been marked. - void repush(JSObject* obj) { - MOZ_ASSERT(gc::TenuredCell::fromPointer(obj)->isMarked(markColor())); - pushTaggedPtr(ObjectTag, obj); - } - - template void markAndTraceChildren(T* thing); - template void markAndPush(StackTag tag, T* thing); - template void markAndScan(T* thing); - template void markImplicitEdgesHelper(T oldThing); - template void markImplicitEdges(T* oldThing); - void eagerlyMarkChildren(JSLinearString* str); - void eagerlyMarkChildren(JSRope* rope); - void eagerlyMarkChildren(JSString* str); - void eagerlyMarkChildren(LazyScript *thing); - void eagerlyMarkChildren(Shape* shape); - void eagerlyMarkChildren(Scope* scope); - void lazilyMarkChildren(ObjectGroup* group); - - // We may not have concrete types yet, so this has to be outside the header. - template - void dispatchToTraceChildren(T* thing); - - // Mark the given GC thing, but do not trace its children. Return true - // if the thing became marked. - template - MOZ_MUST_USE bool mark(T* thing); - - void pushTaggedPtr(StackTag tag, void* ptr) { - checkZone(ptr); - uintptr_t addr = reinterpret_cast(ptr); - MOZ_ASSERT(!(addr & StackTagMask)); - if (!stack.push(addr | uintptr_t(tag))) - delayMarkingChildren(ptr); - } - - void pushValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end) { - checkZone(obj); - - MOZ_ASSERT(start <= end); - uintptr_t tagged = reinterpret_cast(obj) | GCMarker::ValueArrayTag; - uintptr_t startAddr = reinterpret_cast(start); - uintptr_t endAddr = reinterpret_cast(end); - - /* - * Push in the reverse order so obj will be on top. If we cannot push - * the array, we trigger delay marking for the whole object. - */ - if (!stack.push(endAddr, startAddr, tagged)) - delayMarkingChildren(obj); - } - - bool isMarkStackEmpty() { - return stack.isEmpty(); - } - - MOZ_MUST_USE bool restoreValueArray(JSObject* obj, void** vpp, void** endp); - void saveValueRanges(); - inline void processMarkStackTop(SliceBudget& budget); - - /* The mark stack. Pointers in this stack are "gray" in the GC sense. */ - MarkStack stack; - - /* The color is only applied to objects and functions. */ - ActiveThreadData color; - - /* Pointer to the top of the stack of arenas we are delaying marking on. */ - js::gc::Arena* unmarkedArenaStackTop; - - /* - * If the weakKeys table OOMs, disable the linear algorithm and fall back - * to iterating until the next GC. - */ - bool linearWeakMarkingDisabled_; - -#ifdef DEBUG - /* Count of arenas that are currently in the stack. */ - size_t markLaterArenas; - - /* Assert that start and stop are called with correct ordering. */ - bool started; - - /* - * If this is true, all marked objects must belong to a compartment being - * GCed. This is used to look for compartment bugs. - */ - bool strictCompartmentChecking; -#endif // DEBUG -}; - #ifdef DEBUG // Return true if this trace is happening on behalf of gray buffering during // the marking phase of incremental GC. bool IsBufferGrayRootsTracer(JSTracer* trc); + +bool +IsUnmarkGrayTracer(JSTracer* trc); #endif namespace gc { +class Arena; +struct Cell; +class TenuredCell; + /*** Special Cases ***/ void @@ -478,7 +120,7 @@ struct IsPrivateGCThingInValue : public mozilla::EnableIf::value && !mozilla::IsBaseOf::value && !mozilla::IsBaseOf::value && - !mozilla::IsBaseOf::value && + !mozilla::IsBaseOf::value && !mozilla::IsBaseOf::value, T> { static_assert(!mozilla::IsSame::value && !mozilla::IsSame::value, @@ -507,6 +149,56 @@ template void CheckTracedThing(JSTracer* trc, T thing); +namespace gc { + +// Functions for checking and updating GC thing pointers that might have been +// moved by compacting GC. Overloads are also provided that work with Values. +// +// IsForwarded - check whether a pointer refers to an GC thing that has been +// moved. +// +// Forwarded - return a pointer to the new location of a GC thing given a +// pointer to old location. +// +// MaybeForwarded - used before dereferencing a pointer that may refer to a +// moved GC thing without updating it. For JSObjects this will +// also update the object's shape pointer if it has been moved +// to allow slots to be accessed. + +template +inline bool IsForwarded(T* t); +inline bool IsForwarded(const JS::Value& value); + +template +inline T* Forwarded(T* t); + +inline Value Forwarded(const JS::Value& value); + +template +inline T MaybeForwarded(T t); + +inline void +MakeAccessibleAfterMovingGC(void* anyp) {} + +inline void +MakeAccessibleAfterMovingGC(JSObject* obj); // Defined in jsobjinlines.h. + +#ifdef JSGC_HASH_TABLE_CHECKS + +template +inline bool IsGCThingValidAfterMovingGC(T* t); + +template +inline void CheckGCThingAfterMovingGC(T* t); + +template +inline void CheckGCThingAfterMovingGC(const ReadBarriered& t); + +inline void CheckValueAfterMovingGC(const JS::Value& value); + +#endif // JSGC_HASH_TABLE_CHECKS + +} /* namespace gc */ } /* namespace js */ #endif /* gc_Marking_h */ diff --git a/js/src/gc/Nursery-inl.h b/js/src/gc/Nursery-inl.h index 9c8bfac4e0..84d5ec8faf 100644 --- a/js/src/gc/Nursery-inl.h +++ b/js/src/gc/Nursery-inl.h @@ -13,9 +13,18 @@ #include "jscntxt.h" #include "gc/Heap.h" +#include "gc/RelocationOverlay.h" #include "gc/Zone.h" #include "js/TracingAPI.h" #include "vm/Runtime.h" +#include "vm/SharedMem.h" + +template +bool +js::Nursery::isInside(const SharedMem& p) const +{ + return isInside(p.unwrap(/*safe - used for value in comparison above*/)); +} MOZ_ALWAYS_INLINE bool js::Nursery::getForwardedPointer(JSObject** ref) const diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index 417a186a57..216020b4cb 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -13,8 +13,6 @@ #include "mozilla/Unused.h" #include "jscompartment.h" -#include "jsfriendapi.h" -#include "jsgc.h" #include "jsutil.h" #include "gc/GCInternals.h" @@ -29,8 +27,7 @@ #include "vm/TypedArrayObject.h" #include "vm/TypeInference.h" -#include "jsobjinlines.h" - +#include "gc/Marking-inl.h" #include "vm/NativeObject-inl.h" using namespace js; @@ -74,27 +71,43 @@ struct js::Nursery::SweepAction #endif }; +namespace js { +struct NurseryChunk { + char data[Nursery::NurseryChunkUsableSize]; + gc::ChunkTrailer trailer; + static NurseryChunk* fromChunk(gc::Chunk* chunk); + void init(JSRuntime* rt); + void poisonAndInit(JSRuntime* rt, uint8_t poison); + uintptr_t start() const { return uintptr_t(&data); } + uintptr_t end() const { return uintptr_t(&trailer); } + gc::Chunk* toChunk(JSRuntime* rt); +}; +static_assert(sizeof(js::NurseryChunk) == gc::ChunkSize, + "Nursery chunk size must match gc::Chunk size."); + +} /* namespace js */ + inline void -js::Nursery::NurseryChunk::poisonAndInit(JSRuntime* rt, uint8_t poison) +js::NurseryChunk::poisonAndInit(JSRuntime* rt, uint8_t poison) { JS_POISON(this, poison, ChunkSize); init(rt); } inline void -js::Nursery::NurseryChunk::init(JSRuntime* rt) +js::NurseryChunk::init(JSRuntime* rt) { new (&trailer) gc::ChunkTrailer(rt, &rt->gc.storeBuffer); } -/* static */ inline js::Nursery::NurseryChunk* -js::Nursery::NurseryChunk::fromChunk(Chunk* chunk) +/* static */ inline js::NurseryChunk* +js::NurseryChunk::fromChunk(Chunk* chunk) { return reinterpret_cast(chunk); } inline Chunk* -js::Nursery::NurseryChunk::toChunk(JSRuntime* rt) +js::NurseryChunk::toChunk(JSRuntime* rt) { auto chunk = reinterpret_cast(this); chunk->init(rt); @@ -108,7 +121,9 @@ js::Nursery::Nursery(JSRuntime* rt) , currentStartPosition_(0) , currentEnd_(0) , currentChunk_(0) - , maxNurseryChunks_(0) + , maxChunkCount_(0) + , chunkCountLimit_(0) + , timeInChunkAlloc_(0) , previousPromotionRate_(0) , profileThreshold_(0) , enableProfiling_(false) @@ -129,15 +144,18 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock) return false; /* maxNurseryBytes parameter is rounded down to a multiple of chunk size. */ - maxNurseryChunks_ = maxNurseryBytes >> ChunkShift; + chunkCountLimit_ = maxNurseryBytes >> ChunkShift; - /* If no chunks are specified then the nursery is permenantly disabled. */ - if (maxNurseryChunks_ == 0) + /* If no chunks are specified then the nursery is permanently disabled. */ + if (chunkCountLimit_ == 0) return true; - updateNumChunksLocked(1, lock); - if (numChunks() == 0) + maxChunkCount_ = 1; + if (!allocateNextChunk(0, lock)) { + maxChunkCount_ = 0; return false; + } + /* After this point the Nursery has been enabled */ setCurrentChunk(0); setStartPosition(); @@ -184,12 +202,18 @@ void js::Nursery::enable() { MOZ_ASSERT(isEmpty()); - if (isEnabled()) + MOZ_ASSERT(!runtime()->gc.isVerifyPreBarriersEnabled()); + if (isEnabled() || !chunkCountLimit()) return; - updateNumChunks(1); - if (numChunks() == 0) - return; + { + AutoLockGCBgAlloc lock(runtime()); + maxChunkCount_ = 1; + if (!allocateNextChunk(0, lock)) { + maxChunkCount_ = 0; + return; + } + } setCurrentChunk(0); setStartPosition(); @@ -203,8 +227,12 @@ js::Nursery::disable() MOZ_ASSERT(isEmpty()); if (!isEnabled()) return; - updateNumChunks(0); + + freeChunksFrom(0); + maxChunkCount_ = 0; + currentEnd_ = 0; + runtime()->gc.storeBuffer().disable(); } @@ -219,6 +247,23 @@ js::Nursery::isEmpty() const return position() == currentStartPosition_; } +#ifdef JS_GC_ZEAL +void +js::Nursery::enterZealMode() { + if (isEnabled()) + maxChunkCount_ = chunkCountLimit(); +} + +void +js::Nursery::leaveZealMode() { + if (isEnabled()) { + MOZ_ASSERT(isEmpty()); + setCurrentChunk(0); + setStartPosition(); + } +} +#endif // JS_GC_ZEAL + JSObject* js::Nursery::allocateObject(JSContext* cx, size_t size, size_t numDynamic, const js::Class* clasp) { @@ -265,9 +310,23 @@ js::Nursery::allocate(size_t size) MOZ_ASSERT(size % gc::CellSize == 0); if (currentEnd() < position() + size) { - if (currentChunk_ + 1 == numChunks()) + unsigned chunkno = currentChunk_ + 1; + MOZ_ASSERT(chunkno <= chunkCountLimit()); + MOZ_ASSERT(chunkno <= maxChunkCount()); + MOZ_ASSERT(chunkno <= allocatedChunkCount()); + if (chunkno == maxChunkCount()) return nullptr; - setCurrentChunk(currentChunk_ + 1); + if (MOZ_UNLIKELY(chunkno == allocatedChunkCount())) { + mozilla::TimeStamp start = TimeStamp::Now(); + { + AutoLockGCBgAlloc lock(runtime()); + if (!allocateNextChunk(chunkno, lock)) + return nullptr; + } + timeInChunkAlloc_ += TimeStamp::Now() - start; + MOZ_ASSERT(chunkno < allocatedChunkCount()); + } + setCurrentChunk(chunkno); } void* thing = (void*)position(); @@ -424,15 +483,24 @@ js::Nursery::calcPromotionRate(bool *validForTenuring) const { float used = float(previousGC.nurseryUsedBytes); float capacity = float(previousGC.nurseryCapacity); float tenured = float(previousGC.tenuredBytes); + float rate; - if (validForTenuring) { - /* - * We can only use promotion rates if they're likely to be valid, - * they're only valid if the nursury was at least 90% full. - */ - *validForTenuring = used > capacity * 0.9f; + if (previousGC.nurseryUsedBytes > 0) { + if (validForTenuring) { + /* + * We can only use promotion rates if they're likely to be valid, + * they're only valid if the nursury was at least 90% full. + */ + *validForTenuring = used > capacity * 0.9f; + } + rate = tenured / used; + } else { + if (validForTenuring) + *validForTenuring = false; + rate = 0.0f; } - return tenured / used; + + return rate; } void @@ -451,20 +519,28 @@ js::Nursery::renderProfileJSON(JSONPrinter& json) const // requested. (And as a public API, this function should not crash in // such a case.) json.beginObject(); - json.property("status", "no collection"); + json.property("status", "nursery empty"); json.endObject(); return; } json.beginObject(); + json.property("status", "complete"); + json.property("reason", JS::gcreason::ExplainReason(previousGC.reason)); json.property("bytes_tenured", previousGC.tenuredBytes); - json.floatProperty("promotion_rate", calcPromotionRate(nullptr), 0); - json.property("nursery_bytes", previousGC.nurseryUsedBytes); - json.property("new_nursery_bytes", numChunks() * ChunkSize); + json.property("bytes_used", previousGC.nurseryUsedBytes); + json.property("cur_capacity", previousGC.nurseryCapacity); + const size_t newCapacity = spaceToEnd(maxChunkCount()); + if (newCapacity != previousGC.nurseryCapacity) + json.property("new_capacity", newCapacity); + if (previousGC.nurseryLazyCapacity != previousGC.nurseryCapacity) + json.property("lazy_capacity", previousGC.nurseryLazyCapacity); + if (!timeInChunkAlloc_.IsZero()) + json.property("chunk_alloc_us", timeInChunkAlloc_, json.MICROSECONDS); - json.beginObjectProperty("timings"); + json.beginObjectProperty("phase_times"); #define EXTRACT_NAME(name, text) #name, static const char* names[] = { @@ -566,14 +642,13 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) doCollection(reason, tenureCounts); } else { previousGC.nurseryUsedBytes = 0; - previousGC.nurseryCapacity = spaceToEnd(); + previousGC.nurseryCapacity = spaceToEnd(maxChunkCount()); + previousGC.nurseryLazyCapacity = spaceToEnd(allocatedChunkCount()); previousGC.tenuredBytes = 0; } // Resize the nursery. - startProfile(ProfileKey::Resize); maybeResizeNursery(reason); - endProfile(ProfileKey::Resize); // If we are promoting the nursery, or exhausted the store buffer with // pointers to nursery things, which will force a collection well before @@ -589,7 +664,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) for (auto& entry : tenureCounts.entries) { if (entry.count >= 3000) { ObjectGroup* group = entry.group; - if (group->canPreTenure()) { + if (group->canPreTenure() && group->zone()->group()->canEnterWithoutYielding(cx)) { AutoCompartment ac(cx, group); group->setShouldPreTenure(cx); pretenureCount++; @@ -608,7 +683,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) // Disable the nursery if the user changed the configuration setting. The // nursery can only be re-enabled by resetting the configurationa and // restarting firefox. - if (maxNurseryChunks_ == 0) + if (chunkCountLimit_ == 0) disable(); endProfile(ProfileKey::Total); @@ -618,6 +693,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) rt->gc.stats.endNurseryCollection(reason); TraceMinorGCEnd(); + timeInChunkAlloc_ = mozilla::TimeDuration(); if (enableProfiling_ && totalTime >= profileThreshold_) { static int printedHeader = 0; @@ -629,8 +705,8 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ", JS::gcreason::ExplainReason(reason), promotionRate * 100, - numChunks()); - printProfileTimes(profileTimes_); + maxChunkCount()); + printProfileDurations(profileDurations_); if (reportTenurings_) { for (auto& entry : tenureCounts.entries) { @@ -652,7 +728,7 @@ js::Nursery::doCollection(JS::gcreason::Reason reason, AutoDisableProxyCheck disableStrictProxyChecking(rt); mozilla::DebugOnly oomUnsafeRegion; - const size_t initialNurseryCapacity = spaceToEnd(); + const size_t initialNurseryCapacity = spaceToEnd(maxChunkCount()); const size_t initialNurseryUsedBytes = initialNurseryCapacity - freeSpace(); // Move objects pointed to by roots from the nursery to the major heap. @@ -751,6 +827,7 @@ js::Nursery::doCollection(JS::gcreason::Reason reason, previousGC.reason = reason; previousGC.nurseryCapacity = initialNurseryCapacity; + previousGC.nurseryLazyCapacity = spaceToEnd(allocatedChunkCount()); previousGC.nurseryUsedBytes = initialNurseryUsedBytes; previousGC.tenuredBytes = mover.tenuredSize; } @@ -832,12 +909,18 @@ js::Nursery::clear() { #ifdef JS_GC_ZEAL /* Poison the nursery contents so touching a freed object will crash. */ - for (unsigned i = 0; i < numChunks(); i++) + for (unsigned i = 0; i < allocatedChunkCount(); i++) chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) { + /* Only reset the alloc point when we are close to the end. */ + if (currentChunk_ + 1 == maxChunkCount()) + setCurrentChunk(0); + } else +#endif { #ifdef JS_CRASH_DIAGNOSTICS - for (unsigned i = 0; i < numChunks(); ++i) + for (unsigned i = 0; i < allocatedChunkCount(); ++i) chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); #endif setCurrentChunk(0); @@ -849,9 +932,9 @@ js::Nursery::clear() } size_t -js::Nursery::spaceToEnd() const +js::Nursery::spaceToEnd(unsigned chunkCount) const { - unsigned lastChunk = numChunks() - 1; + unsigned lastChunk = chunkCount - 1; MOZ_ASSERT(lastChunk >= currentStartChunk_); MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <= NurseryChunkUsableSize); @@ -859,7 +942,7 @@ js::Nursery::spaceToEnd() const size_t bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) + ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize); - MOZ_ASSERT(bytes <= numChunks() * NurseryChunkUsableSize); + MOZ_ASSERT(bytes <= maxChunkCount() * NurseryChunkUsableSize); return bytes; } @@ -867,14 +950,40 @@ js::Nursery::spaceToEnd() const MOZ_ALWAYS_INLINE void js::Nursery::setCurrentChunk(unsigned chunkno) { - MOZ_ASSERT(chunkno < maxChunks()); - MOZ_ASSERT(chunkno < numChunks()); + MOZ_ASSERT(chunkno < chunkCountLimit()); + MOZ_ASSERT(chunkno < allocatedChunkCount()); currentChunk_ = chunkno; position_ = chunk(chunkno).start(); currentEnd_ = chunk(chunkno).end(); chunk(chunkno).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); } +bool +js::Nursery::allocateNextChunk(const unsigned chunkno, + AutoLockGCBgAlloc& lock) +{ + const unsigned priorCount = allocatedChunkCount(); + const unsigned newCount = priorCount + 1; + + MOZ_ASSERT((chunkno == currentChunk_ + 1) || (chunkno == 0 && allocatedChunkCount() == 0)); + MOZ_ASSERT(chunkno == allocatedChunkCount()); + MOZ_ASSERT(chunkno < chunkCountLimit()); + MOZ_ASSERT(chunkno < maxChunkCount()); + + if (!chunks_.resize(newCount)) + return false; + + Chunk* newChunk; + newChunk = runtime()->gc.getOrAllocChunk(lock); + if (!newChunk) { + chunks_.shrinkTo(priorCount); + return false; + } + + chunks_[chunkno] = NurseryChunk::fromChunk(newChunk); + return true; +} + MOZ_ALWAYS_INLINE void js::Nursery::setStartPosition() { @@ -911,23 +1020,25 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason) float(previousGC.tenuredBytes) / float(previousGC.nurseryCapacity); newMaxNurseryChunks = runtime()->gc.tunables.gcMaxNurseryBytes() >> ChunkShift; - if (newMaxNurseryChunks != maxNurseryChunks_) { - maxNurseryChunks_ = newMaxNurseryChunks; + if (newMaxNurseryChunks != chunkCountLimit_) { + chunkCountLimit_ = newMaxNurseryChunks; /* The configured maximum nursery size is changing */ - const int extraChunks = numChunks() - newMaxNurseryChunks; - if (extraChunks > 0) { + if (maxChunkCount() > newMaxNurseryChunks) { /* We need to shrink the nursery */ - shrinkAllocableSpace(extraChunks); + shrinkAllocableSpace(newMaxNurseryChunks); previousPromotionRate_ = promotionRate; return; } } - if (promotionRate > GrowThreshold) + if (promotionRate > GrowThreshold) { + // The GC nursery is an optimization and so if we fail to allocate + // nursery chunks we do not report an error. growAllocableSpace(); - else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) - shrinkAllocableSpace(1); + } else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) { + shrinkAllocableSpace(maxChunkCount() - 1); + } previousPromotionRate_ = promotionRate; } @@ -935,66 +1046,46 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason) void js::Nursery::growAllocableSpace() { - updateNumChunks(Min(numChunks() * 2, maxNurseryChunks_)); + maxChunkCount_ = Min(maxChunkCount() * 2, chunkCountLimit()); } void -js::Nursery::shrinkAllocableSpace(unsigned removeNumChunks) +js::Nursery::freeChunksFrom(unsigned firstFreeChunk) +{ + MOZ_ASSERT(firstFreeChunk < chunks_.length()); + { + AutoLockGC lock(runtime()); + for (unsigned i = firstFreeChunk; i < chunks_.length(); i++) + runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock); + } + chunks_.shrinkTo(firstFreeChunk); +} + +void +js::Nursery::shrinkAllocableSpace(unsigned newCount) { #ifdef JS_GC_ZEAL if (runtime()->hasZealMode(ZealMode::GenerationalGC)) return; #endif - updateNumChunks(Max(numChunks() - removeNumChunks, 1u)); + + // Don't shrink the nursery to zero (use Nursery::disable() instead) and + // don't attempt to shrink it to the same size. + if ((newCount == 0) || (newCount == maxChunkCount())) + return; + + MOZ_ASSERT(newCount < maxChunkCount()); + + if (newCount < allocatedChunkCount()) + freeChunksFrom(newCount); + + maxChunkCount_ = newCount; } void js::Nursery::minimizeAllocableSpace() { - updateNumChunks(1); -} - -void -js::Nursery::updateNumChunks(unsigned newCount) -{ - if (numChunks() != newCount) { - AutoLockGCBgAlloc lock(runtime()); - updateNumChunksLocked(newCount, lock); - } -} - -void -js::Nursery::updateNumChunksLocked(unsigned newCount, - AutoLockGCBgAlloc& lock) -{ - // The GC nursery is an optimization and so if we fail to allocate nursery - // chunks we do not report an error. - - unsigned priorCount = numChunks(); - MOZ_ASSERT(priorCount != newCount); - - if (newCount < priorCount) { - // Shrink the nursery and free unused chunks. - for (unsigned i = newCount; i < priorCount; i++) - runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock); - chunks_.shrinkTo(newCount); - return; - } - - // Grow the nursery and allocate new chunks. - if (!chunks_.resize(newCount)) - return; - - for (unsigned i = priorCount; i < newCount; i++) { - auto newChunk = runtime()->gc.getOrAllocChunk(lock); - if (!newChunk) { - chunks_.shrinkTo(i); - return; - } - - chunks_[i] = NurseryChunk::fromChunk(newChunk); - chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); - } + shrinkAllocableSpace(1); } void @@ -1033,6 +1124,13 @@ js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj) return dictionaryModeObjects_.append(obj); } +uintptr_t +js::Nursery::currentEnd() const +{ + MOZ_ASSERT(currentEnd_ == chunk(currentChunk_).end()); + return currentEnd_; +} + void js::Nursery::sweepDictionaryModeObjects() { diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 9b31463d3f..5fbe93c221 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -10,19 +10,10 @@ #include "mozilla/EnumeratedArray.h" -#include "jsalloc.h" -#include "jspubtd.h" - -#include "ds/BitArray.h" -#include "gc/Heap.h" -#include "gc/Memory.h" #include "js/Class.h" -#include "js/GCAPI.h" -#include "js/HashTable.h" #include "js/HeapAPI.h" -#include "js/Value.h" +#include "js/TracingAPI.h" #include "js/Vector.h" -#include "vm/SharedMem.h" #define FOR_EACH_NURSERY_PROFILE_TIME(_) \ /* Key Header text */ \ @@ -44,19 +35,22 @@ _(FreeMallocedBuffers, "frSlts") \ _(ClearStoreBuffer, "clrSB") \ _(ClearNursery, "clear") \ - _(Resize, "resize") \ _(Pretenure, "pretnr") +template class SharedMem; + namespace JS { struct Zone; } // namespace JS namespace js { +class AutoLockGCBgAlloc; class ObjectElements; class PlainObject; class NativeObject; class Nursery; +struct NurseryChunk; class HeapSlot; namespace gc { @@ -65,6 +59,8 @@ struct Cell; class MinorCollectionTracer; class RelocationOverlay; struct TenureCountCache; +enum class AllocKind : uint8_t; +class TenuredCell; } /* namespace gc */ namespace jit { @@ -97,7 +93,7 @@ class TenuringTracer : public JSTracer // The store buffers need to be able to call these directly. void traceObject(JSObject* src); void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length); - void traceSlots(JS::Value* vp, uint32_t nslots) { traceSlots(vp, vp + nslots); } + void traceSlots(JS::Value* vp, uint32_t nslots); private: Nursery& nursery() { return nursery_; } @@ -139,15 +135,22 @@ class Nursery [[nodiscard]] bool init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock); - unsigned maxChunks() const { return maxNurseryChunks_; } - unsigned numChunks() const { return chunks_.length(); } + unsigned chunkCountLimit() const { return chunkCountLimit_; } - bool exists() const { return maxChunks() != 0; } - size_t nurserySize() const { return maxChunks() << ChunkShift; } + // Number of allocated (ready to use) chunks. + unsigned allocatedChunkCount() const { return chunks_.length(); } + + // Total number of chunks and the capacity of the nursery. Chunks will be + // lazilly allocated and added to the chunks array up to this limit, after + // that the nursery must be collected, this limit may be raised during + // collection. + unsigned maxChunkCount() const { return maxChunkCount_; } + + bool exists() const { return chunkCountLimit() != 0; } void enable(); void disable(); - bool isEnabled() const { return numChunks() != 0; } + bool isEnabled() const { return maxChunkCount() != 0; } /* Return true if no allocations have been made since the last collection. */ bool isEmpty() const; @@ -159,15 +162,14 @@ class Nursery MOZ_ALWAYS_INLINE bool isInside(gc::Cell* cellp) const = delete; MOZ_ALWAYS_INLINE bool isInside(const void* p) const { for (auto chunk : chunks_) { - if (uintptr_t(p) - chunk->start() < gc::ChunkSize) + if (uintptr_t(p) - uintptr_t(chunk) < gc::ChunkSize) return true; } return false; } + template - bool isInside(const SharedMem& p) const { - return isInside(p.unwrap(/*safe - used for value in comparison above*/)); - } + inline bool isInside(const SharedMem& p) const; /* * Allocate and return a pointer to a new GC object with its |slots| @@ -238,7 +240,7 @@ class Nursery MOZ_MUST_USE bool queueDictionaryModeObjectToSweep(NativeObject* obj); size_t sizeOfHeapCommitted() const { - return numChunks() * gc::ChunkSize; + return allocatedChunkCount() * gc::ChunkSize; } size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const { size_t total = 0; @@ -249,13 +251,16 @@ class Nursery } // The number of bytes from the start position to the end of the nursery. - size_t spaceToEnd() const; + // pass maxChunkCount(), allocatedChunkCount() or chunkCountLimit() + // to calculate the nursery size, current lazy-allocated size or nursery + // limit respectively. + size_t spaceToEnd(unsigned chunkCount) const; // Free space remaining, not counting chunk trailers. MOZ_ALWAYS_INLINE size_t freeSpace() const { MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize); return (currentEnd_ - position_) + - (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize; + (maxChunkCount() - currentChunk_ - 1) * NurseryChunkUsableSize; } /* Print total profile times on shutdown. */ @@ -270,24 +275,19 @@ class Nursery JS::gcreason::Reason minorGCTriggerReason() const { return minorGCTriggerReason_; } void clearMinorGCRequest() { minorGCTriggerReason_ = JS::gcreason::NO_REASON; } + bool needIdleTimeCollection() const { + return minorGCRequested() || + (freeSpace() < kIdleTimeCollectionThreshold); + } + bool enableProfiling() const { return enableProfiling_; } private: /* The amount of space in the mapped nursery available to allocations. */ - static const size_t NurseryChunkUsableSize = gc::ChunkSize - sizeof(gc::ChunkTrailer); + static const size_t NurseryChunkUsableSize = gc::ChunkSize - gc::ChunkTrailerSize; - struct NurseryChunk { - char data[NurseryChunkUsableSize]; - gc::ChunkTrailer trailer; - static NurseryChunk* fromChunk(gc::Chunk* chunk); - void init(JSRuntime* rt); - void poisonAndInit(JSRuntime* rt, uint8_t poison); - uintptr_t start() const { return uintptr_t(&data); } - uintptr_t end() const { return uintptr_t(&trailer); } - gc::Chunk* toChunk(JSRuntime* rt); - }; - static_assert(sizeof(NurseryChunk) == gc::ChunkSize, - "Nursery chunk size must match gc::Chunk size."); + /* Attemp to run a minor GC in the idle time if the free space falls below this threshold. */ + static constexpr size_t kIdleTimeCollectionThreshold = NurseryChunkUsableSize / 4; /* * The start and end pointers are stored under the runtime so that we can @@ -312,8 +312,20 @@ class Nursery /* The index of the chunk that is currently being allocated from. */ unsigned currentChunk_; - /* Maximum number of chunks to allocate for the nursery. */ - unsigned maxNurseryChunks_; + /* + * The nursery may grow the chunks_ vector up to this size without a + * collection. This allows the nursery to grow lazilly. This limit may + * change during maybeResizeNursery() each collection. + */ + unsigned maxChunkCount_; + + /* + * This limit is fixed by configuration. It represents the maximum size + * the nursery is permitted to tune itself to in maybeResizeNursery(); + */ + unsigned chunkCountLimit_; + + mozilla::TimeDuration timeInChunkAlloc_; /* Promotion rate for the previous minor collection. */ float previousPromotionRate_; @@ -350,6 +362,7 @@ class Nursery struct { JS::gcreason::Reason reason; size_t nurseryCapacity; + size_t nurseryLazyCapacity; size_t nurseryUsedBytes; size_t tenuredBytes; } previousGC; @@ -408,7 +421,10 @@ class Nursery using NativeObjectVector = Vector; NativeObjectVector dictionaryModeObjects_; - NurseryChunk* allocChunk(); +#ifdef JS_GC_ZEAL + struct Canary; + Canary* lastCanary_; +#endif NurseryChunk& chunk(unsigned index) const { return *chunks_[index]; @@ -417,24 +433,14 @@ class Nursery void setCurrentChunk(unsigned chunkno); void setStartPosition(); - void updateNumChunks(unsigned newCount); - void updateNumChunksLocked(unsigned newCount, - AutoLockGCBgAlloc& lock); + /* + * Allocate the next chunk, or the first chunk for initialization. + * Callers will probably want to call setCurrentChunk(0) next. + */ + [[nodiscard]] bool allocateNextChunk(unsigned chunkno, + AutoLockGCBgAlloc& lock); - MOZ_ALWAYS_INLINE uintptr_t allocationEnd() const { - MOZ_ASSERT(numChunks() > 0); - return chunks_.back()->end(); - } - - MOZ_ALWAYS_INLINE uintptr_t currentEnd() const { - MOZ_ASSERT(runtime_); - MOZ_ASSERT(currentEnd_ == chunk(currentChunk_).end()); - return currentEnd_; - } - void* addressOfCurrentEnd() const { - MOZ_ASSERT(runtime_); - return (void*)¤tEnd_; - } + MOZ_ALWAYS_INLINE uintptr_t currentEnd() const; uintptr_t position() const { return position_; } void* addressOfPosition() const { return (void*)&position_; } @@ -487,9 +493,13 @@ class Nursery /* Change the allocable space provided by the nursery. */ void maybeResizeNursery(JS::gcreason::Reason reason); void growAllocableSpace(); - void shrinkAllocableSpace(unsigned removeNumChunks); + void shrinkAllocableSpace(unsigned newCount); void minimizeAllocableSpace(); + // Free the chunks starting at firstFreeChunk until the end of the chunks + // vector. Shrinks the vector but does not update maxChunkCount(). + void freeChunksFrom(unsigned firstFreeChunk); + /* Profile recording and printing. */ void startProfile(ProfileKey key); void endProfile(ProfileKey key); @@ -498,6 +508,7 @@ class Nursery friend class TenuringTracer; friend class gc::MinorCollectionTracer; friend class jit::MacroAssembler; + friend struct NurseryChunk; }; } /* namespace js */ diff --git a/js/src/gc/ObjectKind-inl.h b/js/src/gc/ObjectKind-inl.h new file mode 100644 index 0000000000..d2cde1da08 --- /dev/null +++ b/js/src/gc/ObjectKind-inl.h @@ -0,0 +1,174 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal helper functions for getting the AllocKind used to allocate a + * JSObject and related information. + */ + +#ifndef gc_ObjectKind_inl_h +#define gc_ObjectKind_inl_h + +#include "vm/NativeObject.h" + +namespace js { +namespace gc { + +static inline bool +CanBeFinalizedInBackground(AllocKind kind, const Class* clasp) +{ + MOZ_ASSERT(IsObjectAllocKind(kind)); + /* If the class has no finalizer or a finalizer that is safe to call on + * a different thread, we change the alloc kind. For example, + * AllocKind::OBJECT0 calls the finalizer on the active thread, + * AllocKind::OBJECT0_BACKGROUND calls the finalizer on the gcHelperThread. + * IsBackgroundFinalized is called to prevent recursively incrementing + * the alloc kind; kind may already be a background finalize kind. + */ + return (!IsBackgroundFinalized(kind) && + (!clasp->hasFinalize() || (clasp->flags & JSCLASS_BACKGROUND_FINALIZE))); +} + +static inline AllocKind +GetBackgroundAllocKind(AllocKind kind) +{ + MOZ_ASSERT(!IsBackgroundFinalized(kind)); + MOZ_ASSERT(IsObjectAllocKind(kind)); + return AllocKind(size_t(kind) + 1); +} + +/* Capacity for slotsToThingKind */ +const size_t SLOTS_TO_THING_KIND_LIMIT = 17; + +extern const AllocKind slotsToThingKind[]; + +/* Get the best kind to use when making an object with the given slot count. */ +static inline AllocKind +GetGCObjectKind(size_t numSlots) +{ + if (numSlots >= SLOTS_TO_THING_KIND_LIMIT) + return AllocKind::OBJECT16; + return slotsToThingKind[numSlots]; +} + +static inline AllocKind +GetGCObjectKind(const Class* clasp) +{ + if (clasp == FunctionClassPtr) + return AllocKind::FUNCTION; + + MOZ_ASSERT(!clasp->isProxy(), "Proxies should use GetProxyGCObjectKind"); + + uint32_t nslots = JSCLASS_RESERVED_SLOTS(clasp); + if (clasp->flags & JSCLASS_HAS_PRIVATE) + nslots++; + return GetGCObjectKind(nslots); +} + +/* As for GetGCObjectKind, but for dense array allocation. */ +static inline AllocKind +GetGCArrayKind(size_t numElements) +{ + /* + * Dense arrays can use their fixed slots to hold their elements array + * (less two Values worth of ObjectElements header), but if more than the + * maximum number of fixed slots is needed then the fixed slots will be + * unused. + */ + JS_STATIC_ASSERT(ObjectElements::VALUES_PER_HEADER == 2); + if (numElements > NativeObject::MAX_DENSE_ELEMENTS_COUNT || + numElements + ObjectElements::VALUES_PER_HEADER >= SLOTS_TO_THING_KIND_LIMIT) + { + return AllocKind::OBJECT2; + } + return slotsToThingKind[numElements + ObjectElements::VALUES_PER_HEADER]; +} + +static inline AllocKind +GetGCObjectFixedSlotsKind(size_t numFixedSlots) +{ + MOZ_ASSERT(numFixedSlots < SLOTS_TO_THING_KIND_LIMIT); + return slotsToThingKind[numFixedSlots]; +} + +// Get the best kind to use when allocating an object that needs a specific +// number of bytes. +static inline AllocKind +GetGCObjectKindForBytes(size_t nbytes) +{ + MOZ_ASSERT(nbytes <= JSObject::MAX_BYTE_SIZE); + + if (nbytes <= sizeof(NativeObject)) + return AllocKind::OBJECT0; + nbytes -= sizeof(NativeObject); + + size_t dataSlots = AlignBytes(nbytes, sizeof(Value)) / sizeof(Value); + MOZ_ASSERT(nbytes <= dataSlots * sizeof(Value)); + return GetGCObjectKind(dataSlots); +} + +/* Get the number of fixed slots and initial capacity associated with a kind. */ +static inline size_t +GetGCKindSlots(AllocKind thingKind) +{ + /* Using a switch in hopes that thingKind will usually be a compile-time constant. */ + switch (thingKind) { + case AllocKind::FUNCTION: + case AllocKind::OBJECT0: + case AllocKind::OBJECT0_BACKGROUND: + return 0; + case AllocKind::FUNCTION_EXTENDED: + case AllocKind::OBJECT2: + case AllocKind::OBJECT2_BACKGROUND: + return 2; + case AllocKind::OBJECT4: + case AllocKind::OBJECT4_BACKGROUND: + return 4; + case AllocKind::OBJECT8: + case AllocKind::OBJECT8_BACKGROUND: + return 8; + case AllocKind::OBJECT12: + case AllocKind::OBJECT12_BACKGROUND: + return 12; + case AllocKind::OBJECT16: + case AllocKind::OBJECT16_BACKGROUND: + return 16; + default: + MOZ_CRASH("Bad object alloc kind"); + } +} + +static inline size_t +GetGCKindSlots(AllocKind thingKind, const Class* clasp) +{ + size_t nslots = GetGCKindSlots(thingKind); + + /* An object's private data uses the space taken by its last fixed slot. */ + if (clasp->flags & JSCLASS_HAS_PRIVATE) { + MOZ_ASSERT(nslots > 0); + nslots--; + } + + /* + * Functions have a larger alloc kind than AllocKind::OBJECT to reserve + * space for the extra fields in JSFunction, but have no fixed slots. + */ + if (clasp == FunctionClassPtr) + nslots = 0; + + return nslots; +} + +static inline size_t +GetGCKindBytes(AllocKind thingKind) +{ + return sizeof(JSObject_Slots0) + GetGCKindSlots(thingKind) * sizeof(Value); +} + +} // namespace gc +} // namespace js + +#endif // gc_ObjectKind_inl_h diff --git a/js/src/gc/RelocationOverlay.h b/js/src/gc/RelocationOverlay.h new file mode 100644 index 0000000000..55440abc19 --- /dev/null +++ b/js/src/gc/RelocationOverlay.h @@ -0,0 +1,75 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * GC-internal definition of relocation overlay used while moving cells. + */ + +#ifndef gc_RelocationOverlay_h +#define gc_RelocationOverlay_h + +#include "mozilla/Assertions.h" + +#include + +namespace js { +namespace gc { + +struct Cell; + +/* + * This structure overlays a Cell that has been moved and provides a way to find + * its new location. It's used during generational and compacting GC. + */ +class RelocationOverlay +{ + /* The low bit is set so this should never equal a normal pointer. */ + static const uintptr_t Relocated = uintptr_t(0xbad0bad1); + + /* Set to Relocated when moved. */ + uintptr_t magic_; + + /* The location |this| was moved to. */ + Cell* newLocation_; + + /* A list entry to track all relocated things. */ + RelocationOverlay* next_; + + public: + static RelocationOverlay* fromCell(Cell* cell) { + return reinterpret_cast(cell); + } + + bool isForwarded() const { + return magic_ == Relocated; + } + + Cell* forwardingAddress() const { + MOZ_ASSERT(isForwarded()); + return newLocation_; + } + + void forwardTo(Cell* cell); + + RelocationOverlay*& nextRef() { + MOZ_ASSERT(isForwarded()); + return next_; + } + + RelocationOverlay* next() const { + MOZ_ASSERT(isForwarded()); + return next_; + } + + static bool isCellForwarded(Cell* cell) { + return fromCell(cell)->isForwarded(); + } +}; + +} // namespace gc +} // namespace js + +#endif /* gc_RelocationOverlay_h */ diff --git a/js/src/gc/RootMarking.cpp b/js/src/gc/RootMarking.cpp index b793b9f7c3..3930def76a 100644 --- a/js/src/gc/RootMarking.cpp +++ b/js/src/gc/RootMarking.cpp @@ -10,7 +10,6 @@ #endif #include "jscntxt.h" -#include "jsgc.h" #include "jsprf.h" #include "jstypes.h" @@ -26,6 +25,9 @@ #include "jsgcinlines.h" #include "jsobjinlines.h" +#include "gc/Iteration-inl.h" +#include "gc/Nursery-inl.h" + using namespace js; using namespace js::gc; diff --git a/js/src/gc/StoreBuffer-inl.h b/js/src/gc/StoreBuffer-inl.h index 9b18fde6d1..7ef6b10c5c 100644 --- a/js/src/gc/StoreBuffer-inl.h +++ b/js/src/gc/StoreBuffer-inl.h @@ -8,6 +8,7 @@ #include "gc/StoreBuffer.h" +#include "gc/Cell.h" #include "gc/Heap.h" namespace js { diff --git a/js/src/gc/StoreBuffer.cpp b/js/src/gc/StoreBuffer.cpp index 7ae85debee..c850014d2a 100644 --- a/js/src/gc/StoreBuffer.cpp +++ b/js/src/gc/StoreBuffer.cpp @@ -7,8 +7,6 @@ #include "mozilla/Assertions.h" -#include "jscompartment.h" - #include "gc/Statistics.h" #include "vm/ArgumentsObject.h" #include "vm/Runtime.h" diff --git a/js/src/gc/StoreBuffer.h b/js/src/gc/StoreBuffer.h index c2c108140a..d1d80c0564 100644 --- a/js/src/gc/StoreBuffer.h +++ b/js/src/gc/StoreBuffer.h @@ -14,6 +14,7 @@ #include "jsalloc.h" +#include "ds/BitArray.h" #include "ds/LifoAlloc.h" #include "gc/Nursery.h" #include "js/MemoryMetrics.h" @@ -21,6 +22,7 @@ namespace js { namespace gc { +class Arena; class ArenaCellSet; /* diff --git a/js/src/gc/Tracer.cpp b/js/src/gc/Tracer.cpp index 7be4543029..015410e9c5 100644 --- a/js/src/gc/Tracer.cpp +++ b/js/src/gc/Tracer.cpp @@ -8,9 +8,7 @@ #include "mozilla/DebugOnly.h" #include "mozilla/SizePrintfMacros.h" -#include "jsapi.h" #include "jsfun.h" -#include "jsgc.h" #include "jsprf.h" #include "jsscript.h" #include "jsutil.h" @@ -438,3 +436,10 @@ JS_GetTraceThingInfo(char* buf, size_t bufsize, JSTracer* trc, void* thing, JS::CallbackTracer::CallbackTracer(JSContext* cx, WeakMapTraceKind weakTraceKind) : CallbackTracer(cx->runtime(), weakTraceKind) {} + +uint32_t +JSTracer::gcNumberForMarking() const +{ + MOZ_ASSERT(isMarkingTracer()); + return runtime()->gc.gcNumber(); +} diff --git a/js/src/gc/Verifier.cpp b/js/src/gc/Verifier.cpp index 2c31250f81..1c6b21a108 100644 --- a/js/src/gc/Verifier.cpp +++ b/js/src/gc/Verifier.cpp @@ -11,17 +11,17 @@ #include "mozilla/Sprintf.h" #include "jscntxt.h" -#include "jsgc.h" #include "jsprf.h" #include "gc/GCInternals.h" #include "gc/Zone.h" -#include "js/GCAPI.h" #include "js/HashTable.h" #include "jscntxtinlines.h" #include "jsgcinlines.h" +#include "gc/Marking-inl.h" + using namespace js; using namespace js::gc; @@ -564,14 +564,28 @@ CheckHeapTracer::check(AutoLockForExclusiveAccess& lock) return !oom; } +static const char* +GetCellColorName(Cell* cell) +{ + if (cell->isMarkedBlack()) + return "black"; + if (cell->isMarkedGray()) + return "gray"; + return "white"; +} + void HeapCheckTracerBase::dumpCellInfo(Cell* cell) { auto kind = cell->getTraceKind(); - fprintf(stderr, "%s", GCTraceKindToAscii(kind)); - if (kind == JS::TraceKind::Object) - fprintf(stderr, " %s", static_cast(cell)->getClass()->name); + JSObject* obj = kind == JS::TraceKind::Object ? static_cast(cell) : nullptr; + + fprintf(stderr, "%s %s", GetCellColorName(cell), GCTraceKindToAscii(kind)); + if (obj) + fprintf(stderr, " %s", obj->getClass()->name); fprintf(stderr, " %p", cell); + if (obj) + fprintf(stderr, " (compartment %p)", obj->compartment()); } void @@ -629,10 +643,8 @@ CheckHeapTracer::check(AutoLockForExclusiveAccess& lock) if (!traceHeap(lock)) return; - if (failures) { - fprintf(stderr, "Heap check: %" PRIuSIZE " failure(s) out of %" PRIu32 " pointers checked\n", - failures, visited.count()); - } + if (failures) + fprintf(stderr, "Heap check: %zu failure(s)\n", failures); MOZ_RELEASE_ASSERT(failures == 0); } @@ -647,7 +659,7 @@ js::gc::CheckHeapAfterGC(JSRuntime* rt) #endif /* JSGC_HASH_TABLE_CHECKS */ -#ifdef DEBUG +#if defined(JS_GC_ZEAL) || defined(DEBUG) class CheckGrayMarkingTracer final : public HeapCheckTracerBase { @@ -675,10 +687,18 @@ CheckGrayMarkingTracer::checkCell(Cell* cell) if (parent->isMarkedBlack() && cell->isMarkedGray()) { failures++; + fprintf(stderr, "Found black to gray edge to "); dumpCellInfo(cell); fprintf(stderr, "\n"); dumpCellPath(); + +#ifdef DEBUG + if (cell->getTraceKind() == JS::TraceKind::Object) { + fprintf(stderr, "\n"); + DumpObject(static_cast(cell), stderr); + } +#endif } } @@ -709,4 +729,4 @@ js::CheckGrayMarkingState(JSContext* cx) return tracer.check(session.lock); } -#endif // DEBUG +#endif // defined(JS_GC_ZEAL) || defined(DEBUG) diff --git a/js/src/gc/Zone.cpp b/js/src/gc/Zone.cpp index fac11ed781..f07c0ac84e 100644 --- a/js/src/gc/Zone.cpp +++ b/js/src/gc/Zone.cpp @@ -6,8 +6,6 @@ #include "gc/Zone.h" -#include "jsgc.h" - #include "gc/Policy.h" #include "jit/BaselineJIT.h" #include "jit/Ion.h" @@ -17,6 +15,7 @@ #include "jscompartmentinlines.h" #include "jsgcinlines.h" +#include "gc/Marking-inl.h" using namespace js; using namespace js::gc; diff --git a/js/src/gc/Zone.h b/js/src/gc/Zone.h index 97306ff6f6..7ffe3818f2 100644 --- a/js/src/gc/Zone.h +++ b/js/src/gc/Zone.h @@ -28,6 +28,9 @@ class JitZone; namespace gc { +class GCSchedulingState; +class GCSchedulingTunables; + // This class encapsulates the data that determines when we need to do a zone GC. class ZoneHeapThreshold { @@ -196,7 +199,7 @@ struct Zone : public JS::shadow::Zone, bool isPreservingCode() const { return gcPreserveCode_; } bool canCollect(); - + void notifyObservingDebuggers(); void setGCState(GCState state) { diff --git a/js/src/gc/ZoneGroup.cpp b/js/src/gc/ZoneGroup.cpp index 6ade20cb33..e6af70425a 100644 --- a/js/src/gc/ZoneGroup.cpp +++ b/js/src/gc/ZoneGroup.cpp @@ -94,6 +94,12 @@ ZoneGroup::leave() ownerContext_ = CooperatingContext(nullptr); } +bool +ZoneGroup::canEnterWithoutYielding(JSContext* cx) +{ + return ownerContext().context() == cx || ownerContext().context() == nullptr; +} + bool ZoneGroup::ownedByCurrentThread() { diff --git a/js/src/gc/ZoneGroup.h b/js/src/gc/ZoneGroup.h index 499f88578a..e695be6dc5 100644 --- a/js/src/gc/ZoneGroup.h +++ b/js/src/gc/ZoneGroup.h @@ -7,8 +7,6 @@ #ifndef gc_ZoneGroup_h #define gc_ZoneGroup_h -#include "jsgc.h" - #include "gc/Statistics.h" #include "vm/Caches.h" #include "vm/Stack.h" @@ -51,6 +49,7 @@ class ZoneGroup void enter(JSContext* cx); void leave(); + bool canEnterWithoutYielding(JSContext* cx); bool ownedByCurrentThread(); // All zones in the group.