diff --git a/js/ipc/JavaScriptChild.cpp b/js/ipc/JavaScriptChild.cpp index 5b2dbb726e..27dbe964b4 100644 --- a/js/ipc/JavaScriptChild.cpp +++ b/js/ipc/JavaScriptChild.cpp @@ -34,7 +34,7 @@ TraceChild(JSTracer* trc, void* data) JavaScriptChild::~JavaScriptChild() { JSContext* cx = dom::danger::GetJSContext(); - JS_RemoveWeakPointerZoneGroupCallback(cx, UpdateChildWeakPointersBeforeSweepingZoneGroup); + JS_RemoveWeakPointerZonesCallback(cx, UpdateChildWeakPointersBeforeSweepingZoneGroup); JS_RemoveExtraGCRootsTracer(cx, TraceChild, this); } @@ -47,7 +47,7 @@ JavaScriptChild::init() return false; JSContext* cx = dom::danger::GetJSContext(); - JS_AddWeakPointerZoneGroupCallback(cx, UpdateChildWeakPointersBeforeSweepingZoneGroup, this); + JS_AddWeakPointerZonesCallback(cx, UpdateChildWeakPointersBeforeSweepingZoneGroup, this); JS_AddExtraGCRootsTracer(cx, TraceChild, this); return true; } diff --git a/js/ipc/JavaScriptShared.h b/js/ipc/JavaScriptShared.h index 9563ae7861..e7a00962c6 100644 --- a/js/ipc/JavaScriptShared.h +++ b/js/ipc/JavaScriptShared.h @@ -7,6 +7,7 @@ #ifndef mozilla_jsipc_JavaScriptShared_h__ #define mozilla_jsipc_JavaScriptShared_h__ +#include "mozilla/HashFunctions.h" #include "mozilla/dom/DOMTypes.h" #include "mozilla/jsipc/CrossProcessObjectWrappers.h" #include "mozilla/jsipc/PJavaScript.h" @@ -71,7 +72,7 @@ struct ObjectIdHasher { typedef ObjectId Lookup; static js::HashNumber hash(const Lookup& l) { - return l.serialize(); + return mozilla::HashGeneric(l.serialize()); } static bool match(const ObjectId& k, const ObjectId& l) { return k == l; diff --git a/js/public/HashTable.h b/js/public/HashTable.h index d0f7dba41f..ea6b3de0be 100644 --- a/js/public/HashTable.h +++ b/js/public/HashTable.h @@ -568,24 +568,16 @@ class HashSet // h.add(p, k); // } -// Pointer hashing policy that strips the lowest zeroBits when calculating the -// hash to improve key distribution. -template +// Pointer hashing policy that uses HashGeneric() to create good hashes for +// pointers. Note that we don't shift out the lowest k bits to generate a +// good distribution for arena allocated pointers. +template struct PointerHasher { typedef Key Lookup; static HashNumber hash(const Lookup& l) { - size_t word = reinterpret_cast(l) >> zeroBits; - static_assert(sizeof(HashNumber) == 4, - "subsequent code assumes a four-byte hash"); -#if JS_BITS_PER_WORD == 32 - return HashNumber(word); -#else - static_assert(sizeof(word) == 8, - "unexpected word size, new hashing strategy required to " - "properly incorporate all bits"); - return HashNumber((word >> 32) ^ word); -#endif + size_t word = reinterpret_cast(l); + return mozilla::HashGeneric(word); } static bool match(const Key& k, const Lookup& l) { return k == l; @@ -619,7 +611,7 @@ struct DefaultHasher // Specialize hashing policy for pointer types. It assumes that the type is // at least word-aligned. For types with smaller size use PointerHasher. template -struct DefaultHasher : PointerHasher::value> +struct DefaultHasher : PointerHasher {}; // Specialize hashing policy for mozilla::UniquePtr to proxy the UniquePtr's @@ -628,7 +620,7 @@ template struct DefaultHasher> { using Lookup = mozilla::UniquePtr; - using PtrHasher = PointerHasher::value>; + using PtrHasher = PointerHasher; static HashNumber hash(const Lookup& l) { return PtrHasher::hash(l.get()); diff --git a/js/public/HeapAPI.h b/js/public/HeapAPI.h index 43c2d4c974..5912a6e535 100644 --- a/js/public/HeapAPI.h +++ b/js/public/HeapAPI.h @@ -63,6 +63,20 @@ const size_t ArenaHeaderSize = sizeof(size_t) + 2 * sizeof(uintptr_t) + static const uint32_t BLACK = 0; static const uint32_t GRAY = 1; +/* + * Two bits determine the mark color as follows: + * BlackBit GrayOrBlackBit color + * 0 0 white + * 0 1 gray + * 1 0 black + * 1 1 black + */ +enum class ColorBit : uint32_t +{ + BlackBit = 0, + GrayOrBlackBit = 1 +}; + /* * The "location" field in the Chunk trailer is a enum indicating various roles * of the chunk. diff --git a/js/public/UbiNode.h b/js/public/UbiNode.h index 3df3a4840b..bdb921fb95 100644 --- a/js/public/UbiNode.h +++ b/js/public/UbiNode.h @@ -9,6 +9,7 @@ #include "mozilla/Alignment.h" #include "mozilla/Assertions.h" #include "mozilla/Attributes.h" +#include "mozilla/HashFunctions.h" #include "mozilla/Maybe.h" #include "mozilla/MemoryReporting.h" #include "mozilla/Move.h" @@ -425,7 +426,7 @@ class StackFrame { using Lookup = JS::ubi::StackFrame; static js::HashNumber hash(const Lookup& lookup) { - return lookup.identifier(); + return mozilla::HashGeneric(lookup.identifier()); } static bool match(const StackFrame& key, const Lookup& lookup) { @@ -813,7 +814,7 @@ class Node { // This simply uses the stock PointerHasher on the ubi::Node's pointer. // We specialize DefaultHasher below to make this the default. class HashPolicy { - typedef js::PointerHasher::value> PtrHash; + typedef js::PointerHasher PtrHash; public: typedef Node Lookup; diff --git a/js/src/builtin/MapObject.cpp b/js/src/builtin/MapObject.cpp index 4e02790cca..91dbb22282 100644 --- a/js/src/builtin/MapObject.cpp +++ b/js/src/builtin/MapObject.cpp @@ -87,7 +87,7 @@ HashValue(const Value& v, const mozilla::HashCodeScrambler& hcs) return hcs.scramble(v.asRawBits()); MOZ_ASSERT(v.isNull() || !v.isGCThing(), "do not reveal pointers via hash codes"); - return v.asRawBits(); + return mozilla::HashGeneric(v.asRawBits()); } HashNumber diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp index 8ecdd49a5b..408abd95e5 100644 --- a/js/src/gc/Allocator.cpp +++ b/js/src/gc/Allocator.cpp @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -253,8 +254,8 @@ GCRuntime::checkIncrementalZoneState(ExclusiveContext* cx, T* t) if (!cx->isJSContext()) return; - Zone* zone = cx->asJSContext()->zone(); - MOZ_ASSERT_IF(t && zone->wasGCStarted() && (zone->shouldMarkInZone() || zone->isGCSweeping()), + Zone* zone = cx->zone(); + MOZ_ASSERT_IF(t && zone->wasGCStarted() && (zone->isGCMarking() || zone->isGCSweeping()), t->asTenured().arena()->allocatedDuringIncremental); #endif } diff --git a/js/src/gc/Barrier.cpp b/js/src/gc/Barrier.cpp index e814d92b6f..81b7c72e0f 100644 --- a/js/src/gc/Barrier.cpp +++ b/js/src/gc/Barrier.cpp @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -36,8 +37,10 @@ IsMarkedBlack(NativeObject* obj) return true; gc::TenuredCell& tenured = obj->asTenured(); - return (tenured.isMarked(gc::BLACK) && !tenured.isMarked(gc::GRAY)) || - tenured.arena()->allocatedDuringIncremental; + if (tenured.isMarkedAny() || tenured.arena()->allocatedDuringIncremental) + return true; + + return false; } bool diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index cf2f7b036a..9e2abc4ca8 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -34,6 +35,7 @@ class AutoMaybeStartBackgroundAllocation; class MarkingValidator; class AutoTraceSession; struct MovingTracer; +class WeakCacheSweepIterator; enum IncrementalProgress { @@ -121,7 +123,14 @@ class GCSchedulingTunables * subsequently invoke the standard OOM machinery, independent of available * physical memory. */ - size_t gcMaxBytes_; + UnprotectedData gcMaxBytes_; + + /* + * Maximum nursery size for each zone group. + * Initially DefaultNurseryBytes and can be set by + * javascript.options.mem.nursery.max_kb + */ + ActiveThreadData gcMaxNurseryBytes_; /* * The base value used to compute zone->trigger.gcBytes(). When @@ -131,7 +140,9 @@ class GCSchedulingTunables size_t gcZoneAllocThresholdBase_; /* Fraction of threshold.gcBytes() which triggers an incremental GC. */ - double zoneAllocThresholdFactor_; + UnprotectedData zoneAllocThresholdFactor_; + /* The same except when doing so would interrupt an already running GC. */ + UnprotectedData zoneAllocThresholdFactorAvoidInterrupt_; /* * Number of bytes to allocate between incremental slices in GCs triggered @@ -185,8 +196,10 @@ class GCSchedulingTunables public: GCSchedulingTunables() : gcMaxBytes_(0), + gcMaxNurseryBytes_(0), gcZoneAllocThresholdBase_(30 * 1024 * 1024), zoneAllocThresholdFactor_(0.9), + zoneAllocThresholdFactorAvoidInterrupt_(0.95), zoneAllocDelayBytes_(1024 * 1024), dynamicHeapGrowthEnabled_(false), highFrequencyThresholdUsec_(1000 * 1000), @@ -202,8 +215,10 @@ class GCSchedulingTunables {} size_t gcMaxBytes() const { return gcMaxBytes_; } + size_t gcMaxNurseryBytes() const { return gcMaxNurseryBytes_; } size_t gcZoneAllocThresholdBase() const { return gcZoneAllocThresholdBase_; } double zoneAllocThresholdFactor() const { return zoneAllocThresholdFactor_; } + double zoneAllocThresholdFactorAvoidInterrupt() const { return zoneAllocThresholdFactorAvoidInterrupt_; } size_t zoneAllocDelayBytes() const { return zoneAllocDelayBytes_; } bool isDynamicHeapGrowthEnabled() const { return dynamicHeapGrowthEnabled_; } uint64_t highFrequencyThresholdUsec() const { return highFrequencyThresholdUsec_; } @@ -217,7 +232,7 @@ class GCSchedulingTunables unsigned minEmptyChunkCount(const AutoLockGC&) const { return minEmptyChunkCount_; } unsigned maxEmptyChunkCount() const { return maxEmptyChunkCount_; } - MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value, const AutoLockGC& lock); + [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value, const AutoLockGC& lock); }; /* @@ -544,8 +559,8 @@ class GCSchedulingState template struct Callback { - F op; - void* data; + ActiveThreadOrGCTaskData op; + ActiveThreadOrGCTaskData data; Callback() : op(nullptr), data(nullptr) @@ -595,6 +610,56 @@ typedef HashMap, SystemAllocPolicy> R using AllocKinds = mozilla::EnumSet; +template +class MemoryCounter +{ + // Bytes counter to measure memory pressure for GC scheduling. It runs + // from maxBytes down to zero. + mozilla::Atomic bytes_; + + // GC trigger threshold for memory allocations. + js::ActiveThreadData maxBytes_; + + // Whether a GC has been triggered as a result of bytes falling below + // zero. + // + // This should be a bool, but Atomic only supports 32-bit and pointer-sized + // types. + mozilla::Atomic triggered_; + + public: + MemoryCounter() + : bytes_(0), + maxBytes_(0), + triggered_(false) + { } + + void reset() { + bytes_ = maxBytes_; + triggered_ = false; + } + + void setMax(size_t newMax) { + // For compatibility treat any value that exceeds PTRDIFF_T_MAX to + // mean that value. + maxBytes_ = (ptrdiff_t(newMax) >= 0) ? newMax : size_t(-1) >> 1; + reset(); + } + + bool update(T* owner, size_t bytes) { + bytes_ -= ptrdiff_t(bytes); + if (MOZ_UNLIKELY(isTooMuchMalloc())) { + if (!triggered_) + triggered_ = owner->triggerGCForTooMuchMalloc(); + } + return triggered_; + } + + ptrdiff_t bytes() const { return bytes_; } + size_t maxBytes() const { return maxBytes_; } + bool isTooMuchMalloc() const { return bytes_ <= 0; } +}; + class GCRuntime { public: @@ -607,10 +672,10 @@ class GCRuntime void removeRoot(Value* vp); void setMarkStackLimit(size_t limit, AutoLockGC& lock); - MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock); + [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock); uint32_t getParameter(JSGCParamKey key, const AutoLockGC& lock); - MOZ_MUST_USE bool triggerGC(JS::gcreason::Reason reason); + [[nodiscard]] bool triggerGC(JS::gcreason::Reason reason); void maybeAllocTriggerZoneGC(Zone* zone, const AutoLockGC& lock); // The return value indicates if we were able to do the GC. bool triggerZoneGC(Zone* zone, JS::gcreason::Reason reason); @@ -749,14 +814,23 @@ class GCRuntime MOZ_MUST_USE bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data); void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data); + bool triggerGCForTooMuchMalloc() { + if (!triggerGC(JS::gcreason::TOO_MUCH_MALLOC)) + return false; + + // Even though this method may be called off the main thread it is safe + // to access mallocCounter here since triggerGC() will return false in + // that case. + stats().recordTrigger(mallocCounter.bytes(), mallocCounter.maxBytes()); + return true; + } + + int32_t getMallocBytes() const { return mallocCounter.bytes(); } + size_t maxMallocBytesAllocated() const { return mallocCounter.maxBytes(); } + bool isTooMuchMalloc() const { return mallocCounter.isTooMuchMalloc(); } + void resetMallocBytes() { mallocCounter.reset(); } void setMaxMallocBytes(size_t value); -#ifdef MOZ_DEVTOOLS_SERVER - int32_t getMallocBytes() const { return mallocBytesUntilGC; } -#endif - void resetMallocBytes(); - bool isTooMuchMalloc() const { return mallocBytesUntilGC <= 0; } void updateMallocCounter(JS::Zone* zone, size_t nbytes); - void onTooMuchMalloc(); void setGCCallback(JSGCCallback callback, void* data); void callGCCallback(JSGCStatus status) const; @@ -765,10 +839,10 @@ class GCRuntime void callObjectsTenuredCallback(); MOZ_MUST_USE bool addFinalizeCallback(JSFinalizeCallback callback, void* data); void removeFinalizeCallback(JSFinalizeCallback func); - MOZ_MUST_USE bool addWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback, + [[nodiscard]] bool addWeakPointerZonesCallback(JSWeakPointerZonesCallback callback, void* data); - void removeWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback); - MOZ_MUST_USE bool addWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback, + void removeWeakPointerZonesCallback(JSWeakPointerZonesCallback callback); + [[nodiscard]] bool addWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback, void* data); void removeWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback); JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback); @@ -779,20 +853,7 @@ class GCRuntime void setFullCompartmentChecks(bool enable); - bool isManipulatingDeadZones() { return manipulatingDeadZones; } - void setManipulatingDeadZones(bool value) { manipulatingDeadZones = value; } - unsigned objectsMarkedInDeadZonesCount() { return objectsMarkedInDeadZones; } - void incObjectsMarkedInDeadZone() { - MOZ_ASSERT(manipulatingDeadZones); - ++objectsMarkedInDeadZones; - } - - JS::Zone* getCurrentZoneGroup() { return currentZoneGroup; } - void setFoundBlackGrayEdges(TenuredCell& target) { - AutoEnterOOMUnsafeRegion oomUnsafe; - if (!foundBlackGrayEdges.append(&target)) - oomUnsafe.crash("OOM|small: failed to insert into foundBlackGrayEdges"); - } + JS::Zone* getCurrentSweepGroup() { return currentSweepGroup; } uint64_t gcNumber() const { return number; } @@ -834,7 +895,7 @@ class GCRuntime const ChunkPool& emptyChunks(const AutoLockGC& lock) const { return emptyChunks_; } typedef ChainedIter NonEmptyChunksIter; NonEmptyChunksIter allNonEmptyChunks() { - return NonEmptyChunksIter(ChunkPool::Iter(availableChunks_), ChunkPool::Iter(fullChunks_)); + return NonEmptyChunksIter(ChunkPool::Iter(availableChunks_.ref()), ChunkPool::Iter(fullChunks_.ref())); } Chunk* getOrAllocChunk(const AutoLockGC& lock, @@ -870,7 +931,24 @@ class GCRuntime static T* tryNewTenuredThing(ExclusiveContext* cx, AllocKind kind, size_t thingSize); static TenuredCell* refillFreeListInGC(Zone* zone, AllocKind thingKind); + void bufferGrayRoots(); + + /* + * Concurrent sweep infrastructure. + */ + void startTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked); + void joinTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked); + + // Delete an empty zone group after its contents have been merged. + void deleteEmptyZoneGroup(ZoneGroup* group); + private: + enum IncrementalResult + { + Reset = 0, + Ok + }; + // For ArenaLists::allocateFromArena() friend class ArenaLists; Chunk* pickChunk(const AutoLockGC& lock, @@ -926,14 +1004,15 @@ class GCRuntime AutoLockForExclusiveAccess& lock); void purgeRuntime(AutoLockForExclusiveAccess& lock); - MOZ_MUST_USE bool beginMarkPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); + [[nodiscard]] bool beginMarkPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); + bool prepareZonesForCollection(JS::gcreason::Reason reason, bool* isFullOut, + AutoLockForExclusiveAccess& lock); bool shouldPreserveJITCode(JSCompartment* comp, int64_t currentTime, JS::gcreason::Reason reason, bool canAllocateMoreCode); void traceRuntimeForMajorGC(JSTracer* trc, AutoLockForExclusiveAccess& lock); void traceRuntimeAtoms(JSTracer* trc, AutoLockForExclusiveAccess& lock); void traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark, AutoLockForExclusiveAccess& lock); - void bufferGrayRoots(); void maybeDoCycleCollection(); void markCompartments(); IncrementalProgress drainMarkStack(SliceBudget& sliceBudget, gcstats::Phase phase); @@ -945,23 +1024,31 @@ class GCRuntime void markAllWeakReferences(gcstats::Phase phase); void markAllGrayReferences(gcstats::Phase phase); - void beginSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock); - void findZoneGroups(AutoLockForExclusiveAccess& lock); - MOZ_MUST_USE bool findInterZoneEdges(); - void getNextZoneGroup(); - void endMarkingZoneGroup(); - void beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock); + void beginSweepPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); + void groupZonesForSweeping(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); + [[nodiscard]] bool findInterZoneEdges(); + void getNextSweepGroup(); + void endMarkingSweepGroup(); + void beginSweepingSweepGroup(); bool shouldReleaseObservedTypes(); - void endSweepingZoneGroup(); - IncrementalProgress performSweepActions(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock); - static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, Zone* zone, - SliceBudget& budget, AllocKind kind); - static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, Zone* zone, - SliceBudget& budget, AllocKind kind); - static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, Zone* zone, - SliceBudget& budget, AllocKind kind); - static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, Zone* zone, - SliceBudget& budget, AllocKind kind); + void sweepDebuggerOnMainThread(FreeOp* fop); + void sweepJitDataOnMainThread(FreeOp* fop); + void endSweepingSweepGroup(); + IncrementalProgress performSweepActions(SliceBudget& sliceBudget, + AutoLockForExclusiveAccess& lock); + static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, + Zone* zone); + static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, + Zone* zone); + static IncrementalProgress sweepAtomsTable(GCRuntime* gc, FreeOp* fop, SliceBudget& budget); + void startSweepingAtomsTable(); + IncrementalProgress sweepAtomsTable(SliceBudget& budget); + static IncrementalProgress sweepWeakCaches(GCRuntime* gc, FreeOp* fop, SliceBudget& budget); + IncrementalProgress sweepWeakCaches(SliceBudget& budget); + static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, + Zone* zone, AllocKind kind); + static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, + Zone* zone); void endSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock); void sweepZones(FreeOp* fop, bool lastGC); void decommitAllWithoutUnlocking(const AutoLockGC& lock); @@ -993,7 +1080,7 @@ class GCRuntime #endif void callFinalizeCallbacks(FreeOp* fop, JSFinalizeStatus status) const; - void callWeakPointerZoneGroupCallbacks() const; + void callWeakPointerZonesCallbacks() const; void callWeakPointerCompartmentCallbacks(JSCompartment* comp) const; public: @@ -1002,8 +1089,11 @@ class GCRuntime /* Embedders can use this zone however they wish. */ JS::Zone* systemZone; - /* List of compartments and zones (protected by the GC lock). */ - ZoneVector zones; + // List of all zone groups (protected by the GC lock). + private: + ActiveThreadOrGCTaskData groups_; + public: + ZoneGroupVector& groups() { return groups_.ref(); } Nursery nursery; StoreBuffer storeBuffer; @@ -1034,11 +1124,11 @@ class GCRuntime // to the fullChunks pool. During a GC, if all arenas are free, the chunk // is moved back to the emptyChunks pool and scheduled for eventual // release. - ChunkPool availableChunks_; + UnprotectedData availableChunks_; // When all arenas in a chunk are used, it is moved to the fullChunks pool // so as to reduce the cost of operations on the available lists. - ChunkPool fullChunks_; + UnprotectedData fullChunks_; RootedValueMap rootsHash; @@ -1075,7 +1165,7 @@ class GCRuntime Okay, Failed }; - GrayBufferState grayBufferState; + ActiveThreadOrGCTaskData grayBufferState; bool hasBufferedGrayRoots() const { return grayBufferState == GrayBufferState::Okay; } // Clear each zone's gray buffers, but do not change the current state. @@ -1106,9 +1196,6 @@ class GCRuntime /* Incremented on every GC slice. */ uint64_t number; - /* The number at the time of the most recent GC's first slice. */ - uint64_t startNumber; - /* Whether the currently running GC can finish in multiple slices. */ bool isIncremental; @@ -1138,10 +1225,10 @@ class GCRuntime * The current incremental GC phase. This is also used internally in * non-incremental GC. */ - State incrementalState; + ActiveThreadOrGCTaskData incrementalState; /* Indicates that the last incremental slice exhausted the mark stack. */ - bool lastMarkSlice; + ActiveThreadData lastMarkSlice; /* Whether any sweeping will take place in the separate GC helper thread. */ bool sweepOnBackgroundThread; @@ -1149,11 +1236,8 @@ class GCRuntime /* Whether observed type information is being released in the current GC. */ bool releaseObservedTypes; - /* Whether any black->gray edges were found during marking. */ - BlackGrayEdgeVector foundBlackGrayEdges; - - /* Singly linekd list of zones to be swept in the background. */ - ZoneList backgroundSweepZones; + /* Singly linked list of zones to be swept in the background. */ + ActiveThreadOrGCTaskData backgroundSweepZones; /* * Free LIFO blocks are transferred to this allocator before being freed on @@ -1161,14 +1245,9 @@ class GCRuntime */ LifoAlloc blocksToFreeAfterSweeping; - /* - * Free LIFO blocks are transferred to this allocator before being freed - * after minor GC. - */ - LifoAlloc blocksToFreeAfterMinorGC; - - /* Index of current zone group (for stats). */ - unsigned zoneGroupIndex; + private: + /* Index of current sweep group (for stats). */ + ActiveThreadData sweepGroupIndex; /* * Incremental sweep state. @@ -1180,13 +1259,15 @@ class GCRuntime size_t sweepActionIndex; bool abortSweepAfterCurrentGroup; - /* - * Concurrent sweep infrastructure. - */ - void startTask(GCParallelTask& task, gcstats::Phase phase, - AutoLockHelperThreadState& locked); - void joinTask(GCParallelTask& task, gcstats::Phase phase, - AutoLockHelperThreadState& locked); + ActiveThreadData sweepGroups; + ActiveThreadOrGCTaskData currentSweepGroup; + ActiveThreadData>> sweepActions; + ActiveThreadOrGCTaskData sweepZone; + ActiveThreadData> maybeAtomsToSweep; + ActiveThreadOrGCTaskData sweepCache; + ActiveThreadData abortSweepAfterCurrentGroup; + + friend class WeakCacheSweepIterator; /* * List head of arenas allocated during the sweep phase. @@ -1257,20 +1338,10 @@ class GCRuntime Callback gcDoCycleCollectionCallback; Callback tenuredCallback; CallbackVector finalizeCallbacks; - CallbackVector updateWeakPointerZoneGroupCallbacks; + CallbackVector updateWeakPointerZonesCallbacks; CallbackVector updateWeakPointerCompartmentCallbacks; - /* - * Malloc counter to measure memory pressure for GC scheduling. It runs - * from maxMallocBytes down to zero. - */ - mozilla::Atomic mallocBytesUntilGC; - - /* - * Whether a GC has been triggered as a result of mallocBytesUntilGC - * falling below zero. - */ - mozilla::Atomic mallocGCTriggered; + MemoryCounter mallocCounter; /* * The trace operations to trace embedding-specific GC roots. One is for diff --git a/js/src/gc/GenerateStatsPhases.py b/js/src/gc/GenerateStatsPhases.py new file mode 100644 index 0000000000..53a465569c --- /dev/null +++ b/js/src/gc/GenerateStatsPhases.py @@ -0,0 +1,327 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Generate graph structures for GC statistics recording. +# +# Stats phases are nested and form a directed acyclic graph starting +# from a set of root phases. Importantly, a phase may appear under more +# than one parent phase. +# +# For example, the following arrangement is possible: +# +# +---+ +# | A | +# +---+ +# | +# +-------+-------+ +# | | | +# v v v +# +---+ +---+ +---+ +# | B | | C | | D | +# +---+ +---+ +---+ +# | | +# +---+---+ +# | +# v +# +---+ +# | E | +# +---+ +# +# This graph is expanded into a tree (or really a forest) and phases +# with multiple parents are duplicated. +# +# For example, the input example above would be expanded to: +# +# +---+ +# | A | +# +---+ +# | +# +-------+-------+ +# | | | +# v v v +# +---+ +---+ +---+ +# | B | | C | | D | +# +---+ +---+ +---+ +# | | +# v v +# +---+ +---+ +# | E | | E'| +# +---+ +---+ + +import re +import sys +import collections + +class PhaseKind(): + def __init__(self, name, descr, bucket, children = []): + self.name = name + self.descr = descr + self.bucket = bucket + self.children = children + +# The root marking phase appears in several places in the graph. +MarkRootsPhaseKind = PhaseKind("MARK_ROOTS", "Mark Roots", 48, [ + PhaseKind("MARK_CCWS", "Mark Cross Compartment Wrappers", 50), + PhaseKind("MARK_STACK", "Mark C and JS stacks", 51), + PhaseKind("MARK_RUNTIME_DATA", "Mark Runtime-wide Data", 52), + PhaseKind("MARK_EMBEDDING", "Mark Embedding", 53), + PhaseKind("MARK_COMPARTMENTS", "Mark Compartments", 54) +]) + +JoinParallelTasksPhaseKind = PhaseKind("JOIN_PARALLEL_TASKS", "Join Parallel Tasks", 67) + +UnmarkGrayPhaseKind = PhaseKind("UNMARK_GRAY", "Unmark gray", 56) + +PhaseKindGraphRoots = [ + PhaseKind("MUTATOR", "Mutator Running", 0), + PhaseKind("GC_BEGIN", "Begin Callback", 1), + PhaseKind("EVICT_NURSERY_FOR_MAJOR_GC", "Evict Nursery For Major GC", 70, [ + MarkRootsPhaseKind, + ]), + PhaseKind("WAIT_BACKGROUND_THREAD", "Wait Background Thread", 2), + PhaseKind("PREPARE", "Prepare For Collection", 69, [ + PhaseKind("UNMARK", "Unmark", 7), + PhaseKind("BUFFER_GRAY_ROOTS", "Buffer Gray Roots", 49), + PhaseKind("MARK_DISCARD_CODE", "Mark Discard Code", 3), + PhaseKind("RELAZIFY_FUNCTIONS", "Relazify Functions", 4), + PhaseKind("PURGE", "Purge", 5), + PhaseKind("PURGE_SHAPE_TABLES", "Purge ShapeTables", 60), + JoinParallelTasksPhaseKind + ]), + PhaseKind("MARK", "Mark", 6, [ + MarkRootsPhaseKind, + UnmarkGrayPhaseKind, + PhaseKind("MARK_DELAYED", "Mark Delayed", 8) + ]), + PhaseKind("SWEEP", "Sweep", 9, [ + PhaseKind("SWEEP_MARK", "Mark During Sweeping", 10, [ + UnmarkGrayPhaseKind, + PhaseKind("SWEEP_MARK_INCOMING_BLACK", "Mark Incoming Black Pointers", 12, [ + UnmarkGrayPhaseKind, + ]), + PhaseKind("SWEEP_MARK_WEAK", "Mark Weak", 13, [ + UnmarkGrayPhaseKind, + ]), + PhaseKind("SWEEP_MARK_INCOMING_GRAY", "Mark Incoming Gray Pointers", 14), + PhaseKind("SWEEP_MARK_GRAY", "Mark Gray", 15), + PhaseKind("SWEEP_MARK_GRAY_WEAK", "Mark Gray and Weak", 16) + ]), + PhaseKind("FINALIZE_START", "Finalize Start Callbacks", 17, [ + PhaseKind("WEAK_ZONES_CALLBACK", "Per-Slice Weak Callback", 57), + PhaseKind("WEAK_COMPARTMENT_CALLBACK", "Per-Compartment Weak Callback", 58) + ]), + PhaseKind("UPDATE_ATOMS_BITMAP", "Sweep Atoms Bitmap", 68), + PhaseKind("SWEEP_ATOMS_TABLE", "Sweep Atoms Table", 18), + PhaseKind("SWEEP_COMPARTMENTS", "Sweep Compartments", 20, [ + PhaseKind("SWEEP_DISCARD_CODE", "Sweep Discard Code", 21), + PhaseKind("SWEEP_INNER_VIEWS", "Sweep Inner Views", 22), + PhaseKind("SWEEP_CC_WRAPPER", "Sweep Cross Compartment Wrappers", 23), + PhaseKind("SWEEP_BASE_SHAPE", "Sweep Base Shapes", 24), + PhaseKind("SWEEP_INITIAL_SHAPE", "Sweep Initial Shapes", 25), + PhaseKind("SWEEP_TYPE_OBJECT", "Sweep Type Objects", 26), + PhaseKind("SWEEP_BREAKPOINT", "Sweep Breakpoints", 27), + PhaseKind("SWEEP_REGEXP", "Sweep Regexps", 28), + PhaseKind("SWEEP_COMPRESSION", "Sweep Compression Tasks", 62), + PhaseKind("SWEEP_WEAKMAPS", "Sweep WeakMaps", 63), + PhaseKind("SWEEP_UNIQUEIDS", "Sweep Unique IDs", 64), + PhaseKind("SWEEP_JIT_DATA", "Sweep JIT Data", 65), + PhaseKind("SWEEP_WEAK_CACHES", "Sweep Weak Caches", 66), + PhaseKind("SWEEP_MISC", "Sweep Miscellaneous", 29), + PhaseKind("SWEEP_TYPES", "Sweep type information", 30, [ + PhaseKind("SWEEP_TYPES_BEGIN", "Sweep type tables and compilations", 31), + PhaseKind("SWEEP_TYPES_END", "Free type arena", 32), + ]), + JoinParallelTasksPhaseKind + ]), + PhaseKind("SWEEP_OBJECT", "Sweep Object", 33), + PhaseKind("SWEEP_STRING", "Sweep String", 34), + PhaseKind("SWEEP_SCRIPT", "Sweep Script", 35), + PhaseKind("SWEEP_SCOPE", "Sweep Scope", 59), + PhaseKind("SWEEP_REGEXP_SHARED", "Sweep RegExpShared", 61), + PhaseKind("SWEEP_SHAPE", "Sweep Shape", 36), + PhaseKind("SWEEP_JITCODE", "Sweep JIT code", 37), + PhaseKind("FINALIZE_END", "Finalize End Callback", 38), + PhaseKind("DESTROY", "Deallocate", 39), + JoinParallelTasksPhaseKind + ]), + PhaseKind("COMPACT", "Compact", 40, [ + PhaseKind("COMPACT_MOVE", "Compact Move", 41), + PhaseKind("COMPACT_UPDATE", "Compact Update", 42, [ + MarkRootsPhaseKind, + PhaseKind("COMPACT_UPDATE_CELLS", "Compact Update Cells", 43), + JoinParallelTasksPhaseKind + ]), + ]), + PhaseKind("GC_END", "End Callback", 44), + PhaseKind("MINOR_GC", "All Minor GCs", 45, [ + MarkRootsPhaseKind, + ]), + PhaseKind("EVICT_NURSERY", "Minor GCs to Evict Nursery", 46, [ + MarkRootsPhaseKind, + ]), + PhaseKind("TRACE_HEAP", "Trace Heap", 47, [ + MarkRootsPhaseKind, + ]), + PhaseKind("BARRIER", "Barriers", 55, [ + UnmarkGrayPhaseKind + ]) +] + +# Make a linear list of all unique phases by performing a depth first +# search on the phase graph starting at the roots. This will be used to +# generate the PhaseKind enum. + +def findAllPhaseKinds(): + phases = [] + seen = set() + + def dfs(phase): + if phase in seen: + return + phases.append(phase) + seen.add(phase) + for child in phase.children: + dfs(child) + + for phase in PhaseKindGraphRoots: + dfs(phase) + return phases + +AllPhaseKinds = findAllPhaseKinds() + +# Expand the DAG into a tree, duplicating phases which have more than +# one parent. + +class Phase: + def __init__(self, phaseKind, parent): + self.phaseKind = phaseKind + self.parent = parent + self.depth = parent.depth + 1 if parent else 0 + self.children = [] + self.nextSibling = None + self.nextInPhaseKind = None + + self.path = re.sub(r'\W+', '_', phaseKind.name.lower()) + if parent is not None: + self.path = parent.path + '.' + self.path + +def expandPhases(): + phases = [] + phasesForKind = collections.defaultdict(list) + + def traverse(phaseKind, parent): + ep = Phase(phaseKind, parent) + phases.append(ep) + + # Update list of expanded phases for this phase kind. + if phasesForKind[phaseKind]: + phasesForKind[phaseKind][-1].nextInPhaseKind = ep + phasesForKind[phaseKind].append(ep) + + # Recurse over children. + for child in phaseKind.children: + child_ep = traverse(child, ep) + if ep.children: + ep.children[-1].nextSibling = child_ep + ep.children.append(child_ep) + return ep + + for phaseKind in PhaseKindGraphRoots: + traverse(phaseKind, None) + + return phases, phasesForKind + +AllPhases, PhasesForPhaseKind = expandPhases() + +# Name phases based on phase kind name and index if there are multiple phases +# corresponding to a single phase kind. + +for phaseKind in AllPhaseKinds: + phases = PhasesForPhaseKind[phaseKind] + if len(phases) == 1: + phases[0].name = "%s" % phaseKind.name + else: + for index, phase in enumerate(phases): + phase.name = "%s_%d" % (phaseKind.name, index + 1) + +# Find the maximum phase nesting. + +MaxPhaseNesting = max(phase.depth for phase in AllPhases) + 1 + +# Generate code. + +def writeList(out, items): + if items: + out.write(",\n".join(" " + item for item in items) + "\n") + +def writeEnumClass(out, name, type, items, extraItems): + items = [ "FIRST" ] + items + [ "LIMIT" ] + extraItems + items[1] += " = " + items[0] + out.write("enum class %s : %s {\n" % (name, type)); + writeList(out, items) + out.write("};\n") + +def generateHeader(out): + # + # Generate PhaseKind enum. + # + phaseKindNames = map(lambda phaseKind: phaseKind.name, AllPhaseKinds) + extraPhaseKinds = [ + "NONE = LIMIT", + "EXPLICIT_SUSPENSION = LIMIT", + "IMPLICIT_SUSPENSION" + ] + writeEnumClass(out, "PhaseKind", "uint8_t", phaseKindNames, extraPhaseKinds) + out.write("\n") + + # + # Generate Phase enum. + # + phaseNames = map(lambda phase: phase.name, AllPhases) + extraPhases = [ + "NONE = LIMIT", + "EXPLICIT_SUSPENSION = LIMIT", + "IMPLICIT_SUSPENSION" + ] + writeEnumClass(out, "Phase", "uint8_t", phaseNames, extraPhases) + out.write("\n") + + # + # Generate MAX_PHASE_NESTING constant. + # + out.write("static const size_t MAX_PHASE_NESTING = %d;\n" % MaxPhaseNesting) + +def generateCpp(out): + # + # Generate the PhaseKindInfo table. + # + out.write("static const PhaseKindTable phaseKinds = {\n") + for phaseKind in AllPhaseKinds: + phase = PhasesForPhaseKind[phaseKind][0] + out.write(" /* PhaseKind::%s */ PhaseKindInfo { Phase::%s, %d },\n" % + (phaseKind.name, phase.name, phaseKind.bucket)) + out.write("};\n") + out.write("\n") + + # + # Generate the PhaseInfo tree. + # + def name(phase): + return "Phase::" + phase.name if phase else "Phase::NONE" + + out.write("static const PhaseTable phases = {\n") + for phase in AllPhases: + firstChild = phase.children[0] if phase.children else None + phaseKind = phase.phaseKind + out.write(" /* %s */ PhaseInfo { %s, %s, %s, %s, PhaseKind::%s, %d, \"%s\", \"%s\" },\n" % + (name(phase), + name(phase.parent), + name(firstChild), + name(phase.nextSibling), + name(phase.nextInPhaseKind), + phaseKind.name, + phase.depth, + phaseKind.descr, + phase.path)) + out.write("};\n") diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h index 9a8a26d579..58357fdcd7 100644 --- a/js/src/gc/Heap.h +++ b/js/src/gc/Heap.h @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -54,10 +55,6 @@ extern bool CurrentThreadIsIonCompiling(); #endif -// The return value indicates if anything was unmarked. -extern bool -UnmarkGrayCellRecursively(gc::Cell* cell, JS::TraceKind kind); - extern void TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name); @@ -250,6 +247,13 @@ FOR_EACH_ALLOCKIND(EXPAND_ELEMENT) static const size_t MAX_BACKGROUND_FINALIZE_KINDS = size_t(AllocKind::LIMIT) - size_t(AllocKind::OBJECT_LIMIT) / 2; +/* Mark colors to pass to markIfUnmarked. */ +enum class MarkColor : uint32_t +{ + Black = 0, + Gray +}; + class TenuredCell; // A GC cell is the base class for all GC things. @@ -260,8 +264,11 @@ struct Cell MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const; MOZ_ALWAYS_INLINE TenuredCell& asTenured(); - inline JSRuntime* runtimeFromMainThread() const; - inline JS::shadow::Runtime* shadowRuntimeFromMainThread() const; + MOZ_ALWAYS_INLINE bool isMarkedAny() const; + MOZ_ALWAYS_INLINE bool isMarkedBlack() const; + MOZ_ALWAYS_INLINE bool isMarkedGray() const; + + inline JSRuntime* runtimeFromActiveCooperatingThread() const; // Note: Unrestricted access to the runtime of a GC thing from an arbitrary // thread can easily lead to races. Use this method very carefully. @@ -294,6 +301,7 @@ struct Cell return static_cast(this); } + #ifdef DEBUG inline bool isAligned() const; void dump(FILE* fp) const; @@ -315,10 +323,13 @@ class TenuredCell : public Cell static MOZ_ALWAYS_INLINE const TenuredCell* fromPointer(const void* ptr); // Mark bit management. - MOZ_ALWAYS_INLINE bool isMarked(uint32_t color = BLACK) const; + MOZ_ALWAYS_INLINE bool isMarkedAny() const; + MOZ_ALWAYS_INLINE bool isMarkedBlack() const; + MOZ_ALWAYS_INLINE bool isMarkedGray() const; + // The return value indicates if the cell went from unmarked to marked. - MOZ_ALWAYS_INLINE bool markIfUnmarked(uint32_t color = BLACK) const; - MOZ_ALWAYS_INLINE void unmark(uint32_t color) const; + MOZ_ALWAYS_INLINE bool markIfUnmarked(MarkColor color = MarkColor::Black) const; + MOZ_ALWAYS_INLINE void markBlack() const; MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src); // Access to the arena. @@ -881,6 +892,15 @@ static_assert(ArenasPerChunk == 62, "Do not accidentally change our heap's densi static_assert(ArenasPerChunk == 252, "Do not accidentally change our heap's density."); #endif +static inline void +AssertValidColorBit(const TenuredCell* thing, ColorBit colorBit) +{ +#ifdef DEBUG + Arena* arena = thing->arena(); + MOZ_ASSERT(unsigned(colorBit) < arena->getThingSize() / CellBytesPerMarkBit); +#endif +} + /* A chunk bitmap contains enough mark bits for all the cells in a chunk. */ struct ChunkBitmap { @@ -889,26 +909,40 @@ struct ChunkBitmap public: ChunkBitmap() { } - MOZ_ALWAYS_INLINE void getMarkWordAndMask(const Cell* cell, uint32_t color, + MOZ_ALWAYS_INLINE void getMarkWordAndMask(const TenuredCell* cell, ColorBit colorBit, uintptr_t** wordp, uintptr_t* maskp) { - detail::GetGCThingMarkWordAndMask(uintptr_t(cell), color, wordp, maskp); + detail::GetGCThingMarkWordAndMask(uintptr_t(cell), colorBit, wordp, maskp); } - MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarked(const Cell* cell, uint32_t color) { + MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool markBit(const TenuredCell* cell, ColorBit colorBit) { + AssertValidColorBit(cell, colorBit); uintptr_t* word, mask; - getMarkWordAndMask(cell, color, &word, &mask); + getMarkWordAndMask(cell, colorBit, &word, &mask); return *word & mask; } + MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarkedAny(const TenuredCell* cell) { + return markBit(cell, ColorBit::BlackBit) || markBit(cell, ColorBit::GrayOrBlackBit); + } + + MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarkedBlack(const TenuredCell* cell) { + return markBit(cell, ColorBit::BlackBit); + } + + MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarkedGray(const TenuredCell* cell) { + return !markBit(cell, ColorBit::BlackBit) && markBit(cell, ColorBit::GrayOrBlackBit); + } + // The return value indicates if the cell went from unmarked to marked. - MOZ_ALWAYS_INLINE bool markIfUnmarked(const Cell* cell, uint32_t color) { + MOZ_ALWAYS_INLINE bool markIfUnmarked(const TenuredCell* cell, MarkColor color) { uintptr_t* word, mask; getMarkWordAndMask(cell, BLACK, &word, &mask); if (*word & mask) return false; - *word |= mask; - if (color != BLACK) { + if (color == MarkColor::Black) { + *word |= mask; + } else { /* * We use getMarkWordAndMask to recalculate both mask and word as * doing just mask << color may overflow the mask. @@ -921,16 +955,23 @@ struct ChunkBitmap return true; } - MOZ_ALWAYS_INLINE void unmark(const Cell* cell, uint32_t color) { + MOZ_ALWAYS_INLINE void markBlack(const TenuredCell* cell) { uintptr_t* word, mask; getMarkWordAndMask(cell, color, &word, &mask); *word &= ~mask; } - MOZ_ALWAYS_INLINE void copyMarkBit(Cell* dst, const TenuredCell* src, uint32_t color) { - uintptr_t* word, mask; - getMarkWordAndMask(dst, color, &word, &mask); - *word = (*word & ~mask) | (src->isMarked(color) ? mask : 0); + MOZ_ALWAYS_INLINE void copyMarkBit(TenuredCell* dst, const TenuredCell* src, + ColorBit colorBit) { + uintptr_t* srcWord, srcMask; + getMarkWordAndMask(src, colorBit, &srcWord, &srcMask); + + uintptr_t* dstWord, dstMask; + getMarkWordAndMask(dst, colorBit, &dstWord, &dstMask); + + *dstWord &= ~dstMask; + if (*srcWord & srcMask) + *dstWord |= dstMask; } void clear() { @@ -944,7 +985,8 @@ struct ChunkBitmap "that covers bits from two arenas."); uintptr_t* word, unused; - getMarkWordAndMask(reinterpret_cast(arena->address()), BLACK, &word, &unused); + getMarkWordAndMask(reinterpret_cast(arena->address()), + ColorBit::BlackBit, &word, &unused); return word; } }; @@ -1120,15 +1162,6 @@ Arena::chunk() const return Chunk::fromAddress(address()); } -static void -AssertValidColor(const TenuredCell* thing, uint32_t color) -{ -#ifdef DEBUG - Arena* arena = thing->arena(); - MOZ_ASSERT(color < arena->getThingSize() / CellSize); -#endif -} - MOZ_ALWAYS_INLINE const TenuredCell& Cell::asTenured() const { @@ -1143,6 +1176,24 @@ Cell::asTenured() return *static_cast(this); } +MOZ_ALWAYS_INLINE bool +Cell::isMarkedAny() const +{ + return !isTenured() || asTenured().isMarkedAny(); +} + +MOZ_ALWAYS_INLINE bool +Cell::isMarkedBlack() const +{ + return !isTenured() || asTenured().isMarkedBlack(); +} + +MOZ_ALWAYS_INLINE bool +Cell::isMarkedGray() const +{ + return isTenured() && asTenured().isMarkedGray(); +} + inline JSRuntime* Cell::runtimeFromMainThread() const { @@ -1227,17 +1278,29 @@ TenuredCell::fromPointer(const void* ptr) } bool -TenuredCell::isMarked(uint32_t color /* = BLACK */) const +TenuredCell::isMarkedAny() const { MOZ_ASSERT(arena()->allocated()); - AssertValidColor(this, color); - return chunk()->bitmap.isMarked(this, color); + return chunk()->bitmap.isMarkedAny(this); } bool -TenuredCell::markIfUnmarked(uint32_t color /* = BLACK */) const +TenuredCell::isMarkedBlack() const +{ + MOZ_ASSERT(arena()->allocated()); + return chunk()->bitmap.isMarkedBlack(this); +} + +bool +TenuredCell::isMarkedGray() const +{ + MOZ_ASSERT(arena()->allocated()); + return chunk()->bitmap.isMarkedGray(this); +} + +bool +TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const { - AssertValidColor(this, color); return chunk()->bitmap.markIfUnmarked(this, color); } @@ -1320,11 +1383,11 @@ TenuredCell::readBarrier(TenuredCell* thing) MOZ_ASSERT(tmp == thing); } - if (thing->isMarked(GRAY)) { - // There shouldn't be anything marked grey unless we're on the main thread. + if (thing->isMarkedGray()) { + // There shouldn't be anything marked grey unless we're on the active thread. MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())); - if (!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)) - UnmarkGrayCellRecursively(thing, thing->getTraceKind()); + if (!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone)) + JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(thing, thing->getTraceKind())); } } @@ -1380,6 +1443,52 @@ static const int32_t ChunkLocationOffsetFromLastByte = int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask); } /* namespace gc */ + +namespace debug { + +// Utility functions meant to be called from an interactive debugger. +enum class MarkInfo : int { + BLACK = 0, + GRAY = 1, + UNMARKED = -1, + NURSERY = -2, +}; + +// Get the mark color for a cell, in a way easily usable from a debugger. +MOZ_NEVER_INLINE MarkInfo +GetMarkInfo(js::gc::Cell* cell); + +// Sample usage from gdb: +// +// (gdb) p $word = js::debug::GetMarkWordAddress(obj) +// $1 = (uintptr_t *) 0x7fa56d5fe360 +// (gdb) p/x $mask = js::debug::GetMarkMask(obj, js::gc::GRAY) +// $2 = 0x200000000 +// (gdb) watch *$word +// Hardware watchpoint 7: *$word +// (gdb) cond 7 *$word & $mask +// (gdb) cont +// +// Note that this is *not* a watchpoint on a single bit. It is a watchpoint on +// the whole word, which will trigger whenever the word changes and the +// selected bit is set after the change. +// +// So if the bit changing is the desired one, this is exactly what you want. +// But if a different bit changes (either set or cleared), you may still stop +// execution if the $mask bit happened to already be set. gdb does not expose +// enough information to restrict the watchpoint to just a single bit. + +// Return the address of the word containing the mark bits for the given cell, +// or nullptr if the cell is in the nursery. +MOZ_NEVER_INLINE uintptr_t* +GetMarkWordAddress(js::gc::Cell* cell); + +// Return the mask for the given cell and color bit, or 0 if the cell is in the +// nursery. +MOZ_NEVER_INLINE uintptr_t +GetMarkMask(js::gc::Cell* cell, uint32_t colorBit); + +} /* namespace debug */ } /* namespace js */ #endif /* gc_Heap_h */ diff --git a/js/src/gc/Iteration.cpp b/js/src/gc/Iteration.cpp index 2d69faa7a9..3d5be9b9f3 100644 --- a/js/src/gc/Iteration.cpp +++ b/js/src/gc/Iteration.cpp @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -75,8 +76,8 @@ js::IterateChunks(JSContext* cx, void* data, IterateChunkCallback chunkCallback) { AutoPrepareForTracing prep(cx, SkipAtoms); - for (auto chunk = cx->gc.allNonEmptyChunks(); !chunk.done(); chunk.next()) - chunkCallback(cx, data, chunk); + for (auto chunk = cx->runtime()->gc.allNonEmptyChunks(); !chunk.done(); chunk.next()) + chunkCallback(cx->runtime(), data, chunk); } void @@ -106,7 +107,7 @@ IterateGrayObjects(Zone* zone, GCThingCallback cellCallback, void* data) { for (auto kind : ObjectAllocKinds()) { for (GrayObjectIter obj(zone, kind); !obj.done(); obj.next()) { - if (obj->asTenured().isMarked(GRAY)) + if (obj->asTenured().isMarkedGray()) cellCallback(data, JS::GCCellPtr(obj.get())); } } diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp index 0171244436..2e98b9c992 100644 --- a/js/src/gc/Marking.cpp +++ b/js/src/gc/Marking.cpp @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -24,6 +25,8 @@ #include "vm/BigIntType.h" #include "vm/Debugger.h" #include "vm/EnvironmentObject.h" +#include "vm/RegExpObject.h" +#include "vm/RegExpShared.h" #include "vm/Scope.h" #include "vm/Shape.h" #include "vm/Symbol.h" @@ -216,8 +219,10 @@ js::CheckTracedThing(JSTracer* trc, T* thing) Zone* zone = thing->zoneFromAnyThread(); JSRuntime* rt = trc->runtime(); - MOZ_ASSERT_IF(!IsMovingTracer(trc), CurrentThreadCanAccessZone(zone)); - MOZ_ASSERT_IF(!IsMovingTracer(trc), CurrentThreadCanAccessRuntime(rt)); + if (!IsMovingTracer(trc) && !IsBufferGrayRootsTracer(trc) && !IsClearEdgesTracer(trc)) { + MOZ_ASSERT(CurrentThreadCanAccessZone(zone)); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + } MOZ_ASSERT(zone->runtimeFromAnyThread() == trc->runtime()); @@ -231,14 +236,15 @@ js::CheckTracedThing(JSTracer* trc, T* thing) */ bool isGcMarkingTracer = trc->isMarkingTracer(); - MOZ_ASSERT_IF(zone->requireGCTracer(), isGcMarkingTracer || IsBufferGrayRootsTracer(trc)); + MOZ_ASSERT_IF(zone->requireGCTracer(), + isGcMarkingTracer || IsBufferGrayRootsTracer(trc) || IsUnmarkGrayTracer(trc)); if (isGcMarkingTracer) { GCMarker* gcMarker = static_cast(trc); MOZ_ASSERT_IF(gcMarker->shouldCheckCompartments(), zone->isCollecting() || zone->isAtomsZone()); - MOZ_ASSERT_IF(gcMarker->markColor() == GRAY, + MOZ_ASSERT_IF(gcMarker->markColor() == MarkColor::Gray, !zone->isGCMarkingBlack() || zone->isAtomsZone()); MOZ_ASSERT(!(zone->isGCSweeping() || zone->isGCFinished() || zone->isGCCompacting())); @@ -281,35 +287,39 @@ JS_FOR_EACH_TRACEKIND(IMPL_CHECK_TRACED_THING); #undef IMPL_CHECK_TRACED_THING } // namespace js +static bool UnmarkGrayGCThing(JSRuntime* rt, JS::GCCellPtr thing); + static bool ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, Cell* cell) { if (!trc->isMarkingTracer()) return true; - uint32_t color = static_cast(trc)->markColor(); - MOZ_ASSERT(color == BLACK || color == GRAY); + MarkColor color = GCMarker::fromTracer(trc)->markColor(); if (!cell->isTenured()) { - MOZ_ASSERT(color == BLACK); + MOZ_ASSERT(color == MarkColor::Black); return false; } TenuredCell& tenured = cell->asTenured(); JS::Zone* zone = tenured.zone(); - if (color == BLACK) { + if (!src->zone()->isGCMarking() && !zone->isGCMarking()) + return false; + + if (color == MarkColor::Black) { /* * Having black->gray edges violates our promise to the cycle * collector. This can happen if we're collecting a compartment and it * has an edge to an uncollected compartment: it's possible that the * source and destination of the cross-compartment edge should be gray, - * but the source was marked black by the conservative scanner. + * but the source was marked black by the write barrier. */ - if (tenured.isMarked(GRAY)) { + if (tenured.isMarkedGray()) { MOZ_ASSERT(!zone->isCollecting()); - trc->runtime()->gc.setFoundBlackGrayEdges(tenured); + UnmarkGrayGCThing(trc->runtime(), JS::GCCellPtr(cell, cell->getTraceKind())); } - return zone->shouldMarkInZone(); + return zone->isGCMarking(); } else { if (zone->isGCMarkingBlack()) { /* @@ -317,7 +327,7 @@ ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, Cell* cell) * but it will be later, so record the cell so it can be marked gray * at the appropriate time. */ - if (!tenured.isMarked()) + if (!tenured.isMarkedAny()) DelayCrossCompartmentGrayMarking(src); return false; } @@ -619,7 +629,7 @@ js::TraceProcessGlobalRoot(JSTracer* trc, T* thing, const char* name) // permanent atoms, so likewise require no subsquent marking. CheckTracedThing(trc, *ConvertToBase(&thing)); if (trc->isMarkingTracer()) - thing->markIfUnmarked(gc::BLACK); + thing->markIfUnmarked(gc::MarkColor::Black); else DoCallback(trc->asCallbackTracer(), ConvertToBase(&thing), name); } @@ -731,7 +741,7 @@ GCMarker::markImplicitEdgesHelper(T markedThing) return; Zone* zone = gc::TenuredCell::fromPointer(markedThing)->zone(); - MOZ_ASSERT(zone->shouldMarkInZone()); + MOZ_ASSERT(zone->isGCMarking()); MOZ_ASSERT(!zone->isGCSweeping()); auto p = zone->gcWeakKeys.get(JS::GCCellPtr(markedThing)); @@ -788,8 +798,7 @@ ShouldMark(GCMarker* gcmarker, JSObject* obj) // Don't mark things outside a zone if we are in a per-zone GC. It is // faster to check our own arena, which we can do since we know that // the object is tenured. - Zone* zone = obj->asTenured().zone(); - return (zone && zone->shouldMarkInZone()); + return obj->asTenured().zone()->shouldMarkInZone(); } template @@ -978,7 +987,7 @@ js::GCMarker::mark(T* thing) MOZ_ASSERT(!IsInsideNursery(gc::TenuredCell::fromPointer(thing))); return gc::ParticipatesInCC::value ? gc::TenuredCell::fromPointer(thing)->markIfUnmarked(markColor()) - : gc::TenuredCell::fromPointer(thing)->markIfUnmarked(gc::BLACK); + : gc::TenuredCell::fromPointer(thing)->markIfUnmarked(gc::MarkColor::Black); } @@ -1057,7 +1066,9 @@ Shape::traceChildren(JSTracer* trc) inline void js::GCMarker::eagerlyMarkChildren(Shape* shape) { - MOZ_ASSERT(shape->isMarked(this->markColor())); + MOZ_ASSERT_IF(markColor() == MarkColor::Gray, shape->isMarkedGray()); + MOZ_ASSERT_IF(markColor() == MarkColor::Black, shape->isMarkedBlack()); + do { // Special case: if a base shape has a shape table then all its pointers // must point to this shape or an anscestor. Since these pointers will @@ -1110,7 +1121,7 @@ inline void js::GCMarker::eagerlyMarkChildren(JSLinearString* linearStr) { AssertShouldMarkInZone(linearStr); - MOZ_ASSERT(linearStr->isMarked()); + MOZ_ASSERT(linearStr->isMarkedAny()); MOZ_ASSERT(linearStr->JSString::isLinear()); // Use iterative marking to avoid blowing out the stack. @@ -1173,7 +1184,7 @@ js::GCMarker::eagerlyMarkChildren(JSRope* rope) JS_DIAGNOSTICS_ASSERT(rope->getTraceKind() == JS::TraceKind::String); JS_DIAGNOSTICS_ASSERT(rope->JSString::isRope()); AssertShouldMarkInZone(rope); - MOZ_ASSERT(rope->isMarked()); + MOZ_ASSERT(rope->isMarkedAny()); JSRope* next = nullptr; JSString* right = rope->rightChild(); @@ -1272,14 +1283,12 @@ ModuleScope::Data::trace(JSTracer* trc) TraceNullableEdge(trc, &module, "scope module"); TraceBindingNames(trc, trailingNames.start(), length); } - void WasmFunctionScope::Data::trace(JSTracer* trc) { TraceNullableEdge(trc, &instance, "wasm function"); TraceBindingNames(trc, trailingNames.start(), length); } - void Scope::traceChildren(JSTracer* trc) { @@ -1316,7 +1325,6 @@ Scope::traceChildren(JSTracer* trc) case ScopeKind::WasmFunction: reinterpret_cast(data_)->trace(trc); break; - } } inline void @@ -1382,7 +1390,8 @@ js::GCMarker::eagerlyMarkChildren(Scope* scope) case ScopeKind::With: break; - case ScopeKind::WasmFunction: { + + case ScopeKind::WasmFunction: { WasmFunctionScope::Data* data = reinterpret_cast(scope->data_); traverseEdge(scope, static_cast(data->instance)); names = &data->trailingNames; @@ -1390,7 +1399,6 @@ js::GCMarker::eagerlyMarkChildren(Scope* scope) break; } } - if (scope->kind_ == ScopeKind::Function) { for (uint32_t i = 0; i < length; i++) { if (JSAtom* name = names->operator[](i).name()) @@ -2027,7 +2035,7 @@ MarkStack::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const GCMarker::GCMarker(JSRuntime* rt) : JSTracer(rt, JSTracer::TracerKindTag::Marking, ExpandWeakMaps), stack(size_t(-1)), - color(BLACK), + color(MarkColor::Black), unmarkedArenaStackTop(nullptr) #ifdef DEBUG , markLaterArenas(0) @@ -2050,7 +2058,7 @@ GCMarker::start() MOZ_ASSERT(!started); started = true; #endif - color = BLACK; + color = MarkColor::Black; linearWeakMarkingDisabled_ = false; MOZ_ASSERT(!unmarkedArenaStackTop); @@ -2082,7 +2090,7 @@ GCMarker::stop() void GCMarker::reset() { - color = BLACK; + color = MarkColor::Black; stack.reset(); MOZ_ASSERT(isMarkStackEmpty()); @@ -2103,6 +2111,32 @@ GCMarker::reset() MOZ_ASSERT(!markLaterArenas); } + +template +void +GCMarker::pushTaggedPtr(T* ptr) +{ + checkZone(ptr); + if (!stack.push(ptr)) + delayMarkingChildren(ptr); +} + +void +GCMarker::pushValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end) +{ + checkZone(obj); + if (!stack.push(obj, start, end)) + delayMarkingChildren(obj); +} + +void +GCMarker::repush(JSObject* obj) +{ + MOZ_ASSERT_IF(markColor() == MarkColor::Gray, gc::TenuredCell::fromPointer(obj)->isMarkedGray()); + MOZ_ASSERT_IF(markColor() == MarkColor::Black, gc::TenuredCell::fromPointer(obj)->isMarkedBlack()); + pushTaggedPtr(obj); +} + void GCMarker::enterWeakMarkingMode() { @@ -2119,8 +2153,8 @@ GCMarker::enterWeakMarkingMode() if (weakMapAction() == ExpandWeakMaps) { tag_ = TracerKindTag::WeakMarking; - for (GCZoneGroupIter zone(runtime()); !zone.done(); zone.next()) { - for (WeakMapBase* m : zone->gcWeakMapList) { + for (GCSweepGroupIter zone(runtime()); !zone.done(); zone.next()) { + for (WeakMapBase* m : zone->gcWeakMapList()) { if (m->marked) (void) m->traceEntries(this); } @@ -2153,7 +2187,7 @@ GCMarker::markDelayedChildren(Arena* arena) for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) { TenuredCell* t = i.getCell(); - if (always || t->isMarked()) { + if (always || t->isMarkedAny()) { t->markIfUnmarked(); js::TraceChildren(this, t, MapAllocToTraceKind(arena->getAllocKind())); } @@ -2433,7 +2467,7 @@ JSObject* js::TenuringTracer::moveToTenured(JSObject* src) { MOZ_ASSERT(IsInsideNursery(src)); - MOZ_ASSERT(!src->zone()->usedByExclusiveThread); + MOZ_ASSERT(!src->zone()->usedByHelperThread()); AllocKind dstKind = src->allocKindForTenure(nursery()); Zone* zone = src->zone(); @@ -2718,7 +2752,10 @@ IsMarkedInternalCommon(T* thingp) return true; if (zone->isGCCompacting() && IsForwarded(*thingp)) *thingp = Forwarded(*thingp); - return (*thingp)->asTenured().isMarked(); + return true; + } + + return thing.isMarkedAny() || thing.arena()->allocatedDuringIncremental; } template @@ -2769,7 +2806,7 @@ js::gc::IsAboutToBeFinalizedDuringSweep(TenuredCell& tenured) MOZ_ASSERT(tenured.zoneFromAnyThread()->isGCSweeping()); if (tenured.arena()->allocatedDuringIncremental) return false; - return !tenured.isMarked(); + return !tenured.isMarkedAny(); } template @@ -2949,6 +2986,43 @@ struct UnmarkGrayTracer : public JS::CallbackTracer * of the containers, we must add unmark-graying read barriers to these * containers. */ + +#ifdef DEBUG +struct AssertNonGrayTracer : public JS::CallbackTracer { + explicit AssertNonGrayTracer(JSRuntime* rt) : JS::CallbackTracer(rt) {} + void onChild(const JS::GCCellPtr& thing) override { + MOZ_ASSERT(!thing.asCell()->isMarkedGray()); + } +}; +#endif + +class UnmarkGrayTracer : public JS::CallbackTracer +{ + public: + // We set weakMapAction to DoNotTraceWeakMaps because the cycle collector + // will fix up any color mismatches involving weakmaps when it runs. + explicit UnmarkGrayTracer(JSRuntime *rt) + : JS::CallbackTracer(rt, DoNotTraceWeakMaps) + , unmarkedAny(false) + , oom(false) + , stack(rt->gc.unmarkGrayStack) + {} + + void unmark(JS::GCCellPtr cell); + + // Whether we unmarked anything. + bool unmarkedAny; + + // Whether we ran out of memory. + bool oom; + + private: + // Stack of cells to traverse. + Vector& stack; + + void onChild(const JS::GCCellPtr& thing) override; +}; + void UnmarkGrayTracer::onChild(const JS::GCCellPtr& thing) { @@ -2976,7 +3050,7 @@ UnmarkGrayTracer::onChild(const JS::GCCellPtr& thing) } TenuredCell& tenured = cell->asTenured(); - if (!tenured.isMarked(js::gc::GRAY)) + if (!tenured.isMarkedGray()) return; tenured.unmark(js::gc::GRAY); @@ -3013,51 +3087,77 @@ UnmarkGrayTracer::onChild(const JS::GCCellPtr& thing) unmarkedAny |= childTracer.unmarkedAny; } -template static bool -TypedUnmarkGrayCellRecursively(T* t) +UnmarkGrayGCThing(JSRuntime* rt, JS::GCCellPtr thing) { - MOZ_ASSERT(t); + MOZ_ASSERT(thing); - JSRuntime* rt = t->runtimeFromMainThread(); - MOZ_ASSERT(!rt->isHeapCollecting()); - MOZ_ASSERT(!rt->isCycleCollecting()); - - bool unmarkedArg = false; - if (t->isTenured()) { - if (!t->asTenured().isMarked(GRAY)) - return false; - - t->asTenured().unmark(GRAY); - unmarkedArg = true; - } - - UnmarkGrayTracer trc(rt); - gcstats::AutoPhase outerPhase(rt->gc.stats, gcstats::PHASE_BARRIER); - gcstats::AutoPhase innerPhase(rt->gc.stats, gcstats::PHASE_UNMARK_GRAY); - t->traceChildren(&trc); - - return unmarkedArg || trc.unmarkedAny; -} - -struct UnmarkGrayCellRecursivelyFunctor { - template bool operator()(T* t) { return TypedUnmarkGrayCellRecursively(t); } -}; - -bool -js::UnmarkGrayCellRecursively(Cell* cell, JS::TraceKind kind) -{ - return DispatchTraceKindTyped(UnmarkGrayCellRecursivelyFunctor(), cell, kind); -} - -bool -js::UnmarkGrayShapeRecursively(Shape* shape) -{ - return TypedUnmarkGrayCellRecursively(shape); + UnmarkGrayTracer unmarker(rt); + gcstats::AutoPhase innerPhase(rt->gc.stats(), gcstats::PHASE_UNMARK_GRAY); + unmarker.unmark(thing); + return unmarker.unmarkedAny; } JS_FRIEND_API(bool) JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr thing) { - return js::UnmarkGrayCellRecursively(thing.asCell(), thing.kind()); + MOZ_ASSERT(!JS::CurrentThreadIsHeapCollecting()); + MOZ_ASSERT(!JS::CurrentThreadIsHeapCycleCollecting()); + + JSRuntime* rt = thing.asCell()->runtimeFromActiveCooperatingThread(); + gcstats::AutoPhase outerPhase(rt->gc.stats(), gcstats::PHASE_BARRIER); + return UnmarkGrayGCThing(rt, thing); +} + +bool +js::UnmarkGrayShapeRecursively(Shape* shape) +{ + return JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(shape)); +} + +namespace js { +namespace debug { + +MarkInfo +GetMarkInfo(Cell* rawCell) +{ + if (!rawCell->isTenured()) + return MarkInfo::NURSERY; + + TenuredCell* cell = &rawCell->asTenured(); + if (cell->isMarkedGray()) + return MarkInfo::GRAY; + if (cell->isMarkedBlack()) + return MarkInfo::BLACK; + return MarkInfo::UNMARKED; +} + +uintptr_t* +GetMarkWordAddress(Cell* cell) +{ + if (!cell->isTenured()) + return nullptr; + + uintptr_t* wordp; + uintptr_t mask; + js::gc::detail::GetGCThingMarkWordAndMask(uintptr_t(cell), ColorBit::BlackBit, &wordp, &mask); + return wordp; +} + +uintptr_t +GetMarkMask(Cell* cell, uint32_t colorBit) +{ + MOZ_ASSERT(colorBit == 0 || colorBit == 1); + + if (!cell->isTenured()) + return 0; + + ColorBit bit = colorBit == 0 ? ColorBit::BlackBit : ColorBit::GrayOrBlackBit; + uintptr_t* wordp; + uintptr_t mask; + js::gc::detail::GetGCThingMarkWordAndMask(uintptr_t(cell), bit, &wordp, &mask); + return mask; +} + +} } diff --git a/js/src/gc/Marking.h b/js/src/gc/Marking.h index 414079f799..62f0fc11b1 100644 --- a/js/src/gc/Marking.h +++ b/js/src/gc/Marking.h @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -72,9 +73,48 @@ class MarkStack maxCapacity_(maxCapacity) {} - ~MarkStack() { - js_free(stack_); - } + LastTag = TempRopeTag + }; + + static const uintptr_t TagMask = 7; + static_assert(TagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags."); + static_assert(TagMask <= gc::CellAlignMask, "The tag mask must be embeddable in a Cell*."); + + class TaggedPtr + { + uintptr_t bits; + + Cell* ptr() const; + + public: + TaggedPtr(Tag tag, Cell* ptr); + Tag tag() const; + template T* as() const; + JSObject* asValueArrayObject() const; + JSObject* asSavedValueArrayObject() const; + JSRope* asTempRope() const; + }; + + struct ValueArray + { + ValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end); + + HeapSlot* end; + HeapSlot* start; + TaggedPtr ptr; + }; + + struct SavedValueArray + { + SavedValueArray(JSObject* obj, size_t index, HeapSlot::Kind kind); + + uintptr_t kind; + uintptr_t index; + TaggedPtr ptr; + }; + + explicit MarkStack(size_t maxCapacity); + ~MarkStack(); size_t capacity() { return end_ - stack_; } @@ -197,15 +237,15 @@ class GCMarker : public JSTracer */ void setMarkColorGray() { MOZ_ASSERT(isDrained()); - MOZ_ASSERT(color == gc::BLACK); - color = gc::GRAY; + MOZ_ASSERT(color == gc::MarkColor::Black); + color = gc::MarkColor::Gray; } void setMarkColorBlack() { MOZ_ASSERT(isDrained()); - MOZ_ASSERT(color == gc::GRAY); - color = gc::BLACK; + MOZ_ASSERT(color == gc::MarkColor::Gray); + color = gc::MarkColor::Black; } - uint32_t markColor() const { return color; } + gc::MarkColor markColor() const { return color; } void enterWeakMarkingMode(); void leaveWeakMarkingMode(); @@ -329,7 +369,7 @@ class GCMarker : public JSTracer MarkStack stack; /* The color is only applied to objects and functions. */ - uint32_t color; + ActiveThreadData color; /* Pointer to the top of the stack of arenas we are delaying marking on. */ js::gc::Arena* unmarkedArenaStackTop; @@ -360,6 +400,9 @@ class GCMarker : public JSTracer // the marking phase of incremental GC. bool IsBufferGrayRootsTracer(JSTracer* trc); + +bool +IsUnmarkGrayTracer(JSTracer* trc); #endif namespace gc { diff --git a/js/src/gc/Nursery-inl.h b/js/src/gc/Nursery-inl.h index 55c23aebbb..8ebfd82323 100644 --- a/js/src/gc/Nursery-inl.h +++ b/js/src/gc/Nursery-inl.h @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=4 sw=4 et tw=79 ft=cpp: * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index f571c6ef39..e6caf45a47 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sw=4 et tw=78: * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, @@ -153,7 +154,7 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGC& lock) exit(0); } enableProfiling_ = true; - profileThreshold_ = atoi(env); + profileThreshold_ = TimeDuration::FromMicroseconds(atoi(env)); } env = getenv("JS_GC_REPORT_TENURING"); @@ -197,8 +198,7 @@ js::Nursery::enable() setCurrentChunk(0); setStartPosition(); - MOZ_ALWAYS_TRUE(runtime()->gc.storeBuffer.enable()); - return; + MOZ_ALWAYS_TRUE(runtime()->gc.storeBuffer().enable()); } void @@ -209,8 +209,7 @@ js::Nursery::disable() return; updateNumChunks(0); currentEnd_ = 0; - position_ = 0; - runtime()->gc.storeBuffer.disable(); + runtime()->gc.storeBuffer().disable(); } bool @@ -383,11 +382,11 @@ Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint3 void Nursery::setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader, - uint32_t nelems) + uint32_t capacity) { // Only use a direct forwarding pointer if there is enough space for one. setForwardingPointer(oldHeader->elements(), newHeader->elements(), - nelems > ObjectElements::VALUES_PER_HEADER); + capacity > 0); } #ifdef DEBUG @@ -433,9 +432,57 @@ js::TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery) { } +void +js::Nursery::renderProfileJSON(JSONPrinter& json) const +{ + if (!isEnabled()) { + json.beginObject(); + json.property("status", "nursery disabled"); + json.endObject(); + return; + } + + if (previousGC.reason == JS::gcreason::NO_REASON) { + // If the nursery was empty when the last minorGC was requested, then + // no nursery collection will have been performed but JSON may still be + // requested. (And as a public API, this function should not crash in + // such a case.) + json.beginObject(); + json.property("status", "no collection"); + json.endObject(); + return; + } + + json.beginObject(); + + json.property("reason", JS::gcreason::ExplainReason(previousGC.reason)); + json.property("bytes_tenured", previousGC.tenuredBytes); + json.floatProperty("promotion_rate", + 100.0 * previousGC.tenuredBytes / double(previousGC.nurseryUsedBytes), 2); + json.property("nursery_bytes", previousGC.nurseryUsedBytes); + json.property("new_nursery_bytes", numChunks() * ChunkSize); + + json.beginObjectProperty("timings"); + +#define EXTRACT_NAME(name, text) #name, + static const char* names[] = { +FOR_EACH_NURSERY_PROFILE_TIME(EXTRACT_NAME) +#undef EXTRACT_NAME + "" }; + + size_t i = 0; + for (auto time : profileDurations_) + json.property(names[i++], time, json.MICROSECONDS); + + json.endObject(); // timings value + + json.endObject(); +} + /* static */ void js::Nursery::printProfileHeader() { + fprintf(stderr, "MinorGC: Reason PRate Size "); #define PRINT_HEADER(name, text) \ fprintf(stderr, " %6s", text); FOR_EACH_NURSERY_PROFILE_TIME(PRINT_HEADER) @@ -455,11 +502,18 @@ void js::Nursery::printTotalProfileTimes() { if (enableProfiling_) { - fprintf(stderr, "MinorGC TOTALS: %7" PRIu64 " collections: ", minorGcCount_); - printProfileTimes(totalTimes_); + fprintf(stderr, "MinorGC TOTALS: %7" PRIu64 " collections: ", minorGcCount_); + printProfileDurations(totalDurations_); } } +void +js::Nursery::maybeClearProfileDurations() +{ + for (auto& duration : profileDurations_) + duration = mozilla::TimeDuration(); +} + inline void js::Nursery::startProfile(ProfileKey key) { @@ -473,20 +527,6 @@ js::Nursery::endProfile(ProfileKey key) totalTimes_[key] += profileTimes_[key]; } -inline void -js::Nursery::maybeStartProfile(ProfileKey key) -{ - if (enableProfiling_) - startProfile(key); -} - -inline void -js::Nursery::maybeEndProfile(ProfileKey key) -{ - if (enableProfiling_) - endProfile(key); -} - void js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) { @@ -520,38 +560,46 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) TenureCountCache tenureCounts; double promotionRate = 0; + previousGC.reason = JS::gcreason::NO_REASON; if (!isEmpty()) promotionRate = doCollection(rt, reason, tenureCounts); // Resize the nursery. - maybeStartProfile(ProfileKey::Resize); + startProfile(ProfileKey::Resize); maybeResizeNursery(reason, promotionRate); - maybeEndProfile(ProfileKey::Resize); + endProfile(ProfileKey::Resize); // If we are promoting the nursery, or exhausted the store buffer with // pointers to nursery things, which will force a collection well before // the nursery is full, look for object groups that are getting promoted // excessively and try to pretenure them. - maybeStartProfile(ProfileKey::Pretenure); - if (promotionRate > 0.8 || reason == JS::gcreason::FULL_STORE_BUFFER) { - JSContext* cx = rt->contextFromMainThread(); + startProfile(ProfileKey::Pretenure); + uint32_t pretenureCount = 0; + if (promotionRate > 0.8 || IsFullStoreBufferReason(reason)) { + JSContext* cx = TlsContext.get(); for (auto& entry : tenureCounts.entries) { if (entry.count >= 3000) { ObjectGroup* group = entry.group; if (group->canPreTenure()) { AutoCompartment ac(cx, group->compartment()); group->setShouldPreTenure(cx); + pretenureCount++; } } } } - maybeEndProfile(ProfileKey::Pretenure); + endProfile(ProfileKey::Pretenure); // We ignore gcMaxBytes when allocating for minor collection. However, if we // overflowed, we disable the nursery. The next time we allocate, we'll fail // because gcBytes >= gcMaxBytes. if (rt->gc.usage.gcBytes() >= rt->gc.tunables.gcMaxBytes()) disable(); + // Disable the nursery if the user changed the configuration setting. The + // nursery can only be re-enabled by resetting the configurationa and + // restarting firefox. + if (maxNurseryChunks_ == 0) + disable(); endProfile(ProfileKey::Total); minorGcCount_++; @@ -568,7 +616,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) printProfileHeader(); } - fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ", + fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ", JS::gcreason::ExplainReason(reason), promotionRate * 100, numChunks()); @@ -603,87 +651,96 @@ js::Nursery::doCollection(JSRuntime* rt, JS::gcreason::Reason reason, StoreBuffer& sb = rt->gc.storeBuffer; // The MIR graph only contains nursery pointers if cancelIonCompilations() - // is set on the store buffer, in which case we cancel all compilations. - maybeStartProfile(ProfileKey::CancelIonCompilations); + // is set on the store buffer, in which case we cancel all compilations + // of such graphs. + startProfile(ProfileKey::CancelIonCompilations); if (sb.cancelIonCompilations()) - js::CancelOffThreadIonCompile(rt); - maybeEndProfile(ProfileKey::CancelIonCompilations); + js::CancelOffThreadIonCompilesUsingNurseryPointers(rt); + endProfile(ProfileKey::CancelIonCompilations); - maybeStartProfile(ProfileKey::TraceValues); + startProfile(ProfileKey::TraceValues); sb.traceValues(mover); - maybeEndProfile(ProfileKey::TraceValues); + endProfile(ProfileKey::TraceValues); - maybeStartProfile(ProfileKey::TraceCells); + startProfile(ProfileKey::TraceCells); sb.traceCells(mover); - maybeEndProfile(ProfileKey::TraceCells); + endProfile(ProfileKey::TraceCells); - maybeStartProfile(ProfileKey::TraceSlots); + startProfile(ProfileKey::TraceSlots); sb.traceSlots(mover); - maybeEndProfile(ProfileKey::TraceSlots); + endProfile(ProfileKey::TraceSlots); - maybeStartProfile(ProfileKey::TraceWholeCells); + startProfile(ProfileKey::TraceWholeCells); sb.traceWholeCells(mover); - maybeEndProfile(ProfileKey::TraceWholeCells); + endProfile(ProfileKey::TraceWholeCells); - maybeStartProfile(ProfileKey::TraceGenericEntries); + startProfile(ProfileKey::TraceGenericEntries); sb.traceGenericEntries(&mover); - maybeEndProfile(ProfileKey::TraceGenericEntries); + endProfile(ProfileKey::TraceGenericEntries); - maybeStartProfile(ProfileKey::MarkRuntime); + startProfile(ProfileKey::MarkRuntime); rt->gc.traceRuntimeForMinorGC(&mover, session.lock); - maybeEndProfile(ProfileKey::MarkRuntime); + endProfile(ProfileKey::MarkRuntime); - maybeStartProfile(ProfileKey::MarkDebugger); + startProfile(ProfileKey::MarkDebugger); { gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_MARK_ROOTS); Debugger::markAll(&mover); } - maybeEndProfile(ProfileKey::MarkDebugger); + endProfile(ProfileKey::MarkDebugger); - maybeStartProfile(ProfileKey::ClearNewObjectCache); - rt->contextFromMainThread()->caches.newObjectCache.clearNurseryObjects(rt); - maybeEndProfile(ProfileKey::ClearNewObjectCache); + startProfile(ProfileKey::ClearNewObjectCache); + rt->caches().newObjectCache.clearNurseryObjects(rt); + endProfile(ProfileKey::ClearNewObjectCache); // Most of the work is done here. This loop iterates over objects that have // been moved to the major heap. If these objects have any outgoing pointers // to the nursery, then those nursery objects get moved as well, until no // objects are left to move. That is, we iterate to a fixed point. - maybeStartProfile(ProfileKey::CollectToFP); + startProfile(ProfileKey::CollectToFP); collectToFixedPoint(mover, tenureCounts); - maybeEndProfile(ProfileKey::CollectToFP); + endProfile(ProfileKey::CollectToFP); // Sweep compartments to update the array buffer object's view lists. - maybeStartProfile(ProfileKey::SweepArrayBufferViewList); + startProfile(ProfileKey::SweepArrayBufferViewList); for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) c->sweepAfterMinorGC(&mover); - maybeEndProfile(ProfileKey::SweepArrayBufferViewList); + endProfile(ProfileKey::SweepArrayBufferViewList); // Update any slot or element pointers whose destination has been tenured. - maybeStartProfile(ProfileKey::UpdateJitActivations); + startProfile(ProfileKey::UpdateJitActivations); js::jit::UpdateJitActivationsForMinorGC(rt, &mover); forwardedBuffers.finish(); - maybeEndProfile(ProfileKey::UpdateJitActivations); + endProfile(ProfileKey::UpdateJitActivations); - maybeStartProfile(ProfileKey::ObjectsTenuredCallback); + startProfile(ProfileKey::ObjectsTenuredCallback); rt->gc.callObjectsTenuredCallback(); - maybeEndProfile(ProfileKey::ObjectsTenuredCallback); + endProfile(ProfileKey::ObjectsTenuredCallback); // Sweep. - maybeStartProfile(ProfileKey::FreeMallocedBuffers); + startProfile(ProfileKey::FreeMallocedBuffers); freeMallocedBuffers(); - maybeEndProfile(ProfileKey::FreeMallocedBuffers); + endProfile(ProfileKey::FreeMallocedBuffers); - maybeStartProfile(ProfileKey::Sweep); + startProfile(ProfileKey::Sweep); sweep(); - maybeEndProfile(ProfileKey::Sweep); + endProfile(ProfileKey::Sweep); - maybeStartProfile(ProfileKey::ClearStoreBuffer); - rt->gc.storeBuffer.clear(); - maybeEndProfile(ProfileKey::ClearStoreBuffer); + startProfile(ProfileKey::ClearStoreBuffer); + runtime()->gc.storeBuffer().clear(); + endProfile(ProfileKey::ClearStoreBuffer); // Make sure hashtables have been updated after the collection. - maybeStartProfile(ProfileKey::CheckHashTables); - maybeEndProfile(ProfileKey::CheckHashTables); + startProfile(ProfileKey::CheckHashTables); +#ifdef JS_GC_ZEAL + if (rt->hasZealMode(ZealMode::CheckHashTablesOnMinorGC)) + CheckHashTablesAfterMovingGC(rt); +#endif + endProfile(ProfileKey::CheckHashTables); + + previousGC.reason = reason; + previousGC.nurseryUsedBytes = initialNurserySize; + previousGC.tenuredBytes = mover.tenuredSize; // Calculate and return the promotion rate. return mover.tenuredSize / double(initialNurserySize); @@ -743,8 +800,8 @@ void js::Nursery::sweep() { /* Sweep unique id's in all in-use chunks. */ - for (CellsWithUniqueIdSet::Enum e(cellsWithUid_); !e.empty(); e.popFront()) { - JSObject* obj = static_cast(e.front()); + for (Cell* cell : cellsWithUid_) { + JSObject* obj = static_cast(cell); if (!IsForwarded(obj)) obj->zone()->removeUniqueId(obj); else @@ -752,9 +809,13 @@ js::Nursery::sweep() } cellsWithUid_.clear(); - runSweepActions(); sweepDictionaryModeObjects(); +#ifdef JS_GC_ZEAL + /* Poison the nursery contents so touching a freed object will crash. */ + for (unsigned i = 0; i < numChunks(); i++) + chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); + { #ifdef JS_CRASH_DIAGNOSTICS for (unsigned i = 0; i < numChunks(); ++i) @@ -807,6 +868,7 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason, double promotionRat { static const double GrowThreshold = 0.05; static const double ShrinkThreshold = 0.01; + unsigned newMaxNurseryChunks; // Shrink the nursery to its minimum size of we ran out of memory or // received a memory pressure event. @@ -815,10 +877,30 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason, double promotionRat return; } +#ifdef JS_GC_ZEAL + // This zeal mode disabled nursery resizing. + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) + return; +#endif + + newMaxNurseryChunks = runtime()->gc.tunables.gcMaxNurseryBytes() >> ChunkShift; + if (newMaxNurseryChunks != maxNurseryChunks_) { + maxNurseryChunks_ = newMaxNurseryChunks; + /* The configured maximum nursery size is changing */ + int extraChunks = numChunks() - newMaxNurseryChunks; + if (extraChunks > 0) { + /* We need to shrink the nursery */ + shrinkAllocableSpace(extraChunks); + + previousPromotionRate_ = promotionRate; + return; + } + } + if (promotionRate > GrowThreshold) growAllocableSpace(); else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) - shrinkAllocableSpace(); + shrinkAllocableSpace(1); previousPromotionRate_ = promotionRate; } @@ -830,9 +912,13 @@ js::Nursery::growAllocableSpace() } void -js::Nursery::shrinkAllocableSpace() +js::Nursery::shrinkAllocableSpace(unsigned removeNumChunks) { - updateNumChunks(Max(numChunks() - 1, 1u)); +#ifdef JS_GC_ZEAL + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) + return; +#endif + updateNumChunks(Max(numChunks() - removeNumChunks, 1u)); } void diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 2935890c03..21cae4c4de 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sw=4 et tw=78: * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, @@ -198,7 +199,7 @@ class Nursery * sets |*ref| to the new location of the object and returns true. Otherwise * returns false and leaves |*ref| unset. */ - MOZ_ALWAYS_INLINE MOZ_MUST_USE bool getForwardedPointer(JSObject** ref) const; + MOZ_ALWAYS_INLINE [[nodiscard]] static bool getForwardedPointer(JSObject** ref); /* Forward a slots/elements pointer stored in an Ion frame. */ void forwardBufferPointer(HeapSlot** pSlotsElems); @@ -244,7 +245,6 @@ class Nursery // Free space remaining, not counting chunk trailers. MOZ_ALWAYS_INLINE size_t freeSpace() const { - MOZ_ASSERT(isEnabled()); MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize); return (currentEnd_ - position_) + (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize; @@ -253,6 +253,17 @@ class Nursery /* Print total profile times on shutdown. */ void printTotalProfileTimes(); + void* addressOfCurrentEnd() const { return (void*)¤tEnd_; } + void* addressOfPosition() const { return (void*)&position_; } + + void requestMinorGC(JS::gcreason::Reason reason) const; + + bool minorGCRequested() const { return minorGCTriggerReason_ != JS::gcreason::NO_REASON; } + JS::gcreason::Reason minorGCTriggerReason() const { return minorGCTriggerReason_; } + void clearMinorGCRequest() { minorGCTriggerReason_ = JS::gcreason::NO_REASON; } + + bool enableProfiling() const { return enableProfiling_; } + private: /* The amount of space in the mapped nursery available to allocations. */ static const size_t NurseryChunkUsableSize = gc::ChunkSize - sizeof(gc::ChunkTrailer); @@ -324,12 +335,18 @@ class Nursery ProfileTimes totalTimes_; uint64_t minorGcCount_; + struct { + JS::gcreason::Reason reason; + uint64_t nurseryUsedBytes; + uint64_t tenuredBytes; + } previousGC; + /* * The set of externally malloced buffers potentially kept live by objects * stored in the nursery. Any external buffers that do not belong to a * tenured thing at the end of a minor GC must be freed. */ - typedef HashSet, SystemAllocPolicy> MallocedBuffersSet; + typedef HashSet, SystemAllocPolicy> MallocedBuffersSet; MallocedBuffersSet mallocedBuffers; /* A task structure used to free the malloced bufers on a background thread. */ @@ -343,7 +360,7 @@ class Nursery * buffers might overlap each other. For these, an entry in the following * table is used. */ - typedef HashMap, SystemAllocPolicy> ForwardedBufferMap; + typedef HashMap, SystemAllocPolicy> ForwardedBufferMap; ForwardedBufferMap forwardedBuffers; /* @@ -421,7 +438,7 @@ class Nursery void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots); void setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader, - uint32_t nelems); + uint32_t capacity); /* Free malloced pointers owned by freed things in the nursery. */ void freeMallocedBuffers(); @@ -438,16 +455,13 @@ class Nursery /* Change the allocable space provided by the nursery. */ void maybeResizeNursery(JS::gcreason::Reason reason, double promotionRate); void growAllocableSpace(); - void shrinkAllocableSpace(); + void shrinkAllocableSpace(unsigned removeNumChunks); void minimizeAllocableSpace(); /* Profile recording and printing. */ void startProfile(ProfileKey key); void endProfile(ProfileKey key); - void maybeStartProfile(ProfileKey key); - void maybeEndProfile(ProfileKey key); - static void printProfileHeader(); - static void printProfileTimes(const ProfileTimes& times); + static void printProfileDurations(const ProfileDurations& times); friend class TenuringTracer; friend class gc::MinorCollectionTracer; diff --git a/js/src/gc/Policy.h b/js/src/gc/Policy.h index 98913065f9..53bf1dc1a5 100644 --- a/js/src/gc/Policy.h +++ b/js/src/gc/Policy.h @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ diff --git a/js/src/gc/RootMarking.cpp b/js/src/gc/RootMarking.cpp index 223bb70740..b793b9f7c3 100644 --- a/js/src/gc/RootMarking.cpp +++ b/js/src/gc/RootMarking.cpp @@ -359,9 +359,6 @@ js::gc::GCRuntime::traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrM for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) c->traceRoots(trc, traceOrMark); - // Trace SPS. - rt->spsProfiler.trace(trc); - // Trace helper thread roots. HelperThreadState().trace(trc); @@ -442,7 +439,16 @@ class BufferGrayRootsTracer : public JS::CallbackTracer // Set to false if we OOM while buffering gray roots. bool bufferingGrayRootsFailed; - void onChild(const JS::GCCellPtr& thing) override; + void onObjectEdge(JSObject** objp) override { bufferRoot(*objp); } + void onStringEdge(JSString** stringp) override { bufferRoot(*stringp); } + void onScriptEdge(JSScript** scriptp) override { bufferRoot(*scriptp); } + void onSymbolEdge(JS::Symbol** symbolp) override { bufferRoot(*symbolp); } + + void onChild(const JS::GCCellPtr& thing) override { + MOZ_CRASH("Unexpected gray root kind"); + } + + template inline void bufferRoot(T* thing); public: explicit BufferGrayRootsTracer(JSRuntime* rt) @@ -476,8 +482,6 @@ js::gc::GCRuntime::bufferGrayRoots() for (GCZonesIter zone(rt); !zone.done(); zone.next()) MOZ_ASSERT(zone->gcGrayRoots.empty()); - gcstats::AutoPhase ap(stats, gcstats::PHASE_BUFFER_GRAY_ROOTS); - BufferGrayRootsTracer grayBufferer(rt); if (JSTraceDataOp op = grayRootTracer.op) (*op)(&grayBufferer, grayRootTracer.data); @@ -508,8 +512,10 @@ BufferGrayRootsTracer::onChild(const JS::GCCellPtr& thing) gc::TenuredCell* tenured = gc::TenuredCell::fromPointer(thing.asCell()); - Zone* zone = tenured->zone(); - if (zone->isCollecting()) { + // This is run from a helper thread while the mutator is paused so we have + // to use *FromAnyThread methods here. + Zone* zone = tenured->zoneFromAnyThread(); + if (zone->isCollectingFromAnyThread()) { // See the comment on SetMaybeAliveFlag to see why we only do this for // objects and scripts. We rely on gray root buffering for this to work, // but we only need to worry about uncollected dead compartments during diff --git a/js/src/gc/StoreBuffer.h b/js/src/gc/StoreBuffer.h index f9158ef9c2..c2c108140a 100644 --- a/js/src/gc/StoreBuffer.h +++ b/js/src/gc/StoreBuffer.h @@ -7,6 +7,7 @@ #define gc_StoreBuffer_h #include "mozilla/Attributes.h" +#include "mozilla/HashFunctions.h" #include "mozilla/ReentrancyGuard.h" #include @@ -39,7 +40,7 @@ class BufferableRef bool maybeInRememberedSet(const Nursery&) const { return true; } }; -typedef HashSet, SystemAllocPolicy> EdgeSet; +typedef HashSet, SystemAllocPolicy> EdgeSet; /* The size of a single block of store buffer storage space. */ static const size_t LifoAllocBlockSize = 1 << 13; /* 8KiB */ @@ -206,7 +207,7 @@ class StoreBuffer struct PointerEdgeHasher { typedef Edge Lookup; - static HashNumber hash(const Lookup& l) { return uintptr_t(l.edge) >> 3; } + static HashNumber hash(const Lookup& l) { return mozilla::HashGeneric(l.edge); } static bool match(const Edge& k, const Lookup& l) { return k == l; } }; @@ -334,9 +335,13 @@ class StoreBuffer typedef struct Hasher { typedef SlotsEdge Lookup; - static HashNumber hash(const Lookup& l) { return l.objectAndKind_ ^ l.start_ ^ l.count_; } + static HashNumber hash(const Lookup& l) { + return mozilla::HashGeneric(l.objectAndKind_, l.start_, l.count_); + } static bool match(const SlotsEdge& k, const Lookup& l) { return k == l; } } Hasher; + + static const auto FullBufferReason = JS::gcreason::FULL_SLOT_BUFFER; }; template diff --git a/js/src/gc/Verifier.cpp b/js/src/gc/Verifier.cpp index b36c0c8293..bd2753fa8a 100644 --- a/js/src/gc/Verifier.cpp +++ b/js/src/gc/Verifier.cpp @@ -27,7 +27,427 @@ using namespace js::gc; #ifdef JSGC_HASH_TABLE_CHECKS -class CheckHeapTracer : public JS::CallbackTracer +/* + * Write barrier verification + * + * The next few functions are for write barrier verification. + * + * The VerifyBarriers function is a shorthand. It checks if a verification phase + * is currently running. If not, it starts one. Otherwise, it ends the current + * phase and starts a new one. + * + * The user can adjust the frequency of verifications, which causes + * VerifyBarriers to be a no-op all but one out of N calls. However, if the + * |always| parameter is true, it starts a new phase no matter what. + * + * Pre-Barrier Verifier: + * When StartVerifyBarriers is called, a snapshot is taken of all objects in + * the GC heap and saved in an explicit graph data structure. Later, + * EndVerifyBarriers traverses the heap again. Any pointer values that were in + * the snapshot and are no longer found must be marked; otherwise an assertion + * triggers. Note that we must not GC in between starting and finishing a + * verification phase. + */ + +struct EdgeValue +{ + void* thing; + JS::TraceKind kind; + const char* label; +}; + +struct VerifyNode +{ + void* thing; + JS::TraceKind kind; + uint32_t count; + EdgeValue edges[1]; +}; + +typedef HashMap, SystemAllocPolicy> NodeMap; + +/* + * The verifier data structures are simple. The entire graph is stored in a + * single block of memory. At the beginning is a VerifyNode for the root + * node. It is followed by a sequence of EdgeValues--the exact number is given + * in the node. After the edges come more nodes and their edges. + * + * The edgeptr and term fields are used to allocate out of the block of memory + * for the graph. If we run out of memory (i.e., if edgeptr goes beyond term), + * we just abandon the verification. + * + * The nodemap field is a hashtable that maps from the address of the GC thing + * to the VerifyNode that represents it. + */ +class js::VerifyPreTracer final : public JS::CallbackTracer +{ + JS::AutoDisableGenerationalGC noggc; + + void onChild(const JS::GCCellPtr& thing) override; + + public: + /* The gcNumber when the verification began. */ + uint64_t number; + + /* This counts up to gcZealFrequency to decide whether to verify. */ + int count; + + /* This graph represents the initial GC "snapshot". */ + VerifyNode* curnode; + VerifyNode* root; + char* edgeptr; + char* term; + NodeMap nodemap; + + explicit VerifyPreTracer(JSRuntime* rt) + : JS::CallbackTracer(rt), noggc(TlsContext.get()), number(rt->gc.gcNumber()), + count(0), curnode(nullptr), root(nullptr), edgeptr(nullptr), term(nullptr) + {} + + ~VerifyPreTracer() { + js_free(root); + } +}; + +/* + * This function builds up the heap snapshot by adding edges to the current + * node. + */ +void +VerifyPreTracer::onChild(const JS::GCCellPtr& thing) +{ + MOZ_ASSERT(!IsInsideNursery(thing.asCell())); + + // Skip things in other runtimes. + if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime()) + return; + + edgeptr += sizeof(EdgeValue); + if (edgeptr >= term) { + edgeptr = term; + return; + } + + VerifyNode* node = curnode; + uint32_t i = node->count; + + node->edges[i].thing = thing.asCell(); + node->edges[i].kind = thing.kind(); + node->edges[i].label = contextName(); + node->count++; +} + +static VerifyNode* +MakeNode(VerifyPreTracer* trc, void* thing, JS::TraceKind kind) +{ + NodeMap::AddPtr p = trc->nodemap.lookupForAdd(thing); + if (!p) { + VerifyNode* node = (VerifyNode*)trc->edgeptr; + trc->edgeptr += sizeof(VerifyNode) - sizeof(EdgeValue); + if (trc->edgeptr >= trc->term) { + trc->edgeptr = trc->term; + return nullptr; + } + + node->thing = thing; + node->count = 0; + node->kind = kind; + if (!trc->nodemap.add(p, thing, node)) { + trc->edgeptr = trc->term; + return nullptr; + } + + return node; + } + return nullptr; +} + +static VerifyNode* +NextNode(VerifyNode* node) +{ + if (node->count == 0) + return (VerifyNode*)((char*)node + sizeof(VerifyNode) - sizeof(EdgeValue)); + else + return (VerifyNode*)((char*)node + sizeof(VerifyNode) + + sizeof(EdgeValue)*(node->count - 1)); +} + +void +gc::GCRuntime::startVerifyPreBarriers() +{ + if (verifyPreData || isIncrementalGCInProgress()) + return; + + if (IsIncrementalGCUnsafe(rt) != AbortReason::None || + TlsContext.get()->keepAtoms || + rt->hasHelperThreadZones() || + rt->cooperatingContexts().length() != 1) + { + return; + } + + number++; + + VerifyPreTracer* trc = js_new(rt); + if (!trc) + return; + + AutoPrepareForTracing prep(TlsContext.get(), WithAtoms); + + for (auto chunk = allNonEmptyChunks(); !chunk.done(); chunk.next()) + chunk->bitmap.clear(); + + gcstats::AutoPhase ap(stats(), gcstats::PHASE_TRACE_HEAP); + + const size_t size = 64 * 1024 * 1024; + trc->root = (VerifyNode*)js_malloc(size); + if (!trc->root) + goto oom; + trc->edgeptr = (char*)trc->root; + trc->term = trc->edgeptr + size; + + if (!trc->nodemap.init()) + goto oom; + + /* Create the root node. */ + trc->curnode = MakeNode(trc, nullptr, JS::TraceKind(0)); + + incrementalState = State::MarkRoots; + + /* Make all the roots be edges emanating from the root node. */ + traceRuntime(trc, prep.session().lock); + + VerifyNode* node; + node = trc->curnode; + if (trc->edgeptr == trc->term) + goto oom; + + /* For each edge, make a node for it if one doesn't already exist. */ + while ((char*)node < trc->edgeptr) { + for (uint32_t i = 0; i < node->count; i++) { + EdgeValue& e = node->edges[i]; + VerifyNode* child = MakeNode(trc, e.thing, e.kind); + if (child) { + trc->curnode = child; + js::TraceChildren(trc, e.thing, e.kind); + } + if (trc->edgeptr == trc->term) + goto oom; + } + + node = NextNode(node); + } + + verifyPreData = trc; + incrementalState = State::Mark; + marker.start(); + + for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) { + MOZ_ASSERT(!zone->usedByHelperThread()); + zone->setNeedsIncrementalBarrier(true); + zone->arenas.purge(); + } + + return; + +oom: + incrementalState = State::NotActive; + js_delete(trc); + verifyPreData = nullptr; +} + +static bool +IsMarkedOrAllocated(TenuredCell* cell) +{ + return cell->isMarkedAny() || cell->arena()->allocatedDuringIncremental; +} + +struct CheckEdgeTracer : public JS::CallbackTracer { + VerifyNode* node; + explicit CheckEdgeTracer(JSRuntime* rt) : JS::CallbackTracer(rt), node(nullptr) {} + void onChild(const JS::GCCellPtr& thing) override; +}; + +static const uint32_t MAX_VERIFIER_EDGES = 1000; + +/* + * This function is called by EndVerifyBarriers for every heap edge. If the edge + * already existed in the original snapshot, we "cancel it out" by overwriting + * it with nullptr. EndVerifyBarriers later asserts that the remaining + * non-nullptr edges (i.e., the ones from the original snapshot that must have + * been modified) must point to marked objects. + */ +void +CheckEdgeTracer::onChild(const JS::GCCellPtr& thing) +{ + // Skip things in other runtimes. + if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime()) + return; + + /* Avoid n^2 behavior. */ + if (node->count > MAX_VERIFIER_EDGES) + return; + + for (uint32_t i = 0; i < node->count; i++) { + if (node->edges[i].thing == thing.asCell()) { + MOZ_ASSERT(node->edges[i].kind == thing.kind()); + node->edges[i].thing = nullptr; + return; + } + } +} + +void +js::gc::AssertSafeToSkipBarrier(TenuredCell* thing) +{ + mozilla::DebugOnly zone = thing->zoneFromAnyThread(); + MOZ_ASSERT(!zone->needsIncrementalBarrier() || zone->isAtomsZone()); +} + +static bool +IsMarkedOrAllocated(const EdgeValue& edge) +{ + if (!edge.thing || IsMarkedOrAllocated(TenuredCell::fromPointer(edge.thing))) + return true; + + // Permanent atoms and well-known symbols aren't marked during graph traversal. + if (edge.kind == JS::TraceKind::String && static_cast(edge.thing)->isPermanentAtom()) + return true; + if (edge.kind == JS::TraceKind::Symbol && static_cast(edge.thing)->isWellKnownSymbol()) + return true; + + return false; +} + +void +gc::GCRuntime::endVerifyPreBarriers() +{ + VerifyPreTracer* trc = verifyPreData; + + if (!trc) + return; + + MOZ_ASSERT(!JS::IsGenerationalGCEnabled(rt)); + + AutoPrepareForTracing prep(rt->activeContextFromOwnThread(), SkipAtoms); + + bool compartmentCreated = false; + + /* We need to disable barriers before tracing, which may invoke barriers. */ + for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) { + if (!zone->needsIncrementalBarrier()) + compartmentCreated = true; + + zone->setNeedsIncrementalBarrier(false); + } + + /* + * We need to bump gcNumber so that the methodjit knows that jitcode has + * been discarded. + */ + MOZ_ASSERT(trc->number == number); + number++; + + verifyPreData = nullptr; + incrementalState = State::NotActive; + + if (!compartmentCreated && + IsIncrementalGCUnsafe(rt) == AbortReason::None && + !TlsContext.get()->keepAtoms && + !rt->hasHelperThreadZones()) + { + CheckEdgeTracer cetrc(rt); + + /* Start after the roots. */ + VerifyNode* node = NextNode(trc->root); + while ((char*)node < trc->edgeptr) { + cetrc.node = node; + js::TraceChildren(&cetrc, node->thing, node->kind); + + if (node->count <= MAX_VERIFIER_EDGES) { + for (uint32_t i = 0; i < node->count; i++) { + EdgeValue& edge = node->edges[i]; + if (!IsMarkedOrAllocated(edge)) { + char msgbuf[1024]; + SprintfLiteral(msgbuf, + "[barrier verifier] Unmarked edge: %s %p '%s' edge to %s %p", + JS::GCTraceKindToAscii(node->kind), node->thing, + edge.label, + JS::GCTraceKindToAscii(edge.kind), edge.thing); + MOZ_ReportAssertionFailure(msgbuf, __FILE__, __LINE__); + MOZ_CRASH(); + } + } + } + + node = NextNode(node); + } + } + + marker.reset(); + marker.stop(); + + js_delete(trc); +} + +/*** Barrier Verifier Scheduling ***/ + +void +gc::GCRuntime::verifyPreBarriers() +{ + if (verifyPreData) + endVerifyPreBarriers(); + else + startVerifyPreBarriers(); +} + +void +gc::VerifyBarriers(JSRuntime* rt, VerifierType type) +{ + if (type == PreBarrierVerifier) + rt->gc.verifyPreBarriers(); +} + +void +gc::GCRuntime::maybeVerifyPreBarriers(bool always) +{ + if (!hasZealMode(ZealMode::VerifierPre)) + return; + + if (TlsContext.get()->suppressGC) + return; + + if (verifyPreData) { + if (++verifyPreData->count < zealFrequency && !always) + return; + + endVerifyPreBarriers(); + } + + startVerifyPreBarriers(); +} + +void +js::gc::MaybeVerifyBarriers(JSContext* cx, bool always) +{ + GCRuntime* gc = &cx->runtime()->gc; + gc->maybeVerifyPreBarriers(always); +} + +void +js::gc::GCRuntime::finishVerifier() +{ + if (verifyPreData) { + js_delete(verifyPreData.ref()); + verifyPreData = nullptr; + } +} + +#endif /* JS_GC_ZEAL */ + +#if defined(JSGC_HASH_TABLE_CHECKS) || defined(DEBUG) + +class HeapCheckTracerBase : public JS::CallbackTracer { public: explicit CheckHeapTracer(JSRuntime* rt); @@ -152,3 +572,69 @@ js::gc::CheckHeapAfterGC(JSRuntime* rt) } #endif /* JSGC_HASH_TABLE_CHECKS */ + +#ifdef DEBUG + +class CheckGrayMarkingTracer final : public HeapCheckTracerBase +{ + public: + explicit CheckGrayMarkingTracer(JSRuntime* rt); + bool check(AutoLockForExclusiveAccess& lock); + + private: + void checkCell(Cell* cell) override; +}; + +CheckGrayMarkingTracer::CheckGrayMarkingTracer(JSRuntime* rt) + : HeapCheckTracerBase(rt, DoNotTraceWeakMaps) +{ + // Weak gray->black edges are allowed. + setTraceWeakEdges(false); +} + +void +CheckGrayMarkingTracer::checkCell(Cell* cell) +{ + Cell* parent = parentCell(); + if (!cell->isTenured() || !parent || !parent->isTenured()) + return; + + TenuredCell* tenuredCell = &cell->asTenured(); + TenuredCell* tenuredParent = &parent->asTenured(); + if (tenuredParent->isMarkedBlack() && tenuredCell->isMarkedGray()) + { + failures++; + fprintf(stderr, "Found black to gray edge to %s %p\n", + GCTraceKindToAscii(cell->getTraceKind()), cell); + dumpCellPath(); + } +} + +bool +CheckGrayMarkingTracer::check(AutoLockForExclusiveAccess& lock) +{ + if (!traceHeap(lock)) + return true; // Ignore failure. + + return failures == 0; +} + +JS_FRIEND_API(bool) +js::CheckGrayMarkingState(JSContext* cx) +{ + JSRuntime* rt = cx->runtime(); + MOZ_ASSERT(!JS::CurrentThreadIsHeapCollecting()); + MOZ_ASSERT(!rt->gc.isIncrementalGCInProgress()); + if (!rt->gc.areGrayBitsValid()) + return true; + + gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PHASE_TRACE_HEAP); + AutoTraceSession session(rt, JS::HeapState::Tracing); + CheckGrayMarkingTracer tracer(rt); + if (!tracer.init()) + return true; // Ignore failure + + return tracer.check(session.lock); +} + +#endif // DEBUG diff --git a/js/src/gc/Zone.cpp b/js/src/gc/Zone.cpp index 7f2cd83bd7..4dd3b2ba18 100644 --- a/js/src/gc/Zone.cpp +++ b/js/src/gc/Zone.cpp @@ -1,4 +1,5 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -28,23 +29,31 @@ JS::Zone::Zone(JSRuntime* rt) suppressAllocationMetadataBuilder(false), arenas(rt), types(this), - compartments(), - gcGrayRoots(), - gcWeakKeys(SystemAllocPolicy(), rt->randomHashCodeScrambler()), - typeDescrObjects(this, SystemAllocPolicy()), - gcMallocBytes(0), - gcMallocGCTriggered(false), + gcWeakMapList_(group), + compartments_(), + gcGrayRoots_(group), + gcWeakRefs_(group), + weakCaches_(group), + gcWeakKeys_(group, SystemAllocPolicy(), rt->randomHashCodeScrambler()), + gcSweepGroupEdges_(group), + typeDescrObjects_(group, this), + regExps(this), + markedAtoms_(group), + atomCache_(group), + externalStringCache_(group), + functionToStringCache_(group), usage(&rt->gc.usage), gcDelayBytes(0), - propertyTree(this), - baseShapes(this, BaseShapeSet()), - initialShapes(this, InitialShapeSet()), - data(nullptr), - isSystem(false), - usedByExclusiveThread(false), - active(false), - jitZone_(nullptr), - gcState_(NoGC), + propertyTree_(group, this), + baseShapes_(group, this), + initialShapes_(group, this), + nurseryShapes_(group), + data(group, nullptr), + isSystem(group, false), +#ifdef DEBUG + gcLastSweepGroupIndex(group, 0), +#endif + jitZone_(group, nullptr), gcScheduled_(false), gcPreserveCode_(false), jitUsingBarriers_(false), @@ -58,6 +67,7 @@ JS::Zone::Zone(JSRuntime* rt) AutoLockGC lock(rt); threshold.updateAfterGC(8192, GC_NORMAL, rt->gc.tunables, rt->gc.schedulingState, lock); setGCMaxMallocBytes(rt->gc.maxMallocBytesAllocated() * 0.9); + jitCodeCounter.setMax(jit::MaxCodeBytesPerProcess * 0.8); } Zone::~Zone() @@ -79,10 +89,13 @@ Zone::~Zone() bool Zone::init(bool isSystemArg) { isSystem = isSystemArg; - return uniqueIds_.init() && - gcZoneGroupEdges.init() && - gcWeakKeys.init() && - typeDescrObjects.init(); + return uniqueIds().init() && + gcSweepGroupEdges().init() && + gcWeakKeys().init() && + typeDescrObjects().init() && + markedAtoms().init() && + atomCache().init() && + regExps.init(); } void @@ -179,13 +192,13 @@ Zone::sweepBreakpoints(FreeOp* fop) GCPtrNativeObject& dbgobj = bp->debugger->toJSObjectRef(); // If we are sweeping, then we expect the script and the - // debugger object to be swept in the same zone group, except if - // the breakpoint was added after we computed the zone + // debugger object to be swept in the same sweep group, except + // if the breakpoint was added after we computed the sweep // groups. In this case both script and debugger object must be // live. MOZ_ASSERT_IF(isGCSweeping() && dbgobj->zone()->isCollecting(), dbgobj->zone()->isGCSweeping() || - (!scriptGone && dbgobj->asTenured().isMarked())); + (!scriptGone && dbgobj->asTenured().isMarkedAny())); bool dying = scriptGone || IsAboutToBeFinalized(&dbgobj); MOZ_ASSERT_IF(!dying, !IsAboutToBeFinalized(&bp->getHandlerRef())); @@ -230,12 +243,13 @@ Zone::discardJitCode(FreeOp* fop, bool discardBaselineCode) for (auto script = cellIter(); !script.done(); script.next()) { jit::FinishInvalidation(fop, script); - /* - * Discard baseline script if it's not marked as active. Note that - * this also resets the active flag. - */ - if (discardBaselineCode) - jit::FinishDiscardBaselineScript(fop, script); + /* + * Make it impossible to use the control flow graphs cached on the + * BaselineScript. They get deleted. + */ + if (script->hasBaselineScript()) + script->baselineScript()->setControlFlowGraph(nullptr); + } /* * Warm-up counter for scripts are reset on GC. After discarding code we @@ -375,15 +389,30 @@ Zone::addTypeDescrObject(JSContext* cx, HandleObject obj) // Type descriptor objects are always tenured so we don't need post barriers // on the set. MOZ_ASSERT(!IsInsideNursery(obj)); - - if (!typeDescrObjects.put(obj)) { + + if (!typeDescrObjects().put(obj)) { ReportOutOfMemory(cx); return false; } - + return true; } - + +void +Zone::deleteEmptyCompartment(JSCompartment* comp) +{ + MOZ_ASSERT(comp->zone() == this); + MOZ_ASSERT(arenas.checkEmptyArenaLists()); + for (auto& i : compartments()) { + if (i == comp) { + compartments().erase(&i); + comp->destroy(runtimeFromActiveCooperatingThread()->defaultFreeOp()); + return; + } + } + MOZ_CRASH("Compartment not found"); +} + ZoneList::ZoneList() : head(nullptr), tail(nullptr) {} diff --git a/js/src/gc/Zone.h b/js/src/gc/Zone.h index 323aa27758..ae6400fa7a 100644 --- a/js/src/gc/Zone.h +++ b/js/src/gc/Zone.h @@ -7,6 +7,7 @@ #define gc_Zone_h #include "mozilla/Atomics.h" +#include "mozilla/HashFunctions.h" #include "mozilla/MemoryReporting.h" #include "jscntxt.h" @@ -77,7 +78,7 @@ struct UniqueIdGCPolicy { // Maps a Cell* to a unique, 64bit id. using UniqueIdMap = GCHashMap, + PointerHasher, SystemAllocPolicy, UniqueIdGCPolicy>; @@ -193,29 +194,16 @@ struct Zone : public JS::shadow::Zone, bool canCollect(); - void notifyObservingDebuggers(); - - enum GCState { - NoGC, - Mark, - MarkGray, - Sweep, - Finished, - Compact - }; - void setGCState(GCState state) { - MOZ_ASSERT(runtimeFromMainThread()->isHeapBusy()); - MOZ_ASSERT_IF(state != NoGC, canCollect()); - gcState_ = state; - if (state == Finished) - notifyObservingDebuggers(); + void changeGCState(GCState prev, GCState next) { + MOZ_ASSERT(CurrentThreadIsHeapBusy()); + MOZ_ASSERT(gcState() == prev); + MOZ_ASSERT_IF(next != NoGC, canCollect()); + gcState_ = next; } bool isCollecting() const { - if (runtimeFromMainThread()->isHeapCollecting()) - return gcState_ != NoGC; - else - return needsIncrementalBarrier(); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromActiveCooperatingThread())); + return isCollectingFromAnyThread(); } bool isCollectingFromAnyThread() const { @@ -263,9 +251,9 @@ struct Zone : public JS::shadow::Zone, void prepareForCompacting(); #ifdef DEBUG - // For testing purposes, return the index of the zone group which this zone + // For testing purposes, return the index of the sweep group which this zone // was swept in in the last GC. - unsigned lastZoneGroupIndex() { return gcLastZoneGroupIndex; } + unsigned lastSweepGroupIndex() { return gcLastSweepGroupIndex; } #endif using DebuggerVector = js::Vector; @@ -292,6 +280,8 @@ struct Zone : public JS::shadow::Zone, DebuggerVector* getDebuggers() const { return debuggers; } DebuggerVector* getOrCreateDebuggers(JSContext* cx); + void notifyObservingDebuggers(); + void clearTables(); /* @@ -318,17 +308,23 @@ struct Zone : public JS::shadow::Zone, // This zone's gray roots. typedef js::Vector GrayRootVector; - GrayRootVector gcGrayRoots; + private: + js::ZoneGroupOrGCTaskData gcGrayRoots_; + public: + GrayRootVector& gcGrayRoots() { return gcGrayRoots_.ref(); } // This zone's weak edges found via graph traversal during marking, // preserved for re-scanning during sweeping. using WeakEdges = js::Vector; WeakEdges gcWeakRefs; - // List of non-ephemeron weak containers to sweep during beginSweepingZoneGroup. - mozilla::LinkedList> weakCaches_; - void registerWeakCache(WeakCache* cachep) { - weakCaches_.insertBack(cachep); + private: + // List of non-ephemeron weak containers to sweep during beginSweepingSweepGroup. + js::ZoneGroupOrGCTaskData> weakCaches_; + public: + mozilla::LinkedList& weakCaches() { return weakCaches_.ref(); } + void registerWeakCache(detail::WeakCacheBase* cachep) { + weakCaches().insertBack(cachep); } /* @@ -339,9 +335,12 @@ struct Zone : public JS::shadow::Zone, // A set of edges from this zone to other zones. // - // This is used during GC while calculating zone groups to record edges that - // can't be determined by examining this zone by itself. - ZoneSet gcZoneGroupEdges; + // This is used during GC while calculating sweep groups to record edges + // that can't be determined by examining this zone by itself. + js::ZoneGroupData gcSweepGroupEdges_; + + public: + ZoneSet& gcSweepGroupEdges() { return gcSweepGroupEdges_.ref(); } // Keep track of all TypeDescr and related objects in this compartment. // This is used by the GC to trace them all first when compacting, since the @@ -354,18 +353,52 @@ struct Zone : public JS::shadow::Zone, using TypeDescrObjectSet = js::GCHashSet, js::SystemAllocPolicy>; - JS::WeakCache typeDescrObjects; - + private: + js::ZoneGroupData> typeDescrObjects_; + + // Malloc counter to measure memory pressure for GC scheduling. This + // counter should be used only when it's not possible to know the size of + // a free. + js::gc::MemoryCounter gcMallocCounter; + + // Counter of JIT code executable memory for GC scheduling. Also imprecise, + // since wasm can generate code that outlives a zone. + js::gc::MemoryCounter jitCodeCounter; + + public: + js::RegExpZone regExps; + + JS::WeakCache& typeDescrObjects() { return typeDescrObjects_.ref(); } + bool addTypeDescrObject(JSContext* cx, HandleObject obj); + bool triggerGCForTooMuchMalloc() { + JSRuntime* rt = runtimeFromAnyThread(); - // Malloc counter to measure memory pressure for GC scheduling. It runs from - // gcMaxMallocBytes down to zero. This counter should be used only when it's - // not possible to know the size of a free. - mozilla::Atomic gcMallocBytes; + if (CurrentThreadCanAccessRuntime(rt)) { + return rt->gc.triggerZoneGC(this, JS::gcreason::TOO_MUCH_MALLOC, + gcMallocCounter.bytes(), gcMallocCounter.maxBytes()); + } + return false; + } - // GC trigger threshold for allocations on the C heap. - size_t gcMaxMallocBytes; + void resetGCMallocBytes() { gcMallocCounter.reset(); } + void setGCMaxMallocBytes(size_t value) { gcMallocCounter.setMax(value); } + void updateMallocCounter(size_t nbytes) { gcMallocCounter.update(this, nbytes); } + size_t GCMaxMallocBytes() const { return gcMallocCounter.maxBytes(); } + size_t GCMallocBytes() const { return gcMallocCounter.bytes(); } + + void updateJitCodeMallocBytes(size_t size) { jitCodeCounter.update(this, size); } + + // Resets all the memory counters. + void resetAllMallocBytes() { + resetGCMallocBytes(); + jitCodeCounter.reset(); + } + bool isTooMuchMalloc() const { + return gcMallocCounter.isTooMuchMalloc() || + jitCodeCounter.isTooMuchMalloc(); + } // Whether a GC has been triggered as a result of gcMallocBytes falling // below zero. @@ -382,7 +415,7 @@ struct Zone : public JS::shadow::Zone, // Amount of data to allocate before triggering a new incremental slice for // the current GC. - size_t gcDelayBytes; + js::UnprotectedData gcDelayBytes; // Shared Shape property tree. js::PropertyTree propertyTree; @@ -414,11 +447,11 @@ struct Zone : public JS::shadow::Zone, bool active; #ifdef DEBUG - unsigned gcLastZoneGroupIndex; + js::ZoneGroupData gcLastSweepGroupIndex; #endif static js::HashNumber UniqueIdToHash(uint64_t uid) { - return js::HashNumber(uid >> 32) ^ js::HashNumber(uid & 0xFFFFFFFF); + return mozilla::HashGeneric(uid); } // Creates a HashNumber based on getUniqueId. Returns false on OOM. @@ -482,9 +515,9 @@ struct Zone : public JS::shadow::Zone, void transferUniqueId(js::gc::Cell* tgt, js::gc::Cell* src) { MOZ_ASSERT(src != tgt); MOZ_ASSERT(!IsInsideNursery(tgt)); - MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromMainThread())); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromActiveCooperatingThread())); MOZ_ASSERT(js::CurrentThreadCanAccessZone(this)); - uniqueIds_.rekeyIfMoved(src, tgt); + uniqueIds().rekeyIfMoved(src, tgt); } // Remove any unique id associated with this Cell. @@ -521,6 +554,9 @@ struct Zone : public JS::shadow::Zone, keepShapeTables_ = b; } + // Delete an empty compartment after its contents have been merged. + void deleteEmptyCompartment(JSCompartment* comp); + private: js::jit::JitZone* jitZone_; diff --git a/js/src/gc/ZoneGroup.cpp b/js/src/gc/ZoneGroup.cpp new file mode 100644 index 0000000000..9cfd2128cd --- /dev/null +++ b/js/src/gc/ZoneGroup.cpp @@ -0,0 +1,174 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/ZoneGroup.h" + +#include "jscntxt.h" + +#include "jit/IonBuilder.h" +#include "jit/JitCompartment.h" + +using namespace js; + +namespace js { + +ZoneGroup::ZoneGroup(JSRuntime* runtime) + : runtime(runtime), + ownerContext_(TlsContext.get()), + enterCount(1), + zones_(this), + usedByHelperThread(false), +#ifdef DEBUG + ionBailAfter_(this, 0), +#endif + jitZoneGroup(this, nullptr), + debuggerList_(this), + numFinishedBuilders(0), + ionLazyLinkListSize_(0) +{} + +bool +ZoneGroup::init() +{ + AutoLockGC lock(runtime); + + jitZoneGroup = js_new(this); + if (!jitZoneGroup) + return false; + + return true; +} + +ZoneGroup::~ZoneGroup() +{ +#ifdef DEBUG + { + AutoLockHelperThreadState lock; + MOZ_ASSERT(ionLazyLinkListSize_ == 0); + MOZ_ASSERT(ionLazyLinkList().isEmpty()); + } +#endif + + js_delete(jitZoneGroup.ref()); + + if (this == runtime->gc.systemZoneGroup) + runtime->gc.systemZoneGroup = nullptr; +} + +void +ZoneGroup::enter(JSContext* cx) +{ + if (ownerContext().context() == cx) { + MOZ_ASSERT(enterCount); + } else { + if (useExclusiveLocking) { + MOZ_ASSERT(!usedByHelperThread); + while (ownerContext().context() != nullptr) { + cx->yieldToEmbedding(); + } + } + MOZ_RELEASE_ASSERT(ownerContext().context() == nullptr); + MOZ_ASSERT(enterCount == 0); + ownerContext_ = CooperatingContext(cx); + if (cx->generationalDisabled) + nursery().disable(); + + // Finish any Ion compilations in this zone group, in case compilation + // finished for some script in this group while no thread was in this + // group. + jit::AttachFinishedCompilations(this, nullptr); + } + enterCount++; +} + +void +ZoneGroup::leave() +{ + MOZ_ASSERT(ownedByCurrentThread()); + MOZ_ASSERT(enterCount); + if (--enterCount == 0) + ownerContext_ = CooperatingContext(nullptr); +} + +bool +ZoneGroup::ownedByCurrentThread() +{ + MOZ_ASSERT(TlsContext.get()); + return ownerContext().context() == TlsContext.get(); +} + +ZoneGroup::IonBuilderList& +ZoneGroup::ionLazyLinkList() +{ + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime), + "Should only be mutated by the active thread."); + return ionLazyLinkList_.ref(); +} + +void +ZoneGroup::ionLazyLinkListRemove(jit::IonBuilder* builder) +{ + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime), + "Should only be mutated by the active thread."); + MOZ_ASSERT(this == builder->script()->zone()->group()); + MOZ_ASSERT(ionLazyLinkListSize_ > 0); + + builder->removeFrom(ionLazyLinkList()); + ionLazyLinkListSize_--; + + MOZ_ASSERT(ionLazyLinkList().isEmpty() == (ionLazyLinkListSize_ == 0)); +} + +void +ZoneGroup::ionLazyLinkListAdd(jit::IonBuilder* builder) +{ + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime), + "Should only be mutated by the active thread."); + MOZ_ASSERT(this == builder->script()->zone()->group()); + ionLazyLinkList().insertFront(builder); + ionLazyLinkListSize_++; +} + +void +ZoneGroup::deleteEmptyZone(Zone* zone) +{ + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime)); + MOZ_ASSERT(zone->group() == this); + MOZ_ASSERT(zone->compartments().empty()); + for (auto& i : zones()) { + if (i == zone) { + zones().erase(&i); + zone->destroy(runtime->defaultFreeOp()); + return; + } + } + MOZ_CRASH("Zone not found"); +} + +} // namespace js + +JS::AutoRelinquishZoneGroups::AutoRelinquishZoneGroups(JSContext* cx) + : cx(cx) +{ + MOZ_ASSERT(cx == TlsContext.get()); + + AutoEnterOOMUnsafeRegion oomUnsafe; + for (ZoneGroupsIter group(cx->runtime()); !group.done(); group.next()) { + while (group->ownerContext().context() == cx) { + group->leave(); + if (!enterList.append(group)) + oomUnsafe.crash("AutoRelinquishZoneGroups"); + } + } +} + +JS::AutoRelinquishZoneGroups::~AutoRelinquishZoneGroups() +{ + for (size_t i = 0; i < enterList.length(); i++) { + ZoneGroup* group = static_cast(enterList[i]); + group->enter(cx); + } +} diff --git a/js/src/jit/BaselineJIT.cpp b/js/src/jit/BaselineJIT.cpp index 70f7c79eb8..f6ed9fe7c1 100644 --- a/js/src/jit/BaselineJIT.cpp +++ b/js/src/jit/BaselineJIT.cpp @@ -1242,9 +1242,13 @@ MarkActiveBaselineScripts(JSRuntime* rt, const JitActivationIterator& activation void jit::MarkActiveBaselineScripts(Zone* zone) { - JSRuntime* rt = zone->runtimeFromMainThread(); - for (JitActivationIterator iter(rt); !iter.done(); ++iter) { - if (iter->compartment()->zone() == zone) - MarkActiveBaselineScripts(rt, iter); + if (zone->isAtomsZone()) + return; + JSContext* cx = TlsContext.get(); + for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) { + for (JitActivationIterator iter(cx, target); !iter.done(); ++iter) { + if (iter->compartment()->zone() == zone) + MarkActiveBaselineScripts(cx, iter); + } } } diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp index c7a4f36b12..938e233a1c 100644 --- a/js/src/jit/Ion.cpp +++ b/js/src/jit/Ion.cpp @@ -3152,10 +3152,15 @@ jit::InvalidateAll(FreeOp* fop, Zone* zone) MOZ_ASSERT(!HasOffThreadIonCompile(comp)); #endif - for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) { - if (iter->compartment()->zone() == zone) { - JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC"); - InvalidateActivation(fop, iter, true); + if (zone->isAtomsZone()) + return; + JSContext* cx = TlsContext.get(); + for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) { + for (JitActivationIterator iter(cx, target); !iter.done(); ++iter) { + if (iter->compartment()->zone() == zone) { + JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC"); + InvalidateActivation(fop, iter, true); + } } } } @@ -3198,8 +3203,18 @@ jit::Invalidate(TypeZone& types, FreeOp* fop, return; } - for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) - InvalidateActivation(fop, iter, false); + // This method can be called both during GC and during the course of normal + // script execution. In the former case this class will already be on the + // stack, and in the latter case the invalidations will all be on the + // current thread's stack, but the assertion under ActivationIterator can't + // tell that this is a thread local use of the iterator. + JSRuntime::AutoProhibitActiveContextChange apacc(fop->runtime()); + + JSContext* cx = TlsContext.get(); + for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) { + for (JitActivationIterator iter(cx, target); !iter.done(); ++iter) + InvalidateActivation(fop, iter, false); + } // Drop the references added above. If a script was never active, its // IonScript will be immediately destroyed. Otherwise, it will be held live diff --git a/js/src/jit/LoopUnroller.cpp b/js/src/jit/LoopUnroller.cpp index 37a76129a4..2430b6715e 100644 --- a/js/src/jit/LoopUnroller.cpp +++ b/js/src/jit/LoopUnroller.cpp @@ -17,7 +17,7 @@ namespace { struct LoopUnroller { typedef HashMap, SystemAllocPolicy> DefinitionMap; + PointerHasher, SystemAllocPolicy> DefinitionMap; explicit LoopUnroller(MIRGraph& graph) : graph(graph), alloc(graph.alloc()), diff --git a/js/src/jit/OptimizationTracking.cpp b/js/src/jit/OptimizationTracking.cpp index dfc83f16d8..ed0de29e41 100644 --- a/js/src/jit/OptimizationTracking.cpp +++ b/js/src/jit/OptimizationTracking.cpp @@ -211,8 +211,8 @@ static inline HashNumber HashType(TypeSet::Type ty) { if (ty.isObjectUnchecked()) - return PointerHasher::hash(ty.objectKey()); - return HashNumber(ty.raw()); + return PointerHasher::hash(ty.objectKey()); + return mozilla::HashGeneric(ty.raw()); } static HashNumber diff --git a/js/src/jsapi.cpp b/js/src/jsapi.cpp index 0fbb0d40e8..d40790745c 100644 --- a/js/src/jsapi.cpp +++ b/js/src/jsapi.cpp @@ -1364,16 +1364,16 @@ JS_RemoveFinalizeCallback(JSContext* cx, JSFinalizeCallback cb) } JS_PUBLIC_API(bool) -JS_AddWeakPointerZoneGroupCallback(JSContext* cx, JSWeakPointerZoneGroupCallback cb, void* data) +JS_AddWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb, void* data) { - AssertHeapIsIdle(cx); - return cx->gc.addWeakPointerZoneGroupCallback(cb, data); + AssertHeapIsIdle(); + return cx->runtime()->gc.addWeakPointerZonesCallback(cb, data); } JS_PUBLIC_API(void) -JS_RemoveWeakPointerZoneGroupCallback(JSContext* cx, JSWeakPointerZoneGroupCallback cb) +JS_RemoveWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb) { - cx->gc.removeWeakPointerZoneGroupCallback(cb); + cx->runtime()->gc.removeWeakPointerZonesCallback(cb); } JS_PUBLIC_API(bool) diff --git a/js/src/jsapi.h b/js/src/jsapi.h index d8b2a03a5d..41aa126090 100644 --- a/js/src/jsapi.h +++ b/js/src/jsapi.h @@ -599,7 +599,7 @@ typedef void (* JSFinalizeCallback)(JSFreeOp* fop, JSFinalizeStatus status, bool isZoneGC, void* data); typedef void -(* JSWeakPointerZoneGroupCallback)(JSContext* cx, void* data); +(* JSWeakPointerZonesCallback)(JSContext* cx, void* data); typedef void (* JSWeakPointerCompartmentCallback)(JSContext* cx, JSCompartment* comp, void* data); @@ -882,7 +882,50 @@ JS_IsBuiltinFunctionConstructor(JSFunction* fun); extern JS_PUBLIC_API(JSContext*) JS_NewContext(uint32_t maxbytes, uint32_t maxNurseryBytes = JS::DefaultNurseryBytes, - JSContext* parentContext = nullptr); + JSRuntime* parentRuntime = nullptr); + +// The methods below for controlling the active context in a cooperatively +// multithreaded runtime are not threadsafe, and the caller must ensure they +// are called serially if there is a chance for contention between threads. + +// Called from the active context for a runtime, yield execution so that +// this context is no longer active and can no longer use the API. +extern JS_PUBLIC_API(void) +JS_YieldCooperativeContext(JSContext* cx); + +// Called from a context whose runtime has no active context, this thread +// becomes the active context for that runtime and may use the API. +extern JS_PUBLIC_API(void) +JS_ResumeCooperativeContext(JSContext* cx); + +// Create a new context on this thread for cooperative multithreading in the +// same runtime as siblingContext. Called on a runtime (as indicated by +// siblingContet) which has no active context, on success the new context will +// become the runtime's active context. +extern JS_PUBLIC_API(JSContext*) +JS_NewCooperativeContext(JSContext* siblingContext); + +namespace JS { + +// Class to relinquish exclusive access to all zone groups in use by this +// thread. This allows other cooperative threads to enter the zone groups +// and modify their contents. +struct AutoRelinquishZoneGroups +{ + explicit AutoRelinquishZoneGroups(JSContext* cx); + ~AutoRelinquishZoneGroups(); + + private: + JSContext* cx; + mozilla::Vector enterList; +}; + +} // namespace JS + + +// Destroy a context allocated with JS_NewContext or JS_NewCooperativeContext. +// The context must be the current active context in the runtime, and after +// this call the runtime will have no active context. extern JS_PUBLIC_API(void) JS_DestroyContext(JSContext* cx); @@ -1600,10 +1643,10 @@ JS_RemoveFinalizeCallback(JSContext* cx, JSFinalizeCallback cb); */ extern JS_PUBLIC_API(bool) -JS_AddWeakPointerZoneGroupCallback(JSContext* cx, JSWeakPointerZoneGroupCallback cb, void* data); +JS_AddWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb, void* data); extern JS_PUBLIC_API(void) -JS_RemoveWeakPointerZoneGroupCallback(JSContext* cx, JSWeakPointerZoneGroupCallback cb); +JS_RemoveWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb); extern JS_PUBLIC_API(bool) JS_AddWeakPointerCompartmentCallback(JSContext* cx, JSWeakPointerCompartmentCallback cb, diff --git a/js/src/jsfriendapi.cpp b/js/src/jsfriendapi.cpp index d027f3aa53..44c771ce2e 100644 --- a/js/src/jsfriendapi.cpp +++ b/js/src/jsfriendapi.cpp @@ -622,7 +622,7 @@ struct VisitGrayCallbackFunctor { template void operator()(T tp) const { - if ((*tp)->isTenured() && (*tp)->asTenured().isMarked(gc::GRAY)) + if ((*tp)->isTenured() && (*tp)->asTenured().isMarkedGray()) callback_(closure_, JS::GCCellPtr(*tp)); } }; @@ -1083,10 +1083,10 @@ static char MarkDescriptor(void* thing) { gc::TenuredCell* cell = gc::TenuredCell::fromPointer(thing); - if (cell->isMarked(gc::BLACK)) - return cell->isMarked(gc::GRAY) ? 'G' : 'B'; + if (cell->isMarkedAny()) + return cell->isMarkedGray() ? 'G' : 'B'; else - return cell->isMarked(gc::GRAY) ? 'X' : 'W'; + return cell->isMarkedGray() ? 'X' : 'W'; } static void diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index 7aa33c576e..cacaee059b 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -476,14 +476,14 @@ Arena::finalize(FreeOp* fop, AllocKind thingKind, size_t thingSize) if (MOZ_UNLIKELY(MemProfiler::enabled())) { for (ArenaCellIterUnderFinalize i(this); !i.done(); i.next()) { T* t = i.get(); - if (t->asTenured().isMarked()) + if (t->asTenured().isMarkedAny()) MemProfiler::MarkTenured(reinterpret_cast(t)); } } for (ArenaCellIterUnderFinalize i(this); !i.done(); i.next()) { T* t = i.get(); - if (t->asTenured().isMarked()) { + if (t->asTenured().isMarkedAny()) { uint_fast16_t thing = uintptr_t(t) & ArenaMask; if (thing != firstThingOrSuccessorOfLastMarkedThing) { // We just finished passing over one or more free things, @@ -874,12 +874,10 @@ GCRuntime::GCRuntime(JSRuntime* rt) : incrementalState(gc::State::NotActive), lastMarkSlice(false), sweepOnBackgroundThread(false), - blocksToFreeAfterSweeping(JSRuntime::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), - blocksToFreeAfterMinorGC(JSRuntime::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), - zoneGroupIndex(0), - zoneGroups(nullptr), - currentZoneGroup(nullptr), - sweepPhaseIndex(0), + blocksToFreeAfterSweeping((size_t) JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), + sweepGroupIndex(0), + sweepGroups(nullptr), + currentSweepGroup(nullptr), sweepZone(nullptr), sweepActionIndex(0), abortSweepAfterCurrentGroup(false), @@ -1304,28 +1302,27 @@ GCRuntime::callFinalizeCallbacks(FreeOp* fop, JSFinalizeStatus status) const } bool -GCRuntime::addWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback, void* data) +GCRuntime::addWeakPointerZonesCallback(JSWeakPointerZonesCallback callback, void* data) { - return updateWeakPointerZoneGroupCallbacks.append( - Callback(callback, data)); + return updateWeakPointerZonesCallbacks.ref().append( + Callback(callback, data)); } void -GCRuntime::removeWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback) +GCRuntime::removeWeakPointerZonesCallback(JSWeakPointerZonesCallback callback) { - for (auto& p : updateWeakPointerZoneGroupCallbacks) { + for (auto& p : updateWeakPointerZonesCallbacks.ref()) { if (p.op == callback) { - updateWeakPointerZoneGroupCallbacks.erase(&p); + updateWeakPointerZonesCallbacks.ref().erase(&p); break; } } } -void -GCRuntime::callWeakPointerZoneGroupCallbacks() const +GCRuntime::callWeakPointerZonesCallbacks() const { - for (auto const& p : updateWeakPointerZoneGroupCallbacks) - p.op(rt->contextFromMainThread(), p.data); + for (auto const& p : updateWeakPointerZonesCallbacks.ref()) + p.op(TlsContext.get(), p.data); } bool @@ -1843,8 +1840,8 @@ RelocateArena(Arena* arena, SliceBudget& sliceBudget) TenuredCell* src = i.getCell(); MOZ_ASSERT(RelocationOverlay::isCellForwarded(src)); TenuredCell* dest = Forwarded(src); - MOZ_ASSERT(src->isMarked(BLACK) == dest->isMarked(BLACK)); - MOZ_ASSERT(src->isMarked(GRAY) == dest->isMarked(GRAY)); + MOZ_ASSERT(src->isMarkedAny() == dest->isMarkedAny()); + MOZ_ASSERT(src->isMarkedGray() == dest->isMarkedGray()); } #endif } @@ -2426,11 +2423,7 @@ GCRuntime::updatePointersToRelocatedCells(Zone* zone, AutoLockForExclusiveAccess blocksToFreeAfterSweeping.freeAll(); // Call callbacks to get the rest of the system to fixup other untraced pointers. - callWeakPointerZoneGroupCallbacks(); - for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) - callWeakPointerCompartmentCallbacks(comp); - if (rt->sweepZoneCallback) - rt->sweepZoneCallback(zone); + callWeakPointerZonesCallbacks(); } void @@ -3411,7 +3404,7 @@ ArenaLists::checkEmptyArenaList(AllocKind kind) for (Arena* current = arenaLists[kind].head(); current; current = current->next) { for (ArenaCellIterUnderGC i(current); !i.done(); i.next()) { TenuredCell* t = i.getCell(); - MOZ_ASSERT(t->isMarked(), "unmarked cells should have been finalized"); + MOZ_ASSERT(t->isMarkedAny(), "unmarked cells should have been finalized"); if (++num_live <= max_cells) { fprintf(stderr, "ERROR: GC found live Cell %p of kind %s at shutdown\n", t, AllocKindToAscii(kind)); @@ -3879,7 +3872,7 @@ GCRuntime::markWeakReferences(gcstats::Phase phase) void GCRuntime::markWeakReferencesInCurrentGroup(gcstats::Phase phase) { - markWeakReferences(phase); + markWeakReferences(phase); } template @@ -3902,7 +3895,7 @@ GCRuntime::markGrayReferences(gcstats::Phase phase) void GCRuntime::markGrayReferencesInCurrentGroup(gcstats::Phase phase) { - markGrayReferences(phase); + markGrayReferences(phase); } void @@ -3917,6 +3910,300 @@ GCRuntime::markAllGrayReferences(gcstats::Phase phase) markGrayReferences(phase); } +#ifdef JS_GC_ZEAL + +struct GCChunkHasher { + typedef gc::Chunk* Lookup; + + /* + * Strip zeros for better distribution after multiplying by the golden + * ratio. + */ + static HashNumber hash(gc::Chunk* chunk) { + MOZ_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask)); + return HashNumber(uintptr_t(chunk) >> gc::ChunkShift); + } + + static bool match(gc::Chunk* k, gc::Chunk* l) { + MOZ_ASSERT(!(uintptr_t(k) & gc::ChunkMask)); + MOZ_ASSERT(!(uintptr_t(l) & gc::ChunkMask)); + return k == l; + } +}; + +class js::gc::MarkingValidator +{ + public: + explicit MarkingValidator(GCRuntime* gc); + ~MarkingValidator(); + void nonIncrementalMark(AutoLockForExclusiveAccess& lock); + void validate(); + + private: + GCRuntime* gc; + bool initialized; + + typedef HashMap BitmapMap; + BitmapMap map; +}; + +js::gc::MarkingValidator::MarkingValidator(GCRuntime* gc) + : gc(gc), + initialized(false) +{} + +js::gc::MarkingValidator::~MarkingValidator() +{ + if (!map.initialized()) + return; + + for (BitmapMap::Range r(map.all()); !r.empty(); r.popFront()) + js_delete(r.front().value()); +} + +void +js::gc::MarkingValidator::nonIncrementalMark(AutoLockForExclusiveAccess& lock) +{ + /* + * Perform a non-incremental mark for all collecting zones and record + * the results for later comparison. + * + * Currently this does not validate gray marking. + */ + + if (!map.init()) + return; + + JSRuntime* runtime = gc->rt; + GCMarker* gcmarker = &gc->marker; + + gc->waitBackgroundSweepEnd(); + + /* Save existing mark bits. */ + { + AutoLockGC lock(runtime); + for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) { + ChunkBitmap* bitmap = &chunk->bitmap; + ChunkBitmap* entry = js_new(); + if (!entry) + return; + + memcpy((void*)entry->bitmap, (void*)bitmap->bitmap, sizeof(bitmap->bitmap)); + if (!map.putNew(chunk, entry)) + return; + } + } + + /* + * Temporarily clear the weakmaps' mark flags for the compartments we are + * collecting. + */ + + WeakMapSet markedWeakMaps; + if (!markedWeakMaps.init()) + return; + + /* + * For saving, smush all of the keys into one big table and split them back + * up into per-zone tables when restoring. + */ + gc::WeakKeyTable savedWeakKeys(SystemAllocPolicy(), runtime->randomHashCodeScrambler()); + if (!savedWeakKeys.init()) + return; + + for (GCZonesIter zone(runtime); !zone.done(); zone.next()) { + if (!WeakMapBase::saveZoneMarkedWeakMaps(zone, markedWeakMaps)) + return; + + AutoEnterOOMUnsafeRegion oomUnsafe; + for (gc::WeakKeyTable::Range r = zone->gcWeakKeys().all(); !r.empty(); r.popFront()) { + if (!savedWeakKeys.put(Move(r.front().key), Move(r.front().value))) + oomUnsafe.crash("saving weak keys table for validator"); + } + + if (!zone->gcWeakKeys().clear()) + oomUnsafe.crash("clearing weak keys table for validator"); + } + + /* + * After this point, the function should run to completion, so we shouldn't + * do anything fallible. + */ + initialized = true; + + /* Re-do all the marking, but non-incrementally. */ + js::gc::State state = gc->incrementalState; + gc->incrementalState = State::MarkRoots; + + { + gcstats::AutoPhase ap(gc->stats(), gcstats::PHASE_MARK); + { + gcstats::AutoPhase ap(gc->stats(), gcstats::PHASE_UNMARK); + + for (GCZonesIter zone(runtime); !zone.done(); zone.next()) + WeakMapBase::unmarkZone(zone); + + MOZ_ASSERT(gcmarker->isDrained()); + gcmarker->reset(); + + AutoLockGC lock(runtime); + for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) + chunk->bitmap.clear(); + } + + gc->traceRuntimeForMajorGC(gcmarker, lock); + + gc->incrementalState = State::Mark; + auto unlimited = SliceBudget::unlimited(); + MOZ_RELEASE_ASSERT(gc->marker.drainMarkStack(unlimited)); + } + + gc->incrementalState = State::Sweep; + { + gcstats::AutoPhase ap1(gc->stats(), gcstats::PHASE_SWEEP); + gcstats::AutoPhase ap2(gc->stats(), gcstats::PHASE_SWEEP_MARK); + + gc->markAllWeakReferences(gcstats::PHASE_SWEEP_MARK_WEAK); + + /* Update zone state for gray marking. */ + for (GCZonesIter zone(runtime); !zone.done(); zone.next()) { + MOZ_ASSERT(zone->isGCMarkingBlack()); + zone->setGCState(Zone::MarkGray); + } + gc->marker.setMarkColorGray(); + + gc->markAllGrayReferences(gcstats::PHASE_SWEEP_MARK_GRAY); + gc->markAllWeakReferences(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK); + + /* Restore zone state. */ + for (GCZonesIter zone(runtime); !zone.done(); zone.next()) { + MOZ_ASSERT(zone->isGCMarkingGray()); + zone->setGCState(Zone::Mark); + } + MOZ_ASSERT(gc->marker.isDrained()); + gc->marker.setMarkColorBlack(); + } + + /* Take a copy of the non-incremental mark state and restore the original. */ + { + AutoLockGC lock(runtime); + for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) { + ChunkBitmap* bitmap = &chunk->bitmap; + ChunkBitmap* entry = map.lookup(chunk)->value(); + Swap(*entry, *bitmap); + } + } + + for (GCZonesIter zone(runtime); !zone.done(); zone.next()) { + WeakMapBase::unmarkZone(zone); + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!zone->gcWeakKeys().clear()) + oomUnsafe.crash("clearing weak keys table for validator"); + } + + WeakMapBase::restoreMarkedWeakMaps(markedWeakMaps); + + for (gc::WeakKeyTable::Range r = savedWeakKeys.all(); !r.empty(); r.popFront()) { + AutoEnterOOMUnsafeRegion oomUnsafe; + Zone* zone = gc::TenuredCell::fromPointer(r.front().key.asCell())->zone(); + if (!zone->gcWeakKeys().put(Move(r.front().key), Move(r.front().value))) + oomUnsafe.crash("restoring weak keys table for validator"); + } + + gc->incrementalState = state; +} + +void +js::gc::MarkingValidator::validate() +{ + /* + * Validates the incremental marking for a single compartment by comparing + * the mark bits to those previously recorded for a non-incremental mark. + */ + + if (!initialized) + return; + + gc->waitBackgroundSweepEnd(); + + AutoLockGC lock(gc->rt); + for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) { + BitmapMap::Ptr ptr = map.lookup(chunk); + if (!ptr) + continue; /* Allocated after we did the non-incremental mark. */ + + ChunkBitmap* bitmap = ptr->value(); + ChunkBitmap* incBitmap = &chunk->bitmap; + + for (size_t i = 0; i < ArenasPerChunk; i++) { + if (chunk->decommittedArenas.get(i)) + continue; + Arena* arena = &chunk->arenas[i]; + if (!arena->allocated()) + continue; + if (!arena->zone->isGCSweeping()) + continue; + if (arena->allocatedDuringIncremental) + continue; + + AllocKind kind = arena->getAllocKind(); + uintptr_t thing = arena->thingsStart(); + uintptr_t end = arena->thingsEnd(); + while (thing < end) { + Cell* cell = (Cell*)thing; + + /* + * If a non-incremental GC wouldn't have collected a cell, then + * an incremental GC won't collect it. + */ + if (bitmap->isMarkedAny(cell)) + MOZ_RELEASE_ASSERT(incBitmap->isMarkedAny(cell)); + + /* + * If the cycle collector isn't allowed to collect an object + * after a non-incremental GC has run, then it isn't allowed to + * collected it after an incremental GC. + */ + if (!bitmap->isMarkedGray(cell)) + MOZ_RELEASE_ASSERT(!incBitmap->isMarkedGray(cell)); + + thing += Arena::thingSize(kind); + } + } + } +} + +#endif // JS_GC_ZEAL + +void +GCRuntime::computeNonIncrementalMarkingForValidation(AutoLockForExclusiveAccess& lock) +{ +#ifdef JS_GC_ZEAL + MOZ_ASSERT(!markingValidator); + if (isIncremental && hasZealMode(ZealMode::IncrementalMarkingValidator)) + markingValidator = js_new(this); + if (markingValidator) + markingValidator->nonIncrementalMark(lock); +#endif +} + +void +GCRuntime::validateIncrementalMarking() +{ +#ifdef JS_GC_ZEAL + if (markingValidator) + markingValidator->validate(); +#endif +} + +void +GCRuntime::finishMarkingValidation() +{ +#ifdef JS_GC_ZEAL + js_delete(markingValidator.ref()); + markingValidator = nullptr; +#endif +} static void DropStringWrappers(JSRuntime* rt) { @@ -3983,7 +4270,7 @@ JSCompartment::findOutgoingEdges(ZoneComponentFinder& finder) bool needsEdge = true; if (key.is()) { TenuredCell& other = key.as()->asTenured(); - needsEdge = !other.isMarked(BLACK) || other.isMarked(GRAY); + needsEdge = !other.isMarkedAny() || other.isMarkedGray(); } key.applyToWrapped(AddOutgoingEdgeFunctor(needsEdge, finder)); } @@ -4004,7 +4291,7 @@ Zone::findOutgoingEdges(ZoneComponentFinder& finder) for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) comp->findOutgoingEdges(finder); - for (ZoneSet::Range r = gcZoneGroupEdges.all(); !r.empty(); r.popFront()) { + for (ZoneSet::Range r = gcSweepGroupEdges().all(); !r.empty(); r.popFront()) { if (r.front()->shouldMarkInZone()) finder.addEdgeTo(r.front()); } @@ -4034,11 +4321,11 @@ GCRuntime::findInterZoneEdges() } void -GCRuntime::findZoneGroups(AutoLockForExclusiveAccess& lock) +GCRuntime::groupZonesForSweeping(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock) { #ifdef DEBUG for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) - MOZ_ASSERT(zone->gcZoneGroupEdges.empty()); + MOZ_ASSERT(zone->gcSweepGroupEdges().empty()); #endif JSContext* cx = rt->contextFromMainThread(); @@ -4050,22 +4337,22 @@ GCRuntime::findZoneGroups(AutoLockForExclusiveAccess& lock) MOZ_ASSERT(zone->shouldMarkInZone()); finder.addNode(zone); } - zoneGroups = finder.getResultsList(); - currentZoneGroup = zoneGroups; - zoneGroupIndex = 0; + sweepGroups = finder.getResultsList(); + currentSweepGroup = sweepGroups; + sweepGroupIndex = 0; for (GCZonesIter zone(rt); !zone.done(); zone.next()) - zone->gcZoneGroupEdges.clear(); + zone->gcSweepGroupEdges().clear(); #ifdef DEBUG - for (Zone* head = currentZoneGroup; head; head = head->nextGroup()) { + for (Zone* head = currentSweepGroup; head; head = head->nextGroup()) { for (Zone* zone = head; zone; zone = zone->nextNodeInGroup()) MOZ_ASSERT(zone->shouldMarkInZone()); } - MOZ_ASSERT_IF(!isIncremental, !currentZoneGroup->nextGroup()); + MOZ_ASSERT_IF(!isIncremental, !currentSweepGroup->nextGroup()); for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) - MOZ_ASSERT(zone->gcZoneGroupEdges.empty()); + MOZ_ASSERT(zone->gcSweepGroupEdges().empty()); #endif } @@ -4073,26 +4360,26 @@ static void ResetGrayList(JSCompartment* comp); void -GCRuntime::getNextZoneGroup() +GCRuntime::getNextSweepGroup() { - currentZoneGroup = currentZoneGroup->nextGroup(); - ++zoneGroupIndex; - if (!currentZoneGroup) { + currentSweepGroup = currentSweepGroup->nextGroup(); + ++sweepGroupIndex; + if (!currentSweepGroup) { abortSweepAfterCurrentGroup = false; return; } - for (Zone* zone = currentZoneGroup; zone; zone = zone->nextNodeInGroup()) { + for (Zone* zone = currentSweepGroup; zone; zone = zone->nextNodeInGroup()) { MOZ_ASSERT(zone->shouldMarkInZone()); MOZ_ASSERT(!zone->isQueuedForBackgroundSweep()); } if (!isIncremental) - ZoneComponentFinder::mergeGroups(currentZoneGroup); + ZoneComponentFinder::mergeGroups(currentSweepGroup); if (abortSweepAfterCurrentGroup) { MOZ_ASSERT(!isIncremental); - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { MOZ_ASSERT(!zone->gcNextGraphComponent); MOZ_ASSERT(zone->shouldMarkInZone()); zone->setNeedsIncrementalBarrier(false, Zone::UpdateJit); @@ -4104,7 +4391,7 @@ GCRuntime::getNextZoneGroup() ResetGrayList(comp); abortSweepAfterCurrentGroup = false; - currentZoneGroup = nullptr; + currentSweepGroup = nullptr; } } @@ -4250,11 +4537,11 @@ MarkIncomingCrossCompartmentPointers(JSRuntime* rt, const uint32_t color) MOZ_ASSERT(dst->compartment() == c); if (color == GRAY) { - if (IsMarkedUnbarriered(rt, &src) && src->asTenured().isMarked(GRAY)) + if (IsMarkedUnbarriered(rt, &src) && src->asTenured().isMarkedGray()) TraceManuallyBarrieredEdge(&rt->gc.marker, &dst, "cross-compartment gray pointer"); } else { - if (IsMarkedUnbarriered(rt, &src) && !src->asTenured().isMarked(GRAY)) + if (IsMarkedUnbarriered(rt, &src) && !src->asTenured().isMarkedGray()) TraceManuallyBarrieredEdge(&rt->gc.marker, &dst, "cross-compartment black pointer"); } @@ -4350,8 +4637,8 @@ js::NotifyGCPostSwap(JSObject* a, JSObject* b, unsigned removedFlags) DelayCrossCompartmentGrayMarking(a); } -void -GCRuntime::endMarkingZoneGroup() +IncrementalProgress +GCRuntime::endMarkingSweepGroup(FreeOp* fop, SliceBudget& budget) { gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_MARK); @@ -4369,7 +4656,7 @@ GCRuntime::endMarkingZoneGroup() * these will be marked through, as they are not marked with * MarkCrossCompartmentXXX. */ - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { MOZ_ASSERT(zone->isGCMarkingBlack()); zone->setGCState(Zone::MarkGray); } @@ -4383,7 +4670,7 @@ GCRuntime::endMarkingZoneGroup() markWeakReferencesInCurrentGroup(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK); /* Restore marking state. */ - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { MOZ_ASSERT(zone->isGCMarkingGray()); zone->setGCState(Zone::Mark); } @@ -4479,6 +4766,58 @@ SweepMiscTask::run() } } +static void +SweepCompressionTasks(JSRuntime* runtime) +{ + AutoLockHelperThreadState lock; + + // Attach finished compression tasks. + auto& finished = HelperThreadState().compressionFinishedList(lock); + for (size_t i = 0; i < finished.length(); i++) { + if (finished[i]->runtimeMatches(runtime)) { + UniquePtr task(Move(finished[i])); + HelperThreadState().remove(finished, &i); + task->complete(); + } + } + + // Sweep pending tasks that are holding onto should-be-dead ScriptSources. + auto& pending = HelperThreadState().compressionPendingList(lock); + for (size_t i = 0; i < pending.length(); i++) { + if (pending[i]->shouldCancel()) + HelperThreadState().remove(pending, &i); + } +} + +static void +SweepWeakMaps(JSRuntime* runtime) +{ + for (GCSweepGroupIter zone(runtime); !zone.done(); zone.next()) { + /* Clear all weakrefs that point to unmarked things. */ + for (auto edge : zone->gcWeakRefs()) { + /* Edges may be present multiple times, so may already be nulled. */ + if (*edge && IsAboutToBeFinalizedDuringSweep(**edge)) + *edge = nullptr; + } + zone->gcWeakRefs().clear(); + + /* No need to look up any more weakmap keys from this sweep group. */ + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!zone->gcWeakKeys().clear()) + oomUnsafe.crash("clearing weak keys in beginSweepingSweepGroup()"); + + zone->sweepWeakMaps(); + } +} + +static void +SweepUniqueIds(JSRuntime* runtime) +{ + FreeOp fop(nullptr); + for (GCSweepGroupIter zone(runtime); !zone.done(); zone.next()) + zone->sweepUniqueIds(&fop); +} + void GCRuntime::startTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked) @@ -4498,7 +4837,127 @@ GCRuntime::joinTask(GCParallelTask& task, gcstats::Phase phase, task.joinWithLockHeld(locked); } -using WeakCacheTaskVector = mozilla::Vector; +void +GCRuntime::sweepDebuggerOnMainThread(FreeOp* fop) +{ + // Detach unreachable debuggers and global objects from each other. + // This can modify weakmaps and so must happen before weakmap sweeping. + Debugger::sweepAll(fop); + + gcstats::AutoPhase ap(stats(), gcstats::PHASE_SWEEP_COMPARTMENTS); + + // Sweep debug environment information. This performs lookups in the Zone's + // unique IDs table and so must not happen in parallel with sweeping that + // table. + { + gcstats::AutoPhase ap2(stats(), gcstats::PHASE_SWEEP_MISC); + for (GCCompartmentGroupIter c(rt); !c.done(); c.next()) + c->sweepDebugEnvironments(); + } + + // Sweep breakpoints. This is done here to be with the other debug sweeping, + // although note that it can cause JIT code to be patched. + { + gcstats::AutoPhase ap(stats(), gcstats::PHASE_SWEEP_BREAKPOINT); + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) + zone->sweepBreakpoints(fop); + } +} + +void +GCRuntime::sweepJitDataOnMainThread(FreeOp* fop) +{ + + { + gcstats::AutoPhase ap(stats(), gcstats::PHASE_SWEEP_JIT_DATA); + + // Cancel any active or pending off thread compilations. + js::CancelOffThreadIonCompile(rt, JS::Zone::Sweep); + + for (GCCompartmentGroupIter c(rt); !c.done(); c.next()) + c->sweepJitCompartment(fop); + + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { + if (jit::JitZone* jitZone = zone->jitZone()) + jitZone->sweep(fop); + } + // Bug 1071218: the following method has not yet been refactored to + // work on a single zone-group at once. + + // Sweep entries containing about-to-be-finalized JitCode and + // update relocated TypeSet::Types inside the JitcodeGlobalTable. + jit::JitRuntime::SweepJitcodeGlobalTable(rt); + } + + { + gcstats::AutoPhase apdc(stats(), gcstats::PHASE_SWEEP_DISCARD_CODE); + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) + zone->discardJitCode(fop); + } + + { + gcstats::AutoPhase ap1(stats(), gcstats::PHASE_SWEEP_TYPES); + gcstats::AutoPhase ap2(stats(), gcstats::PHASE_SWEEP_TYPES_BEGIN); + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) + zone->beginSweepTypes(fop, releaseObservedTypes && !zone->isPreservingCode()); + } +} + +using WeakCacheTaskVector = mozilla::Vector; + +enum WeakCacheLocation +{ + RuntimeWeakCache, + ZoneWeakCache +}; + +// Call a functor for all weak caches that need to be swept in the current +// sweep group. +template +static inline bool +IterateWeakCaches(JSRuntime* rt, Functor f) +{ + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { + for (JS::detail::WeakCacheBase* cache : zone->weakCaches()) { + if (!f(cache, ZoneWeakCache)) + return false; + } + } + + for (JS::detail::WeakCacheBase* cache : rt->weakCaches()) { + if (!f(cache, RuntimeWeakCache)) + return false; + } + + return true; +} + +static bool +PrepareWeakCacheTasks(JSRuntime* rt, WeakCacheTaskVector* immediateTasks) +{ + // Start incremental sweeping for caches that support it or add to a vector + // of sweep tasks to run on a helper thread. + + MOZ_ASSERT(immediateTasks->empty()); + + bool ok = IterateWeakCaches(rt, [&] (JS::detail::WeakCacheBase* cache, + WeakCacheLocation location) + { + if (!cache->needsSweep()) + return true; + + // Caches that support incremental sweeping will be swept later. + if (location == ZoneWeakCache && cache->setNeedsIncrementalBarrier(true)) + return true; + + return immediateTasks->emplaceBack(rt, *cache); + }); + + if (!ok) + immediateTasks->clearAndFree(); + + return ok; +} static void SweepWeakCachesFromMainThread(JSRuntime* rt) @@ -4526,8 +4985,17 @@ PrepareWeakCacheTasks(JSRuntime* rt) return out; } -void -GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock) + ~AutoRunParallelTask() { + runtime()->gc.joinTask(*this, phase_, lock_); + } + + void run() override { + func_(runtime()); + } +}; + +IncrementalProgress +GCRuntime::beginSweepingSweepGroup(FreeOp* fop, SliceBudget& budget) { /* * Begin sweeping the group of zones in gcCurrentZoneGroup, @@ -4535,7 +5003,7 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock) */ bool sweepingAtoms = false; - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { /* Set the GC state to sweeping. */ MOZ_ASSERT(zone->shouldMarkInZone()); zone->setGCState(Zone::Sweep); @@ -4550,7 +5018,7 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock) rt->sweepZoneCallback(zone); #ifdef DEBUG - zone->gcLastZoneGroupIndex = zoneGroupIndex; + zone->gcLastSweepGroupIndex = sweepGroupIndex; #endif } @@ -4581,12 +5049,12 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock) gcstats::AutoPhase ap(stats, gcstats::PHASE_FINALIZE_START); callFinalizeCallbacks(&fop, JSFINALIZE_GROUP_START); { - gcstats::AutoPhase ap2(stats, gcstats::PHASE_WEAK_ZONEGROUP_CALLBACK); - callWeakPointerZoneGroupCallbacks(); + AutoPhase ap2(stats(), PHASE_WEAK_ZONEGROUP_CALLBACK); + callWeakPointerZonesCallbacks(); } { - gcstats::AutoPhase ap2(stats, gcstats::PHASE_WEAK_COMPARTMENT_CALLBACK); - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { + AutoPhase ap2(stats(), PHASE_WEAK_COMPARTMENT_CALLBACK); + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) callWeakPointerCompartmentCallbacks(comp); } @@ -4700,17 +5168,10 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock) * Objects are finalized immediately but this may change in the future. */ - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { - gcstats::AutoSCC scc(stats, zoneGroupIndex); - zone->arenas.queueForegroundObjectsForSweep(&fop); - } - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { - gcstats::AutoSCC scc(stats, zoneGroupIndex); - for (unsigned i = 0; i < ArrayLength(IncrementalFinalizePhases); ++i) - zone->arenas.queueForForegroundSweep(&fop, IncrementalFinalizePhases[i]); - } - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { - gcstats::AutoSCC scc(stats, zoneGroupIndex); + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { + gcstats::AutoSCC scc(stats(), sweepGroupIndex); + zone->arenas.queueForForegroundSweep(fop, ForegroundObjectFinalizePhase); + zone->arenas.queueForForegroundSweep(fop, ForegroundNonObjectFinalizePhase); for (unsigned i = 0; i < ArrayLength(BackgroundFinalizePhases); ++i) zone->arenas.queueForBackgroundSweep(&fop, BackgroundFinalizePhases[i]); } @@ -4724,7 +5185,19 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock) sweepActionIndex = 0; { - gcstats::AutoPhase ap(stats, gcstats::PHASE_FINALIZE_END); + return NotFinished; + } + + return Finished; +} +#endif + +IncrementalProgress +GCRuntime::endSweepingSweepGroup(FreeOp* fop, SliceBudget& budget) +{ + { + gcstats::AutoPhase ap(stats(), gcstats::PHASE_FINALIZE_END); + FreeOp fop(rt); callFinalizeCallbacks(&fop, JSFINALIZE_GROUP_END); } } @@ -4733,7 +5206,7 @@ void GCRuntime::endSweepingZoneGroup() { /* Update the GC state for zones we have swept. */ - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) { MOZ_ASSERT(zone->isGCSweeping()); AutoLockGC lock(rt); zone->setGCState(Zone::Finished); @@ -4743,7 +5216,7 @@ GCRuntime::endSweepingZoneGroup() /* Start background thread to sweep zones if required. */ ZoneList zones; - for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) + for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) zones.append(zone); if (sweepOnBackgroundThread) queueZonesForBackgroundSweep(zones); @@ -4826,7 +5299,7 @@ GCRuntime::drainMarkStack(SliceBudget& sliceBudget, gcstats::Phase phase) static void SweepThing(Shape* shape) { - if (!shape->isMarked()) + if (!shape->isMarkedAny()) shape->sweep(); } @@ -4981,7 +5454,44 @@ GCRuntime::initializeSweepActions() AddSweepPhase(&ok); AddSweepAction(&ok, GCRuntime::sweepShapeTree); - return ok; + using Action = typename RemoveLastTemplateParameter< + SweepActionForEach, AllocKinds, Args...>>::Type; + return js::MakeUnique(kinds, Move(action)); +} + +} // namespace sweepaction + +bool +GCRuntime::initSweepActions() +{ + using namespace sweepaction; + using sweepaction::Call; + + sweepActions.ref() = + RepeatForZoneGroup(rt, + Sequence( + Call(&GCRuntime::endMarkingSweepGroup), + Call(&GCRuntime::beginSweepingSweepGroup), +#ifdef JS_GC_ZEAL + Call(&GCRuntime::maybeYieldForSweepingZeal), +#endif + Call(&GCRuntime::sweepAtomsTable), + Call(&GCRuntime::sweepWeakCaches), + ForEachZoneInZoneGroup(rt, + ForEachAllocKind(ForegroundObjectFinalizePhase.kinds, + Call(&GCRuntime::finalizeAllocKind))), + ForEachZoneInZoneGroup(rt, + Sequence( + Call(&GCRuntime::sweepTypeInformation), + Call(&GCRuntime::mergeSweptObjectArenas))), + ForEachZoneInZoneGroup(rt, + ForEachAllocKind(ForegroundNonObjectFinalizePhase.kinds, + Call(&GCRuntime::finalizeAllocKind))), + ForEachZoneInZoneGroup(rt, + Call(&GCRuntime::sweepShapeTree)), + Call(&GCRuntime::endSweepingSweepGroup))); + + return sweepActions != nullptr; } IncrementalProgress @@ -5816,7 +6326,7 @@ GCRuntime::maybeDoCycleCollection() for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) { ++compartmentsTotal; GlobalObject* global = c->unsafeUnbarrieredMaybeGlobal(); - if (global && global->asTenured().isMarked(GRAY)) + if (global && global->asTenured().isMarkedGray()) ++compartmentsGray; } double grayFraction = double(compartmentsGray) / double(compartmentsTotal); diff --git a/js/src/jsgcinlines.h b/js/src/jsgcinlines.h index 1c96d37c00..dfea46278b 100644 --- a/js/src/jsgcinlines.h +++ b/js/src/jsgcinlines.h @@ -434,15 +434,14 @@ class GCZonesIter typedef CompartmentsIterT GCCompartmentsIter; -/* Iterates over all zones in the current zone group. */ -class GCZoneGroupIter { - private: +/* Iterates over all zones in the current sweep group. */ +class GCSweepGroupIter { JS::Zone* current; public: - explicit GCZoneGroupIter(JSRuntime* rt) { + explicit GCSweepGroupIter(JSRuntime* rt) { MOZ_ASSERT(CurrentThreadIsPerformingGC()); - current = rt->gc.getCurrentZoneGroup(); + current = rt->gc.getCurrentSweepGroup(); } bool done() const { return !current; } @@ -461,7 +460,7 @@ class GCZoneGroupIter { JS::Zone* operator->() const { return get(); } }; -typedef CompartmentsIterT GCCompartmentGroupIter; +typedef CompartmentsIterT GCCompartmentGroupIter; inline void RelocationOverlay::forwardTo(Cell* cell) diff --git a/js/src/jsscript.cpp b/js/src/jsscript.cpp index 41fabb1724..857c4170a1 100644 --- a/js/src/jsscript.cpp +++ b/js/src/jsscript.cpp @@ -3417,7 +3417,7 @@ js::detail::CopyScript(JSContext* cx, HandleScript src, HandleScript dst, /* NB: Keep this in sync with XDRScript. */ /* Some embeddings are not careful to use ExposeObjectToActiveJS as needed. */ - MOZ_ASSERT(!src->sourceObject()->asTenured().isMarked(gc::GRAY)); + MOZ_ASSERT(!src->sourceObject()->asTenured().isMarkedGray()); uint32_t nconsts = src->hasConsts() ? src->consts()->length : 0; uint32_t nobjects = src->hasObjects() ? src->objects()->length : 0; @@ -4161,24 +4161,27 @@ JSScript::argumentsOptimizationFailed(JSContext* cx, HandleScript script) * assumption of !script->needsArgsObj(); * - type inference data for the script assuming script->needsArgsObj */ - for (AllScriptFramesIter i(cx); !i.done(); ++i) { - /* - * We cannot reliably create an arguments object for Ion activations of - * this script. To maintain the invariant that "script->needsArgsObj - * implies fp->hasArgsObj", the Ion bail mechanism will create an - * arguments object right after restoring the BaselineFrame and before - * entering Baseline code (in jit::FinishBailoutToBaseline). - */ - if (i.isIon()) - continue; - AbstractFramePtr frame = i.abstractFramePtr(); - if (frame.isFunctionFrame() && frame.script() == script) { - /* We crash on OOM since cleaning up here would be complicated. */ - AutoEnterOOMUnsafeRegion oomUnsafe; - ArgumentsObject* argsobj = ArgumentsObject::createExpected(cx, frame); - if (!argsobj) - oomUnsafe.crash("JSScript::argumentsOptimizationFailed"); - SetFrameArgumentsObject(cx, frame, script, argsobj); + JSRuntime::AutoProhibitActiveContextChange apacc(cx->runtime()); + for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) { + for (AllScriptFramesIter i(cx, target); !i.done(); ++i) { + /* + * We cannot reliably create an arguments object for Ion activations of + * this script. To maintain the invariant that "script->needsArgsObj + * implies fp->hasArgsObj", the Ion bail mechanism will create an + * arguments object right after restoring the BaselineFrame and before + * entering Baseline code (in jit::FinishBailoutToBaseline). + */ + if (i.isIon()) + continue; + AbstractFramePtr frame = i.abstractFramePtr(); + if (frame.isFunctionFrame() && frame.script() == script) { + /* We crash on OOM since cleaning up here would be complicated. */ + AutoEnterOOMUnsafeRegion oomUnsafe; + ArgumentsObject* argsobj = ArgumentsObject::createExpected(cx, frame); + if (!argsobj) + oomUnsafe.crash("JSScript::argumentsOptimizationFailed"); + SetFrameArgumentsObject(cx, frame, script, argsobj); + } } } diff --git a/js/src/jsweakmap.cpp b/js/src/jsweakmap.cpp index 03d1a0847e..2fdc0448b7 100644 --- a/js/src/jsweakmap.cpp +++ b/js/src/jsweakmap.cpp @@ -138,7 +138,7 @@ ObjectValueMap::findZoneEdges() JS::AutoSuppressGCAnalysis nogc; for (Range r = all(); !r.empty(); r.popFront()) { JSObject* key = r.front().key(); - if (key->asTenured().isMarked(BLACK) && !key->asTenured().isMarked(GRAY)) + if (key->asTenured().isMarkedAny() && !key->asTenured().isMarkedGray()) continue; JSObject* delegate = getDelegate(key); if (!delegate) @@ -146,7 +146,7 @@ ObjectValueMap::findZoneEdges() Zone* delegateZone = delegate->zone(); if (delegateZone == zone || !delegateZone->shouldMarkInZone()) continue; - if (!delegateZone->gcZoneGroupEdges.put(key->zone())) + if (!delegateZone->gcSweepGroupEdges().put(key->zone())) return false; } return true; diff --git a/js/src/proxy/Wrapper.cpp b/js/src/proxy/Wrapper.cpp index 5ce1f22c9a..7de6d1f623 100644 --- a/js/src/proxy/Wrapper.cpp +++ b/js/src/proxy/Wrapper.cpp @@ -329,8 +329,16 @@ Wrapper::wrappedObject(JSObject* wrapper) { MOZ_ASSERT(wrapper->is()); JSObject* target = wrapper->as().target(); - if (target) - JS::ExposeObjectToActiveJS(target); + // Eagerly unmark gray wrapper targets so we can assert that we don't create + // black to gray edges. An incremental GC will eventually mark the targets + // of black wrappers black but while it is in progress we can observe gray + // targets. Expose rather than returning a gray object in this case. + if (target) { + if (wrapper->isMarkedAny() && !wrapper->isMarkedGray()) + MOZ_ASSERT(JS::ObjectIsNotGray(target)); + if (!wrapper->isMarkedGray()) + JS::ExposeObjectToActiveJS(target); + } return target; } diff --git a/js/src/shell/js.cpp b/js/src/shell/js.cpp index d526ea98a6..7b63a78ef1 100644 --- a/js/src/shell/js.cpp +++ b/js/src/shell/js.cpp @@ -1713,8 +1713,7 @@ Evaluate(JSContext* cx, unsigned argc, Value* vp) .setFileAndLine("@evaluate", 1); global = JS_GetGlobalForObject(cx, &args.callee()); - if (!global) - return false; + MOZ_ASSERT(global); if (args.length() == 2) { RootedObject opts(cx, &args[1].toObject()); @@ -1753,6 +1752,41 @@ Evaluate(JSContext* cx, unsigned argc, Value* vp) return false; } } + + if (!JS_GetProperty(cx, opts, "zoneGroup", &v)) + return false; + if (!v.isUndefined()) { + if (global != JS_GetGlobalForObject(cx, &args.callee())) { + JS_ReportErrorASCII(cx, "zoneGroup and global cannot both be specified."); + return false; + } + + // Find all eligible globals to execute in: any global in another + // zone group which has not been entered by a cooperative thread. + JS::AutoObjectVector eligibleGlobals(cx); + for (CompartmentsIter c(cx->runtime(), SkipAtoms); !c.done(); c.next()) { + if (!c->zone()->group()->ownerContext().context() && + c->maybeGlobal() && + !cx->runtime()->isSelfHostingGlobal(c->maybeGlobal())) + { + if (!eligibleGlobals.append(c->maybeGlobal())) + return false; + } + } + + if (eligibleGlobals.empty()) { + JS_ReportErrorASCII(cx, "zoneGroup can only be used if another" + " cooperative thread has called cooperativeYield(true)."); + return false; + } + + // Pick an eligible global to use based on the value of the zoneGroup property. + int32_t which; + if (!ToInt32(cx, v, &which)) + return false; + which = Min(Max(which, 0), eligibleGlobals.length() - 1); + global = eligibleGlobals[which]; + } if (!JS_GetProperty(cx, opts, "catchTermination", &v)) return false; @@ -3440,6 +3474,137 @@ EvalInContext(JSContext* cx, unsigned argc, Value* vp) return true; } +struct CooperationState +{ + CooperationState() + : lock(mutexid::ShellThreadCooperation) + , idle(false) + , numThreads(0) + , yieldCount(0) + , singleThreaded(false) + {} + + Mutex lock; + ConditionVariable cvar; + bool idle; + size_t numThreads; + uint64_t yieldCount; + bool singleThreaded; +}; +static CooperationState* cooperationState = nullptr; + +static void +CooperativeBeginWait(JSContext* cx) +{ + MOZ_ASSERT(cx == TlsContext.get()); + JS_YieldCooperativeContext(cx); +} + +static void +CooperativeEndWait(JSContext* cx) +{ + MOZ_ASSERT(cx == TlsContext.get()); + LockGuard lock(cooperationState->lock); + + cooperationState->cvar.wait(lock, [&] { return cooperationState->idle; }); + + JS_ResumeCooperativeContext(cx); + cooperationState->idle = false; + cooperationState->yieldCount++; + cooperationState->cvar.notify_all(); +} + +static void +CooperativeYield() +{ + LockGuard lock(cooperationState->lock); + MOZ_ASSERT(!cooperationState->idle); + cooperationState->idle = true; + cooperationState->cvar.notify_all(); + + // Wait until another thread takes over control before returning, if there + // is another thread to do so. + if (cooperationState->numThreads) { + uint64_t count = cooperationState->yieldCount; + cooperationState->cvar.wait(lock, [&] { return cooperationState->yieldCount != count; }); + } +} + +static bool +CooperativeYieldThread(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + + if (!cx->runtime()->gc.canChangeActiveContext(cx)) { + JS_ReportErrorASCII(cx, "Cooperating multithreading context switches are not currently allowed"); + return false; + } + + if (GetShellContext(cx)->isWorker) { + JS_ReportErrorASCII(cx, "Worker threads cannot yield"); + return false; + } + + if (cooperationState->singleThreaded) { + JS_ReportErrorASCII(cx, "Yielding is not allowed while single threaded"); + return false; + } + // To avoid contention issues between threads, yields are not allowed while + // a thread has access to zone groups other than its original one, i.e. if + // the thread is inside an evaluate() call with a different zone group. + // This is not a limit which the browser has, but is necessary in the + // shell: the shell can have arbitrary interleavings between cooperative + // threads, whereas the browser has more control over which threads are + // running at different times. + for (ZoneGroupsIter group(cx->runtime()); !group.done(); group.next()) { + if (group->ownerContext().context() == cx && group != cx->zone()->group()) { + JS_ReportErrorASCII(cx, "Yielding is not allowed while owning multiple zone groups"); + return false; + } + } + + { + Maybe artzg; + if ((args.length() > 0) && ToBoolean(args[0])) + artzg.emplace(cx); + + CooperativeBeginWait(cx); + CooperativeYield(); + CooperativeEndWait(cx); + } + + args.rval().setUndefined(); + return true; +} + +static void +CooperativeBeginSingleThreadedExecution(JSContext* cx) +{ + MOZ_ASSERT(!cooperationState->singleThreaded); + + // Yield until all other threads have exited any zone groups they are in. + while (true) { + bool done = true; + for (ZoneGroupsIter group(cx->runtime()); !group.done(); group.next()) { + if (!group->ownedByCurrentThread() && group->ownerContext().context()) + done = false; + } + if (done) + break; + CooperativeBeginWait(cx); + CooperativeYield(); + CooperativeEndWait(cx); + } + + cooperationState->singleThreaded = true; +} + +static void +CooperativeEndSingleThreadedExecution(JSContext* cx) +{ + if (cooperationState) + cooperationState->singleThreaded = false; +} struct WorkerInput { JSContext* context; @@ -6271,6 +6436,16 @@ static const JSFunctionSpecWithHelp shell_functions[] = { "evalInWorker(str)", " Evaluate 'str' in a separate thread with its own runtime.\n"), + JS_FN_HELP("evalInCooperativeThread", EvalInCooperativeThread, 1, 0, +"evalInCooperativeThread(str)", +" Evaluate 'str' in a separate cooperatively scheduled thread using the same runtime.\n"), + + JS_FN_HELP("cooperativeYield", CooperativeYieldThread, 1, 0, +"cooperativeYield(leaveZoneGroup)", +" Yield execution to another cooperatively scheduled thread using the same runtime.\n" +" If leaveZoneGroup is specified then other threads may execute code in the\n" +" current thread's zone group via evaluate(..., {zoneGroup:N}).\n"), + JS_FN_HELP("getSharedArrayBuffer", GetSharedArrayBuffer, 0, 0, "getSharedArrayBuffer()", " Retrieve the SharedArrayBuffer object from the cross-worker mailbox.\n" diff --git a/js/src/vm/Caches.h b/js/src/vm/Caches.h index bb7774ac10..7c34295c95 100644 --- a/js/src/vm/Caches.h +++ b/js/src/vm/Caches.h @@ -31,7 +31,7 @@ namespace js { struct GSNCache { typedef HashMap, + PointerHasher, SystemAllocPolicy> Map; jsbytecode* code; diff --git a/js/src/vm/Debugger.cpp b/js/src/vm/Debugger.cpp index 344ee3a4d1..7a421b567d 100644 --- a/js/src/vm/Debugger.cpp +++ b/js/src/vm/Debugger.cpp @@ -2558,21 +2558,23 @@ UpdateExecutionObservabilityOfScriptsInZone(JSContext* cx, Zone* zone, // // Mark active baseline scripts in the observable set so that they don't // get discarded. They will be recompiled. - for (JitActivationIterator actIter(rt); !actIter.done(); ++actIter) { - if (actIter->compartment()->zone() != zone) - continue; + for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) { + for (JitActivationIterator actIter(cx, target); !actIter.done(); ++actIter) { + if (actIter->compartment()->zone() != zone) + continue; - for (JitFrameIterator iter(actIter); !iter.done(); ++iter) { - switch (iter.type()) { - case JitFrame_BaselineJS: - MarkBaselineScriptActiveIfObservable(iter.script(), obs); - break; - case JitFrame_IonJS: - MarkBaselineScriptActiveIfObservable(iter.script(), obs); - for (InlineFrameIterator inlineIter(rt, &iter); inlineIter.more(); ++inlineIter) - MarkBaselineScriptActiveIfObservable(inlineIter.script(), obs); - break; - default:; + for (JitFrameIterator iter(actIter); !iter.done(); ++iter) { + switch (iter.type()) { + case JitFrame_BaselineJS: + MarkBaselineScriptActiveIfObservable(iter.script(), obs); + break; + case JitFrame_IonJS: + MarkBaselineScriptActiveIfObservable(iter.script(), obs); + for (InlineFrameIterator inlineIter(cx, &iter); inlineIter.more(); ++inlineIter) + MarkBaselineScriptActiveIfObservable(inlineIter.script(), obs); + break; + default:; + } } } } diff --git a/js/src/vm/ObjectGroup.cpp b/js/src/vm/ObjectGroup.cpp index 408e346608..56039661bd 100644 --- a/js/src/vm/ObjectGroup.cpp +++ b/js/src/vm/ObjectGroup.cpp @@ -412,12 +412,9 @@ struct ObjectGroupCompartment::NewEntry } static inline HashNumber hash(const Lookup& lookup) { - MOZ_ASSERT(lookup.proto.hasUniqueId()); - MOZ_ASSERT(lookup.hasAssocId()); - HashNumber hash = uintptr_t(lookup.clasp); - hash = mozilla::RotateLeft(hash, 4) ^ Zone::UniqueIdToHash(lookup.proto.uniqueId()); - hash = mozilla::RotateLeft(hash, 4) ^ Zone::UniqueIdToHash(lookup.getAssocId()); - return hash; + HashNumber hash = MovableCellHasher::hash(lookup.proto); + hash = mozilla::AddToHash(hash, MovableCellHasher::hash(lookup.associated)); + return mozilla::AddToHash(hash, mozilla::HashGeneric(lookup.clasp)); } static inline bool match(const ObjectGroupCompartment::NewEntry& key, const Lookup& lookup) { @@ -1116,8 +1113,8 @@ struct ObjectGroupCompartment::PlainObjectKey }; static inline HashNumber hash(const Lookup& lookup) { - return (HashNumber) (HashId(lookup.properties[lookup.nproperties - 1].id) ^ - lookup.nproperties); + HashNumber hash = HashId(lookup.properties[lookup.nproperties - 1].id); + return mozilla::AddToHash(hash, lookup.nproperties); } static inline bool match(const PlainObjectKey& v, const Lookup& lookup) { diff --git a/js/src/vm/RegExpShared.h b/js/src/vm/RegExpShared.h new file mode 100644 index 0000000000..e1e6dad802 --- /dev/null +++ b/js/src/vm/RegExpShared.h @@ -0,0 +1,410 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The compiled representation of a RegExp, potentially shared among RegExp instances created + * during separate evaluations of a single RegExp literal in source code. + */ + +#ifndef vm_RegExpShared_h +#define vm_RegExpShared_h + +#include "mozilla/Assertions.h" +#include "mozilla/MemoryReporting.h" + +#include "jsalloc.h" +#include "jsatom.h" + +#include "builtin/SelfHostingDefines.h" +#include "gc/Barrier.h" +#include "gc/Heap.h" +#include "gc/Marking.h" +#include "js/UbiNode.h" +#include "js/Vector.h" +#include "irregexp/InfallibleVector.h" +#include "vm/ArrayObject.h" + +struct JSContext; + +namespace js { + +class ArrayObject; +class MatchPairs; +class RegExpCompartment; +class RegExpShared; +class RegExpStatics; + +using RootedRegExpShared = JS::Rooted; +using HandleRegExpShared = JS::Handle; +using MutableHandleRegExpShared = JS::MutableHandle; + +enum RegExpFlag : uint8_t +{ + IgnoreCaseFlag = 0x01, + GlobalFlag = 0x02, + MultilineFlag = 0x04, + StickyFlag = 0x08, + UnicodeFlag = 0x10, + DotAllFlag = 0x20, + HasIndicesFlag = 0x40, + + NoFlags = 0x00, + AllFlags = 0x7f +}; + +static_assert(IgnoreCaseFlag == REGEXP_IGNORECASE_FLAG && + GlobalFlag == REGEXP_GLOBAL_FLAG && + MultilineFlag == REGEXP_MULTILINE_FLAG && + StickyFlag == REGEXP_STICKY_FLAG && + UnicodeFlag == REGEXP_UNICODE_FLAG && + DotAllFlag == REGEXP_DOTALL_FLAG, + "Flag values should be in sync with self-hosted JS"); + +enum RegExpRunStatus +{ + RegExpRunStatus_Error, + RegExpRunStatus_Success, + RegExpRunStatus_Success_NotFound +}; + +/* + * A RegExpShared is the compiled representation of a regexp. A RegExpShared is + * potentially pointed to by multiple RegExpObjects. Additionally, C++ code may + * have pointers to RegExpShareds on the stack. The RegExpShareds are kept in a + * table so that they can be reused when compiling the same regex string. + * + * To save memory, a RegExpShared is not created for a RegExpObject until it is + * needed for execution. When a RegExpShared needs to be created, it is looked + * up in a per-compartment table to allow reuse between objects. + * + * During a GC, RegExpShared instances are marked and swept like GC things. + * Usually, RegExpObjects clear their pointers to their RegExpShareds rather + * than explicitly tracing them, so that the RegExpShared and any jitcode can + * be reclaimed quicker. However, the RegExpShareds are traced through by + * objects when we are preserving jitcode in their zone, to avoid the same + * recompilation inefficiencies as normal Ion and baseline compilation. + */ +class RegExpShared : public gc::TenuredCell +{ + public: + enum CompilationMode { + Normal, + MatchOnly + }; + + enum ForceByteCodeEnum { + DontForceByteCode, + ForceByteCode + }; + + using JitCodeTable = UniquePtr; + using JitCodeTables = Vector; + + private: + friend class RegExpStatics; + friend class RegExpZone; + + struct RegExpCompilation + { + ReadBarriered jitCode; + uint8_t* byteCode; + + RegExpCompilation() : byteCode(nullptr) {} + + bool compiled(ForceByteCodeEnum force = DontForceByteCode) const { + return byteCode || (force == DontForceByteCode && jitCode); + } + }; + + /* Source to the RegExp, for lazy compilation. */ + GCPtr source; + + RegExpFlag flags; + bool canStringMatch; + size_t parenCount; + + uint32_t numNamedCaptures_; + GCPtr groupsTemplate_; + + RegExpCompilation compilationArray[4]; + + static int CompilationIndex(CompilationMode mode, bool latin1) { + switch (mode) { + case Normal: return latin1 ? 0 : 1; + case MatchOnly: return latin1 ? 2 : 3; + } + MOZ_CRASH(); + } + + // Tables referenced by JIT code. + JitCodeTables tables; + + /* Internal functions. */ + RegExpShared(JSAtom* source, RegExpFlag flags); + + static bool compile(JSContext* cx, MutableHandleRegExpShared res, HandleLinearString input, + CompilationMode mode, ForceByteCodeEnum force); + static bool compile(JSContext* cx, MutableHandleRegExpShared res, HandleAtom pattern, + HandleLinearString input, CompilationMode mode, ForceByteCodeEnum force); + + static bool compileIfNecessary(JSContext* cx, MutableHandleRegExpShared res, + HandleLinearString input, CompilationMode mode, + ForceByteCodeEnum force); + + const RegExpCompilation& compilation(CompilationMode mode, bool latin1) const { + return compilationArray[CompilationIndex(mode, latin1)]; + } + + RegExpCompilation& compilation(CompilationMode mode, bool latin1) { + return compilationArray[CompilationIndex(mode, latin1)]; + } + + public: + ~RegExpShared() = delete; + + // Execute this RegExp on input starting from searchIndex, filling in + // matches if specified and otherwise only determining if there is a match. + static RegExpRunStatus execute(JSContext* cx, MutableHandleRegExpShared res, + HandleLinearString input, size_t searchIndex, + MatchPairs* matches, size_t* endIndex); + + // Register a table with this RegExpShared, and take ownership. + bool addTable(JitCodeTable table) { + return tables.append(Move(table)); + } + + /* Accessors */ + + size_t getParenCount() const { + MOZ_ASSERT(isCompiled()); + return parenCount; + } + + /* Accounts for the "0" (whole match) pair. */ + size_t pairCount() const { return getParenCount() + 1; } + + // not public due to circular inclusion problems + static bool initializeNamedCaptures(JSContext* cx, MutableHandleRegExpShared re, irregexp::CharacterVectorVector* names, irregexp::IntegerVector* indices); + PlainObject* getGroupsTemplate() { return groupsTemplate_; } + uint32_t numNamedCaptures() const { return numNamedCaptures_; } + JSAtom* getSource() const { return source; } + RegExpFlag getFlags() const { return flags; } + bool hasIndices() const { return flags & HasIndicesFlag; } + bool global() const { return flags & GlobalFlag; } + bool ignoreCase() const { return flags & IgnoreCaseFlag; } + bool multiline() const { return flags & MultilineFlag; } + bool dotAll() const { return flags & DotAllFlag; } + bool unicode() const { return flags & UnicodeFlag; } + bool sticky() const { return flags & StickyFlag; } + + bool isCompiled(CompilationMode mode, bool latin1, + ForceByteCodeEnum force = DontForceByteCode) const { + return compilation(mode, latin1).compiled(force); + } + bool isCompiled() const { + return isCompiled(Normal, true) || isCompiled(Normal, false) + || isCompiled(MatchOnly, true) || isCompiled(MatchOnly, false); + } + + void traceChildren(JSTracer* trc); + void discardJitCode(); + void finalize(FreeOp* fop); + + static size_t offsetOfSource() { + return offsetof(RegExpShared, source); + } + + static size_t offsetOfFlags() { + return offsetof(RegExpShared, flags); + } + + static size_t offsetOfParenCount() { + return offsetof(RegExpShared, parenCount); + } + + static size_t offsetOfLatin1JitCode(CompilationMode mode) { + return offsetof(RegExpShared, compilationArray) + + (CompilationIndex(mode, true) * sizeof(RegExpCompilation)) + + offsetof(RegExpCompilation, jitCode); + } + static size_t offsetOfTwoByteJitCode(CompilationMode mode) { + return offsetof(RegExpShared, compilationArray) + + (CompilationIndex(mode, false) * sizeof(RegExpCompilation)) + + offsetof(RegExpCompilation, jitCode); + } + + static size_t offsetOfGroupsTemplate() { + return offsetof(RegExpShared, groupsTemplate_); + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf); + +#ifdef DEBUG + static bool dumpBytecode(JSContext* cx, MutableHandleRegExpShared res, bool match_only, + HandleLinearString input); +#endif +}; + +class RegExpZone +{ + struct Key { + JSAtom* atom; + uint16_t flag; + + Key() {} + Key(JSAtom* atom, RegExpFlag flag) + : atom(atom), flag(flag) + { } + MOZ_IMPLICIT Key(const ReadBarriered& shared) + : atom(shared.unbarrieredGet()->getSource()), + flag(shared.unbarrieredGet()->getFlags()) + { } + + typedef Key Lookup; + static HashNumber hash(const Lookup& l) { + HashNumber hash = DefaultHasher::hash(l.atom); + return mozilla::AddToHash(hash, l.flag); + } + static bool match(Key l, Key r) { + return l.atom == r.atom && l.flag == r.flag; + } + }; + + /* + * The set of all RegExpShareds in the zone. On every GC, every RegExpShared + * that was not marked is deleted and removed from the set. + */ + using Set = JS::WeakCache, Key, ZoneAllocPolicy>>; + Set set_; + + public: + explicit RegExpZone(Zone* zone); + + ~RegExpZone() { + MOZ_ASSERT_IF(set_.initialized(), set_.empty()); + } + + bool init(); + + bool empty() const { return set_.empty(); } + + bool get(JSContext* cx, HandleAtom source, RegExpFlag flags, MutableHandleRegExpShared shared); + + /* Like 'get', but compile 'maybeOpt' (if non-null). */ + bool get(JSContext* cx, HandleAtom source, JSString* maybeOpt, + MutableHandleRegExpShared shared); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf); +}; + +class RegExpCompartment +{ +public: + enum ResultTemplateKind { Normal, WithIndices, Indices, NumKinds }; + +private: + /* + * The template objects that the result of re.exec() is based on, if + * there is a result. These are used in CreateRegExpMatchResult. + * There are three template objects, each of which is an ArrayObject + * with some additional properties. We decide which to use based on + * the |hasIndices| (/d) flag. + * + * Normal: Has |index|, |input|, and |groups| properties. + * Used for the result object if |hasIndices| is not set. + * + * WithIndices: Has |index|, |input|, |groups|, and |indices| properties. + * Used for the result object if |hasIndices| is set. + * + * Indices: Has a |groups| property. If |hasIndices| is set, used + * for the |.indices| property of the result object. + */ + ReadBarriered matchResultTemplateObjects_[ResultTemplateKind::NumKinds]; + + /* + * The shape of RegExp.prototype object that satisfies following: + * * RegExp.prototype.flags getter is not modified + * * RegExp.prototype.global getter is not modified + * * RegExp.prototype.ignoreCase getter is not modified + * * RegExp.prototype.multiline getter is not modified + * * RegExp.prototype.sticky getter is not modified + * * RegExp.prototype.unicode getter is not modified + * * RegExp.prototype.exec is an own data property + * * RegExp.prototype[@@match] is an own data property + * * RegExp.prototype[@@search] is an own data property + */ + ReadBarriered optimizableRegExpPrototypeShape_; + + /* + * The shape of RegExp instance that satisfies following: + * * lastProperty is lastIndex + * * prototype is RegExp.prototype + */ + ReadBarriered optimizableRegExpInstanceShape_; + + ArrayObject* createMatchResultTemplateObject(JSContext* cx, ResultTemplateKind kind); + + public: + explicit RegExpCompartment(Zone* zone); + + void sweep(JSRuntime* rt); + + /* Get or create template object used to base the result of .exec() on. */ + ArrayObject* getOrCreateMatchResultTemplateObject(JSContext* cx, ResultTemplateKind kind = ResultTemplateKind::Normal) { + if (matchResultTemplateObjects_[kind]) + return matchResultTemplateObjects_[kind]; + return createMatchResultTemplateObject(cx, kind); + } + + Shape* getOptimizableRegExpPrototypeShape() { + return optimizableRegExpPrototypeShape_; + } + void setOptimizableRegExpPrototypeShape(Shape* shape) { + optimizableRegExpPrototypeShape_ = shape; + } + Shape* getOptimizableRegExpInstanceShape() { + return optimizableRegExpInstanceShape_; + } + void setOptimizableRegExpInstanceShape(Shape* shape) { + optimizableRegExpInstanceShape_ = shape; + } + + static size_t offsetOfOptimizableRegExpPrototypeShape() { + return offsetof(RegExpCompartment, optimizableRegExpPrototypeShape_); + } + static size_t offsetOfOptimizableRegExpInstanceShape() { + return offsetof(RegExpCompartment, optimizableRegExpInstanceShape_); + } +}; + +} /* namespace js */ + +namespace JS { +namespace ubi { + +template <> +class Concrete : TracerConcrete +{ + protected: + explicit Concrete(js::RegExpShared* ptr) : TracerConcrete(ptr) { } + + public: + static void construct(void* storage, js::RegExpShared* ptr) { + new (storage) Concrete(ptr); + } + + CoarseType coarseType() const final { return CoarseType::Other; } + + Size size(mozilla::MallocSizeOf mallocSizeOf) const override; + + const char16_t* typeName() const override { return concreteTypeName; } + static const char16_t concreteTypeName[]; +}; + +} // namespace ubi +} // namespace JS + +#endif /* vm_RegExpShared_h */ \ No newline at end of file diff --git a/js/src/vm/Runtime.h b/js/src/vm/Runtime.h index 6f781d101c..02df80e18c 100644 --- a/js/src/vm/Runtime.h +++ b/js/src/vm/Runtime.h @@ -87,7 +87,7 @@ namespace js { extern MOZ_COLD void ReportOutOfMemory(ExclusiveContext* cx); -/* Different signature because the return type has MOZ_MUST_USE_TYPE. */ +/* Different signature because the return type has [[nodiscard]]_TYPE. */ extern MOZ_COLD mozilla::GenericErrorResult ReportOutOfMemoryResult(ExclusiveContext* cx); diff --git a/js/src/vm/SavedFrame.h b/js/src/vm/SavedFrame.h index 6995cbb359..373adc6746 100644 --- a/js/src/vm/SavedFrame.h +++ b/js/src/vm/SavedFrame.h @@ -187,7 +187,7 @@ struct SavedFrame::HashPolicy { typedef SavedFrame::Lookup Lookup; typedef MovableCellHasher SavedFramePtrHasher; - typedef PointerHasher JSPrincipalsPtrHasher; + typedef PointerHasher JSPrincipalsPtrHasher; static bool hasHash(const Lookup& l); static bool ensureHash(const Lookup& l); diff --git a/js/src/vm/Shape.cpp b/js/src/vm/Shape.cpp index c71cef5a7e..07d9c58910 100644 --- a/js/src/vm/Shape.cpp +++ b/js/src/vm/Shape.cpp @@ -1504,6 +1504,447 @@ EmptyShape::new_(ExclusiveContext* cx, Handle base, uint32_t return shape; } +MOZ_ALWAYS_INLINE HashNumber +ShapeHasher::hash(const Lookup& l) +{ + return l.hash(); +} + +MOZ_ALWAYS_INLINE bool +ShapeHasher::match(const Key k, const Lookup& l) +{ + return k->matches(l); +} + +static KidsHash* +HashChildren(Shape* kid1, Shape* kid2) +{ + KidsHash* hash = js_new(); + if (!hash || !hash->init(2)) { + js_delete(hash); + return nullptr; + } + + hash->putNewInfallible(StackShape(kid1), kid1); + hash->putNewInfallible(StackShape(kid2), kid2); + return hash; +} + +bool +PropertyTree::insertChild(JSContext* cx, Shape* parent, Shape* child) +{ + MOZ_ASSERT(!parent->inDictionary()); + MOZ_ASSERT(!child->parent); + MOZ_ASSERT(!child->inDictionary()); + MOZ_ASSERT(child->zone() == parent->zone()); + MOZ_ASSERT(cx->zone() == zone_); + + KidsPointer* kidp = &parent->kids; + + if (kidp->isNull()) { + child->setParent(parent); + kidp->setShape(child); + return true; + } + + if (kidp->isShape()) { + Shape* shape = kidp->toShape(); + MOZ_ASSERT(shape != child); + MOZ_ASSERT(!shape->matches(child)); + + KidsHash* hash = HashChildren(shape, child); + if (!hash) { + ReportOutOfMemory(cx); + return false; + } + kidp->setHash(hash); + child->setParent(parent); + return true; + } + + if (!kidp->toHash()->putNew(StackShape(child), child)) { + ReportOutOfMemory(cx); + return false; + } + + child->setParent(parent); + return true; +} + +void +Shape::removeChild(Shape* child) +{ + MOZ_ASSERT(!child->inDictionary()); + MOZ_ASSERT(child->parent == this); + + KidsPointer* kidp = &kids; + + if (kidp->isShape()) { + MOZ_ASSERT(kidp->toShape() == child); + kidp->setNull(); + child->parent = nullptr; + return; + } + + KidsHash* hash = kidp->toHash(); + MOZ_ASSERT(hash->count() >= 2); /* otherwise kidp->isShape() should be true */ + +#ifdef DEBUG + size_t oldCount = hash->count(); +#endif + + hash->remove(StackShape(child)); + child->parent = nullptr; + + MOZ_ASSERT(hash->count() == oldCount - 1); + + if (hash->count() == 1) { + /* Convert from HASH form back to SHAPE form. */ + KidsHash::Range r = hash->all(); + Shape* otherChild = r.front(); + MOZ_ASSERT((r.popFront(), r.empty())); /* No more elements! */ + kidp->setShape(otherChild); + js_delete(hash); + } +} + +MOZ_ALWAYS_INLINE Shape* +PropertyTree::inlinedGetChild(JSContext* cx, Shape* parent, Handle child) +{ + MOZ_ASSERT(parent); + + Shape* existingShape = nullptr; + + /* + * The property tree has extremely low fan-out below its root in + * popular embeddings with real-world workloads. Patterns such as + * defining closures that capture a constructor's environment as + * getters or setters on the new object that is passed in as + * |this| can significantly increase fan-out below the property + * tree root -- see bug 335700 for details. + */ + KidsPointer* kidp = &parent->kids; + if (kidp->isShape()) { + Shape* kid = kidp->toShape(); + if (kid->matches(child)) + existingShape = kid; + } else if (kidp->isHash()) { + if (KidsHash::Ptr p = kidp->toHash()->lookup(child)) + existingShape = *p; + } else { + /* If kidp->isNull(), we always insert. */ + } + + if (existingShape) { + JS::Zone* zone = existingShape->zone(); + if (zone->needsIncrementalBarrier()) { + /* + * We need a read barrier for the shape tree, since these are weak + * pointers. + */ + Shape* tmp = existingShape; + TraceManuallyBarrieredEdge(zone->barrierTracer(), &tmp, "read barrier"); + MOZ_ASSERT(tmp == existingShape); + return existingShape; + } + if (!zone->isGCSweepingOrCompacting() || + !IsAboutToBeFinalizedUnbarriered(&existingShape)) + { + if (existingShape->isMarkedGray()) + UnmarkGrayShapeRecursively(existingShape); + return existingShape; + } + /* + * The shape we've found is unreachable and due to be finalized, so + * remove our weak reference to it and don't use it. + */ + MOZ_ASSERT(parent->isMarkedAny()); + parent->removeChild(existingShape); + } + + RootedShape parentRoot(cx, parent); + Shape* shape = Shape::new_(cx, child, parentRoot->numFixedSlots()); + if (!shape) + return nullptr; + + if (!insertChild(cx, parentRoot, shape)) + return nullptr; + + return shape; +} + +Shape* +PropertyTree::getChild(JSContext* cx, Shape* parent, Handle child) +{ + return inlinedGetChild(cx, parent, child); +} + +void +Shape::sweep() +{ + /* + * We detach the child from the parent if the parent is reachable. + * + * This test depends on shape arenas not being freed until after we finish + * incrementally sweeping them. If that were not the case the parent pointer + * could point to a marked cell that had been deallocated and then + * reallocated, since allocating a cell in a zone that is being marked will + * set the mark bit for that cell. + */ + if (parent && parent->isMarkedAny()) { + if (inDictionary()) { + if (parent->listp == &parent) + parent->listp = nullptr; + } else { + parent->removeChild(this); + } + } +} + +void +Shape::finalize(FreeOp* fop) +{ + if (!inDictionary() && kids.isHash()) + fop->delete_(kids.toHash()); +} + +void +Shape::fixupDictionaryShapeAfterMovingGC() +{ + if (!listp) + return; + + // The listp field either points to the parent field of the next shape in + // the list if there is one. Otherwise if this shape is the last in the + // list then it points to the shape_ field of the object the list is for. + // We can tell which it is because the base shape is owned if this is the + // last property and not otherwise. + bool listpPointsIntoShape = !MaybeForwarded(base())->isOwned(); + +#ifdef DEBUG + // Check that we got this right by interrogating the arena. + // We use a fake cell pointer for this: it might not point to the beginning + // of a cell, but will point into the right arena and will have the right + // alignment. + Cell* cell = reinterpret_cast(uintptr_t(listp) & ~CellAlignMask); + AllocKind kind = TenuredCell::fromPointer(cell)->getAllocKind(); + MOZ_ASSERT_IF(listpPointsIntoShape, IsShapeAllocKind(kind)); + MOZ_ASSERT_IF(!listpPointsIntoShape, IsObjectAllocKind(kind)); +#endif + + if (listpPointsIntoShape) { + // listp points to the parent field of the next shape. + Shape* next = reinterpret_cast(uintptr_t(listp) - offsetof(Shape, parent)); + if (gc::IsForwarded(next)) + listp = &gc::Forwarded(next)->parent; + } else { + // listp points to the shape_ field of an object. + JSObject* last = reinterpret_cast(uintptr_t(listp) - ShapedObject::offsetOfShape()); + if (gc::IsForwarded(last)) + listp = &gc::Forwarded(last)->as().shape_; + } +} + +void +Shape::fixupShapeTreeAfterMovingGC() +{ + if (kids.isNull()) + return; + + if (kids.isShape()) { + if (gc::IsForwarded(kids.toShape())) + kids.setShape(gc::Forwarded(kids.toShape())); + return; + } + + MOZ_ASSERT(kids.isHash()); + KidsHash* kh = kids.toHash(); + for (KidsHash::Enum e(*kh); !e.empty(); e.popFront()) { + Shape* key = e.front(); + if (IsForwarded(key)) + key = Forwarded(key); + + BaseShape* base = key->base(); + if (IsForwarded(base)) + base = Forwarded(base); + UnownedBaseShape* unowned = base->unowned(); + if (IsForwarded(unowned)) + unowned = Forwarded(unowned); + + GetterOp getter = key->getter(); + if (key->hasGetterObject()) + getter = GetterOp(MaybeForwarded(key->getterObject())); + + SetterOp setter = key->setter(); + if (key->hasSetterObject()) + setter = SetterOp(MaybeForwarded(key->setterObject())); + + StackShape lookup(unowned, + const_cast(key)->propidRef(), + key->slotInfo & Shape::SLOT_MASK, + key->attrs, + key->flags); + lookup.updateGetterSetter(getter, setter); + e.rekeyFront(lookup, key); + } +} + +void +Shape::fixupAfterMovingGC() +{ + if (inDictionary()) + fixupDictionaryShapeAfterMovingGC(); + else + fixupShapeTreeAfterMovingGC(); +} + +void +NurseryShapesRef::trace(JSTracer* trc) +{ + auto& shapes = zone_->nurseryShapes(); + for (auto shape : shapes) + shape->fixupGetterSetterForBarrier(trc); + shapes.clearAndFree(); +} + +void +Shape::fixupGetterSetterForBarrier(JSTracer* trc) +{ + if (!hasGetterValue() && !hasSetterValue()) + return; + + JSObject* priorGetter = asAccessorShape().getterObj; + JSObject* priorSetter = asAccessorShape().setterObj; + if (!priorGetter && !priorSetter) + return; + + JSObject* postGetter = priorGetter; + JSObject* postSetter = priorSetter; + if (priorGetter) + TraceManuallyBarrieredEdge(trc, &postGetter, "getterObj"); + if (priorSetter) + TraceManuallyBarrieredEdge(trc, &postSetter, "setterObj"); + if (priorGetter == postGetter && priorSetter == postSetter) + return; + + if (parent && !parent->inDictionary() && parent->kids.isHash()) { + // Relocating the getterObj or setterObj will have changed our location + // in our parent's KidsHash, so take care to update it. We must do this + // before we update the shape itself, since the shape is used to match + // the original entry in the hash set. + + StackShape original(this); + StackShape updated(this); + updated.rawGetter = reinterpret_cast(postGetter); + updated.rawSetter = reinterpret_cast(postSetter); + + KidsHash* kh = parent->kids.toHash(); + MOZ_ALWAYS_TRUE(kh->rekeyAs(original, updated, this)); + } + + asAccessorShape().getterObj = postGetter; + asAccessorShape().setterObj = postSetter; + + MOZ_ASSERT_IF(parent && !parent->inDictionary() && parent->kids.isHash(), + parent->kids.toHash()->has(StackShape(this))); +} + +#ifdef DEBUG + +void +KidsPointer::checkConsistency(Shape* aKid) const +{ + if (isShape()) { + MOZ_ASSERT(toShape() == aKid); + } else { + MOZ_ASSERT(isHash()); + KidsHash* hash = toHash(); + KidsHash::Ptr ptr = hash->lookup(StackShape(aKid)); + MOZ_ASSERT(*ptr == aKid); + } +} + +void +Shape::dump(FILE* fp) const +{ + jsid propid = this->propid(); + + MOZ_ASSERT(!JSID_IS_VOID(propid)); + + if (JSID_IS_INT(propid)) { + fprintf(fp, "[%ld]", (long) JSID_TO_INT(propid)); + } else if (JSID_IS_ATOM(propid)) { + if (JSLinearString* str = JSID_TO_ATOM(propid)) + FileEscapedString(fp, str, '"'); + else + fputs("", fp); + } else { + MOZ_ASSERT(JSID_IS_SYMBOL(propid)); + JSID_TO_SYMBOL(propid)->dump(fp); + } + + fprintf(fp, " g/s %p/%p slot %d attrs %x ", + JS_FUNC_TO_DATA_PTR(void*, getter()), + JS_FUNC_TO_DATA_PTR(void*, setter()), + hasSlot() ? slot() : -1, attrs); + + if (attrs) { + int first = 1; + fputs("(", fp); +#define DUMP_ATTR(name, display) if (attrs & JSPROP_##name) fputs(&(" " #display)[first], fp), first = 0 + DUMP_ATTR(ENUMERATE, enumerate); + DUMP_ATTR(READONLY, readonly); + DUMP_ATTR(PERMANENT, permanent); + DUMP_ATTR(GETTER, getter); + DUMP_ATTR(SETTER, setter); + DUMP_ATTR(SHARED, shared); +#undef DUMP_ATTR + fputs(") ", fp); + } + + fprintf(fp, "flags %x ", flags); + if (flags) { + int first = 1; + fputs("(", fp); +#define DUMP_FLAG(name, display) if (flags & name) fputs(&(" " #display)[first], fp), first = 0 + DUMP_FLAG(IN_DICTIONARY, in_dictionary); +#undef DUMP_FLAG + fputs(") ", fp); + } +} + +void +Shape::dumpSubtree(int level, FILE* fp) const +{ + if (!parent) { + MOZ_ASSERT(level == 0); + MOZ_ASSERT(JSID_IS_EMPTY(propid_)); + fprintf(fp, "class %s emptyShape\n", getObjectClass()->name); + } else { + fprintf(fp, "%*sid ", level, ""); + dump(fp); + } + + if (!kids.isNull()) { + ++level; + if (kids.isShape()) { + Shape* kid = kids.toShape(); + MOZ_ASSERT(kid->parent == this); + kid->dumpSubtree(level, fp); + } else { + const KidsHash& hash = *kids.toHash(); + for (KidsHash::Range range = hash.all(); !range.empty(); range.popFront()) { + Shape* kid = range.front(); + + MOZ_ASSERT(kid->parent == this); + kid->dumpSubtree(level, fp); + } + } + } +} + +#endif + static bool IsOriginalProto(GlobalObject* global, JSProtoKey key, JSObject& proto) { diff --git a/js/src/vm/Shape.h b/js/src/vm/Shape.h index bb813997f0..dfd807d09d 100644 --- a/js/src/vm/Shape.h +++ b/js/src/vm/Shape.h @@ -8,6 +8,7 @@ #include "mozilla/Attributes.h" #include "mozilla/GuardObjects.h" +#include "mozilla/HashFunctions.h" #include "mozilla/MathAlgorithms.h" #include "mozilla/Maybe.h" #include "mozilla/MemoryReporting.h" @@ -560,8 +561,13 @@ struct StackBaseShape : public DefaultHasher> } }; - static inline HashNumber hash(const Lookup& lookup); - static inline bool match(ReadBarriered key, const Lookup& lookup); + static HashNumber hash(const Lookup& lookup) { + return mozilla::HashGeneric(lookup.flags, lookup.clasp); + } + static inline bool match(const ReadBarriered& key, const Lookup& lookup) { + return key.unbarrieredGet()->flags == lookup.flags && + key.unbarrieredGet()->clasp_ == lookup.clasp; + } }; static MOZ_ALWAYS_INLINE js::HashNumber @@ -1220,6 +1226,32 @@ class InitialShapeProto void setProto(TaggedProto proto) { proto_ = proto; } + + bool operator==(const InitialShapeProto& other) const { + return key_ == other.key_ && proto_ == other.proto_; + } +}; + +template <> +struct MovableCellHasher>> +{ + using Key = InitialShapeProto>; + using Lookup = InitialShapeProto; + + static bool hasHash(const Lookup& l) { + return MovableCellHasher::hasHash(l.proto()); + } + static bool ensureHash(const Lookup& l) { + return MovableCellHasher::ensureHash(l.proto()); + } + static HashNumber hash(const Lookup& l) { + HashNumber hash = MovableCellHasher::hash(l.proto()); + return mozilla::AddToHash(hash, l.key()); + } + static bool match(const Key& k, const Lookup& l) { + return k.key() == l.key() && + MovableCellHasher::match(k.proto().unbarrieredGet(), l.proto()); + } }; /* @@ -1258,9 +1290,20 @@ struct InitialShapeEntry inline InitialShapeEntry(); inline InitialShapeEntry(Shape* shape, const Lookup::ShapeProto& proto); - static inline HashNumber hash(const Lookup& lookup); - static inline bool match(const InitialShapeEntry& key, const Lookup& lookup); - static void rekey(InitialShapeEntry& k, const InitialShapeEntry& newKey) { k = newKey; } + static HashNumber hash(const Lookup& lookup) { + HashNumber hash = MovableCellHasher::hash(lookup.proto); + return mozilla::AddToHash(hash, mozilla::HashGeneric(lookup.clasp, lookup.nfixed)); + } + static inline bool match(const InitialShapeEntry& key, const Lookup& lookup) { + const Shape* shape = key.shape.unbarrieredGet(); + return lookup.clasp == shape->getObjectClass() + && lookup.nfixed == shape->numFixedSlots() + && lookup.baseFlags == shape->getObjectFlags() + && MovableCellHasher::match(key.proto, lookup.proto); + } + static void rekey(InitialShapeEntry& k, const InitialShapeEntry& newKey) { + k = newKey; + } bool needsSweep() { Shape* ushape = shape.unbarrieredGet(); diff --git a/js/src/vm/Stack.cpp b/js/src/vm/Stack.cpp index b0f05d6997..5934c68651 100644 --- a/js/src/vm/Stack.cpp +++ b/js/src/vm/Stack.cpp @@ -598,6 +598,21 @@ FrameIter::Data::Data(JSContext* cx, DebuggerEvalOption debuggerEvalOption, { } +FrameIter::Data::Data(JSContext* cx, const CooperatingContext& target, + DebuggerEvalOption debuggerEvalOption) + : cx_(cx), + debuggerEvalOption_(debuggerEvalOption), + principals_(nullptr), + state_(DONE), + pc_(nullptr), + interpFrames_(nullptr), + activations_(cx, target), + jitFrames_(), + ionInlineFrameNo_(0), + wasmFrames_() +{ +} + FrameIter::Data::Data(const FrameIter::Data& other) : cx_(other.cx_), debuggerEvalOption_(other.debuggerEvalOption_), @@ -612,6 +627,16 @@ FrameIter::Data::Data(const FrameIter::Data& other) { } +FrameIter::FrameIter(JSContext* cx, const CooperatingContext& target, + DebuggerEvalOption debuggerEvalOption) + : data_(cx, target, debuggerEvalOption), + ionInlineFrames_(cx, (js::jit::JitFrameIterator*) nullptr) +{ + // settleOnActivation can only GC if principals are given. + JS::AutoSuppressGCAnalysis nogc; + settleOnActivation(); +} + FrameIter::FrameIter(JSContext* cx, DebuggerEvalOption debuggerEvalOption) : data_(cx, debuggerEvalOption, nullptr), ionInlineFrames_(cx, (js::jit::JitFrameIterator*) nullptr) diff --git a/js/src/vm/Stack.h b/js/src/vm/Stack.h index 42faa1158a..816dbe63b9 100644 --- a/js/src/vm/Stack.h +++ b/js/src/vm/Stack.h @@ -7,6 +7,7 @@ #define vm_Stack_h #include "mozilla/Atomics.h" +#include "mozilla/HashFunctions.h" #include "mozilla/Maybe.h" #include "mozilla/MemoryReporting.h" #include "mozilla/Variant.h" @@ -1087,7 +1088,7 @@ struct DefaultHasher { typedef AbstractFramePtr Lookup; static js::HashNumber hash(const Lookup& key) { - return size_t(key.raw()); + return mozilla::HashGeneric(key.raw()); } static bool match(const AbstractFramePtr& k, const Lookup& l) { @@ -1787,11 +1788,13 @@ class FrameIter wasm::FrameIterator wasmFrames_; Data(JSContext* cx, DebuggerEvalOption debuggerEvalOption, JSPrincipals* principals); + Data(JSContext* cx, const CooperatingContext& target, DebuggerEvalOption debuggerEvalOption); Data(const Data& other); }; explicit FrameIter(JSContext* cx, DebuggerEvalOption = FOLLOW_DEBUGGER_EVAL_PREV_LINK); + FrameIter(JSContext* cx, const CooperatingContext&, DebuggerEvalOption); FrameIter(JSContext* cx, DebuggerEvalOption, JSPrincipals*); FrameIter(const FrameIter& iter); MOZ_IMPLICIT FrameIter(const Data& data); @@ -1947,6 +1950,14 @@ class ScriptFrameIter : public FrameIter settle(); } + ScriptFrameIter(JSContext* cx, + const CooperatingContext& target, + DebuggerEvalOption debuggerEvalOption) + : FrameIter(cx, target, debuggerEvalOption) + { + settle(); + } + ScriptFrameIter(JSContext* cx, DebuggerEvalOption debuggerEvalOption, JSPrincipals* prin) @@ -2069,6 +2080,10 @@ class AllScriptFramesIter : public ScriptFrameIter explicit AllScriptFramesIter(JSContext* cx) : ScriptFrameIter(cx, ScriptFrameIter::IGNORE_DEBUGGER_EVAL_PREV_LINK) {} + + explicit AllScriptFramesIter(JSContext* cx, const CooperatingContext& target) + : ScriptFrameIter(cx, target, ScriptFrameIter::IGNORE_DEBUGGER_EVAL_PREV_LINK) + {} }; /* Popular inline definitions. */ diff --git a/js/src/vm/TraceLogging.h b/js/src/vm/TraceLogging.h index 65eda7f6b7..65590f5dfd 100644 --- a/js/src/vm/TraceLogging.h +++ b/js/src/vm/TraceLogging.h @@ -328,8 +328,19 @@ class TraceLoggerThreadState bool offThreadEnabled; bool graphSpewingEnabled; bool spewErrors; - ThreadLoggerHashMap threadLoggers; - mozilla::LinkedList traceLoggerMainThreadList; + mozilla::LinkedList threadLoggers; + + typedef HashMap, + SystemAllocPolicy> PointerHashMap; + typedef HashMap, + SystemAllocPolicy> TextIdHashMap; + PointerHashMap pointerMap; + TextIdHashMap textIdPayloads; + uint32_t nextTextId; public: uint64_t startupTime; diff --git a/js/src/vm/TypeInference.cpp b/js/src/vm/TypeInference.cpp index d086b556e8..5970c8b385 100644 --- a/js/src/vm/TypeInference.cpp +++ b/js/src/vm/TypeInference.cpp @@ -2963,7 +2963,7 @@ ObjectGroup::maybeClearNewScriptOnOOM() { MOZ_ASSERT(zone()->isGCSweepingOrCompacting()); - if (!isMarked()) + if (!isMarkedAny()) return; TypeNewScript* newScript = anyNewScript(); @@ -4002,97 +4002,91 @@ TypeNewScript::rollbackPartiallyInitializedObjects(JSContext* cx, ObjectGroup* g RootedFunction function(cx, this->function()); Vector pcOffsets(cx); - for (ScriptFrameIter iter(cx); !iter.done(); ++iter) { - { - AutoEnterOOMUnsafeRegion oomUnsafe; - if (!pcOffsets.append(iter.script()->pcToOffset(iter.pc()))) - oomUnsafe.crash("rollbackPartiallyInitializedObjects"); - } + JSRuntime::AutoProhibitActiveContextChange apacc(cx->runtime()); + for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) { + for (AllScriptFramesIter iter(cx, target); !iter.done(); ++iter) { + { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!pcOffsets.append(iter.script()->pcToOffset(iter.pc()))) + oomUnsafe.crash("rollbackPartiallyInitializedObjects"); + } - if (!iter.isConstructing()) { - continue; - } + if (!iter.isConstructing() || !iter.matchCallee(cx, function)) + continue; - MOZ_ASSERT(iter.calleeTemplate()->maybeCanonicalFunction()); + // Derived class constructors initialize their this-binding later and + // we shouldn't run the definite properties analysis on them. + MOZ_ASSERT(!iter.script()->isDerivedClassConstructor()); - if (iter.calleeTemplate()->maybeCanonicalFunction() != function) { - continue; - } + Value thisv = iter.thisArgument(cx); + if (!thisv.isObject() || + thisv.toObject().hasLazyGroup() || + thisv.toObject().group() != group) + { + continue; + } - // Derived class constructors initialize their this-binding later and - // we shouldn't run the definite properties analysis on them. - MOZ_ASSERT(!iter.script()->isDerivedClassConstructor()); + if (thisv.toObject().is()) { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!UnboxedPlainObject::convertToNative(cx, &thisv.toObject())) + oomUnsafe.crash("rollbackPartiallyInitializedObjects"); + } - Value thisv = iter.thisArgument(cx); - if (!thisv.isObject() || - thisv.toObject().hasLazyGroup() || - thisv.toObject().group() != group) - { - continue; - } + // Found a matching frame. + RootedPlainObject obj(cx, &thisv.toObject().as()); - if (thisv.toObject().is()) { - AutoEnterOOMUnsafeRegion oomUnsafe; - if (!UnboxedPlainObject::convertToNative(cx, &thisv.toObject())) - oomUnsafe.crash("rollbackPartiallyInitializedObjects"); - } + // Whether all identified 'new' properties have been initialized. + bool finished = false; - // Found a matching frame. - RootedPlainObject obj(cx, &thisv.toObject().as()); + // If not finished, number of properties that have been added. + uint32_t numProperties = 0; - // Whether all identified 'new' properties have been initialized. - bool finished = false; + // Whether the current SETPROP is within an inner frame which has + // finished entirely. + bool pastProperty = false; - // If not finished, number of properties that have been added. - uint32_t numProperties = 0; + // Index in pcOffsets of the outermost frame. + int callDepth = pcOffsets.length() - 1; - // Whether the current SETPROP is within an inner frame which has - // finished entirely. - bool pastProperty = false; + // Index in pcOffsets of the frame currently being checked for a SETPROP. + int setpropDepth = callDepth; - // Index in pcOffsets of the outermost frame. - int callDepth = pcOffsets.length() - 1; - - // Index in pcOffsets of the frame currently being checked for a SETPROP. - int setpropDepth = callDepth; - - for (Initializer* init = initializerList;; init++) { - if (init->kind == Initializer::SETPROP) { - if (!pastProperty && pcOffsets[setpropDepth] < init->offset) { - // Have not yet reached this setprop. + for (Initializer* init = initializerList;; init++) { + if (init->kind == Initializer::SETPROP) { + if (!pastProperty && pcOffsets[setpropDepth] < init->offset) { + // Have not yet reached this setprop. + break; + } + // This setprop has executed, reset state for the next one. + numProperties++; + pastProperty = false; + setpropDepth = callDepth; + } else if (init->kind == Initializer::SETPROP_FRAME) { + if (!pastProperty) { + if (pcOffsets[setpropDepth] < init->offset) { + // Have not yet reached this inner call. + break; + } else if (pcOffsets[setpropDepth] > init->offset) { + // Have advanced past this inner call. + pastProperty = true; + } else if (setpropDepth == 0) { + // Have reached this call but not yet in it. + break; + } else { + // Somewhere inside this inner call. + setpropDepth--; + } + } + } else { + MOZ_ASSERT(init->kind == Initializer::DONE); + finished = true; break; } - // This setprop has executed, reset state for the next one. - numProperties++; - pastProperty = false; - setpropDepth = callDepth; - } else if (init->kind == Initializer::SETPROP_FRAME) { - if (!pastProperty) { - if (pcOffsets[setpropDepth] < init->offset) { - // Have not yet reached this inner call. - break; - } else if (pcOffsets[setpropDepth] > init->offset) { - // Have advanced past this inner call. - pastProperty = true; - } else if (setpropDepth == 0) { - // Have reached this call but not yet in it. - break; - } else { - // Somewhere inside this inner call. - setpropDepth--; - } - } - } else { - MOZ_ASSERT(init->kind == Initializer::DONE); - finished = true; - break; + if (!finished) { + (void) NativeObject::rollbackProperties(cx, obj, numProperties); + found = true; } } - - if (!finished) { - (void) NativeObject::rollbackProperties(cx, obj, numProperties); - found = true; - } } return found; diff --git a/js/xpconnect/src/XPCJSContext.cpp b/js/xpconnect/src/XPCJSContext.cpp index a1dcd4dcc0..5e4479637e 100644 --- a/js/xpconnect/src/XPCJSContext.cpp +++ b/js/xpconnect/src/XPCJSContext.cpp @@ -827,7 +827,7 @@ XPCJSContext::FinalizeCallback(JSFreeOp* fop, } /* static */ void -XPCJSContext::WeakPointerZoneGroupCallback(JSContext* cx, void* data) +XPCJSContext::WeakPointerZonesCallback(JSContext* cx, void* data) { // Called before each sweeping slice -- after processing any final marking // triggered by barriers -- to clear out any references to things that are @@ -1514,7 +1514,7 @@ XPCJSContext::~XPCJSContext() // callbacks if we aren't careful. Null out the relevant callbacks. js::SetActivityCallback(Context(), nullptr, nullptr); JS_RemoveFinalizeCallback(Context(), FinalizeCallback); - JS_RemoveWeakPointerZoneGroupCallback(Context(), WeakPointerZoneGroupCallback); + JS_RemoveWeakPointerZonesCallback(Context(), WeakPointerZonesCallback); JS_RemoveWeakPointerCompartmentCallback(Context(), WeakPointerCompartmentCallback); // Clear any pending exception. It might be an XPCWrappedJS, and if we try @@ -3357,7 +3357,7 @@ XPCJSContext::Initialize() mPrevDoCycleCollectionCallback = JS::SetDoCycleCollectionCallback(cx, DoCycleCollectionCallback); JS_AddFinalizeCallback(cx, FinalizeCallback, nullptr); - JS_AddWeakPointerZoneGroupCallback(cx, WeakPointerZoneGroupCallback, this); + JS_AddWeakPointerZonesCallback(cx, WeakPointerZonesCallback, this); JS_AddWeakPointerCompartmentCallback(cx, WeakPointerCompartmentCallback, this); JS_SetWrapObjectCallbacks(cx, &WrapObjectCallbacks); js::SetPreserveWrapperCallback(cx, PreserveWrapper); diff --git a/js/xpconnect/src/xpcprivate.h b/js/xpconnect/src/xpcprivate.h index fc9a254a5c..381cec1646 100644 --- a/js/xpconnect/src/xpcprivate.h +++ b/js/xpconnect/src/xpcprivate.h @@ -540,7 +540,7 @@ public: JSFinalizeStatus status, bool isZoneGC, void* data); - static void WeakPointerZoneGroupCallback(JSContext* cx, void* data); + static void WeakPointerZonesCallback(JSContext* cx, void* data); static void WeakPointerCompartmentCallback(JSContext* cx, JSCompartment* comp, void* data); inline void AddVariantRoot(XPCTraceableVariant* variant); @@ -953,11 +953,11 @@ public: typedef js::HashMap, - js::PointerHasher, + js::PointerHasher, js::SystemAllocPolicy> InterpositionMap; typedef js::HashSet, + js::PointerHasher, js::SystemAllocPolicy> AddonSet; // Gets the appropriate scope object for XBL in this scope. The context