diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp index 408abd95e5..e422dd6a99 100644 --- a/js/src/gc/Allocator.cpp +++ b/js/src/gc/Allocator.cpp @@ -236,7 +236,7 @@ GCRuntime::gcIfNeededPerAllocation(JSContext* cx) // an incremental GC, we're growing faster than we're GCing, so stop // the world and do a full, non-incremental GC right now, if possible. if (isIncrementalGCInProgress() && - cx->zone()->usage.gcBytes() > cx->zone()->threshold.gcTriggerBytes()) + cx->zone()->usage.gcBytes() > cx->zone()->threshold.AllocThresholdFactorTriggerBytes(tunables)) { PrepareZoneForGC(cx->zone()); AutoKeepAtoms keepAtoms(cx->perThreadData); @@ -426,7 +426,7 @@ GCRuntime::allocateArena(Chunk* chunk, Zone* zone, AllocKind thingKind, // Trigger an incremental slice if needed. if (checkThresholds) - maybeAllocTriggerZoneGC(zone, lock); + maybeAllocTriggerZoneGC(zone, lock, ArenaSize); return arena; } diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index 9e2abc4ca8..a062f8b1a9 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -35,6 +35,7 @@ class AutoMaybeStartBackgroundAllocation; class MarkingValidator; class AutoTraceSession; struct MovingTracer; +class SweepGroupsIter; class WeakCacheSweepIterator; enum IncrementalProgress @@ -125,6 +126,13 @@ class GCSchedulingTunables */ UnprotectedData gcMaxBytes_; + /* + * JSGC_MAX_MALLOC_BYTES + * + * Initial malloc bytes threshold. + */ + UnprotectedData maxMallocBytes_; + /* * Maximum nursery size for each zone group. * Initially DefaultNurseryBytes and can be set by @@ -133,16 +141,24 @@ class GCSchedulingTunables ActiveThreadData gcMaxNurseryBytes_; /* - * The base value used to compute zone->trigger.gcBytes(). When - * usage.gcBytes() surpasses threshold.gcBytes() for a zone, the zone may - * be scheduled for a GC, depending on the exact circumstances. + * The base value used to compute zone->threshold.gcTriggerBytes(). When + * usage.gcBytes() surpasses threshold.gcTriggerBytes() for a zone, the + * zone may be scheduled for a GC, depending on the exact circumstances. */ size_t gcZoneAllocThresholdBase_; - /* Fraction of threshold.gcBytes() which triggers an incremental GC. */ - UnprotectedData zoneAllocThresholdFactor_; - /* The same except when doing so would interrupt an already running GC. */ - UnprotectedData zoneAllocThresholdFactorAvoidInterrupt_; + /* + * JSGC_ALLOCATION_THRESHOLD_FACTOR + * + * Fraction of threshold.gcBytes() which triggers an incremental GC. + */ + UnprotectedData allocThresholdFactor_; + /* + * JSGC_ALLOCATION_THRESHOLD_FACTOR_AVOID_INTERRUPT + * + * The same except when doing so would interrupt an already running GC. + */ + UnprotectedData allocThresholdFactorAvoidInterrupt_; /* * Number of bytes to allocate between incremental slices in GCs triggered @@ -194,31 +210,14 @@ class GCSchedulingTunables uint32_t maxEmptyChunkCount_; public: - GCSchedulingTunables() - : gcMaxBytes_(0), - gcMaxNurseryBytes_(0), - gcZoneAllocThresholdBase_(30 * 1024 * 1024), - zoneAllocThresholdFactor_(0.9), - zoneAllocThresholdFactorAvoidInterrupt_(0.95), - zoneAllocDelayBytes_(1024 * 1024), - dynamicHeapGrowthEnabled_(false), - highFrequencyThresholdUsec_(1000 * 1000), - highFrequencyLowLimitBytes_(100 * 1024 * 1024), - highFrequencyHighLimitBytes_(500 * 1024 * 1024), - highFrequencyHeapGrowthMax_(3.0), - highFrequencyHeapGrowthMin_(1.5), - lowFrequencyHeapGrowth_(1.5), - dynamicMarkSliceEnabled_(false), - refreshFrameSlicesEnabled_(true), - minEmptyChunkCount_(1), - maxEmptyChunkCount_(30) - {} + GCSchedulingTunables(); size_t gcMaxBytes() const { return gcMaxBytes_; } + size_t maxMallocBytes() const { return maxMallocBytes_; } size_t gcMaxNurseryBytes() const { return gcMaxNurseryBytes_; } size_t gcZoneAllocThresholdBase() const { return gcZoneAllocThresholdBase_; } - double zoneAllocThresholdFactor() const { return zoneAllocThresholdFactor_; } - double zoneAllocThresholdFactorAvoidInterrupt() const { return zoneAllocThresholdFactorAvoidInterrupt_; } + float allocThresholdFactor() const { return allocThresholdFactor_; } + float allocThresholdFactorAvoidInterrupt() const { return allocThresholdFactorAvoidInterrupt_; } size_t zoneAllocDelayBytes() const { return zoneAllocDelayBytes_; } bool isDynamicHeapGrowthEnabled() const { return dynamicHeapGrowthEnabled_; } uint64_t highFrequencyThresholdUsec() const { return highFrequencyThresholdUsec_; } @@ -233,6 +232,15 @@ class GCSchedulingTunables unsigned maxEmptyChunkCount() const { return maxEmptyChunkCount_; } [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value, const AutoLockGC& lock); + void resetParameter(JSGCParamKey key, const AutoLockGC& lock); + + void setMaxMallocBytes(size_t value); + +private: + void setHighFrequencyLowLimit(uint64_t value); + void setHighFrequencyHighLimit(uint64_t value); + void setMinEmptyChunkCount(uint32_t value); + void setMaxEmptyChunkCount(uint32_t value); }; /* @@ -610,54 +618,64 @@ typedef HashMap, SystemAllocPolicy> R using AllocKinds = mozilla::EnumSet; -template +enum TriggerKind +{ + NoTrigger = 0, + IncrementalTrigger, + NonIncrementalTrigger +}; + class MemoryCounter { - // Bytes counter to measure memory pressure for GC scheduling. It runs - // from maxBytes down to zero. - mozilla::Atomic bytes_; + // Bytes counter to measure memory pressure for GC scheduling. It counts + // upwards from zero. + mozilla::Atomic bytes_; // GC trigger threshold for memory allocations. - js::ActiveThreadData maxBytes_; + size_t maxBytes_; - // Whether a GC has been triggered as a result of bytes falling below - // zero. - // - // This should be a bool, but Atomic only supports 32-bit and pointer-sized - // types. - mozilla::Atomic triggered_; + // The counter value at the start of a GC. + ActiveThreadData bytesAtStartOfGC_; + + // Which kind of GC has been triggered if any. + mozilla::Atomic triggered_; public: - MemoryCounter() - : bytes_(0), - maxBytes_(0), - triggered_(false) - { } + MemoryCounter(); - void reset() { - bytes_ = maxBytes_; - triggered_ = false; - } - - void setMax(size_t newMax) { - // For compatibility treat any value that exceeds PTRDIFF_T_MAX to - // mean that value. - maxBytes_ = (ptrdiff_t(newMax) >= 0) ? newMax : size_t(-1) >> 1; - reset(); - } - - bool update(T* owner, size_t bytes) { - bytes_ -= ptrdiff_t(bytes); - if (MOZ_UNLIKELY(isTooMuchMalloc())) { - if (!triggered_) - triggered_ = owner->triggerGCForTooMuchMalloc(); - } - return triggered_; - } - - ptrdiff_t bytes() const { return bytes_; } + size_t bytes() const { return bytes_; } size_t maxBytes() const { return maxBytes_; } - bool isTooMuchMalloc() const { return bytes_ <= 0; } + TriggerKind triggered() const { return triggered_; } + + void setMax(size_t newMax, const AutoLockGC& lock); + + void update(size_t bytes) { + bytes_ += bytes; + } + + void adopt(MemoryCounter& other); + + TriggerKind shouldTriggerGC(const GCSchedulingTunables& tunables) const { + if (MOZ_LIKELY(bytes_ < maxBytes_ * tunables.allocThresholdFactor())) + return NoTrigger; + + if (bytes_ < maxBytes_) + return IncrementalTrigger; + + return NonIncrementalTrigger; + } + + bool shouldResetIncrementalGC(const GCSchedulingTunables& tunables) const { + return bytes_ > maxBytes_ * tunables.allocThresholdFactorAvoidInterrupt(); + } + + void recordTrigger(TriggerKind trigger); + + void updateOnGCStart(); + void updateOnGCEnd(const GCSchedulingTunables& tunables, const AutoLockGC& lock); + + private: + void reset(); }; class GCRuntime @@ -673,10 +691,11 @@ class GCRuntime void setMarkStackLimit(size_t limit, AutoLockGC& lock); [[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock); + void resetParameter(JSGCParamKey key, AutoLockGC& lock); uint32_t getParameter(JSGCParamKey key, const AutoLockGC& lock); [[nodiscard]] bool triggerGC(JS::gcreason::Reason reason); - void maybeAllocTriggerZoneGC(Zone* zone, const AutoLockGC& lock); + void maybeAllocTriggerZoneGC(Zone* zone, const AutoLockGC& lock, size_t nbytes = 0); // The return value indicates if we were able to do the GC. bool triggerZoneGC(Zone* zone, JS::gcreason::Reason reason); void maybeGC(Zone* zone); @@ -827,10 +846,26 @@ class GCRuntime int32_t getMallocBytes() const { return mallocCounter.bytes(); } size_t maxMallocBytesAllocated() const { return mallocCounter.maxBytes(); } - bool isTooMuchMalloc() const { return mallocCounter.isTooMuchMalloc(); } - void resetMallocBytes() { mallocCounter.reset(); } - void setMaxMallocBytes(size_t value); - void updateMallocCounter(JS::Zone* zone, size_t nbytes); + void setMaxMallocBytes(size_t value, const AutoLockGC& lock); + + bool updateMallocCounter(size_t nbytes) { + mallocCounter.update(nbytes); + TriggerKind trigger = mallocCounter.shouldTriggerGC(tunables); + if (MOZ_LIKELY(trigger == NoTrigger) || trigger <= mallocCounter.triggered()) + return false; + if (!triggerGC(JS::gcreason::TOO_MUCH_MALLOC)) + return false; + + // Even though this method may be called off the main thread it is safe + // to access mallocCounter here since triggerGC() will return false in + // that case. + stats().recordTrigger(mallocCounter.bytes(), mallocCounter.maxBytes()); + + mallocCounter.recordTrigger(trigger); + return true; + } + + void updateMallocCountersOnGCStart(); void setGCCallback(JSGCCallback callback, void* data); void callGCCallback(JSGCStatus status) const; @@ -854,6 +889,11 @@ class GCRuntime void setFullCompartmentChecks(bool enable); JS::Zone* getCurrentSweepGroup() { return currentSweepGroup; } + void setFoundBlackGrayEdges(TenuredCell& target) { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!foundBlackGrayEdges.ref().append(&target)) + oomUnsafe.crash("OOM|small: failed to insert into foundBlackGrayEdges"); + } uint64_t gcNumber() const { return number; } @@ -939,8 +979,7 @@ class GCRuntime void startTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked); void joinTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked); - // Delete an empty zone group after its contents have been merged. - void deleteEmptyZoneGroup(ZoneGroup* group); + void mergeCompartments(JSCompartment* source, JSCompartment* target); private: enum IncrementalResult @@ -949,6 +988,9 @@ class GCRuntime Ok }; + // Delete an empty zone group after its contents have been merged. + void deleteEmptyZoneGroup(ZoneGroup* group); + // For ArenaLists::allocateFromArena() friend class ArenaLists; Chunk* pickChunk(const AutoLockGC& lock, @@ -1005,7 +1047,7 @@ class GCRuntime void purgeRuntime(AutoLockForExclusiveAccess& lock); [[nodiscard]] bool beginMarkPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); - bool prepareZonesForCollection(JS::gcreason::Reason reason, bool* isFullOut, + bool prepareZonesForCollection(JS::gcreason::Reason reason, bool* isFullOut, AutoLockForExclusiveAccess& lock); bool shouldPreserveJITCode(JSCompartment* comp, int64_t currentTime, JS::gcreason::Reason reason, bool canAllocateMoreCode); @@ -1028,27 +1070,24 @@ class GCRuntime void groupZonesForSweeping(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); [[nodiscard]] bool findInterZoneEdges(); void getNextSweepGroup(); - void endMarkingSweepGroup(); - void beginSweepingSweepGroup(); + IncrementalProgress endMarkingSweepGroup(FreeOp* fop, SliceBudget& budget); + IncrementalProgress beginSweepingSweepGroup(FreeOp* fop, SliceBudget& budget); +#ifdef JS_GC_ZEAL + IncrementalProgress maybeYieldForSweepingZeal(FreeOp* fop, SliceBudget& budget); +#endif bool shouldReleaseObservedTypes(); void sweepDebuggerOnMainThread(FreeOp* fop); void sweepJitDataOnMainThread(FreeOp* fop); - void endSweepingSweepGroup(); - IncrementalProgress performSweepActions(SliceBudget& sliceBudget, - AutoLockForExclusiveAccess& lock); - static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, - Zone* zone); - static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, - Zone* zone); - static IncrementalProgress sweepAtomsTable(GCRuntime* gc, FreeOp* fop, SliceBudget& budget); + IncrementalProgress endSweepingSweepGroup(FreeOp* fop, SliceBudget& budget); + IncrementalProgress performSweepActions(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock); + IncrementalProgress sweepTypeInformation(FreeOp* fop, SliceBudget& budget, Zone* zone); + IncrementalProgress mergeSweptObjectArenas(FreeOp* fop, SliceBudget& budget, Zone* zone); void startSweepingAtomsTable(); - IncrementalProgress sweepAtomsTable(SliceBudget& budget); - static IncrementalProgress sweepWeakCaches(GCRuntime* gc, FreeOp* fop, SliceBudget& budget); - IncrementalProgress sweepWeakCaches(SliceBudget& budget); - static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, - Zone* zone, AllocKind kind); - static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, - Zone* zone); + IncrementalProgress sweepAtomsTable(FreeOp* fop, SliceBudget& budget); + IncrementalProgress sweepWeakCaches(FreeOp* fop, SliceBudget& budget); + IncrementalProgress finalizeAllocKind(FreeOp* fop, SliceBudget& budget, Zone* zone, + AllocKind kind); + IncrementalProgress sweepShapeTree(FreeOp* fop, SliceBudget& budget, Zone* zone); void endSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock); void sweepZones(FreeOp* fop, bool lastGC); void decommitAllWithoutUnlocking(const AutoLockGC& lock); @@ -1227,15 +1266,30 @@ class GCRuntime */ ActiveThreadOrGCTaskData incrementalState; + /* The incremental state at the start of this slice. */ + ActiveThreadData initialState; + +#ifdef JS_GC_ZEAL + /* Whether to pay attention the zeal settings in this incremental slice. */ + ActiveThreadData useZeal; +#endif + + /* Indicates that the last incremental slice exhausted the mark stack. */ ActiveThreadData lastMarkSlice; + /* Whether it's currently safe to yield to the mutator in an incremental GC. */ + ActiveThreadData safeToYield; + /* Whether any sweeping will take place in the separate GC helper thread. */ bool sweepOnBackgroundThread; /* Whether observed type information is being released in the current GC. */ bool releaseObservedTypes; + /* Whether any black->gray edges were found during marking. */ + ActiveThreadData foundBlackGrayEdges; + /* Singly linked list of zones to be swept in the background. */ ActiveThreadOrGCTaskData backgroundSweepZones; @@ -1267,6 +1321,7 @@ class GCRuntime ActiveThreadOrGCTaskData sweepCache; ActiveThreadData abortSweepAfterCurrentGroup; + friend class SweepGroupsIter; friend class WeakCacheSweepIterator; /* @@ -1341,7 +1396,7 @@ class GCRuntime CallbackVector updateWeakPointerZonesCallbacks; CallbackVector updateWeakPointerCompartmentCallbacks; - MemoryCounter mallocCounter; + MemoryCounter mallocCounter; /* * The trace operations to trace embedding-specific GC roots. One is for diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h index 58357fdcd7..d77d49dcf6 100644 --- a/js/src/gc/Heap.h +++ b/js/src/gc/Heap.h @@ -55,6 +55,10 @@ extern bool CurrentThreadIsIonCompiling(); #endif +// The return value indicates if anything was unmarked. +extern bool +UnmarkGrayCellRecursively(gc::Cell* cell, JS::TraceKind kind); + extern void TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name); @@ -1387,7 +1391,7 @@ TenuredCell::readBarrier(TenuredCell* thing) // There shouldn't be anything marked grey unless we're on the active thread. MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())); if (!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone)) - JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(thing, thing->getTraceKind())); + UnmarkGrayCellRecursively(thing, thing->getTraceKind()); } } diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp index 2e98b9c992..2b35533b21 100644 --- a/js/src/gc/Marking.cpp +++ b/js/src/gc/Marking.cpp @@ -236,8 +236,7 @@ js::CheckTracedThing(JSTracer* trc, T* thing) */ bool isGcMarkingTracer = trc->isMarkingTracer(); - MOZ_ASSERT_IF(zone->requireGCTracer(), - isGcMarkingTracer || IsBufferGrayRootsTracer(trc) || IsUnmarkGrayTracer(trc)); + MOZ_ASSERT_IF(zone->requireGCTracer(), isGcMarkingTracer || IsBufferGrayRootsTracer(trc)); if (isGcMarkingTracer) { GCMarker* gcMarker = static_cast(trc); @@ -287,8 +286,6 @@ JS_FOR_EACH_TRACEKIND(IMPL_CHECK_TRACED_THING); #undef IMPL_CHECK_TRACED_THING } // namespace js -static bool UnmarkGrayGCThing(JSRuntime* rt, JS::GCCellPtr thing); - static bool ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, Cell* cell) { @@ -304,8 +301,6 @@ ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, Cell* cell) TenuredCell& tenured = cell->asTenured(); JS::Zone* zone = tenured.zone(); - if (!src->zone()->isGCMarking() && !zone->isGCMarking()) - return false; if (color == MarkColor::Black) { /* @@ -317,7 +312,7 @@ ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, Cell* cell) */ if (tenured.isMarkedGray()) { MOZ_ASSERT(!zone->isCollecting()); - UnmarkGrayGCThing(trc->runtime(), JS::GCCellPtr(cell, cell->getTraceKind())); + trc->runtime()->gc.setFoundBlackGrayEdges(tenured); } return zone->isGCMarking(); } else { @@ -2034,7 +2029,7 @@ MarkStack::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const */ GCMarker::GCMarker(JSRuntime* rt) : JSTracer(rt, JSTracer::TracerKindTag::Marking, ExpandWeakMaps), - stack(size_t(-1)), + stack(), color(MarkColor::Black), unmarkedArenaStackTop(nullptr) #ifdef DEBUG @@ -3087,32 +3082,43 @@ UnmarkGrayTracer::onChild(const JS::GCCellPtr& thing) unmarkedAny |= childTracer.unmarkedAny; } +template static bool -UnmarkGrayGCThing(JSRuntime* rt, JS::GCCellPtr thing) +TypedUnmarkGrayCellRecursively(T* t) { - MOZ_ASSERT(thing); + MOZ_ASSERT(t); - UnmarkGrayTracer unmarker(rt); - gcstats::AutoPhase innerPhase(rt->gc.stats(), gcstats::PHASE_UNMARK_GRAY); - unmarker.unmark(thing); - return unmarker.unmarkedAny; -} - -JS_FRIEND_API(bool) -JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr thing) -{ + JSRuntime* rt = t->runtimeFromActiveCooperatingThread(); MOZ_ASSERT(!JS::CurrentThreadIsHeapCollecting()); MOZ_ASSERT(!JS::CurrentThreadIsHeapCycleCollecting()); - JSRuntime* rt = thing.asCell()->runtimeFromActiveCooperatingThread(); - gcstats::AutoPhase outerPhase(rt->gc.stats(), gcstats::PHASE_BARRIER); - return UnmarkGrayGCThing(rt, thing); + UnmarkGrayTracer unmarker(rt); + gcstats::AutoPhase outerPhase(rt->gc.stats(), gcstats::PHASE_BARRIER); + gcstats::AutoPhase innerPhase(rt->gc.stats(), gcstats::PHASE_UNMARK_GRAY); + unmarker.unmark(JS::GCCellPtr(t, MapTypeToTraceKind::kind)); + return unmarker.unmarkedAny; +} + +struct UnmarkGrayCellRecursivelyFunctor { + template bool operator()(T* t) { return TypedUnmarkGrayCellRecursively(t); } +}; + +bool +js::UnmarkGrayCellRecursively(Cell* cell, JS::TraceKind kind) +{ + return DispatchTraceKindTyped(UnmarkGrayCellRecursivelyFunctor(), cell, kind); } bool js::UnmarkGrayShapeRecursively(Shape* shape) { - return JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(shape)); + return TypedUnmarkGrayCellRecursively(shape); +} + +JS_FRIEND_API(bool) +JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr thing) +{ + return js::UnmarkGrayCellRecursively(thing.asCell(), thing.kind()); } namespace js { diff --git a/js/src/gc/Marking.h b/js/src/gc/Marking.h index 62f0fc11b1..8e33520e08 100644 --- a/js/src/gc/Marking.h +++ b/js/src/gc/Marking.h @@ -113,9 +113,11 @@ class MarkStack TaggedPtr ptr; }; - explicit MarkStack(size_t maxCapacity); + explicit MarkStack(size_t maxCapacity = DefaultCapacity); ~MarkStack(); + static const size_t DefaultCapacity = SIZE_MAX; + size_t capacity() { return end_ - stack_; } ptrdiff_t position() const { return tos_ - stack_; } @@ -400,9 +402,6 @@ class GCMarker : public JSTracer // the marking phase of incremental GC. bool IsBufferGrayRootsTracer(JSTracer* trc); - -bool -IsUnmarkGrayTracer(JSTracer* trc); #endif namespace gc { diff --git a/js/src/gc/Zone.cpp b/js/src/gc/Zone.cpp index 4dd3b2ba18..8cdc12dd3d 100644 --- a/js/src/gc/Zone.cpp +++ b/js/src/gc/Zone.cpp @@ -66,8 +66,8 @@ JS::Zone::Zone(JSRuntime* rt) AutoLockGC lock(rt); threshold.updateAfterGC(8192, GC_NORMAL, rt->gc.tunables, rt->gc.schedulingState, lock); - setGCMaxMallocBytes(rt->gc.maxMallocBytesAllocated() * 0.9); - jitCodeCounter.setMax(jit::MaxCodeBytesPerProcess * 0.8); + setGCMaxMallocBytes(rt->gc.tunables.maxMallocBytes(), lock); + jitCodeCounter.setMax(jit::MaxCodeBytesPerProcess * 0.8, lock); } Zone::~Zone() diff --git a/js/src/gc/Zone.h b/js/src/gc/Zone.h index ae6400fa7a..2eee24a2be 100644 --- a/js/src/gc/Zone.h +++ b/js/src/gc/Zone.h @@ -45,7 +45,10 @@ class ZoneHeapThreshold double gcHeapGrowthFactor() const { return gcHeapGrowthFactor_; } size_t gcTriggerBytes() const { return gcTriggerBytes_; } - double allocTrigger(bool highFrequencyGC) const; + size_t AllocThresholdFactorTriggerBytes(GCSchedulingTunables& tunables) const { + return gcTriggerBytes_ * tunables.allocThresholdFactor(); + } + double eagerAllocTrigger(bool highFrequencyGC) const; void updateAfterGC(size_t lastBytes, JSGCInvocationKind gckind, const GCSchedulingTunables& tunables, const GCSchedulingState& state, @@ -193,16 +196,19 @@ struct Zone : public JS::shadow::Zone, bool isPreservingCode() const { return gcPreserveCode_; } bool canCollect(); + + void notifyObservingDebuggers(); - void changeGCState(GCState prev, GCState next) { + void setGCState(GCState state) { MOZ_ASSERT(CurrentThreadIsHeapBusy()); - MOZ_ASSERT(gcState() == prev); - MOZ_ASSERT_IF(next != NoGC, canCollect()); - gcState_ = next; + MOZ_ASSERT_IF(state != NoGC, canCollect()); + gcState_ = state; + if (state == Finished) + notifyObservingDebuggers(); } bool isCollecting() const { - MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromActiveCooperatingThread())); + MOZ_ASSERT(js::CurrentThreadCanAccessRuntime(runtimeFromActiveCooperatingThread())); return isCollectingFromAnyThread(); } @@ -280,8 +286,6 @@ struct Zone : public JS::shadow::Zone, DebuggerVector* getDebuggers() const { return debuggers; } DebuggerVector* getOrCreateDebuggers(JSContext* cx); - void notifyObservingDebuggers(); - void clearTables(); /* @@ -359,11 +363,35 @@ struct Zone : public JS::shadow::Zone, // Malloc counter to measure memory pressure for GC scheduling. This // counter should be used only when it's not possible to know the size of // a free. - js::gc::MemoryCounter gcMallocCounter; + js::gc::MemoryCounter gcMallocCounter; // Counter of JIT code executable memory for GC scheduling. Also imprecise, // since wasm can generate code that outlives a zone. - js::gc::MemoryCounter jitCodeCounter; + js::gc::MemoryCounter jitCodeCounter; + + void updateMemoryCounter(js::gc::MemoryCounter& counter, size_t nbytes) { + JSRuntime* rt = runtimeFromAnyThread(); + + counter.update(nbytes); + auto trigger = counter.shouldTriggerGC(rt->gc.tunables); + if (MOZ_LIKELY(trigger == js::gc::NoTrigger) || trigger <= counter.triggered()) + return; + + if (!js::CurrentThreadCanAccessRuntime(rt)) + return; + + bool wouldInterruptGC = rt->gc.isIncrementalGCInProgress() && !isCollecting(); + if (wouldInterruptGC && !counter.shouldResetIncrementalGC(rt->gc.tunables)) + return; + + if (!rt->gc.triggerZoneGC(this, JS::gcreason::TOO_MUCH_MALLOC, + counter.bytes(), counter.maxBytes())) + { + return; + } + + counter.recordTrigger(trigger); + } public: js::RegExpZone regExps; @@ -372,32 +400,37 @@ struct Zone : public JS::shadow::Zone, bool addTypeDescrObject(JSContext* cx, HandleObject obj); - bool triggerGCForTooMuchMalloc() { - JSRuntime* rt = runtimeFromAnyThread(); - - if (CurrentThreadCanAccessRuntime(rt)) { - return rt->gc.triggerZoneGC(this, JS::gcreason::TOO_MUCH_MALLOC, - gcMallocCounter.bytes(), gcMallocCounter.maxBytes()); - } - return false; + void setGCMaxMallocBytes(size_t value, const js::AutoLockGC& lock) { + gcMallocCounter.setMax(value, lock); + } + void updateMallocCounter(size_t nbytes) { + updateMemoryCounter(gcMallocCounter, nbytes); + } + void adoptMallocBytes(Zone* other) { + gcMallocCounter.adopt(other->gcMallocCounter); } - void resetGCMallocBytes() { gcMallocCounter.reset(); } - void setGCMaxMallocBytes(size_t value) { gcMallocCounter.setMax(value); } - void updateMallocCounter(size_t nbytes) { gcMallocCounter.update(this, nbytes); } size_t GCMaxMallocBytes() const { return gcMallocCounter.maxBytes(); } size_t GCMallocBytes() const { return gcMallocCounter.bytes(); } - void updateJitCodeMallocBytes(size_t size) { jitCodeCounter.update(this, size); } - - // Resets all the memory counters. - void resetAllMallocBytes() { - resetGCMallocBytes(); - jitCodeCounter.reset(); + void updateJitCodeMallocBytes(size_t nbytes) { + updateMemoryCounter(jitCodeCounter, nbytes); } - bool isTooMuchMalloc() const { - return gcMallocCounter.isTooMuchMalloc() || - jitCodeCounter.isTooMuchMalloc(); + + void updateAllGCMallocCountersOnGCStart() { + gcMallocCounter.updateOnGCStart(); + jitCodeCounter.updateOnGCStart(); + } + void updateAllGCMallocCountersOnGCEnd(const js::AutoLockGC& lock) { + auto& gc = runtimeFromAnyThread()->gc; + gcMallocCounter.updateOnGCEnd(gc.tunables, lock); + jitCodeCounter.updateOnGCEnd(gc.tunables, lock); + } + + js::gc::TriggerKind shouldTriggerGCForTooMuchMalloc() { + auto& gc = runtimeFromAnyThread()->gc; + return std::max(gcMallocCounter.shouldTriggerGC(gc.tunables), + jitCodeCounter.shouldTriggerGC(gc.tunables)); } // Whether a GC has been triggered as a result of gcMallocBytes falling @@ -415,7 +448,7 @@ struct Zone : public JS::shadow::Zone, // Amount of data to allocate before triggering a new incremental slice for // the current GC. - js::UnprotectedData gcDelayBytes; + js::ActiveThreadData gcDelayBytes; // Shared Shape property tree. js::PropertyTree propertyTree; @@ -515,7 +548,7 @@ struct Zone : public JS::shadow::Zone, void transferUniqueId(js::gc::Cell* tgt, js::gc::Cell* src) { MOZ_ASSERT(src != tgt); MOZ_ASSERT(!IsInsideNursery(tgt)); - MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromActiveCooperatingThread())); + MOZ_ASSERT(js::CurrentThreadCanAccessRuntime(runtimeFromActiveCooperatingThread())); MOZ_ASSERT(js::CurrentThreadCanAccessZone(this)); uniqueIds().rekeyIfMoved(src, tgt); } @@ -557,6 +590,28 @@ struct Zone : public JS::shadow::Zone, // Delete an empty compartment after its contents have been merged. void deleteEmptyCompartment(JSCompartment* comp); + /* + * This variation of calloc will call the large-allocation-failure callback + * on OOM and retry the allocation. + */ + template + T* pod_callocCanGC(size_t numElems) { + T* p = pod_calloc(numElems); + if (MOZ_LIKELY(!!p)) + return p; + size_t bytes; + if (MOZ_UNLIKELY(!js::CalculateAllocSize(numElems, &bytes))) { + reportAllocationOverflow(); + return nullptr; + } + JSRuntime* rt = runtimeFromActiveCooperatingThread(); + p = static_cast(rt->onOutOfMemoryCanGC(js::AllocFunction::Calloc, bytes)); + if (!p) + return nullptr; + updateMallocCounter(bytes); + return p; + } + private: js::jit::JitZone* jitZone_; diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index cacaee059b..b86f415117 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -1319,6 +1319,7 @@ GCRuntime::removeWeakPointerZonesCallback(JSWeakPointerZonesCallback callback) } } +void GCRuntime::callWeakPointerZonesCallbacks() const { for (auto const& p : updateWeakPointerZonesCallbacks.ref()) @@ -4998,7 +4999,7 @@ IncrementalProgress GCRuntime::beginSweepingSweepGroup(FreeOp* fop, SliceBudget& budget) { /* - * Begin sweeping the group of zones in gcCurrentZoneGroup, + * Begin sweeping the group of zones in gccurrentSweepGroup, * performing actions that must be done before yielding to caller. */ @@ -5067,8 +5068,29 @@ GCRuntime::beginSweepingSweepGroup(FreeOp* fop, SliceBudget& budget) } { - gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_COMPARTMENTS); - gcstats::AutoSCC scc(stats, zoneGroupIndex); + AutoLockHelperThreadState lock; + + Maybe updateAtomsBitmap; + if (sweepingAtoms) + updateAtomsBitmap.emplace(rt, UpdateAtomsBitmap, PHASE_UPDATE_ATOMS_BITMAP, lock); + + AutoPhase ap(stats(), PHASE_SWEEP_COMPARTMENTS); + AutoSCC scc(stats(), sweepGroupIndex); + + AutoRunParallelTask sweepCCWrappers(rt, SweepCCWrappers, PHASE_SWEEP_CC_WRAPPER, lock); + AutoRunParallelTask sweepObjectGroups(rt, SweepObjectGroups, PHASE_SWEEP_TYPE_OBJECT, lock); + AutoRunParallelTask sweepRegExps(rt, SweepRegExps, PHASE_SWEEP_REGEXP, lock); + AutoRunParallelTask sweepMisc(rt, SweepMisc, PHASE_SWEEP_MISC, lock); + AutoRunParallelTask sweepCompTasks(rt, SweepCompressionTasks, PHASE_SWEEP_COMPRESSION, lock); + AutoRunParallelTask sweepWeakMaps(rt, SweepWeakMaps, PHASE_SWEEP_WEAKMAPS, lock); + AutoRunParallelTask sweepUniqueIds(rt, SweepUniqueIds, PHASE_SWEEP_UNIQUEIDS, lock); + + WeakCacheTaskVector sweepCacheTasks; + if (!PrepareWeakCacheTasks(rt, &sweepCacheTasks)) + SweepWeakCachesOnMainThread(rt); + + for (auto& task : sweepCacheTasks) + startTask(task, PHASE_SWEEP_WEAK_CACHES, lock); { AutoLockHelperThreadState helperLock; @@ -5257,9 +5279,12 @@ GCRuntime::beginSweepPhase(bool destroyingRuntime, AutoLockForExclusiveAccess& l AssertNoWrappersInGrayList(rt); DropStringWrappers(rt); - findZoneGroups(lock); - endMarkingZoneGroup(); - beginSweepingZoneGroup(lock); + groupZonesForSweeping(reason, lock); + sweepActions->assertFinished(); + + // We must not yield after this point until we start sweeping the first sweep + // group. + safeToYield = false; } bool @@ -5382,8 +5407,193 @@ GCRuntime::mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceB return Finished; } -/* static */ IncrementalProgress -GCRuntime::finalizeAllocKind(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget, +void +GCRuntime::startSweepingAtomsTable() +{ + auto& maybeAtoms = maybeAtomsToSweep.ref(); + MOZ_ASSERT(maybeAtoms.isNothing()); + + AtomSet* atomsTable = rt->atomsForSweeping(); + if (!atomsTable) + return; + + // Create a secondary table to hold new atoms added while we're sweeping + // the main table incrementally. + if (!rt->createAtomsAddedWhileSweepingTable()) { + atomsTable->sweep(); + return; + } + + // Initialize remaining atoms to sweep. + maybeAtoms.emplace(*atomsTable); +} + +IncrementalProgress +GCRuntime::sweepAtomsTable(FreeOp* fop, SliceBudget& budget) +{ + if (!atomsZone->isGCSweeping()) + return Finished; + + gcstats::AutoPhase ap(stats(), gcstats::PHASE_SWEEP_ATOMS_TABLE); + + auto& maybeAtoms = maybeAtomsToSweep.ref(); + if (!maybeAtoms) + return Finished; + + MOZ_ASSERT(rt->atomsAddedWhileSweeping()); + + // Sweep the table incrementally until we run out of work or budget. + auto& atomsToSweep = *maybeAtoms; + while (!atomsToSweep.empty()) { + budget.step(); + if (budget.isOverBudget()) + return NotFinished; + + JSAtom* atom = atomsToSweep.front().asPtrUnbarriered(); + if (IsAboutToBeFinalizedUnbarriered(&atom)) + atomsToSweep.removeFront(); + atomsToSweep.popFront(); + } + + // Add any new atoms from the secondary table. + AutoEnterOOMUnsafeRegion oomUnsafe; + AtomSet* atomsTable = rt->atomsForSweeping(); + MOZ_ASSERT(atomsTable); + for (auto r = rt->atomsAddedWhileSweeping()->all(); !r.empty(); r.popFront()) { + if (!atomsTable->putNew(AtomHasher::Lookup(r.front().asPtrUnbarriered()), r.front())) + oomUnsafe.crash("Adding atom from secondary table after sweep"); + } + rt->destroyAtomsAddedWhileSweepingTable(); + + maybeAtoms.reset(); + return Finished; +} + +class js::gc::WeakCacheSweepIterator +{ + JS::Zone*& sweepZone; + JS::detail::WeakCacheBase*& sweepCache; + + public: + explicit WeakCacheSweepIterator(GCRuntime* gc) + : sweepZone(gc->sweepZone.ref()), sweepCache(gc->sweepCache.ref()) + { + // Initialize state when we start sweeping a sweep group. + if (!sweepZone) { + sweepZone = gc->currentSweepGroup; + MOZ_ASSERT(!sweepCache); + sweepCache = sweepZone->weakCaches().getFirst(); + settle(); + } + + checkState(); + } + + bool empty(AutoLockHelperThreadState& lock) { + return !sweepZone; + } + + JS::detail::WeakCacheBase* next(AutoLockHelperThreadState& lock) { + if (empty(lock)) + return nullptr; + + JS::detail::WeakCacheBase* result = sweepCache; + sweepCache = sweepCache->getNext(); + settle(); + checkState(); + return result; + } + + void settle() { + while (sweepZone) { + while (sweepCache && !sweepCache->needsIncrementalBarrier()) + sweepCache = sweepCache->getNext(); + + if (sweepCache) + break; + + sweepZone = sweepZone->nextNodeInGroup(); + if (sweepZone) + sweepCache = sweepZone->weakCaches().getFirst(); + } + } + + private: + void checkState() { + MOZ_ASSERT((!sweepZone && !sweepCache) || + (sweepCache && sweepCache->needsIncrementalBarrier())); + } +}; + +class IncrementalSweepWeakCacheTask : public GCParallelTask +{ + WeakCacheSweepIterator& work_; + SliceBudget& budget_; + AutoLockHelperThreadState& lock_; + JS::detail::WeakCacheBase* cache_; + + public: + IncrementalSweepWeakCacheTask(JSRuntime* rt, WeakCacheSweepIterator& work, SliceBudget& budget, + AutoLockHelperThreadState& lock) + : GCParallelTask(rt), work_(work), budget_(budget), lock_(lock), + cache_(work.next(lock)) + { + MOZ_ASSERT(cache_); + runtime()->gc.startTask(*this, gcstats::PHASE_SWEEP_WEAK_CACHES, lock_); + } + + ~IncrementalSweepWeakCacheTask() { + runtime()->gc.joinTask(*this, gcstats::PHASE_SWEEP_WEAK_CACHES, lock_); + } + + private: + void run() override { + do { + MOZ_ASSERT(cache_->needsIncrementalBarrier()); + size_t steps = cache_->sweep(); + cache_->setNeedsIncrementalBarrier(false); + + AutoLockHelperThreadState lock; + budget_.step(steps); + if (budget_.isOverBudget()) + break; + + cache_ = work_.next(lock); + } while(cache_); + } +}; + +static const size_t MaxWeakCacheSweepTasks = 8; + +static size_t +WeakCacheSweepTaskCount() +{ + size_t targetTaskCount = HelperThreadState().cpuCount; + return Min(targetTaskCount, MaxWeakCacheSweepTasks); +} + +IncrementalProgress +GCRuntime::sweepWeakCaches(FreeOp* fop, SliceBudget& budget) +{ + WeakCacheSweepIterator work(this); + + { + AutoLockHelperThreadState lock; + gcstats::AutoPhase ap(stats(), gcstats::PHASE_SWEEP_COMPARTMENTS); + + Maybe tasks[MaxWeakCacheSweepTasks]; + for (size_t i = 0; !work.empty(lock) && i < WeakCacheSweepTaskCount(); i++) + tasks[i].emplace(rt, work, budget, lock); + + // Tasks run until budget or work is exhausted. + } + + AutoLockHelperThreadState lock; + return work.empty(lock) ? Finished : NotFinished; +} + +IncrementalProgress +GCRuntime::finalizeAllocKind(FreeOp* fop, SliceBudget& budget, Zone* zone, AllocKind kind) { // Set the number of things per arena for this AllocKind. @@ -5424,8 +5634,251 @@ GCRuntime::sweepShapeTree(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& b static void AddSweepPhase(bool* ok) { - if (*ok) - *ok = SweepPhases.emplaceBack(); + using Iter = decltype(mozilla::DeclVal().begin()); + using Elem = decltype(*mozilla::DeclVal()); + + Iter iter; + const Iter end; + + public: + explicit ContainerIter(const Container& container) + : iter(container.begin()), end(container.end()) + {} + + bool done() const { + return iter == end; + } + + Elem get() const { + return *iter; + } + + void next() { + MOZ_ASSERT(!done()); + ++iter; + } +}; + +// IncrementalIter is a template class that makes a normal iterator into one +// that can be used to perform incremental work by using external state that +// persists between instantiations. The state is only initialised on the first +// use and subsequent uses carry on from the previous state. +template +struct IncrementalIter +{ + using State = Maybe; + using Elem = decltype(mozilla::DeclVal().get()); + + private: + State& maybeIter; + + public: + template + explicit IncrementalIter(State& maybeIter, Args&&... args) + : maybeIter(maybeIter) + { + if (maybeIter.isNothing()) + maybeIter.emplace(mozilla::Forward(args)...); + } + + ~IncrementalIter() { + if (done()) + maybeIter.reset(); + } + + bool done() const { + return maybeIter.ref().done(); + } + + Elem get() const { + return maybeIter.ref().get(); + } + + void next() { + maybeIter.ref().next(); + } +}; + +// Iterate through the sweep groups created by GCRuntime::groupZonesForSweeping(). +class js::gc::SweepGroupsIter +{ + GCRuntime* gc; + + public: + explicit SweepGroupsIter(JSRuntime* rt) + : gc(&rt->gc) + { + MOZ_ASSERT(gc->currentSweepGroup); + } + + bool done() const { + return !gc->currentSweepGroup; + } + + Zone* get() const { + return gc->currentSweepGroup; + } + + void next() { + MOZ_ASSERT(!done()); + gc->getNextSweepGroup(); + } +}; + +namespace sweepaction { + +// Implementation of the SweepAction interface that calls a method on GCRuntime. +template +class SweepActionCall final : public SweepAction +{ + using Method = IncrementalProgress (GCRuntime::*)(Args...); + + Method method; + + public: + explicit SweepActionCall(Method m) : method(m) {} + IncrementalProgress run(GCRuntime* gc, Args... args) override { + return (gc->*method)(args...); + } + void assertFinished() const override { } +}; + +// Implementation of the SweepAction interface that calls a list of actions in +// sequence. +template +class SweepActionSequence final : public SweepAction +{ + using Action = SweepAction; + using ActionVector = Vector, 0, SystemAllocPolicy>; + using Iter = IncrementalIter>; + + ActionVector actions; + typename Iter::State iterState; + + public: + bool init(UniquePtr* acts, size_t count) { + for (size_t i = 0; i < count; i++) { + if (!actions.emplaceBack(Move(acts[i]))) + return false; + } + return true; + } + + IncrementalProgress run(Args... args) override { + for (Iter iter(iterState, actions); !iter.done(); iter.next()) { + if (iter.get()->run(args...) == NotFinished) + return NotFinished; + } + return Finished; + } + + void assertFinished() const override { + MOZ_ASSERT(iterState.isNothing()); + for (const auto& action : actions) + action->assertFinished(); + } +}; + +template +class SweepActionForEach final : public SweepAction +{ + using Elem = decltype(mozilla::DeclVal().get()); + using Action = SweepAction; + using IncrIter = IncrementalIter; + + Init iterInit; + UniquePtr action; + typename IncrIter::State iterState; + + public: + SweepActionForEach(const Init& init, UniquePtr action) + : iterInit(init), action(Move(action)) + {} + + IncrementalProgress run(Args... args) override { + for (IncrIter iter(iterState, iterInit); !iter.done(); iter.next()) { + if (action->run(args..., iter.get()) == NotFinished) + return NotFinished; + } + return Finished; + } + + void assertFinished() const override { + MOZ_ASSERT(iterState.isNothing()); + action->assertFinished(); + } +}; + +template +class SweepActionRepeatFor final : public SweepAction +{ + protected: + using Action = SweepAction; + using IncrIter = IncrementalIter; + + Init iterInit; + UniquePtr action; + typename IncrIter::State iterState; + + public: + SweepActionRepeatFor(const Init& init, UniquePtr action) + : iterInit(init), action(Move(action)) + {} + + IncrementalProgress run(Args... args) override { + for (IncrIter iter(iterState, iterInit); !iter.done(); iter.next()) { + if (action->run(args...) == NotFinished) + return NotFinished; + } + return Finished; + } + + void assertFinished() const override { + MOZ_ASSERT(iterState.isNothing()); + action->assertFinished(); + } +}; +// Helper class to remove the last template parameter from the instantiation of +// a variadic template. For example: +// +// RemoveLastTemplateParameter>::Type ==> Foo +// +// This works by recursively instantiating the Impl template with the contents +// of the parameter pack so long as there are at least two parameters. The +// specialization that matches when only one parameter remains discards it and +// instantiates the target template with parameters previously processed. +template +class RemoveLastTemplateParameter {}; + +template