diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index de3db9bc31..cf2f7b036a 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -35,6 +35,12 @@ class MarkingValidator; class AutoTraceSession; struct MovingTracer; +enum IncrementalProgress +{ + NotFinished = 0, + Finished +}; + class ChunkPool { Chunk* head_; @@ -737,6 +743,8 @@ class GCRuntime bool isShrinkingGC() const { return invocationKind == GC_SHRINK; } + static bool initializeSweepActions(); + void setGrayRootsTracer(JSTraceDataOp traceOp, void* data); MOZ_MUST_USE bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data); void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data); @@ -863,12 +871,6 @@ class GCRuntime static TenuredCell* refillFreeListInGC(Zone* zone, AllocKind thingKind); private: - enum IncrementalProgress - { - NotFinished = 0, - Finished - }; - // For ArenaLists::allocateFromArena() friend class ArenaLists; Chunk* pickChunk(const AutoLockGC& lock, @@ -951,7 +953,15 @@ class GCRuntime void beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock); bool shouldReleaseObservedTypes(); void endSweepingZoneGroup(); - IncrementalProgress sweepPhase(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock); + IncrementalProgress performSweepActions(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock); + static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, Zone* zone, + SliceBudget& budget, AllocKind kind); + static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, Zone* zone, + SliceBudget& budget, AllocKind kind); + static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, Zone* zone, + SliceBudget& budget, AllocKind kind); + static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, Zone* zone, + SliceBudget& budget, AllocKind kind); void endSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock); void sweepZones(FreeOp* fop, bool lastGC); void decommitAllWithoutUnlocking(const AutoLockGC& lock); @@ -1165,10 +1175,9 @@ class GCRuntime */ JS::Zone* zoneGroups; JS::Zone* currentZoneGroup; - bool sweepingTypes; - unsigned finalizePhase; + size_t sweepPhaseIndex; JS::Zone* sweepZone; - AllocKind sweepKind; + size_t sweepActionIndex; bool abortSweepAfterCurrentGroup; /* diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index 7fb035794b..575c075532 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -384,6 +384,42 @@ static const FinalizePhase BackgroundFinalizePhases[] = { } }; +// Incremental sweeping is controlled by a list of actions that describe what +// happens and in what order. Due to the incremental nature of sweeping an +// action does not necessarily run to completion so the current state is tracked +// in the GCRuntime by the performSweepActions() method. +// +// Actions are performed in phases run per sweep group, and each action is run +// for every zone in the group, i.e. as if by the following pseudocode: +// +// for each sweep group: +// for each phase: +// for each zone in sweep group: +// for each action in phase: +// perform_action + +struct SweepAction +{ + using Func = IncrementalProgress (*)(GCRuntime* gc, FreeOp* fop, Zone* zone, + SliceBudget& budget, AllocKind kind); + + Func func; + AllocKind kind; + + SweepAction(Func func, AllocKind kind) : func(func), kind(kind) {} +}; + +using SweepActionVector = Vector; +using SweepPhaseVector = Vector; + +static SweepPhaseVector SweepPhases; + +bool +js::gc::InitializeStaticData() +{ + return GCRuntime::initializeSweepActions(); +} + template<> JSObject* ArenaCellIterImpl::get() const @@ -841,8 +877,9 @@ GCRuntime::GCRuntime(JSRuntime* rt) : zoneGroupIndex(0), zoneGroups(nullptr), currentZoneGroup(nullptr), + sweepPhaseIndex(0), sweepZone(nullptr), - sweepKind(AllocKind::FIRST), + sweepActionIndex(0), abortSweepAfterCurrentGroup(false), arenasAllocatedDuringSweep(nullptr), startedCompacting(false), @@ -883,7 +920,7 @@ static const uint64_t JIT_SCRIPT_RELEASE_TYPES_PERIOD = 20; bool GCRuntime::init(uint32_t maxbytes, uint32_t maxNurseryBytes) { - InitMemorySubsystem(); + MOZ_ASSERT(SystemPageSize()); if (!rootsHash.init(256)) return false; @@ -4667,11 +4704,9 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock) zone->arenas.queueForegroundThingsForSweep(&fop); } - sweepingTypes = true; - - finalizePhase = 0; + sweepPhaseIndex = 0; sweepZone = currentZoneGroup; - sweepKind = AllocKind::FIRST; + sweepActionIndex = 0; { gcstats::AutoPhase ap(stats, gcstats::PHASE_FINALIZE_END); @@ -4765,7 +4800,7 @@ ArenaLists::foregroundFinalize(FreeOp* fop, AllocKind thingKind, SliceBudget& sl return true; } -GCRuntime::IncrementalProgress +IncrementalProgress GCRuntime::drainMarkStack(SliceBudget& sliceBudget, gcstats::Phase phase) { /* Run a marking slice and return whether the stack is now empty. */ @@ -4810,105 +4845,160 @@ SweepArenaList(Arena** arenasToSweep, SliceBudget& sliceBudget, Args... args) return true; } -GCRuntime::IncrementalProgress -GCRuntime::sweepPhase(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock) +/* static */ IncrementalProgress +GCRuntime::sweepTypeInformation(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget, + AllocKind kind) +{ + // Sweep dead type information stored in scripts and object groups, but + // don't finalize them yet. We have to sweep dead information from both live + // and dead scripts and object groups, so that no dead references remain in + // them. Type inference can end up crawling these zones again, such as for + // TypeCompartment::markSetsUnknown, and if this happens after sweeping for + // the sweep group finishes we won't be able to determine which things in + // the zone are live. + + MOZ_ASSERT(kind == AllocKind::LIMIT); + + gcstats::AutoPhase ap1(gc->stats, gcstats::PHASE_SWEEP_COMPARTMENTS); + gcstats::AutoPhase ap2(gc->stats, gcstats::PHASE_SWEEP_TYPES); + + ArenaLists& al = zone->arenas; + + AutoClearTypeInferenceStateOnOOM oom(zone); + + if (!SweepArenaList(&al.gcScriptArenasToUpdate, budget, &oom)) + return NotFinished; + + if (!SweepArenaList(&al.gcObjectGroupArenasToUpdate, budget, &oom)) + return NotFinished; + + // Finish sweeping type information in the zone. + { + gcstats::AutoPhase ap(gc->stats, gcstats::PHASE_SWEEP_TYPES_END); + zone->types.endSweep(gc->rt); + } + + return Finished; +} + +/* static */ IncrementalProgress +GCRuntime::mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget, + AllocKind kind) +{ + // Foreground finalized objects have already been finalized, and now their + // arenas can be reclaimed by freeing empty ones and making non-empty ones + // available for allocation. + + MOZ_ASSERT(kind == AllocKind::LIMIT); + zone->arenas.mergeForegroundSweptObjectArenas(); + return Finished; +} + +/* static */ IncrementalProgress +GCRuntime::finalizeAllocKind(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget, + AllocKind kind) +{ + // Set the number of things per arena for this AllocKind. + size_t thingsPerArena = Arena::thingsPerArena(kind); + auto& sweepList = gc->incrementalSweepList; + sweepList.setThingsPerArena(thingsPerArena); + + if (!zone->arenas.foregroundFinalize(fop, kind, budget, sweepList)) + return NotFinished; + + // Reset the slots of the sweep list that we used. + sweepList.reset(thingsPerArena); + + return Finished; +} + +/* static */ IncrementalProgress +GCRuntime::sweepShapeTree(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget, + AllocKind kind) +{ + // Remove dead shapes from the shape tree, but don't finalize them yet. + + MOZ_ASSERT(kind == AllocKind::LIMIT); + + gcstats::AutoPhase ap(gc->stats, gcstats::PHASE_SWEEP_SHAPE); + + ArenaLists& al = zone->arenas; + + if (!SweepArenaList(&al.gcShapeArenasToUpdate, budget)) + return NotFinished; + + if (!SweepArenaList(&al.gcAccessorShapeArenasToUpdate, budget)) + return NotFinished; + + return Finished; +} + +static void +AddSweepPhase(bool* ok) +{ + if (*ok) + *ok = SweepPhases.emplaceBack(); +} + +static void +AddSweepAction(bool* ok, SweepAction::Func func, AllocKind kind = AllocKind::LIMIT) +{ + if (*ok) + *ok = SweepPhases.back().emplaceBack(func, kind); +} + +/* static */ bool +GCRuntime::initializeSweepActions() +{ + bool ok = true; + + AddSweepPhase(&ok); + + AddSweepAction(&ok, GCRuntime::sweepTypeInformation); + AddSweepAction(&ok, GCRuntime::mergeSweptObjectArenas); + + for (const auto& finalizePhase : IncrementalFinalizePhases) { + AddSweepPhase(&ok); + for (auto kind : finalizePhase.kinds) + AddSweepAction(&ok, GCRuntime::finalizeAllocKind, kind); + } + + AddSweepPhase(&ok); + AddSweepAction(&ok, GCRuntime::sweepShapeTree); + + return ok; +} + +IncrementalProgress +GCRuntime::performSweepActions(SliceBudget& budget, AutoLockForExclusiveAccess& lock) { AutoSetThreadIsSweeping threadIsSweeping; gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP); FreeOp fop(rt); - if (drainMarkStack(sliceBudget, gcstats::PHASE_SWEEP_MARK) == NotFinished) + if (drainMarkStack(budget, gcstats::PHASE_SWEEP_MARK) == NotFinished) return NotFinished; for (;;) { - // Sweep dead type information stored in scripts and object groups, but - // don't finalize them yet. We have to sweep dead information from both - // live and dead scripts and object groups, so that no dead references - // remain in them. Type inference can end up crawling these zones - // again, such as for TypeCompartment::markSetsUnknown, and if this - // happens after sweeping for the zone group finishes we won't be able - // to determine which things in the zone are live. - if (sweepingTypes) { - gcstats::AutoPhase ap1(stats, gcstats::PHASE_SWEEP_COMPARTMENTS); - gcstats::AutoPhase ap2(stats, gcstats::PHASE_SWEEP_TYPES); - + for (; sweepPhaseIndex < SweepPhases.length(); sweepPhaseIndex++) { + const auto& actions = SweepPhases[sweepPhaseIndex]; for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) { - ArenaLists& al = sweepZone->arenas; - - AutoClearTypeInferenceStateOnOOM oom(sweepZone); - - if (!SweepArenaList(&al.gcScriptArenasToUpdate, sliceBudget, &oom)) - return NotFinished; - - if (!SweepArenaList( - &al.gcObjectGroupArenasToUpdate, sliceBudget, &oom)) - { - return NotFinished; - } - - // Finish sweeping type information in the zone. - { - gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_TYPES_END); - sweepZone->types.endSweep(rt); - } - - // Foreground finalized objects have already been finalized, - // and now their arenas can be reclaimed by freeing empty ones - // and making non-empty ones available for allocation. - al.mergeForegroundSweptObjectArenas(); - } - - sweepZone = currentZoneGroup; - sweepingTypes = false; - } - - /* Finalize foreground finalized things. */ - for (; finalizePhase < ArrayLength(IncrementalFinalizePhases) ; ++finalizePhase) { - gcstats::AutoPhase ap(stats, IncrementalFinalizePhases[finalizePhase].statsPhase); - - for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) { - Zone* zone = sweepZone; - - for (auto kind : SomeAllocKinds(sweepKind, AllocKind::LIMIT)) { - if (!IncrementalFinalizePhases[finalizePhase].kinds.contains(kind)) - continue; - - /* Set the number of things per arena for this AllocKind. */ - size_t thingsPerArena = Arena::thingsPerArena(kind); - incrementalSweepList.setThingsPerArena(thingsPerArena); - - if (!zone->arenas.foregroundFinalize(&fop, kind, sliceBudget, - incrementalSweepList)) - { - sweepKind = kind; + for (; sweepActionIndex < actions.length(); sweepActionIndex++) { + const auto& action = actions[sweepActionIndex]; + if (action.func(this, &fop, sweepZone, budget, action.kind) == NotFinished) return NotFinished; - } - - /* Reset the slots of the sweep list that we used. */ - incrementalSweepList.reset(thingsPerArena); } - sweepKind = AllocKind::FIRST; + // Reset action index to first. + sweepActionIndex = 0; } sweepZone = currentZoneGroup; } - /* Remove dead shapes from the shape tree, but don't finalize them yet. */ - { - gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_SHAPE); - - for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) { - ArenaLists& al = sweepZone->arenas; - - if (!SweepArenaList(&al.gcShapeArenasToUpdate, sliceBudget)) - return NotFinished; - - if (!SweepArenaList(&al.gcAccessorShapeArenasToUpdate, sliceBudget)) - return NotFinished; - } - } - + // Reset phase index. + sweepPhaseIndex = 0; + endSweepingZoneGroup(); getNextZoneGroup(); if (!currentZoneGroup) @@ -4999,7 +5089,7 @@ GCRuntime::beginCompactPhase() startedCompacting = true; } -GCRuntime::IncrementalProgress +IncrementalProgress GCRuntime::compactPhase(JS::gcreason::Reason reason, SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock) { @@ -5381,7 +5471,7 @@ GCRuntime::incrementalCollectSlice(SliceBudget& budget, JS::gcreason::Reason rea MOZ_FALLTHROUGH; case State::Sweep: - if (sweepPhase(budget, lock) == NotFinished) + if (performSweepActions(budget, lock) == NotFinished) break; endSweepPhase(destroyingRuntime, lock); diff --git a/js/src/jsgc.h b/js/src/jsgc.h index 4d6e086521..ffd6102747 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -838,6 +838,9 @@ class ArenaLists /* The number of GC cycles an empty chunk can survive before been released. */ const size_t MAX_EMPTY_CHUNK_AGE = 4; +extern bool +InitializeStaticData(); + } /* namespace gc */ class InterpreterFrame; diff --git a/js/src/vm/Initialization.cpp b/js/src/vm/Initialization.cpp index 6c4303d5e2..dc645f3ed9 100644 --- a/js/src/vm/Initialization.cpp +++ b/js/src/vm/Initialization.cpp @@ -102,6 +102,8 @@ JS::detail::InitWithFailureDiagnostic(bool isDebugBuild) RETURN_IF_FAIL(js::wasm::InitInstanceStaticData()); js::gc::InitMemorySubsystem(); // Ensure gc::SystemPageSize() works. + RETURN_IF_FAIL(js::gc::InitializeStaticData()); + RETURN_IF_FAIL(js::jit::InitProcessExecutableMemory()); MOZ_ALWAYS_TRUE(js::MemoryProtectionExceptionHandler::install());