wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-05 23:38:38 +09:00
Code Issues Projects Releases Packages Wiki Activity

56 GC.

Browse source 
Trying narrow when atom crash started between 55 and 60 and if its easy to revert.
This commit is contained in:
win7-7 2026-01-14 00:38:31 +02:00 • committed by wuggy
commit 17e93e902c
51 changed files with 3856 additions and 660 deletions
Download patch file Download diff file Expand all files Collapse all files

4
js/ipc/JavaScriptChild.cpp
View file

@ -34,7 +34,7 @@ TraceChild(JSTracer* trc, void* data)
JavaScriptChild::~JavaScriptChild()
{
JSContext* cx = dom::danger::GetJSContext();
JS_RemoveWeakPointerZoneGroupCallback(cx, UpdateChildWeakPointersBeforeSweepingZoneGroup);
JS_RemoveWeakPointerZonesCallback(cx, UpdateChildWeakPointersBeforeSweepingZoneGroup);
JS_RemoveExtraGCRootsTracer(cx, TraceChild, this);
}
@ -47,7 +47,7 @@ JavaScriptChild::init()
return false;
JSContext* cx = dom::danger::GetJSContext();
JS_AddWeakPointerZoneGroupCallback(cx, UpdateChildWeakPointersBeforeSweepingZoneGroup, this);
JS_AddWeakPointerZonesCallback(cx, UpdateChildWeakPointersBeforeSweepingZoneGroup, this);
JS_AddExtraGCRootsTracer(cx, TraceChild, this);
return true;
}

3
js/ipc/JavaScriptShared.h
View file

@ -7,6 +7,7 @@
#ifndef mozilla_jsipc_JavaScriptShared_h__
#define mozilla_jsipc_JavaScriptShared_h__
#include "mozilla/HashFunctions.h"
#include "mozilla/dom/DOMTypes.h"
#include "mozilla/jsipc/CrossProcessObjectWrappers.h"
#include "mozilla/jsipc/PJavaScript.h"
@ -71,7 +72,7 @@ struct ObjectIdHasher
{
typedef ObjectId Lookup;
static js::HashNumber hash(const Lookup& l) {
return l.serialize();
return mozilla::HashGeneric(l.serialize());
}
static bool match(const ObjectId& k, const ObjectId& l) {
return k == l;

24
js/public/HashTable.h
View file

@ -568,24 +568,16 @@ class HashSet
// h.add(p, k);
// }
// Pointer hashing policy that strips the lowest zeroBits when calculating the
// hash to improve key distribution.
template <typename Key, size_t zeroBits>
// Pointer hashing policy that uses HashGeneric() to create good hashes for
// pointers. Note that we don't shift out the lowest k bits to generate a
// good distribution for arena allocated pointers.
template <typename Key>
struct PointerHasher
{
typedef Key Lookup;
static HashNumber hash(const Lookup& l) {
size_t word = reinterpret_cast<size_t>(l) >> zeroBits;
static_assert(sizeof(HashNumber) == 4,
"subsequent code assumes a four-byte hash");
#if JS_BITS_PER_WORD == 32
return HashNumber(word);
#else
static_assert(sizeof(word) == 8,
"unexpected word size, new hashing strategy required to "
"properly incorporate all bits");
return HashNumber((word >> 32) ^ word);
#endif
size_t word = reinterpret_cast<size_t>(l);
return mozilla::HashGeneric(word);
}
static bool match(const Key& k, const Lookup& l) {
return k == l;
@ -619,7 +611,7 @@ struct DefaultHasher
// Specialize hashing policy for pointer types. It assumes that the type is
// at least word-aligned. For types with smaller size use PointerHasher.
template <class T>
struct DefaultHasher<T*> : PointerHasher<T*, mozilla::tl::FloorLog2<sizeof(void*)>::value>
struct DefaultHasher<T*> : PointerHasher<T*>
{};
// Specialize hashing policy for mozilla::UniquePtr to proxy the UniquePtr's
@ -628,7 +620,7 @@ template <class T, class D>
struct DefaultHasher<mozilla::UniquePtr<T, D>>
{
using Lookup = mozilla::UniquePtr<T, D>;
using PtrHasher = PointerHasher<T*, mozilla::tl::FloorLog2<sizeof(void*)>::value>;
using PtrHasher = PointerHasher<T*>;
static HashNumber hash(const Lookup& l) {
return PtrHasher::hash(l.get());

14
js/public/HeapAPI.h
View file

@ -63,6 +63,20 @@ const size_t ArenaHeaderSize = sizeof(size_t) + 2 * sizeof(uintptr_t) +
static const uint32_t BLACK = 0;
static const uint32_t GRAY = 1;
/*
* Two bits determine the mark color as follows:
* BlackBit GrayOrBlackBit color
* 0 0 white
* 0 1 gray
* 1 0 black
* 1 1 black
*/
enum class ColorBit : uint32_t
{
BlackBit = 0,
GrayOrBlackBit = 1
};
/*
* The "location" field in the Chunk trailer is a enum indicating various roles
* of the chunk.

5
js/public/UbiNode.h
View file

@ -9,6 +9,7 @@
#include "mozilla/Alignment.h"
#include "mozilla/Assertions.h"
#include "mozilla/Attributes.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/Maybe.h"
#include "mozilla/MemoryReporting.h"
#include "mozilla/Move.h"
@ -425,7 +426,7 @@ class StackFrame {
using Lookup = JS::ubi::StackFrame;
static js::HashNumber hash(const Lookup& lookup) {
return lookup.identifier();
return mozilla::HashGeneric(lookup.identifier());
}
static bool match(const StackFrame& key, const Lookup& lookup) {
@ -813,7 +814,7 @@ class Node {
// This simply uses the stock PointerHasher on the ubi::Node's pointer.
// We specialize DefaultHasher below to make this the default.
class HashPolicy {
typedef js::PointerHasher<void*, mozilla::tl::FloorLog2<sizeof(void*)>::value> PtrHash;
typedef js::PointerHasher<void*> PtrHash;
public:
typedef Node Lookup;

2
js/src/builtin/MapObject.cpp
View file

@ -87,7 +87,7 @@ HashValue(const Value& v, const mozilla::HashCodeScrambler& hcs)
return hcs.scramble(v.asRawBits());
MOZ_ASSERT(v.isNull() || !v.isGCThing(), "do not reveal pointers via hash codes");
return v.asRawBits();
return mozilla::HashGeneric(v.asRawBits());
}
HashNumber

5
js/src/gc/Allocator.cpp
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -253,8 +254,8 @@ GCRuntime::checkIncrementalZoneState(ExclusiveContext* cx, T* t)
if (!cx->isJSContext())
return;
Zone* zone = cx->asJSContext()->zone();
MOZ_ASSERT_IF(t && zone->wasGCStarted() && (zone->shouldMarkInZone() || zone->isGCSweeping()),
Zone* zone = cx->zone();
MOZ_ASSERT_IF(t && zone->wasGCStarted() && (zone->isGCMarking() || zone->isGCSweeping()),
t->asTenured().arena()->allocatedDuringIncremental);
#endif
}

7
js/src/gc/Barrier.cpp
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -36,8 +37,10 @@ IsMarkedBlack(NativeObject* obj)
return true;
gc::TenuredCell& tenured = obj->asTenured();
return (tenured.isMarked(gc::BLACK) && !tenured.isMarked(gc::GRAY)) ||
tenured.arena()->allocatedDuringIncremental;
if (tenured.isMarkedAny() || tenured.arena()->allocatedDuringIncremental)
return true;
return false;
}
bool

255
js/src/gc/GCRuntime.h
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -34,6 +35,7 @@ class AutoMaybeStartBackgroundAllocation;
class MarkingValidator;
class AutoTraceSession;
struct MovingTracer;
class WeakCacheSweepIterator;
enum IncrementalProgress
{
@ -121,7 +123,14 @@ class GCSchedulingTunables
* subsequently invoke the standard OOM machinery, independent of available
* physical memory.
*/
size_t gcMaxBytes_;
UnprotectedData<size_t> gcMaxBytes_;
/*
* Maximum nursery size for each zone group.
* Initially DefaultNurseryBytes and can be set by
* javascript.options.mem.nursery.max_kb
*/
ActiveThreadData<size_t> gcMaxNurseryBytes_;
/*
* The base value used to compute zone->trigger.gcBytes(). When
@ -131,7 +140,9 @@ class GCSchedulingTunables
size_t gcZoneAllocThresholdBase_;
/* Fraction of threshold.gcBytes() which triggers an incremental GC. */
double zoneAllocThresholdFactor_;
UnprotectedData<double> zoneAllocThresholdFactor_;
/* The same except when doing so would interrupt an already running GC. */
UnprotectedData<double> zoneAllocThresholdFactorAvoidInterrupt_;
/*
* Number of bytes to allocate between incremental slices in GCs triggered
@ -185,8 +196,10 @@ class GCSchedulingTunables
public:
GCSchedulingTunables()
: gcMaxBytes_(0),
gcMaxNurseryBytes_(0),
gcZoneAllocThresholdBase_(30 * 1024 * 1024),
zoneAllocThresholdFactor_(0.9),
zoneAllocThresholdFactorAvoidInterrupt_(0.95),
zoneAllocDelayBytes_(1024 * 1024),
dynamicHeapGrowthEnabled_(false),
highFrequencyThresholdUsec_(1000 * 1000),
@ -202,8 +215,10 @@ class GCSchedulingTunables
{}
size_t gcMaxBytes() const { return gcMaxBytes_; }
size_t gcMaxNurseryBytes() const { return gcMaxNurseryBytes_; }
size_t gcZoneAllocThresholdBase() const { return gcZoneAllocThresholdBase_; }
double zoneAllocThresholdFactor() const { return zoneAllocThresholdFactor_; }
double zoneAllocThresholdFactorAvoidInterrupt() const { return zoneAllocThresholdFactorAvoidInterrupt_; }
size_t zoneAllocDelayBytes() const { return zoneAllocDelayBytes_; }
bool isDynamicHeapGrowthEnabled() const { return dynamicHeapGrowthEnabled_; }
uint64_t highFrequencyThresholdUsec() const { return highFrequencyThresholdUsec_; }
@ -217,7 +232,7 @@ class GCSchedulingTunables
unsigned minEmptyChunkCount(const AutoLockGC&) const { return minEmptyChunkCount_; }
unsigned maxEmptyChunkCount() const { return maxEmptyChunkCount_; }
MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value, const AutoLockGC& lock);
[[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value, const AutoLockGC& lock);
};
/*
@ -544,8 +559,8 @@ class GCSchedulingState
template<typename F>
struct Callback {
F op;
void* data;
ActiveThreadOrGCTaskData<F> op;
ActiveThreadOrGCTaskData<void*> data;
Callback()
: op(nullptr), data(nullptr)
@ -595,6 +610,56 @@ typedef HashMap<Value*, const char*, DefaultHasher<Value*>, SystemAllocPolicy> R
using AllocKinds = mozilla::EnumSet<AllocKind>;
template <typename T>
class MemoryCounter
{
// Bytes counter to measure memory pressure for GC scheduling. It runs
// from maxBytes down to zero.
mozilla::Atomic<ptrdiff_t, mozilla::ReleaseAcquire> bytes_;
// GC trigger threshold for memory allocations.
js::ActiveThreadData<size_t> maxBytes_;
// Whether a GC has been triggered as a result of bytes falling below
// zero.
//
// This should be a bool, but Atomic only supports 32-bit and pointer-sized
// types.
mozilla::Atomic<uint32_t, mozilla::ReleaseAcquire> triggered_;
public:
MemoryCounter()
: bytes_(0),
maxBytes_(0),
triggered_(false)
{ }
void reset() {
bytes_ = maxBytes_;
triggered_ = false;
}
void setMax(size_t newMax) {
// For compatibility treat any value that exceeds PTRDIFF_T_MAX to
// mean that value.
maxBytes_ = (ptrdiff_t(newMax) >= 0) ? newMax : size_t(-1) >> 1;
reset();
}
bool update(T* owner, size_t bytes) {
bytes_ -= ptrdiff_t(bytes);
if (MOZ_UNLIKELY(isTooMuchMalloc())) {
if (!triggered_)
triggered_ = owner->triggerGCForTooMuchMalloc();
}
return triggered_;
}
ptrdiff_t bytes() const { return bytes_; }
size_t maxBytes() const { return maxBytes_; }
bool isTooMuchMalloc() const { return bytes_ <= 0; }
};
class GCRuntime
{
public:
@ -607,10 +672,10 @@ class GCRuntime
void removeRoot(Value* vp);
void setMarkStackLimit(size_t limit, AutoLockGC& lock);
MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock);
[[nodiscard]] bool setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock);
uint32_t getParameter(JSGCParamKey key, const AutoLockGC& lock);
MOZ_MUST_USE bool triggerGC(JS::gcreason::Reason reason);
[[nodiscard]] bool triggerGC(JS::gcreason::Reason reason);
void maybeAllocTriggerZoneGC(Zone* zone, const AutoLockGC& lock);
// The return value indicates if we were able to do the GC.
bool triggerZoneGC(Zone* zone, JS::gcreason::Reason reason);
@ -749,14 +814,23 @@ class GCRuntime
MOZ_MUST_USE bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data);
void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data);
bool triggerGCForTooMuchMalloc() {
if (!triggerGC(JS::gcreason::TOO_MUCH_MALLOC))
return false;
// Even though this method may be called off the main thread it is safe
// to access mallocCounter here since triggerGC() will return false in
// that case.
stats().recordTrigger(mallocCounter.bytes(), mallocCounter.maxBytes());
return true;
}
int32_t getMallocBytes() const { return mallocCounter.bytes(); }
size_t maxMallocBytesAllocated() const { return mallocCounter.maxBytes(); }
bool isTooMuchMalloc() const { return mallocCounter.isTooMuchMalloc(); }
void resetMallocBytes() { mallocCounter.reset(); }
void setMaxMallocBytes(size_t value);
#ifdef MOZ_DEVTOOLS_SERVER
int32_t getMallocBytes() const { return mallocBytesUntilGC; }
#endif
void resetMallocBytes();
bool isTooMuchMalloc() const { return mallocBytesUntilGC <= 0; }
void updateMallocCounter(JS::Zone* zone, size_t nbytes);
void onTooMuchMalloc();
void setGCCallback(JSGCCallback callback, void* data);
void callGCCallback(JSGCStatus status) const;
@ -765,10 +839,10 @@ class GCRuntime
void callObjectsTenuredCallback();
MOZ_MUST_USE bool addFinalizeCallback(JSFinalizeCallback callback, void* data);
void removeFinalizeCallback(JSFinalizeCallback func);
MOZ_MUST_USE bool addWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback,
[[nodiscard]] bool addWeakPointerZonesCallback(JSWeakPointerZonesCallback callback,
void* data);
void removeWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback);
MOZ_MUST_USE bool addWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback,
void removeWeakPointerZonesCallback(JSWeakPointerZonesCallback callback);
[[nodiscard]] bool addWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback,
void* data);
void removeWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback);
JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback);
@ -779,20 +853,7 @@ class GCRuntime
void setFullCompartmentChecks(bool enable);
bool isManipulatingDeadZones() { return manipulatingDeadZones; }
void setManipulatingDeadZones(bool value) { manipulatingDeadZones = value; }
unsigned objectsMarkedInDeadZonesCount() { return objectsMarkedInDeadZones; }
void incObjectsMarkedInDeadZone() {
MOZ_ASSERT(manipulatingDeadZones);
++objectsMarkedInDeadZones;
}
JS::Zone* getCurrentZoneGroup() { return currentZoneGroup; }
void setFoundBlackGrayEdges(TenuredCell& target) {
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!foundBlackGrayEdges.append(&target))
oomUnsafe.crash("OOM|small: failed to insert into foundBlackGrayEdges");
}
JS::Zone* getCurrentSweepGroup() { return currentSweepGroup; }
uint64_t gcNumber() const { return number; }
@ -834,7 +895,7 @@ class GCRuntime
const ChunkPool& emptyChunks(const AutoLockGC& lock) const { return emptyChunks_; }
typedef ChainedIter<Chunk*, ChunkPool::Iter, ChunkPool::Iter> NonEmptyChunksIter;
NonEmptyChunksIter allNonEmptyChunks() {
return NonEmptyChunksIter(ChunkPool::Iter(availableChunks_), ChunkPool::Iter(fullChunks_));
return NonEmptyChunksIter(ChunkPool::Iter(availableChunks_.ref()), ChunkPool::Iter(fullChunks_.ref()));
}
Chunk* getOrAllocChunk(const AutoLockGC& lock,
@ -870,7 +931,24 @@ class GCRuntime
static T* tryNewTenuredThing(ExclusiveContext* cx, AllocKind kind, size_t thingSize);
static TenuredCell* refillFreeListInGC(Zone* zone, AllocKind thingKind);
void bufferGrayRoots();
/*
* Concurrent sweep infrastructure.
*/
void startTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked);
void joinTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked);
// Delete an empty zone group after its contents have been merged.
void deleteEmptyZoneGroup(ZoneGroup* group);
private:
enum IncrementalResult
{
Reset = 0,
Ok
};
// For ArenaLists::allocateFromArena()
friend class ArenaLists;
Chunk* pickChunk(const AutoLockGC& lock,
@ -926,14 +1004,15 @@ class GCRuntime
AutoLockForExclusiveAccess& lock);
void purgeRuntime(AutoLockForExclusiveAccess& lock);
MOZ_MUST_USE bool beginMarkPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock);
[[nodiscard]] bool beginMarkPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock);
bool prepareZonesForCollection(JS::gcreason::Reason reason, bool* isFullOut,
AutoLockForExclusiveAccess& lock);
bool shouldPreserveJITCode(JSCompartment* comp, int64_t currentTime,
JS::gcreason::Reason reason, bool canAllocateMoreCode);
void traceRuntimeForMajorGC(JSTracer* trc, AutoLockForExclusiveAccess& lock);
void traceRuntimeAtoms(JSTracer* trc, AutoLockForExclusiveAccess& lock);
void traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark,
AutoLockForExclusiveAccess& lock);
void bufferGrayRoots();
void maybeDoCycleCollection();
void markCompartments();
IncrementalProgress drainMarkStack(SliceBudget& sliceBudget, gcstats::Phase phase);
@ -945,23 +1024,31 @@ class GCRuntime
void markAllWeakReferences(gcstats::Phase phase);
void markAllGrayReferences(gcstats::Phase phase);
void beginSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock);
void findZoneGroups(AutoLockForExclusiveAccess& lock);
MOZ_MUST_USE bool findInterZoneEdges();
void getNextZoneGroup();
void endMarkingZoneGroup();
void beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock);
void beginSweepPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock);
void groupZonesForSweeping(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock);
[[nodiscard]] bool findInterZoneEdges();
void getNextSweepGroup();
void endMarkingSweepGroup();
void beginSweepingSweepGroup();
bool shouldReleaseObservedTypes();
void endSweepingZoneGroup();
IncrementalProgress performSweepActions(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock);
static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, Zone* zone,
SliceBudget& budget, AllocKind kind);
static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, Zone* zone,
SliceBudget& budget, AllocKind kind);
static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, Zone* zone,
SliceBudget& budget, AllocKind kind);
static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, Zone* zone,
SliceBudget& budget, AllocKind kind);
void sweepDebuggerOnMainThread(FreeOp* fop);
void sweepJitDataOnMainThread(FreeOp* fop);
void endSweepingSweepGroup();
IncrementalProgress performSweepActions(SliceBudget& sliceBudget,
AutoLockForExclusiveAccess& lock);
static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, SliceBudget& budget,
Zone* zone);
static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, SliceBudget& budget,
Zone* zone);
static IncrementalProgress sweepAtomsTable(GCRuntime* gc, FreeOp* fop, SliceBudget& budget);
void startSweepingAtomsTable();
IncrementalProgress sweepAtomsTable(SliceBudget& budget);
static IncrementalProgress sweepWeakCaches(GCRuntime* gc, FreeOp* fop, SliceBudget& budget);
IncrementalProgress sweepWeakCaches(SliceBudget& budget);
static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, SliceBudget& budget,
Zone* zone, AllocKind kind);
static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, SliceBudget& budget,
Zone* zone);
void endSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock);
void sweepZones(FreeOp* fop, bool lastGC);
void decommitAllWithoutUnlocking(const AutoLockGC& lock);
@ -993,7 +1080,7 @@ class GCRuntime
#endif
void callFinalizeCallbacks(FreeOp* fop, JSFinalizeStatus status) const;
void callWeakPointerZoneGroupCallbacks() const;
void callWeakPointerZonesCallbacks() const;
void callWeakPointerCompartmentCallbacks(JSCompartment* comp) const;
public:
@ -1002,8 +1089,11 @@ class GCRuntime
/* Embedders can use this zone however they wish. */
JS::Zone* systemZone;
/* List of compartments and zones (protected by the GC lock). */
ZoneVector zones;
// List of all zone groups (protected by the GC lock).
private:
ActiveThreadOrGCTaskData<ZoneGroupVector> groups_;
public:
ZoneGroupVector& groups() { return groups_.ref(); }
Nursery nursery;
StoreBuffer storeBuffer;
@ -1034,11 +1124,11 @@ class GCRuntime
// to the fullChunks pool. During a GC, if all arenas are free, the chunk
// is moved back to the emptyChunks pool and scheduled for eventual
// release.
ChunkPool availableChunks_;
UnprotectedData<ChunkPool> availableChunks_;
// When all arenas in a chunk are used, it is moved to the fullChunks pool
// so as to reduce the cost of operations on the available lists.
ChunkPool fullChunks_;
UnprotectedData<ChunkPool> fullChunks_;
RootedValueMap rootsHash;
@ -1075,7 +1165,7 @@ class GCRuntime
Okay,
Failed
};
GrayBufferState grayBufferState;
ActiveThreadOrGCTaskData<GrayBufferState> grayBufferState;
bool hasBufferedGrayRoots() const { return grayBufferState == GrayBufferState::Okay; }
// Clear each zone's gray buffers, but do not change the current state.
@ -1106,9 +1196,6 @@ class GCRuntime
/* Incremented on every GC slice. */
uint64_t number;
/* The number at the time of the most recent GC's first slice. */
uint64_t startNumber;
/* Whether the currently running GC can finish in multiple slices. */
bool isIncremental;
@ -1138,10 +1225,10 @@ class GCRuntime
* The current incremental GC phase. This is also used internally in
* non-incremental GC.
*/
State incrementalState;
ActiveThreadOrGCTaskData<State> incrementalState;
/* Indicates that the last incremental slice exhausted the mark stack. */
bool lastMarkSlice;
ActiveThreadData<bool> lastMarkSlice;
/* Whether any sweeping will take place in the separate GC helper thread. */
bool sweepOnBackgroundThread;
@ -1149,11 +1236,8 @@ class GCRuntime
/* Whether observed type information is being released in the current GC. */
bool releaseObservedTypes;
/* Whether any black->gray edges were found during marking. */
BlackGrayEdgeVector foundBlackGrayEdges;
/* Singly linekd list of zones to be swept in the background. */
ZoneList backgroundSweepZones;
/* Singly linked list of zones to be swept in the background. */
ActiveThreadOrGCTaskData<ZoneList> backgroundSweepZones;
/*
* Free LIFO blocks are transferred to this allocator before being freed on
@ -1161,14 +1245,9 @@ class GCRuntime
*/
LifoAlloc blocksToFreeAfterSweeping;
/*
* Free LIFO blocks are transferred to this allocator before being freed
* after minor GC.
*/
LifoAlloc blocksToFreeAfterMinorGC;
/* Index of current zone group (for stats). */
unsigned zoneGroupIndex;
private:
/* Index of current sweep group (for stats). */
ActiveThreadData<unsigned> sweepGroupIndex;
/*
* Incremental sweep state.
@ -1180,13 +1259,15 @@ class GCRuntime
size_t sweepActionIndex;
bool abortSweepAfterCurrentGroup;
/*
* Concurrent sweep infrastructure.
*/
void startTask(GCParallelTask& task, gcstats::Phase phase,
AutoLockHelperThreadState& locked);
void joinTask(GCParallelTask& task, gcstats::Phase phase,
AutoLockHelperThreadState& locked);
ActiveThreadData<JS::Zone*> sweepGroups;
ActiveThreadOrGCTaskData<JS::Zone*> currentSweepGroup;
ActiveThreadData<UniquePtr<SweepAction<GCRuntime*, FreeOp*, SliceBudget&>>> sweepActions;
ActiveThreadOrGCTaskData<JS::Zone*> sweepZone;
ActiveThreadData<mozilla::Maybe<AtomSet::Enum>> maybeAtomsToSweep;
ActiveThreadOrGCTaskData<JS::detail::WeakCacheBase*> sweepCache;
ActiveThreadData<bool> abortSweepAfterCurrentGroup;
friend class WeakCacheSweepIterator;
/*
* List head of arenas allocated during the sweep phase.
@ -1257,20 +1338,10 @@ class GCRuntime
Callback<JS::DoCycleCollectionCallback> gcDoCycleCollectionCallback;
Callback<JSObjectsTenuredCallback> tenuredCallback;
CallbackVector<JSFinalizeCallback> finalizeCallbacks;
CallbackVector<JSWeakPointerZoneGroupCallback> updateWeakPointerZoneGroupCallbacks;
CallbackVector<JSWeakPointerZonesCallback> updateWeakPointerZonesCallbacks;
CallbackVector<JSWeakPointerCompartmentCallback> updateWeakPointerCompartmentCallbacks;
/*
* Malloc counter to measure memory pressure for GC scheduling. It runs
* from maxMallocBytes down to zero.
*/
mozilla::Atomic<ptrdiff_t, mozilla::ReleaseAcquire> mallocBytesUntilGC;
/*
* Whether a GC has been triggered as a result of mallocBytesUntilGC
* falling below zero.
*/
mozilla::Atomic<bool, mozilla::ReleaseAcquire> mallocGCTriggered;
MemoryCounter<GCRuntime> mallocCounter;
/*
* The trace operations to trace embedding-specific GC roots. One is for

327
js/src/gc/GenerateStatsPhases.py Normal file
View file

@ -0,0 +1,327 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Generate graph structures for GC statistics recording.
#
# Stats phases are nested and form a directed acyclic graph starting
# from a set of root phases. Importantly, a phase may appear under more
# than one parent phase.
#
# For example, the following arrangement is possible:
#
# +---+
# | A |
# +---+
# |
# +-------+-------+
# | | |
# v v v
# +---+ +---+ +---+
# | B | | C | | D |
# +---+ +---+ +---+
# | |
# +---+---+
# |
# v
# +---+
# | E |
# +---+
#
# This graph is expanded into a tree (or really a forest) and phases
# with multiple parents are duplicated.
#
# For example, the input example above would be expanded to:
#
# +---+
# | A |
# +---+
# |
# +-------+-------+
# | | |
# v v v
# +---+ +---+ +---+
# | B | | C | | D |
# +---+ +---+ +---+
# | |
# v v
# +---+ +---+
# | E | | E'|
# +---+ +---+
import re
import sys
import collections
class PhaseKind():
def __init__(self, name, descr, bucket, children = []):
self.name = name
self.descr = descr
self.bucket = bucket
self.children = children
# The root marking phase appears in several places in the graph.
MarkRootsPhaseKind = PhaseKind("MARK_ROOTS", "Mark Roots", 48, [
PhaseKind("MARK_CCWS", "Mark Cross Compartment Wrappers", 50),
PhaseKind("MARK_STACK", "Mark C and JS stacks", 51),
PhaseKind("MARK_RUNTIME_DATA", "Mark Runtime-wide Data", 52),
PhaseKind("MARK_EMBEDDING", "Mark Embedding", 53),
PhaseKind("MARK_COMPARTMENTS", "Mark Compartments", 54)
])
JoinParallelTasksPhaseKind = PhaseKind("JOIN_PARALLEL_TASKS", "Join Parallel Tasks", 67)
UnmarkGrayPhaseKind = PhaseKind("UNMARK_GRAY", "Unmark gray", 56)
PhaseKindGraphRoots = [
PhaseKind("MUTATOR", "Mutator Running", 0),
PhaseKind("GC_BEGIN", "Begin Callback", 1),
PhaseKind("EVICT_NURSERY_FOR_MAJOR_GC", "Evict Nursery For Major GC", 70, [
MarkRootsPhaseKind,
]),
PhaseKind("WAIT_BACKGROUND_THREAD", "Wait Background Thread", 2),
PhaseKind("PREPARE", "Prepare For Collection", 69, [
PhaseKind("UNMARK", "Unmark", 7),
PhaseKind("BUFFER_GRAY_ROOTS", "Buffer Gray Roots", 49),
PhaseKind("MARK_DISCARD_CODE", "Mark Discard Code", 3),
PhaseKind("RELAZIFY_FUNCTIONS", "Relazify Functions", 4),
PhaseKind("PURGE", "Purge", 5),
PhaseKind("PURGE_SHAPE_TABLES", "Purge ShapeTables", 60),
JoinParallelTasksPhaseKind
]),
PhaseKind("MARK", "Mark", 6, [
MarkRootsPhaseKind,
UnmarkGrayPhaseKind,
PhaseKind("MARK_DELAYED", "Mark Delayed", 8)
]),
PhaseKind("SWEEP", "Sweep", 9, [
PhaseKind("SWEEP_MARK", "Mark During Sweeping", 10, [
UnmarkGrayPhaseKind,
PhaseKind("SWEEP_MARK_INCOMING_BLACK", "Mark Incoming Black Pointers", 12, [
UnmarkGrayPhaseKind,
]),
PhaseKind("SWEEP_MARK_WEAK", "Mark Weak", 13, [
UnmarkGrayPhaseKind,
]),
PhaseKind("SWEEP_MARK_INCOMING_GRAY", "Mark Incoming Gray Pointers", 14),
PhaseKind("SWEEP_MARK_GRAY", "Mark Gray", 15),
PhaseKind("SWEEP_MARK_GRAY_WEAK", "Mark Gray and Weak", 16)
]),
PhaseKind("FINALIZE_START", "Finalize Start Callbacks", 17, [
PhaseKind("WEAK_ZONES_CALLBACK", "Per-Slice Weak Callback", 57),
PhaseKind("WEAK_COMPARTMENT_CALLBACK", "Per-Compartment Weak Callback", 58)
]),
PhaseKind("UPDATE_ATOMS_BITMAP", "Sweep Atoms Bitmap", 68),
PhaseKind("SWEEP_ATOMS_TABLE", "Sweep Atoms Table", 18),
PhaseKind("SWEEP_COMPARTMENTS", "Sweep Compartments", 20, [
PhaseKind("SWEEP_DISCARD_CODE", "Sweep Discard Code", 21),
PhaseKind("SWEEP_INNER_VIEWS", "Sweep Inner Views", 22),
PhaseKind("SWEEP_CC_WRAPPER", "Sweep Cross Compartment Wrappers", 23),
PhaseKind("SWEEP_BASE_SHAPE", "Sweep Base Shapes", 24),
PhaseKind("SWEEP_INITIAL_SHAPE", "Sweep Initial Shapes", 25),
PhaseKind("SWEEP_TYPE_OBJECT", "Sweep Type Objects", 26),
PhaseKind("SWEEP_BREAKPOINT", "Sweep Breakpoints", 27),
PhaseKind("SWEEP_REGEXP", "Sweep Regexps", 28),
PhaseKind("SWEEP_COMPRESSION", "Sweep Compression Tasks", 62),
PhaseKind("SWEEP_WEAKMAPS", "Sweep WeakMaps", 63),
PhaseKind("SWEEP_UNIQUEIDS", "Sweep Unique IDs", 64),
PhaseKind("SWEEP_JIT_DATA", "Sweep JIT Data", 65),
PhaseKind("SWEEP_WEAK_CACHES", "Sweep Weak Caches", 66),
PhaseKind("SWEEP_MISC", "Sweep Miscellaneous", 29),
PhaseKind("SWEEP_TYPES", "Sweep type information", 30, [
PhaseKind("SWEEP_TYPES_BEGIN", "Sweep type tables and compilations", 31),
PhaseKind("SWEEP_TYPES_END", "Free type arena", 32),
]),
JoinParallelTasksPhaseKind
]),
PhaseKind("SWEEP_OBJECT", "Sweep Object", 33),
PhaseKind("SWEEP_STRING", "Sweep String", 34),
PhaseKind("SWEEP_SCRIPT", "Sweep Script", 35),
PhaseKind("SWEEP_SCOPE", "Sweep Scope", 59),
PhaseKind("SWEEP_REGEXP_SHARED", "Sweep RegExpShared", 61),
PhaseKind("SWEEP_SHAPE", "Sweep Shape", 36),
PhaseKind("SWEEP_JITCODE", "Sweep JIT code", 37),
PhaseKind("FINALIZE_END", "Finalize End Callback", 38),
PhaseKind("DESTROY", "Deallocate", 39),
JoinParallelTasksPhaseKind
]),
PhaseKind("COMPACT", "Compact", 40, [
PhaseKind("COMPACT_MOVE", "Compact Move", 41),
PhaseKind("COMPACT_UPDATE", "Compact Update", 42, [
MarkRootsPhaseKind,
PhaseKind("COMPACT_UPDATE_CELLS", "Compact Update Cells", 43),
JoinParallelTasksPhaseKind
]),
]),
PhaseKind("GC_END", "End Callback", 44),
PhaseKind("MINOR_GC", "All Minor GCs", 45, [
MarkRootsPhaseKind,
]),
PhaseKind("EVICT_NURSERY", "Minor GCs to Evict Nursery", 46, [
MarkRootsPhaseKind,
]),
PhaseKind("TRACE_HEAP", "Trace Heap", 47, [
MarkRootsPhaseKind,
]),
PhaseKind("BARRIER", "Barriers", 55, [
UnmarkGrayPhaseKind
])
]
# Make a linear list of all unique phases by performing a depth first
# search on the phase graph starting at the roots. This will be used to
# generate the PhaseKind enum.
def findAllPhaseKinds():
phases = []
seen = set()
def dfs(phase):
if phase in seen:
return
phases.append(phase)
seen.add(phase)
for child in phase.children:
dfs(child)
for phase in PhaseKindGraphRoots:
dfs(phase)
return phases
AllPhaseKinds = findAllPhaseKinds()
# Expand the DAG into a tree, duplicating phases which have more than
# one parent.
class Phase:
def __init__(self, phaseKind, parent):
self.phaseKind = phaseKind
self.parent = parent
self.depth = parent.depth + 1 if parent else 0
self.children = []
self.nextSibling = None
self.nextInPhaseKind = None
self.path = re.sub(r'\W+', '_', phaseKind.name.lower())
if parent is not None:
self.path = parent.path + '.' + self.path
def expandPhases():
phases = []
phasesForKind = collections.defaultdict(list)
def traverse(phaseKind, parent):
ep = Phase(phaseKind, parent)
phases.append(ep)
# Update list of expanded phases for this phase kind.
if phasesForKind[phaseKind]:
phasesForKind[phaseKind][-1].nextInPhaseKind = ep
phasesForKind[phaseKind].append(ep)
# Recurse over children.
for child in phaseKind.children:
child_ep = traverse(child, ep)
if ep.children:
ep.children[-1].nextSibling = child_ep
ep.children.append(child_ep)
return ep
for phaseKind in PhaseKindGraphRoots:
traverse(phaseKind, None)
return phases, phasesForKind
AllPhases, PhasesForPhaseKind = expandPhases()
# Name phases based on phase kind name and index if there are multiple phases
# corresponding to a single phase kind.
for phaseKind in AllPhaseKinds:
phases = PhasesForPhaseKind[phaseKind]
if len(phases) == 1:
phases[0].name = "%s" % phaseKind.name
else:
for index, phase in enumerate(phases):
phase.name = "%s_%d" % (phaseKind.name, index + 1)
# Find the maximum phase nesting.
MaxPhaseNesting = max(phase.depth for phase in AllPhases) + 1
# Generate code.
def writeList(out, items):
if items:
out.write(",\n".join(" " + item for item in items) + "\n")
def writeEnumClass(out, name, type, items, extraItems):
items = [ "FIRST" ] + items + [ "LIMIT" ] + extraItems
items[1] += " = " + items[0]
out.write("enum class %s : %s {\n" % (name, type));
writeList(out, items)
out.write("};\n")
def generateHeader(out):
#
# Generate PhaseKind enum.
#
phaseKindNames = map(lambda phaseKind: phaseKind.name, AllPhaseKinds)
extraPhaseKinds = [
"NONE = LIMIT",
"EXPLICIT_SUSPENSION = LIMIT",
"IMPLICIT_SUSPENSION"
]
writeEnumClass(out, "PhaseKind", "uint8_t", phaseKindNames, extraPhaseKinds)
out.write("\n")
#
# Generate Phase enum.
#
phaseNames = map(lambda phase: phase.name, AllPhases)
extraPhases = [
"NONE = LIMIT",
"EXPLICIT_SUSPENSION = LIMIT",
"IMPLICIT_SUSPENSION"
]
writeEnumClass(out, "Phase", "uint8_t", phaseNames, extraPhases)
out.write("\n")
#
# Generate MAX_PHASE_NESTING constant.
#
out.write("static const size_t MAX_PHASE_NESTING = %d;\n" % MaxPhaseNesting)
def generateCpp(out):
#
# Generate the PhaseKindInfo table.
#
out.write("static const PhaseKindTable phaseKinds = {\n")
for phaseKind in AllPhaseKinds:
phase = PhasesForPhaseKind[phaseKind][0]
out.write(" /* PhaseKind::%s */ PhaseKindInfo { Phase::%s, %d },\n" %
(phaseKind.name, phase.name, phaseKind.bucket))
out.write("};\n")
out.write("\n")
#
# Generate the PhaseInfo tree.
#
def name(phase):
return "Phase::" + phase.name if phase else "Phase::NONE"
out.write("static const PhaseTable phases = {\n")
for phase in AllPhases:
firstChild = phase.children[0] if phase.children else None
phaseKind = phase.phaseKind
out.write(" /* %s */ PhaseInfo { %s, %s, %s, %s, PhaseKind::%s, %d, \"%s\", \"%s\" },\n" %
(name(phase),
name(phase.parent),
name(firstChild),
name(phase.nextSibling),
name(phase.nextInPhaseKind),
phaseKind.name,
phase.depth,
phaseKind.descr,
phase.path))
out.write("};\n")

189
js/src/gc/Heap.h
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -54,10 +55,6 @@ extern bool
CurrentThreadIsIonCompiling();
#endif
// The return value indicates if anything was unmarked.
extern bool
UnmarkGrayCellRecursively(gc::Cell* cell, JS::TraceKind kind);
extern void
TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name);
@ -250,6 +247,13 @@ FOR_EACH_ALLOCKIND(EXPAND_ELEMENT)
static const size_t MAX_BACKGROUND_FINALIZE_KINDS =
size_t(AllocKind::LIMIT) - size_t(AllocKind::OBJECT_LIMIT) / 2;
/* Mark colors to pass to markIfUnmarked. */
enum class MarkColor : uint32_t
{
Black = 0,
Gray
};
class TenuredCell;
// A GC cell is the base class for all GC things.
@ -260,8 +264,11 @@ struct Cell
MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const;
MOZ_ALWAYS_INLINE TenuredCell& asTenured();
inline JSRuntime* runtimeFromMainThread() const;
inline JS::shadow::Runtime* shadowRuntimeFromMainThread() const;
MOZ_ALWAYS_INLINE bool isMarkedAny() const;
MOZ_ALWAYS_INLINE bool isMarkedBlack() const;
MOZ_ALWAYS_INLINE bool isMarkedGray() const;
inline JSRuntime* runtimeFromActiveCooperatingThread() const;
// Note: Unrestricted access to the runtime of a GC thing from an arbitrary
// thread can easily lead to races. Use this method very carefully.
@ -294,6 +301,7 @@ struct Cell
return static_cast<const T*>(this);
}
#ifdef DEBUG
inline bool isAligned() const;
void dump(FILE* fp) const;
@ -315,10 +323,13 @@ class TenuredCell : public Cell
static MOZ_ALWAYS_INLINE const TenuredCell* fromPointer(const void* ptr);
// Mark bit management.
MOZ_ALWAYS_INLINE bool isMarked(uint32_t color = BLACK) const;
MOZ_ALWAYS_INLINE bool isMarkedAny() const;
MOZ_ALWAYS_INLINE bool isMarkedBlack() const;
MOZ_ALWAYS_INLINE bool isMarkedGray() const;
// The return value indicates if the cell went from unmarked to marked.
MOZ_ALWAYS_INLINE bool markIfUnmarked(uint32_t color = BLACK) const;
MOZ_ALWAYS_INLINE void unmark(uint32_t color) const;
MOZ_ALWAYS_INLINE bool markIfUnmarked(MarkColor color = MarkColor::Black) const;
MOZ_ALWAYS_INLINE void markBlack() const;
MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src);
// Access to the arena.
@ -881,6 +892,15 @@ static_assert(ArenasPerChunk == 62, "Do not accidentally change our heap's densi
static_assert(ArenasPerChunk == 252, "Do not accidentally change our heap's density.");
#endif
static inline void
AssertValidColorBit(const TenuredCell* thing, ColorBit colorBit)
{
#ifdef DEBUG
Arena* arena = thing->arena();
MOZ_ASSERT(unsigned(colorBit) < arena->getThingSize() / CellBytesPerMarkBit);
#endif
}
/* A chunk bitmap contains enough mark bits for all the cells in a chunk. */
struct ChunkBitmap
{
@ -889,26 +909,40 @@ struct ChunkBitmap
public:
ChunkBitmap() { }
MOZ_ALWAYS_INLINE void getMarkWordAndMask(const Cell* cell, uint32_t color,
MOZ_ALWAYS_INLINE void getMarkWordAndMask(const TenuredCell* cell, ColorBit colorBit,
uintptr_t** wordp, uintptr_t* maskp)
{
detail::GetGCThingMarkWordAndMask(uintptr_t(cell), color, wordp, maskp);
detail::GetGCThingMarkWordAndMask(uintptr_t(cell), colorBit, wordp, maskp);
}
MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarked(const Cell* cell, uint32_t color) {
MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool markBit(const TenuredCell* cell, ColorBit colorBit) {
AssertValidColorBit(cell, colorBit);
uintptr_t* word, mask;
getMarkWordAndMask(cell, color, &word, &mask);
getMarkWordAndMask(cell, colorBit, &word, &mask);
return *word & mask;
}
MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarkedAny(const TenuredCell* cell) {
return markBit(cell, ColorBit::BlackBit) || markBit(cell, ColorBit::GrayOrBlackBit);
}
MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarkedBlack(const TenuredCell* cell) {
return markBit(cell, ColorBit::BlackBit);
}
MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarkedGray(const TenuredCell* cell) {
return !markBit(cell, ColorBit::BlackBit) && markBit(cell, ColorBit::GrayOrBlackBit);
}
// The return value indicates if the cell went from unmarked to marked.
MOZ_ALWAYS_INLINE bool markIfUnmarked(const Cell* cell, uint32_t color) {
MOZ_ALWAYS_INLINE bool markIfUnmarked(const TenuredCell* cell, MarkColor color) {
uintptr_t* word, mask;
getMarkWordAndMask(cell, BLACK, &word, &mask);
if (*word & mask)
return false;
*word |= mask;
if (color != BLACK) {
if (color == MarkColor::Black) {
*word |= mask;
} else {
/*
* We use getMarkWordAndMask to recalculate both mask and word as
* doing just mask << color may overflow the mask.
@ -921,16 +955,23 @@ struct ChunkBitmap
return true;
}
MOZ_ALWAYS_INLINE void unmark(const Cell* cell, uint32_t color) {
MOZ_ALWAYS_INLINE void markBlack(const TenuredCell* cell) {
uintptr_t* word, mask;
getMarkWordAndMask(cell, color, &word, &mask);
*word &= ~mask;
}
MOZ_ALWAYS_INLINE void copyMarkBit(Cell* dst, const TenuredCell* src, uint32_t color) {
uintptr_t* word, mask;
getMarkWordAndMask(dst, color, &word, &mask);
*word = (*word & ~mask) | (src->isMarked(color) ? mask : 0);
MOZ_ALWAYS_INLINE void copyMarkBit(TenuredCell* dst, const TenuredCell* src,
ColorBit colorBit) {
uintptr_t* srcWord, srcMask;
getMarkWordAndMask(src, colorBit, &srcWord, &srcMask);
uintptr_t* dstWord, dstMask;
getMarkWordAndMask(dst, colorBit, &dstWord, &dstMask);
*dstWord &= ~dstMask;
if (*srcWord & srcMask)
*dstWord |= dstMask;
}
void clear() {
@ -944,7 +985,8 @@ struct ChunkBitmap
"that covers bits from two arenas.");
uintptr_t* word, unused;
getMarkWordAndMask(reinterpret_cast<Cell*>(arena->address()), BLACK, &word, &unused);
getMarkWordAndMask(reinterpret_cast<TenuredCell*>(arena->address()),
ColorBit::BlackBit, &word, &unused);
return word;
}
};
@ -1120,15 +1162,6 @@ Arena::chunk() const
return Chunk::fromAddress(address());
}
static void
AssertValidColor(const TenuredCell* thing, uint32_t color)
{
#ifdef DEBUG
Arena* arena = thing->arena();
MOZ_ASSERT(color < arena->getThingSize() / CellSize);
#endif
}
MOZ_ALWAYS_INLINE const TenuredCell&
Cell::asTenured() const
{
@ -1143,6 +1176,24 @@ Cell::asTenured()
return *static_cast<TenuredCell*>(this);
}
MOZ_ALWAYS_INLINE bool
Cell::isMarkedAny() const
{
return !isTenured() || asTenured().isMarkedAny();
}
MOZ_ALWAYS_INLINE bool
Cell::isMarkedBlack() const
{
return !isTenured() || asTenured().isMarkedBlack();
}
MOZ_ALWAYS_INLINE bool
Cell::isMarkedGray() const
{
return isTenured() && asTenured().isMarkedGray();
}
inline JSRuntime*
Cell::runtimeFromMainThread() const
{
@ -1227,17 +1278,29 @@ TenuredCell::fromPointer(const void* ptr)
}
bool
TenuredCell::isMarked(uint32_t color /* = BLACK */) const
TenuredCell::isMarkedAny() const
{
MOZ_ASSERT(arena()->allocated());
AssertValidColor(this, color);
return chunk()->bitmap.isMarked(this, color);
return chunk()->bitmap.isMarkedAny(this);
}
bool
TenuredCell::markIfUnmarked(uint32_t color /* = BLACK */) const
TenuredCell::isMarkedBlack() const
{
MOZ_ASSERT(arena()->allocated());
return chunk()->bitmap.isMarkedBlack(this);
}
bool
TenuredCell::isMarkedGray() const
{
MOZ_ASSERT(arena()->allocated());
return chunk()->bitmap.isMarkedGray(this);
}
bool
TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const
{
AssertValidColor(this, color);
return chunk()->bitmap.markIfUnmarked(this, color);
}
@ -1320,11 +1383,11 @@ TenuredCell::readBarrier(TenuredCell* thing)
MOZ_ASSERT(tmp == thing);
}
if (thing->isMarked(GRAY)) {
// There shouldn't be anything marked grey unless we're on the main thread.
if (thing->isMarkedGray()) {
// There shouldn't be anything marked grey unless we're on the active thread.
MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread()));
if (!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone))
UnmarkGrayCellRecursively(thing, thing->getTraceKind());
if (!RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(shadowZone))
JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(thing, thing->getTraceKind()));
}
}
@ -1380,6 +1443,52 @@ static const int32_t ChunkLocationOffsetFromLastByte =
int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask);
} /* namespace gc */
namespace debug {
// Utility functions meant to be called from an interactive debugger.
enum class MarkInfo : int {
BLACK = 0,
GRAY = 1,
UNMARKED = -1,
NURSERY = -2,
};
// Get the mark color for a cell, in a way easily usable from a debugger.
MOZ_NEVER_INLINE MarkInfo
GetMarkInfo(js::gc::Cell* cell);
// Sample usage from gdb:
//
// (gdb) p $word = js::debug::GetMarkWordAddress(obj)
// $1 = (uintptr_t *) 0x7fa56d5fe360
// (gdb) p/x $mask = js::debug::GetMarkMask(obj, js::gc::GRAY)
// $2 = 0x200000000
// (gdb) watch *$word
// Hardware watchpoint 7: *$word
// (gdb) cond 7 *$word & $mask
// (gdb) cont
//
// Note that this is *not* a watchpoint on a single bit. It is a watchpoint on
// the whole word, which will trigger whenever the word changes and the
// selected bit is set after the change.
//
// So if the bit changing is the desired one, this is exactly what you want.
// But if a different bit changes (either set or cleared), you may still stop
// execution if the $mask bit happened to already be set. gdb does not expose
// enough information to restrict the watchpoint to just a single bit.
// Return the address of the word containing the mark bits for the given cell,
// or nullptr if the cell is in the nursery.
MOZ_NEVER_INLINE uintptr_t*
GetMarkWordAddress(js::gc::Cell* cell);
// Return the mask for the given cell and color bit, or 0 if the cell is in the
// nursery.
MOZ_NEVER_INLINE uintptr_t
GetMarkMask(js::gc::Cell* cell, uint32_t colorBit);
} /* namespace debug */
} /* namespace js */
#endif /* gc_Heap_h */

7
js/src/gc/Iteration.cpp
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -75,8 +76,8 @@ js::IterateChunks(JSContext* cx, void* data, IterateChunkCallback chunkCallback)
{
AutoPrepareForTracing prep(cx, SkipAtoms);
for (auto chunk = cx->gc.allNonEmptyChunks(); !chunk.done(); chunk.next())
chunkCallback(cx, data, chunk);
for (auto chunk = cx->runtime()->gc.allNonEmptyChunks(); !chunk.done(); chunk.next())
chunkCallback(cx->runtime(), data, chunk);
}
void
@ -106,7 +107,7 @@ IterateGrayObjects(Zone* zone, GCThingCallback cellCallback, void* data)
{
for (auto kind : ObjectAllocKinds()) {
for (GrayObjectIter obj(zone, kind); !obj.done(); obj.next()) {
if (obj->asTenured().isMarked(GRAY))
if (obj->asTenured().isMarkedGray())
cellCallback(data, JS::GCCellPtr(obj.get()));
}
}

250
js/src/gc/Marking.cpp
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -24,6 +25,8 @@
#include "vm/BigIntType.h"
#include "vm/Debugger.h"
#include "vm/EnvironmentObject.h"
#include "vm/RegExpObject.h"
#include "vm/RegExpShared.h"
#include "vm/Scope.h"
#include "vm/Shape.h"
#include "vm/Symbol.h"
@ -216,8 +219,10 @@ js::CheckTracedThing(JSTracer* trc, T* thing)
Zone* zone = thing->zoneFromAnyThread();
JSRuntime* rt = trc->runtime();
MOZ_ASSERT_IF(!IsMovingTracer(trc), CurrentThreadCanAccessZone(zone));
MOZ_ASSERT_IF(!IsMovingTracer(trc), CurrentThreadCanAccessRuntime(rt));
if (!IsMovingTracer(trc) && !IsBufferGrayRootsTracer(trc) && !IsClearEdgesTracer(trc)) {
MOZ_ASSERT(CurrentThreadCanAccessZone(zone));
MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
}
MOZ_ASSERT(zone->runtimeFromAnyThread() == trc->runtime());
@ -231,14 +236,15 @@ js::CheckTracedThing(JSTracer* trc, T* thing)
*/
bool isGcMarkingTracer = trc->isMarkingTracer();
MOZ_ASSERT_IF(zone->requireGCTracer(), isGcMarkingTracer || IsBufferGrayRootsTracer(trc));
MOZ_ASSERT_IF(zone->requireGCTracer(),
isGcMarkingTracer || IsBufferGrayRootsTracer(trc) || IsUnmarkGrayTracer(trc));
if (isGcMarkingTracer) {
GCMarker* gcMarker = static_cast<GCMarker*>(trc);
MOZ_ASSERT_IF(gcMarker->shouldCheckCompartments(),
zone->isCollecting() || zone->isAtomsZone());
MOZ_ASSERT_IF(gcMarker->markColor() == GRAY,
MOZ_ASSERT_IF(gcMarker->markColor() == MarkColor::Gray,
!zone->isGCMarkingBlack() || zone->isAtomsZone());
MOZ_ASSERT(!(zone->isGCSweeping() || zone->isGCFinished() || zone->isGCCompacting()));
@ -281,35 +287,39 @@ JS_FOR_EACH_TRACEKIND(IMPL_CHECK_TRACED_THING);
#undef IMPL_CHECK_TRACED_THING
} // namespace js
static bool UnmarkGrayGCThing(JSRuntime* rt, JS::GCCellPtr thing);
static bool
ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, Cell* cell)
{
if (!trc->isMarkingTracer())
return true;
uint32_t color = static_cast<GCMarker*>(trc)->markColor();
MOZ_ASSERT(color == BLACK || color == GRAY);
MarkColor color = GCMarker::fromTracer(trc)->markColor();
if (!cell->isTenured()) {
MOZ_ASSERT(color == BLACK);
MOZ_ASSERT(color == MarkColor::Black);
return false;
}
TenuredCell& tenured = cell->asTenured();
JS::Zone* zone = tenured.zone();
if (color == BLACK) {
if (!src->zone()->isGCMarking() && !zone->isGCMarking())
return false;
if (color == MarkColor::Black) {
/*
* Having black->gray edges violates our promise to the cycle
* collector. This can happen if we're collecting a compartment and it
* has an edge to an uncollected compartment: it's possible that the
* source and destination of the cross-compartment edge should be gray,
* but the source was marked black by the conservative scanner.
* but the source was marked black by the write barrier.
*/
if (tenured.isMarked(GRAY)) {
if (tenured.isMarkedGray()) {
MOZ_ASSERT(!zone->isCollecting());
trc->runtime()->gc.setFoundBlackGrayEdges(tenured);
UnmarkGrayGCThing(trc->runtime(), JS::GCCellPtr(cell, cell->getTraceKind()));
}
return zone->shouldMarkInZone();
return zone->isGCMarking();
} else {
if (zone->isGCMarkingBlack()) {
/*
@ -317,7 +327,7 @@ ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, Cell* cell)
* but it will be later, so record the cell so it can be marked gray
* at the appropriate time.
*/
if (!tenured.isMarked())
if (!tenured.isMarkedAny())
DelayCrossCompartmentGrayMarking(src);
return false;
}
@ -619,7 +629,7 @@ js::TraceProcessGlobalRoot(JSTracer* trc, T* thing, const char* name)
// permanent atoms, so likewise require no subsquent marking.
CheckTracedThing(trc, *ConvertToBase(&thing));
if (trc->isMarkingTracer())
thing->markIfUnmarked(gc::BLACK);
thing->markIfUnmarked(gc::MarkColor::Black);
else
DoCallback(trc->asCallbackTracer(), ConvertToBase(&thing), name);
}
@ -731,7 +741,7 @@ GCMarker::markImplicitEdgesHelper(T markedThing)
return;
Zone* zone = gc::TenuredCell::fromPointer(markedThing)->zone();
MOZ_ASSERT(zone->shouldMarkInZone());
MOZ_ASSERT(zone->isGCMarking());
MOZ_ASSERT(!zone->isGCSweeping());
auto p = zone->gcWeakKeys.get(JS::GCCellPtr(markedThing));
@ -788,8 +798,7 @@ ShouldMark<JSObject*>(GCMarker* gcmarker, JSObject* obj)
// Don't mark things outside a zone if we are in a per-zone GC. It is
// faster to check our own arena, which we can do since we know that
// the object is tenured.
Zone* zone = obj->asTenured().zone();
return (zone && zone->shouldMarkInZone());
return obj->asTenured().zone()->shouldMarkInZone();
}
template <typename T>
@ -978,7 +987,7 @@ js::GCMarker::mark(T* thing)
MOZ_ASSERT(!IsInsideNursery(gc::TenuredCell::fromPointer(thing)));
return gc::ParticipatesInCC<T>::value
? gc::TenuredCell::fromPointer(thing)->markIfUnmarked(markColor())
: gc::TenuredCell::fromPointer(thing)->markIfUnmarked(gc::BLACK);
: gc::TenuredCell::fromPointer(thing)->markIfUnmarked(gc::MarkColor::Black);
}
@ -1057,7 +1066,9 @@ Shape::traceChildren(JSTracer* trc)
inline void
js::GCMarker::eagerlyMarkChildren(Shape* shape)
{
MOZ_ASSERT(shape->isMarked(this->markColor()));
MOZ_ASSERT_IF(markColor() == MarkColor::Gray, shape->isMarkedGray());
MOZ_ASSERT_IF(markColor() == MarkColor::Black, shape->isMarkedBlack());
do {
// Special case: if a base shape has a shape table then all its pointers
// must point to this shape or an anscestor. Since these pointers will
@ -1110,7 +1121,7 @@ inline void
js::GCMarker::eagerlyMarkChildren(JSLinearString* linearStr)
{
AssertShouldMarkInZone(linearStr);
MOZ_ASSERT(linearStr->isMarked());
MOZ_ASSERT(linearStr->isMarkedAny());
MOZ_ASSERT(linearStr->JSString::isLinear());
// Use iterative marking to avoid blowing out the stack.
@ -1173,7 +1184,7 @@ js::GCMarker::eagerlyMarkChildren(JSRope* rope)
JS_DIAGNOSTICS_ASSERT(rope->getTraceKind() == JS::TraceKind::String);
JS_DIAGNOSTICS_ASSERT(rope->JSString::isRope());
AssertShouldMarkInZone(rope);
MOZ_ASSERT(rope->isMarked());
MOZ_ASSERT(rope->isMarkedAny());
JSRope* next = nullptr;
JSString* right = rope->rightChild();
@ -1272,14 +1283,12 @@ ModuleScope::Data::trace(JSTracer* trc)
TraceNullableEdge(trc, &module, "scope module");
TraceBindingNames(trc, trailingNames.start(), length);
}
void
WasmFunctionScope::Data::trace(JSTracer* trc)
{
TraceNullableEdge(trc, &instance, "wasm function");
TraceBindingNames(trc, trailingNames.start(), length);
}
void
Scope::traceChildren(JSTracer* trc)
{
@ -1316,7 +1325,6 @@ Scope::traceChildren(JSTracer* trc)
case ScopeKind::WasmFunction:
reinterpret_cast<WasmFunctionScope::Data*>(data_)->trace(trc);
break;
}
}
inline void
@ -1382,7 +1390,8 @@ js::GCMarker::eagerlyMarkChildren(Scope* scope)
case ScopeKind::With:
break;
case ScopeKind::WasmFunction: {
case ScopeKind::WasmFunction: {
WasmFunctionScope::Data* data = reinterpret_cast<WasmFunctionScope::Data*>(scope->data_);
traverseEdge(scope, static_cast<JSObject*>(data->instance));
names = &data->trailingNames;
@ -1390,7 +1399,6 @@ js::GCMarker::eagerlyMarkChildren(Scope* scope)
break;
}
}
if (scope->kind_ == ScopeKind::Function) {
for (uint32_t i = 0; i < length; i++) {
if (JSAtom* name = names->operator[](i).name())
@ -2027,7 +2035,7 @@ MarkStack::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const
GCMarker::GCMarker(JSRuntime* rt)
: JSTracer(rt, JSTracer::TracerKindTag::Marking, ExpandWeakMaps),
stack(size_t(-1)),
color(BLACK),
color(MarkColor::Black),
unmarkedArenaStackTop(nullptr)
#ifdef DEBUG
, markLaterArenas(0)
@ -2050,7 +2058,7 @@ GCMarker::start()
MOZ_ASSERT(!started);
started = true;
#endif
color = BLACK;
color = MarkColor::Black;
linearWeakMarkingDisabled_ = false;
MOZ_ASSERT(!unmarkedArenaStackTop);
@ -2082,7 +2090,7 @@ GCMarker::stop()
void
GCMarker::reset()
{
color = BLACK;
color = MarkColor::Black;
stack.reset();
MOZ_ASSERT(isMarkStackEmpty());
@ -2103,6 +2111,32 @@ GCMarker::reset()
MOZ_ASSERT(!markLaterArenas);
}
template <typename T>
void
GCMarker::pushTaggedPtr(T* ptr)
{
checkZone(ptr);
if (!stack.push(ptr))
delayMarkingChildren(ptr);
}
void
GCMarker::pushValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end)
{
checkZone(obj);
if (!stack.push(obj, start, end))
delayMarkingChildren(obj);
}
void
GCMarker::repush(JSObject* obj)
{
MOZ_ASSERT_IF(markColor() == MarkColor::Gray, gc::TenuredCell::fromPointer(obj)->isMarkedGray());
MOZ_ASSERT_IF(markColor() == MarkColor::Black, gc::TenuredCell::fromPointer(obj)->isMarkedBlack());
pushTaggedPtr(obj);
}
void
GCMarker::enterWeakMarkingMode()
{
@ -2119,8 +2153,8 @@ GCMarker::enterWeakMarkingMode()
if (weakMapAction() == ExpandWeakMaps) {
tag_ = TracerKindTag::WeakMarking;
for (GCZoneGroupIter zone(runtime()); !zone.done(); zone.next()) {
for (WeakMapBase* m : zone->gcWeakMapList) {
for (GCSweepGroupIter zone(runtime()); !zone.done(); zone.next()) {
for (WeakMapBase* m : zone->gcWeakMapList()) {
if (m->marked)
(void) m->traceEntries(this);
}
@ -2153,7 +2187,7 @@ GCMarker::markDelayedChildren(Arena* arena)
for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) {
TenuredCell* t = i.getCell();
if (always || t->isMarked()) {
if (always || t->isMarkedAny()) {
t->markIfUnmarked();
js::TraceChildren(this, t, MapAllocToTraceKind(arena->getAllocKind()));
}
@ -2433,7 +2467,7 @@ JSObject*
js::TenuringTracer::moveToTenured(JSObject* src)
{
MOZ_ASSERT(IsInsideNursery(src));
MOZ_ASSERT(!src->zone()->usedByExclusiveThread);
MOZ_ASSERT(!src->zone()->usedByHelperThread());
AllocKind dstKind = src->allocKindForTenure(nursery());
Zone* zone = src->zone();
@ -2718,7 +2752,10 @@ IsMarkedInternalCommon(T* thingp)
return true;
if (zone->isGCCompacting() && IsForwarded(*thingp))
*thingp = Forwarded(*thingp);
return (*thingp)->asTenured().isMarked();
return true;
}
return thing.isMarkedAny() || thing.arena()->allocatedDuringIncremental;
}
template <typename T>
@ -2769,7 +2806,7 @@ js::gc::IsAboutToBeFinalizedDuringSweep(TenuredCell& tenured)
MOZ_ASSERT(tenured.zoneFromAnyThread()->isGCSweeping());
if (tenured.arena()->allocatedDuringIncremental)
return false;
return !tenured.isMarked();
return !tenured.isMarkedAny();
}
template <typename T>
@ -2949,6 +2986,43 @@ struct UnmarkGrayTracer : public JS::CallbackTracer
* of the containers, we must add unmark-graying read barriers to these
* containers.
*/
#ifdef DEBUG
struct AssertNonGrayTracer : public JS::CallbackTracer {
explicit AssertNonGrayTracer(JSRuntime* rt) : JS::CallbackTracer(rt) {}
void onChild(const JS::GCCellPtr& thing) override {
MOZ_ASSERT(!thing.asCell()->isMarkedGray());
}
};
#endif
class UnmarkGrayTracer : public JS::CallbackTracer
{
public:
// We set weakMapAction to DoNotTraceWeakMaps because the cycle collector
// will fix up any color mismatches involving weakmaps when it runs.
explicit UnmarkGrayTracer(JSRuntime *rt)
: JS::CallbackTracer(rt, DoNotTraceWeakMaps)
, unmarkedAny(false)
, oom(false)
, stack(rt->gc.unmarkGrayStack)
{}
void unmark(JS::GCCellPtr cell);
// Whether we unmarked anything.
bool unmarkedAny;
// Whether we ran out of memory.
bool oom;
private:
// Stack of cells to traverse.
Vector<JS::GCCellPtr, 0, SystemAllocPolicy>& stack;
void onChild(const JS::GCCellPtr& thing) override;
};
void
UnmarkGrayTracer::onChild(const JS::GCCellPtr& thing)
{
@ -2976,7 +3050,7 @@ UnmarkGrayTracer::onChild(const JS::GCCellPtr& thing)
}
TenuredCell& tenured = cell->asTenured();
if (!tenured.isMarked(js::gc::GRAY))
if (!tenured.isMarkedGray())
return;
tenured.unmark(js::gc::GRAY);
@ -3013,51 +3087,77 @@ UnmarkGrayTracer::onChild(const JS::GCCellPtr& thing)
unmarkedAny |= childTracer.unmarkedAny;
}
template <typename T>
static bool
TypedUnmarkGrayCellRecursively(T* t)
UnmarkGrayGCThing(JSRuntime* rt, JS::GCCellPtr thing)
{
MOZ_ASSERT(t);
MOZ_ASSERT(thing);
JSRuntime* rt = t->runtimeFromMainThread();
MOZ_ASSERT(!rt->isHeapCollecting());
MOZ_ASSERT(!rt->isCycleCollecting());
bool unmarkedArg = false;
if (t->isTenured()) {
if (!t->asTenured().isMarked(GRAY))
return false;
t->asTenured().unmark(GRAY);
unmarkedArg = true;
}
UnmarkGrayTracer trc(rt);
gcstats::AutoPhase outerPhase(rt->gc.stats, gcstats::PHASE_BARRIER);
gcstats::AutoPhase innerPhase(rt->gc.stats, gcstats::PHASE_UNMARK_GRAY);
t->traceChildren(&trc);
return unmarkedArg || trc.unmarkedAny;
}
struct UnmarkGrayCellRecursivelyFunctor {
template <typename T> bool operator()(T* t) { return TypedUnmarkGrayCellRecursively(t); }
};
bool
js::UnmarkGrayCellRecursively(Cell* cell, JS::TraceKind kind)
{
return DispatchTraceKindTyped(UnmarkGrayCellRecursivelyFunctor(), cell, kind);
}
bool
js::UnmarkGrayShapeRecursively(Shape* shape)
{
return TypedUnmarkGrayCellRecursively(shape);
UnmarkGrayTracer unmarker(rt);
gcstats::AutoPhase innerPhase(rt->gc.stats(), gcstats::PHASE_UNMARK_GRAY);
unmarker.unmark(thing);
return unmarker.unmarkedAny;
}
JS_FRIEND_API(bool)
JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr thing)
{
return js::UnmarkGrayCellRecursively(thing.asCell(), thing.kind());
MOZ_ASSERT(!JS::CurrentThreadIsHeapCollecting());
MOZ_ASSERT(!JS::CurrentThreadIsHeapCycleCollecting());
JSRuntime* rt = thing.asCell()->runtimeFromActiveCooperatingThread();
gcstats::AutoPhase outerPhase(rt->gc.stats(), gcstats::PHASE_BARRIER);
return UnmarkGrayGCThing(rt, thing);
}
bool
js::UnmarkGrayShapeRecursively(Shape* shape)
{
return JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr(shape));
}
namespace js {
namespace debug {
MarkInfo
GetMarkInfo(Cell* rawCell)
{
if (!rawCell->isTenured())
return MarkInfo::NURSERY;
TenuredCell* cell = &rawCell->asTenured();
if (cell->isMarkedGray())
return MarkInfo::GRAY;
if (cell->isMarkedBlack())
return MarkInfo::BLACK;
return MarkInfo::UNMARKED;
}
uintptr_t*
GetMarkWordAddress(Cell* cell)
{
if (!cell->isTenured())
return nullptr;
uintptr_t* wordp;
uintptr_t mask;
js::gc::detail::GetGCThingMarkWordAndMask(uintptr_t(cell), ColorBit::BlackBit, &wordp, &mask);
return wordp;
}
uintptr_t
GetMarkMask(Cell* cell, uint32_t colorBit)
{
MOZ_ASSERT(colorBit == 0 || colorBit == 1);
if (!cell->isTenured())
return 0;
ColorBit bit = colorBit == 0 ? ColorBit::BlackBit : ColorBit::GrayOrBlackBit;
uintptr_t* wordp;
uintptr_t mask;
js::gc::detail::GetGCThingMarkWordAndMask(uintptr_t(cell), bit, &wordp, &mask);
return mask;
}
}
}

61
js/src/gc/Marking.h
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -72,9 +73,48 @@ class MarkStack
maxCapacity_(maxCapacity)
{}
~MarkStack() {
js_free(stack_);
}
LastTag = TempRopeTag
};
static const uintptr_t TagMask = 7;
static_assert(TagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags.");
static_assert(TagMask <= gc::CellAlignMask, "The tag mask must be embeddable in a Cell*.");
class TaggedPtr
{
uintptr_t bits;
Cell* ptr() const;
public:
TaggedPtr(Tag tag, Cell* ptr);
Tag tag() const;
template <typename T> T* as() const;
JSObject* asValueArrayObject() const;
JSObject* asSavedValueArrayObject() const;
JSRope* asTempRope() const;
};
struct ValueArray
{
ValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end);
HeapSlot* end;
HeapSlot* start;
TaggedPtr ptr;
};
struct SavedValueArray
{
SavedValueArray(JSObject* obj, size_t index, HeapSlot::Kind kind);
uintptr_t kind;
uintptr_t index;
TaggedPtr ptr;
};
explicit MarkStack(size_t maxCapacity);
~MarkStack();
size_t capacity() { return end_ - stack_; }
@ -197,15 +237,15 @@ class GCMarker : public JSTracer
*/
void setMarkColorGray() {
MOZ_ASSERT(isDrained());
MOZ_ASSERT(color == gc::BLACK);
color = gc::GRAY;
MOZ_ASSERT(color == gc::MarkColor::Black);
color = gc::MarkColor::Gray;
}
void setMarkColorBlack() {
MOZ_ASSERT(isDrained());
MOZ_ASSERT(color == gc::GRAY);
color = gc::BLACK;
MOZ_ASSERT(color == gc::MarkColor::Gray);
color = gc::MarkColor::Black;
}
uint32_t markColor() const { return color; }
gc::MarkColor markColor() const { return color; }
void enterWeakMarkingMode();
void leaveWeakMarkingMode();
@ -329,7 +369,7 @@ class GCMarker : public JSTracer
MarkStack stack;
/* The color is only applied to objects and functions. */
uint32_t color;
ActiveThreadData<gc::MarkColor> color;
/* Pointer to the top of the stack of arenas we are delaying marking on. */
js::gc::Arena* unmarkedArenaStackTop;
@ -360,6 +400,9 @@ class GCMarker : public JSTracer
// the marking phase of incremental GC.
bool
IsBufferGrayRootsTracer(JSTracer* trc);
bool
IsUnmarkGrayTracer(JSTracer* trc);
#endif
namespace gc {

1
js/src/gc/Nursery-inl.h
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=4 sw=4 et tw=79 ft=cpp:
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this

234
js/src/gc/Nursery.cpp
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sw=4 et tw=78:
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
@ -153,7 +154,7 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGC& lock)
exit(0);
}
enableProfiling_ = true;
profileThreshold_ = atoi(env);
profileThreshold_ = TimeDuration::FromMicroseconds(atoi(env));
}
env = getenv("JS_GC_REPORT_TENURING");
@ -197,8 +198,7 @@ js::Nursery::enable()
setCurrentChunk(0);
setStartPosition();
MOZ_ALWAYS_TRUE(runtime()->gc.storeBuffer.enable());
return;
MOZ_ALWAYS_TRUE(runtime()->gc.storeBuffer().enable());
}
void
@ -209,8 +209,7 @@ js::Nursery::disable()
return;
updateNumChunks(0);
currentEnd_ = 0;
position_ = 0;
runtime()->gc.storeBuffer.disable();
runtime()->gc.storeBuffer().disable();
}
bool
@ -383,11 +382,11 @@ Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint3
void
Nursery::setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
uint32_t nelems)
uint32_t capacity)
{
// Only use a direct forwarding pointer if there is enough space for one.
setForwardingPointer(oldHeader->elements(), newHeader->elements(),
nelems > ObjectElements::VALUES_PER_HEADER);
capacity > 0);
}
#ifdef DEBUG
@ -433,9 +432,57 @@ js::TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery)
{
}
void
js::Nursery::renderProfileJSON(JSONPrinter& json) const
{
if (!isEnabled()) {
json.beginObject();
json.property("status", "nursery disabled");
json.endObject();
return;
}
if (previousGC.reason == JS::gcreason::NO_REASON) {
// If the nursery was empty when the last minorGC was requested, then
// no nursery collection will have been performed but JSON may still be
// requested. (And as a public API, this function should not crash in
// such a case.)
json.beginObject();
json.property("status", "no collection");
json.endObject();
return;
}
json.beginObject();
json.property("reason", JS::gcreason::ExplainReason(previousGC.reason));
json.property("bytes_tenured", previousGC.tenuredBytes);
json.floatProperty("promotion_rate",
100.0 * previousGC.tenuredBytes / double(previousGC.nurseryUsedBytes), 2);
json.property("nursery_bytes", previousGC.nurseryUsedBytes);
json.property("new_nursery_bytes", numChunks() * ChunkSize);
json.beginObjectProperty("timings");
#define EXTRACT_NAME(name, text) #name,
static const char* names[] = {
FOR_EACH_NURSERY_PROFILE_TIME(EXTRACT_NAME)
#undef EXTRACT_NAME
"" };
size_t i = 0;
for (auto time : profileDurations_)
json.property(names[i++], time, json.MICROSECONDS);
json.endObject(); // timings value
json.endObject();
}
/* static */ void
js::Nursery::printProfileHeader()
{
fprintf(stderr, "MinorGC: Reason PRate Size ");
#define PRINT_HEADER(name, text) \
fprintf(stderr, " %6s", text);
FOR_EACH_NURSERY_PROFILE_TIME(PRINT_HEADER)
@ -455,11 +502,18 @@ void
js::Nursery::printTotalProfileTimes()
{
if (enableProfiling_) {
fprintf(stderr, "MinorGC TOTALS: %7" PRIu64 " collections: ", minorGcCount_);
printProfileTimes(totalTimes_);
fprintf(stderr, "MinorGC TOTALS: %7" PRIu64 " collections: ", minorGcCount_);
printProfileDurations(totalDurations_);
}
}
void
js::Nursery::maybeClearProfileDurations()
{
for (auto& duration : profileDurations_)
duration = mozilla::TimeDuration();
}
inline void
js::Nursery::startProfile(ProfileKey key)
{
@ -473,20 +527,6 @@ js::Nursery::endProfile(ProfileKey key)
totalTimes_[key] += profileTimes_[key];
}
inline void
js::Nursery::maybeStartProfile(ProfileKey key)
{
if (enableProfiling_)
startProfile(key);
}
inline void
js::Nursery::maybeEndProfile(ProfileKey key)
{
if (enableProfiling_)
endProfile(key);
}
void
js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason)
{
@ -520,38 +560,46 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason)
TenureCountCache tenureCounts;
double promotionRate = 0;
previousGC.reason = JS::gcreason::NO_REASON;
if (!isEmpty())
promotionRate = doCollection(rt, reason, tenureCounts);
// Resize the nursery.
maybeStartProfile(ProfileKey::Resize);
startProfile(ProfileKey::Resize);
maybeResizeNursery(reason, promotionRate);
maybeEndProfile(ProfileKey::Resize);
endProfile(ProfileKey::Resize);
// If we are promoting the nursery, or exhausted the store buffer with
// pointers to nursery things, which will force a collection well before
// the nursery is full, look for object groups that are getting promoted
// excessively and try to pretenure them.
maybeStartProfile(ProfileKey::Pretenure);
if (promotionRate > 0.8 || reason == JS::gcreason::FULL_STORE_BUFFER) {
JSContext* cx = rt->contextFromMainThread();
startProfile(ProfileKey::Pretenure);
uint32_t pretenureCount = 0;
if (promotionRate > 0.8 || IsFullStoreBufferReason(reason)) {
JSContext* cx = TlsContext.get();
for (auto& entry : tenureCounts.entries) {
if (entry.count >= 3000) {
ObjectGroup* group = entry.group;
if (group->canPreTenure()) {
AutoCompartment ac(cx, group->compartment());
group->setShouldPreTenure(cx);
pretenureCount++;
}
}
}
}
maybeEndProfile(ProfileKey::Pretenure);
endProfile(ProfileKey::Pretenure);
// We ignore gcMaxBytes when allocating for minor collection. However, if we
// overflowed, we disable the nursery. The next time we allocate, we'll fail
// because gcBytes >= gcMaxBytes.
if (rt->gc.usage.gcBytes() >= rt->gc.tunables.gcMaxBytes())
disable();
// Disable the nursery if the user changed the configuration setting. The
// nursery can only be re-enabled by resetting the configurationa and
// restarting firefox.
if (maxNurseryChunks_ == 0)
disable();
endProfile(ProfileKey::Total);
minorGcCount_++;
@ -568,7 +616,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason)
printProfileHeader();
}
fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ",
fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ",
JS::gcreason::ExplainReason(reason),
promotionRate * 100,
numChunks());
@ -603,87 +651,96 @@ js::Nursery::doCollection(JSRuntime* rt, JS::gcreason::Reason reason,
StoreBuffer& sb = rt->gc.storeBuffer;
// The MIR graph only contains nursery pointers if cancelIonCompilations()
// is set on the store buffer, in which case we cancel all compilations.
maybeStartProfile(ProfileKey::CancelIonCompilations);
// is set on the store buffer, in which case we cancel all compilations
// of such graphs.
startProfile(ProfileKey::CancelIonCompilations);
if (sb.cancelIonCompilations())
js::CancelOffThreadIonCompile(rt);
maybeEndProfile(ProfileKey::CancelIonCompilations);
js::CancelOffThreadIonCompilesUsingNurseryPointers(rt);
endProfile(ProfileKey::CancelIonCompilations);
maybeStartProfile(ProfileKey::TraceValues);
startProfile(ProfileKey::TraceValues);
sb.traceValues(mover);
maybeEndProfile(ProfileKey::TraceValues);
endProfile(ProfileKey::TraceValues);
maybeStartProfile(ProfileKey::TraceCells);
startProfile(ProfileKey::TraceCells);
sb.traceCells(mover);
maybeEndProfile(ProfileKey::TraceCells);
endProfile(ProfileKey::TraceCells);
maybeStartProfile(ProfileKey::TraceSlots);
startProfile(ProfileKey::TraceSlots);
sb.traceSlots(mover);
maybeEndProfile(ProfileKey::TraceSlots);
endProfile(ProfileKey::TraceSlots);
maybeStartProfile(ProfileKey::TraceWholeCells);
startProfile(ProfileKey::TraceWholeCells);
sb.traceWholeCells(mover);
maybeEndProfile(ProfileKey::TraceWholeCells);
endProfile(ProfileKey::TraceWholeCells);
maybeStartProfile(ProfileKey::TraceGenericEntries);
startProfile(ProfileKey::TraceGenericEntries);
sb.traceGenericEntries(&mover);
maybeEndProfile(ProfileKey::TraceGenericEntries);
endProfile(ProfileKey::TraceGenericEntries);
maybeStartProfile(ProfileKey::MarkRuntime);
startProfile(ProfileKey::MarkRuntime);
rt->gc.traceRuntimeForMinorGC(&mover, session.lock);
maybeEndProfile(ProfileKey::MarkRuntime);
endProfile(ProfileKey::MarkRuntime);
maybeStartProfile(ProfileKey::MarkDebugger);
startProfile(ProfileKey::MarkDebugger);
{
gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_MARK_ROOTS);
Debugger::markAll(&mover);
}
maybeEndProfile(ProfileKey::MarkDebugger);
endProfile(ProfileKey::MarkDebugger);
maybeStartProfile(ProfileKey::ClearNewObjectCache);
rt->contextFromMainThread()->caches.newObjectCache.clearNurseryObjects(rt);
maybeEndProfile(ProfileKey::ClearNewObjectCache);
startProfile(ProfileKey::ClearNewObjectCache);
rt->caches().newObjectCache.clearNurseryObjects(rt);
endProfile(ProfileKey::ClearNewObjectCache);
// Most of the work is done here. This loop iterates over objects that have
// been moved to the major heap. If these objects have any outgoing pointers
// to the nursery, then those nursery objects get moved as well, until no
// objects are left to move. That is, we iterate to a fixed point.
maybeStartProfile(ProfileKey::CollectToFP);
startProfile(ProfileKey::CollectToFP);
collectToFixedPoint(mover, tenureCounts);
maybeEndProfile(ProfileKey::CollectToFP);
endProfile(ProfileKey::CollectToFP);
// Sweep compartments to update the array buffer object's view lists.
maybeStartProfile(ProfileKey::SweepArrayBufferViewList);
startProfile(ProfileKey::SweepArrayBufferViewList);
for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next())
c->sweepAfterMinorGC(&mover);
maybeEndProfile(ProfileKey::SweepArrayBufferViewList);
endProfile(ProfileKey::SweepArrayBufferViewList);
// Update any slot or element pointers whose destination has been tenured.
maybeStartProfile(ProfileKey::UpdateJitActivations);
startProfile(ProfileKey::UpdateJitActivations);
js::jit::UpdateJitActivationsForMinorGC(rt, &mover);
forwardedBuffers.finish();
maybeEndProfile(ProfileKey::UpdateJitActivations);
endProfile(ProfileKey::UpdateJitActivations);
maybeStartProfile(ProfileKey::ObjectsTenuredCallback);
startProfile(ProfileKey::ObjectsTenuredCallback);
rt->gc.callObjectsTenuredCallback();
maybeEndProfile(ProfileKey::ObjectsTenuredCallback);
endProfile(ProfileKey::ObjectsTenuredCallback);
// Sweep.
maybeStartProfile(ProfileKey::FreeMallocedBuffers);
startProfile(ProfileKey::FreeMallocedBuffers);
freeMallocedBuffers();
maybeEndProfile(ProfileKey::FreeMallocedBuffers);
endProfile(ProfileKey::FreeMallocedBuffers);
maybeStartProfile(ProfileKey::Sweep);
startProfile(ProfileKey::Sweep);
sweep();
maybeEndProfile(ProfileKey::Sweep);
endProfile(ProfileKey::Sweep);
maybeStartProfile(ProfileKey::ClearStoreBuffer);
rt->gc.storeBuffer.clear();
maybeEndProfile(ProfileKey::ClearStoreBuffer);
startProfile(ProfileKey::ClearStoreBuffer);
runtime()->gc.storeBuffer().clear();
endProfile(ProfileKey::ClearStoreBuffer);
// Make sure hashtables have been updated after the collection.
maybeStartProfile(ProfileKey::CheckHashTables);
maybeEndProfile(ProfileKey::CheckHashTables);
startProfile(ProfileKey::CheckHashTables);
#ifdef JS_GC_ZEAL
if (rt->hasZealMode(ZealMode::CheckHashTablesOnMinorGC))
CheckHashTablesAfterMovingGC(rt);
#endif
endProfile(ProfileKey::CheckHashTables);
previousGC.reason = reason;
previousGC.nurseryUsedBytes = initialNurserySize;
previousGC.tenuredBytes = mover.tenuredSize;
// Calculate and return the promotion rate.
return mover.tenuredSize / double(initialNurserySize);
@ -743,8 +800,8 @@ void
js::Nursery::sweep()
{
/* Sweep unique id's in all in-use chunks. */
for (CellsWithUniqueIdSet::Enum e(cellsWithUid_); !e.empty(); e.popFront()) {
JSObject* obj = static_cast<JSObject*>(e.front());
for (Cell* cell : cellsWithUid_) {
JSObject* obj = static_cast<JSObject*>(cell);
if (!IsForwarded(obj))
obj->zone()->removeUniqueId(obj);
else
@ -752,9 +809,13 @@ js::Nursery::sweep()
}
cellsWithUid_.clear();
runSweepActions();
sweepDictionaryModeObjects();
#ifdef JS_GC_ZEAL
/* Poison the nursery contents so touching a freed object will crash. */
for (unsigned i = 0; i < numChunks(); i++)
chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN);
{
#ifdef JS_CRASH_DIAGNOSTICS
for (unsigned i = 0; i < numChunks(); ++i)
@ -807,6 +868,7 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason, double promotionRat
{
static const double GrowThreshold = 0.05;
static const double ShrinkThreshold = 0.01;
unsigned newMaxNurseryChunks;
// Shrink the nursery to its minimum size of we ran out of memory or
// received a memory pressure event.
@ -815,10 +877,30 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason, double promotionRat
return;
}
#ifdef JS_GC_ZEAL
// This zeal mode disabled nursery resizing.
if (runtime()->hasZealMode(ZealMode::GenerationalGC))
return;
#endif
newMaxNurseryChunks = runtime()->gc.tunables.gcMaxNurseryBytes() >> ChunkShift;
if (newMaxNurseryChunks != maxNurseryChunks_) {
maxNurseryChunks_ = newMaxNurseryChunks;
/* The configured maximum nursery size is changing */
int extraChunks = numChunks() - newMaxNurseryChunks;
if (extraChunks > 0) {
/* We need to shrink the nursery */
shrinkAllocableSpace(extraChunks);
previousPromotionRate_ = promotionRate;
return;
}
}
if (promotionRate > GrowThreshold)
growAllocableSpace();
else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold)
shrinkAllocableSpace();
shrinkAllocableSpace(1);
previousPromotionRate_ = promotionRate;
}
@ -830,9 +912,13 @@ js::Nursery::growAllocableSpace()
}
void
js::Nursery::shrinkAllocableSpace()
js::Nursery::shrinkAllocableSpace(unsigned removeNumChunks)
{
updateNumChunks(Max(numChunks() - 1, 1u));
#ifdef JS_GC_ZEAL
if (runtime()->hasZealMode(ZealMode::GenerationalGC))
return;
#endif
updateNumChunks(Max(numChunks() - removeNumChunks, 1u));
}
void

34
js/src/gc/Nursery.h
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sw=4 et tw=78:
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
@ -198,7 +199,7 @@ class Nursery
* sets |*ref| to the new location of the object and returns true. Otherwise
* returns false and leaves |*ref| unset.
*/
MOZ_ALWAYS_INLINE MOZ_MUST_USE bool getForwardedPointer(JSObject** ref) const;
MOZ_ALWAYS_INLINE [[nodiscard]] static bool getForwardedPointer(JSObject** ref);
/* Forward a slots/elements pointer stored in an Ion frame. */
void forwardBufferPointer(HeapSlot** pSlotsElems);
@ -244,7 +245,6 @@ class Nursery
// Free space remaining, not counting chunk trailers.
MOZ_ALWAYS_INLINE size_t freeSpace() const {
MOZ_ASSERT(isEnabled());
MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize);
return (currentEnd_ - position_) +
(numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize;
@ -253,6 +253,17 @@ class Nursery
/* Print total profile times on shutdown. */
void printTotalProfileTimes();
void* addressOfCurrentEnd() const { return (void*)&currentEnd_; }
void* addressOfPosition() const { return (void*)&position_; }
void requestMinorGC(JS::gcreason::Reason reason) const;
bool minorGCRequested() const { return minorGCTriggerReason_ != JS::gcreason::NO_REASON; }
JS::gcreason::Reason minorGCTriggerReason() const { return minorGCTriggerReason_; }
void clearMinorGCRequest() { minorGCTriggerReason_ = JS::gcreason::NO_REASON; }
bool enableProfiling() const { return enableProfiling_; }
private:
/* The amount of space in the mapped nursery available to allocations. */
static const size_t NurseryChunkUsableSize = gc::ChunkSize - sizeof(gc::ChunkTrailer);
@ -324,12 +335,18 @@ class Nursery
ProfileTimes totalTimes_;
uint64_t minorGcCount_;
struct {
JS::gcreason::Reason reason;
uint64_t nurseryUsedBytes;
uint64_t tenuredBytes;
} previousGC;
/*
* The set of externally malloced buffers potentially kept live by objects
* stored in the nursery. Any external buffers that do not belong to a
* tenured thing at the end of a minor GC must be freed.
*/
typedef HashSet<void*, PointerHasher<void*, 3>, SystemAllocPolicy> MallocedBuffersSet;
typedef HashSet<void*, PointerHasher<void*>, SystemAllocPolicy> MallocedBuffersSet;
MallocedBuffersSet mallocedBuffers;
/* A task structure used to free the malloced bufers on a background thread. */
@ -343,7 +360,7 @@ class Nursery
* buffers might overlap each other. For these, an entry in the following
* table is used.
*/
typedef HashMap<void*, void*, PointerHasher<void*, 1>, SystemAllocPolicy> ForwardedBufferMap;
typedef HashMap<void*, void*, PointerHasher<void*>, SystemAllocPolicy> ForwardedBufferMap;
ForwardedBufferMap forwardedBuffers;
/*
@ -421,7 +438,7 @@ class Nursery
void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots);
void setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
uint32_t nelems);
uint32_t capacity);
/* Free malloced pointers owned by freed things in the nursery. */
void freeMallocedBuffers();
@ -438,16 +455,13 @@ class Nursery
/* Change the allocable space provided by the nursery. */
void maybeResizeNursery(JS::gcreason::Reason reason, double promotionRate);
void growAllocableSpace();
void shrinkAllocableSpace();
void shrinkAllocableSpace(unsigned removeNumChunks);
void minimizeAllocableSpace();
/* Profile recording and printing. */
void startProfile(ProfileKey key);
void endProfile(ProfileKey key);
void maybeStartProfile(ProfileKey key);
void maybeEndProfile(ProfileKey key);
static void printProfileHeader();
static void printProfileTimes(const ProfileTimes& times);
static void printProfileDurations(const ProfileDurations& times);
friend class TenuringTracer;
friend class gc::MinorCollectionTracer;

1
js/src/gc/Policy.h
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

22
js/src/gc/RootMarking.cpp
View file

@ -359,9 +359,6 @@ js::gc::GCRuntime::traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrM
for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next())
c->traceRoots(trc, traceOrMark);
// Trace SPS.
rt->spsProfiler.trace(trc);
// Trace helper thread roots.
HelperThreadState().trace(trc);
@ -442,7 +439,16 @@ class BufferGrayRootsTracer : public JS::CallbackTracer
// Set to false if we OOM while buffering gray roots.
bool bufferingGrayRootsFailed;
void onChild(const JS::GCCellPtr& thing) override;
void onObjectEdge(JSObject** objp) override { bufferRoot(*objp); }
void onStringEdge(JSString** stringp) override { bufferRoot(*stringp); }
void onScriptEdge(JSScript** scriptp) override { bufferRoot(*scriptp); }
void onSymbolEdge(JS::Symbol** symbolp) override { bufferRoot(*symbolp); }
void onChild(const JS::GCCellPtr& thing) override {
MOZ_CRASH("Unexpected gray root kind");
}
template <typename T> inline void bufferRoot(T* thing);
public:
explicit BufferGrayRootsTracer(JSRuntime* rt)
@ -476,8 +482,6 @@ js::gc::GCRuntime::bufferGrayRoots()
for (GCZonesIter zone(rt); !zone.done(); zone.next())
MOZ_ASSERT(zone->gcGrayRoots.empty());
gcstats::AutoPhase ap(stats, gcstats::PHASE_BUFFER_GRAY_ROOTS);
BufferGrayRootsTracer grayBufferer(rt);
if (JSTraceDataOp op = grayRootTracer.op)
(*op)(&grayBufferer, grayRootTracer.data);
@ -508,8 +512,10 @@ BufferGrayRootsTracer::onChild(const JS::GCCellPtr& thing)
gc::TenuredCell* tenured = gc::TenuredCell::fromPointer(thing.asCell());
Zone* zone = tenured->zone();
if (zone->isCollecting()) {
// This is run from a helper thread while the mutator is paused so we have
// to use *FromAnyThread methods here.
Zone* zone = tenured->zoneFromAnyThread();
if (zone->isCollectingFromAnyThread()) {
// See the comment on SetMaybeAliveFlag to see why we only do this for
// objects and scripts. We rely on gray root buffering for this to work,
// but we only need to worry about uncollected dead compartments during

11
js/src/gc/StoreBuffer.h
View file

@ -7,6 +7,7 @@
#define gc_StoreBuffer_h
#include "mozilla/Attributes.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/ReentrancyGuard.h"
#include <algorithm>
@ -39,7 +40,7 @@ class BufferableRef
bool maybeInRememberedSet(const Nursery&) const { return true; }
};
typedef HashSet<void*, PointerHasher<void*, 3>, SystemAllocPolicy> EdgeSet;
typedef HashSet<void*, PointerHasher<void*>, SystemAllocPolicy> EdgeSet;
/* The size of a single block of store buffer storage space. */
static const size_t LifoAllocBlockSize = 1 << 13; /* 8KiB */
@ -206,7 +207,7 @@ class StoreBuffer
struct PointerEdgeHasher
{
typedef Edge Lookup;
static HashNumber hash(const Lookup& l) { return uintptr_t(l.edge) >> 3; }
static HashNumber hash(const Lookup& l) { return mozilla::HashGeneric(l.edge); }
static bool match(const Edge& k, const Lookup& l) { return k == l; }
};
@ -334,9 +335,13 @@ class StoreBuffer
typedef struct Hasher {
typedef SlotsEdge Lookup;
static HashNumber hash(const Lookup& l) { return l.objectAndKind_ ^ l.start_ ^ l.count_; }
static HashNumber hash(const Lookup& l) {
return mozilla::HashGeneric(l.objectAndKind_, l.start_, l.count_);
}
static bool match(const SlotsEdge& k, const Lookup& l) { return k == l; }
} Hasher;
static const auto FullBufferReason = JS::gcreason::FULL_SLOT_BUFFER;
};
template <typename Buffer, typename Edge>

488
js/src/gc/Verifier.cpp
View file

@ -27,7 +27,427 @@ using namespace js::gc;
#ifdef JSGC_HASH_TABLE_CHECKS
class CheckHeapTracer : public JS::CallbackTracer
/*
* Write barrier verification
*
* The next few functions are for write barrier verification.
*
* The VerifyBarriers function is a shorthand. It checks if a verification phase
* is currently running. If not, it starts one. Otherwise, it ends the current
* phase and starts a new one.
*
* The user can adjust the frequency of verifications, which causes
* VerifyBarriers to be a no-op all but one out of N calls. However, if the
* |always| parameter is true, it starts a new phase no matter what.
*
* Pre-Barrier Verifier:
* When StartVerifyBarriers is called, a snapshot is taken of all objects in
* the GC heap and saved in an explicit graph data structure. Later,
* EndVerifyBarriers traverses the heap again. Any pointer values that were in
* the snapshot and are no longer found must be marked; otherwise an assertion
* triggers. Note that we must not GC in between starting and finishing a
* verification phase.
*/
struct EdgeValue
{
void* thing;
JS::TraceKind kind;
const char* label;
};
struct VerifyNode
{
void* thing;
JS::TraceKind kind;
uint32_t count;
EdgeValue edges[1];
};
typedef HashMap<void*, VerifyNode*, DefaultHasher<void*>, SystemAllocPolicy> NodeMap;
/*
* The verifier data structures are simple. The entire graph is stored in a
* single block of memory. At the beginning is a VerifyNode for the root
* node. It is followed by a sequence of EdgeValues--the exact number is given
* in the node. After the edges come more nodes and their edges.
*
* The edgeptr and term fields are used to allocate out of the block of memory
* for the graph. If we run out of memory (i.e., if edgeptr goes beyond term),
* we just abandon the verification.
*
* The nodemap field is a hashtable that maps from the address of the GC thing
* to the VerifyNode that represents it.
*/
class js::VerifyPreTracer final : public JS::CallbackTracer
{
JS::AutoDisableGenerationalGC noggc;
void onChild(const JS::GCCellPtr& thing) override;
public:
/* The gcNumber when the verification began. */
uint64_t number;
/* This counts up to gcZealFrequency to decide whether to verify. */
int count;
/* This graph represents the initial GC "snapshot". */
VerifyNode* curnode;
VerifyNode* root;
char* edgeptr;
char* term;
NodeMap nodemap;
explicit VerifyPreTracer(JSRuntime* rt)
: JS::CallbackTracer(rt), noggc(TlsContext.get()), number(rt->gc.gcNumber()),
count(0), curnode(nullptr), root(nullptr), edgeptr(nullptr), term(nullptr)
{}
~VerifyPreTracer() {
js_free(root);
}
};
/*
* This function builds up the heap snapshot by adding edges to the current
* node.
*/
void
VerifyPreTracer::onChild(const JS::GCCellPtr& thing)
{
MOZ_ASSERT(!IsInsideNursery(thing.asCell()));
// Skip things in other runtimes.
if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime())
return;
edgeptr += sizeof(EdgeValue);
if (edgeptr >= term) {
edgeptr = term;
return;
}
VerifyNode* node = curnode;
uint32_t i = node->count;
node->edges[i].thing = thing.asCell();
node->edges[i].kind = thing.kind();
node->edges[i].label = contextName();
node->count++;
}
static VerifyNode*
MakeNode(VerifyPreTracer* trc, void* thing, JS::TraceKind kind)
{
NodeMap::AddPtr p = trc->nodemap.lookupForAdd(thing);
if (!p) {
VerifyNode* node = (VerifyNode*)trc->edgeptr;
trc->edgeptr += sizeof(VerifyNode) - sizeof(EdgeValue);
if (trc->edgeptr >= trc->term) {
trc->edgeptr = trc->term;
return nullptr;
}
node->thing = thing;
node->count = 0;
node->kind = kind;
if (!trc->nodemap.add(p, thing, node)) {
trc->edgeptr = trc->term;
return nullptr;
}
return node;
}
return nullptr;
}
static VerifyNode*
NextNode(VerifyNode* node)
{
if (node->count == 0)
return (VerifyNode*)((char*)node + sizeof(VerifyNode) - sizeof(EdgeValue));
else
return (VerifyNode*)((char*)node + sizeof(VerifyNode) +
sizeof(EdgeValue)*(node->count - 1));
}
void
gc::GCRuntime::startVerifyPreBarriers()
{
if (verifyPreData || isIncrementalGCInProgress())
return;
if (IsIncrementalGCUnsafe(rt) != AbortReason::None ||
TlsContext.get()->keepAtoms ||
rt->hasHelperThreadZones() ||
rt->cooperatingContexts().length() != 1)
{
return;
}
number++;
VerifyPreTracer* trc = js_new<VerifyPreTracer>(rt);
if (!trc)
return;
AutoPrepareForTracing prep(TlsContext.get(), WithAtoms);
for (auto chunk = allNonEmptyChunks(); !chunk.done(); chunk.next())
chunk->bitmap.clear();
gcstats::AutoPhase ap(stats(), gcstats::PHASE_TRACE_HEAP);
const size_t size = 64 * 1024 * 1024;
trc->root = (VerifyNode*)js_malloc(size);
if (!trc->root)
goto oom;
trc->edgeptr = (char*)trc->root;
trc->term = trc->edgeptr + size;
if (!trc->nodemap.init())
goto oom;
/* Create the root node. */
trc->curnode = MakeNode(trc, nullptr, JS::TraceKind(0));
incrementalState = State::MarkRoots;
/* Make all the roots be edges emanating from the root node. */
traceRuntime(trc, prep.session().lock);
VerifyNode* node;
node = trc->curnode;
if (trc->edgeptr == trc->term)
goto oom;
/* For each edge, make a node for it if one doesn't already exist. */
while ((char*)node < trc->edgeptr) {
for (uint32_t i = 0; i < node->count; i++) {
EdgeValue& e = node->edges[i];
VerifyNode* child = MakeNode(trc, e.thing, e.kind);
if (child) {
trc->curnode = child;
js::TraceChildren(trc, e.thing, e.kind);
}
if (trc->edgeptr == trc->term)
goto oom;
}
node = NextNode(node);
}
verifyPreData = trc;
incrementalState = State::Mark;
marker.start();
for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
MOZ_ASSERT(!zone->usedByHelperThread());
zone->setNeedsIncrementalBarrier(true);
zone->arenas.purge();
}
return;
oom:
incrementalState = State::NotActive;
js_delete(trc);
verifyPreData = nullptr;
}
static bool
IsMarkedOrAllocated(TenuredCell* cell)
{
return cell->isMarkedAny() || cell->arena()->allocatedDuringIncremental;
}
struct CheckEdgeTracer : public JS::CallbackTracer {
VerifyNode* node;
explicit CheckEdgeTracer(JSRuntime* rt) : JS::CallbackTracer(rt), node(nullptr) {}
void onChild(const JS::GCCellPtr& thing) override;
};
static const uint32_t MAX_VERIFIER_EDGES = 1000;
/*
* This function is called by EndVerifyBarriers for every heap edge. If the edge
* already existed in the original snapshot, we "cancel it out" by overwriting
* it with nullptr. EndVerifyBarriers later asserts that the remaining
* non-nullptr edges (i.e., the ones from the original snapshot that must have
* been modified) must point to marked objects.
*/
void
CheckEdgeTracer::onChild(const JS::GCCellPtr& thing)
{
// Skip things in other runtimes.
if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime())
return;
/* Avoid n^2 behavior. */
if (node->count > MAX_VERIFIER_EDGES)
return;
for (uint32_t i = 0; i < node->count; i++) {
if (node->edges[i].thing == thing.asCell()) {
MOZ_ASSERT(node->edges[i].kind == thing.kind());
node->edges[i].thing = nullptr;
return;
}
}
}
void
js::gc::AssertSafeToSkipBarrier(TenuredCell* thing)
{
mozilla::DebugOnly<Zone*> zone = thing->zoneFromAnyThread();
MOZ_ASSERT(!zone->needsIncrementalBarrier() || zone->isAtomsZone());
}
static bool
IsMarkedOrAllocated(const EdgeValue& edge)
{
if (!edge.thing || IsMarkedOrAllocated(TenuredCell::fromPointer(edge.thing)))
return true;
// Permanent atoms and well-known symbols aren't marked during graph traversal.
if (edge.kind == JS::TraceKind::String && static_cast<JSString*>(edge.thing)->isPermanentAtom())
return true;
if (edge.kind == JS::TraceKind::Symbol && static_cast<JS::Symbol*>(edge.thing)->isWellKnownSymbol())
return true;
return false;
}
void
gc::GCRuntime::endVerifyPreBarriers()
{
VerifyPreTracer* trc = verifyPreData;
if (!trc)
return;
MOZ_ASSERT(!JS::IsGenerationalGCEnabled(rt));
AutoPrepareForTracing prep(rt->activeContextFromOwnThread(), SkipAtoms);
bool compartmentCreated = false;
/* We need to disable barriers before tracing, which may invoke barriers. */
for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
if (!zone->needsIncrementalBarrier())
compartmentCreated = true;
zone->setNeedsIncrementalBarrier(false);
}
/*
* We need to bump gcNumber so that the methodjit knows that jitcode has
* been discarded.
*/
MOZ_ASSERT(trc->number == number);
number++;
verifyPreData = nullptr;
incrementalState = State::NotActive;
if (!compartmentCreated &&
IsIncrementalGCUnsafe(rt) == AbortReason::None &&
!TlsContext.get()->keepAtoms &&
!rt->hasHelperThreadZones())
{
CheckEdgeTracer cetrc(rt);
/* Start after the roots. */
VerifyNode* node = NextNode(trc->root);
while ((char*)node < trc->edgeptr) {
cetrc.node = node;
js::TraceChildren(&cetrc, node->thing, node->kind);
if (node->count <= MAX_VERIFIER_EDGES) {
for (uint32_t i = 0; i < node->count; i++) {
EdgeValue& edge = node->edges[i];
if (!IsMarkedOrAllocated(edge)) {
char msgbuf[1024];
SprintfLiteral(msgbuf,
"[barrier verifier] Unmarked edge: %s %p '%s' edge to %s %p",
JS::GCTraceKindToAscii(node->kind), node->thing,
edge.label,
JS::GCTraceKindToAscii(edge.kind), edge.thing);
MOZ_ReportAssertionFailure(msgbuf, __FILE__, __LINE__);
MOZ_CRASH();
}
}
}
node = NextNode(node);
}
}
marker.reset();
marker.stop();
js_delete(trc);
}
/*** Barrier Verifier Scheduling ***/
void
gc::GCRuntime::verifyPreBarriers()
{
if (verifyPreData)
endVerifyPreBarriers();
else
startVerifyPreBarriers();
}
void
gc::VerifyBarriers(JSRuntime* rt, VerifierType type)
{
if (type == PreBarrierVerifier)
rt->gc.verifyPreBarriers();
}
void
gc::GCRuntime::maybeVerifyPreBarriers(bool always)
{
if (!hasZealMode(ZealMode::VerifierPre))
return;
if (TlsContext.get()->suppressGC)
return;
if (verifyPreData) {
if (++verifyPreData->count < zealFrequency && !always)
return;
endVerifyPreBarriers();
}
startVerifyPreBarriers();
}
void
js::gc::MaybeVerifyBarriers(JSContext* cx, bool always)
{
GCRuntime* gc = &cx->runtime()->gc;
gc->maybeVerifyPreBarriers(always);
}
void
js::gc::GCRuntime::finishVerifier()
{
if (verifyPreData) {
js_delete(verifyPreData.ref());
verifyPreData = nullptr;
}
}
#endif /* JS_GC_ZEAL */
#if defined(JSGC_HASH_TABLE_CHECKS) || defined(DEBUG)
class HeapCheckTracerBase : public JS::CallbackTracer
{
public:
explicit CheckHeapTracer(JSRuntime* rt);
@ -152,3 +572,69 @@ js::gc::CheckHeapAfterGC(JSRuntime* rt)
}
#endif /* JSGC_HASH_TABLE_CHECKS */
#ifdef DEBUG
class CheckGrayMarkingTracer final : public HeapCheckTracerBase
{
public:
explicit CheckGrayMarkingTracer(JSRuntime* rt);
bool check(AutoLockForExclusiveAccess& lock);
private:
void checkCell(Cell* cell) override;
};
CheckGrayMarkingTracer::CheckGrayMarkingTracer(JSRuntime* rt)
: HeapCheckTracerBase(rt, DoNotTraceWeakMaps)
{
// Weak gray->black edges are allowed.
setTraceWeakEdges(false);
}
void
CheckGrayMarkingTracer::checkCell(Cell* cell)
{
Cell* parent = parentCell();
if (!cell->isTenured() || !parent || !parent->isTenured())
return;
TenuredCell* tenuredCell = &cell->asTenured();
TenuredCell* tenuredParent = &parent->asTenured();
if (tenuredParent->isMarkedBlack() && tenuredCell->isMarkedGray())
{
failures++;
fprintf(stderr, "Found black to gray edge to %s %p\n",
GCTraceKindToAscii(cell->getTraceKind()), cell);
dumpCellPath();
}
}
bool
CheckGrayMarkingTracer::check(AutoLockForExclusiveAccess& lock)
{
if (!traceHeap(lock))
return true; // Ignore failure.
return failures == 0;
}
JS_FRIEND_API(bool)
js::CheckGrayMarkingState(JSContext* cx)
{
JSRuntime* rt = cx->runtime();
MOZ_ASSERT(!JS::CurrentThreadIsHeapCollecting());
MOZ_ASSERT(!rt->gc.isIncrementalGCInProgress());
if (!rt->gc.areGrayBitsValid())
return true;
gcstats::AutoPhase ap(rt->gc.stats(), gcstats::PHASE_TRACE_HEAP);
AutoTraceSession session(rt, JS::HeapState::Tracing);
CheckGrayMarkingTracer tracer(rt);
if (!tracer.init())
return true; // Ignore failure
return tracer.check(session.lock);
}
#endif // DEBUG

93
js/src/gc/Zone.cpp
View file

@ -1,4 +1,5 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -28,23 +29,31 @@ JS::Zone::Zone(JSRuntime* rt)
suppressAllocationMetadataBuilder(false),
arenas(rt),
types(this),
compartments(),
gcGrayRoots(),
gcWeakKeys(SystemAllocPolicy(), rt->randomHashCodeScrambler()),
typeDescrObjects(this, SystemAllocPolicy()),
gcMallocBytes(0),
gcMallocGCTriggered(false),
gcWeakMapList_(group),
compartments_(),
gcGrayRoots_(group),
gcWeakRefs_(group),
weakCaches_(group),
gcWeakKeys_(group, SystemAllocPolicy(), rt->randomHashCodeScrambler()),
gcSweepGroupEdges_(group),
typeDescrObjects_(group, this),
regExps(this),
markedAtoms_(group),
atomCache_(group),
externalStringCache_(group),
functionToStringCache_(group),
usage(&rt->gc.usage),
gcDelayBytes(0),
propertyTree(this),
baseShapes(this, BaseShapeSet()),
initialShapes(this, InitialShapeSet()),
data(nullptr),
isSystem(false),
usedByExclusiveThread(false),
active(false),
jitZone_(nullptr),
gcState_(NoGC),
propertyTree_(group, this),
baseShapes_(group, this),
initialShapes_(group, this),
nurseryShapes_(group),
data(group, nullptr),
isSystem(group, false),
#ifdef DEBUG
gcLastSweepGroupIndex(group, 0),
#endif
jitZone_(group, nullptr),
gcScheduled_(false),
gcPreserveCode_(false),
jitUsingBarriers_(false),
@ -58,6 +67,7 @@ JS::Zone::Zone(JSRuntime* rt)
AutoLockGC lock(rt);
threshold.updateAfterGC(8192, GC_NORMAL, rt->gc.tunables, rt->gc.schedulingState, lock);
setGCMaxMallocBytes(rt->gc.maxMallocBytesAllocated() * 0.9);
jitCodeCounter.setMax(jit::MaxCodeBytesPerProcess * 0.8);
}
Zone::~Zone()
@ -79,10 +89,13 @@ Zone::~Zone()
bool Zone::init(bool isSystemArg)
{
isSystem = isSystemArg;
return uniqueIds_.init() &&
gcZoneGroupEdges.init() &&
gcWeakKeys.init() &&
typeDescrObjects.init();
return uniqueIds().init() &&
gcSweepGroupEdges().init() &&
gcWeakKeys().init() &&
typeDescrObjects().init() &&
markedAtoms().init() &&
atomCache().init() &&
regExps.init();
}
void
@ -179,13 +192,13 @@ Zone::sweepBreakpoints(FreeOp* fop)
GCPtrNativeObject& dbgobj = bp->debugger->toJSObjectRef();
// If we are sweeping, then we expect the script and the
// debugger object to be swept in the same zone group, except if
// the breakpoint was added after we computed the zone
// debugger object to be swept in the same sweep group, except
// if the breakpoint was added after we computed the sweep
// groups. In this case both script and debugger object must be
// live.
MOZ_ASSERT_IF(isGCSweeping() && dbgobj->zone()->isCollecting(),
dbgobj->zone()->isGCSweeping() ||
(!scriptGone && dbgobj->asTenured().isMarked()));
(!scriptGone && dbgobj->asTenured().isMarkedAny()));
bool dying = scriptGone || IsAboutToBeFinalized(&dbgobj);
MOZ_ASSERT_IF(!dying, !IsAboutToBeFinalized(&bp->getHandlerRef()));
@ -230,12 +243,13 @@ Zone::discardJitCode(FreeOp* fop, bool discardBaselineCode)
for (auto script = cellIter<JSScript>(); !script.done(); script.next()) {
jit::FinishInvalidation(fop, script);
/*
* Discard baseline script if it's not marked as active. Note that
* this also resets the active flag.
*/
if (discardBaselineCode)
jit::FinishDiscardBaselineScript(fop, script);
/*
* Make it impossible to use the control flow graphs cached on the
* BaselineScript. They get deleted.
*/
if (script->hasBaselineScript())
script->baselineScript()->setControlFlowGraph(nullptr);
}
/*
* Warm-up counter for scripts are reset on GC. After discarding code we
@ -375,15 +389,30 @@ Zone::addTypeDescrObject(JSContext* cx, HandleObject obj)
// Type descriptor objects are always tenured so we don't need post barriers
// on the set.
MOZ_ASSERT(!IsInsideNursery(obj));
if (!typeDescrObjects.put(obj)) {
if (!typeDescrObjects().put(obj)) {
ReportOutOfMemory(cx);
return false;
}
return true;
}
void
Zone::deleteEmptyCompartment(JSCompartment* comp)
{
MOZ_ASSERT(comp->zone() == this);
MOZ_ASSERT(arenas.checkEmptyArenaLists());
for (auto& i : compartments()) {
if (i == comp) {
compartments().erase(&i);
comp->destroy(runtimeFromActiveCooperatingThread()->defaultFreeOp());
return;
}
}
MOZ_CRASH("Compartment not found");
}
ZoneList::ZoneList()
: head(nullptr), tail(nullptr)
{}

124
js/src/gc/Zone.h
View file

@ -7,6 +7,7 @@
#define gc_Zone_h
#include "mozilla/Atomics.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/MemoryReporting.h"
#include "jscntxt.h"
@ -77,7 +78,7 @@ struct UniqueIdGCPolicy {
// Maps a Cell* to a unique, 64bit id.
using UniqueIdMap = GCHashMap<Cell*,
uint64_t,
PointerHasher<Cell*, 3>,
PointerHasher<Cell*>,
SystemAllocPolicy,
UniqueIdGCPolicy>;
@ -193,29 +194,16 @@ struct Zone : public JS::shadow::Zone,
bool canCollect();
void notifyObservingDebuggers();
enum GCState {
NoGC,
Mark,
MarkGray,
Sweep,
Finished,
Compact
};
void setGCState(GCState state) {
MOZ_ASSERT(runtimeFromMainThread()->isHeapBusy());
MOZ_ASSERT_IF(state != NoGC, canCollect());
gcState_ = state;
if (state == Finished)
notifyObservingDebuggers();
void changeGCState(GCState prev, GCState next) {
MOZ_ASSERT(CurrentThreadIsHeapBusy());
MOZ_ASSERT(gcState() == prev);
MOZ_ASSERT_IF(next != NoGC, canCollect());
gcState_ = next;
}
bool isCollecting() const {
if (runtimeFromMainThread()->isHeapCollecting())
return gcState_ != NoGC;
else
return needsIncrementalBarrier();
MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromActiveCooperatingThread()));
return isCollectingFromAnyThread();
}
bool isCollectingFromAnyThread() const {
@ -263,9 +251,9 @@ struct Zone : public JS::shadow::Zone,
void prepareForCompacting();
#ifdef DEBUG
// For testing purposes, return the index of the zone group which this zone
// For testing purposes, return the index of the sweep group which this zone
// was swept in in the last GC.
unsigned lastZoneGroupIndex() { return gcLastZoneGroupIndex; }
unsigned lastSweepGroupIndex() { return gcLastSweepGroupIndex; }
#endif
using DebuggerVector = js::Vector<js::Debugger*, 0, js::SystemAllocPolicy>;
@ -292,6 +280,8 @@ struct Zone : public JS::shadow::Zone,
DebuggerVector* getDebuggers() const { return debuggers; }
DebuggerVector* getOrCreateDebuggers(JSContext* cx);
void notifyObservingDebuggers();
void clearTables();
/*
@ -318,17 +308,23 @@ struct Zone : public JS::shadow::Zone,
// This zone's gray roots.
typedef js::Vector<js::gc::Cell*, 0, js::SystemAllocPolicy> GrayRootVector;
GrayRootVector gcGrayRoots;
private:
js::ZoneGroupOrGCTaskData<GrayRootVector> gcGrayRoots_;
public:
GrayRootVector& gcGrayRoots() { return gcGrayRoots_.ref(); }
// This zone's weak edges found via graph traversal during marking,
// preserved for re-scanning during sweeping.
using WeakEdges = js::Vector<js::gc::TenuredCell**, 0, js::SystemAllocPolicy>;
WeakEdges gcWeakRefs;
// List of non-ephemeron weak containers to sweep during beginSweepingZoneGroup.
mozilla::LinkedList<WeakCache<void*>> weakCaches_;
void registerWeakCache(WeakCache<void*>* cachep) {
weakCaches_.insertBack(cachep);
private:
// List of non-ephemeron weak containers to sweep during beginSweepingSweepGroup.
js::ZoneGroupOrGCTaskData<mozilla::LinkedList<detail::WeakCacheBase>> weakCaches_;
public:
mozilla::LinkedList<detail::WeakCacheBase>& weakCaches() { return weakCaches_.ref(); }
void registerWeakCache(detail::WeakCacheBase* cachep) {
weakCaches().insertBack(cachep);
}
/*
@ -339,9 +335,12 @@ struct Zone : public JS::shadow::Zone,
// A set of edges from this zone to other zones.
//
// This is used during GC while calculating zone groups to record edges that
// can't be determined by examining this zone by itself.
ZoneSet gcZoneGroupEdges;
// This is used during GC while calculating sweep groups to record edges
// that can't be determined by examining this zone by itself.
js::ZoneGroupData<ZoneSet> gcSweepGroupEdges_;
public:
ZoneSet& gcSweepGroupEdges() { return gcSweepGroupEdges_.ref(); }
// Keep track of all TypeDescr and related objects in this compartment.
// This is used by the GC to trace them all first when compacting, since the
@ -354,18 +353,52 @@ struct Zone : public JS::shadow::Zone,
using TypeDescrObjectSet = js::GCHashSet<JSObject*,
js::MovableCellHasher<JSObject*>,
js::SystemAllocPolicy>;
JS::WeakCache<TypeDescrObjectSet> typeDescrObjects;
private:
js::ZoneGroupData<JS::WeakCache<TypeDescrObjectSet>> typeDescrObjects_;
// Malloc counter to measure memory pressure for GC scheduling. This
// counter should be used only when it's not possible to know the size of
// a free.
js::gc::MemoryCounter<Zone> gcMallocCounter;
// Counter of JIT code executable memory for GC scheduling. Also imprecise,
// since wasm can generate code that outlives a zone.
js::gc::MemoryCounter<Zone> jitCodeCounter;
public:
js::RegExpZone regExps;
JS::WeakCache<TypeDescrObjectSet>& typeDescrObjects() { return typeDescrObjects_.ref(); }
bool addTypeDescrObject(JSContext* cx, HandleObject obj);
bool triggerGCForTooMuchMalloc() {
JSRuntime* rt = runtimeFromAnyThread();
// Malloc counter to measure memory pressure for GC scheduling. It runs from
// gcMaxMallocBytes down to zero. This counter should be used only when it's
// not possible to know the size of a free.
mozilla::Atomic<ptrdiff_t, mozilla::ReleaseAcquire> gcMallocBytes;
if (CurrentThreadCanAccessRuntime(rt)) {
return rt->gc.triggerZoneGC(this, JS::gcreason::TOO_MUCH_MALLOC,
gcMallocCounter.bytes(), gcMallocCounter.maxBytes());
}
return false;
}
// GC trigger threshold for allocations on the C heap.
size_t gcMaxMallocBytes;
void resetGCMallocBytes() { gcMallocCounter.reset(); }
void setGCMaxMallocBytes(size_t value) { gcMallocCounter.setMax(value); }
void updateMallocCounter(size_t nbytes) { gcMallocCounter.update(this, nbytes); }
size_t GCMaxMallocBytes() const { return gcMallocCounter.maxBytes(); }
size_t GCMallocBytes() const { return gcMallocCounter.bytes(); }
void updateJitCodeMallocBytes(size_t size) { jitCodeCounter.update(this, size); }
// Resets all the memory counters.
void resetAllMallocBytes() {
resetGCMallocBytes();
jitCodeCounter.reset();
}
bool isTooMuchMalloc() const {
return gcMallocCounter.isTooMuchMalloc() ||
jitCodeCounter.isTooMuchMalloc();
}
// Whether a GC has been triggered as a result of gcMallocBytes falling
// below zero.
@ -382,7 +415,7 @@ struct Zone : public JS::shadow::Zone,
// Amount of data to allocate before triggering a new incremental slice for
// the current GC.
size_t gcDelayBytes;
js::UnprotectedData<size_t> gcDelayBytes;
// Shared Shape property tree.
js::PropertyTree propertyTree;
@ -414,11 +447,11 @@ struct Zone : public JS::shadow::Zone,
bool active;
#ifdef DEBUG
unsigned gcLastZoneGroupIndex;
js::ZoneGroupData<unsigned> gcLastSweepGroupIndex;
#endif
static js::HashNumber UniqueIdToHash(uint64_t uid) {
return js::HashNumber(uid >> 32) ^ js::HashNumber(uid & 0xFFFFFFFF);
return mozilla::HashGeneric(uid);
}
// Creates a HashNumber based on getUniqueId. Returns false on OOM.
@ -482,9 +515,9 @@ struct Zone : public JS::shadow::Zone,
void transferUniqueId(js::gc::Cell* tgt, js::gc::Cell* src) {
MOZ_ASSERT(src != tgt);
MOZ_ASSERT(!IsInsideNursery(tgt));
MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromMainThread()));
MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromActiveCooperatingThread()));
MOZ_ASSERT(js::CurrentThreadCanAccessZone(this));
uniqueIds_.rekeyIfMoved(src, tgt);
uniqueIds().rekeyIfMoved(src, tgt);
}
// Remove any unique id associated with this Cell.
@ -521,6 +554,9 @@ struct Zone : public JS::shadow::Zone,
keepShapeTables_ = b;
}
// Delete an empty compartment after its contents have been merged.
void deleteEmptyCompartment(JSCompartment* comp);
private:
js::jit::JitZone* jitZone_;

174
js/src/gc/ZoneGroup.cpp Normal file
View file

@ -0,0 +1,174 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "gc/ZoneGroup.h"
#include "jscntxt.h"
#include "jit/IonBuilder.h"
#include "jit/JitCompartment.h"
using namespace js;
namespace js {
ZoneGroup::ZoneGroup(JSRuntime* runtime)
: runtime(runtime),
ownerContext_(TlsContext.get()),
enterCount(1),
zones_(this),
usedByHelperThread(false),
#ifdef DEBUG
ionBailAfter_(this, 0),
#endif
jitZoneGroup(this, nullptr),
debuggerList_(this),
numFinishedBuilders(0),
ionLazyLinkListSize_(0)
{}
bool
ZoneGroup::init()
{
AutoLockGC lock(runtime);
jitZoneGroup = js_new<jit::JitZoneGroup>(this);
if (!jitZoneGroup)
return false;
return true;
}
ZoneGroup::~ZoneGroup()
{
#ifdef DEBUG
{
AutoLockHelperThreadState lock;
MOZ_ASSERT(ionLazyLinkListSize_ == 0);
MOZ_ASSERT(ionLazyLinkList().isEmpty());
}
#endif
js_delete(jitZoneGroup.ref());
if (this == runtime->gc.systemZoneGroup)
runtime->gc.systemZoneGroup = nullptr;
}
void
ZoneGroup::enter(JSContext* cx)
{
if (ownerContext().context() == cx) {
MOZ_ASSERT(enterCount);
} else {
if (useExclusiveLocking) {
MOZ_ASSERT(!usedByHelperThread);
while (ownerContext().context() != nullptr) {
cx->yieldToEmbedding();
}
}
MOZ_RELEASE_ASSERT(ownerContext().context() == nullptr);
MOZ_ASSERT(enterCount == 0);
ownerContext_ = CooperatingContext(cx);
if (cx->generationalDisabled)
nursery().disable();
// Finish any Ion compilations in this zone group, in case compilation
// finished for some script in this group while no thread was in this
// group.
jit::AttachFinishedCompilations(this, nullptr);
}
enterCount++;
}
void
ZoneGroup::leave()
{
MOZ_ASSERT(ownedByCurrentThread());
MOZ_ASSERT(enterCount);
if (--enterCount == 0)
ownerContext_ = CooperatingContext(nullptr);
}
bool
ZoneGroup::ownedByCurrentThread()
{
MOZ_ASSERT(TlsContext.get());
return ownerContext().context() == TlsContext.get();
}
ZoneGroup::IonBuilderList&
ZoneGroup::ionLazyLinkList()
{
MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime),
"Should only be mutated by the active thread.");
return ionLazyLinkList_.ref();
}
void
ZoneGroup::ionLazyLinkListRemove(jit::IonBuilder* builder)
{
MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime),
"Should only be mutated by the active thread.");
MOZ_ASSERT(this == builder->script()->zone()->group());
MOZ_ASSERT(ionLazyLinkListSize_ > 0);
builder->removeFrom(ionLazyLinkList());
ionLazyLinkListSize_--;
MOZ_ASSERT(ionLazyLinkList().isEmpty() == (ionLazyLinkListSize_ == 0));
}
void
ZoneGroup::ionLazyLinkListAdd(jit::IonBuilder* builder)
{
MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime),
"Should only be mutated by the active thread.");
MOZ_ASSERT(this == builder->script()->zone()->group());
ionLazyLinkList().insertFront(builder);
ionLazyLinkListSize_++;
}
void
ZoneGroup::deleteEmptyZone(Zone* zone)
{
MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime));
MOZ_ASSERT(zone->group() == this);
MOZ_ASSERT(zone->compartments().empty());
for (auto& i : zones()) {
if (i == zone) {
zones().erase(&i);
zone->destroy(runtime->defaultFreeOp());
return;
}
}
MOZ_CRASH("Zone not found");
}
} // namespace js
JS::AutoRelinquishZoneGroups::AutoRelinquishZoneGroups(JSContext* cx)
: cx(cx)
{
MOZ_ASSERT(cx == TlsContext.get());
AutoEnterOOMUnsafeRegion oomUnsafe;
for (ZoneGroupsIter group(cx->runtime()); !group.done(); group.next()) {
while (group->ownerContext().context() == cx) {
group->leave();
if (!enterList.append(group))
oomUnsafe.crash("AutoRelinquishZoneGroups");
}
}
}
JS::AutoRelinquishZoneGroups::~AutoRelinquishZoneGroups()
{
for (size_t i = 0; i < enterList.length(); i++) {
ZoneGroup* group = static_cast<ZoneGroup*>(enterList[i]);
group->enter(cx);
}
}

12
js/src/jit/BaselineJIT.cpp
View file

@ -1242,9 +1242,13 @@ MarkActiveBaselineScripts(JSRuntime* rt, const JitActivationIterator& activation
void
jit::MarkActiveBaselineScripts(Zone* zone)
{
JSRuntime* rt = zone->runtimeFromMainThread();
for (JitActivationIterator iter(rt); !iter.done(); ++iter) {
if (iter->compartment()->zone() == zone)
MarkActiveBaselineScripts(rt, iter);
if (zone->isAtomsZone())
return;
JSContext* cx = TlsContext.get();
for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) {
for (JitActivationIterator iter(cx, target); !iter.done(); ++iter) {
if (iter->compartment()->zone() == zone)
MarkActiveBaselineScripts(cx, iter);
}
}
}

27
js/src/jit/Ion.cpp
View file

@ -3152,10 +3152,15 @@ jit::InvalidateAll(FreeOp* fop, Zone* zone)
MOZ_ASSERT(!HasOffThreadIonCompile(comp));
#endif
for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) {
if (iter->compartment()->zone() == zone) {
JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC");
InvalidateActivation(fop, iter, true);
if (zone->isAtomsZone())
return;
JSContext* cx = TlsContext.get();
for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) {
for (JitActivationIterator iter(cx, target); !iter.done(); ++iter) {
if (iter->compartment()->zone() == zone) {
JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC");
InvalidateActivation(fop, iter, true);
}
}
}
}
@ -3198,8 +3203,18 @@ jit::Invalidate(TypeZone& types, FreeOp* fop,
return;
}
for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter)
InvalidateActivation(fop, iter, false);
// This method can be called both during GC and during the course of normal
// script execution. In the former case this class will already be on the
// stack, and in the latter case the invalidations will all be on the
// current thread's stack, but the assertion under ActivationIterator can't
// tell that this is a thread local use of the iterator.
JSRuntime::AutoProhibitActiveContextChange apacc(fop->runtime());
JSContext* cx = TlsContext.get();
for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) {
for (JitActivationIterator iter(cx, target); !iter.done(); ++iter)
InvalidateActivation(fop, iter, false);
}
// Drop the references added above. If a script was never active, its
// IonScript will be immediately destroyed. Otherwise, it will be held live

2
js/src/jit/LoopUnroller.cpp
View file

@ -17,7 +17,7 @@ namespace {
struct LoopUnroller
{
typedef HashMap<MDefinition*, MDefinition*,
PointerHasher<MDefinition*, 2>, SystemAllocPolicy> DefinitionMap;
PointerHasher<MDefinition*>, SystemAllocPolicy> DefinitionMap;
explicit LoopUnroller(MIRGraph& graph)
: graph(graph), alloc(graph.alloc()),

4
js/src/jit/OptimizationTracking.cpp
View file

@ -211,8 +211,8 @@ static inline HashNumber
HashType(TypeSet::Type ty)
{
if (ty.isObjectUnchecked())
return PointerHasher<TypeSet::ObjectKey*, 3>::hash(ty.objectKey());
return HashNumber(ty.raw());
return PointerHasher<TypeSet::ObjectKey*>::hash(ty.objectKey());
return mozilla::HashGeneric(ty.raw());
}
static HashNumber

10
js/src/jsapi.cpp
View file

@ -1364,16 +1364,16 @@ JS_RemoveFinalizeCallback(JSContext* cx, JSFinalizeCallback cb)
}
JS_PUBLIC_API(bool)
JS_AddWeakPointerZoneGroupCallback(JSContext* cx, JSWeakPointerZoneGroupCallback cb, void* data)
JS_AddWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb, void* data)
{
AssertHeapIsIdle(cx);
return cx->gc.addWeakPointerZoneGroupCallback(cb, data);
AssertHeapIsIdle();
return cx->runtime()->gc.addWeakPointerZonesCallback(cb, data);
}
JS_PUBLIC_API(void)
JS_RemoveWeakPointerZoneGroupCallback(JSContext* cx, JSWeakPointerZoneGroupCallback cb)
JS_RemoveWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb)
{
cx->gc.removeWeakPointerZoneGroupCallback(cb);
cx->runtime()->gc.removeWeakPointerZonesCallback(cb);
}
JS_PUBLIC_API(bool)

51
js/src/jsapi.h
View file

@ -599,7 +599,7 @@ typedef void
(* JSFinalizeCallback)(JSFreeOp* fop, JSFinalizeStatus status, bool isZoneGC, void* data);
typedef void
(* JSWeakPointerZoneGroupCallback)(JSContext* cx, void* data);
(* JSWeakPointerZonesCallback)(JSContext* cx, void* data);
typedef void
(* JSWeakPointerCompartmentCallback)(JSContext* cx, JSCompartment* comp, void* data);
@ -882,7 +882,50 @@ JS_IsBuiltinFunctionConstructor(JSFunction* fun);
extern JS_PUBLIC_API(JSContext*)
JS_NewContext(uint32_t maxbytes,
uint32_t maxNurseryBytes = JS::DefaultNurseryBytes,
JSContext* parentContext = nullptr);
JSRuntime* parentRuntime = nullptr);
// The methods below for controlling the active context in a cooperatively
// multithreaded runtime are not threadsafe, and the caller must ensure they
// are called serially if there is a chance for contention between threads.
// Called from the active context for a runtime, yield execution so that
// this context is no longer active and can no longer use the API.
extern JS_PUBLIC_API(void)
JS_YieldCooperativeContext(JSContext* cx);
// Called from a context whose runtime has no active context, this thread
// becomes the active context for that runtime and may use the API.
extern JS_PUBLIC_API(void)
JS_ResumeCooperativeContext(JSContext* cx);
// Create a new context on this thread for cooperative multithreading in the
// same runtime as siblingContext. Called on a runtime (as indicated by
// siblingContet) which has no active context, on success the new context will
// become the runtime's active context.
extern JS_PUBLIC_API(JSContext*)
JS_NewCooperativeContext(JSContext* siblingContext);
namespace JS {
// Class to relinquish exclusive access to all zone groups in use by this
// thread. This allows other cooperative threads to enter the zone groups
// and modify their contents.
struct AutoRelinquishZoneGroups
{
explicit AutoRelinquishZoneGroups(JSContext* cx);
~AutoRelinquishZoneGroups();
private:
JSContext* cx;
mozilla::Vector<void*> enterList;
};
} // namespace JS
// Destroy a context allocated with JS_NewContext or JS_NewCooperativeContext.
// The context must be the current active context in the runtime, and after
// this call the runtime will have no active context.
extern JS_PUBLIC_API(void)
JS_DestroyContext(JSContext* cx);
@ -1600,10 +1643,10 @@ JS_RemoveFinalizeCallback(JSContext* cx, JSFinalizeCallback cb);
*/
extern JS_PUBLIC_API(bool)
JS_AddWeakPointerZoneGroupCallback(JSContext* cx, JSWeakPointerZoneGroupCallback cb, void* data);
JS_AddWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb, void* data);
extern JS_PUBLIC_API(void)
JS_RemoveWeakPointerZoneGroupCallback(JSContext* cx, JSWeakPointerZoneGroupCallback cb);
JS_RemoveWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb);
extern JS_PUBLIC_API(bool)
JS_AddWeakPointerCompartmentCallback(JSContext* cx, JSWeakPointerCompartmentCallback cb,

8
js/src/jsfriendapi.cpp
View file

@ -622,7 +622,7 @@ struct VisitGrayCallbackFunctor {
template <class T>
void operator()(T tp) const {
if ((*tp)->isTenured() && (*tp)->asTenured().isMarked(gc::GRAY))
if ((*tp)->isTenured() && (*tp)->asTenured().isMarkedGray())
callback_(closure_, JS::GCCellPtr(*tp));
}
};
@ -1083,10 +1083,10 @@ static char
MarkDescriptor(void* thing)
{
gc::TenuredCell* cell = gc::TenuredCell::fromPointer(thing);
if (cell->isMarked(gc::BLACK))
return cell->isMarked(gc::GRAY) ? 'G' : 'B';
if (cell->isMarkedAny())
return cell->isMarkedGray() ? 'G' : 'B';
else
return cell->isMarked(gc::GRAY) ? 'X' : 'W';
return cell->isMarkedGray() ? 'X' : 'W';
}
static void

668
js/src/jsgc.cpp
View file

@ -476,14 +476,14 @@ Arena::finalize(FreeOp* fop, AllocKind thingKind, size_t thingSize)
if (MOZ_UNLIKELY(MemProfiler::enabled())) {
for (ArenaCellIterUnderFinalize i(this); !i.done(); i.next()) {
T* t = i.get<T>();
if (t->asTenured().isMarked())
if (t->asTenured().isMarkedAny())
MemProfiler::MarkTenured(reinterpret_cast<void*>(t));
}
}
for (ArenaCellIterUnderFinalize i(this); !i.done(); i.next()) {
T* t = i.get<T>();
if (t->asTenured().isMarked()) {
if (t->asTenured().isMarkedAny()) {
uint_fast16_t thing = uintptr_t(t) & ArenaMask;
if (thing != firstThingOrSuccessorOfLastMarkedThing) {
// We just finished passing over one or more free things,
@ -874,12 +874,10 @@ GCRuntime::GCRuntime(JSRuntime* rt) :
incrementalState(gc::State::NotActive),
lastMarkSlice(false),
sweepOnBackgroundThread(false),
blocksToFreeAfterSweeping(JSRuntime::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
blocksToFreeAfterMinorGC(JSRuntime::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
zoneGroupIndex(0),
zoneGroups(nullptr),
currentZoneGroup(nullptr),
sweepPhaseIndex(0),
blocksToFreeAfterSweeping((size_t) JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
sweepGroupIndex(0),
sweepGroups(nullptr),
currentSweepGroup(nullptr),
sweepZone(nullptr),
sweepActionIndex(0),
abortSweepAfterCurrentGroup(false),
@ -1304,28 +1302,27 @@ GCRuntime::callFinalizeCallbacks(FreeOp* fop, JSFinalizeStatus status) const
}
bool
GCRuntime::addWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback, void* data)
GCRuntime::addWeakPointerZonesCallback(JSWeakPointerZonesCallback callback, void* data)
{
return updateWeakPointerZoneGroupCallbacks.append(
Callback<JSWeakPointerZoneGroupCallback>(callback, data));
return updateWeakPointerZonesCallbacks.ref().append(
Callback<JSWeakPointerZonesCallback>(callback, data));
}
void
GCRuntime::removeWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback)
GCRuntime::removeWeakPointerZonesCallback(JSWeakPointerZonesCallback callback)
{
for (auto& p : updateWeakPointerZoneGroupCallbacks) {
for (auto& p : updateWeakPointerZonesCallbacks.ref()) {
if (p.op == callback) {
updateWeakPointerZoneGroupCallbacks.erase(&p);
updateWeakPointerZonesCallbacks.ref().erase(&p);
break;
}
}
}
void
GCRuntime::callWeakPointerZoneGroupCallbacks() const
GCRuntime::callWeakPointerZonesCallbacks() const
{
for (auto const& p : updateWeakPointerZoneGroupCallbacks)
p.op(rt->contextFromMainThread(), p.data);
for (auto const& p : updateWeakPointerZonesCallbacks.ref())
p.op(TlsContext.get(), p.data);
}
bool
@ -1843,8 +1840,8 @@ RelocateArena(Arena* arena, SliceBudget& sliceBudget)
TenuredCell* src = i.getCell();
MOZ_ASSERT(RelocationOverlay::isCellForwarded(src));
TenuredCell* dest = Forwarded(src);
MOZ_ASSERT(src->isMarked(BLACK) == dest->isMarked(BLACK));
MOZ_ASSERT(src->isMarked(GRAY) == dest->isMarked(GRAY));
MOZ_ASSERT(src->isMarkedAny() == dest->isMarkedAny());
MOZ_ASSERT(src->isMarkedGray() == dest->isMarkedGray());
}
#endif
}
@ -2426,11 +2423,7 @@ GCRuntime::updatePointersToRelocatedCells(Zone* zone, AutoLockForExclusiveAccess
blocksToFreeAfterSweeping.freeAll();
// Call callbacks to get the rest of the system to fixup other untraced pointers.
callWeakPointerZoneGroupCallbacks();
for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
callWeakPointerCompartmentCallbacks(comp);
if (rt->sweepZoneCallback)
rt->sweepZoneCallback(zone);
callWeakPointerZonesCallbacks();
}
void
@ -3411,7 +3404,7 @@ ArenaLists::checkEmptyArenaList(AllocKind kind)
for (Arena* current = arenaLists[kind].head(); current; current = current->next) {
for (ArenaCellIterUnderGC i(current); !i.done(); i.next()) {
TenuredCell* t = i.getCell();
MOZ_ASSERT(t->isMarked(), "unmarked cells should have been finalized");
MOZ_ASSERT(t->isMarkedAny(), "unmarked cells should have been finalized");
if (++num_live <= max_cells) {
fprintf(stderr, "ERROR: GC found live Cell %p of kind %s at shutdown\n",
t, AllocKindToAscii(kind));
@ -3879,7 +3872,7 @@ GCRuntime::markWeakReferences(gcstats::Phase phase)
void
GCRuntime::markWeakReferencesInCurrentGroup(gcstats::Phase phase)
{
markWeakReferences<GCZoneGroupIter>(phase);
markWeakReferences<GCSweepGroupIter>(phase);
}
template <class ZoneIterT, class CompartmentIterT>
@ -3902,7 +3895,7 @@ GCRuntime::markGrayReferences(gcstats::Phase phase)
void
GCRuntime::markGrayReferencesInCurrentGroup(gcstats::Phase phase)
{
markGrayReferences<GCZoneGroupIter, GCCompartmentGroupIter>(phase);
markGrayReferences<GCSweepGroupIter, GCCompartmentGroupIter>(phase);
}
void
@ -3917,6 +3910,300 @@ GCRuntime::markAllGrayReferences(gcstats::Phase phase)
markGrayReferences<GCZonesIter, GCCompartmentsIter>(phase);
}
#ifdef JS_GC_ZEAL
struct GCChunkHasher {
typedef gc::Chunk* Lookup;
/*
* Strip zeros for better distribution after multiplying by the golden
* ratio.
*/
static HashNumber hash(gc::Chunk* chunk) {
MOZ_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask));
return HashNumber(uintptr_t(chunk) >> gc::ChunkShift);
}
static bool match(gc::Chunk* k, gc::Chunk* l) {
MOZ_ASSERT(!(uintptr_t(k) & gc::ChunkMask));
MOZ_ASSERT(!(uintptr_t(l) & gc::ChunkMask));
return k == l;
}
};
class js::gc::MarkingValidator
{
public:
explicit MarkingValidator(GCRuntime* gc);
~MarkingValidator();
void nonIncrementalMark(AutoLockForExclusiveAccess& lock);
void validate();
private:
GCRuntime* gc;
bool initialized;
typedef HashMap<Chunk*, ChunkBitmap*, GCChunkHasher, SystemAllocPolicy> BitmapMap;
BitmapMap map;
};
js::gc::MarkingValidator::MarkingValidator(GCRuntime* gc)
: gc(gc),
initialized(false)
{}
js::gc::MarkingValidator::~MarkingValidator()
{
if (!map.initialized())
return;
for (BitmapMap::Range r(map.all()); !r.empty(); r.popFront())
js_delete(r.front().value());
}
void
js::gc::MarkingValidator::nonIncrementalMark(AutoLockForExclusiveAccess& lock)
{
/*
* Perform a non-incremental mark for all collecting zones and record
* the results for later comparison.
*
* Currently this does not validate gray marking.
*/
if (!map.init())
return;
JSRuntime* runtime = gc->rt;
GCMarker* gcmarker = &gc->marker;
gc->waitBackgroundSweepEnd();
/* Save existing mark bits. */
{
AutoLockGC lock(runtime);
for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) {
ChunkBitmap* bitmap = &chunk->bitmap;
ChunkBitmap* entry = js_new<ChunkBitmap>();
if (!entry)
return;
memcpy((void*)entry->bitmap, (void*)bitmap->bitmap, sizeof(bitmap->bitmap));
if (!map.putNew(chunk, entry))
return;
}
}
/*
* Temporarily clear the weakmaps' mark flags for the compartments we are
* collecting.
*/
WeakMapSet markedWeakMaps;
if (!markedWeakMaps.init())
return;
/*
* For saving, smush all of the keys into one big table and split them back
* up into per-zone tables when restoring.
*/
gc::WeakKeyTable savedWeakKeys(SystemAllocPolicy(), runtime->randomHashCodeScrambler());
if (!savedWeakKeys.init())
return;
for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
if (!WeakMapBase::saveZoneMarkedWeakMaps(zone, markedWeakMaps))
return;
AutoEnterOOMUnsafeRegion oomUnsafe;
for (gc::WeakKeyTable::Range r = zone->gcWeakKeys().all(); !r.empty(); r.popFront()) {
if (!savedWeakKeys.put(Move(r.front().key), Move(r.front().value)))
oomUnsafe.crash("saving weak keys table for validator");
}
if (!zone->gcWeakKeys().clear())
oomUnsafe.crash("clearing weak keys table for validator");
}
/*
* After this point, the function should run to completion, so we shouldn't
* do anything fallible.
*/
initialized = true;
/* Re-do all the marking, but non-incrementally. */
js::gc::State state = gc->incrementalState;
gc->incrementalState = State::MarkRoots;
{
gcstats::AutoPhase ap(gc->stats(), gcstats::PHASE_MARK);
{
gcstats::AutoPhase ap(gc->stats(), gcstats::PHASE_UNMARK);
for (GCZonesIter zone(runtime); !zone.done(); zone.next())
WeakMapBase::unmarkZone(zone);
MOZ_ASSERT(gcmarker->isDrained());
gcmarker->reset();
AutoLockGC lock(runtime);
for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next())
chunk->bitmap.clear();
}
gc->traceRuntimeForMajorGC(gcmarker, lock);
gc->incrementalState = State::Mark;
auto unlimited = SliceBudget::unlimited();
MOZ_RELEASE_ASSERT(gc->marker.drainMarkStack(unlimited));
}
gc->incrementalState = State::Sweep;
{
gcstats::AutoPhase ap1(gc->stats(), gcstats::PHASE_SWEEP);
gcstats::AutoPhase ap2(gc->stats(), gcstats::PHASE_SWEEP_MARK);
gc->markAllWeakReferences(gcstats::PHASE_SWEEP_MARK_WEAK);
/* Update zone state for gray marking. */
for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
MOZ_ASSERT(zone->isGCMarkingBlack());
zone->setGCState(Zone::MarkGray);
}
gc->marker.setMarkColorGray();
gc->markAllGrayReferences(gcstats::PHASE_SWEEP_MARK_GRAY);
gc->markAllWeakReferences(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK);
/* Restore zone state. */
for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
MOZ_ASSERT(zone->isGCMarkingGray());
zone->setGCState(Zone::Mark);
}
MOZ_ASSERT(gc->marker.isDrained());
gc->marker.setMarkColorBlack();
}
/* Take a copy of the non-incremental mark state and restore the original. */
{
AutoLockGC lock(runtime);
for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) {
ChunkBitmap* bitmap = &chunk->bitmap;
ChunkBitmap* entry = map.lookup(chunk)->value();
Swap(*entry, *bitmap);
}
}
for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
WeakMapBase::unmarkZone(zone);
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!zone->gcWeakKeys().clear())
oomUnsafe.crash("clearing weak keys table for validator");
}
WeakMapBase::restoreMarkedWeakMaps(markedWeakMaps);
for (gc::WeakKeyTable::Range r = savedWeakKeys.all(); !r.empty(); r.popFront()) {
AutoEnterOOMUnsafeRegion oomUnsafe;
Zone* zone = gc::TenuredCell::fromPointer(r.front().key.asCell())->zone();
if (!zone->gcWeakKeys().put(Move(r.front().key), Move(r.front().value)))
oomUnsafe.crash("restoring weak keys table for validator");
}
gc->incrementalState = state;
}
void
js::gc::MarkingValidator::validate()
{
/*
* Validates the incremental marking for a single compartment by comparing
* the mark bits to those previously recorded for a non-incremental mark.
*/
if (!initialized)
return;
gc->waitBackgroundSweepEnd();
AutoLockGC lock(gc->rt);
for (auto chunk = gc->allNonEmptyChunks(lock); !chunk.done(); chunk.next()) {
BitmapMap::Ptr ptr = map.lookup(chunk);
if (!ptr)
continue; /* Allocated after we did the non-incremental mark. */
ChunkBitmap* bitmap = ptr->value();
ChunkBitmap* incBitmap = &chunk->bitmap;
for (size_t i = 0; i < ArenasPerChunk; i++) {
if (chunk->decommittedArenas.get(i))
continue;
Arena* arena = &chunk->arenas[i];
if (!arena->allocated())
continue;
if (!arena->zone->isGCSweeping())
continue;
if (arena->allocatedDuringIncremental)
continue;
AllocKind kind = arena->getAllocKind();
uintptr_t thing = arena->thingsStart();
uintptr_t end = arena->thingsEnd();
while (thing < end) {
Cell* cell = (Cell*)thing;
/*
* If a non-incremental GC wouldn't have collected a cell, then
* an incremental GC won't collect it.
*/
if (bitmap->isMarkedAny(cell))
MOZ_RELEASE_ASSERT(incBitmap->isMarkedAny(cell));
/*
* If the cycle collector isn't allowed to collect an object
* after a non-incremental GC has run, then it isn't allowed to
* collected it after an incremental GC.
*/
if (!bitmap->isMarkedGray(cell))
MOZ_RELEASE_ASSERT(!incBitmap->isMarkedGray(cell));
thing += Arena::thingSize(kind);
}
}
}
}
#endif // JS_GC_ZEAL
void
GCRuntime::computeNonIncrementalMarkingForValidation(AutoLockForExclusiveAccess& lock)
{
#ifdef JS_GC_ZEAL
MOZ_ASSERT(!markingValidator);
if (isIncremental && hasZealMode(ZealMode::IncrementalMarkingValidator))
markingValidator = js_new<MarkingValidator>(this);
if (markingValidator)
markingValidator->nonIncrementalMark(lock);
#endif
}
void
GCRuntime::validateIncrementalMarking()
{
#ifdef JS_GC_ZEAL
if (markingValidator)
markingValidator->validate();
#endif
}
void
GCRuntime::finishMarkingValidation()
{
#ifdef JS_GC_ZEAL
js_delete(markingValidator.ref());
markingValidator = nullptr;
#endif
}
static void
DropStringWrappers(JSRuntime* rt)
{
@ -3983,7 +4270,7 @@ JSCompartment::findOutgoingEdges(ZoneComponentFinder& finder)
bool needsEdge = true;
if (key.is<JSObject*>()) {
TenuredCell& other = key.as<JSObject*>()->asTenured();
needsEdge = !other.isMarked(BLACK) || other.isMarked(GRAY);
needsEdge = !other.isMarkedAny() || other.isMarkedGray();
}
key.applyToWrapped(AddOutgoingEdgeFunctor(needsEdge, finder));
}
@ -4004,7 +4291,7 @@ Zone::findOutgoingEdges(ZoneComponentFinder& finder)
for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next())
comp->findOutgoingEdges(finder);
for (ZoneSet::Range r = gcZoneGroupEdges.all(); !r.empty(); r.popFront()) {
for (ZoneSet::Range r = gcSweepGroupEdges().all(); !r.empty(); r.popFront()) {
if (r.front()->shouldMarkInZone())
finder.addEdgeTo(r.front());
}
@ -4034,11 +4321,11 @@ GCRuntime::findInterZoneEdges()
}
void
GCRuntime::findZoneGroups(AutoLockForExclusiveAccess& lock)
GCRuntime::groupZonesForSweeping(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock)
{
#ifdef DEBUG
for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next())
MOZ_ASSERT(zone->gcZoneGroupEdges.empty());
MOZ_ASSERT(zone->gcSweepGroupEdges().empty());
#endif
JSContext* cx = rt->contextFromMainThread();
@ -4050,22 +4337,22 @@ GCRuntime::findZoneGroups(AutoLockForExclusiveAccess& lock)
MOZ_ASSERT(zone->shouldMarkInZone());
finder.addNode(zone);
}
zoneGroups = finder.getResultsList();
currentZoneGroup = zoneGroups;
zoneGroupIndex = 0;
sweepGroups = finder.getResultsList();
currentSweepGroup = sweepGroups;
sweepGroupIndex = 0;
for (GCZonesIter zone(rt); !zone.done(); zone.next())
zone->gcZoneGroupEdges.clear();
zone->gcSweepGroupEdges().clear();
#ifdef DEBUG
for (Zone* head = currentZoneGroup; head; head = head->nextGroup()) {
for (Zone* head = currentSweepGroup; head; head = head->nextGroup()) {
for (Zone* zone = head; zone; zone = zone->nextNodeInGroup())
MOZ_ASSERT(zone->shouldMarkInZone());
}
MOZ_ASSERT_IF(!isIncremental, !currentZoneGroup->nextGroup());
MOZ_ASSERT_IF(!isIncremental, !currentSweepGroup->nextGroup());
for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next())
MOZ_ASSERT(zone->gcZoneGroupEdges.empty());
MOZ_ASSERT(zone->gcSweepGroupEdges().empty());
#endif
}
@ -4073,26 +4360,26 @@ static void
ResetGrayList(JSCompartment* comp);
void
GCRuntime::getNextZoneGroup()
GCRuntime::getNextSweepGroup()
{
currentZoneGroup = currentZoneGroup->nextGroup();
++zoneGroupIndex;
if (!currentZoneGroup) {
currentSweepGroup = currentSweepGroup->nextGroup();
++sweepGroupIndex;
if (!currentSweepGroup) {
abortSweepAfterCurrentGroup = false;
return;
}
for (Zone* zone = currentZoneGroup; zone; zone = zone->nextNodeInGroup()) {
for (Zone* zone = currentSweepGroup; zone; zone = zone->nextNodeInGroup()) {
MOZ_ASSERT(zone->shouldMarkInZone());
MOZ_ASSERT(!zone->isQueuedForBackgroundSweep());
}
if (!isIncremental)
ZoneComponentFinder::mergeGroups(currentZoneGroup);
ZoneComponentFinder::mergeGroups(currentSweepGroup);
if (abortSweepAfterCurrentGroup) {
MOZ_ASSERT(!isIncremental);
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
MOZ_ASSERT(!zone->gcNextGraphComponent);
MOZ_ASSERT(zone->shouldMarkInZone());
zone->setNeedsIncrementalBarrier(false, Zone::UpdateJit);
@ -4104,7 +4391,7 @@ GCRuntime::getNextZoneGroup()
ResetGrayList(comp);
abortSweepAfterCurrentGroup = false;
currentZoneGroup = nullptr;
currentSweepGroup = nullptr;
}
}
@ -4250,11 +4537,11 @@ MarkIncomingCrossCompartmentPointers(JSRuntime* rt, const uint32_t color)
MOZ_ASSERT(dst->compartment() == c);
if (color == GRAY) {
if (IsMarkedUnbarriered(rt, &src) && src->asTenured().isMarked(GRAY))
if (IsMarkedUnbarriered(rt, &src) && src->asTenured().isMarkedGray())
TraceManuallyBarrieredEdge(&rt->gc.marker, &dst,
"cross-compartment gray pointer");
} else {
if (IsMarkedUnbarriered(rt, &src) && !src->asTenured().isMarked(GRAY))
if (IsMarkedUnbarriered(rt, &src) && !src->asTenured().isMarkedGray())
TraceManuallyBarrieredEdge(&rt->gc.marker, &dst,
"cross-compartment black pointer");
}
@ -4350,8 +4637,8 @@ js::NotifyGCPostSwap(JSObject* a, JSObject* b, unsigned removedFlags)
DelayCrossCompartmentGrayMarking(a);
}
void
GCRuntime::endMarkingZoneGroup()
IncrementalProgress
GCRuntime::endMarkingSweepGroup(FreeOp* fop, SliceBudget& budget)
{
gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_MARK);
@ -4369,7 +4656,7 @@ GCRuntime::endMarkingZoneGroup()
* these will be marked through, as they are not marked with
* MarkCrossCompartmentXXX.
*/
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
MOZ_ASSERT(zone->isGCMarkingBlack());
zone->setGCState(Zone::MarkGray);
}
@ -4383,7 +4670,7 @@ GCRuntime::endMarkingZoneGroup()
markWeakReferencesInCurrentGroup(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK);
/* Restore marking state. */
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
MOZ_ASSERT(zone->isGCMarkingGray());
zone->setGCState(Zone::Mark);
}
@ -4479,6 +4766,58 @@ SweepMiscTask::run()
}
}
static void
SweepCompressionTasks(JSRuntime* runtime)
{
AutoLockHelperThreadState lock;
// Attach finished compression tasks.
auto& finished = HelperThreadState().compressionFinishedList(lock);
for (size_t i = 0; i < finished.length(); i++) {
if (finished[i]->runtimeMatches(runtime)) {
UniquePtr<SourceCompressionTask> task(Move(finished[i]));
HelperThreadState().remove(finished, &i);
task->complete();
}
}
// Sweep pending tasks that are holding onto should-be-dead ScriptSources.
auto& pending = HelperThreadState().compressionPendingList(lock);
for (size_t i = 0; i < pending.length(); i++) {
if (pending[i]->shouldCancel())
HelperThreadState().remove(pending, &i);
}
}
static void
SweepWeakMaps(JSRuntime* runtime)
{
for (GCSweepGroupIter zone(runtime); !zone.done(); zone.next()) {
/* Clear all weakrefs that point to unmarked things. */
for (auto edge : zone->gcWeakRefs()) {
/* Edges may be present multiple times, so may already be nulled. */
if (*edge && IsAboutToBeFinalizedDuringSweep(**edge))
*edge = nullptr;
}
zone->gcWeakRefs().clear();
/* No need to look up any more weakmap keys from this sweep group. */
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!zone->gcWeakKeys().clear())
oomUnsafe.crash("clearing weak keys in beginSweepingSweepGroup()");
zone->sweepWeakMaps();
}
}
static void
SweepUniqueIds(JSRuntime* runtime)
{
FreeOp fop(nullptr);
for (GCSweepGroupIter zone(runtime); !zone.done(); zone.next())
zone->sweepUniqueIds(&fop);
}
void
GCRuntime::startTask(GCParallelTask& task, gcstats::Phase phase,
AutoLockHelperThreadState& locked)
@ -4498,7 +4837,127 @@ GCRuntime::joinTask(GCParallelTask& task, gcstats::Phase phase,
task.joinWithLockHeld(locked);
}
using WeakCacheTaskVector = mozilla::Vector<SweepWeakCacheTask, 0, SystemAllocPolicy>;
void
GCRuntime::sweepDebuggerOnMainThread(FreeOp* fop)
{
// Detach unreachable debuggers and global objects from each other.
// This can modify weakmaps and so must happen before weakmap sweeping.
Debugger::sweepAll(fop);
gcstats::AutoPhase ap(stats(), gcstats::PHASE_SWEEP_COMPARTMENTS);
// Sweep debug environment information. This performs lookups in the Zone's
// unique IDs table and so must not happen in parallel with sweeping that
// table.
{
gcstats::AutoPhase ap2(stats(), gcstats::PHASE_SWEEP_MISC);
for (GCCompartmentGroupIter c(rt); !c.done(); c.next())
c->sweepDebugEnvironments();
}
// Sweep breakpoints. This is done here to be with the other debug sweeping,
// although note that it can cause JIT code to be patched.
{
gcstats::AutoPhase ap(stats(), gcstats::PHASE_SWEEP_BREAKPOINT);
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next())
zone->sweepBreakpoints(fop);
}
}
void
GCRuntime::sweepJitDataOnMainThread(FreeOp* fop)
{
{
gcstats::AutoPhase ap(stats(), gcstats::PHASE_SWEEP_JIT_DATA);
// Cancel any active or pending off thread compilations.
js::CancelOffThreadIonCompile(rt, JS::Zone::Sweep);
for (GCCompartmentGroupIter c(rt); !c.done(); c.next())
c->sweepJitCompartment(fop);
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
if (jit::JitZone* jitZone = zone->jitZone())
jitZone->sweep(fop);
}
// Bug 1071218: the following method has not yet been refactored to
// work on a single zone-group at once.
// Sweep entries containing about-to-be-finalized JitCode and
// update relocated TypeSet::Types inside the JitcodeGlobalTable.
jit::JitRuntime::SweepJitcodeGlobalTable(rt);
}
{
gcstats::AutoPhase apdc(stats(), gcstats::PHASE_SWEEP_DISCARD_CODE);
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next())
zone->discardJitCode(fop);
}
{
gcstats::AutoPhase ap1(stats(), gcstats::PHASE_SWEEP_TYPES);
gcstats::AutoPhase ap2(stats(), gcstats::PHASE_SWEEP_TYPES_BEGIN);
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next())
zone->beginSweepTypes(fop, releaseObservedTypes && !zone->isPreservingCode());
}
}
using WeakCacheTaskVector = mozilla::Vector<ImmediateSweepWeakCacheTask, 0, SystemAllocPolicy>;
enum WeakCacheLocation
{
RuntimeWeakCache,
ZoneWeakCache
};
// Call a functor for all weak caches that need to be swept in the current
// sweep group.
template <typename Functor>
static inline bool
IterateWeakCaches(JSRuntime* rt, Functor f)
{
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
for (JS::detail::WeakCacheBase* cache : zone->weakCaches()) {
if (!f(cache, ZoneWeakCache))
return false;
}
}
for (JS::detail::WeakCacheBase* cache : rt->weakCaches()) {
if (!f(cache, RuntimeWeakCache))
return false;
}
return true;
}
static bool
PrepareWeakCacheTasks(JSRuntime* rt, WeakCacheTaskVector* immediateTasks)
{
// Start incremental sweeping for caches that support it or add to a vector
// of sweep tasks to run on a helper thread.
MOZ_ASSERT(immediateTasks->empty());
bool ok = IterateWeakCaches(rt, [&] (JS::detail::WeakCacheBase* cache,
WeakCacheLocation location)
{
if (!cache->needsSweep())
return true;
// Caches that support incremental sweeping will be swept later.
if (location == ZoneWeakCache && cache->setNeedsIncrementalBarrier(true))
return true;
return immediateTasks->emplaceBack(rt, *cache);
});
if (!ok)
immediateTasks->clearAndFree();
return ok;
}
static void
SweepWeakCachesFromMainThread(JSRuntime* rt)
@ -4526,8 +4985,17 @@ PrepareWeakCacheTasks(JSRuntime* rt)
return out;
}
void
GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock)
~AutoRunParallelTask() {
runtime()->gc.joinTask(*this, phase_, lock_);
}
void run() override {
func_(runtime());
}
};
IncrementalProgress
GCRuntime::beginSweepingSweepGroup(FreeOp* fop, SliceBudget& budget)
{
/*
* Begin sweeping the group of zones in gcCurrentZoneGroup,
@ -4535,7 +5003,7 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock)
*/
bool sweepingAtoms = false;
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
/* Set the GC state to sweeping. */
MOZ_ASSERT(zone->shouldMarkInZone());
zone->setGCState(Zone::Sweep);
@ -4550,7 +5018,7 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock)
rt->sweepZoneCallback(zone);
#ifdef DEBUG
zone->gcLastZoneGroupIndex = zoneGroupIndex;
zone->gcLastSweepGroupIndex = sweepGroupIndex;
#endif
}
@ -4581,12 +5049,12 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock)
gcstats::AutoPhase ap(stats, gcstats::PHASE_FINALIZE_START);
callFinalizeCallbacks(&fop, JSFINALIZE_GROUP_START);
{
gcstats::AutoPhase ap2(stats, gcstats::PHASE_WEAK_ZONEGROUP_CALLBACK);
callWeakPointerZoneGroupCallbacks();
AutoPhase ap2(stats(), PHASE_WEAK_ZONEGROUP_CALLBACK);
callWeakPointerZonesCallbacks();
}
{
gcstats::AutoPhase ap2(stats, gcstats::PHASE_WEAK_COMPARTMENT_CALLBACK);
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
AutoPhase ap2(stats(), PHASE_WEAK_COMPARTMENT_CALLBACK);
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
callWeakPointerCompartmentCallbacks(comp);
}
@ -4700,17 +5168,10 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock)
* Objects are finalized immediately but this may change in the future.
*/
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
gcstats::AutoSCC scc(stats, zoneGroupIndex);
zone->arenas.queueForegroundObjectsForSweep(&fop);
}
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
gcstats::AutoSCC scc(stats, zoneGroupIndex);
for (unsigned i = 0; i < ArrayLength(IncrementalFinalizePhases); ++i)
zone->arenas.queueForForegroundSweep(&fop, IncrementalFinalizePhases[i]);
}
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
gcstats::AutoSCC scc(stats, zoneGroupIndex);
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
gcstats::AutoSCC scc(stats(), sweepGroupIndex);
zone->arenas.queueForForegroundSweep(fop, ForegroundObjectFinalizePhase);
zone->arenas.queueForForegroundSweep(fop, ForegroundNonObjectFinalizePhase);
for (unsigned i = 0; i < ArrayLength(BackgroundFinalizePhases); ++i)
zone->arenas.queueForBackgroundSweep(&fop, BackgroundFinalizePhases[i]);
}
@ -4724,7 +5185,19 @@ GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock)
sweepActionIndex = 0;
{
gcstats::AutoPhase ap(stats, gcstats::PHASE_FINALIZE_END);
return NotFinished;
}
return Finished;
}
#endif
IncrementalProgress
GCRuntime::endSweepingSweepGroup(FreeOp* fop, SliceBudget& budget)
{
{
gcstats::AutoPhase ap(stats(), gcstats::PHASE_FINALIZE_END);
FreeOp fop(rt);
callFinalizeCallbacks(&fop, JSFINALIZE_GROUP_END);
}
}
@ -4733,7 +5206,7 @@ void
GCRuntime::endSweepingZoneGroup()
{
/* Update the GC state for zones we have swept. */
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
MOZ_ASSERT(zone->isGCSweeping());
AutoLockGC lock(rt);
zone->setGCState(Zone::Finished);
@ -4743,7 +5216,7 @@ GCRuntime::endSweepingZoneGroup()
/* Start background thread to sweep zones if required. */
ZoneList zones;
for (GCZoneGroupIter zone(rt); !zone.done(); zone.next())
for (GCSweepGroupIter zone(rt); !zone.done(); zone.next())
zones.append(zone);
if (sweepOnBackgroundThread)
queueZonesForBackgroundSweep(zones);
@ -4826,7 +5299,7 @@ GCRuntime::drainMarkStack(SliceBudget& sliceBudget, gcstats::Phase phase)
static void
SweepThing(Shape* shape)
{
if (!shape->isMarked())
if (!shape->isMarkedAny())
shape->sweep();
}
@ -4981,7 +5454,44 @@ GCRuntime::initializeSweepActions()
AddSweepPhase(&ok);
AddSweepAction(&ok, GCRuntime::sweepShapeTree);
return ok;
using Action = typename RemoveLastTemplateParameter<
SweepActionForEach<ContainerIter<AllocKinds>, AllocKinds, Args...>>::Type;
return js::MakeUnique<Action>(kinds, Move(action));
}
} // namespace sweepaction
bool
GCRuntime::initSweepActions()
{
using namespace sweepaction;
using sweepaction::Call;
sweepActions.ref() =
RepeatForZoneGroup(rt,
Sequence(
Call(&GCRuntime::endMarkingSweepGroup),
Call(&GCRuntime::beginSweepingSweepGroup),
#ifdef JS_GC_ZEAL
Call(&GCRuntime::maybeYieldForSweepingZeal),
#endif
Call(&GCRuntime::sweepAtomsTable),
Call(&GCRuntime::sweepWeakCaches),
ForEachZoneInZoneGroup(rt,
ForEachAllocKind(ForegroundObjectFinalizePhase.kinds,
Call(&GCRuntime::finalizeAllocKind))),
ForEachZoneInZoneGroup(rt,
Sequence(
Call(&GCRuntime::sweepTypeInformation),
Call(&GCRuntime::mergeSweptObjectArenas))),
ForEachZoneInZoneGroup(rt,
ForEachAllocKind(ForegroundNonObjectFinalizePhase.kinds,
Call(&GCRuntime::finalizeAllocKind))),
ForEachZoneInZoneGroup(rt,
Call(&GCRuntime::sweepShapeTree)),
Call(&GCRuntime::endSweepingSweepGroup)));
return sweepActions != nullptr;
}
IncrementalProgress
@ -5816,7 +6326,7 @@ GCRuntime::maybeDoCycleCollection()
for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
++compartmentsTotal;
GlobalObject* global = c->unsafeUnbarrieredMaybeGlobal();
if (global && global->asTenured().isMarked(GRAY))
if (global && global->asTenured().isMarkedGray())
++compartmentsGray;
}
double grayFraction = double(compartmentsGray) / double(compartmentsTotal);

11
js/src/jsgcinlines.h
View file

@ -434,15 +434,14 @@ class GCZonesIter
typedef CompartmentsIterT<GCZonesIter> GCCompartmentsIter;
/* Iterates over all zones in the current zone group. */
class GCZoneGroupIter {
private:
/* Iterates over all zones in the current sweep group. */
class GCSweepGroupIter {
JS::Zone* current;
public:
explicit GCZoneGroupIter(JSRuntime* rt) {
explicit GCSweepGroupIter(JSRuntime* rt) {
MOZ_ASSERT(CurrentThreadIsPerformingGC());
current = rt->gc.getCurrentZoneGroup();
current = rt->gc.getCurrentSweepGroup();
}
bool done() const { return !current; }
@ -461,7 +460,7 @@ class GCZoneGroupIter {
JS::Zone* operator->() const { return get(); }
};
typedef CompartmentsIterT<GCZoneGroupIter> GCCompartmentGroupIter;
typedef CompartmentsIterT<GCSweepGroupIter> GCCompartmentGroupIter;
inline void
RelocationOverlay::forwardTo(Cell* cell)

41
js/src/jsscript.cpp
View file

@ -3417,7 +3417,7 @@ js::detail::CopyScript(JSContext* cx, HandleScript src, HandleScript dst,
/* NB: Keep this in sync with XDRScript. */
/* Some embeddings are not careful to use ExposeObjectToActiveJS as needed. */
MOZ_ASSERT(!src->sourceObject()->asTenured().isMarked(gc::GRAY));
MOZ_ASSERT(!src->sourceObject()->asTenured().isMarkedGray());
uint32_t nconsts = src->hasConsts() ? src->consts()->length : 0;
uint32_t nobjects = src->hasObjects() ? src->objects()->length : 0;
@ -4161,24 +4161,27 @@ JSScript::argumentsOptimizationFailed(JSContext* cx, HandleScript script)
* assumption of !script->needsArgsObj();
* - type inference data for the script assuming script->needsArgsObj
*/
for (AllScriptFramesIter i(cx); !i.done(); ++i) {
/*
* We cannot reliably create an arguments object for Ion activations of
* this script. To maintain the invariant that "script->needsArgsObj
* implies fp->hasArgsObj", the Ion bail mechanism will create an
* arguments object right after restoring the BaselineFrame and before
* entering Baseline code (in jit::FinishBailoutToBaseline).
*/
if (i.isIon())
continue;
AbstractFramePtr frame = i.abstractFramePtr();
if (frame.isFunctionFrame() && frame.script() == script) {
/* We crash on OOM since cleaning up here would be complicated. */
AutoEnterOOMUnsafeRegion oomUnsafe;
ArgumentsObject* argsobj = ArgumentsObject::createExpected(cx, frame);
if (!argsobj)
oomUnsafe.crash("JSScript::argumentsOptimizationFailed");
SetFrameArgumentsObject(cx, frame, script, argsobj);
JSRuntime::AutoProhibitActiveContextChange apacc(cx->runtime());
for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) {
for (AllScriptFramesIter i(cx, target); !i.done(); ++i) {
/*
* We cannot reliably create an arguments object for Ion activations of
* this script. To maintain the invariant that "script->needsArgsObj
* implies fp->hasArgsObj", the Ion bail mechanism will create an
* arguments object right after restoring the BaselineFrame and before
* entering Baseline code (in jit::FinishBailoutToBaseline).
*/
if (i.isIon())
continue;
AbstractFramePtr frame = i.abstractFramePtr();
if (frame.isFunctionFrame() && frame.script() == script) {
/* We crash on OOM since cleaning up here would be complicated. */
AutoEnterOOMUnsafeRegion oomUnsafe;
ArgumentsObject* argsobj = ArgumentsObject::createExpected(cx, frame);
if (!argsobj)
oomUnsafe.crash("JSScript::argumentsOptimizationFailed");
SetFrameArgumentsObject(cx, frame, script, argsobj);
}
}
}

4
js/src/jsweakmap.cpp
View file

@ -138,7 +138,7 @@ ObjectValueMap::findZoneEdges()
JS::AutoSuppressGCAnalysis nogc;
for (Range r = all(); !r.empty(); r.popFront()) {
JSObject* key = r.front().key();
if (key->asTenured().isMarked(BLACK) && !key->asTenured().isMarked(GRAY))
if (key->asTenured().isMarkedAny() && !key->asTenured().isMarkedGray())
continue;
JSObject* delegate = getDelegate(key);
if (!delegate)
@ -146,7 +146,7 @@ ObjectValueMap::findZoneEdges()
Zone* delegateZone = delegate->zone();
if (delegateZone == zone || !delegateZone->shouldMarkInZone())
continue;
if (!delegateZone->gcZoneGroupEdges.put(key->zone()))
if (!delegateZone->gcSweepGroupEdges().put(key->zone()))
return false;
}
return true;

12
js/src/proxy/Wrapper.cpp
View file

@ -329,8 +329,16 @@ Wrapper::wrappedObject(JSObject* wrapper)
{
MOZ_ASSERT(wrapper->is<WrapperObject>());
JSObject* target = wrapper->as<ProxyObject>().target();
if (target)
JS::ExposeObjectToActiveJS(target);
// Eagerly unmark gray wrapper targets so we can assert that we don't create
// black to gray edges. An incremental GC will eventually mark the targets
// of black wrappers black but while it is in progress we can observe gray
// targets. Expose rather than returning a gray object in this case.
if (target) {
if (wrapper->isMarkedAny() && !wrapper->isMarkedGray())
MOZ_ASSERT(JS::ObjectIsNotGray(target));
if (!wrapper->isMarkedGray())
JS::ExposeObjectToActiveJS(target);
}
return target;
}

179
js/src/shell/js.cpp
View file

@ -1713,8 +1713,7 @@ Evaluate(JSContext* cx, unsigned argc, Value* vp)
.setFileAndLine("@evaluate", 1);
global = JS_GetGlobalForObject(cx, &args.callee());
if (!global)
return false;
MOZ_ASSERT(global);
if (args.length() == 2) {
RootedObject opts(cx, &args[1].toObject());
@ -1753,6 +1752,41 @@ Evaluate(JSContext* cx, unsigned argc, Value* vp)
return false;
}
}
if (!JS_GetProperty(cx, opts, "zoneGroup", &v))
return false;
if (!v.isUndefined()) {
if (global != JS_GetGlobalForObject(cx, &args.callee())) {
JS_ReportErrorASCII(cx, "zoneGroup and global cannot both be specified.");
return false;
}
// Find all eligible globals to execute in: any global in another
// zone group which has not been entered by a cooperative thread.
JS::AutoObjectVector eligibleGlobals(cx);
for (CompartmentsIter c(cx->runtime(), SkipAtoms); !c.done(); c.next()) {
if (!c->zone()->group()->ownerContext().context() &&
c->maybeGlobal() &&
!cx->runtime()->isSelfHostingGlobal(c->maybeGlobal()))
{
if (!eligibleGlobals.append(c->maybeGlobal()))
return false;
}
}
if (eligibleGlobals.empty()) {
JS_ReportErrorASCII(cx, "zoneGroup can only be used if another"
" cooperative thread has called cooperativeYield(true).");
return false;
}
// Pick an eligible global to use based on the value of the zoneGroup property.
int32_t which;
if (!ToInt32(cx, v, &which))
return false;
which = Min<int32_t>(Max(which, 0), eligibleGlobals.length() - 1);
global = eligibleGlobals[which];
}
if (!JS_GetProperty(cx, opts, "catchTermination", &v))
return false;
@ -3440,6 +3474,137 @@ EvalInContext(JSContext* cx, unsigned argc, Value* vp)
return true;
}
struct CooperationState
{
CooperationState()
: lock(mutexid::ShellThreadCooperation)
, idle(false)
, numThreads(0)
, yieldCount(0)
, singleThreaded(false)
{}
Mutex lock;
ConditionVariable cvar;
bool idle;
size_t numThreads;
uint64_t yieldCount;
bool singleThreaded;
};
static CooperationState* cooperationState = nullptr;
static void
CooperativeBeginWait(JSContext* cx)
{
MOZ_ASSERT(cx == TlsContext.get());
JS_YieldCooperativeContext(cx);
}
static void
CooperativeEndWait(JSContext* cx)
{
MOZ_ASSERT(cx == TlsContext.get());
LockGuard<Mutex> lock(cooperationState->lock);
cooperationState->cvar.wait(lock, [&] { return cooperationState->idle; });
JS_ResumeCooperativeContext(cx);
cooperationState->idle = false;
cooperationState->yieldCount++;
cooperationState->cvar.notify_all();
}
static void
CooperativeYield()
{
LockGuard<Mutex> lock(cooperationState->lock);
MOZ_ASSERT(!cooperationState->idle);
cooperationState->idle = true;
cooperationState->cvar.notify_all();
// Wait until another thread takes over control before returning, if there
// is another thread to do so.
if (cooperationState->numThreads) {
uint64_t count = cooperationState->yieldCount;
cooperationState->cvar.wait(lock, [&] { return cooperationState->yieldCount != count; });
}
}
static bool
CooperativeYieldThread(JSContext* cx, unsigned argc, Value* vp)
{
CallArgs args = CallArgsFromVp(argc, vp);
if (!cx->runtime()->gc.canChangeActiveContext(cx)) {
JS_ReportErrorASCII(cx, "Cooperating multithreading context switches are not currently allowed");
return false;
}
if (GetShellContext(cx)->isWorker) {
JS_ReportErrorASCII(cx, "Worker threads cannot yield");
return false;
}
if (cooperationState->singleThreaded) {
JS_ReportErrorASCII(cx, "Yielding is not allowed while single threaded");
return false;
}
// To avoid contention issues between threads, yields are not allowed while
// a thread has access to zone groups other than its original one, i.e. if
// the thread is inside an evaluate() call with a different zone group.
// This is not a limit which the browser has, but is necessary in the
// shell: the shell can have arbitrary interleavings between cooperative
// threads, whereas the browser has more control over which threads are
// running at different times.
for (ZoneGroupsIter group(cx->runtime()); !group.done(); group.next()) {
if (group->ownerContext().context() == cx && group != cx->zone()->group()) {
JS_ReportErrorASCII(cx, "Yielding is not allowed while owning multiple zone groups");
return false;
}
}
{
Maybe<JS::AutoRelinquishZoneGroups> artzg;
if ((args.length() > 0) && ToBoolean(args[0]))
artzg.emplace(cx);
CooperativeBeginWait(cx);
CooperativeYield();
CooperativeEndWait(cx);
}
args.rval().setUndefined();
return true;
}
static void
CooperativeBeginSingleThreadedExecution(JSContext* cx)
{
MOZ_ASSERT(!cooperationState->singleThreaded);
// Yield until all other threads have exited any zone groups they are in.
while (true) {
bool done = true;
for (ZoneGroupsIter group(cx->runtime()); !group.done(); group.next()) {
if (!group->ownedByCurrentThread() && group->ownerContext().context())
done = false;
}
if (done)
break;
CooperativeBeginWait(cx);
CooperativeYield();
CooperativeEndWait(cx);
}
cooperationState->singleThreaded = true;
}
static void
CooperativeEndSingleThreadedExecution(JSContext* cx)
{
if (cooperationState)
cooperationState->singleThreaded = false;
}
struct WorkerInput
{
JSContext* context;
@ -6271,6 +6436,16 @@ static const JSFunctionSpecWithHelp shell_functions[] = {
"evalInWorker(str)",
" Evaluate 'str' in a separate thread with its own runtime.\n"),
JS_FN_HELP("evalInCooperativeThread", EvalInCooperativeThread, 1, 0,
"evalInCooperativeThread(str)",
" Evaluate 'str' in a separate cooperatively scheduled thread using the same runtime.\n"),
JS_FN_HELP("cooperativeYield", CooperativeYieldThread, 1, 0,
"cooperativeYield(leaveZoneGroup)",
" Yield execution to another cooperatively scheduled thread using the same runtime.\n"
" If leaveZoneGroup is specified then other threads may execute code in the\n"
" current thread's zone group via evaluate(..., {zoneGroup:N}).\n"),
JS_FN_HELP("getSharedArrayBuffer", GetSharedArrayBuffer, 0, 0,
"getSharedArrayBuffer()",
" Retrieve the SharedArrayBuffer object from the cross-worker mailbox.\n"

2
js/src/vm/Caches.h
View file

@ -31,7 +31,7 @@ namespace js {
struct GSNCache {
typedef HashMap<jsbytecode*,
jssrcnote*,
PointerHasher<jsbytecode*, 0>,
PointerHasher<jsbytecode*>,
SystemAllocPolicy> Map;
jsbytecode* code;

30
js/src/vm/Debugger.cpp
View file

@ -2558,21 +2558,23 @@ UpdateExecutionObservabilityOfScriptsInZone(JSContext* cx, Zone* zone,
//
// Mark active baseline scripts in the observable set so that they don't
// get discarded. They will be recompiled.
for (JitActivationIterator actIter(rt); !actIter.done(); ++actIter) {
if (actIter->compartment()->zone() != zone)
continue;
for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) {
for (JitActivationIterator actIter(cx, target); !actIter.done(); ++actIter) {
if (actIter->compartment()->zone() != zone)
continue;
for (JitFrameIterator iter(actIter); !iter.done(); ++iter) {
switch (iter.type()) {
case JitFrame_BaselineJS:
MarkBaselineScriptActiveIfObservable(iter.script(), obs);
break;
case JitFrame_IonJS:
MarkBaselineScriptActiveIfObservable(iter.script(), obs);
for (InlineFrameIterator inlineIter(rt, &iter); inlineIter.more(); ++inlineIter)
MarkBaselineScriptActiveIfObservable(inlineIter.script(), obs);
break;
default:;
for (JitFrameIterator iter(actIter); !iter.done(); ++iter) {
switch (iter.type()) {
case JitFrame_BaselineJS:
MarkBaselineScriptActiveIfObservable(iter.script(), obs);
break;
case JitFrame_IonJS:
MarkBaselineScriptActiveIfObservable(iter.script(), obs);
for (InlineFrameIterator inlineIter(cx, &iter); inlineIter.more(); ++inlineIter)
MarkBaselineScriptActiveIfObservable(inlineIter.script(), obs);
break;
default:;
}
}
}
}

13
js/src/vm/ObjectGroup.cpp
View file

@ -412,12 +412,9 @@ struct ObjectGroupCompartment::NewEntry
}
static inline HashNumber hash(const Lookup& lookup) {
MOZ_ASSERT(lookup.proto.hasUniqueId());
MOZ_ASSERT(lookup.hasAssocId());
HashNumber hash = uintptr_t(lookup.clasp);
hash = mozilla::RotateLeft(hash, 4) ^ Zone::UniqueIdToHash(lookup.proto.uniqueId());
hash = mozilla::RotateLeft(hash, 4) ^ Zone::UniqueIdToHash(lookup.getAssocId());
return hash;
HashNumber hash = MovableCellHasher<TaggedProto>::hash(lookup.proto);
hash = mozilla::AddToHash(hash, MovableCellHasher<JSObject*>::hash(lookup.associated));
return mozilla::AddToHash(hash, mozilla::HashGeneric(lookup.clasp));
}
static inline bool match(const ObjectGroupCompartment::NewEntry& key, const Lookup& lookup) {
@ -1116,8 +1113,8 @@ struct ObjectGroupCompartment::PlainObjectKey
};
static inline HashNumber hash(const Lookup& lookup) {
return (HashNumber) (HashId(lookup.properties[lookup.nproperties - 1].id) ^
lookup.nproperties);
HashNumber hash = HashId(lookup.properties[lookup.nproperties - 1].id);
return mozilla::AddToHash(hash, lookup.nproperties);
}
static inline bool match(const PlainObjectKey& v, const Lookup& lookup) {

410
js/src/vm/RegExpShared.h Normal file
View file

@ -0,0 +1,410 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
* The compiled representation of a RegExp, potentially shared among RegExp instances created
* during separate evaluations of a single RegExp literal in source code.
*/
#ifndef vm_RegExpShared_h
#define vm_RegExpShared_h
#include "mozilla/Assertions.h"
#include "mozilla/MemoryReporting.h"
#include "jsalloc.h"
#include "jsatom.h"
#include "builtin/SelfHostingDefines.h"
#include "gc/Barrier.h"
#include "gc/Heap.h"
#include "gc/Marking.h"
#include "js/UbiNode.h"
#include "js/Vector.h"
#include "irregexp/InfallibleVector.h"
#include "vm/ArrayObject.h"
struct JSContext;
namespace js {
class ArrayObject;
class MatchPairs;
class RegExpCompartment;
class RegExpShared;
class RegExpStatics;
using RootedRegExpShared = JS::Rooted<RegExpShared*>;
using HandleRegExpShared = JS::Handle<RegExpShared*>;
using MutableHandleRegExpShared = JS::MutableHandle<RegExpShared*>;
enum RegExpFlag : uint8_t
{
IgnoreCaseFlag = 0x01,
GlobalFlag = 0x02,
MultilineFlag = 0x04,
StickyFlag = 0x08,
UnicodeFlag = 0x10,
DotAllFlag = 0x20,
HasIndicesFlag = 0x40,
NoFlags = 0x00,
AllFlags = 0x7f
};
static_assert(IgnoreCaseFlag == REGEXP_IGNORECASE_FLAG &&
GlobalFlag == REGEXP_GLOBAL_FLAG &&
MultilineFlag == REGEXP_MULTILINE_FLAG &&
StickyFlag == REGEXP_STICKY_FLAG &&
UnicodeFlag == REGEXP_UNICODE_FLAG &&
DotAllFlag == REGEXP_DOTALL_FLAG,
"Flag values should be in sync with self-hosted JS");
enum RegExpRunStatus
{
RegExpRunStatus_Error,
RegExpRunStatus_Success,
RegExpRunStatus_Success_NotFound
};
/*
* A RegExpShared is the compiled representation of a regexp. A RegExpShared is
* potentially pointed to by multiple RegExpObjects. Additionally, C++ code may
* have pointers to RegExpShareds on the stack. The RegExpShareds are kept in a
* table so that they can be reused when compiling the same regex string.
*
* To save memory, a RegExpShared is not created for a RegExpObject until it is
* needed for execution. When a RegExpShared needs to be created, it is looked
* up in a per-compartment table to allow reuse between objects.
*
* During a GC, RegExpShared instances are marked and swept like GC things.
* Usually, RegExpObjects clear their pointers to their RegExpShareds rather
* than explicitly tracing them, so that the RegExpShared and any jitcode can
* be reclaimed quicker. However, the RegExpShareds are traced through by
* objects when we are preserving jitcode in their zone, to avoid the same
* recompilation inefficiencies as normal Ion and baseline compilation.
*/
class RegExpShared : public gc::TenuredCell
{
public:
enum CompilationMode {
Normal,
MatchOnly
};
enum ForceByteCodeEnum {
DontForceByteCode,
ForceByteCode
};
using JitCodeTable = UniquePtr<uint8_t[], JS::FreePolicy>;
using JitCodeTables = Vector<JitCodeTable, 0, SystemAllocPolicy>;
private:
friend class RegExpStatics;
friend class RegExpZone;
struct RegExpCompilation
{
ReadBarriered<jit::JitCode*> jitCode;
uint8_t* byteCode;
RegExpCompilation() : byteCode(nullptr) {}
bool compiled(ForceByteCodeEnum force = DontForceByteCode) const {
return byteCode || (force == DontForceByteCode && jitCode);
}
};
/* Source to the RegExp, for lazy compilation. */
GCPtr<JSAtom*> source;
RegExpFlag flags;
bool canStringMatch;
size_t parenCount;
uint32_t numNamedCaptures_;
GCPtr<PlainObject*> groupsTemplate_;
RegExpCompilation compilationArray[4];
static int CompilationIndex(CompilationMode mode, bool latin1) {
switch (mode) {
case Normal: return latin1 ? 0 : 1;
case MatchOnly: return latin1 ? 2 : 3;
}
MOZ_CRASH();
}
// Tables referenced by JIT code.
JitCodeTables tables;
/* Internal functions. */
RegExpShared(JSAtom* source, RegExpFlag flags);
static bool compile(JSContext* cx, MutableHandleRegExpShared res, HandleLinearString input,
CompilationMode mode, ForceByteCodeEnum force);
static bool compile(JSContext* cx, MutableHandleRegExpShared res, HandleAtom pattern,
HandleLinearString input, CompilationMode mode, ForceByteCodeEnum force);
static bool compileIfNecessary(JSContext* cx, MutableHandleRegExpShared res,
HandleLinearString input, CompilationMode mode,
ForceByteCodeEnum force);
const RegExpCompilation& compilation(CompilationMode mode, bool latin1) const {
return compilationArray[CompilationIndex(mode, latin1)];
}
RegExpCompilation& compilation(CompilationMode mode, bool latin1) {
return compilationArray[CompilationIndex(mode, latin1)];
}
public:
~RegExpShared() = delete;
// Execute this RegExp on input starting from searchIndex, filling in
// matches if specified and otherwise only determining if there is a match.
static RegExpRunStatus execute(JSContext* cx, MutableHandleRegExpShared res,
HandleLinearString input, size_t searchIndex,
MatchPairs* matches, size_t* endIndex);
// Register a table with this RegExpShared, and take ownership.
bool addTable(JitCodeTable table) {
return tables.append(Move(table));
}
/* Accessors */
size_t getParenCount() const {
MOZ_ASSERT(isCompiled());
return parenCount;
}
/* Accounts for the "0" (whole match) pair. */
size_t pairCount() const { return getParenCount() + 1; }
// not public due to circular inclusion problems
static bool initializeNamedCaptures(JSContext* cx, MutableHandleRegExpShared re, irregexp::CharacterVectorVector* names, irregexp::IntegerVector* indices);
PlainObject* getGroupsTemplate() { return groupsTemplate_; }
uint32_t numNamedCaptures() const { return numNamedCaptures_; }
JSAtom* getSource() const { return source; }
RegExpFlag getFlags() const { return flags; }
bool hasIndices() const { return flags & HasIndicesFlag; }
bool global() const { return flags & GlobalFlag; }
bool ignoreCase() const { return flags & IgnoreCaseFlag; }
bool multiline() const { return flags & MultilineFlag; }
bool dotAll() const { return flags & DotAllFlag; }
bool unicode() const { return flags & UnicodeFlag; }
bool sticky() const { return flags & StickyFlag; }
bool isCompiled(CompilationMode mode, bool latin1,
ForceByteCodeEnum force = DontForceByteCode) const {
return compilation(mode, latin1).compiled(force);
}
bool isCompiled() const {
return isCompiled(Normal, true) || isCompiled(Normal, false)
|| isCompiled(MatchOnly, true) || isCompiled(MatchOnly, false);
}
void traceChildren(JSTracer* trc);
void discardJitCode();
void finalize(FreeOp* fop);
static size_t offsetOfSource() {
return offsetof(RegExpShared, source);
}
static size_t offsetOfFlags() {
return offsetof(RegExpShared, flags);
}
static size_t offsetOfParenCount() {
return offsetof(RegExpShared, parenCount);
}
static size_t offsetOfLatin1JitCode(CompilationMode mode) {
return offsetof(RegExpShared, compilationArray)
+ (CompilationIndex(mode, true) * sizeof(RegExpCompilation))
+ offsetof(RegExpCompilation, jitCode);
}
static size_t offsetOfTwoByteJitCode(CompilationMode mode) {
return offsetof(RegExpShared, compilationArray)
+ (CompilationIndex(mode, false) * sizeof(RegExpCompilation))
+ offsetof(RegExpCompilation, jitCode);
}
static size_t offsetOfGroupsTemplate() {
return offsetof(RegExpShared, groupsTemplate_);
}
size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
#ifdef DEBUG
static bool dumpBytecode(JSContext* cx, MutableHandleRegExpShared res, bool match_only,
HandleLinearString input);
#endif
};
class RegExpZone
{
struct Key {
JSAtom* atom;
uint16_t flag;
Key() {}
Key(JSAtom* atom, RegExpFlag flag)
: atom(atom), flag(flag)
{ }
MOZ_IMPLICIT Key(const ReadBarriered<RegExpShared*>& shared)
: atom(shared.unbarrieredGet()->getSource()),
flag(shared.unbarrieredGet()->getFlags())
{ }
typedef Key Lookup;
static HashNumber hash(const Lookup& l) {
HashNumber hash = DefaultHasher<JSAtom*>::hash(l.atom);
return mozilla::AddToHash(hash, l.flag);
}
static bool match(Key l, Key r) {
return l.atom == r.atom && l.flag == r.flag;
}
};
/*
* The set of all RegExpShareds in the zone. On every GC, every RegExpShared
* that was not marked is deleted and removed from the set.
*/
using Set = JS::WeakCache<JS::GCHashSet<ReadBarriered<RegExpShared*>, Key, ZoneAllocPolicy>>;
Set set_;
public:
explicit RegExpZone(Zone* zone);
~RegExpZone() {
MOZ_ASSERT_IF(set_.initialized(), set_.empty());
}
bool init();
bool empty() const { return set_.empty(); }
bool get(JSContext* cx, HandleAtom source, RegExpFlag flags, MutableHandleRegExpShared shared);
/* Like 'get', but compile 'maybeOpt' (if non-null). */
bool get(JSContext* cx, HandleAtom source, JSString* maybeOpt,
MutableHandleRegExpShared shared);
size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
};
class RegExpCompartment
{
public:
enum ResultTemplateKind { Normal, WithIndices, Indices, NumKinds };
private:
/*
* The template objects that the result of re.exec() is based on, if
* there is a result. These are used in CreateRegExpMatchResult.
* There are three template objects, each of which is an ArrayObject
* with some additional properties. We decide which to use based on
* the |hasIndices| (/d) flag.
*
* Normal: Has |index|, |input|, and |groups| properties.
* Used for the result object if |hasIndices| is not set.
*
* WithIndices: Has |index|, |input|, |groups|, and |indices| properties.
* Used for the result object if |hasIndices| is set.
*
* Indices: Has a |groups| property. If |hasIndices| is set, used
* for the |.indices| property of the result object.
*/
ReadBarriered<ArrayObject*> matchResultTemplateObjects_[ResultTemplateKind::NumKinds];
/*
* The shape of RegExp.prototype object that satisfies following:
* * RegExp.prototype.flags getter is not modified
* * RegExp.prototype.global getter is not modified
* * RegExp.prototype.ignoreCase getter is not modified
* * RegExp.prototype.multiline getter is not modified
* * RegExp.prototype.sticky getter is not modified
* * RegExp.prototype.unicode getter is not modified
* * RegExp.prototype.exec is an own data property
* * RegExp.prototype[@@match] is an own data property
* * RegExp.prototype[@@search] is an own data property
*/
ReadBarriered<Shape*> optimizableRegExpPrototypeShape_;
/*
* The shape of RegExp instance that satisfies following:
* * lastProperty is lastIndex
* * prototype is RegExp.prototype
*/
ReadBarriered<Shape*> optimizableRegExpInstanceShape_;
ArrayObject* createMatchResultTemplateObject(JSContext* cx, ResultTemplateKind kind);
public:
explicit RegExpCompartment(Zone* zone);
void sweep(JSRuntime* rt);
/* Get or create template object used to base the result of .exec() on. */
ArrayObject* getOrCreateMatchResultTemplateObject(JSContext* cx, ResultTemplateKind kind = ResultTemplateKind::Normal) {
if (matchResultTemplateObjects_[kind])
return matchResultTemplateObjects_[kind];
return createMatchResultTemplateObject(cx, kind);
}
Shape* getOptimizableRegExpPrototypeShape() {
return optimizableRegExpPrototypeShape_;
}
void setOptimizableRegExpPrototypeShape(Shape* shape) {
optimizableRegExpPrototypeShape_ = shape;
}
Shape* getOptimizableRegExpInstanceShape() {
return optimizableRegExpInstanceShape_;
}
void setOptimizableRegExpInstanceShape(Shape* shape) {
optimizableRegExpInstanceShape_ = shape;
}
static size_t offsetOfOptimizableRegExpPrototypeShape() {
return offsetof(RegExpCompartment, optimizableRegExpPrototypeShape_);
}
static size_t offsetOfOptimizableRegExpInstanceShape() {
return offsetof(RegExpCompartment, optimizableRegExpInstanceShape_);
}
};
} /* namespace js */
namespace JS {
namespace ubi {
template <>
class Concrete<js::RegExpShared> : TracerConcrete<js::RegExpShared>
{
protected:
explicit Concrete(js::RegExpShared* ptr) : TracerConcrete<js::RegExpShared>(ptr) { }
public:
static void construct(void* storage, js::RegExpShared* ptr) {
new (storage) Concrete(ptr);
}
CoarseType coarseType() const final { return CoarseType::Other; }
Size size(mozilla::MallocSizeOf mallocSizeOf) const override;
const char16_t* typeName() const override { return concreteTypeName; }
static const char16_t concreteTypeName[];
};
} // namespace ubi
} // namespace JS
#endif /* vm_RegExpShared_h */

2
js/src/vm/Runtime.h
View file

@ -87,7 +87,7 @@ namespace js {
extern MOZ_COLD void
ReportOutOfMemory(ExclusiveContext* cx);
/* Different signature because the return type has MOZ_MUST_USE_TYPE. */
/* Different signature because the return type has [[nodiscard]]_TYPE. */
extern MOZ_COLD mozilla::GenericErrorResult<OOM&>
ReportOutOfMemoryResult(ExclusiveContext* cx);

2
js/src/vm/SavedFrame.h
View file

@ -187,7 +187,7 @@ struct SavedFrame::HashPolicy
{
typedef SavedFrame::Lookup Lookup;
typedef MovableCellHasher<SavedFrame*> SavedFramePtrHasher;
typedef PointerHasher<JSPrincipals*, 3> JSPrincipalsPtrHasher;
typedef PointerHasher<JSPrincipals*> JSPrincipalsPtrHasher;
static bool hasHash(const Lookup& l);
static bool ensureHash(const Lookup& l);

441
js/src/vm/Shape.cpp
View file

@ -1504,6 +1504,447 @@ EmptyShape::new_(ExclusiveContext* cx, Handle<UnownedBaseShape*> base, uint32_t
return shape;
}
MOZ_ALWAYS_INLINE HashNumber
ShapeHasher::hash(const Lookup& l)
{
return l.hash();
}
MOZ_ALWAYS_INLINE bool
ShapeHasher::match(const Key k, const Lookup& l)
{
return k->matches(l);
}
static KidsHash*
HashChildren(Shape* kid1, Shape* kid2)
{
KidsHash* hash = js_new<KidsHash>();
if (!hash || !hash->init(2)) {
js_delete(hash);
return nullptr;
}
hash->putNewInfallible(StackShape(kid1), kid1);
hash->putNewInfallible(StackShape(kid2), kid2);
return hash;
}
bool
PropertyTree::insertChild(JSContext* cx, Shape* parent, Shape* child)
{
MOZ_ASSERT(!parent->inDictionary());
MOZ_ASSERT(!child->parent);
MOZ_ASSERT(!child->inDictionary());
MOZ_ASSERT(child->zone() == parent->zone());
MOZ_ASSERT(cx->zone() == zone_);
KidsPointer* kidp = &parent->kids;
if (kidp->isNull()) {
child->setParent(parent);
kidp->setShape(child);
return true;
}
if (kidp->isShape()) {
Shape* shape = kidp->toShape();
MOZ_ASSERT(shape != child);
MOZ_ASSERT(!shape->matches(child));
KidsHash* hash = HashChildren(shape, child);
if (!hash) {
ReportOutOfMemory(cx);
return false;
}
kidp->setHash(hash);
child->setParent(parent);
return true;
}
if (!kidp->toHash()->putNew(StackShape(child), child)) {
ReportOutOfMemory(cx);
return false;
}
child->setParent(parent);
return true;
}
void
Shape::removeChild(Shape* child)
{
MOZ_ASSERT(!child->inDictionary());
MOZ_ASSERT(child->parent == this);
KidsPointer* kidp = &kids;
if (kidp->isShape()) {
MOZ_ASSERT(kidp->toShape() == child);
kidp->setNull();
child->parent = nullptr;
return;
}
KidsHash* hash = kidp->toHash();
MOZ_ASSERT(hash->count() >= 2); /* otherwise kidp->isShape() should be true */
#ifdef DEBUG
size_t oldCount = hash->count();
#endif
hash->remove(StackShape(child));
child->parent = nullptr;
MOZ_ASSERT(hash->count() == oldCount - 1);
if (hash->count() == 1) {
/* Convert from HASH form back to SHAPE form. */
KidsHash::Range r = hash->all();
Shape* otherChild = r.front();
MOZ_ASSERT((r.popFront(), r.empty())); /* No more elements! */
kidp->setShape(otherChild);
js_delete(hash);
}
}
MOZ_ALWAYS_INLINE Shape*
PropertyTree::inlinedGetChild(JSContext* cx, Shape* parent, Handle<StackShape> child)
{
MOZ_ASSERT(parent);
Shape* existingShape = nullptr;
/*
* The property tree has extremely low fan-out below its root in
* popular embeddings with real-world workloads. Patterns such as
* defining closures that capture a constructor's environment as
* getters or setters on the new object that is passed in as
* |this| can significantly increase fan-out below the property
* tree root -- see bug 335700 for details.
*/
KidsPointer* kidp = &parent->kids;
if (kidp->isShape()) {
Shape* kid = kidp->toShape();
if (kid->matches(child))
existingShape = kid;
} else if (kidp->isHash()) {
if (KidsHash::Ptr p = kidp->toHash()->lookup(child))
existingShape = *p;
} else {
/* If kidp->isNull(), we always insert. */
}
if (existingShape) {
JS::Zone* zone = existingShape->zone();
if (zone->needsIncrementalBarrier()) {
/*
* We need a read barrier for the shape tree, since these are weak
* pointers.
*/
Shape* tmp = existingShape;
TraceManuallyBarrieredEdge(zone->barrierTracer(), &tmp, "read barrier");
MOZ_ASSERT(tmp == existingShape);
return existingShape;
}
if (!zone->isGCSweepingOrCompacting() ||
!IsAboutToBeFinalizedUnbarriered(&existingShape))
{
if (existingShape->isMarkedGray())
UnmarkGrayShapeRecursively(existingShape);
return existingShape;
}
/*
* The shape we've found is unreachable and due to be finalized, so
* remove our weak reference to it and don't use it.
*/
MOZ_ASSERT(parent->isMarkedAny());
parent->removeChild(existingShape);
}
RootedShape parentRoot(cx, parent);
Shape* shape = Shape::new_(cx, child, parentRoot->numFixedSlots());
if (!shape)
return nullptr;
if (!insertChild(cx, parentRoot, shape))
return nullptr;
return shape;
}
Shape*
PropertyTree::getChild(JSContext* cx, Shape* parent, Handle<StackShape> child)
{
return inlinedGetChild(cx, parent, child);
}
void
Shape::sweep()
{
/*
* We detach the child from the parent if the parent is reachable.
*
* This test depends on shape arenas not being freed until after we finish
* incrementally sweeping them. If that were not the case the parent pointer
* could point to a marked cell that had been deallocated and then
* reallocated, since allocating a cell in a zone that is being marked will
* set the mark bit for that cell.
*/
if (parent && parent->isMarkedAny()) {
if (inDictionary()) {
if (parent->listp == &parent)
parent->listp = nullptr;
} else {
parent->removeChild(this);
}
}
}
void
Shape::finalize(FreeOp* fop)
{
if (!inDictionary() && kids.isHash())
fop->delete_(kids.toHash());
}
void
Shape::fixupDictionaryShapeAfterMovingGC()
{
if (!listp)
return;
// The listp field either points to the parent field of the next shape in
// the list if there is one. Otherwise if this shape is the last in the
// list then it points to the shape_ field of the object the list is for.
// We can tell which it is because the base shape is owned if this is the
// last property and not otherwise.
bool listpPointsIntoShape = !MaybeForwarded(base())->isOwned();
#ifdef DEBUG
// Check that we got this right by interrogating the arena.
// We use a fake cell pointer for this: it might not point to the beginning
// of a cell, but will point into the right arena and will have the right
// alignment.
Cell* cell = reinterpret_cast<Cell*>(uintptr_t(listp) & ~CellAlignMask);
AllocKind kind = TenuredCell::fromPointer(cell)->getAllocKind();
MOZ_ASSERT_IF(listpPointsIntoShape, IsShapeAllocKind(kind));
MOZ_ASSERT_IF(!listpPointsIntoShape, IsObjectAllocKind(kind));
#endif
if (listpPointsIntoShape) {
// listp points to the parent field of the next shape.
Shape* next = reinterpret_cast<Shape*>(uintptr_t(listp) - offsetof(Shape, parent));
if (gc::IsForwarded(next))
listp = &gc::Forwarded(next)->parent;
} else {
// listp points to the shape_ field of an object.
JSObject* last = reinterpret_cast<JSObject*>(uintptr_t(listp) - ShapedObject::offsetOfShape());
if (gc::IsForwarded(last))
listp = &gc::Forwarded(last)->as<NativeObject>().shape_;
}
}
void
Shape::fixupShapeTreeAfterMovingGC()
{
if (kids.isNull())
return;
if (kids.isShape()) {
if (gc::IsForwarded(kids.toShape()))
kids.setShape(gc::Forwarded(kids.toShape()));
return;
}
MOZ_ASSERT(kids.isHash());
KidsHash* kh = kids.toHash();
for (KidsHash::Enum e(*kh); !e.empty(); e.popFront()) {
Shape* key = e.front();
if (IsForwarded(key))
key = Forwarded(key);
BaseShape* base = key->base();
if (IsForwarded(base))
base = Forwarded(base);
UnownedBaseShape* unowned = base->unowned();
if (IsForwarded(unowned))
unowned = Forwarded(unowned);
GetterOp getter = key->getter();
if (key->hasGetterObject())
getter = GetterOp(MaybeForwarded(key->getterObject()));
SetterOp setter = key->setter();
if (key->hasSetterObject())
setter = SetterOp(MaybeForwarded(key->setterObject()));
StackShape lookup(unowned,
const_cast<Shape*>(key)->propidRef(),
key->slotInfo & Shape::SLOT_MASK,
key->attrs,
key->flags);
lookup.updateGetterSetter(getter, setter);
e.rekeyFront(lookup, key);
}
}
void
Shape::fixupAfterMovingGC()
{
if (inDictionary())
fixupDictionaryShapeAfterMovingGC();
else
fixupShapeTreeAfterMovingGC();
}
void
NurseryShapesRef::trace(JSTracer* trc)
{
auto& shapes = zone_->nurseryShapes();
for (auto shape : shapes)
shape->fixupGetterSetterForBarrier(trc);
shapes.clearAndFree();
}
void
Shape::fixupGetterSetterForBarrier(JSTracer* trc)
{
if (!hasGetterValue() && !hasSetterValue())
return;
JSObject* priorGetter = asAccessorShape().getterObj;
JSObject* priorSetter = asAccessorShape().setterObj;
if (!priorGetter && !priorSetter)
return;
JSObject* postGetter = priorGetter;
JSObject* postSetter = priorSetter;
if (priorGetter)
TraceManuallyBarrieredEdge(trc, &postGetter, "getterObj");
if (priorSetter)
TraceManuallyBarrieredEdge(trc, &postSetter, "setterObj");
if (priorGetter == postGetter && priorSetter == postSetter)
return;
if (parent && !parent->inDictionary() && parent->kids.isHash()) {
// Relocating the getterObj or setterObj will have changed our location
// in our parent's KidsHash, so take care to update it. We must do this
// before we update the shape itself, since the shape is used to match
// the original entry in the hash set.
StackShape original(this);
StackShape updated(this);
updated.rawGetter = reinterpret_cast<GetterOp>(postGetter);
updated.rawSetter = reinterpret_cast<SetterOp>(postSetter);
KidsHash* kh = parent->kids.toHash();
MOZ_ALWAYS_TRUE(kh->rekeyAs(original, updated, this));
}
asAccessorShape().getterObj = postGetter;
asAccessorShape().setterObj = postSetter;
MOZ_ASSERT_IF(parent && !parent->inDictionary() && parent->kids.isHash(),
parent->kids.toHash()->has(StackShape(this)));
}
#ifdef DEBUG
void
KidsPointer::checkConsistency(Shape* aKid) const
{
if (isShape()) {
MOZ_ASSERT(toShape() == aKid);
} else {
MOZ_ASSERT(isHash());
KidsHash* hash = toHash();
KidsHash::Ptr ptr = hash->lookup(StackShape(aKid));
MOZ_ASSERT(*ptr == aKid);
}
}
void
Shape::dump(FILE* fp) const
{
jsid propid = this->propid();
MOZ_ASSERT(!JSID_IS_VOID(propid));
if (JSID_IS_INT(propid)) {
fprintf(fp, "[%ld]", (long) JSID_TO_INT(propid));
} else if (JSID_IS_ATOM(propid)) {
if (JSLinearString* str = JSID_TO_ATOM(propid))
FileEscapedString(fp, str, '"');
else
fputs("<error>", fp);
} else {
MOZ_ASSERT(JSID_IS_SYMBOL(propid));
JSID_TO_SYMBOL(propid)->dump(fp);
}
fprintf(fp, " g/s %p/%p slot %d attrs %x ",
JS_FUNC_TO_DATA_PTR(void*, getter()),
JS_FUNC_TO_DATA_PTR(void*, setter()),
hasSlot() ? slot() : -1, attrs);
if (attrs) {
int first = 1;
fputs("(", fp);
#define DUMP_ATTR(name, display) if (attrs & JSPROP_##name) fputs(&(" " #display)[first], fp), first = 0
DUMP_ATTR(ENUMERATE, enumerate);
DUMP_ATTR(READONLY, readonly);
DUMP_ATTR(PERMANENT, permanent);
DUMP_ATTR(GETTER, getter);
DUMP_ATTR(SETTER, setter);
DUMP_ATTR(SHARED, shared);
#undef DUMP_ATTR
fputs(") ", fp);
}
fprintf(fp, "flags %x ", flags);
if (flags) {
int first = 1;
fputs("(", fp);
#define DUMP_FLAG(name, display) if (flags & name) fputs(&(" " #display)[first], fp), first = 0
DUMP_FLAG(IN_DICTIONARY, in_dictionary);
#undef DUMP_FLAG
fputs(") ", fp);
}
}
void
Shape::dumpSubtree(int level, FILE* fp) const
{
if (!parent) {
MOZ_ASSERT(level == 0);
MOZ_ASSERT(JSID_IS_EMPTY(propid_));
fprintf(fp, "class %s emptyShape\n", getObjectClass()->name);
} else {
fprintf(fp, "%*sid ", level, "");
dump(fp);
}
if (!kids.isNull()) {
++level;
if (kids.isShape()) {
Shape* kid = kids.toShape();
MOZ_ASSERT(kid->parent == this);
kid->dumpSubtree(level, fp);
} else {
const KidsHash& hash = *kids.toHash();
for (KidsHash::Range range = hash.all(); !range.empty(); range.popFront()) {
Shape* kid = range.front();
MOZ_ASSERT(kid->parent == this);
kid->dumpSubtree(level, fp);
}
}
}
}
#endif
static bool
IsOriginalProto(GlobalObject* global, JSProtoKey key, JSObject& proto)
{

53
js/src/vm/Shape.h
View file

@ -8,6 +8,7 @@
#include "mozilla/Attributes.h"
#include "mozilla/GuardObjects.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/Maybe.h"
#include "mozilla/MemoryReporting.h"
@ -560,8 +561,13 @@ struct StackBaseShape : public DefaultHasher<ReadBarriered<UnownedBaseShape*>>
}
};
static inline HashNumber hash(const Lookup& lookup);
static inline bool match(ReadBarriered<UnownedBaseShape*> key, const Lookup& lookup);
static HashNumber hash(const Lookup& lookup) {
return mozilla::HashGeneric(lookup.flags, lookup.clasp);
}
static inline bool match(const ReadBarriered<UnownedBaseShape*>& key, const Lookup& lookup) {
return key.unbarrieredGet()->flags == lookup.flags &&
key.unbarrieredGet()->clasp_ == lookup.clasp;
}
};
static MOZ_ALWAYS_INLINE js::HashNumber
@ -1220,6 +1226,32 @@ class InitialShapeProto
void setProto(TaggedProto proto) {
proto_ = proto;
}
bool operator==(const InitialShapeProto& other) const {
return key_ == other.key_ && proto_ == other.proto_;
}
};
template <>
struct MovableCellHasher<InitialShapeProto<ReadBarriered<TaggedProto>>>
{
using Key = InitialShapeProto<ReadBarriered<TaggedProto>>;
using Lookup = InitialShapeProto<TaggedProto>;
static bool hasHash(const Lookup& l) {
return MovableCellHasher<TaggedProto>::hasHash(l.proto());
}
static bool ensureHash(const Lookup& l) {
return MovableCellHasher<TaggedProto>::ensureHash(l.proto());
}
static HashNumber hash(const Lookup& l) {
HashNumber hash = MovableCellHasher<TaggedProto>::hash(l.proto());
return mozilla::AddToHash(hash, l.key());
}
static bool match(const Key& k, const Lookup& l) {
return k.key() == l.key() &&
MovableCellHasher<TaggedProto>::match(k.proto().unbarrieredGet(), l.proto());
}
};
/*
@ -1258,9 +1290,20 @@ struct InitialShapeEntry
inline InitialShapeEntry();
inline InitialShapeEntry(Shape* shape, const Lookup::ShapeProto& proto);
static inline HashNumber hash(const Lookup& lookup);
static inline bool match(const InitialShapeEntry& key, const Lookup& lookup);
static void rekey(InitialShapeEntry& k, const InitialShapeEntry& newKey) { k = newKey; }
static HashNumber hash(const Lookup& lookup) {
HashNumber hash = MovableCellHasher<ShapeProto>::hash(lookup.proto);
return mozilla::AddToHash(hash, mozilla::HashGeneric(lookup.clasp, lookup.nfixed));
}
static inline bool match(const InitialShapeEntry& key, const Lookup& lookup) {
const Shape* shape = key.shape.unbarrieredGet();
return lookup.clasp == shape->getObjectClass()
&& lookup.nfixed == shape->numFixedSlots()
&& lookup.baseFlags == shape->getObjectFlags()
&& MovableCellHasher<ShapeProto>::match(key.proto, lookup.proto);
}
static void rekey(InitialShapeEntry& k, const InitialShapeEntry& newKey) {
k = newKey;
}
bool needsSweep() {
Shape* ushape = shape.unbarrieredGet();

25
js/src/vm/Stack.cpp
View file

@ -598,6 +598,21 @@ FrameIter::Data::Data(JSContext* cx, DebuggerEvalOption debuggerEvalOption,
{
}
FrameIter::Data::Data(JSContext* cx, const CooperatingContext& target,
DebuggerEvalOption debuggerEvalOption)
: cx_(cx),
debuggerEvalOption_(debuggerEvalOption),
principals_(nullptr),
state_(DONE),
pc_(nullptr),
interpFrames_(nullptr),
activations_(cx, target),
jitFrames_(),
ionInlineFrameNo_(0),
wasmFrames_()
{
}
FrameIter::Data::Data(const FrameIter::Data& other)
: cx_(other.cx_),
debuggerEvalOption_(other.debuggerEvalOption_),
@ -612,6 +627,16 @@ FrameIter::Data::Data(const FrameIter::Data& other)
{
}
FrameIter::FrameIter(JSContext* cx, const CooperatingContext& target,
DebuggerEvalOption debuggerEvalOption)
: data_(cx, target, debuggerEvalOption),
ionInlineFrames_(cx, (js::jit::JitFrameIterator*) nullptr)
{
// settleOnActivation can only GC if principals are given.
JS::AutoSuppressGCAnalysis nogc;
settleOnActivation();
}
FrameIter::FrameIter(JSContext* cx, DebuggerEvalOption debuggerEvalOption)
: data_(cx, debuggerEvalOption, nullptr),
ionInlineFrames_(cx, (js::jit::JitFrameIterator*) nullptr)

17
js/src/vm/Stack.h
View file

@ -7,6 +7,7 @@
#define vm_Stack_h
#include "mozilla/Atomics.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/Maybe.h"
#include "mozilla/MemoryReporting.h"
#include "mozilla/Variant.h"
@ -1087,7 +1088,7 @@ struct DefaultHasher<AbstractFramePtr> {
typedef AbstractFramePtr Lookup;
static js::HashNumber hash(const Lookup& key) {
return size_t(key.raw());
return mozilla::HashGeneric(key.raw());
}
static bool match(const AbstractFramePtr& k, const Lookup& l) {
@ -1787,11 +1788,13 @@ class FrameIter
wasm::FrameIterator wasmFrames_;
Data(JSContext* cx, DebuggerEvalOption debuggerEvalOption, JSPrincipals* principals);
Data(JSContext* cx, const CooperatingContext& target, DebuggerEvalOption debuggerEvalOption);
Data(const Data& other);
};
explicit FrameIter(JSContext* cx,
DebuggerEvalOption = FOLLOW_DEBUGGER_EVAL_PREV_LINK);
FrameIter(JSContext* cx, const CooperatingContext&, DebuggerEvalOption);
FrameIter(JSContext* cx, DebuggerEvalOption, JSPrincipals*);
FrameIter(const FrameIter& iter);
MOZ_IMPLICIT FrameIter(const Data& data);
@ -1947,6 +1950,14 @@ class ScriptFrameIter : public FrameIter
settle();
}
ScriptFrameIter(JSContext* cx,
const CooperatingContext& target,
DebuggerEvalOption debuggerEvalOption)
: FrameIter(cx, target, debuggerEvalOption)
{
settle();
}
ScriptFrameIter(JSContext* cx,
DebuggerEvalOption debuggerEvalOption,
JSPrincipals* prin)
@ -2069,6 +2080,10 @@ class AllScriptFramesIter : public ScriptFrameIter
explicit AllScriptFramesIter(JSContext* cx)
: ScriptFrameIter(cx, ScriptFrameIter::IGNORE_DEBUGGER_EVAL_PREV_LINK)
{}
explicit AllScriptFramesIter(JSContext* cx, const CooperatingContext& target)
: ScriptFrameIter(cx, target, ScriptFrameIter::IGNORE_DEBUGGER_EVAL_PREV_LINK)
{}
};
/* Popular inline definitions. */

15
js/src/vm/TraceLogging.h
View file

@ -328,8 +328,19 @@ class TraceLoggerThreadState
bool offThreadEnabled;
bool graphSpewingEnabled;
bool spewErrors;
ThreadLoggerHashMap threadLoggers;
mozilla::LinkedList<TraceLoggerMainThread> traceLoggerMainThreadList;
mozilla::LinkedList<TraceLoggerThread> threadLoggers;
typedef HashMap<const void*,
TraceLoggerEventPayload*,
PointerHasher<const void*>,
SystemAllocPolicy> PointerHashMap;
typedef HashMap<uint32_t,
TraceLoggerEventPayload*,
DefaultHasher<uint32_t>,
SystemAllocPolicy> TextIdHashMap;
PointerHashMap pointerMap;
TextIdHashMap textIdPayloads;
uint32_t nextTextId;
public:
uint64_t startupTime;

148
js/src/vm/TypeInference.cpp
View file

@ -2963,7 +2963,7 @@ ObjectGroup::maybeClearNewScriptOnOOM()
{
MOZ_ASSERT(zone()->isGCSweepingOrCompacting());
if (!isMarked())
if (!isMarkedAny())
return;
TypeNewScript* newScript = anyNewScript();
@ -4002,97 +4002,91 @@ TypeNewScript::rollbackPartiallyInitializedObjects(JSContext* cx, ObjectGroup* g
RootedFunction function(cx, this->function());
Vector<uint32_t, 32> pcOffsets(cx);
for (ScriptFrameIter iter(cx); !iter.done(); ++iter) {
{
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!pcOffsets.append(iter.script()->pcToOffset(iter.pc())))
oomUnsafe.crash("rollbackPartiallyInitializedObjects");
}
JSRuntime::AutoProhibitActiveContextChange apacc(cx->runtime());
for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) {
for (AllScriptFramesIter iter(cx, target); !iter.done(); ++iter) {
{
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!pcOffsets.append(iter.script()->pcToOffset(iter.pc())))
oomUnsafe.crash("rollbackPartiallyInitializedObjects");
}
if (!iter.isConstructing()) {
continue;
}
if (!iter.isConstructing() || !iter.matchCallee(cx, function))
continue;
MOZ_ASSERT(iter.calleeTemplate()->maybeCanonicalFunction());
// Derived class constructors initialize their this-binding later and
// we shouldn't run the definite properties analysis on them.
MOZ_ASSERT(!iter.script()->isDerivedClassConstructor());
if (iter.calleeTemplate()->maybeCanonicalFunction() != function) {
continue;
}
Value thisv = iter.thisArgument(cx);
if (!thisv.isObject() ||
thisv.toObject().hasLazyGroup() ||
thisv.toObject().group() != group)
{
continue;
}
// Derived class constructors initialize their this-binding later and
// we shouldn't run the definite properties analysis on them.
MOZ_ASSERT(!iter.script()->isDerivedClassConstructor());
if (thisv.toObject().is<UnboxedPlainObject>()) {
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!UnboxedPlainObject::convertToNative(cx, &thisv.toObject()))
oomUnsafe.crash("rollbackPartiallyInitializedObjects");
}
Value thisv = iter.thisArgument(cx);
if (!thisv.isObject() ||
thisv.toObject().hasLazyGroup() ||
thisv.toObject().group() != group)
{
continue;
}
// Found a matching frame.
RootedPlainObject obj(cx, &thisv.toObject().as<PlainObject>());
if (thisv.toObject().is<UnboxedPlainObject>()) {
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!UnboxedPlainObject::convertToNative(cx, &thisv.toObject()))
oomUnsafe.crash("rollbackPartiallyInitializedObjects");
}
// Whether all identified 'new' properties have been initialized.
bool finished = false;
// Found a matching frame.
RootedPlainObject obj(cx, &thisv.toObject().as<PlainObject>());
// If not finished, number of properties that have been added.
uint32_t numProperties = 0;
// Whether all identified 'new' properties have been initialized.
bool finished = false;
// Whether the current SETPROP is within an inner frame which has
// finished entirely.
bool pastProperty = false;
// If not finished, number of properties that have been added.
uint32_t numProperties = 0;
// Index in pcOffsets of the outermost frame.
int callDepth = pcOffsets.length() - 1;
// Whether the current SETPROP is within an inner frame which has
// finished entirely.
bool pastProperty = false;
// Index in pcOffsets of the frame currently being checked for a SETPROP.
int setpropDepth = callDepth;
// Index in pcOffsets of the outermost frame.
int callDepth = pcOffsets.length() - 1;
// Index in pcOffsets of the frame currently being checked for a SETPROP.
int setpropDepth = callDepth;
for (Initializer* init = initializerList;; init++) {
if (init->kind == Initializer::SETPROP) {
if (!pastProperty && pcOffsets[setpropDepth] < init->offset) {
// Have not yet reached this setprop.
for (Initializer* init = initializerList;; init++) {
if (init->kind == Initializer::SETPROP) {
if (!pastProperty && pcOffsets[setpropDepth] < init->offset) {
// Have not yet reached this setprop.
break;
}
// This setprop has executed, reset state for the next one.
numProperties++;
pastProperty = false;
setpropDepth = callDepth;
} else if (init->kind == Initializer::SETPROP_FRAME) {
if (!pastProperty) {
if (pcOffsets[setpropDepth] < init->offset) {
// Have not yet reached this inner call.
break;
} else if (pcOffsets[setpropDepth] > init->offset) {
// Have advanced past this inner call.
pastProperty = true;
} else if (setpropDepth == 0) {
// Have reached this call but not yet in it.
break;
} else {
// Somewhere inside this inner call.
setpropDepth--;
}
}
} else {
MOZ_ASSERT(init->kind == Initializer::DONE);
finished = true;
break;
}
// This setprop has executed, reset state for the next one.
numProperties++;
pastProperty = false;
setpropDepth = callDepth;
} else if (init->kind == Initializer::SETPROP_FRAME) {
if (!pastProperty) {
if (pcOffsets[setpropDepth] < init->offset) {
// Have not yet reached this inner call.
break;
} else if (pcOffsets[setpropDepth] > init->offset) {
// Have advanced past this inner call.
pastProperty = true;
} else if (setpropDepth == 0) {
// Have reached this call but not yet in it.
break;
} else {
// Somewhere inside this inner call.
setpropDepth--;
}
}
} else {
MOZ_ASSERT(init->kind == Initializer::DONE);
finished = true;
break;
if (!finished) {
(void) NativeObject::rollbackProperties(cx, obj, numProperties);
found = true;
}
}
if (!finished) {
(void) NativeObject::rollbackProperties(cx, obj, numProperties);
found = true;
}
}
return found;

6
js/xpconnect/src/XPCJSContext.cpp
View file

@ -827,7 +827,7 @@ XPCJSContext::FinalizeCallback(JSFreeOp* fop,
}
/* static */ void
XPCJSContext::WeakPointerZoneGroupCallback(JSContext* cx, void* data)
XPCJSContext::WeakPointerZonesCallback(JSContext* cx, void* data)
{
// Called before each sweeping slice -- after processing any final marking
// triggered by barriers -- to clear out any references to things that are
@ -1514,7 +1514,7 @@ XPCJSContext::~XPCJSContext()
// callbacks if we aren't careful. Null out the relevant callbacks.
js::SetActivityCallback(Context(), nullptr, nullptr);
JS_RemoveFinalizeCallback(Context(), FinalizeCallback);
JS_RemoveWeakPointerZoneGroupCallback(Context(), WeakPointerZoneGroupCallback);
JS_RemoveWeakPointerZonesCallback(Context(), WeakPointerZonesCallback);
JS_RemoveWeakPointerCompartmentCallback(Context(), WeakPointerCompartmentCallback);
// Clear any pending exception. It might be an XPCWrappedJS, and if we try
@ -3357,7 +3357,7 @@ XPCJSContext::Initialize()
mPrevDoCycleCollectionCallback = JS::SetDoCycleCollectionCallback(cx,
DoCycleCollectionCallback);
JS_AddFinalizeCallback(cx, FinalizeCallback, nullptr);
JS_AddWeakPointerZoneGroupCallback(cx, WeakPointerZoneGroupCallback, this);
JS_AddWeakPointerZonesCallback(cx, WeakPointerZonesCallback, this);
JS_AddWeakPointerCompartmentCallback(cx, WeakPointerCompartmentCallback, this);
JS_SetWrapObjectCallbacks(cx, &WrapObjectCallbacks);
js::SetPreserveWrapperCallback(cx, PreserveWrapper);

6
js/xpconnect/src/xpcprivate.h
View file

@ -540,7 +540,7 @@ public:
JSFinalizeStatus status,
bool isZoneGC,
void* data);
static void WeakPointerZoneGroupCallback(JSContext* cx, void* data);
static void WeakPointerZonesCallback(JSContext* cx, void* data);
static void WeakPointerCompartmentCallback(JSContext* cx, JSCompartment* comp, void* data);
inline void AddVariantRoot(XPCTraceableVariant* variant);
@ -953,11 +953,11 @@ public:
typedef js::HashMap<JSAddonId*,
nsCOMPtr<nsIAddonInterposition>,
js::PointerHasher<JSAddonId*, 3>,
js::PointerHasher<JSAddonId*>,
js::SystemAllocPolicy> InterpositionMap;
typedef js::HashSet<JSAddonId*,
js::PointerHasher<JSAddonId*, 3>,
js::PointerHasher<JSAddonId*>,
js::SystemAllocPolicy> AddonSet;
// Gets the appropriate scope object for XBL in this scope. The context