wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-08 00:38:39 +09:00
Code Issues Projects Releases Packages Wiki Activity

903519 part 6 to 13 (partially)

Browse source 
903519 Part 6: Strings in the nursery: tracing and tenuring.

903519 Part 7: Strings in the nursery: barriers.

903519 Part 8: Discard nursery keys from EvalCache.

903519 Part 9:  Strings in the nursery: allow any thread to access zone of permanent atoms.

903519 Part 10: Strings in the nursery: MIR node.

903519 Part 11: Change Relocated marker to not confuse string vs object bit.

903519 Part 12: Default nursery strings to off, add ability to enable.

903519 Part 13: Strings in the nursery: JIT, partial due CodeGenerator.cpp differences startin from: // Follow any base pointer if the input is itself a dependent string.

This is due file changed: 1434230: Spectre mitigations for strings
This commit is contained in:
win7-7 2024-01-29 23:31:35 +02:00 • committed by wuggy
commit 1154dbddf6
31 changed files with 718 additions and 121 deletions
Download patch file Download diff file Expand all files Collapse all files

19
js/public/HeapAPI.h
View file

@ -23,6 +23,12 @@ namespace gc {
struct Cell;
/*
* The low bit is set so this should never equal a normal pointer, and the high
* bit is set so this should never equal the upper 32 bits of a 64-bit pointer.
*/
const uint32_t Relocated = uintptr_t(0xbad0bad1);
const size_t ArenaShift = 12;
const size_t ArenaSize = size_t(1) << ArenaShift;
const size_t ArenaMask = ArenaSize - 1;
@ -411,10 +417,15 @@ GetTenuredGCThingZone(GCCellPtr thing)
return js::gc::detail::GetGCThingZone(thing.unsafeAsUIntPtr());
}
extern JS_PUBLIC_API(Zone*)
GetNurseryStringZone(JSString* str);
static MOZ_ALWAYS_INLINE Zone*
GetStringZone(JSString* str)
{
return js::gc::detail::GetGCThingZone(uintptr_t(str));
if (!js::gc::IsInsideNursery(reinterpret_cast<js::gc::Cell*>(str)))
return js::gc::detail::GetGCThingZone(reinterpret_cast<uintptr_t>(str));
return GetNurseryStringZone(str);
}
extern JS_PUBLIC_API(Zone*)
@ -431,6 +442,12 @@ GCThingIsMarkedGray(GCCellPtr thing)
extern JS_PUBLIC_API(JS::TraceKind)
GCThingTraceKind(void* thing);
extern JS_PUBLIC_API(void)
EnableNurseryStrings(JSContext* cx);
extern JS_PUBLIC_API(void)
DisableNurseryStrings(JSContext* cx);
/*
* Returns true when writes to GC thing pointers (and reads from weak pointers)
* must call an incremental barrier. This is generally only true when running

22
js/public/RootingAPI.h
View file

@ -201,6 +201,7 @@ template <typename T> class PersistentRooted;
JS_FRIEND_API(bool) isGCEnabled();
JS_FRIEND_API(void) HeapObjectPostBarrier(JSObject** objp, JSObject* prev, JSObject* next);
JS_FRIEND_API(void) HeapStringPostBarrier(JSString** objp, JSString* prev, JSString* next);
#ifdef JS_DEBUG
/**
@ -210,12 +211,12 @@ JS_FRIEND_API(void) HeapObjectPostBarrier(JSObject** objp, JSObject* prev, JSObj
extern JS_FRIEND_API(void)
AssertGCThingMustBeTenured(JSObject* obj);
extern JS_FRIEND_API(void)
AssertGCThingIsNotAnObjectSubclass(js::gc::Cell* cell);
AssertGCThingIsNotNurseryAllocable(js::gc::Cell* cell);
#else
inline void
AssertGCThingMustBeTenured(JSObject* obj) {}
inline void
AssertGCThingIsNotAnObjectSubclass(js::gc::Cell* cell) {}
AssertGCThingIsNotNurseryAllocable(js::gc::Cell* cell) {}
#endif
/**
@ -612,7 +613,7 @@ struct BarrierMethods<T*>
}
static void postBarrier(T** vp, T* prev, T* next) {
if (next)
JS::AssertGCThingIsNotAnObjectSubclass(reinterpret_cast<js::gc::Cell*>(next));
JS::AssertGCThingIsNotNurseryAllocable(reinterpret_cast<js::gc::Cell*>(next));
}
static void exposeToJS(T* t) {
if (t)
@ -660,6 +661,21 @@ struct BarrierMethods<JSFunction*>
}
};
template <>
struct BarrierMethods<JSString*>
{
static JSString* initial() { return nullptr; }
static gc::Cell* asGCThingOrNull(JSString* v) {
if (!v)
return nullptr;
MOZ_ASSERT(uintptr_t(v) > 32);
return reinterpret_cast<gc::Cell*>(v);
}
static void postBarrier(JSString** vp, JSString* prev, JSString* next) {
JS::HeapStringPostBarrier(vp, prev, next);
}
};
// Provide hash codes for Cell kinds that may be relocated and, thus, not have
// a stable address to use as the base for a hash code. Instead of the address,
// this hasher uses Cell::getUniqueId to provide exact matches and as a base

77
js/src/gc/Allocator.cpp
View file

@ -75,7 +75,7 @@ template JSObject* js::Allocate<JSObject, CanGC>(ExclusiveContext* cx, gc::Alloc
size_t nDynamicSlots, gc::InitialHeap heap,
const Class* clasp);
// Attempt to allocate a new GC thing out of the nursery. If there is not enough
// Attempt to allocate a new string out of the nursery. If there is not enough
// room in the nursery or there is an OOM, this method will return nullptr.
template <AllowGC allowGC>
JSObject*
@ -127,6 +127,81 @@ GCRuntime::tryNewTenuredObject(ExclusiveContext* cx, AllocKind kind, size_t thin
return obj;
}
// Attempt to allocate a new JSObject out of the nursery. If there is not
// enough room in the nursery or there is an OOM, this method will return
// nullptr.
template <AllowGC allowGC>
JSString*
GCRuntime::tryNewNurseryString(JSContext* cx, size_t thingSize, AllocKind kind)
{
MOZ_ASSERT(IsNurseryAllocable(kind));
MOZ_ASSERT(cx->isNurseryAllocAllowed());
MOZ_ASSERT(!cx->isJSContext());
MOZ_ASSERT(!IsAtomsCompartment(cx->compartment()));
Cell* cell = cx->nursery().allocateString(cx, cx->zone(), thingSize, kind);
if (cell)
return static_cast<JSString*>(cell);
if (allowGC && !cx->suppressGC) {
cx->runtime()->gc.minorGC(JS::gcreason::OUT_OF_NURSERY);
// Exceeding gcMaxBytes while tenuring can disable the Nursery.
if (cx->nursery().isEnabled()) {
cell = cx->nursery().allocateString(cx, cx->zone(), thingSize, kind);
MOZ_ASSERT(cell);
return static_cast<JSString*>(cell);
}
}
return nullptr;
}
template <typename StringAllocT, AllowGC allowGC /* = CanGC */>
StringAllocT*
js::AllocateString(JSContext* cx, InitialHeap heap)
{
static_assert(mozilla::IsConvertible<StringAllocT*, JSString*>::value, "must be JSString derived");
AllocKind kind = MapTypeToFinalizeKind<StringAllocT>::kind;
size_t size = sizeof(StringAllocT);
MOZ_ASSERT(size == Arena::thingSize(kind));
MOZ_ASSERT(size == sizeof(JSString) || size == sizeof(JSFatInlineString));
// Off-thread alloc cannot trigger GC or make runtime assertions.
if (cx->isJSContext()) {
StringAllocT* str = GCRuntime::tryNewTenuredThing<StringAllocT, NoGC>(cx, kind, size);
if (MOZ_UNLIKELY(allowGC && !str))
ReportOutOfMemory(cx);
return str;
}
JSRuntime* rt = cx->runtime();
if (!rt->gc.checkAllocatorState<allowGC>(cx, kind))
return nullptr;
if (cx->nursery().isEnabled() && heap != TenuredHeap && cx->nursery().canAllocateStrings()) {
auto str = static_cast<StringAllocT*>(rt->gc.tryNewNurseryString<allowGC>(cx, size, kind));
if (str)
return str;
// Our most common non-jit allocation path is NoGC; thus, if we fail the
// alloc and cannot GC, we *must* return nullptr here so that the caller
// will do a CanGC allocation to clear the nursery. Failing to do so will
// cause all allocations on this path to land in Tenured, and we will not
// get the benefit of the nursery.
if (!allowGC)
return nullptr;
}
return GCRuntime::tryNewTenuredThing<StringAllocT, allowGC>(cx, kind, size);
}
#define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType, bgfinal, nursery) \
template type* js::AllocateString<type, NoGC>(JSContext* cx, InitialHeap heap);\
template type* js::AllocateString<type, CanGC>(JSContext* cx, InitialHeap heap);
FOR_EACH_NURSERY_STRING_ALLOCKIND(DECL_ALLOCATOR_INSTANCES)
#undef DECL_ALLOCATOR_INSTANCES
template <typename T, AllowGC allowGC /* = CanGC */>
T*
js::Allocate(ExclusiveContext* cx)

7
js/src/gc/Barrier.cpp
View file

@ -199,6 +199,13 @@ JS::HeapObjectPostBarrier(JSObject** objp, JSObject* prev, JSObject* next)
js::InternalBarrierMethods<JSObject*>::postBarrier(objp, prev, next);
}
JS_PUBLIC_API(void)
JS::HeapStringPostBarrier(JSString** strp, JSString* prev, JSString* next)
{
MOZ_ASSERT(strp);
js::InternalBarrierMethods<JSString*>::postBarrier(strp, prev, next);
}
JS_PUBLIC_API(void)
JS::HeapValuePostBarrier(JS::Value* valuep, const Value& prev, const Value& next)
{

14
js/src/gc/Barrier.h
View file

@ -293,18 +293,18 @@ struct InternalBarrierMethods<Value>
// If the target needs an entry, add it.
js::gc::StoreBuffer* sb;
if (next.isObject() && (sb = reinterpret_cast<gc::Cell*>(&next.toObject())->storeBuffer())) {
if ((next.isObject() || next.isString()) && (sb = next.toGCThing()->storeBuffer())) {
// If we know that the prev has already inserted an entry, we can
// skip doing the lookup to add the new entry. Note that we cannot
// safely assert the presence of the entry because it may have been
// added via a different store buffer.
if (prev.isObject() && reinterpret_cast<gc::Cell*>(&prev.toObject())->storeBuffer())
if ((prev.isObject() || prev.isString()) && prev.toGCThing()->storeBuffer())
return;
sb->putValue(vp);
return;
}
// Remove the prev entry if the new value does not need it.
if (prev.isObject() && (sb = reinterpret_cast<gc::Cell*>(&prev.toObject())->storeBuffer()))
if ((prev.isObject() || prev.isString()) && (sb = prev.toGCThing()->storeBuffer()))
sb->unputValue(vp);
}
@ -679,9 +679,11 @@ class HeapSlot : public WriteBarrieredBase<Value>
private:
void post(NativeObject* owner, Kind kind, uint32_t slot, const Value& target) {
MOZ_ASSERT(preconditionForWriteBarrierPost(owner, kind, slot, target));
if (this->value.isObject()) {
gc::Cell* cell = reinterpret_cast<gc::Cell*>(&this->value.toObject());
#ifdef DEBUG
assertPreconditionForWriteBarrierPost(owner, kind, slot, target);
#endif
if (this->value.isObject() || this->value.isString()) {
gc::Cell* cell = this->value.toGCThing();
if (cell->storeBuffer())
cell->storeBuffer()->putSlot(owner, kind, slot, 1);
}

3
js/src/gc/GCRuntime.h
View file

@ -785,6 +785,9 @@ class GCRuntime
void traceRuntime(JSTracer* trc, AutoTraceSession& session);
void traceRuntimeForMinorGC(JSTracer* trc, AutoTraceSession& session);
void purgeRuntimeForMinorGC();
void notifyDidPaint();
void shrinkBuffers();
void onOutOfMallocMemory();

2
js/src/gc/Heap.h
View file

@ -875,6 +875,8 @@ InFreeList(Arena* arena, void* thing)
static const int32_t ChunkLocationOffsetFromLastByte =
int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask);
static const int32_t ChunkStoreBufferOffsetFromLastByte =
int32_t(gc::ChunkStoreBufferOffset) - int32_t(gc::ChunkMask);
} /* namespace gc */

128
js/src/gc/Marking.cpp
View file

@ -796,6 +796,18 @@ ShouldMark<JSObject*>(GCMarker* gcmarker, JSObject* obj)
return obj->asTenured().zone()->shouldMarkInZone();
}
// JSStrings can also be in the nursery. See ShouldMark<JSObject*> for comments.
template <>
bool
ShouldMark<JSString*>(GCMarker* gcmarker, JSString* str)
{
if (IsOwnedByOtherRuntime(gcmarker->runtime(), str))
return false;
if (IsInsideNursery(str))
return false;
return str->asTenured().zone()->shouldMarkInZone();
}
template <typename T>
void
DoMarking(GCMarker* gcmarker, T* thing)
@ -2312,12 +2324,13 @@ TenuringTracer::traverse(JSObject** objp)
// We only ever visit the internals of objects after moving them to tenured.
MOZ_ASSERT(!nursery().isInside(objp));
JSObject* obj = *objp;
if (!IsInsideNursery(obj) || nursery().getForwardedPointer(objp))
Cell** cellp = reinterpret_cast<Cell**>(objp);
if (!IsInsideNursery(*cellp) || nursery().getForwardedPointer(cellp))
return;
// Take a fast path for tenuring a plain object which is by far the most
// common case.
JSObject* obj = *objp;
if (obj->is<PlainObject>()) {
*objp = movePlainObjectToTenured(&obj->as<PlainObject>());
return;
@ -2326,6 +2339,18 @@ TenuringTracer::traverse(JSObject** objp)
*objp = moveToTenuredSlow(obj);
}
template <>
void
TenuringTracer::traverse(JSString** strp)
{
// We only ever visit the internals of strings after moving them to tenured.
MOZ_ASSERT(!nursery().isInside(strp));
Cell** cellp = reinterpret_cast<Cell**>(strp);
if (IsInsideNursery(*cellp) && !nursery().getForwardedPointer(cellp))
*strp = moveToTenured(*strp);
}
template <typename S>
struct TenuringTraversalFunctor : public IdentityDefaultAdaptor<S> {
template <typename T> S operator()(T* t, TenuringTracer* trc) {
@ -2411,6 +2436,12 @@ TraceWholeCell(TenuringTracer& mover, JSObject* object)
}
}
static inline void
TraceWholeCell(TenuringTracer& mover, JSString* str)
{
str->traceChildren(&mover);
}
static inline void
TraceWholeCell(TenuringTracer& mover, JSScript* script)
{
@ -2450,6 +2481,9 @@ js::gc::StoreBuffer::traceWholeCells(TenuringTracer& mover)
case JS::TraceKind::Object:
TraceBufferedCells<JSObject>(mover, arena, cells);
break;
case JS::TraceKind::String:
TraceBufferedCells<JSString>(mover, arena, cells);
break;
case JS::TraceKind::Script:
TraceBufferedCells<JSScript>(mover, arena, cells);
break;
@ -2472,8 +2506,22 @@ js::gc::StoreBuffer::CellPtrEdge::trace(TenuringTracer& mover) const
// XXX: We should check if the cell pointer is valid here too
MOZ_ASSERT(IsCellPointerValid(*edge));
MOZ_ASSERT((*edge)->getTraceKind() == JS::TraceKind::Object);
mover.traverse(reinterpret_cast<JSObject**>(edge));
#ifdef DEBUG
auto traceKind = (*edge)->getTraceKind();
MOZ_ASSERT(traceKind == JS::TraceKind::Object || traceKind == JS::TraceKind::String);
#endif
// Bug 1376646: Make separate store buffers for strings and objects, and
// only check IsInsideNursery once.
if (!IsInsideNursery(*edge))
return;
if (JSString::nurseryCellIsString(*edge))
mover.traverse(reinterpret_cast<JSString**>(edge));
else
mover.traverse(reinterpret_cast<JSObject**>(edge));
}
void
@ -2538,6 +2586,11 @@ inline void
js::TenuringTracer::traceSlots(JS::Value* vp, uint32_t nslots)
{
traceSlots(vp, vp + nslots);
void
js::TenuringTracer::traceString(JSString* str)
{
str->traceChildren(this);
}
#ifdef DEBUG
@ -2765,10 +2818,43 @@ js::TenuringTracer::moveElementsToTenured(NativeObject* dst, NativeObject* src,
return nslots * sizeof(HeapSlot);
}
inline void
js::TenuringTracer::insertIntoStringFixupList(RelocationOverlay* entry) {
*stringTail = entry;
stringTail = &entry->nextRef();
*stringTail = nullptr;
}
JSString*
js::TenuringTracer::moveToTenured(JSString* src)
{
MOZ_ASSERT(IsInsideNursery(src));
MOZ_ASSERT(!src->zone()->usedByHelperThread());
AllocKind dstKind = src->getAllocKind();
Zone* zone = src->zone();
TenuredCell* t = zone->arenas.allocateFromFreeList(dstKind, Arena::thingSize(dstKind));
if (!t) {
AutoEnterOOMUnsafeRegion oomUnsafe;
t = runtime()->gc.refillFreeListInGC(zone, dstKind);
if (!t)
oomUnsafe.crash(ChunkSize, "Failed to allocate string while tenuring.");
}
JSString* dst = reinterpret_cast<JSString*>(t);
tenuredSize += moveStringToTenured(dst, src, dstKind);
RelocationOverlay* overlay = RelocationOverlay::fromCell(src);
overlay->forwardTo(dst);
insertIntoStringFixupList(overlay);
TracePromoteToTenured(src, dst);
return dst;
}
void
js::Nursery::collectToFixedPoint(TenuringTracer& mover, TenureCountCache& tenureCounts)
{
for (RelocationOverlay* p = mover.head; p; p = p->next()) {
for (RelocationOverlay* p = mover.objHead; p; p = p->next()) {
JSObject* obj = static_cast<JSObject*>(p->forwardingAddress());
mover.traceObject(obj);
@ -2780,6 +2866,31 @@ js::Nursery::collectToFixedPoint(TenuringTracer& mover, TenureCountCache& tenure
entry.count = 1;
}
}
for (RelocationOverlay* p = mover.stringHead; p; p = p->next())
mover.traceString(static_cast<JSString*>(p->forwardingAddress()));
}
size_t
js::TenuringTracer::moveStringToTenured(JSString* dst, JSString* src, AllocKind dstKind)
{
size_t size = Arena::thingSize(dstKind);
// At the moment, strings always have the same AllocKind between src and
// dst. This may change in the future.
MOZ_ASSERT(dst->asTenured().getAllocKind() == src->getAllocKind());
// Copy the Cell contents.
MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(size));
js_memcpy(dst, src, size);
if (!src->isInline() && src->isLinear()) {
if (src->isUndepended() || !src->hasBase()) {
void* chars = src->asLinear().nonInlineCharsRaw();
nursery().removeMallocedBuffer(chars);
}
}
return size;
}
@ -2842,7 +2953,8 @@ IsMarkedInternal(JSRuntime* rt, JSObject** thingp)
if (IsInsideNursery(*thingp)) {
MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
return rt->gc.nursery.getForwardedPointer(thingp);
Cell** cellp = reinterpret_cast<Cell**>(thingp);
return rt->gc.nursery.getForwardedPointer(cellp);
}
return IsMarkedInternalCommon(thingp);
}
@ -2888,8 +3000,8 @@ IsAboutToBeFinalizedInternal(T** thingp)
Nursery& nursery = rt->gc.nursery;
if (IsInsideNursery(thing)) {
MOZ_ASSERT(rt->isHeapMinorCollecting());
return !nursery.getForwardedPointer(reinterpret_cast<JSObject**>(thingp));
return JS::CurrentThreadIsHeapMinorCollecting() &&
!Nursery::getForwardedPointer(reinterpret_cast<Cell**>(thingp));
}
Zone* zone = thing->asTenured().zoneFromAnyThread();

6
js/src/gc/Nursery-inl.h
View file

@ -26,15 +26,15 @@ js::Nursery::isInside(const SharedMem<T>& p) const
return isInside(p.unwrap(/*safe - used for value in comparison above*/));
}
MOZ_ALWAYS_INLINE bool
js::Nursery::getForwardedPointer(JSObject** ref) const
MOZ_ALWAYS_INLINE /* static */ bool
js::Nursery::getForwardedPointer(js::gc::Cell** ref)
{
MOZ_ASSERT(ref);
MOZ_ASSERT(isInside((void*)*ref));
const gc::RelocationOverlay* overlay = reinterpret_cast<const gc::RelocationOverlay*>(*ref);
if (!overlay->isForwarded())
return false;
*ref = static_cast<JSObject*>(overlay->forwardingAddress());
*ref = overlay->forwardingAddress();
return true;
}

62
js/src/gc/Nursery.cpp
View file

@ -127,11 +127,18 @@ js::Nursery::Nursery(JSRuntime* rt)
, previousPromotionRate_(0)
, profileThreshold_(0)
, enableProfiling_(false)
, canAllocateStrings_(false)
, reportTenurings_(0)
, minorGcCount_(0)
, freeMallocedBuffersTask(nullptr)
, sweepActions_(nullptr)
{}
#ifdef JS_GC_ZEAL
, lastCanary_(nullptr)
#endif
{
const char* env = getenv("MOZ_ENABLE_NURSERY_STRINGS");
if (env && *env)
canAllocateStrings_ = true;
}
bool
js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock)
@ -236,6 +243,20 @@ js::Nursery::disable()
runtime()->gc.storeBuffer().disable();
}
void
js::Nursery::enableStrings()
{
MOZ_ASSERT(isEmpty());
canAllocateStrings_ = true;
}
void
js::Nursery::disableStrings()
{
MOZ_ASSERT(isEmpty());
canAllocateStrings_ = false;
}
bool
js::Nursery::isEmpty() const
{
@ -350,7 +371,7 @@ js::Nursery::allocateBuffer(Zone* zone, size_t nbytes)
}
void* buffer = zone->pod_malloc<uint8_t>(nbytes);
if (buffer && !mallocedBuffers.putNew(buffer)) {
if (buffer && !registerMallocedBuffer(buffer)) {
js_free(buffer);
return nullptr;
}
@ -473,8 +494,10 @@ js::TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery)
: JSTracer(rt, JSTracer::TracerKindTag::Tenuring, TraceWeakMapKeysValues)
, nursery_(*nursery)
, tenuredSize(0)
, head(nullptr)
, tail(&head)
, objHead(nullptr)
, objTail(&objHead)
, stringHead(nullptr)
, stringTail(&stringHead)
{
}
@ -776,9 +799,9 @@ js::Nursery::doCollection(JS::gcreason::Reason reason,
}
endProfile(ProfileKey::MarkDebugger);
startProfile(ProfileKey::ClearNewObjectCache);
rt->caches().newObjectCache.clearNurseryObjects(rt);
endProfile(ProfileKey::ClearNewObjectCache);
maybeStartProfile(ProfileKey::SweepCaches);
rt->contextFromMainThread()->gc.purgeRuntimeForMinorGC();
maybeEndProfile(ProfileKey::SweepCaches);
// Most of the work is done here. This loop iterates over objects that have
// been moved to the major heap. If these objects have any outgoing pointers
@ -851,6 +874,13 @@ js::Nursery::FreeMallocedBuffersTask::run()
buffers_.clear();
}
bool
js::Nursery::registerMallocedBuffer(void* buffer)
{
MOZ_ASSERT(buffer);
return mallocedBuffers.putNew(buffer);
}
void
js::Nursery::freeMallocedBuffers()
{
@ -1142,3 +1172,19 @@ js::Nursery::sweepDictionaryModeObjects()
}
dictionaryModeObjects_.clear();
}
JS_PUBLIC_API(void)
JS::EnableNurseryStrings(JSContext* cx)
{
AutoEmptyNursery empty(cx);
ReleaseAllJITCode(cx->runtime()->defaultFreeOp());
cx->runtime()->gc.nursery().enableStrings();
}
JS_PUBLIC_API(void)
JS::DisableNurseryStrings(JSContext* cx)
{
AutoEmptyNursery empty(cx);
ReleaseAllJITCode(cx->runtime()->defaultFreeOp());
cx->runtime()->gc.nursery().disableStrings();
}

51
js/src/gc/Nursery.h
View file

@ -27,7 +27,7 @@
_(CheckHashTables, "ckTbls") \
_(MarkRuntime, "mkRntm") \
_(MarkDebugger, "mkDbgr") \
_(ClearNewObjectCache, "clrNOC") \
_(SweepCaches, "swpCch") \
_(CollectToFP, "collct") \
_(ObjectsTenuredCallback, "tenCB") \
_(Sweep, "sweep") \
@ -75,16 +75,18 @@ class TenuringTracer : public JSTracer
// Amount of data moved to the tenured generation during collection.
size_t tenuredSize;
// This list is threaded through the Nursery using the space from already
// moved things. The list is used to fix up the moved things and to find
// things held live by intra-Nursery pointers.
gc::RelocationOverlay* head;
gc::RelocationOverlay** tail;
// These lists are threaded through the Nursery using the space from
// already moved things. The lists are used to fix up the moved things and
// to find things held live by intra-Nursery pointers.
gc::RelocationOverlay* objHead;
gc::RelocationOverlay** objTail;
gc::RelocationOverlay* stringHead;
gc::RelocationOverlay** stringTail;
TenuringTracer(JSRuntime* rt, Nursery* nursery);
public:
const Nursery& nursery() const { return nursery_; }
Nursery& nursery() { return nursery_; }
// Returns true if the pointer was updated.
template <typename T> void traverse(T** thingp);
@ -93,19 +95,23 @@ class TenuringTracer : public JSTracer
// The store buffers need to be able to call these directly.
void traceObject(JSObject* src);
void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length);
void traceSlots(JS::Value* vp, uint32_t nslots);
void traceString(JSString* src);
private:
Nursery& nursery() { return nursery_; }
inline void insertIntoFixupList(gc::RelocationOverlay* entry);
inline void insertIntoObjectFixupList(gc::RelocationOverlay* entry);
inline void insertIntoStringFixupList(gc::RelocationOverlay* entry);
template <typename T>
inline T* allocTenured(JS::Zone* zone, gc::AllocKind kind);
inline JSObject* movePlainObjectToTenured(PlainObject* src);
JSObject* moveToTenuredSlow(JSObject* src);
JSString* moveToTenuredSlow(JSString* src);
size_t moveElementsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind);
size_t moveSlotsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind);
size_t moveStringToTenured(JSString* dst, JSString* src, gc::AllocKind dstKind);
void traceSlots(JS::Value* vp, JS::Value* end);
};
@ -152,6 +158,10 @@ class Nursery
void disable();
bool isEnabled() const { return maxChunkCount() != 0; }
void enableStrings();
void disableStrings();
bool canAllocateStrings() const { return canAllocateStrings_; }
/* Return true if no allocations have been made since the last collection. */
bool isEmpty() const;
@ -207,11 +217,11 @@ class Nursery
void collect(JSRuntime* rt, JS::gcreason::Reason reason);
/*
* Check if the thing at |*ref| in the Nursery has been forwarded. If so,
* sets |*ref| to the new location of the object and returns true. Otherwise
* returns false and leaves |*ref| unset.
* If the thing at |*ref| in the Nursery has been forwarded, set |*ref| to
* the new location and return true. Otherwise return false and leave
* |*ref| unset.
*/
MOZ_ALWAYS_INLINE [[nodiscard]] static bool getForwardedPointer(JSObject** ref);
MOZ_ALWAYS_INLINE [[nodiscard]] static bool getForwardedPointer(js::gc::Cell** ref);
/* Forward a slots/elements pointer stored in an Ion frame. */
void forwardBufferPointer(HeapSlot** pSlotsElems);
@ -219,6 +229,13 @@ class Nursery
inline void maybeSetForwardingPointer(JSTracer* trc, void* oldData, void* newData, bool direct);
inline void setForwardingPointerWhileTenuring(void* oldData, void* newData, bool direct);
/*
* Register a malloced buffer that is held by a nursery object, which
* should be freed at the end of a minor GC. Buffers are unregistered when
* their owning objects are tenured.
*/
bool registerMallocedBuffer(void* buffer);
/* Mark a malloced buffer as no longer needing to be freed. */
void removeMallocedBuffer(void* buffer) {
mallocedBuffers.remove(buffer);
@ -334,7 +351,10 @@ class Nursery
int64_t profileThreshold_;
bool enableProfiling_;
/* Report ObjectGroups with at lest this many instances tenured. */
/* Whether we will nursery-allocate strings. */
bool canAllocateStrings_;
/* Report ObjectGroups with at least this many instances tenured. */
int64_t reportTenurings_;
/* Profiling data. */
@ -447,9 +467,6 @@ class Nursery
JSRuntime* runtime() const { return runtime_; }
/* Allocates a new GC thing from the tenured generation during minor GC. */
gc::TenuredCell* allocateFromTenured(JS::Zone* zone, gc::AllocKind thingKind);
/* Common internal allocator function. */
void* allocate(size_t size);

18
js/src/gc/NurseryAwareHashMap.h
View file

@ -138,14 +138,18 @@ class NurseryAwareHashMap
// Update and relocate the key, if the value is still needed.
//
// Note that this currently assumes that all Value will contain a
// strong reference to Key, as per its use as the
// CrossCompartmentWrapperMap. We may need to make the following
// behavior more dynamic if we use this map in other nursery-aware
// contexts.
// Non-string Values will contain a strong reference to Key, as per
// its use in the CrossCompartmentWrapperMap, so the key will never
// be dying here. Strings do *not* have any sort of pointer from
// wrapper to wrappee, as they are just copies. The wrapper map
// entry is merely used as a cache to avoid re-copying the string,
// and currently that entire cache is flushed on major GC.
Key copy(key);
mozilla::DebugOnly<bool> sweepKey = JS::GCPolicy<Key>::needsSweep(&copy);
MOZ_ASSERT(!sweepKey);
bool sweepKey = JS::GCPolicy<Key>::needsSweep(&copy);
if (sweepKey) {
map.remove(key);
continue;
}
map.rekeyIfMoved(key, copy);
}
nurseryEntries.clear();

2
js/src/gc/StoreBuffer.h
View file

@ -400,6 +400,8 @@ class StoreBuffer
void clear();
const Nursery& nursery() const { return nursery_; }
/* Get the overflowed status. */
bool isAboutToOverflow() const { return aboutToOverflow_; }

4
js/src/jit/BaselineCompiler.cpp
View file

@ -2708,7 +2708,7 @@ BaselineCompiler::emit_JSOP_SETALIASEDVAR()
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, objReg, temp, &skipBarrier);
masm.branchValueIsNurseryObject(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.call(&postBarrierSlot_); // Won't clobber R0
@ -3127,7 +3127,7 @@ BaselineCompiler::emitFormalArgAccess(uint32_t arg, bool get)
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &skipBarrier);
masm.branchValueIsNurseryObject(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.call(&postBarrierSlot_);

124
js/src/jit/CodeGenerator.cpp
View file

@ -1002,7 +1002,104 @@ CodeGenerator::visitValueToObjectOrNull(LValueToObjectOrNull* lir)
masm.bind(ool->rejoin());
}
enum class FieldToBarrier {
REGEXP_PENDING_INPUT,
REGEXP_MATCHES_INPUT,
DEPENDENT_STRING_BASE
};
static void
EmitStoreBufferMutation(MacroAssembler& masm, Register holder, FieldToBarrier field,
Register buffer,
LiveGeneralRegisterSet& liveVolatiles,
void (*fun)(js::gc::StoreBuffer*, js::gc::Cell**))
{
Label callVM;
Label exit;
// Call into the VM to barrier the write. The only registers that need to
// be preserved are those in liveVolatiles, so once they are saved on the
// stack all volatile registers are available for use.
masm.bind(&callVM);
masm.PushRegsInMask(liveVolatiles);
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
regs.takeUnchecked(buffer);
regs.takeUnchecked(holder);
Register addrReg = regs.takeAny();
switch (field) {
case FieldToBarrier::REGEXP_PENDING_INPUT:
masm.computeEffectiveAddress(Address(holder, RegExpStatics::offsetOfPendingInput()), addrReg);
break;
case FieldToBarrier::REGEXP_MATCHES_INPUT:
masm.computeEffectiveAddress(Address(holder, RegExpStatics::offsetOfMatchesInput()), addrReg);
break;
case FieldToBarrier::DEPENDENT_STRING_BASE:
masm.leaNewDependentStringBase(holder, addrReg);
break;
}
bool needExtraReg = !regs.hasAny<GeneralRegisterSet::DefaultType>();
if (needExtraReg) {
masm.push(holder);
masm.setupUnalignedABICall(holder);
} else {
masm.setupUnalignedABICall(regs.takeAny());
}
masm.passABIArg(buffer);
masm.passABIArg(addrReg);
masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, fun), MoveOp::GENERAL,
CheckUnsafeCallWithABI::DontCheckOther);
if (needExtraReg)
masm.pop(holder);
masm.PopRegsInMask(liveVolatiles);
masm.bind(&exit);
}
// Warning: this function modifies prev and next.
static void
EmitPostWriteBarrierS(MacroAssembler& masm,
Register string, FieldToBarrier field,
Register prev, Register next,
LiveGeneralRegisterSet& liveVolatiles)
{
Label exit;
Label checkRemove, putCell;
// if (next && (buffer = next->storeBuffer()))
// but we never pass in nullptr for next.
Register storebuffer = next;
masm.loadStoreBuffer(next, storebuffer);
masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &checkRemove);
// if (prev && prev->storeBuffer())
masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &putCell);
masm.loadStoreBuffer(prev, prev);
masm.branchPtr(Assembler::NotEqual, prev, ImmWord(0), &exit);
// buffer->putCell(cellp)
masm.bind(&putCell);
EmitStoreBufferMutation(masm, string, field, storebuffer, liveVolatiles,
JSString::addCellAddressToStoreBuffer);
masm.jump(&exit);
// if (prev && (buffer = prev->storeBuffer()))
masm.bind(&checkRemove);
masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &exit);
masm.loadStoreBuffer(prev, storebuffer);
masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &exit);
EmitStoreBufferMutation(masm, string, field, storebuffer, liveVolatiles,
JSString::removeCellAddressFromStoreBuffer);
masm.bind(&exit);
}
typedef JSObject* (*CloneRegExpObjectFn)(JSContext*, JSObject*);
static const VMFunction CloneRegExpObjectInfo =
FunctionInfo<CloneRegExpObjectFn>(CloneRegExpObject, "CloneRegExpObject");
@ -1219,8 +1316,22 @@ PrepareAndExecuteRegExp(JSContext* cx, MacroAssembler& masm, Register regexp, Re
masm.patchableCallPreBarrier(matchesInputAddress, MIRType::String);
masm.patchableCallPreBarrier(lazySourceAddress, MIRType::String);
if (temp1.volatile_())
volatileRegs.add(temp1);
// Writing into RegExpStatics tenured memory; must post-barrier.
masm.loadPtr(pendingInputAddress, temp2);
masm.storePtr(input, pendingInputAddress);
masm.movePtr(input, temp3);
EmitPostWriteBarrierS(masm, temp1, FieldToBarrier::REGEXP_PENDING_INPUT,
temp2 /* prev */, temp3 /* next */, volatileRegs);
masm.loadPtr(matchesInputAddress, temp2);
masm.storePtr(input, matchesInputAddress);
masm.movePtr(input, temp3);
EmitPostWriteBarrierS(masm, temp1, FieldToBarrier::REGEXP_MATCHES_INPUT,
temp2 /* prev */, temp3 /* next */, volatileRegs);
masm.storePtr(lastIndex, Address(temp1, RegExpStatics::offsetOfLazyIndex()));
masm.store32(Imm32(1), Address(temp1, RegExpStatics::offsetOfPendingLazyEvaluation()));
@ -1263,6 +1374,7 @@ public:
bool latin1, Register string,
Register base, Register temp1, Register temp2,
BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
bool stringsCanBeInNursery,
Label* failure);
// Generate fallback path for creating DependentString.
@ -1274,6 +1386,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
bool latin1, Register string,
Register base, Register temp1, Register temp2,
BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
bool stringsCanBeInNursery,
Label* failure)
{
string_ = string;
@ -1311,7 +1424,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
masm.branch32(Assembler::Above, temp1, Imm32(maxThinInlineLength), &fatInline);
int32_t thinFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_THIN_INLINE_FLAGS;
masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString]);
masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString], stringsCanBeInNursery);
masm.bind(&joins_[FallbackKind::InlineString]);
masm.store32(Imm32(thinFlags), Address(string, JSString::offsetOfFlags()));
masm.jump(&stringAllocated);
@ -1319,7 +1432,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
masm.bind(&fatInline);
int32_t fatFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_FAT_INLINE_FLAGS;
masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString]);
masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString], stringsCanBeInNursery);
masm.bind(&joins_[FallbackKind::FatInlineString]);
masm.store32(Imm32(fatFlags), Address(string, JSString::offsetOfFlags()));
@ -1364,7 +1477,9 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
// Make a dependent string.
int32_t flags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::DEPENDENT_FLAGS;
masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString]);
masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString], stringsCanBeInNursery);
// Warning: string may be tenured (if the fallback case is hit), so
// stores into it must be post barriered.
masm.bind(&joins_[FallbackKind::NotInlineString]);
masm.store32(Imm32(flags), Address(string, JSString::offsetOfFlags()));
masm.store32(temp1, Address(string, JSString::offsetOfLength()));
@ -1377,6 +1492,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
masm.computeEffectiveAddress(BaseIndex(temp1, temp2, TimesTwo), temp1);
masm.storePtr(temp1, Address(string, JSString::offsetOfNonInlineChars()));
masm.storePtr(base, Address(string, JSDependentString::offsetOfBase()));
masm.movePtr(base, temp1);
// Follow any base pointer if the input is itself a dependent string.
// Watch for undepended strings, which have a base pointer but don't
@ -1386,7 +1502,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
Imm32(JSString::HAS_BASE_BIT), &noBase);
masm.branchTest32(Assembler::NonZero, Address(base, JSString::offsetOfFlags()),
Imm32(JSString::FLAT_BIT), &noBase);
masm.loadPtr(Address(base, JSDependentString::offsetOfBase()), temp1);
masm.loadPtr(Address(base, JSDependentString::offsetOfBase()), temp2);
masm.storePtr(temp1, Address(string, JSDependentString::offsetOfBase()));
masm.bind(&noBase);
}

8
js/src/jit/IonBuilder.cpp
View file

@ -13462,7 +13462,7 @@ IonBuilder::storeUnboxedValue(MDefinition* obj, MDefinition* elements, int32_t e
break;
case JSVAL_TYPE_STRING:
store = MStoreUnboxedString::New(alloc(), elements, scaledOffset, value,
store = MStoreUnboxedString::New(alloc(), elements, scaledOffset, value, obj,
elementsOffset, preBarrier);
break;
@ -15035,9 +15035,9 @@ IonBuilder::storeReferenceTypedObjectValue(MDefinition* typedObj,
store = MStoreUnboxedObjectOrNull::New(alloc(), elements, scaledOffset, value, typedObj, adjustment);
break;
case ReferenceTypeDescr::TYPE_STRING:
// Strings are not nursery allocated, so these writes do not need post
// barriers.
store = MStoreUnboxedString::New(alloc(), elements, scaledOffset, value, adjustment);
// See previous comment. The StoreUnboxedString type policy may insert
// ToString instructions that require a post barrier.
store = MStoreUnboxedString::New(alloc(), elements, scaledOffset, value, typedObj, adjustment);
break;
}

26
js/src/jit/MIR.h
View file

@ -8618,9 +8618,9 @@ class MFallibleStoreElement
};
// Store an unboxed object or null pointer to a v\ector.
// Store an unboxed object or null pointer to an elements vector.
class MStoreUnboxedObjectOrNull
: public MAryInstruction<4>,
: public MQuaternaryInstruction,
public StoreUnboxedObjectOrNullPolicy::Data
{
int32_t offsetAdjustment_;
@ -8663,29 +8663,33 @@ class MStoreUnboxedObjectOrNull
ALLOW_CLONE(MStoreUnboxedObjectOrNull)
};
// Store an unboxed object or null pointer to a vector.
// Store an unboxed string to an elements vector.
class MStoreUnboxedString
: public MAryInstruction<3>,
public MixPolicy<SingleObjectPolicy, ConvertToStringPolicy<2> >::Data
: public MQuaternaryInstruction,
public StoreUnboxedStringPolicy::Data
{
int32_t offsetAdjustment_;
bool preBarrier_;
MStoreUnboxedString(MDefinition* elements, MDefinition* index, MDefinition* value,
MStoreUnboxedString(MDefinition* elements, MDefinition* index,
MDefinition* value, MDefinition* typedObj,
int32_t offsetAdjustment = 0, bool preBarrier = true)
: offsetAdjustment_(offsetAdjustment), preBarrier_(preBarrier)
: MQuaternaryInstruction(classOpcode, elements, index, value, typedObj),
offsetAdjustment_(offsetAdjustment),
preBarrier_(preBarrier)
{
initOperand(0, elements);
initOperand(1, index);
initOperand(2, value);
MOZ_ASSERT(IsValidElementsType(elements, offsetAdjustment));
MOZ_ASSERT(index->type() == MIRType::Int32);
MOZ_ASSERT(typedObj->type() == MIRType::Object);
}
public:
INSTRUCTION_HEADER(StoreUnboxedString)
TRIVIAL_NEW_WRAPPERS
NAMED_OPERANDS((0, elements), (1, index), (2, value))
NAMED_OPERANDS((0, elements), (1, index), (2, value), (3, typedObj));
int32_t offsetAdjustment() const {
return offsetAdjustment_;
@ -8697,6 +8701,12 @@ class MStoreUnboxedString
return AliasSet::Store(AliasSet::UnboxedElement);
}
// For StoreUnboxedStringPolicy, to replace the original output with the
// output of a post barrier (if one is needed.)
void setValue(MDefinition* def) {
replaceOperand(2, def);
}
ALLOW_CLONE(MStoreUnboxedString)
};

28
js/src/jit/TypePolicy.cpp
View file

@ -973,6 +973,33 @@ StoreUnboxedObjectOrNullPolicy::adjustInputs(TempAllocator& alloc, MInstruction*
return true;
}
bool
StoreUnboxedStringPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins)
{
if (!ObjectPolicy<0>::staticAdjustInputs(alloc, ins))
return false;
// Change the value input to a ToString instruction if it might be
// a non-null primitive.
if (!ConvertToStringPolicy<2>::staticAdjustInputs(alloc, ins))
return false;
if (!ObjectPolicy<3>::staticAdjustInputs(alloc, ins))
return false;
// Insert a post barrier for the instruction's object and whatever its new
// value is.
MStoreUnboxedString* store = ins->toStoreUnboxedString();
MOZ_ASSERT(store->typedObj()->type() == MIRType::Object);
MDefinition* value = store->value();
MOZ_ASSERT(value->type() == MIRType::String);
MInstruction* barrier = MPostWriteBarrier::New(alloc, store->typedObj(), value);
store->block()->insertBefore(store, barrier);
return true;
}
bool
ClampPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins)
{
@ -1086,6 +1113,7 @@ FilterTypeSetPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins)
_(StoreTypedArrayHolePolicy) \
_(StoreUnboxedScalarPolicy) \
_(StoreUnboxedObjectOrNullPolicy) \
_(StoreUnboxedStringPolicy) \
_(TestPolicy) \
_(ToDoublePolicy) \
_(ToInt32Policy) \

7
js/src/jit/TypePolicy.h
View file

@ -455,6 +455,13 @@ class StoreUnboxedObjectOrNullPolicy final : public TypePolicy
virtual MOZ_MUST_USE bool adjustInputs(TempAllocator& alloc, MInstruction* def) override;
};
class StoreUnboxedStringPolicy final : public TypePolicy
{
public:
EMPTY_DATA_;
virtual [[nodiscard]] bool adjustInputs(TempAllocator& alloc, MInstruction* def) override;
};
// Accepts integers and doubles. Everything else is boxed.
class ClampPolicy final : public TypePolicy
{

7
js/src/jsapi.cpp
View file

@ -7618,6 +7618,13 @@ JS::GetObjectZone(JSObject* obj)
return obj->zone();
}
JS_PUBLIC_API(Zone*)
JS::GetNurseryStringZone(JSString* str)
{
MOZ_ASSERT(!str->isTenured());
return str->zone();
}
JS_PUBLIC_API(JS::TraceKind)
JS::GCThingTraceKind(void* thing)
{

2
js/src/jscompartment.cpp
View file

@ -751,6 +751,8 @@ JSCompartment::sweepAfterMinorGC(JSTracer* trc)
table.sweepAfterMinorGC();
crossCompartmentWrappers.sweepAfterMinorGC(trc);
dtoaCache.purge();
}
void

2
js/src/jscompartment.h
View file

@ -175,7 +175,7 @@ class CrossCompartmentKey
using ReturnType = bool;
ReturnType operator()(JSObject** tp) { return !IsInsideNursery(*tp); }
ReturnType operator()(JSScript** tp) { return true; }
ReturnType operator()(JSString** tp) { return true; }
ReturnType operator()(JSString** tp) { return !IsInsideNursery(*tp); }
};
return const_cast<CrossCompartmentKey*>(this)->applyToWrapped(IsTenuredFunctor());
}

1
js/src/jsfriendapi.h
View file

@ -19,6 +19,7 @@
#include "js/CallArgs.h"
#include "js/CallNonGenericMethod.h"
#include "js/Class.h"
#include "js/HeapAPI.h"
#include "js/Utility.h"
#if JS_STACK_GROWTH_DIRECTION > 0

50
js/src/jsgc.cpp
View file

@ -3646,6 +3646,19 @@ class MOZ_RAII js::gc::AutoRunParallelTask : public GCParallelTask
}
};
void
GCRuntime::purgeRuntimeForMinorGC(AutoLockForExclusiveAccess& lock)
{
// If external strings become nursery allocable, remember to call
// zone->externalStringCache().purge() (and delete this assert.)
MOZ_ASSERT(!IsNurseryAllocable(AllocKind::EXTERNAL_STRING));
for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next())
zone->functionToStringCache().purge();
rt->caches().purgeForMinorGC(rt);
}
void
GCRuntime::purgeRuntime()
{
@ -3658,16 +3671,7 @@ GCRuntime::purgeRuntime()
rt->interpreterStack().purge(rt);
JSContext* cx = rt->contextFromMainThread();
cx->caches.gsnCache.purge();
cx->caches.envCoordinateNameCache.purge();
cx->caches.newObjectCache.purge();
cx->caches.nativeIterCache.purge();
cx->caches.uncompressedSourceCache.purge();
if (cx->caches.evalCache.initialized())
cx->caches.evalCache.clear();
rt->mainThread.frontendCollectionPool.purge();
rt->caches().purge();
if (auto cache = rt->maybeThisRuntimeSharedImmutableStrings())
cache->purge();
@ -6398,11 +6402,7 @@ GCRuntime::compactPhase(JS::gcreason::Reason reason, SliceBudget& sliceBudget,
releaseRelocatedArenas(relocatedArenas);
// Clear caches that can contain cell pointers.
JSContext* cx = rt->contextFromMainThread();
cx->caches.newObjectCache.purge();
cx->caches.nativeIterCache.purge();
if (cx->caches.evalCache.initialized())
cx->caches.evalCache.clear();
rt->caches().purgeForCompaction();
#ifdef DEBUG
CheckHashTablesAfterMovingGC(rt);
@ -7790,10 +7790,10 @@ JS::AssertGCThingMustBeTenured(JSObject* obj)
}
JS_FRIEND_API(void)
JS::AssertGCThingIsNotAnObjectSubclass(Cell* cell)
JS::AssertGCThingIsNotNurseryAllocable(Cell* cell)
{
MOZ_ASSERT(cell);
MOZ_ASSERT(cell->getTraceKind() != JS::TraceKind::Object);
MOZ_ASSERT(!cell->is<JSObject>() && !cell->is<JSString>());
}
JS_FRIEND_API(void)
@ -7801,10 +7801,18 @@ js::gc::AssertGCThingHasType(js::gc::Cell* cell, JS::TraceKind kind)
{
if (!cell)
MOZ_ASSERT(kind == JS::TraceKind::Null);
else if (IsInsideNursery(cell))
MOZ_ASSERT(kind == JS::TraceKind::Object);
else
MOZ_ASSERT(MapAllocToTraceKind(cell->asTenured().getAllocKind()) == kind);
return;
}
MOZ_ASSERT(IsCellPointerValid(cell));
if (IsInsideNursery(cell)) {
MOZ_ASSERT(kind == (JSString::nurseryCellIsString(cell) ? JS::TraceKind::String
: JS::TraceKind::Object));
return;
}
MOZ_ASSERT(MapAllocToTraceKind(cell->asTenured().getAllocKind()) == kind);
}
JS_PUBLIC_API(size_t)

9
js/src/vm/Caches.h
View file

@ -65,6 +65,15 @@ struct EvalCacheEntry
JSScript* script;
JSScript* callerScript;
jsbytecode* pc;
// We sweep this cache before a nursery collection to remove entries with
// string keys in the nursery.
//
// The entire cache is purged on a major GC, so we don't need to sweep it
// then.
bool needsSweep() {
return !str->isTenured();
}
};
struct EvalCacheLookup

8
js/src/vm/MemoryMetrics.cpp
View file

@ -14,6 +14,7 @@
#include "jsscript.h"
#include "gc/Heap.h"
#include "gc/Nursery.h"
#include "jit/BaselineJIT.h"
#include "jit/Ion.h"
#include "vm/ArrayObject.h"
@ -514,13 +515,16 @@ StatsCellCallback(JSRuntime* rt, void* data, void* thing, JS::TraceKind traceKin
case JS::TraceKind::String: {
JSString* str = static_cast<JSString*>(thing);
size_t size = thingSize;
if (!str->isTenured())
size += Nursery::stringHeaderSize();
JS::StringInfo info;
if (str->hasLatin1Chars()) {
info.gcHeapLatin1 = thingSize;
info.gcHeapLatin1 = size;
info.mallocHeapLatin1 = str->sizeOfExcludingThis(rtStats->mallocSizeOf_);
} else {
info.gcHeapTwoByte = thingSize;
info.gcHeapTwoByte = size;
info.mallocHeapTwoByte = str->sizeOfExcludingThis(rtStats->mallocSizeOf_);
}
info.numCopies = 1;

17
js/src/vm/Scope.h
View file

@ -251,7 +251,14 @@ class Scope : public js::gc::TenuredCell
friend class GCMarker;
// The kind determines data_.
ScopeKind kind_;
//
// The memory here must be fully initialized, since otherwise the magic_
// value for gc::RelocationOverlay will land in the padding and may be
// stale.
union {
ScopeKind kind_;
uintptr_t paddedKind_;
};
// The enclosing scope or nullptr.
GCPtrScope enclosing_;
@ -264,11 +271,13 @@ class Scope : public js::gc::TenuredCell
uintptr_t data_;
Scope(ScopeKind kind, Scope* enclosing, Shape* environmentShape)
: kind_(kind),
enclosing_(enclosing),
: enclosing_(enclosing),
environmentShape_(environmentShape),
data_(0)
{ }
{
paddedKind_ = 0;
kind_ = kind;
}
static Scope* create(ExclusiveContext* cx, ScopeKind kind, HandleScope enclosing,
HandleShape envShape);

28
js/src/vm/String-inl.h
View file

@ -81,13 +81,9 @@ NewInlineString(ExclusiveContext* cx, HandleLinearString base, size_t start, siz
}
static inline void
StringWriteBarrierPost(js::ExclusiveContext* maybecx, JSString** strp)
{
}
static inline void
StringWriteBarrierPostRemove(js::ExclusiveContext* maybecx, JSString** strp)
StringWriteBarrierPost(JSContext* maybecx, JSString** strp, JSString* prev, JSString* next)
{
js::BarrierMethods<JSString*>::postBarrier(strp, prev, next);
}
} /* namespace js */
@ -112,8 +108,8 @@ JSRope::init(js::ExclusiveContext* cx, JSString* left, JSString* right, size_t l
d.u1.flags |= LATIN1_CHARS_BIT;
d.s.u2.left = left;
d.s.u3.right = right;
js::StringWriteBarrierPost(cx, &d.s.u2.left);
js::StringWriteBarrierPost(cx, &d.s.u3.right);
js::BarrierMethods<JSString*>::postBarrier(&d.s.u2.left, nullptr, left);
js::BarrierMethods<JSString*>::postBarrier(&d.s.u3.right, nullptr, right);
}
template <js::AllowGC allowGC>
@ -147,7 +143,7 @@ JSDependentString::init(js::ExclusiveContext* cx, JSLinearString* base, size_t s
d.s.u2.nonInlineCharsTwoByte = base->twoByteChars(nogc) + start;
}
d.s.u3.base = base;
js::StringWriteBarrierPost(cx, reinterpret_cast<JSString**>(&d.s.u3.base));
js::BarrierMethods<JSString*>::postBarrier(reinterpret_cast<JSString**>(&d.s.u3.base), nullptr, base);
}
MOZ_ALWAYS_INLINE JSLinearString*
@ -233,6 +229,20 @@ JSFlatString::new_(js::ExclusiveContext* cx, const CharT* chars, size_t length)
if (!str)
return nullptr;
if (!str->isTenured()) {
// The chars pointer is only considered to be handed over to this
// function on a successful return. If the following registration
// fails, the string is partially initialized and must be made valid,
// or its finalizer may attempt to free uninitialized memory.
void* ptr = const_cast<void*>(static_cast<const void*>(chars));
if (!cx->runtime()->gc.nursery().registerMallocedBuffer(ptr)) {
str->init((JS::Latin1Char*)nullptr, 0);
if (allowGC)
ReportOutOfMemory(cx);
return nullptr;
}
}
str->init(chars, length);
return str;
}

27
js/src/vm/String.cpp
View file

@ -14,6 +14,7 @@
#include "mozilla/Unused.h"
#include "gc/Marking.h"
#include "gc/Nursery.h"
#include "js/UbiNode.h"
#include "vm/SPSProfiler.h"
@ -79,9 +80,9 @@ JS::ubi::Concrete<JSString>::size(mozilla::MallocSizeOf mallocSizeOf) const
else
size = str.isFatInline() ? sizeof(JSFatInlineString) : sizeof(JSString);
// We can't use mallocSizeof on things in the nursery. At the moment,
// strings are never in the nursery, but that may change.
MOZ_ASSERT(!IsInsideNursery(&str));
if (IsInsideNursery(&str))
size += Nursery::stringHeaderSize();
size += str.sizeOfExcludingThis(mallocSizeOf);
return size;
@ -470,6 +471,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx)
JSString::writeBarrierPre(str->d.s.u3.right);
}
JSString* child = str->d.s.u2.left;
js::BarrierMethods<JSString*>::postBarrier(&str->d.s.u2.left, child, nullptr);
MOZ_ASSERT(child->isRope());
str->setNonInlineChars(left.nonInlineChars<CharT>(nogc));
child->d.u1.flattenData = uintptr_t(str) | Tag_VisitRightChild;
@ -486,8 +488,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx)
JS_STATIC_ASSERT(!(EXTENSIBLE_FLAGS & DEPENDENT_FLAGS));
left.d.u1.flags ^= (EXTENSIBLE_FLAGS | DEPENDENT_FLAGS);
left.d.s.u3.base = (JSLinearString*)this; /* will be true on exit */
StringWriteBarrierPostRemove(maybecx, &left.d.s.u2.left);
StringWriteBarrierPost(maybecx, (JSString**)&left.d.s.u3.base);
BarrierMethods<JSString*>::postBarrier((JSString**)&left.d.s.u3.base, nullptr, this);
goto visit_right_child;
}
}
@ -498,6 +499,15 @@ JSRope::flattenInternal(ExclusiveContext* maybecx)
return nullptr;
}
if (!isTenured() && maybecx) {
JSRuntime* rt = maybecx->runtime();
if (!rt->gc.nursery().registerMallocedBuffer(wholeChars)) {
js_free(wholeChars);
ReportOutOfMemory(maybecx);
return nullptr;
}
}
pos = wholeChars;
first_visit_node: {
if (b == WithIncrementalBarrier) {
@ -506,8 +516,8 @@ JSRope::flattenInternal(ExclusiveContext* maybecx)
}
JSString& left = *str->d.s.u2.left;
js::BarrierMethods<JSString*>::postBarrier(&str->d.s.u2.left, &left, nullptr);
str->setNonInlineChars(pos);
StringWriteBarrierPostRemove(maybecx, &str->d.s.u2.left);
if (left.isRope()) {
/* Return to this node when 'left' done, then goto visit_right_child. */
left.d.u1.flattenData = uintptr_t(str) | Tag_VisitRightChild;
@ -519,6 +529,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx)
}
visit_right_child: {
JSString& right = *str->d.s.u3.right;
BarrierMethods<JSString*>::postBarrier(&str->d.s.u3.right, &right, nullptr);
if (right.isRope()) {
/* Return to this node when 'right' done, then goto finish_node. */
right.d.u1.flattenData = uintptr_t(str) | Tag_FinishNode;
@ -539,8 +550,6 @@ JSRope::flattenInternal(ExclusiveContext* maybecx)
str->d.u1.flags = EXTENSIBLE_FLAGS | LATIN1_CHARS_BIT;
str->setNonInlineChars(wholeChars);
str->d.s.u3.capacity = wholeCapacity;
StringWriteBarrierPostRemove(maybecx, &str->d.s.u2.left);
StringWriteBarrierPostRemove(maybecx, &str->d.s.u3.right);
return &this->asFlat();
}
uintptr_t flattenData = str->d.u1.flattenData;
@ -550,7 +559,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx)
str->d.u1.flags = DEPENDENT_FLAGS | LATIN1_CHARS_BIT;
str->d.u1.length = pos - str->asLinear().nonInlineChars<CharT>(nogc);
str->d.s.u3.base = (JSLinearString*)this; /* will be true on exit */
StringWriteBarrierPost(maybecx, (JSString**)&str->d.s.u3.base);
BarrierMethods<JSString*>::postBarrier((JSString**)&str->d.s.u3.base, nullptr, this);
str = (JSString*)(flattenData & ~Tag_Mask);
if ((flattenData & Tag_Mask) == Tag_VisitRightChild)
goto visit_right_child;

77
js/src/vm/String.h
View file

@ -15,6 +15,7 @@
#include "gc/Barrier.h"
#include "gc/Heap.h"
#include "gc/Nursery.h"
#include "gc/Marking.h"
#include "gc/Rooting.h"
#include "js/CharacterEncoding.h"
@ -460,6 +461,14 @@ class JSString : public js::gc::TenuredCell
return *(JSAtom*)this;
}
// Used for distinguishing strings from objects in the nursery. The caller
// must ensure that cell is in the nursery (and not forwarded).
MOZ_ALWAYS_INLINE
static bool nurseryCellIsString(js::gc::Cell* cell) {
MOZ_ASSERT(!cell->isTenured());
return !static_cast<JSString*>(cell)->isAtom();
}
/* Only called by the GC for dependent or undepended strings. */
inline bool hasBase() const {
@ -496,6 +505,53 @@ class JSString : public js::gc::TenuredCell
static const JS::TraceKind TraceKind = JS::TraceKind::String;
JS::Zone* zone() const {
if (isTenured()) {
// Allow permanent atoms to be accessed across zones and runtimes.
if (isPermanentAtom())
return zoneFromAnyThread();
return asTenured().zone();
return js::Nursery::getStringZone(this);
}
// Implement TenuredZone members needed for template instantiations.
JS::Zone* zoneFromAnyThread() const {
if (isTenured())
return asTenured().zoneFromAnyThread();
return js::Nursery::getStringZone(this);
}
void fixupAfterMovingGC() {}
js::gc::AllocKind getAllocKind() const {
using js::gc::AllocKind;
AllocKind kind;
if (isAtom())
if (isFatInline())
kind = AllocKind::FAT_INLINE_ATOM;
else
kind = AllocKind::ATOM;
else if (isFatInline())
kind = AllocKind::FAT_INLINE_STRING;
else if (isExternal())
kind = AllocKind::EXTERNAL_STRING;
else
kind = AllocKind::STRING;
#if DEBUG
if (isTenured()) {
// Normally, the kinds should match, but an EXTERNAL_STRING arena
// may contain strings that have been flattened (see
// JSExternalString::ensureFlat).
AllocKind tenuredKind = asTenured().getAllocKind();
MOZ_ASSERT(kind == tenuredKind ||
(tenuredKind == AllocKind::EXTERNAL_STRING && kind == AllocKind::STRING));
}
#endif
return kind;
}
#ifdef DEBUG
void dump(FILE* fp);
void dumpCharsNoNewline(FILE* fp);
@ -513,19 +569,35 @@ class JSString : public js::gc::TenuredCell
void traceChildren(JSTracer* trc);
static MOZ_ALWAYS_INLINE void readBarrier(JSString* thing) {
if (thing->isPermanentAtom())
if (thing->isPermanentAtom() || js::gc::IsInsideNursery(thing))
return;
TenuredCell::readBarrier(thing);
}
static MOZ_ALWAYS_INLINE void writeBarrierPre(JSString* thing) {
if (!thing || thing->isPermanentAtom())
if (!thing || thing->isPermanentAtom() || js::gc::IsInsideNursery(thing))
return;
TenuredCell::writeBarrierPre(thing);
}
static void writeBarrierPost(void* cellp, JSString* prev, JSString* next) {
// See JSObject::writeBarrierPost for a description of the logic here.
MOZ_ASSERT(cellp);
js::gc::StoreBuffer* buffer;
if (next && (buffer = next->storeBuffer())) {
if (prev && prev->storeBuffer())
return;
buffer->putCell(static_cast<js::gc::Cell**>(cellp));
return;
}
if (prev && (buffer = prev->storeBuffer()))
buffer->unputCell(static_cast<js::gc::Cell**>(cellp));
}
private:
JSString() = delete;
JSString(const JSString& other) = delete;
@ -600,6 +672,7 @@ class JSLinearString : public JSString
{
friend class JSString;
friend class js::AutoStableStringChars;
friend class js::TenuringTracer;
/* Vacuous and therefore unimplemented. */
JSLinearString* ensureLinear(js::ExclusiveContext* cx) = delete;

3
js/src/vm/UnboxedObject-inl.h
View file

@ -65,8 +65,9 @@ SetUnboxedValueNoTypeChange(JSObject* unboxedObject,
return;
case JSVAL_TYPE_STRING: {
MOZ_ASSERT(!IsInsideNursery(v.toString()));
JSString** np = reinterpret_cast<JSString**>(p);
if (IsInsideNursery(v.toString()) && !IsInsideNursery(unboxedObject))
unboxedObject->zone()->group()->storeBuffer().putWholeCell(unboxedObject);
if (preBarrier)
JSString::writeBarrierPre(*np);
*np = v.toString();