diff --git a/js/public/HeapAPI.h b/js/public/HeapAPI.h index cda27e935f..836a29c583 100644 --- a/js/public/HeapAPI.h +++ b/js/public/HeapAPI.h @@ -23,6 +23,12 @@ namespace gc { struct Cell; +/* + * The low bit is set so this should never equal a normal pointer, and the high + * bit is set so this should never equal the upper 32 bits of a 64-bit pointer. + */ +const uint32_t Relocated = uintptr_t(0xbad0bad1); + const size_t ArenaShift = 12; const size_t ArenaSize = size_t(1) << ArenaShift; const size_t ArenaMask = ArenaSize - 1; @@ -411,10 +417,15 @@ GetTenuredGCThingZone(GCCellPtr thing) return js::gc::detail::GetGCThingZone(thing.unsafeAsUIntPtr()); } +extern JS_PUBLIC_API(Zone*) +GetNurseryStringZone(JSString* str); + static MOZ_ALWAYS_INLINE Zone* GetStringZone(JSString* str) { - return js::gc::detail::GetGCThingZone(uintptr_t(str)); + if (!js::gc::IsInsideNursery(reinterpret_cast(str))) + return js::gc::detail::GetGCThingZone(reinterpret_cast(str)); + return GetNurseryStringZone(str); } extern JS_PUBLIC_API(Zone*) @@ -431,6 +442,12 @@ GCThingIsMarkedGray(GCCellPtr thing) extern JS_PUBLIC_API(JS::TraceKind) GCThingTraceKind(void* thing); +extern JS_PUBLIC_API(void) +EnableNurseryStrings(JSContext* cx); + +extern JS_PUBLIC_API(void) +DisableNurseryStrings(JSContext* cx); + /* * Returns true when writes to GC thing pointers (and reads from weak pointers) * must call an incremental barrier. This is generally only true when running diff --git a/js/public/RootingAPI.h b/js/public/RootingAPI.h index 7b37ab1c4c..4dc9780003 100644 --- a/js/public/RootingAPI.h +++ b/js/public/RootingAPI.h @@ -201,6 +201,7 @@ template class PersistentRooted; JS_FRIEND_API(bool) isGCEnabled(); JS_FRIEND_API(void) HeapObjectPostBarrier(JSObject** objp, JSObject* prev, JSObject* next); +JS_FRIEND_API(void) HeapStringPostBarrier(JSString** objp, JSString* prev, JSString* next); #ifdef JS_DEBUG /** @@ -210,12 +211,12 @@ JS_FRIEND_API(void) HeapObjectPostBarrier(JSObject** objp, JSObject* prev, JSObj extern JS_FRIEND_API(void) AssertGCThingMustBeTenured(JSObject* obj); extern JS_FRIEND_API(void) -AssertGCThingIsNotAnObjectSubclass(js::gc::Cell* cell); +AssertGCThingIsNotNurseryAllocable(js::gc::Cell* cell); #else inline void AssertGCThingMustBeTenured(JSObject* obj) {} inline void -AssertGCThingIsNotAnObjectSubclass(js::gc::Cell* cell) {} +AssertGCThingIsNotNurseryAllocable(js::gc::Cell* cell) {} #endif /** @@ -612,7 +613,7 @@ struct BarrierMethods } static void postBarrier(T** vp, T* prev, T* next) { if (next) - JS::AssertGCThingIsNotAnObjectSubclass(reinterpret_cast(next)); + JS::AssertGCThingIsNotNurseryAllocable(reinterpret_cast(next)); } static void exposeToJS(T* t) { if (t) @@ -660,6 +661,21 @@ struct BarrierMethods } }; +template <> +struct BarrierMethods +{ + static JSString* initial() { return nullptr; } + static gc::Cell* asGCThingOrNull(JSString* v) { + if (!v) + return nullptr; + MOZ_ASSERT(uintptr_t(v) > 32); + return reinterpret_cast(v); + } + static void postBarrier(JSString** vp, JSString* prev, JSString* next) { + JS::HeapStringPostBarrier(vp, prev, next); + } +}; + // Provide hash codes for Cell kinds that may be relocated and, thus, not have // a stable address to use as the base for a hash code. Instead of the address, // this hasher uses Cell::getUniqueId to provide exact matches and as a base diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp index e8780722a5..c973908003 100644 --- a/js/src/gc/Allocator.cpp +++ b/js/src/gc/Allocator.cpp @@ -75,7 +75,7 @@ template JSObject* js::Allocate(ExclusiveContext* cx, gc::Alloc size_t nDynamicSlots, gc::InitialHeap heap, const Class* clasp); -// Attempt to allocate a new GC thing out of the nursery. If there is not enough +// Attempt to allocate a new string out of the nursery. If there is not enough // room in the nursery or there is an OOM, this method will return nullptr. template JSObject* @@ -127,6 +127,81 @@ GCRuntime::tryNewTenuredObject(ExclusiveContext* cx, AllocKind kind, size_t thin return obj; } +// Attempt to allocate a new JSObject out of the nursery. If there is not +// enough room in the nursery or there is an OOM, this method will return +// nullptr. +template +JSString* +GCRuntime::tryNewNurseryString(JSContext* cx, size_t thingSize, AllocKind kind) +{ + MOZ_ASSERT(IsNurseryAllocable(kind)); + MOZ_ASSERT(cx->isNurseryAllocAllowed()); + MOZ_ASSERT(!cx->isJSContext()); + MOZ_ASSERT(!IsAtomsCompartment(cx->compartment())); + + Cell* cell = cx->nursery().allocateString(cx, cx->zone(), thingSize, kind); + if (cell) + return static_cast(cell); + + if (allowGC && !cx->suppressGC) { + cx->runtime()->gc.minorGC(JS::gcreason::OUT_OF_NURSERY); + + // Exceeding gcMaxBytes while tenuring can disable the Nursery. + if (cx->nursery().isEnabled()) { + cell = cx->nursery().allocateString(cx, cx->zone(), thingSize, kind); + MOZ_ASSERT(cell); + return static_cast(cell); + } + } + return nullptr; +} + +template +StringAllocT* +js::AllocateString(JSContext* cx, InitialHeap heap) +{ + static_assert(mozilla::IsConvertible::value, "must be JSString derived"); + + AllocKind kind = MapTypeToFinalizeKind::kind; + size_t size = sizeof(StringAllocT); + MOZ_ASSERT(size == Arena::thingSize(kind)); + MOZ_ASSERT(size == sizeof(JSString) || size == sizeof(JSFatInlineString)); + + // Off-thread alloc cannot trigger GC or make runtime assertions. + if (cx->isJSContext()) { + StringAllocT* str = GCRuntime::tryNewTenuredThing(cx, kind, size); + if (MOZ_UNLIKELY(allowGC && !str)) + ReportOutOfMemory(cx); + return str; + } + + JSRuntime* rt = cx->runtime(); + if (!rt->gc.checkAllocatorState(cx, kind)) + return nullptr; + + if (cx->nursery().isEnabled() && heap != TenuredHeap && cx->nursery().canAllocateStrings()) { + auto str = static_cast(rt->gc.tryNewNurseryString(cx, size, kind)); + if (str) + return str; + + // Our most common non-jit allocation path is NoGC; thus, if we fail the + // alloc and cannot GC, we *must* return nullptr here so that the caller + // will do a CanGC allocation to clear the nursery. Failing to do so will + // cause all allocations on this path to land in Tenured, and we will not + // get the benefit of the nursery. + if (!allowGC) + return nullptr; + } + + return GCRuntime::tryNewTenuredThing(cx, kind, size); +} + +#define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType, bgfinal, nursery) \ + template type* js::AllocateString(JSContext* cx, InitialHeap heap);\ + template type* js::AllocateString(JSContext* cx, InitialHeap heap); +FOR_EACH_NURSERY_STRING_ALLOCKIND(DECL_ALLOCATOR_INSTANCES) +#undef DECL_ALLOCATOR_INSTANCES + template T* js::Allocate(ExclusiveContext* cx) diff --git a/js/src/gc/Barrier.cpp b/js/src/gc/Barrier.cpp index add28a0ff2..c4b90402d1 100644 --- a/js/src/gc/Barrier.cpp +++ b/js/src/gc/Barrier.cpp @@ -199,6 +199,13 @@ JS::HeapObjectPostBarrier(JSObject** objp, JSObject* prev, JSObject* next) js::InternalBarrierMethods::postBarrier(objp, prev, next); } +JS_PUBLIC_API(void) +JS::HeapStringPostBarrier(JSString** strp, JSString* prev, JSString* next) +{ + MOZ_ASSERT(strp); + js::InternalBarrierMethods::postBarrier(strp, prev, next); +} + JS_PUBLIC_API(void) JS::HeapValuePostBarrier(JS::Value* valuep, const Value& prev, const Value& next) { diff --git a/js/src/gc/Barrier.h b/js/src/gc/Barrier.h index 3a5fac2c38..fbf12a7a89 100644 --- a/js/src/gc/Barrier.h +++ b/js/src/gc/Barrier.h @@ -293,18 +293,18 @@ struct InternalBarrierMethods // If the target needs an entry, add it. js::gc::StoreBuffer* sb; - if (next.isObject() && (sb = reinterpret_cast(&next.toObject())->storeBuffer())) { + if ((next.isObject() || next.isString()) && (sb = next.toGCThing()->storeBuffer())) { // If we know that the prev has already inserted an entry, we can // skip doing the lookup to add the new entry. Note that we cannot // safely assert the presence of the entry because it may have been // added via a different store buffer. - if (prev.isObject() && reinterpret_cast(&prev.toObject())->storeBuffer()) + if ((prev.isObject() || prev.isString()) && prev.toGCThing()->storeBuffer()) return; sb->putValue(vp); return; } // Remove the prev entry if the new value does not need it. - if (prev.isObject() && (sb = reinterpret_cast(&prev.toObject())->storeBuffer())) + if ((prev.isObject() || prev.isString()) && (sb = prev.toGCThing()->storeBuffer())) sb->unputValue(vp); } @@ -679,9 +679,11 @@ class HeapSlot : public WriteBarrieredBase private: void post(NativeObject* owner, Kind kind, uint32_t slot, const Value& target) { - MOZ_ASSERT(preconditionForWriteBarrierPost(owner, kind, slot, target)); - if (this->value.isObject()) { - gc::Cell* cell = reinterpret_cast(&this->value.toObject()); +#ifdef DEBUG + assertPreconditionForWriteBarrierPost(owner, kind, slot, target); +#endif + if (this->value.isObject() || this->value.isString()) { + gc::Cell* cell = this->value.toGCThing(); if (cell->storeBuffer()) cell->storeBuffer()->putSlot(owner, kind, slot, 1); } diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index 98f363ab7a..c6e06d1731 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -785,6 +785,9 @@ class GCRuntime void traceRuntime(JSTracer* trc, AutoTraceSession& session); void traceRuntimeForMinorGC(JSTracer* trc, AutoTraceSession& session); + void purgeRuntimeForMinorGC(); + + void notifyDidPaint(); void shrinkBuffers(); void onOutOfMallocMemory(); diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h index 84eda138e8..70729f7016 100644 --- a/js/src/gc/Heap.h +++ b/js/src/gc/Heap.h @@ -875,6 +875,8 @@ InFreeList(Arena* arena, void* thing) static const int32_t ChunkLocationOffsetFromLastByte = int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask); +static const int32_t ChunkStoreBufferOffsetFromLastByte = + int32_t(gc::ChunkStoreBufferOffset) - int32_t(gc::ChunkMask); } /* namespace gc */ diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp index 5cc8cd0e0a..63dc92d2ba 100644 --- a/js/src/gc/Marking.cpp +++ b/js/src/gc/Marking.cpp @@ -796,6 +796,18 @@ ShouldMark(GCMarker* gcmarker, JSObject* obj) return obj->asTenured().zone()->shouldMarkInZone(); } +// JSStrings can also be in the nursery. See ShouldMark for comments. +template <> +bool +ShouldMark(GCMarker* gcmarker, JSString* str) +{ + if (IsOwnedByOtherRuntime(gcmarker->runtime(), str)) + return false; + if (IsInsideNursery(str)) + return false; + return str->asTenured().zone()->shouldMarkInZone(); +} + template void DoMarking(GCMarker* gcmarker, T* thing) @@ -2312,12 +2324,13 @@ TenuringTracer::traverse(JSObject** objp) // We only ever visit the internals of objects after moving them to tenured. MOZ_ASSERT(!nursery().isInside(objp)); - JSObject* obj = *objp; - if (!IsInsideNursery(obj) || nursery().getForwardedPointer(objp)) + Cell** cellp = reinterpret_cast(objp); + if (!IsInsideNursery(*cellp) || nursery().getForwardedPointer(cellp)) return; // Take a fast path for tenuring a plain object which is by far the most // common case. + JSObject* obj = *objp; if (obj->is()) { *objp = movePlainObjectToTenured(&obj->as()); return; @@ -2326,6 +2339,18 @@ TenuringTracer::traverse(JSObject** objp) *objp = moveToTenuredSlow(obj); } +template <> +void +TenuringTracer::traverse(JSString** strp) +{ + // We only ever visit the internals of strings after moving them to tenured. + MOZ_ASSERT(!nursery().isInside(strp)); + + Cell** cellp = reinterpret_cast(strp); + if (IsInsideNursery(*cellp) && !nursery().getForwardedPointer(cellp)) + *strp = moveToTenured(*strp); +} + template struct TenuringTraversalFunctor : public IdentityDefaultAdaptor { template S operator()(T* t, TenuringTracer* trc) { @@ -2411,6 +2436,12 @@ TraceWholeCell(TenuringTracer& mover, JSObject* object) } } +static inline void +TraceWholeCell(TenuringTracer& mover, JSString* str) +{ + str->traceChildren(&mover); +} + static inline void TraceWholeCell(TenuringTracer& mover, JSScript* script) { @@ -2450,6 +2481,9 @@ js::gc::StoreBuffer::traceWholeCells(TenuringTracer& mover) case JS::TraceKind::Object: TraceBufferedCells(mover, arena, cells); break; + case JS::TraceKind::String: + TraceBufferedCells(mover, arena, cells); + break; case JS::TraceKind::Script: TraceBufferedCells(mover, arena, cells); break; @@ -2472,8 +2506,22 @@ js::gc::StoreBuffer::CellPtrEdge::trace(TenuringTracer& mover) const // XXX: We should check if the cell pointer is valid here too MOZ_ASSERT(IsCellPointerValid(*edge)); - MOZ_ASSERT((*edge)->getTraceKind() == JS::TraceKind::Object); - mover.traverse(reinterpret_cast(edge)); + +#ifdef DEBUG + auto traceKind = (*edge)->getTraceKind(); + MOZ_ASSERT(traceKind == JS::TraceKind::Object || traceKind == JS::TraceKind::String); +#endif + + // Bug 1376646: Make separate store buffers for strings and objects, and + // only check IsInsideNursery once. + + if (!IsInsideNursery(*edge)) + return; + + if (JSString::nurseryCellIsString(*edge)) + mover.traverse(reinterpret_cast(edge)); + else + mover.traverse(reinterpret_cast(edge)); } void @@ -2538,6 +2586,11 @@ inline void js::TenuringTracer::traceSlots(JS::Value* vp, uint32_t nslots) { traceSlots(vp, vp + nslots); + +void +js::TenuringTracer::traceString(JSString* str) +{ + str->traceChildren(this); } #ifdef DEBUG @@ -2765,10 +2818,43 @@ js::TenuringTracer::moveElementsToTenured(NativeObject* dst, NativeObject* src, return nslots * sizeof(HeapSlot); } +inline void +js::TenuringTracer::insertIntoStringFixupList(RelocationOverlay* entry) { + *stringTail = entry; + stringTail = &entry->nextRef(); + *stringTail = nullptr; +} + +JSString* +js::TenuringTracer::moveToTenured(JSString* src) +{ + MOZ_ASSERT(IsInsideNursery(src)); + MOZ_ASSERT(!src->zone()->usedByHelperThread()); + + AllocKind dstKind = src->getAllocKind(); + Zone* zone = src->zone(); + + TenuredCell* t = zone->arenas.allocateFromFreeList(dstKind, Arena::thingSize(dstKind)); + if (!t) { + AutoEnterOOMUnsafeRegion oomUnsafe; + t = runtime()->gc.refillFreeListInGC(zone, dstKind); + if (!t) + oomUnsafe.crash(ChunkSize, "Failed to allocate string while tenuring."); + } + JSString* dst = reinterpret_cast(t); + tenuredSize += moveStringToTenured(dst, src, dstKind); + + RelocationOverlay* overlay = RelocationOverlay::fromCell(src); + overlay->forwardTo(dst); + insertIntoStringFixupList(overlay); + + TracePromoteToTenured(src, dst); + return dst; +} void js::Nursery::collectToFixedPoint(TenuringTracer& mover, TenureCountCache& tenureCounts) { - for (RelocationOverlay* p = mover.head; p; p = p->next()) { + for (RelocationOverlay* p = mover.objHead; p; p = p->next()) { JSObject* obj = static_cast(p->forwardingAddress()); mover.traceObject(obj); @@ -2780,6 +2866,31 @@ js::Nursery::collectToFixedPoint(TenuringTracer& mover, TenureCountCache& tenure entry.count = 1; } } + for (RelocationOverlay* p = mover.stringHead; p; p = p->next()) + mover.traceString(static_cast(p->forwardingAddress())); +} + +size_t +js::TenuringTracer::moveStringToTenured(JSString* dst, JSString* src, AllocKind dstKind) +{ + size_t size = Arena::thingSize(dstKind); + + // At the moment, strings always have the same AllocKind between src and + // dst. This may change in the future. + MOZ_ASSERT(dst->asTenured().getAllocKind() == src->getAllocKind()); + + // Copy the Cell contents. + MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(size)); + js_memcpy(dst, src, size); + + if (!src->isInline() && src->isLinear()) { + if (src->isUndepended() || !src->hasBase()) { + void* chars = src->asLinear().nonInlineCharsRaw(); + nursery().removeMallocedBuffer(chars); + } + } + + return size; } @@ -2842,7 +2953,8 @@ IsMarkedInternal(JSRuntime* rt, JSObject** thingp) if (IsInsideNursery(*thingp)) { MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); - return rt->gc.nursery.getForwardedPointer(thingp); + Cell** cellp = reinterpret_cast(thingp); + return rt->gc.nursery.getForwardedPointer(cellp); } return IsMarkedInternalCommon(thingp); } @@ -2888,8 +3000,8 @@ IsAboutToBeFinalizedInternal(T** thingp) Nursery& nursery = rt->gc.nursery; if (IsInsideNursery(thing)) { - MOZ_ASSERT(rt->isHeapMinorCollecting()); - return !nursery.getForwardedPointer(reinterpret_cast(thingp)); + return JS::CurrentThreadIsHeapMinorCollecting() && + !Nursery::getForwardedPointer(reinterpret_cast(thingp)); } Zone* zone = thing->asTenured().zoneFromAnyThread(); diff --git a/js/src/gc/Nursery-inl.h b/js/src/gc/Nursery-inl.h index 84d5ec8faf..fc88669d28 100644 --- a/js/src/gc/Nursery-inl.h +++ b/js/src/gc/Nursery-inl.h @@ -26,15 +26,15 @@ js::Nursery::isInside(const SharedMem& p) const return isInside(p.unwrap(/*safe - used for value in comparison above*/)); } -MOZ_ALWAYS_INLINE bool -js::Nursery::getForwardedPointer(JSObject** ref) const +MOZ_ALWAYS_INLINE /* static */ bool +js::Nursery::getForwardedPointer(js::gc::Cell** ref) { MOZ_ASSERT(ref); MOZ_ASSERT(isInside((void*)*ref)); const gc::RelocationOverlay* overlay = reinterpret_cast(*ref); if (!overlay->isForwarded()) return false; - *ref = static_cast(overlay->forwardingAddress()); + *ref = overlay->forwardingAddress(); return true; } diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index 0721514a09..663a23c07c 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -127,11 +127,18 @@ js::Nursery::Nursery(JSRuntime* rt) , previousPromotionRate_(0) , profileThreshold_(0) , enableProfiling_(false) + , canAllocateStrings_(false) , reportTenurings_(0) , minorGcCount_(0) , freeMallocedBuffersTask(nullptr) - , sweepActions_(nullptr) -{} +#ifdef JS_GC_ZEAL + , lastCanary_(nullptr) +#endif +{ + const char* env = getenv("MOZ_ENABLE_NURSERY_STRINGS"); + if (env && *env) + canAllocateStrings_ = true; +} bool js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock) @@ -236,6 +243,20 @@ js::Nursery::disable() runtime()->gc.storeBuffer().disable(); } +void +js::Nursery::enableStrings() +{ + MOZ_ASSERT(isEmpty()); + canAllocateStrings_ = true; +} + +void +js::Nursery::disableStrings() +{ + MOZ_ASSERT(isEmpty()); + canAllocateStrings_ = false; +} + bool js::Nursery::isEmpty() const { @@ -350,7 +371,7 @@ js::Nursery::allocateBuffer(Zone* zone, size_t nbytes) } void* buffer = zone->pod_malloc(nbytes); - if (buffer && !mallocedBuffers.putNew(buffer)) { + if (buffer && !registerMallocedBuffer(buffer)) { js_free(buffer); return nullptr; } @@ -473,8 +494,10 @@ js::TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery) : JSTracer(rt, JSTracer::TracerKindTag::Tenuring, TraceWeakMapKeysValues) , nursery_(*nursery) , tenuredSize(0) - , head(nullptr) - , tail(&head) + , objHead(nullptr) + , objTail(&objHead) + , stringHead(nullptr) + , stringTail(&stringHead) { } @@ -776,9 +799,9 @@ js::Nursery::doCollection(JS::gcreason::Reason reason, } endProfile(ProfileKey::MarkDebugger); - startProfile(ProfileKey::ClearNewObjectCache); - rt->caches().newObjectCache.clearNurseryObjects(rt); - endProfile(ProfileKey::ClearNewObjectCache); + maybeStartProfile(ProfileKey::SweepCaches); + rt->contextFromMainThread()->gc.purgeRuntimeForMinorGC(); + maybeEndProfile(ProfileKey::SweepCaches); // Most of the work is done here. This loop iterates over objects that have // been moved to the major heap. If these objects have any outgoing pointers @@ -851,6 +874,13 @@ js::Nursery::FreeMallocedBuffersTask::run() buffers_.clear(); } +bool +js::Nursery::registerMallocedBuffer(void* buffer) +{ + MOZ_ASSERT(buffer); + return mallocedBuffers.putNew(buffer); +} + void js::Nursery::freeMallocedBuffers() { @@ -1142,3 +1172,19 @@ js::Nursery::sweepDictionaryModeObjects() } dictionaryModeObjects_.clear(); } + +JS_PUBLIC_API(void) +JS::EnableNurseryStrings(JSContext* cx) +{ + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->runtime()->defaultFreeOp()); + cx->runtime()->gc.nursery().enableStrings(); +} + +JS_PUBLIC_API(void) +JS::DisableNurseryStrings(JSContext* cx) +{ + AutoEmptyNursery empty(cx); + ReleaseAllJITCode(cx->runtime()->defaultFreeOp()); + cx->runtime()->gc.nursery().disableStrings(); +} diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 5fbe93c221..6ca66fd09b 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -27,7 +27,7 @@ _(CheckHashTables, "ckTbls") \ _(MarkRuntime, "mkRntm") \ _(MarkDebugger, "mkDbgr") \ - _(ClearNewObjectCache, "clrNOC") \ + _(SweepCaches, "swpCch") \ _(CollectToFP, "collct") \ _(ObjectsTenuredCallback, "tenCB") \ _(Sweep, "sweep") \ @@ -75,16 +75,18 @@ class TenuringTracer : public JSTracer // Amount of data moved to the tenured generation during collection. size_t tenuredSize; - // This list is threaded through the Nursery using the space from already - // moved things. The list is used to fix up the moved things and to find - // things held live by intra-Nursery pointers. - gc::RelocationOverlay* head; - gc::RelocationOverlay** tail; + // These lists are threaded through the Nursery using the space from + // already moved things. The lists are used to fix up the moved things and + // to find things held live by intra-Nursery pointers. + gc::RelocationOverlay* objHead; + gc::RelocationOverlay** objTail; + gc::RelocationOverlay* stringHead; + gc::RelocationOverlay** stringTail; TenuringTracer(JSRuntime* rt, Nursery* nursery); public: - const Nursery& nursery() const { return nursery_; } + Nursery& nursery() { return nursery_; } // Returns true if the pointer was updated. template void traverse(T** thingp); @@ -93,19 +95,23 @@ class TenuringTracer : public JSTracer // The store buffers need to be able to call these directly. void traceObject(JSObject* src); void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length); + void traceSlots(JS::Value* vp, uint32_t nslots); + void traceString(JSString* src); private: - Nursery& nursery() { return nursery_; } - inline void insertIntoFixupList(gc::RelocationOverlay* entry); + inline void insertIntoObjectFixupList(gc::RelocationOverlay* entry); + inline void insertIntoStringFixupList(gc::RelocationOverlay* entry); template inline T* allocTenured(JS::Zone* zone, gc::AllocKind kind); inline JSObject* movePlainObjectToTenured(PlainObject* src); JSObject* moveToTenuredSlow(JSObject* src); + JSString* moveToTenuredSlow(JSString* src); size_t moveElementsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind); size_t moveSlotsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind); + size_t moveStringToTenured(JSString* dst, JSString* src, gc::AllocKind dstKind); void traceSlots(JS::Value* vp, JS::Value* end); }; @@ -152,6 +158,10 @@ class Nursery void disable(); bool isEnabled() const { return maxChunkCount() != 0; } + void enableStrings(); + void disableStrings(); + bool canAllocateStrings() const { return canAllocateStrings_; } + /* Return true if no allocations have been made since the last collection. */ bool isEmpty() const; @@ -207,11 +217,11 @@ class Nursery void collect(JSRuntime* rt, JS::gcreason::Reason reason); /* - * Check if the thing at |*ref| in the Nursery has been forwarded. If so, - * sets |*ref| to the new location of the object and returns true. Otherwise - * returns false and leaves |*ref| unset. + * If the thing at |*ref| in the Nursery has been forwarded, set |*ref| to + * the new location and return true. Otherwise return false and leave + * |*ref| unset. */ - MOZ_ALWAYS_INLINE [[nodiscard]] static bool getForwardedPointer(JSObject** ref); + MOZ_ALWAYS_INLINE [[nodiscard]] static bool getForwardedPointer(js::gc::Cell** ref); /* Forward a slots/elements pointer stored in an Ion frame. */ void forwardBufferPointer(HeapSlot** pSlotsElems); @@ -219,6 +229,13 @@ class Nursery inline void maybeSetForwardingPointer(JSTracer* trc, void* oldData, void* newData, bool direct); inline void setForwardingPointerWhileTenuring(void* oldData, void* newData, bool direct); + /* + * Register a malloced buffer that is held by a nursery object, which + * should be freed at the end of a minor GC. Buffers are unregistered when + * their owning objects are tenured. + */ + bool registerMallocedBuffer(void* buffer); + /* Mark a malloced buffer as no longer needing to be freed. */ void removeMallocedBuffer(void* buffer) { mallocedBuffers.remove(buffer); @@ -334,7 +351,10 @@ class Nursery int64_t profileThreshold_; bool enableProfiling_; - /* Report ObjectGroups with at lest this many instances tenured. */ + /* Whether we will nursery-allocate strings. */ + bool canAllocateStrings_; + + /* Report ObjectGroups with at least this many instances tenured. */ int64_t reportTenurings_; /* Profiling data. */ @@ -447,9 +467,6 @@ class Nursery JSRuntime* runtime() const { return runtime_; } - /* Allocates a new GC thing from the tenured generation during minor GC. */ - gc::TenuredCell* allocateFromTenured(JS::Zone* zone, gc::AllocKind thingKind); - /* Common internal allocator function. */ void* allocate(size_t size); diff --git a/js/src/gc/NurseryAwareHashMap.h b/js/src/gc/NurseryAwareHashMap.h index 9f9486deaa..c3a5d1ca15 100644 --- a/js/src/gc/NurseryAwareHashMap.h +++ b/js/src/gc/NurseryAwareHashMap.h @@ -138,14 +138,18 @@ class NurseryAwareHashMap // Update and relocate the key, if the value is still needed. // - // Note that this currently assumes that all Value will contain a - // strong reference to Key, as per its use as the - // CrossCompartmentWrapperMap. We may need to make the following - // behavior more dynamic if we use this map in other nursery-aware - // contexts. + // Non-string Values will contain a strong reference to Key, as per + // its use in the CrossCompartmentWrapperMap, so the key will never + // be dying here. Strings do *not* have any sort of pointer from + // wrapper to wrappee, as they are just copies. The wrapper map + // entry is merely used as a cache to avoid re-copying the string, + // and currently that entire cache is flushed on major GC. Key copy(key); - mozilla::DebugOnly sweepKey = JS::GCPolicy::needsSweep(©); - MOZ_ASSERT(!sweepKey); + bool sweepKey = JS::GCPolicy::needsSweep(©); + if (sweepKey) { + map.remove(key); + continue; + } map.rekeyIfMoved(key, copy); } nurseryEntries.clear(); diff --git a/js/src/gc/StoreBuffer.h b/js/src/gc/StoreBuffer.h index d1d80c0564..228792bff3 100644 --- a/js/src/gc/StoreBuffer.h +++ b/js/src/gc/StoreBuffer.h @@ -400,6 +400,8 @@ class StoreBuffer void clear(); + const Nursery& nursery() const { return nursery_; } + /* Get the overflowed status. */ bool isAboutToOverflow() const { return aboutToOverflow_; } diff --git a/js/src/jit/BaselineCompiler.cpp b/js/src/jit/BaselineCompiler.cpp index 7673af7d5d..6b4d62aa0d 100644 --- a/js/src/jit/BaselineCompiler.cpp +++ b/js/src/jit/BaselineCompiler.cpp @@ -2708,7 +2708,7 @@ BaselineCompiler::emit_JSOP_SETALIASEDVAR() Label skipBarrier; masm.branchPtrInNurseryChunk(Assembler::Equal, objReg, temp, &skipBarrier); - masm.branchValueIsNurseryObject(Assembler::NotEqual, R0, temp, &skipBarrier); + masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier); masm.call(&postBarrierSlot_); // Won't clobber R0 @@ -3127,7 +3127,7 @@ BaselineCompiler::emitFormalArgAccess(uint32_t arg, bool get) Label skipBarrier; masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &skipBarrier); - masm.branchValueIsNurseryObject(Assembler::NotEqual, R0, temp, &skipBarrier); + masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier); masm.call(&postBarrierSlot_); diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index dbc023150c..e338bf8f5a 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -1002,7 +1002,104 @@ CodeGenerator::visitValueToObjectOrNull(LValueToObjectOrNull* lir) masm.bind(ool->rejoin()); } +enum class FieldToBarrier { + REGEXP_PENDING_INPUT, + REGEXP_MATCHES_INPUT, + DEPENDENT_STRING_BASE +}; + +static void +EmitStoreBufferMutation(MacroAssembler& masm, Register holder, FieldToBarrier field, + Register buffer, + LiveGeneralRegisterSet& liveVolatiles, + void (*fun)(js::gc::StoreBuffer*, js::gc::Cell**)) +{ + Label callVM; + Label exit; + + // Call into the VM to barrier the write. The only registers that need to + // be preserved are those in liveVolatiles, so once they are saved on the + // stack all volatile registers are available for use. + masm.bind(&callVM); + masm.PushRegsInMask(liveVolatiles); + + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile()); + regs.takeUnchecked(buffer); + regs.takeUnchecked(holder); + Register addrReg = regs.takeAny(); + + switch (field) { + case FieldToBarrier::REGEXP_PENDING_INPUT: + masm.computeEffectiveAddress(Address(holder, RegExpStatics::offsetOfPendingInput()), addrReg); + break; + + case FieldToBarrier::REGEXP_MATCHES_INPUT: + masm.computeEffectiveAddress(Address(holder, RegExpStatics::offsetOfMatchesInput()), addrReg); + break; + + case FieldToBarrier::DEPENDENT_STRING_BASE: + masm.leaNewDependentStringBase(holder, addrReg); + break; + } + + bool needExtraReg = !regs.hasAny(); + if (needExtraReg) { + masm.push(holder); + masm.setupUnalignedABICall(holder); + } else { + masm.setupUnalignedABICall(regs.takeAny()); + } + masm.passABIArg(buffer); + masm.passABIArg(addrReg); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, fun), MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + + if (needExtraReg) + masm.pop(holder); + masm.PopRegsInMask(liveVolatiles); + masm.bind(&exit); +} + +// Warning: this function modifies prev and next. +static void +EmitPostWriteBarrierS(MacroAssembler& masm, + Register string, FieldToBarrier field, + Register prev, Register next, + LiveGeneralRegisterSet& liveVolatiles) +{ + Label exit; + Label checkRemove, putCell; + + // if (next && (buffer = next->storeBuffer())) + // but we never pass in nullptr for next. + Register storebuffer = next; + masm.loadStoreBuffer(next, storebuffer); + masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &checkRemove); + + // if (prev && prev->storeBuffer()) + masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &putCell); + masm.loadStoreBuffer(prev, prev); + masm.branchPtr(Assembler::NotEqual, prev, ImmWord(0), &exit); + + // buffer->putCell(cellp) + masm.bind(&putCell); + EmitStoreBufferMutation(masm, string, field, storebuffer, liveVolatiles, + JSString::addCellAddressToStoreBuffer); + masm.jump(&exit); + + // if (prev && (buffer = prev->storeBuffer())) + masm.bind(&checkRemove); + masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &exit); + masm.loadStoreBuffer(prev, storebuffer); + masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &exit); + EmitStoreBufferMutation(masm, string, field, storebuffer, liveVolatiles, + JSString::removeCellAddressFromStoreBuffer); + + masm.bind(&exit); +} + typedef JSObject* (*CloneRegExpObjectFn)(JSContext*, JSObject*); + static const VMFunction CloneRegExpObjectInfo = FunctionInfo(CloneRegExpObject, "CloneRegExpObject"); @@ -1219,8 +1316,22 @@ PrepareAndExecuteRegExp(JSContext* cx, MacroAssembler& masm, Register regexp, Re masm.patchableCallPreBarrier(matchesInputAddress, MIRType::String); masm.patchableCallPreBarrier(lazySourceAddress, MIRType::String); + + if (temp1.volatile_()) + volatileRegs.add(temp1); + + // Writing into RegExpStatics tenured memory; must post-barrier. + masm.loadPtr(pendingInputAddress, temp2); masm.storePtr(input, pendingInputAddress); + masm.movePtr(input, temp3); + EmitPostWriteBarrierS(masm, temp1, FieldToBarrier::REGEXP_PENDING_INPUT, + temp2 /* prev */, temp3 /* next */, volatileRegs); + + masm.loadPtr(matchesInputAddress, temp2); masm.storePtr(input, matchesInputAddress); + masm.movePtr(input, temp3); + EmitPostWriteBarrierS(masm, temp1, FieldToBarrier::REGEXP_MATCHES_INPUT, + temp2 /* prev */, temp3 /* next */, volatileRegs); masm.storePtr(lastIndex, Address(temp1, RegExpStatics::offsetOfLazyIndex())); masm.store32(Imm32(1), Address(temp1, RegExpStatics::offsetOfPendingLazyEvaluation())); @@ -1263,6 +1374,7 @@ public: bool latin1, Register string, Register base, Register temp1, Register temp2, BaseIndex startIndexAddress, BaseIndex limitIndexAddress, + bool stringsCanBeInNursery, Label* failure); // Generate fallback path for creating DependentString. @@ -1274,6 +1386,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names, bool latin1, Register string, Register base, Register temp1, Register temp2, BaseIndex startIndexAddress, BaseIndex limitIndexAddress, + bool stringsCanBeInNursery, Label* failure) { string_ = string; @@ -1311,7 +1424,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names, masm.branch32(Assembler::Above, temp1, Imm32(maxThinInlineLength), &fatInline); int32_t thinFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_THIN_INLINE_FLAGS; - masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString]); + masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString], stringsCanBeInNursery); masm.bind(&joins_[FallbackKind::InlineString]); masm.store32(Imm32(thinFlags), Address(string, JSString::offsetOfFlags())); masm.jump(&stringAllocated); @@ -1319,7 +1432,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names, masm.bind(&fatInline); int32_t fatFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_FAT_INLINE_FLAGS; - masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString]); + masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString], stringsCanBeInNursery); masm.bind(&joins_[FallbackKind::FatInlineString]); masm.store32(Imm32(fatFlags), Address(string, JSString::offsetOfFlags())); @@ -1364,7 +1477,9 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names, // Make a dependent string. int32_t flags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::DEPENDENT_FLAGS; - masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString]); + masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString], stringsCanBeInNursery); + // Warning: string may be tenured (if the fallback case is hit), so + // stores into it must be post barriered. masm.bind(&joins_[FallbackKind::NotInlineString]); masm.store32(Imm32(flags), Address(string, JSString::offsetOfFlags())); masm.store32(temp1, Address(string, JSString::offsetOfLength())); @@ -1377,6 +1492,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names, masm.computeEffectiveAddress(BaseIndex(temp1, temp2, TimesTwo), temp1); masm.storePtr(temp1, Address(string, JSString::offsetOfNonInlineChars())); masm.storePtr(base, Address(string, JSDependentString::offsetOfBase())); + masm.movePtr(base, temp1); // Follow any base pointer if the input is itself a dependent string. // Watch for undepended strings, which have a base pointer but don't @@ -1386,7 +1502,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names, Imm32(JSString::HAS_BASE_BIT), &noBase); masm.branchTest32(Assembler::NonZero, Address(base, JSString::offsetOfFlags()), Imm32(JSString::FLAT_BIT), &noBase); - masm.loadPtr(Address(base, JSDependentString::offsetOfBase()), temp1); + masm.loadPtr(Address(base, JSDependentString::offsetOfBase()), temp2); masm.storePtr(temp1, Address(string, JSDependentString::offsetOfBase())); masm.bind(&noBase); } diff --git a/js/src/jit/IonBuilder.cpp b/js/src/jit/IonBuilder.cpp index 5cbbd3ca26..e93e282a14 100644 --- a/js/src/jit/IonBuilder.cpp +++ b/js/src/jit/IonBuilder.cpp @@ -13462,7 +13462,7 @@ IonBuilder::storeUnboxedValue(MDefinition* obj, MDefinition* elements, int32_t e break; case JSVAL_TYPE_STRING: - store = MStoreUnboxedString::New(alloc(), elements, scaledOffset, value, + store = MStoreUnboxedString::New(alloc(), elements, scaledOffset, value, obj, elementsOffset, preBarrier); break; @@ -15035,9 +15035,9 @@ IonBuilder::storeReferenceTypedObjectValue(MDefinition* typedObj, store = MStoreUnboxedObjectOrNull::New(alloc(), elements, scaledOffset, value, typedObj, adjustment); break; case ReferenceTypeDescr::TYPE_STRING: - // Strings are not nursery allocated, so these writes do not need post - // barriers. - store = MStoreUnboxedString::New(alloc(), elements, scaledOffset, value, adjustment); + // See previous comment. The StoreUnboxedString type policy may insert + // ToString instructions that require a post barrier. + store = MStoreUnboxedString::New(alloc(), elements, scaledOffset, value, typedObj, adjustment); break; } diff --git a/js/src/jit/MIR.h b/js/src/jit/MIR.h index f432ae37f2..4f7375ca11 100644 --- a/js/src/jit/MIR.h +++ b/js/src/jit/MIR.h @@ -8618,9 +8618,9 @@ class MFallibleStoreElement }; -// Store an unboxed object or null pointer to a v\ector. +// Store an unboxed object or null pointer to an elements vector. class MStoreUnboxedObjectOrNull - : public MAryInstruction<4>, + : public MQuaternaryInstruction, public StoreUnboxedObjectOrNullPolicy::Data { int32_t offsetAdjustment_; @@ -8663,29 +8663,33 @@ class MStoreUnboxedObjectOrNull ALLOW_CLONE(MStoreUnboxedObjectOrNull) }; -// Store an unboxed object or null pointer to a vector. +// Store an unboxed string to an elements vector. class MStoreUnboxedString - : public MAryInstruction<3>, - public MixPolicy >::Data + : public MQuaternaryInstruction, + public StoreUnboxedStringPolicy::Data { int32_t offsetAdjustment_; bool preBarrier_; - MStoreUnboxedString(MDefinition* elements, MDefinition* index, MDefinition* value, + MStoreUnboxedString(MDefinition* elements, MDefinition* index, + MDefinition* value, MDefinition* typedObj, int32_t offsetAdjustment = 0, bool preBarrier = true) - : offsetAdjustment_(offsetAdjustment), preBarrier_(preBarrier) + : MQuaternaryInstruction(classOpcode, elements, index, value, typedObj), + offsetAdjustment_(offsetAdjustment), + preBarrier_(preBarrier) { initOperand(0, elements); initOperand(1, index); initOperand(2, value); MOZ_ASSERT(IsValidElementsType(elements, offsetAdjustment)); MOZ_ASSERT(index->type() == MIRType::Int32); + MOZ_ASSERT(typedObj->type() == MIRType::Object); } public: INSTRUCTION_HEADER(StoreUnboxedString) TRIVIAL_NEW_WRAPPERS - NAMED_OPERANDS((0, elements), (1, index), (2, value)) + NAMED_OPERANDS((0, elements), (1, index), (2, value), (3, typedObj)); int32_t offsetAdjustment() const { return offsetAdjustment_; @@ -8697,6 +8701,12 @@ class MStoreUnboxedString return AliasSet::Store(AliasSet::UnboxedElement); } + // For StoreUnboxedStringPolicy, to replace the original output with the + // output of a post barrier (if one is needed.) + void setValue(MDefinition* def) { + replaceOperand(2, def); + } + ALLOW_CLONE(MStoreUnboxedString) }; diff --git a/js/src/jit/TypePolicy.cpp b/js/src/jit/TypePolicy.cpp index 5023d00f77..49c12db05f 100644 --- a/js/src/jit/TypePolicy.cpp +++ b/js/src/jit/TypePolicy.cpp @@ -973,6 +973,33 @@ StoreUnboxedObjectOrNullPolicy::adjustInputs(TempAllocator& alloc, MInstruction* return true; } +bool +StoreUnboxedStringPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) +{ + if (!ObjectPolicy<0>::staticAdjustInputs(alloc, ins)) + return false; + + // Change the value input to a ToString instruction if it might be + // a non-null primitive. + if (!ConvertToStringPolicy<2>::staticAdjustInputs(alloc, ins)) + return false; + + if (!ObjectPolicy<3>::staticAdjustInputs(alloc, ins)) + return false; + + // Insert a post barrier for the instruction's object and whatever its new + // value is. + MStoreUnboxedString* store = ins->toStoreUnboxedString(); + + MOZ_ASSERT(store->typedObj()->type() == MIRType::Object); + + MDefinition* value = store->value(); + MOZ_ASSERT(value->type() == MIRType::String); + MInstruction* barrier = MPostWriteBarrier::New(alloc, store->typedObj(), value); + store->block()->insertBefore(store, barrier); + return true; +} + bool ClampPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) { @@ -1086,6 +1113,7 @@ FilterTypeSetPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) _(StoreTypedArrayHolePolicy) \ _(StoreUnboxedScalarPolicy) \ _(StoreUnboxedObjectOrNullPolicy) \ + _(StoreUnboxedStringPolicy) \ _(TestPolicy) \ _(ToDoublePolicy) \ _(ToInt32Policy) \ diff --git a/js/src/jit/TypePolicy.h b/js/src/jit/TypePolicy.h index 0b9d2b37ef..5dcf5189ab 100644 --- a/js/src/jit/TypePolicy.h +++ b/js/src/jit/TypePolicy.h @@ -455,6 +455,13 @@ class StoreUnboxedObjectOrNullPolicy final : public TypePolicy virtual MOZ_MUST_USE bool adjustInputs(TempAllocator& alloc, MInstruction* def) override; }; +class StoreUnboxedStringPolicy final : public TypePolicy +{ + public: + EMPTY_DATA_; + virtual [[nodiscard]] bool adjustInputs(TempAllocator& alloc, MInstruction* def) override; +}; + // Accepts integers and doubles. Everything else is boxed. class ClampPolicy final : public TypePolicy { diff --git a/js/src/jsapi.cpp b/js/src/jsapi.cpp index d40790745c..287de9eaae 100644 --- a/js/src/jsapi.cpp +++ b/js/src/jsapi.cpp @@ -7618,6 +7618,13 @@ JS::GetObjectZone(JSObject* obj) return obj->zone(); } +JS_PUBLIC_API(Zone*) +JS::GetNurseryStringZone(JSString* str) +{ + MOZ_ASSERT(!str->isTenured()); + return str->zone(); +} + JS_PUBLIC_API(JS::TraceKind) JS::GCThingTraceKind(void* thing) { diff --git a/js/src/jscompartment.cpp b/js/src/jscompartment.cpp index ef8404d74e..d7afc1e097 100644 --- a/js/src/jscompartment.cpp +++ b/js/src/jscompartment.cpp @@ -751,6 +751,8 @@ JSCompartment::sweepAfterMinorGC(JSTracer* trc) table.sweepAfterMinorGC(); crossCompartmentWrappers.sweepAfterMinorGC(trc); + + dtoaCache.purge(); } void diff --git a/js/src/jscompartment.h b/js/src/jscompartment.h index 977069f1e7..92ffb6b53a 100644 --- a/js/src/jscompartment.h +++ b/js/src/jscompartment.h @@ -175,7 +175,7 @@ class CrossCompartmentKey using ReturnType = bool; ReturnType operator()(JSObject** tp) { return !IsInsideNursery(*tp); } ReturnType operator()(JSScript** tp) { return true; } - ReturnType operator()(JSString** tp) { return true; } + ReturnType operator()(JSString** tp) { return !IsInsideNursery(*tp); } }; return const_cast(this)->applyToWrapped(IsTenuredFunctor()); } diff --git a/js/src/jsfriendapi.h b/js/src/jsfriendapi.h index c463019dac..6f4c4bfa8d 100644 --- a/js/src/jsfriendapi.h +++ b/js/src/jsfriendapi.h @@ -19,6 +19,7 @@ #include "js/CallArgs.h" #include "js/CallNonGenericMethod.h" #include "js/Class.h" +#include "js/HeapAPI.h" #include "js/Utility.h" #if JS_STACK_GROWTH_DIRECTION > 0 diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index 086b793fab..c53ebf32af 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -3646,6 +3646,19 @@ class MOZ_RAII js::gc::AutoRunParallelTask : public GCParallelTask } }; +void +GCRuntime::purgeRuntimeForMinorGC(AutoLockForExclusiveAccess& lock) +{ + // If external strings become nursery allocable, remember to call + // zone->externalStringCache().purge() (and delete this assert.) + MOZ_ASSERT(!IsNurseryAllocable(AllocKind::EXTERNAL_STRING)); + + for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) + zone->functionToStringCache().purge(); + + rt->caches().purgeForMinorGC(rt); +} + void GCRuntime::purgeRuntime() { @@ -3658,16 +3671,7 @@ GCRuntime::purgeRuntime() rt->interpreterStack().purge(rt); - JSContext* cx = rt->contextFromMainThread(); - cx->caches.gsnCache.purge(); - cx->caches.envCoordinateNameCache.purge(); - cx->caches.newObjectCache.purge(); - cx->caches.nativeIterCache.purge(); - cx->caches.uncompressedSourceCache.purge(); - if (cx->caches.evalCache.initialized()) - cx->caches.evalCache.clear(); - - rt->mainThread.frontendCollectionPool.purge(); + rt->caches().purge(); if (auto cache = rt->maybeThisRuntimeSharedImmutableStrings()) cache->purge(); @@ -6398,11 +6402,7 @@ GCRuntime::compactPhase(JS::gcreason::Reason reason, SliceBudget& sliceBudget, releaseRelocatedArenas(relocatedArenas); // Clear caches that can contain cell pointers. - JSContext* cx = rt->contextFromMainThread(); - cx->caches.newObjectCache.purge(); - cx->caches.nativeIterCache.purge(); - if (cx->caches.evalCache.initialized()) - cx->caches.evalCache.clear(); + rt->caches().purgeForCompaction(); #ifdef DEBUG CheckHashTablesAfterMovingGC(rt); @@ -7790,10 +7790,10 @@ JS::AssertGCThingMustBeTenured(JSObject* obj) } JS_FRIEND_API(void) -JS::AssertGCThingIsNotAnObjectSubclass(Cell* cell) +JS::AssertGCThingIsNotNurseryAllocable(Cell* cell) { MOZ_ASSERT(cell); - MOZ_ASSERT(cell->getTraceKind() != JS::TraceKind::Object); + MOZ_ASSERT(!cell->is() && !cell->is()); } JS_FRIEND_API(void) @@ -7801,10 +7801,18 @@ js::gc::AssertGCThingHasType(js::gc::Cell* cell, JS::TraceKind kind) { if (!cell) MOZ_ASSERT(kind == JS::TraceKind::Null); - else if (IsInsideNursery(cell)) - MOZ_ASSERT(kind == JS::TraceKind::Object); - else - MOZ_ASSERT(MapAllocToTraceKind(cell->asTenured().getAllocKind()) == kind); + return; + } + + MOZ_ASSERT(IsCellPointerValid(cell)); + + if (IsInsideNursery(cell)) { + MOZ_ASSERT(kind == (JSString::nurseryCellIsString(cell) ? JS::TraceKind::String + : JS::TraceKind::Object)); + return; + } + + MOZ_ASSERT(MapAllocToTraceKind(cell->asTenured().getAllocKind()) == kind); } JS_PUBLIC_API(size_t) diff --git a/js/src/vm/Caches.h b/js/src/vm/Caches.h index 7c34295c95..dd5e3666d2 100644 --- a/js/src/vm/Caches.h +++ b/js/src/vm/Caches.h @@ -65,6 +65,15 @@ struct EvalCacheEntry JSScript* script; JSScript* callerScript; jsbytecode* pc; + + // We sweep this cache before a nursery collection to remove entries with + // string keys in the nursery. + // + // The entire cache is purged on a major GC, so we don't need to sweep it + // then. + bool needsSweep() { + return !str->isTenured(); + } }; struct EvalCacheLookup diff --git a/js/src/vm/MemoryMetrics.cpp b/js/src/vm/MemoryMetrics.cpp index 2a28cf23c0..472035808f 100644 --- a/js/src/vm/MemoryMetrics.cpp +++ b/js/src/vm/MemoryMetrics.cpp @@ -14,6 +14,7 @@ #include "jsscript.h" #include "gc/Heap.h" +#include "gc/Nursery.h" #include "jit/BaselineJIT.h" #include "jit/Ion.h" #include "vm/ArrayObject.h" @@ -514,13 +515,16 @@ StatsCellCallback(JSRuntime* rt, void* data, void* thing, JS::TraceKind traceKin case JS::TraceKind::String: { JSString* str = static_cast(thing); + size_t size = thingSize; + if (!str->isTenured()) + size += Nursery::stringHeaderSize(); JS::StringInfo info; if (str->hasLatin1Chars()) { - info.gcHeapLatin1 = thingSize; + info.gcHeapLatin1 = size; info.mallocHeapLatin1 = str->sizeOfExcludingThis(rtStats->mallocSizeOf_); } else { - info.gcHeapTwoByte = thingSize; + info.gcHeapTwoByte = size; info.mallocHeapTwoByte = str->sizeOfExcludingThis(rtStats->mallocSizeOf_); } info.numCopies = 1; diff --git a/js/src/vm/Scope.h b/js/src/vm/Scope.h index 36abf2fbb7..a480a8c1e9 100644 --- a/js/src/vm/Scope.h +++ b/js/src/vm/Scope.h @@ -251,7 +251,14 @@ class Scope : public js::gc::TenuredCell friend class GCMarker; // The kind determines data_. - ScopeKind kind_; + // + // The memory here must be fully initialized, since otherwise the magic_ + // value for gc::RelocationOverlay will land in the padding and may be + // stale. + union { + ScopeKind kind_; + uintptr_t paddedKind_; + }; // The enclosing scope or nullptr. GCPtrScope enclosing_; @@ -264,11 +271,13 @@ class Scope : public js::gc::TenuredCell uintptr_t data_; Scope(ScopeKind kind, Scope* enclosing, Shape* environmentShape) - : kind_(kind), - enclosing_(enclosing), + : enclosing_(enclosing), environmentShape_(environmentShape), data_(0) - { } + { + paddedKind_ = 0; + kind_ = kind; + } static Scope* create(ExclusiveContext* cx, ScopeKind kind, HandleScope enclosing, HandleShape envShape); diff --git a/js/src/vm/String-inl.h b/js/src/vm/String-inl.h index 52261d0f75..70f05fbd62 100644 --- a/js/src/vm/String-inl.h +++ b/js/src/vm/String-inl.h @@ -81,13 +81,9 @@ NewInlineString(ExclusiveContext* cx, HandleLinearString base, size_t start, siz } static inline void -StringWriteBarrierPost(js::ExclusiveContext* maybecx, JSString** strp) -{ -} - -static inline void -StringWriteBarrierPostRemove(js::ExclusiveContext* maybecx, JSString** strp) +StringWriteBarrierPost(JSContext* maybecx, JSString** strp, JSString* prev, JSString* next) { + js::BarrierMethods::postBarrier(strp, prev, next); } } /* namespace js */ @@ -112,8 +108,8 @@ JSRope::init(js::ExclusiveContext* cx, JSString* left, JSString* right, size_t l d.u1.flags |= LATIN1_CHARS_BIT; d.s.u2.left = left; d.s.u3.right = right; - js::StringWriteBarrierPost(cx, &d.s.u2.left); - js::StringWriteBarrierPost(cx, &d.s.u3.right); + js::BarrierMethods::postBarrier(&d.s.u2.left, nullptr, left); + js::BarrierMethods::postBarrier(&d.s.u3.right, nullptr, right); } template @@ -147,7 +143,7 @@ JSDependentString::init(js::ExclusiveContext* cx, JSLinearString* base, size_t s d.s.u2.nonInlineCharsTwoByte = base->twoByteChars(nogc) + start; } d.s.u3.base = base; - js::StringWriteBarrierPost(cx, reinterpret_cast(&d.s.u3.base)); + js::BarrierMethods::postBarrier(reinterpret_cast(&d.s.u3.base), nullptr, base); } MOZ_ALWAYS_INLINE JSLinearString* @@ -233,6 +229,20 @@ JSFlatString::new_(js::ExclusiveContext* cx, const CharT* chars, size_t length) if (!str) return nullptr; + if (!str->isTenured()) { + // The chars pointer is only considered to be handed over to this + // function on a successful return. If the following registration + // fails, the string is partially initialized and must be made valid, + // or its finalizer may attempt to free uninitialized memory. + void* ptr = const_cast(static_cast(chars)); + if (!cx->runtime()->gc.nursery().registerMallocedBuffer(ptr)) { + str->init((JS::Latin1Char*)nullptr, 0); + if (allowGC) + ReportOutOfMemory(cx); + return nullptr; + } + } + str->init(chars, length); return str; } diff --git a/js/src/vm/String.cpp b/js/src/vm/String.cpp index 538e9c09eb..1cf020eaca 100644 --- a/js/src/vm/String.cpp +++ b/js/src/vm/String.cpp @@ -14,6 +14,7 @@ #include "mozilla/Unused.h" #include "gc/Marking.h" +#include "gc/Nursery.h" #include "js/UbiNode.h" #include "vm/SPSProfiler.h" @@ -79,9 +80,9 @@ JS::ubi::Concrete::size(mozilla::MallocSizeOf mallocSizeOf) const else size = str.isFatInline() ? sizeof(JSFatInlineString) : sizeof(JSString); - // We can't use mallocSizeof on things in the nursery. At the moment, - // strings are never in the nursery, but that may change. - MOZ_ASSERT(!IsInsideNursery(&str)); + if (IsInsideNursery(&str)) + size += Nursery::stringHeaderSize(); + size += str.sizeOfExcludingThis(mallocSizeOf); return size; @@ -470,6 +471,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) JSString::writeBarrierPre(str->d.s.u3.right); } JSString* child = str->d.s.u2.left; + js::BarrierMethods::postBarrier(&str->d.s.u2.left, child, nullptr); MOZ_ASSERT(child->isRope()); str->setNonInlineChars(left.nonInlineChars(nogc)); child->d.u1.flattenData = uintptr_t(str) | Tag_VisitRightChild; @@ -486,8 +488,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) JS_STATIC_ASSERT(!(EXTENSIBLE_FLAGS & DEPENDENT_FLAGS)); left.d.u1.flags ^= (EXTENSIBLE_FLAGS | DEPENDENT_FLAGS); left.d.s.u3.base = (JSLinearString*)this; /* will be true on exit */ - StringWriteBarrierPostRemove(maybecx, &left.d.s.u2.left); - StringWriteBarrierPost(maybecx, (JSString**)&left.d.s.u3.base); + BarrierMethods::postBarrier((JSString**)&left.d.s.u3.base, nullptr, this); goto visit_right_child; } } @@ -498,6 +499,15 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) return nullptr; } + if (!isTenured() && maybecx) { + JSRuntime* rt = maybecx->runtime(); + if (!rt->gc.nursery().registerMallocedBuffer(wholeChars)) { + js_free(wholeChars); + ReportOutOfMemory(maybecx); + return nullptr; + } + } + pos = wholeChars; first_visit_node: { if (b == WithIncrementalBarrier) { @@ -506,8 +516,8 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) } JSString& left = *str->d.s.u2.left; + js::BarrierMethods::postBarrier(&str->d.s.u2.left, &left, nullptr); str->setNonInlineChars(pos); - StringWriteBarrierPostRemove(maybecx, &str->d.s.u2.left); if (left.isRope()) { /* Return to this node when 'left' done, then goto visit_right_child. */ left.d.u1.flattenData = uintptr_t(str) | Tag_VisitRightChild; @@ -519,6 +529,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) } visit_right_child: { JSString& right = *str->d.s.u3.right; + BarrierMethods::postBarrier(&str->d.s.u3.right, &right, nullptr); if (right.isRope()) { /* Return to this node when 'right' done, then goto finish_node. */ right.d.u1.flattenData = uintptr_t(str) | Tag_FinishNode; @@ -539,8 +550,6 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) str->d.u1.flags = EXTENSIBLE_FLAGS | LATIN1_CHARS_BIT; str->setNonInlineChars(wholeChars); str->d.s.u3.capacity = wholeCapacity; - StringWriteBarrierPostRemove(maybecx, &str->d.s.u2.left); - StringWriteBarrierPostRemove(maybecx, &str->d.s.u3.right); return &this->asFlat(); } uintptr_t flattenData = str->d.u1.flattenData; @@ -550,7 +559,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) str->d.u1.flags = DEPENDENT_FLAGS | LATIN1_CHARS_BIT; str->d.u1.length = pos - str->asLinear().nonInlineChars(nogc); str->d.s.u3.base = (JSLinearString*)this; /* will be true on exit */ - StringWriteBarrierPost(maybecx, (JSString**)&str->d.s.u3.base); + BarrierMethods::postBarrier((JSString**)&str->d.s.u3.base, nullptr, this); str = (JSString*)(flattenData & ~Tag_Mask); if ((flattenData & Tag_Mask) == Tag_VisitRightChild) goto visit_right_child; diff --git a/js/src/vm/String.h b/js/src/vm/String.h index 0c763cf14a..309e0ceafc 100644 --- a/js/src/vm/String.h +++ b/js/src/vm/String.h @@ -15,6 +15,7 @@ #include "gc/Barrier.h" #include "gc/Heap.h" +#include "gc/Nursery.h" #include "gc/Marking.h" #include "gc/Rooting.h" #include "js/CharacterEncoding.h" @@ -460,6 +461,14 @@ class JSString : public js::gc::TenuredCell return *(JSAtom*)this; } + // Used for distinguishing strings from objects in the nursery. The caller + // must ensure that cell is in the nursery (and not forwarded). + MOZ_ALWAYS_INLINE + static bool nurseryCellIsString(js::gc::Cell* cell) { + MOZ_ASSERT(!cell->isTenured()); + return !static_cast(cell)->isAtom(); + } + /* Only called by the GC for dependent or undepended strings. */ inline bool hasBase() const { @@ -496,6 +505,53 @@ class JSString : public js::gc::TenuredCell static const JS::TraceKind TraceKind = JS::TraceKind::String; + JS::Zone* zone() const { + if (isTenured()) { + // Allow permanent atoms to be accessed across zones and runtimes. + if (isPermanentAtom()) + return zoneFromAnyThread(); + return asTenured().zone(); + return js::Nursery::getStringZone(this); + } + + // Implement TenuredZone members needed for template instantiations. + + JS::Zone* zoneFromAnyThread() const { + if (isTenured()) + return asTenured().zoneFromAnyThread(); + return js::Nursery::getStringZone(this); + } + + void fixupAfterMovingGC() {} + + js::gc::AllocKind getAllocKind() const { + using js::gc::AllocKind; + AllocKind kind; + if (isAtom()) + if (isFatInline()) + kind = AllocKind::FAT_INLINE_ATOM; + else + kind = AllocKind::ATOM; + else if (isFatInline()) + kind = AllocKind::FAT_INLINE_STRING; + else if (isExternal()) + kind = AllocKind::EXTERNAL_STRING; + else + kind = AllocKind::STRING; + +#if DEBUG + if (isTenured()) { + // Normally, the kinds should match, but an EXTERNAL_STRING arena + // may contain strings that have been flattened (see + // JSExternalString::ensureFlat). + AllocKind tenuredKind = asTenured().getAllocKind(); + MOZ_ASSERT(kind == tenuredKind || + (tenuredKind == AllocKind::EXTERNAL_STRING && kind == AllocKind::STRING)); + } +#endif + return kind; + } + #ifdef DEBUG void dump(FILE* fp); void dumpCharsNoNewline(FILE* fp); @@ -513,19 +569,35 @@ class JSString : public js::gc::TenuredCell void traceChildren(JSTracer* trc); static MOZ_ALWAYS_INLINE void readBarrier(JSString* thing) { - if (thing->isPermanentAtom()) + if (thing->isPermanentAtom() || js::gc::IsInsideNursery(thing)) return; TenuredCell::readBarrier(thing); } static MOZ_ALWAYS_INLINE void writeBarrierPre(JSString* thing) { - if (!thing || thing->isPermanentAtom()) + if (!thing || thing->isPermanentAtom() || js::gc::IsInsideNursery(thing)) return; TenuredCell::writeBarrierPre(thing); } + static void writeBarrierPost(void* cellp, JSString* prev, JSString* next) { + // See JSObject::writeBarrierPost for a description of the logic here. + MOZ_ASSERT(cellp); + + js::gc::StoreBuffer* buffer; + if (next && (buffer = next->storeBuffer())) { + if (prev && prev->storeBuffer()) + return; + buffer->putCell(static_cast(cellp)); + return; + } + + if (prev && (buffer = prev->storeBuffer())) + buffer->unputCell(static_cast(cellp)); + } + private: JSString() = delete; JSString(const JSString& other) = delete; @@ -600,6 +672,7 @@ class JSLinearString : public JSString { friend class JSString; friend class js::AutoStableStringChars; + friend class js::TenuringTracer; /* Vacuous and therefore unimplemented. */ JSLinearString* ensureLinear(js::ExclusiveContext* cx) = delete; diff --git a/js/src/vm/UnboxedObject-inl.h b/js/src/vm/UnboxedObject-inl.h index 2d355c6946..c2e7f7189e 100644 --- a/js/src/vm/UnboxedObject-inl.h +++ b/js/src/vm/UnboxedObject-inl.h @@ -65,8 +65,9 @@ SetUnboxedValueNoTypeChange(JSObject* unboxedObject, return; case JSVAL_TYPE_STRING: { - MOZ_ASSERT(!IsInsideNursery(v.toString())); JSString** np = reinterpret_cast(p); + if (IsInsideNursery(v.toString()) && !IsInsideNursery(unboxedObject)) + unboxedObject->zone()->group()->storeBuffer().putWholeCell(unboxedObject); if (preBarrier) JSString::writeBarrierPre(*np); *np = v.toString();