diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp index 1407e5905d..fc7dbc84e1 100644 --- a/js/src/gc/Allocator.cpp +++ b/js/src/gc/Allocator.cpp @@ -229,8 +229,8 @@ js::Allocate(ExclusiveContext* cx) } #define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ - template type* js::Allocate(ExclusiveContext* cx);\ - template type* js::Allocate(ExclusiveContext* cx); + template type* js::Allocate(JSContext* cx);\ + template type* js::Allocate(JSContext* cx); FOR_EACH_NONOBJECT_NONNURSERY_ALLOCKIND(DECL_ALLOCATOR_INSTANCES) #undef DECL_ALLOCATOR_INSTANCES diff --git a/js/src/gc/AtomMarking-inl.h b/js/src/gc/AtomMarking-inl.h index a65394a0dc..cba5d7ef8f 100644 --- a/js/src/gc/AtomMarking-inl.h +++ b/js/src/gc/AtomMarking-inl.h @@ -16,8 +16,10 @@ namespace gc { inline size_t GetAtomBit(TenuredCell* thing) { - (void)thing; - return 0; + MOZ_ASSERT(thing->zoneFromAnyThread()->isAtomsZone()); + Arena* arena = thing->arena(); + size_t arenaBit = (reinterpret_cast(thing) - arena->address()) / CellBytesPerMarkBit; + return arena->atomBitmapStart() * JS_BITS_PER_WORD + arenaBit; } inline bool @@ -52,7 +54,23 @@ AtomMarkingRuntime::inlinedMarkAtom(JSContext* cx, T* thing) if (ThingIsPermanent(thing)) return; - (void)cell; + size_t bit = GetAtomBit(cell); + MOZ_ASSERT(bit / JS_BITS_PER_WORD < allocatedWords); + + cx->zone()->markedAtoms().setBit(bit); + + if (!cx->helperThread()) { + // Trigger a read barrier on the atom, in case there is an incremental + // GC in progress. This is necessary if the atom is being marked + // because a reference to it was obtained from another zone which is + // not being collected by the incremental GC. + T::readBarrier(thing); + } + + // Children of the thing also need to be marked in the context's zone. + // We don't have a JSTracer for this so manually handle the cases in which + // an atom can reference other atoms. + markChildren(cx, thing); } } // namespace gc diff --git a/js/src/gc/AtomMarking.cpp b/js/src/gc/AtomMarking.cpp index f008ee25e4..8214c316f3 100644 --- a/js/src/gc/AtomMarking.cpp +++ b/js/src/gc/AtomMarking.cpp @@ -47,20 +47,51 @@ namespace gc { void AtomMarkingRuntime::registerArena(Arena* arena) { - (void)arena; + MOZ_ASSERT(arena->getThingSize() != 0); + MOZ_ASSERT(arena->getThingSize() % CellAlignBytes == 0); + MOZ_ASSERT(arena->zone->isAtomsZone()); + MOZ_ASSERT(arena->zone->runtimeFromAnyThread()->currentThreadHasExclusiveAccess()); + + // We need to find a range of bits from the atoms bitmap for this arena. + + // Look for a free range of bits compatible with this arena. + if (freeArenaIndexes.ref().length()) { + arena->atomBitmapStart() = freeArenaIndexes.ref().popCopy(); + return; + } + + // Allocate a range of bits from the end for this arena. + arena->atomBitmapStart() = allocatedWords; + allocatedWords += ArenaBitmapWords; } void AtomMarkingRuntime::unregisterArena(Arena* arena) { - (void)arena; + MOZ_ASSERT(arena->zone->isAtomsZone()); + + // Leak these atom bits if we run out of memory. + mozilla::Unused << freeArenaIndexes.ref().emplaceBack(arena->atomBitmapStart()); } bool AtomMarkingRuntime::computeBitmapFromChunkMarkBits(JSRuntime* runtime, DenseBitmap& bitmap) { - (void)runtime; - return bitmap.ensureSpace(0); + MOZ_ASSERT(runtime->currentThreadHasExclusiveAccess()); + + if (!bitmap.ensureSpace(allocatedWords)) + return false; + + Zone* atomsZone = runtime->unsafeAtomsCompartment()->zone(); + for (auto thingKind : AllAllocKinds()) { + for (ArenaIter aiter(atomsZone, thingKind); !aiter.done(); aiter.next()) { + Arena* arena = aiter.get(); + uintptr_t* chunkWords = arena->chunk()->bitmap.arenaBits(arena); + bitmap.copyBitsFrom(arena->atomBitmapStart(), ArenaBitmapWords, chunkWords); + } + } + + return true; } void @@ -81,8 +112,19 @@ template static void BitwiseOrIntoChunkMarkBits(JSRuntime* runtime, Bitmap& bitmap) { - (void)runtime; - (void)bitmap; + // Make sure that by copying the mark bits for one arena in word sizes we + // do not affect the mark bits for other arenas. + static_assert(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD, + "ArenaBitmapWords must evenly divide ArenaBitmapBits"); + + Zone* atomsZone = runtime->unsafeAtomsCompartment()->zone(); + for (auto thingKind : AllAllocKinds()) { + for (ArenaIter aiter(atomsZone, thingKind); !aiter.done(); aiter.next()) { + Arena* arena = aiter.get(); + uintptr_t* chunkWords = arena->chunk()->bitmap.arenaBits(arena); + bitmap.bitwiseOrRangeInto(arena->atomBitmapStart(), ArenaBitmapWords, chunkWords); + } + } } void diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index dd58aab6ef..c4a01f287c 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -1615,8 +1615,6 @@ class GCRuntime friend class AutoEnterIteration; }; -MOZ_MUST_USE bool InitializeStaticData(); - /* Prevent compartments and zones from being collected during iteration. */ class MOZ_RAII AutoEnterIteration { GCRuntime* gc; diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp index 48361c04b4..08695bf37f 100644 --- a/js/src/gc/Marking.cpp +++ b/js/src/gc/Marking.cpp @@ -158,22 +158,23 @@ IsMovingTracer(JSTracer *trc) } #endif -bool ThingIsPermanentAtomOrWellKnownSymbol(JSString* str) { +template bool ThingIsPermanentAtomOrWellKnownSymbol(T* thing) { return false; } +template <> bool ThingIsPermanentAtomOrWellKnownSymbol(JSString* str) { return str->isPermanentAtom(); } -bool ThingIsPermanentAtomOrWellKnownSymbol(JSFlatString* str) { +template <> bool ThingIsPermanentAtomOrWellKnownSymbol(JSFlatString* str) { return str->isPermanentAtom(); } -bool ThingIsPermanentAtomOrWellKnownSymbol(JSLinearString* str) { +template <> bool ThingIsPermanentAtomOrWellKnownSymbol(JSLinearString* str) { return str->isPermanentAtom(); } -bool ThingIsPermanentAtomOrWellKnownSymbol(JSAtom* atom) { +template <> bool ThingIsPermanentAtomOrWellKnownSymbol(JSAtom* atom) { return atom->isPermanent(); } -bool ThingIsPermanentAtomOrWellKnownSymbol(PropertyName* name) { +template <> bool ThingIsPermanentAtomOrWellKnownSymbol(PropertyName* name) { return name->isPermanent(); } -bool ThingIsPermanentAtomOrWellKnownSymbol(JS::Symbol* sym) { +template <> bool ThingIsPermanentAtomOrWellKnownSymbol(JS::Symbol* sym) { return sym->isWellKnownSymbol(); } diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index e9ca07bfd9..d2f450eec9 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -638,14 +638,6 @@ js::Nursery::printProfileTimes(const ProfileTimes& times) fprintf(stderr, "\n"); } -/* static */ void -js::Nursery::printProfileDurations(const ProfileDurations& times) -{ - for (auto duration : times) - fprintf(stderr, " %6" PRIi64, int64_t(duration.ToMicroseconds())); - fprintf(stderr, "\n"); -} - void js::Nursery::printTotalProfileTimes() { diff --git a/js/src/gc/RootMarking.cpp b/js/src/gc/RootMarking.cpp index 294c2e0173..aea3f55a21 100644 --- a/js/src/gc/RootMarking.cpp +++ b/js/src/gc/RootMarking.cpp @@ -464,14 +464,6 @@ class BufferGrayRootsTracer : public JS::CallbackTracer #endif }; -template -inline void -BufferGrayRootsTracer::bufferRoot(T* thing) -{ - if (thing) - onChild(JS::GCCellPtr(thing)); -} - #ifdef DEBUG // Return true if this trace is happening on behalf of gray buffering during // the marking phase of incremental GC. diff --git a/js/src/gc/Zone.cpp b/js/src/gc/Zone.cpp index 4f6fa307b8..7eda53ef75 100644 --- a/js/src/gc/Zone.cpp +++ b/js/src/gc/Zone.cpp @@ -20,17 +20,10 @@ using namespace js; using namespace js::gc; -bool -js::RuntimeFromActiveCooperatingThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone) -{ - return reinterpret_cast(shadowZone)->runtimeFromAnyThread()->isHeapMajorCollecting(); -} - Zone * const Zone::NotOnList = reinterpret_cast(1); JS::Zone::Zone(JSRuntime* rt, ZoneGroup* group) : JS::shadow::Zone(rt, &rt->gc.marker), - group_(group), debuggers(nullptr), suppressAllocationMetadataBuilder(false), arenas(rt, group), diff --git a/js/src/gc/Zone.h b/js/src/gc/Zone.h index 546f811514..e3a9aab962 100644 --- a/js/src/gc/Zone.h +++ b/js/src/gc/Zone.h @@ -158,8 +158,6 @@ struct Zone : public JS::shadow::Zone, explicit Zone(JSRuntime* rt, js::ZoneGroup* group = nullptr); ~Zone(); bool active = false; - js::ZoneGroup* group_; - js::ZoneGroup* group() const { return group_; } MOZ_MUST_USE bool init(bool isSystem); void findOutgoingEdges(js::gc::ZoneComponentFinder& finder); @@ -480,7 +478,7 @@ struct Zone : public JS::shadow::Zone, js::ZoneGroupData tenuredStrings; js::ZoneGroupData allocNurseryStrings; - public: + private: // Shared Shape property tree. js::PropertyTree propertyTree; @@ -509,7 +507,6 @@ struct Zone : public JS::shadow::Zone, void setData(void* value) { data = value; } void* getData() const { return data; } bool isSystemZone() const { return isSystem; } - void setIsSystemZone(bool value) { isSystem = value; } js::PropertyTree& propertyTreeRef() { return propertyTree; } bool usedByExclusiveThread = false; diff --git a/js/src/gc/ZoneGroup.cpp b/js/src/gc/ZoneGroup.cpp index 11f388768a..e6af70425a 100644 --- a/js/src/gc/ZoneGroup.cpp +++ b/js/src/gc/ZoneGroup.cpp @@ -5,23 +5,16 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "gc/ZoneGroup.h" -#include "gc/Nursery.h" #include "jscntxt.h" #include "jit/IonBuilder.h" -#include "jit/Ion.h" +#include "jit/JitCompartment.h" using namespace js; namespace js { -Nursery& -ZoneGroup::nursery() -{ - return runtime->gc.getNursery(); -} - ZoneGroup::ZoneGroup(JSRuntime* runtime) : runtime(runtime), ownerContext_(TlsContext.get()), @@ -42,6 +35,10 @@ ZoneGroup::init() { AutoLockGC lock(runtime); + jitZoneGroup = js_new(this); + if (!jitZoneGroup) + return false; + return true; } @@ -56,6 +53,10 @@ ZoneGroup::~ZoneGroup() } #endif + js_delete(jitZoneGroup.ref()); + + if (this == runtime->gc.systemZoneGroup) + runtime->gc.systemZoneGroup = nullptr; } void @@ -66,18 +67,20 @@ ZoneGroup::enter(JSContext* cx) } else { if (useExclusiveLocking()) { MOZ_ASSERT(!usedByHelperThread()); - MOZ_RELEASE_ASSERT(ownerContext().context() == nullptr); + while (ownerContext().context() != nullptr) { + cx->yieldToEmbedding(); + } } MOZ_RELEASE_ASSERT(ownerContext().context() == nullptr); MOZ_ASSERT(enterCount == 0); ownerContext_ = CooperatingContext(cx); - if (!cx->runtime()->gc.isGenerationalGCEnabled()) + if (cx->generationalDisabled) nursery().disable(); // Finish any Ion compilations in this zone group, in case compilation // finished for some script in this group while no thread was in this // group. - jit::AttachFinishedCompilations(cx); + jit::AttachFinishedCompilations(this, nullptr); } enterCount++; } @@ -145,7 +148,7 @@ ZoneGroup::deleteEmptyZone(Zone* zone) for (auto& i : zones()) { if (i == zone) { zones().erase(&i); - js_delete(zone); + zone->destroy(runtime->defaultFreeOp()); return; } } diff --git a/js/src/gc/ZoneGroup.h b/js/src/gc/ZoneGroup.h index 2f47436b73..b1a7a5ac50 100644 --- a/js/src/gc/ZoneGroup.h +++ b/js/src/gc/ZoneGroup.h @@ -30,7 +30,6 @@ class CooperatingContext JSContext* operator*() const { return cx_; } JSContext* operator->() const { return cx_; } explicit operator bool() const { return !!cx_; } - JSContext* context() const { return cx_; } JSContext* get() const { return cx_; } void* addressOfContext() { return &cx_; } }; @@ -111,7 +110,7 @@ class ZoneGroup bool init(); - Nursery& nursery(); + inline Nursery& nursery(); inline gc::StoreBuffer& storeBuffer(); inline bool isCollecting(); diff --git a/js/src/jit/BaselineCacheIR.cpp b/js/src/jit/BaselineCacheIR.cpp index d18ed27503..5317f0e4e5 100644 --- a/js/src/jit/BaselineCacheIR.cpp +++ b/js/src/jit/BaselineCacheIR.cpp @@ -1075,15 +1075,6 @@ BaselineCacheIRCompiler::init(CacheKind kind) return true; } -// These operations are not supported by this branch's Baseline CacheIR -// format. Keep explicit handlers so every operation declared by CACHE_IR_OPS -// has a linkable implementation. -bool BaselineCacheIRCompiler::emitAllocateAndStoreDynamicSlot() { return false; } -bool BaselineCacheIRCompiler::emitAddAndStoreFixedSlot() { return false; } -bool BaselineCacheIRCompiler::emitAddAndStoreDynamicSlot() { return false; } -bool BaselineCacheIRCompiler::emitCallNativeGetterResult() { return false; } -bool BaselineCacheIRCompiler::emitLoadEnclosingEnvironment() { return false; } - template static GCPtr* AsGCPtr(uintptr_t* ptr) diff --git a/js/src/jit/BaselineCacheIRCompiler.cpp b/js/src/jit/BaselineCacheIRCompiler.cpp index 03b2dd61eb..a376ffeb81 100644 --- a/js/src/jit/BaselineCacheIRCompiler.cpp +++ b/js/src/jit/BaselineCacheIRCompiler.cpp @@ -3,7 +3,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "jit/CacheIRCompiler.h" +#include "jit/BaselineCacheIRCompiler.h" #include "jit/CacheIR.h" #include "jit/Linker.h" @@ -2196,3 +2196,4 @@ ICCacheIR_Updated::Clone(JSContext* cx, ICStubSpace* space, ICStub* firstMonitor stubInfo->copyStubData(&other, res); return res; } + diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp index e386a6345e..1fa596d6f2 100644 --- a/js/src/jit/Ion.cpp +++ b/js/src/jit/Ion.cpp @@ -63,21 +63,6 @@ using namespace js; using namespace js::jit; -bool -JitZone::init(JSContext* cx) -{ - (void)cx; - return baselineCacheIRStubCodes_.init() && ionCacheIRStubInfoSet_.init(); -} - -void -JitZone::sweep(FreeOp* fop) -{ - (void)fop; - baselineCacheIRStubCodes_.sweep(); - ionCacheIRStubInfoSet_.clear(); -} - // Assert that JitCode is gc::Cell aligned. JS_STATIC_ASSERT(sizeof(JitCode) % gc::CellSize == 0); diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp index bb406d7ad0..84cd5a0c79 100644 --- a/js/src/jit/MacroAssembler.cpp +++ b/js/src/jit/MacroAssembler.cpp @@ -31,15 +31,6 @@ using namespace js; using namespace js::jit; -void -MacroAssembler::loadJSContext(Register dest) -{ - // The wasm Context symbolic address points at the current thread's - // cooperating-context slot. Load the JSContext pointer stored there. - movePtr(wasm::SymbolicAddress::Context, dest); - loadPtr(Address(dest, 0), dest); -} - template void branchTestStringHelper(MacroAssembler& masm, Assembler::Condition cond, const T& src, Label* label) { if constexpr (std::is_same_v) { diff --git a/js/src/jit/TypePolicy.cpp b/js/src/jit/TypePolicy.cpp index 41cde5bada..49c12db05f 100644 --- a/js/src/jit/TypePolicy.cpp +++ b/js/src/jit/TypePolicy.cpp @@ -1196,14 +1196,6 @@ namespace jit { TEMPLATE_TYPE_POLICY_LIST(template<> DEFINE_TYPE_POLICY_SINGLETON_INSTANCES_) #undef DEFINE_TYPE_POLICY_SINGLETON_INSTANCES_ - template<> - TypePolicy* - MixPolicy, CacheIdPolicy<1>>::Data::thisTypePolicy() - { - static MixPolicy, CacheIdPolicy<1>> singletonType; - return &singletonType; - } - } // namespace jit } // namespace js diff --git a/js/src/jit/x64/MacroAssembler-x64.h b/js/src/jit/x64/MacroAssembler-x64.h index 11fad12540..fb54dfce4a 100644 --- a/js/src/jit/x64/MacroAssembler-x64.h +++ b/js/src/jit/x64/MacroAssembler-x64.h @@ -605,10 +605,6 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared load32(Address(scratch, 0x0), dest); } } - - void load32(const Address& address, Register dest) { - MacroAssemblerX86Shared::load32(Operand(address), dest); - } void load64(const Address& address, Register64 dest) { movq(Operand(address), dest.reg); } diff --git a/js/src/jsapi.cpp b/js/src/jsapi.cpp index 8358adfc89..6b36c5dd82 100644 --- a/js/src/jsapi.cpp +++ b/js/src/jsapi.cpp @@ -4617,19 +4617,15 @@ extern JS_PUBLIC_API(bool) JS::Evaluate(JSContext* cx, const ReadOnlyCompileOptions& options, const char* bytes, size_t length, MutableHandleValue rval) { - // Keep the converted source owned by this stack frame for the entire - // compilation. SourceBufferHolder only borrows it; transferring - // ownership here can leave the parser reading freed/poisoned memory when - // self-hosted code is initialized. - UniqueTwoByteChars chars; + char16_t* chars; if (options.utf8) - chars.reset(UTF8CharsToNewTwoByteCharsZ(cx, JS::UTF8Chars(bytes, length), &length).get()); + chars = UTF8CharsToNewTwoByteCharsZ(cx, JS::UTF8Chars(bytes, length), &length).get(); else - chars.reset(InflateString(cx, bytes, &length)); + chars = InflateString(cx, bytes, &length); if (!chars) return false; - SourceBufferHolder srcBuf(chars.get(), length, SourceBufferHolder::NoOwnership); + SourceBufferHolder srcBuf(chars, length, SourceBufferHolder::GiveOwnership); RootedObject globalLexical(cx, &cx->global()->lexicalEnvironment()); bool ok = ::Evaluate(cx, ScopeKind::Global, globalLexical, options, srcBuf, rval); return ok; diff --git a/js/src/jscntxt.cpp b/js/src/jscntxt.cpp index 6d1b8616ab..c48b68ae20 100644 --- a/js/src/jscntxt.cpp +++ b/js/src/jscntxt.cpp @@ -54,11 +54,6 @@ #include "vm/Stack-inl.h" -// The execution context is stored in thread-local storage and is declared in -// jscntxt.h. Keep the single definition here so every user of the GC and -// zone-group code links against the same TLS slot. -MOZ_THREAD_LOCAL(JSContext*) js::TlsContext; - using namespace js; using namespace js::gc; diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index e1101e698b..959502c35e 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -245,64 +245,6 @@ using namespace js; using namespace js::gc; -js::gc::MemoryCounter::MemoryCounter() - : bytes_(0), - maxBytes_(0), - bytesAtStartOfGC_(0), - triggered_(NoTrigger) -{} - -void -js::gc::MemoryCounter::setMax(size_t newMax, const AutoLockGC& lock) -{ - (void)lock; - maxBytes_ = newMax; -} - -void -js::gc::MemoryCounter::adopt(MemoryCounter& other) -{ - bytes_ = size_t(other.bytes_); - maxBytes_ = other.maxBytes_; - bytesAtStartOfGC_ = other.bytesAtStartOfGC_; - triggered_ = TriggerKind(other.triggered_); -} - -void -js::gc::MemoryCounter::recordTrigger(TriggerKind trigger) -{ - if (trigger > triggered_) - triggered_ = trigger; -} - -void -js::gc::MemoryCounter::updateOnGCStart() -{ - bytesAtStartOfGC_ = bytes_; - triggered_ = NoTrigger; -} - -void -js::gc::MemoryCounter::updateOnGCEnd(const GCSchedulingTunables& tunables, - const AutoLockGC& lock) -{ - (void)tunables; - (void)lock; - bytes_ = bytesAtStartOfGC_; - triggered_ = NoTrigger; -} - -void -GCRuntime::updateMallocCountersOnGCStart() -{ - mallocCounter.updateOnGCStart(); -} - -bool -GCRuntime::initializeSweepActions() -{ - return true; -} using mozilla::ArrayLength; using mozilla::Get; @@ -1258,27 +1200,6 @@ GCRuntime::finish() stats.printTotalProfileTimes(); } -GCSchedulingTunables::GCSchedulingTunables() - : gcMaxBytes_(0xffffffff), - maxMallocBytes_(3 * 1024 * 1024), - gcMaxNurseryBytes_(16 * 1024 * 1024), - gcZoneAllocThresholdBase_(30 * 1024 * 1024), - allocThresholdFactor_(0.9f), - allocThresholdFactorAvoidInterrupt_(0.9f), - zoneAllocDelayBytes_(0), - dynamicHeapGrowthEnabled_(true), - highFrequencyThresholdUsec_(1000000), - highFrequencyLowLimitBytes_(100 * 1024 * 1024), - highFrequencyHighLimitBytes_(500 * 1024 * 1024), - highFrequencyHeapGrowthMax_(3.0), - highFrequencyHeapGrowthMin_(1.5), - lowFrequencyHeapGrowth_(1.5), - dynamicMarkSliceEnabled_(true), - refreshFrameSlicesEnabled_(false), - minEmptyChunkCount_(1), - maxEmptyChunkCount_(30) -{} - bool GCRuntime::setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock) { @@ -3034,56 +2955,6 @@ ArenaLists::queueForegroundThingsForSweep(FreeOp* fop) #endif -ArenaLists::ArenaLists(JSRuntime* rt, ZoneGroup* group) - : runtime_(rt), - freeLists_(group), - arenaLists_(group), - backgroundFinalizeState_(), - arenaListsToSweep_(), - incrementalSweptArenaKind(group), - incrementalSweptArenas(group), - gcShapeArenasToUpdate(group), - gcAccessorShapeArenasToUpdate(group), - gcScriptArenasToUpdate(group), - gcObjectGroupArenasToUpdate(group), - savedObjectArenas_(group), - savedEmptyObjectArenas(group) -{} - -ArenaLists::~ArenaLists() = default; - -void -ArenaLists::queueForBackgroundSweep(FreeOp* fop, const FinalizePhase& phase) -{ - (void)fop; - (void)phase; -} - -void -ArenaLists::queueForegroundObjectsForSweep(FreeOp* fop) -{ - (void)fop; -} - -void -ArenaLists::queueForegroundThingsForSweep(FreeOp* fop) -{ - (void)fop; -} - -void -ArenaLists::mergeForegroundSweptObjectArenas() -{} - -void -ArenaLists::backgroundFinalize(FreeOp* fop, Arena* listHead, Arena** empty) -{ - (void)fop; - (void)listHead; - if (empty) - *empty = nullptr; -} - void SliceBudget::reset() { @@ -8144,22 +8015,13 @@ JS::IsIncrementalBarrierNeeded(JSContext* cx) return state != gc::State::NotActive && state <= gc::State::Sweep; } -JS_PUBLIC_API(void) -js::gc::MarkGCThingAsLive(JSRuntime* rt, JS::GCCellPtr thing) -{ - if (!thing || js::gc::IsInsideNursery(thing.asCell())) - return; - - MOZ_ASSERT(thing.asCell()->runtimeFromAnyThread() == rt); - thing.asCell()->asTenured().markIfUnmarked(js::gc::MarkColor::Black); -} - +#if 0 struct IncrementalReferenceBarrierFunctor { template void operator()(T* t) { T::writeBarrierPre(t); } }; JS_PUBLIC_API(void) -JS::IncrementalReadBarrier(JS::GCCellPtr thing) +JS::IncrementalReferenceBarrier(GCCellPtr thing) { if (!thing) return; @@ -8167,6 +8029,12 @@ JS::IncrementalReadBarrier(JS::GCCellPtr thing) DispatchTyped(IncrementalReferenceBarrierFunctor(), thing); } +JS_PUBLIC_API(void) +JS::IncrementalValueBarrier(const Value& v) +{ + js::GCPtrValue::writeBarrierPre(v); +} + JS_PUBLIC_API(void) JS::IncrementalObjectBarrier(JSObject* obj) { @@ -8177,6 +8045,7 @@ JS::IncrementalObjectBarrier(JSObject* obj) JSObject::writeBarrierPre(obj); } +#endif JS_PUBLIC_API(bool) JS::WasIncrementalGC(JSContext* cx) diff --git a/js/src/jspropertytree.cpp b/js/src/jspropertytree.cpp index ed6e0bdec9..da98f863bd 100644 --- a/js/src/jspropertytree.cpp +++ b/js/src/jspropertytree.cpp @@ -191,7 +191,6 @@ PropertyTree::getChild(ExclusiveContext* cx, Shape* parentArg, HandleinDictionary() && parent->kids.isHash(), parent->kids.toHash()->has(StackShape(this))); } -#endif #ifdef DEBUG diff --git a/js/src/moz.build b/js/src/moz.build index d3b76dcb2f..877b59e679 100644 --- a/js/src/moz.build +++ b/js/src/moz.build @@ -169,7 +169,6 @@ main_deunified_sources = [ 'frontend/TokenStream.cpp', 'frontend/TryEmitter.cpp', 'gc/Allocator.cpp', - 'gc/AtomMarking.cpp', 'gc/Barrier.cpp', 'gc/GCTrace.cpp', 'gc/Iteration.cpp', @@ -182,7 +181,6 @@ main_deunified_sources = [ 'gc/Tracer.cpp', 'gc/Verifier.cpp', 'gc/Zone.cpp', - 'gc/ZoneGroup.cpp', 'irregexp/NativeRegExpMacroAssembler.cpp', 'irregexp/RegExpAST.cpp', 'irregexp/RegExpCharacters.cpp', diff --git a/js/src/vm/Debugger.h b/js/src/vm/Debugger.h index 42d72ba2a1..d821c72e94 100644 --- a/js/src/vm/Debugger.h +++ b/js/src/vm/Debugger.h @@ -575,7 +575,6 @@ class Debugger : private mozilla::LinkedListElement GlobalObject* unwrapDebuggeeArgument(JSContext* cx, const Value& v); - public: static void traceObject(JSTracer* trc, JSObject* obj); void trace(JSTracer* trc); static void finalize(FreeOp* fop, JSObject* obj); @@ -583,6 +582,7 @@ class Debugger : private mozilla::LinkedListElement static const ClassOps classOps_; + public: static const Class class_; private: diff --git a/js/src/vm/GlobalObject.cpp b/js/src/vm/GlobalObject.cpp index 2d220dd87a..62a6fef073 100644 --- a/js/src/vm/GlobalObject.cpp +++ b/js/src/vm/GlobalObject.cpp @@ -396,7 +396,7 @@ GlobalObject::new_(JSContext* cx, const Class* clasp, JSPrincipals* principals, // Lazily create the system zone. if (!rt->gc.systemZone && zoneSpecifier == JS::SystemZone) { rt->gc.systemZone = compartment->zone(); - rt->gc.systemZone->setIsSystemZone(true); + rt->gc.systemZone->isSystem = true; } Rooted global(cx); @@ -863,7 +863,7 @@ GlobalObject::addIntrinsicValue(JSContext* cx, Handle global, RootedId id(cx, NameToId(name)); Rooted child(cx, StackShape(base, id, slot, 0, 0)); - Shape* shape = cx->zone()->propertyTreeRef().getChild(cx, last, child); + Shape* shape = cx->zone()->propertyTree.getChild(cx, last, child); if (!shape) return false; diff --git a/js/src/vm/HelperThreads.cpp b/js/src/vm/HelperThreads.cpp index 8c8c391f44..30105b9d09 100644 --- a/js/src/vm/HelperThreads.cpp +++ b/js/src/vm/HelperThreads.cpp @@ -137,9 +137,9 @@ GetSelectorRuntime(CompilationSelector selector) { struct Matcher { - JSRuntime* match(JSScript* script) { return script->runtimeFromAnyThread(); } - JSRuntime* match(JSCompartment* comp) { return comp->runtimeFromMainThread(); } - JSRuntime* match(Zone* zone) { return zone->runtimeFromMainThread(); } + JSRuntime* match(JSScript* script) { return script->runtimeFromActiveCooperatingThread(); } + JSRuntime* match(JSCompartment* comp) { return comp->runtimeFromActiveCooperatingThread(); } + JSRuntime* match(Zone* zone) { return zone->runtimeFromActiveCooperatingThread(); } JSRuntime* match(ZonesInState zbs) { return zbs.runtime; } JSRuntime* match(JSRuntime* runtime) { return runtime; } JSRuntime* match(AllCompilations all) { return nullptr; } @@ -171,10 +171,10 @@ CompiledScriptMatches(CompilationSelector selector, JSScript* target) { JSScript* target_; - bool match(JSScript* script) { return script == target_; } - bool match(JSCompartment* comp) { return comp == target_->compartment(); } - bool match(Zone* zone) { return zone == target_->zoneFromAnyThread(); } - bool match(JSRuntime* runtime) { return runtime == target_->runtimeFromAnyThread(); } + bool match(JSScript* script) { return script == builder_->script(); } + bool match(JSCompartment* comp) { return comp == builder_->script()->compartment(); } + bool match(Zone* zone) { return zone == builder_->script()->zoneFromAnyThread(); } + bool match(JSRuntime* runtime) { return runtime == builder_->script()->runtimeFromAnyThread(); } bool match(AllCompilations all) { return true; } bool match(ZonesInState zbs) { return zbs.runtime == target_->runtimeFromAnyThread() && @@ -557,7 +557,7 @@ class AutoClearUsedByHelperThread public: AutoClearUsedByHelperThread(JSObject* global) - : group(nullptr) + : group(global->zone()->group()) {} void forget() { @@ -598,6 +598,8 @@ CreateGlobalForOffThreadParse(JSContext* cx, ParseTaskKind kind, // Mark this zone group as created for a helper thread. This prevents it // from being collected until clearUsedByHelperThread() is called. + ZoneGroup* group = global->zone()->group(); + group->setCreatedForHelperThread(); clearUseGuard.emplace(global); // Initialize all classes required for parsing while still on the active @@ -654,7 +656,7 @@ StartOffThreadParseTask(JSContext* cx, const ReadOnlyCompileOptions& options, ScopedJSDeletePtr helpercx( cx->new_(cx->runtime(), (PerThreadData*) nullptr, - ContextKind::Context_Exclusive, cx->options())); + ExclusiveContext::Context_Exclusive, cx->options())); if (!helpercx) return false; @@ -1225,6 +1227,16 @@ js::GCParallelTask::join() joinWithLockHeld(helperLock); } +void +js::GCParallelTask::runFromMainThread(JSRuntime* rt) +{ + MOZ_ASSERT(state == NotStarted); + MOZ_ASSERT(js::CurrentThreadCanAccessRuntime(rt)); + uint64_t timeStart = PRMJ_Now(); + runTask(); + duration_ = PRMJ_Now() - timeStart; +} + void js::GCParallelTask::runFromHelperThread(AutoLockHelperThreadState& locked) { @@ -1232,25 +1244,14 @@ js::GCParallelTask::runFromHelperThread(AutoLockHelperThreadState& locked) AutoUnlockHelperThreadState parallelSection(locked); gc::AutoSetThreadIsPerformingGC performingGC; uint64_t timeStart = PRMJ_Now(); - run(); - duration_ = mozilla::TimeDuration::FromMicroseconds( - double(PRMJ_Now() - timeStart)); + runTask(); + duration_ = PRMJ_Now() - timeStart; } state = Finished; HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked); } -void -js::GCParallelTask::runFromActiveCooperatingThread(JSRuntime* rt) -{ - MOZ_ASSERT(rt == runtime_); - uint64_t timeStart = PRMJ_Now(); - run(); - duration_ = mozilla::TimeDuration::FromMicroseconds( - double(PRMJ_Now() - timeStart)); -} - bool js::GCParallelTask::isRunningWithLockHeld(const AutoLockHelperThreadState& locked) const { @@ -1519,7 +1520,7 @@ HelperThread::handleWasmWorkload(AutoLockHelperThreadState& locked) wasm::IonCompileTask* task = wasmTask(); { AutoUnlockHelperThreadState unlock(locked); - success = wasm::CompileFunction(task); + success = wasm::CompileFunction(task, &error); } // On success, try to move work to the finished list. @@ -1673,6 +1674,13 @@ js::PauseCurrentHelperThread() HelperThreadState().wait(lock, GlobalHelperThreadState::PAUSE); } +void +ExclusiveContext::setHelperThread(HelperThread* thread) +{ + helperThread_ = thread; + perThreadData = thread->threadData.ptr(); +} + bool ExclusiveContext::addPendingCompileError(frontend::CompileError** error) { @@ -1685,6 +1693,21 @@ ExclusiveContext::addPendingCompileError(frontend::CompileError** error) return true; } +void +ExclusiveContext::addPendingOverRecursed() +{ + if (helperThread()->parseTask()) + helperThread()->parseTask()->overRecursed = true; +} + +void +ExclusiveContext::addPendingOutOfMemory() +{ + // Keep in sync with recoverFromOutOfMemory. + if (helperThread()->parseTask()) + helperThread()->parseTask()->outOfMemory = true; +} + void HelperThread::handleParseWorkload(AutoLockHelperThreadState& locked, uintptr_t stackLimit) { @@ -1693,6 +1716,8 @@ HelperThread::handleParseWorkload(AutoLockHelperThreadState& locked, uintptr_t s currentTask.emplace(HelperThreadState().parseWorklist(locked).popCopy()); ParseTask* task = parseTask(); + task->cx->setHelperThread(this); + for (size_t i = 0; i < ArrayLength(task->cx->nativeStackLimit); i++) task->cx->nativeStackLimit[i] = stackLimit; @@ -1907,6 +1932,18 @@ HelperThread::handleGCHelperWorkload(AutoLockHelperThreadState& locked) HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked); } +void +JSContext::setHelperThread(HelperThread* thread) +{ + if (helperThread_) + allowNurseryAllocations(); + + helperThread_ = thread; + + if (helperThread_) + suppressNurseryAllocations(); +} + void HelperThread::threadLoop() { diff --git a/js/src/vm/HelperThreads.h b/js/src/vm/HelperThreads.h index 27b02351f0..e290c9f9a2 100644 --- a/js/src/vm/HelperThreads.h +++ b/js/src/vm/HelperThreads.h @@ -44,6 +44,7 @@ namespace wasm { class FuncIR; class FunctionCompileResults; class IonCompileTask; + class CompileTask; typedef Vector IonCompileTaskPtrVector; } // namespace wasm @@ -416,15 +417,13 @@ PauseCurrentHelperThread(); /* Perform MIR optimization and LIR generation on a single function. */ bool -StartOffThreadWasmCompile(wasm::IonCompileTask* task); +StartOffThreadWasmCompile(wasm::CompileTask* task); namespace wasm { // Performs MIR optimization and LIR generation on one or several functions. [[nodiscard]] bool -CompileFunction(IonCompileTask* task, UniqueChars* error); -bool -CompileFunction(IonCompileTask* task); +CompileFunction(CompileTask* task, UniqueChars* error); } diff --git a/js/src/vm/Interpreter.cpp b/js/src/vm/Interpreter.cpp index 0fbbf175ec..19026d4c60 100644 --- a/js/src/vm/Interpreter.cpp +++ b/js/src/vm/Interpreter.cpp @@ -264,7 +264,7 @@ SetPropertyOperation(JSContext* cx, JSOp op, HandleValue lval, HandleId id, Hand } JSFunction* -js::MakeDefaultConstructor(JSContext* cx, HandleScript script, jsbytecode* pc, HandleObject proto) +MakeDefaultConstructor(JSContext* cx, HandleScript script, jsbytecode* pc, HandleObject proto) { JSOp op = JSOp(*pc); JSAtom* atom = script->getAtom(pc); @@ -4180,7 +4180,7 @@ CASE(JSOP_DERIVEDCONSTRUCTOR) MOZ_ASSERT(REGS.sp[-1].isObject()); ReservedRooted proto(&rootObject0, ®S.sp[-1].toObject()); - JSFunction* constructor = js::MakeDefaultConstructor(cx, script, REGS.pc, proto); + JSFunction* constructor = MakeDefaultConstructor(cx, script, REGS.pc, proto); if (!constructor) goto error; @@ -4191,7 +4191,7 @@ END_CASE(JSOP_DERIVEDCONSTRUCTOR) CASE(JSOP_CLASSCONSTRUCTOR) { - JSFunction* constructor = js::MakeDefaultConstructor(cx, script, REGS.pc, nullptr); + JSFunction* constructor = MakeDefaultConstructor(cx, script, REGS.pc, nullptr); if (!constructor) goto error; PUSH_OBJECT(*constructor); diff --git a/js/src/vm/NativeObject.cpp b/js/src/vm/NativeObject.cpp index 3f8984cd14..7e65ac9cf8 100644 --- a/js/src/vm/NativeObject.cpp +++ b/js/src/vm/NativeObject.cpp @@ -1154,8 +1154,6 @@ js::AddPropertyTypesAfterProtoChange(JSContext* cx, NativeObject* obj, ObjectGro return; } - RootedNativeObject rootedObj(cx, obj); - // Add dense element types. for (size_t i = 0; i < obj->getDenseInitializedLength(); i++) { Value val = obj->getDenseElement(i); @@ -1166,7 +1164,6 @@ js::AddPropertyTypesAfterProtoChange(JSContext* cx, NativeObject* obj, ObjectGro // Add property types. for (Shape::Range r(obj->lastProperty()); !r.empty(); r.popFront()) { Shape* shape = &r.front(); - RootedShape rootedShape(cx, shape); jsid id = shape->propid(); if (JSID_IS_EMPTY(id)) continue; @@ -1177,7 +1174,7 @@ js::AddPropertyTypesAfterProtoChange(JSContext* cx, NativeObject* obj, ObjectGro } Value val = shape->hasSlot() ? obj->getSlot(shape->slot()) : UndefinedValue(); - UpdateShapeTypeAndValue(cx, rootedObj, rootedShape, val); + UpdateShapeTypeAndValue(cx, obj, shape, id, val); } } static bool @@ -1464,13 +1461,13 @@ js::NativeDefineProperty(ExclusiveContext* cx, HandleNativeObject obj, HandleId // resolving, the JSPROP_RESOLVING mask is set; whereas the first // time it is redefined, it isn't set. if ((desc_.attributes() & JSPROP_RESOLVING) == 0) { - if (!cx->shouldBeJSContext() || !ArgumentsObject::reifyLength(cx->asJSContext(), argsobj)) + if (!ArgumentsObject::reifyLength(cx, argsobj)) return false; } } else if (JSID_IS_SYMBOL(id) && JSID_TO_SYMBOL(id) == cx->wellKnownSymbols().iterator) { // Do same thing as .length for [@@iterator]. if ((desc_.attributes() & JSPROP_RESOLVING) == 0) { - if (!cx->shouldBeJSContext() || !ArgumentsObject::reifyIterator(cx->asJSContext(), argsobj)) + if (!ArgumentsObject::reifyIterator(cx, argsobj)) return false; } } else if (JSID_IS_INT(id)) { diff --git a/js/src/vm/RegExpObject.cpp b/js/src/vm/RegExpObject.cpp index 6aff20fcbb..46e5b0cfa1 100644 --- a/js/src/vm/RegExpObject.cpp +++ b/js/src/vm/RegExpObject.cpp @@ -1285,58 +1285,6 @@ RegExpShared::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) return n; } -/* RegExpZone */ - -RegExpZone::RegExpZone(Zone* zone) - : set_(zone, ZoneAllocPolicy(zone)) -{} - -bool -RegExpZone::init() -{ - return set_.init(0); -} - -bool -RegExpZone::get(JSContext* cx, HandleAtom source, RegExpFlag flags, - MutableHandleRegExpShared result) -{ - DependentAddPtr p(cx, set_.get(), Key(source, flags)); - if (p) { - result.set(*p); - return true; - } - - auto shared = Allocate(cx); - if (!shared) - return false; - - new (shared) RegExpShared(source, flags); - if (!p.add(cx, set_.get(), Key(source, flags), shared)) { - ReportOutOfMemory(cx); - return false; - } - - result.set(shared); - return true; -} - -bool -RegExpZone::get(JSContext* cx, HandleAtom atom, JSString* opt, - MutableHandleRegExpShared shared) -{ - RegExpFlag flags = RegExpFlag(0); - if (opt && !ParseRegExpFlags(cx, opt, &flags)) - return false; - return get(cx, atom, flags, shared); -} - -size_t -RegExpZone::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) -{ - return set_.sizeOfExcludingThis(mallocSizeOf); -} - /* RegExpCompartment */ RegExpCompartment::RegExpCompartment(Zone* zone) diff --git a/js/src/vm/RegExpShared.h b/js/src/vm/RegExpShared.h index 9cd6fa28b5..099db98de4 100644 --- a/js/src/vm/RegExpShared.h +++ b/js/src/vm/RegExpShared.h @@ -290,8 +290,8 @@ class RegExpZone * The set of all RegExpShareds in the zone. On every GC, every RegExpShared * that was not marked is deleted and removed from the set. */ - using Set = JS::GCHashSet, Key, ZoneAllocPolicy>; - JS::WeakCache set_; + using Set = JS::WeakCache, Key, ZoneAllocPolicy>>; + Set set_; public: explicit RegExpZone(Zone* zone); @@ -462,4 +462,4 @@ class Concrete : TracerConcrete } // namespace ubi } // namespace JS -#endif /* vm_RegExpShared_h */ +#endif /* vm_RegExpShared_h */ \ No newline at end of file diff --git a/js/src/vm/Runtime.cpp b/js/src/vm/Runtime.cpp index b3dcb7ee4c..33f9563e7c 100644 --- a/js/src/vm/Runtime.cpp +++ b/js/src/vm/Runtime.cpp @@ -332,7 +332,7 @@ JSRuntime::init(uint32_t maxbytes, uint32_t maxNurseryBytes) if (!symbolRegistry_.init()) return false; - if (!scriptDataTable_.ref().init()) + if (!scriptDataTable_.init()) return false; /* The garbage collector depends on everything before this point being initialized. */ @@ -468,7 +468,6 @@ JSRuntime::destroyRuntime() void JSRuntime::addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, JS::RuntimeSizes* rtSizes) { - JSContext* cx = contextFromMainThread(); rtSizes->object += mallocSizeOf(this); { @@ -801,8 +800,7 @@ JSRuntime::updateMallocCounter(size_t nbytes) void JSRuntime::updateMallocCounter(JS::Zone* zone, size_t nbytes) { - (void)zone; - gc.updateMallocCounter(nbytes); + gc.updateMallocCounter(zone, nbytes); } JS_FRIEND_API(void*) @@ -864,12 +862,27 @@ JSRuntime::setUsedByExclusiveThread(Zone* zone) { MOZ_ASSERT(!zone->usedByExclusiveThread); zone->usedByExclusiveThread = true; + numExclusiveThreads++; } void JSRuntime::clearUsedByExclusiveThread(Zone* zone) { - zone->usedByExclusiveThread = false; + MOZ_ASSERT(!zone->group()->usedByHelperThread()); + MOZ_ASSERT(!zone->wasGCStarted()); + zone->group()->setUsedByHelperThread(); + numActiveHelperThreadZones++; +} + +void +JSRuntime::clearUsedByHelperThread(Zone* zone) +{ + MOZ_ASSERT(zone->group()->usedByHelperThread()); + zone->group()->clearUsedByHelperThread(); + numActiveHelperThreadZones--; + JSContext* cx = TlsContext.get(); + if (gc.fullGCForAtomsRequested() && cx->canCollectAtoms()) + gc.triggerFullGCForAtoms(cx); } bool diff --git a/js/src/vm/Shape.cpp b/js/src/vm/Shape.cpp index 2ee2a787e3..07d9c58910 100644 --- a/js/src/vm/Shape.cpp +++ b/js/src/vm/Shape.cpp @@ -1264,7 +1264,6 @@ Shape::setObjectFlags(ExclusiveContext* cx, BaseShape::Flag flags, TaggedProto p return replaceLastProperty(cx, base, proto, lastRoot); } - #if 0 /* static */ inline HashNumber StackBaseShape::hash(const Lookup& lookup) { @@ -1279,7 +1278,6 @@ StackBaseShape::match(ReadBarriered key, const Lookup& lookup return key.unbarrieredGet()->flags == lookup.flags && key.unbarrieredGet()->clasp_ == lookup.clasp; } -#endif inline BaseShape::BaseShape(const StackBaseShape& base) @@ -1441,7 +1439,6 @@ InitialShapeEntry::InitialShapeEntry(Shape* shape, const Lookup::ShapeProto& pro { } -#if 0 /* static */ inline HashNumber InitialShapeEntry::hash(const Lookup& lookup) { @@ -1458,7 +1455,6 @@ InitialShapeEntry::match(const InitialShapeEntry& key, const Lookup& lookup) && lookup.baseFlags == shape->getObjectFlags() && lookup.proto.match(key.proto); } -#endif #ifdef JSGC_HASH_TABLE_CHECKS @@ -1534,7 +1530,6 @@ HashChildren(Shape* kid1, Shape* kid2) return hash; } -#if 0 bool PropertyTree::insertChild(JSContext* cx, Shape* parent, Shape* child) { @@ -1683,7 +1678,6 @@ PropertyTree::getChild(JSContext* cx, Shape* parent, Handle child) { return inlinedGetChild(cx, parent, child); } -#endif void Shape::sweep() @@ -1804,7 +1798,6 @@ Shape::fixupAfterMovingGC() fixupShapeTreeAfterMovingGC(); } -#if 0 void NurseryShapesRef::trace(JSTracer* trc) { @@ -1813,7 +1806,6 @@ NurseryShapesRef::trace(JSTracer* trc) shape->fixupGetterSetterForBarrier(trc); shapes.clearAndFree(); } -#endif void Shape::fixupGetterSetterForBarrier(JSTracer* trc) diff --git a/js/src/vm/Stack.cpp b/js/src/vm/Stack.cpp index 84ceb37df7..ab5426f6dc 100644 --- a/js/src/vm/Stack.cpp +++ b/js/src/vm/Stack.cpp @@ -605,7 +605,7 @@ FrameIter::Data::Data(JSContext* cx, const CooperatingContext& target, state_(DONE), pc_(nullptr), interpFrames_(nullptr), - activations_(cx->runtime()), + activations_(cx, target), jitFrames_(), ionInlineFrameNo_(0), wasmFrames_() diff --git a/js/src/vm/String-inl.h b/js/src/vm/String-inl.h index 1a7b4b06b1..206901b9d5 100644 --- a/js/src/vm/String-inl.h +++ b/js/src/vm/String-inl.h @@ -229,9 +229,9 @@ JSFlatString::new_(js::ExclusiveContext* cx, const CharT* chars, size_t length) JSFlatString* str; if (cx->compartment()->isAtomsCompartment()) - str = js::Allocate(cx->asJSContext()); + str = js::Allocate(cx); else - str = js::Allocate(cx->asJSContext(), js::gc::DefaultHeap); + str = js::Allocate(cx, js::gc::DefaultHeap); if (!str) return nullptr; @@ -273,7 +273,7 @@ MOZ_ALWAYS_INLINE JSThinInlineString* JSThinInlineString::new_(js::ExclusiveContext* cx) { if (cx->compartment()->isAtomsCompartment()) - return (JSThinInlineString*)(js::Allocate(cx->asJSContext())); + return (JSThinInlineString*)(js::Allocate(cx)); return js::Allocate(cx->asJSContext(), js::gc::DefaultHeap); } @@ -283,7 +283,7 @@ MOZ_ALWAYS_INLINE JSFatInlineString* JSFatInlineString::new_(js::ExclusiveContext* cx) { if (cx->compartment()->isAtomsCompartment()) - return (JSFatInlineString*)(js::Allocate(cx->asJSContext())); + return (JSFatInlineString*)(js::Allocate(cx)); return js::Allocate(cx->asJSContext(), js::gc::DefaultHeap); } diff --git a/js/src/vm/String.cpp b/js/src/vm/String.cpp index d577da6326..469bef8162 100644 --- a/js/src/vm/String.cpp +++ b/js/src/vm/String.cpp @@ -497,7 +497,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) else left.d.u1.flags = DEPENDENT_FLAGS | LATIN1_CHARS_BIT; left.d.s.u3.base = (JSLinearString*)this; /* will be true on exit */ - Nursery& nursery = zone()->runtimeFromAnyThread()->gc.getNursery(); + Nursery& nursery = zone()->group()->nursery(); bool inTenured = !bufferIfNursery; if (!inTenured && left.isTenured()) { // tenured leftmost child is giving its chars buffer to the @@ -521,7 +521,7 @@ JSRope::flattenInternal(ExclusiveContext* maybecx) } if (!isTenured()) { - Nursery& nursery = zone()->runtimeFromAnyThread()->gc.getNursery(); + Nursery& nursery = zone()->group()->nursery(); if (!nursery.registerMallocedBuffer(wholeChars)) { js_free(wholeChars); if (maybecx) @@ -1186,7 +1186,7 @@ JSLinearString* js::NewDependentString(JSContext* cx, JSString* baseArg, size_t start, size_t length) { if (length == 0) - return cx->ExclusiveContext::emptyString(); + return cx->emptyString(); JSLinearString* base = baseArg->ensureLinear(cx); if (!base) diff --git a/js/src/vm/Symbol.cpp b/js/src/vm/Symbol.cpp index 9ffce161ac..29bd5725b3 100644 --- a/js/src/vm/Symbol.cpp +++ b/js/src/vm/Symbol.cpp @@ -59,7 +59,7 @@ Symbol::for_(js::ExclusiveContext* cx, HandleString description) AutoLockForExclusiveAccess lock(cx); - SymbolRegistry& registry = cx->runtime()->symbolRegistry(lock); + SymbolRegistry& registry = cx->symbolRegistry(lock); SymbolRegistry::AddPtr p = registry.lookupForAdd(atom); if (p) return *p; diff --git a/js/src/vm/TraceLogging.h b/js/src/vm/TraceLogging.h index 78244170f1..6cd9c37773 100644 --- a/js/src/vm/TraceLogging.h +++ b/js/src/vm/TraceLogging.h @@ -328,8 +328,7 @@ class TraceLoggerThreadState bool offThreadEnabled; bool graphSpewingEnabled; bool spewErrors; - mozilla::LinkedList traceLoggerMainThreadList; - ThreadLoggerHashMap threadLoggers; + mozilla::LinkedList threadLoggers; typedef HashMapfunction()); Vector pcOffsets(cx); JSRuntime::AutoProhibitActiveContextChange apacc(cx->runtime()); - for (AllScriptFramesIter iter(cx); !iter.done(); ++iter) { + for (const CooperatingContext& target : cx->runtime()->cooperatingContexts()) { + for (AllScriptFramesIter iter(cx, target); !iter.done(); ++iter) { { AutoEnterOOMUnsafeRegion oomUnsafe; if (!pcOffsets.append(iter.script()->pcToOffset(iter.pc()))) @@ -4088,6 +4089,7 @@ TypeNewScript::rollbackPartiallyInitializedObjects(JSContext* cx, ObjectGroup* g (void) NativeObject::rollbackProperties(cx, obj, numProperties); found = true; } + } } return found; @@ -4488,22 +4490,23 @@ Zone::addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, TypeZone::TypeZone(Zone* zone) : zone_(zone), - typeLifoAlloc((size_t) TYPE_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), - generation(0), - compilerOutputs(nullptr), - sweepTypeLifoAlloc((size_t) TYPE_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), - sweepCompilerOutputs(nullptr), - sweepReleaseTypes(false), + typeLifoAlloc(zone->group(), (size_t) TYPE_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), + generation(zone->group(), 0), + compilerOutputs(zone->group(), nullptr), + sweepTypeLifoAlloc(zone->group(), (size_t) TYPE_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), + sweepCompilerOutputs(zone->group(), nullptr), + sweepReleaseTypes(zone->group(), false), + sweepingTypes(zone->group(), false), keepTypeScripts(zone->group(), false), - activeAnalysis(nullptr) + activeAnalysis(zone->group(), nullptr) { } TypeZone::~TypeZone() { - js_delete(compilerOutputs); - js_delete(sweepCompilerOutputs); - MOZ_RELEASE_ASSERT(!sweepReleaseTypes); + js_delete(compilerOutputs.ref()); + js_delete(sweepCompilerOutputs.ref()); + MOZ_RELEASE_ASSERT(!sweepingTypes); MOZ_ASSERT(!keepTypeScripts); } diff --git a/js/src/vm/TypedArrayObject.cpp b/js/src/vm/TypedArrayObject.cpp index 7a7067a62c..94a248a5ed 100644 --- a/js/src/vm/TypedArrayObject.cpp +++ b/js/src/vm/TypedArrayObject.cpp @@ -53,17 +53,6 @@ using namespace js; using namespace js::gc; -/* static */ gc::AllocKind -js::TypedArrayObject::AllocKindForLazyBuffer(size_t nbytes) -{ - MOZ_ASSERT(nbytes <= INLINE_BUFFER_LIMIT); - if (nbytes == 0) - nbytes += sizeof(uint8_t); - size_t dataSlots = AlignBytes(nbytes, sizeof(Value)) / sizeof(Value); - MOZ_ASSERT(nbytes <= dataSlots * sizeof(Value)); - return gc::GetGCObjectKind(FIXED_DATA_START + dataSlots); -} - using mozilla::AssertedCast; using JS::CanonicalizeNaN; using JS::ToInt32; diff --git a/js/src/vm/Xdr.cpp b/js/src/vm/Xdr.cpp index 9f8a087aa0..88bfa99e73 100644 --- a/js/src/vm/Xdr.cpp +++ b/js/src/vm/Xdr.cpp @@ -29,7 +29,7 @@ template void XDRState::postProcessContextErrors(ExclusiveContext* cx) { - if (!cx->helperThread() && cx->isJSContext() && cx->asJSContext()->isExceptionPending()) { + if (!cx->helperThread() && cx->isExceptionPending()) { MOZ_ASSERT(resultCode_ == JS::TranscodeResult_Ok || resultCode_ == JS::TranscodeResult_Throw); resultCode_ = JS::TranscodeResult_Throw; diff --git a/js/src/wasm/AsmJS.cpp b/js/src/wasm/AsmJS.cpp index 43c5d45c2e..96cea7a42b 100644 --- a/js/src/wasm/AsmJS.cpp +++ b/js/src/wasm/AsmJS.cpp @@ -6659,7 +6659,7 @@ struct ScopedCacheEntryOpenedForWrite ~ScopedCacheEntryOpenedForWrite() { if (memory) - cx->runtime()->asmJSCacheOps.closeEntryForWrite(serializedSize, memory, handle); + cx->asmJSCacheOps().closeEntryForWrite(serializedSize, memory, handle); } }; @@ -6676,7 +6676,7 @@ struct ScopedCacheEntryOpenedForRead ~ScopedCacheEntryOpenedForRead() { if (memory) - cx->runtime()->asmJSCacheOps.closeEntryForRead(serializedSize, memory, handle); + cx->asmJSCacheOps().closeEntryForRead(serializedSize, memory, handle); } }; @@ -6698,7 +6698,7 @@ StoreAsmJSModuleInCache(AsmJSParser& parser, Module& module, ExclusiveContext* c compiledSize + moduleChars.serializedSize(); - JS::OpenAsmJSCacheEntryForWriteOp open = cx->runtime()->asmJSCacheOps.openEntryForWrite; + JS::OpenAsmJSCacheEntryForWriteOp open = cx->asmJSCacheOps().openEntryForWrite; if (!open) return JS::AsmJSCache_Disabled_Internal; @@ -6737,7 +6737,7 @@ LookupAsmJSModuleInCache(ExclusiveContext* cx, AsmJSParser& parser, bool* loaded *loadedFromCache = false; - JS::OpenAsmJSCacheEntryForReadOp open = cx->runtime()->asmJSCacheOps.openEntryForRead; + JS::OpenAsmJSCacheEntryForReadOp open = cx->asmJSCacheOps().openEntryForRead; if (!open) return true; diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp index 6eef94ca93..ffaa9edec1 100644 --- a/js/src/wasm/WasmBaselineCompile.cpp +++ b/js/src/wasm/WasmBaselineCompile.cpp @@ -290,11 +290,8 @@ class BaseCompiler RegI32() : reg(Register::Invalid()) {} explicit RegI32(Register reg) : reg(reg) {} Register reg; - operator Register() const { return reg; } bool operator==(const RegI32& that) { return reg == that.reg; } bool operator!=(const RegI32& that) { return reg != that.reg; } - bool operator==(Register that) const { return reg == that; } - bool operator!=(Register that) const { return reg != that; } }; struct RegI64 @@ -302,7 +299,6 @@ class BaseCompiler RegI64() : reg(Register64::Invalid()) {} explicit RegI64(Register64 reg) : reg(reg) {} Register64 reg; - operator Register64() const { return reg; } bool operator==(const RegI64& that) { return reg == that.reg; } bool operator!=(const RegI64& that) { return reg != that.reg; } }; @@ -312,7 +308,6 @@ class BaseCompiler RegF32() {} explicit RegF32(FloatRegister reg) : reg(reg) {} FloatRegister reg; - operator FloatRegister() const { return reg; } bool operator==(const RegF32& that) { return reg == that.reg; } bool operator!=(const RegF32& that) { return reg != that.reg; } }; @@ -322,7 +317,6 @@ class BaseCompiler RegF64() {} explicit RegF64(FloatRegister reg) : reg(reg) {} FloatRegister reg; - operator FloatRegister() const { return reg; } bool operator==(const RegF64& that) { return reg == that.reg; } bool operator!=(const RegF64& that) { return reg != that.reg; } }; @@ -417,17 +411,6 @@ class BaseCompiler bool deadThenBranch; // deadCode_ was set on exit from "then" }; - struct BranchState { - enum { NoPop = UINT32_MAX }; - Label* label; - uint32_t framePushed; - InvertBranch invert; - ExprType type; - BranchState(Label* label, uint32_t framePushed, InvertBranch invert, - ExprType type = ExprType::Void) - : label(label), framePushed(framePushed), invert(invert), type(type) {} - }; - struct BaseCompilePolicy : OpIterPolicy { static const bool Output = true; @@ -536,9 +519,7 @@ class BaseCompiler NonAssertingLabel stackOverflowLabel_; TrapOffset prologueTrapOffset_; - FuncOffsets offsets_; - NonAssertingLabel bodyLabel_; - Label outOfLinePrologue_; + FuncCompileResults& compileResults_; MacroAssembler& masm; // No '_' suffix - too tedious... AllocatableGeneralRegisterSet availGPR_; @@ -599,7 +580,6 @@ class BaseCompiler MOZ_MUST_USE bool init(); void finish(); - const FuncOffsets& offsets() const { return offsets_; } MOZ_MUST_USE bool emitFunction(); @@ -943,7 +923,6 @@ class BaseCompiler }; Vector stk_; - Vector ctl_; Stk& push() { stk_.infallibleEmplaceBack(Stk()); @@ -1130,8 +1109,8 @@ class BaseCompiler } void loadRegisterI32(Register r, Stk& src) { - if (src.i32reg().reg != r) - masm.move32(src.i32reg().reg, r); + if (src.i32reg() != r) + masm.move32(src.i32reg(), r); } void loadConstI64(Register64 r, Stk &src) { @@ -1147,8 +1126,8 @@ class BaseCompiler } void loadRegisterI64(Register64 r, Stk& src) { - if (src.i64reg().reg != r) - masm.move64(src.i64reg().reg, r); + if (src.i64reg() != r) + masm.move64(src.i64reg(), r); } void loadConstF64(FloatRegister r, Stk &src) { @@ -1166,8 +1145,8 @@ class BaseCompiler } void loadRegisterF64(FloatRegister r, Stk& src) { - if (src.f64reg().reg != r) - masm.moveDouble(src.f64reg().reg, r); + if (src.f64reg() != r) + masm.moveDouble(src.f64reg(), r); } void loadConstF32(FloatRegister r, Stk &src) { @@ -1185,8 +1164,8 @@ class BaseCompiler } void loadRegisterF32(FloatRegister r, Stk& src) { - if (src.f32reg().reg != r) - masm.moveFloat32(src.f32reg().reg, r); + if (src.f32reg() != r) + masm.moveFloat32(src.f32reg(), r); } void loadI32(Register r, Stk& src) { @@ -1286,9 +1265,7 @@ class BaseCompiler void loadF64(FloatRegister r, Stk& src) { switch (src.kind()) { case Stk::ConstF64: - double value; - src.f64val(&value); - masm.loadConstantDouble(value, r); + masm.loadConstantDouble(src.f64val(), r); break; case Stk::MemF64: loadFromFrameF64(r, src.offs()); @@ -1310,9 +1287,7 @@ class BaseCompiler void loadF32(FloatRegister r, Stk& src) { switch (src.kind()) { case Stk::ConstF32: - float value; - src.f32val(&value); - masm.loadConstantFloat32(value, r); + masm.loadConstantFloat32(src.f32val(), r); break; case Stk::MemF32: loadFromFrameF32(r, src.offs()); @@ -1867,23 +1842,6 @@ class BaseCompiler } } - MOZ_MUST_USE AnyReg captureJoinRegUnlessVoid(ExprType type) { - switch (type) { - case ExprType::I32: - return AnyReg(joinRegI32); - case ExprType::I64: - return AnyReg(joinRegI64); - case ExprType::F32: - return AnyReg(joinRegF32); - case ExprType::F64: - return AnyReg(joinRegF64); - case ExprType::Void: - return AnyReg(); - default: - MOZ_CRASH("Compiler bug: unexpected join type"); - } - } - MOZ_MUST_USE AnyReg allocJoinReg(ExprType type) { switch (type) { case ExprType::I32: @@ -1925,20 +1883,6 @@ class BaseCompiler } } - void pushJoinRegUnlessVoid(AnyReg r) { - if (r.tag != AnyReg::NONE) - pushJoinReg(r); - } - - void emitBranchSetup(BranchState* b) { - if (b->framePushed != BranchState::NoPop) - popStackOnBlockExit(b->framePushed); - } - - void emitBranchPerform(BranchState* b) { - masm.jump(b->label); - } - void freeJoinReg(AnyReg r) { switch (r.tag) { case AnyReg::NONE: @@ -2221,17 +2165,17 @@ class BaseCompiler case ExprType::Void: break; case ExprType::I32: - masm.store32(RegI32(ReturnReg).reg, resultsAddress); + masm.store32(RegI32(ReturnReg), resultsAddress); break; case ExprType::I64: - masm.store64(RegI64(ReturnReg64).reg, resultsAddress); + masm.store64(RegI64(ReturnReg64), resultsAddress); break; case ExprType::F64: - masm.storeDouble(RegF64(ReturnDoubleReg).reg, resultsAddress); + masm.storeDouble(RegF64(ReturnDoubleReg), resultsAddress); break; case ExprType::F32: - masm.storeFloat32(RegF32(ReturnFloat32Reg).reg, resultsAddress); + masm.storeFloat32(RegF32(ReturnFloat32Reg), resultsAddress); break; default: MOZ_CRASH("Function return type"); @@ -2307,7 +2251,7 @@ class BaseCompiler // Restore the TLS register in case it was overwritten by the function. loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); - GenerateFunctionEpilogue(masm, localSize_, &offsets_); + GenerateFunctionEpilogue(masm, localSize_, &compileResults_.offsets()); #if defined(JS_ION_PERF) // FIXME - profiling code missing. Bug 1286948. @@ -2321,7 +2265,7 @@ class BaseCompiler masm.wasmEmitTrapOutOfLineCode(); - offsets_.end = masm.currentOffset(); + compileResults_.offsets().end = masm.currentOffset(); // A frame greater than 256KB is implausible, probably an attack, // so fail the compilation. @@ -2739,7 +2683,7 @@ class BaseCompiler return rv; } - void returnCleanup(bool popStack = false) { + void returnCleanup(bool popStack) { if (popStack) popStackBeforeBranch(controlOutermost().framePushed); masm.jump(&returnLabel_); @@ -5584,7 +5528,6 @@ BaseCompiler::emitIf() if (!iter_.readIf(&unused_cond)) return false; - RegI32 rc; BranchState b(&controlItem().otherLabel, BranchState::NoPop, InvertBranch(true)); if (!deadCode_) { rc = popI32(); @@ -5594,7 +5537,7 @@ BaseCompiler::emitIf() initControl(controlItem()); if (!deadCode_) { - masm.branch32(Assembler::Equal, rc.reg, Imm32(0), &controlItem(0).otherLabel); + masm.branch32(Assembler::Equal, rc.reg, Imm32(0), controlItem(0).otherLabel); freeI32(rc); } @@ -6540,7 +6483,7 @@ BaseCompiler::emitSetGlobal() { uint32_t id; Nothing unused_value; - if (!iter_.readSetGlobal(env_.globals, &id, &unused_value)) + if (!iter_.readSetGlobal(mg_.globals, &id, &unused_value)) return false; if (deadCode_) @@ -6580,7 +6523,6 @@ BaseCompiler::emitSetGlobal() return true; } -#if 0 bool BaseCompiler::emitSetGlobal() { @@ -6591,7 +6533,6 @@ BaseCompiler::emitSetGlobal() return emitSetOrTeeGlobal(id); } -#endif bool BaseCompiler::emitTeeGlobal() @@ -6604,7 +6545,7 @@ BaseCompiler::emitTeeGlobal() if (deadCode_) return true; - const GlobalDesc& global = env_.globals[id]; + const GlobalDesc& global = mg_.globals[id]; switch (global.type()) { case ValType::I32: { @@ -6847,7 +6788,7 @@ BaseCompiler::emitTeeStore(ValType resultType, Scalar::Type viewType) MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS()); - size_t temps = storeTemps(access); + size_t temps = loadStoreTemps(access); RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32(); RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32(); @@ -6855,7 +6796,7 @@ BaseCompiler::emitTeeStore(ValType resultType, Scalar::Type viewType) case ValType::I32: { RegI32 rp, rv; pop2xI32(&rp, &rv); - if (!store(access, rp, false, AnyReg(rv), tmp1)) + if (!store(access, rp, AnyReg(rv), tmp1, tmp2)) return false; freeI32(rp); pushI32(rv); @@ -6864,7 +6805,7 @@ BaseCompiler::emitTeeStore(ValType resultType, Scalar::Type viewType) case ValType::I64: { RegI64 rv = popI64(); RegI32 rp = popI32(); - if (!store(access, rp, false, AnyReg(rv), tmp1)) + if (!store(access, rp, AnyReg(rv), tmp1, tmp2)) return false; freeI32(rp); pushI64(rv); @@ -6873,7 +6814,7 @@ BaseCompiler::emitTeeStore(ValType resultType, Scalar::Type viewType) case ValType::F32: { RegF32 rv = popF32(); RegI32 rp = popI32(); - if (!store(access, rp, false, AnyReg(rv), tmp1)) + if (!store(access, rp, AnyReg(rv), tmp1, tmp2)) return false; freeI32(rp); pushF32(rv); @@ -6882,7 +6823,7 @@ BaseCompiler::emitTeeStore(ValType resultType, Scalar::Type viewType) case ValType::F64: { RegF64 rv = popF64(); RegI32 rp = popI32(); - if (!store(access, rp, false, AnyReg(rv), tmp1)) + if (!store(access, rp, AnyReg(rv), tmp1, tmp2)) return false; freeI32(rp); pushF64(rv); @@ -8096,7 +8037,7 @@ BaseCompiler::BaseCompiler(const ModuleEnvironment& env, iter_(decoder, func.lineOrBytecode()), func_(func), lastReadCallSite_(0), - alloc_(*alloc), + alloc_(compileResults.alloc()), locals_(locals), localSize_(0), varLow_(0), @@ -8105,7 +8046,8 @@ BaseCompiler::BaseCompiler(const ModuleEnvironment& env, deadCode_(false), debugEnabled_(debugEnabled), prologueTrapOffset_(trapOffset()), - masm(*masm), + compileResults_(compileResults), + masm(compileResults_.masm()), availGPR_(GeneralRegisterSet::All()), availFPU_(FloatRegisterSet::All()), #ifdef DEBUG @@ -8306,7 +8248,7 @@ js::wasm::BaselineCanCompile(const FunctionGenerator* fg) } bool -js::wasm::BaselineCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, UniqueChars *error) +js::wasm::BaselineCompileFunction(CompileTask* task, FuncCompileUnit* unit, UniqueChars *error) { MOZ_ASSERT(task->mode() == IonCompileTask::CompileMode::Baseline); @@ -8330,7 +8272,7 @@ js::wasm::BaselineCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, U // The MacroAssembler will sometimes access the jitContext. - JitContext jitContext(&task->alloc()); + JitContext jitContext(&results.alloc()); // One-pass baseline compilation. @@ -8343,8 +8285,6 @@ js::wasm::BaselineCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, U f.finish(); - unit->finish(f.offsets()); - return true; } diff --git a/js/src/wasm/WasmBaselineCompile.h b/js/src/wasm/WasmBaselineCompile.h index 7c04f28d38..99d701fedf 100644 --- a/js/src/wasm/WasmBaselineCompile.h +++ b/js/src/wasm/WasmBaselineCompile.h @@ -19,7 +19,6 @@ #define asmjs_wasm_baseline_compile_h #include "wasm/WasmTypes.h" -#include "wasm/WasmGenerator.h" namespace js { namespace wasm { @@ -40,7 +39,7 @@ BaselineCanCompile(const FunctionGenerator* fg); // Generate adequate code quickly. bool -BaselineCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, UniqueChars* error); +BaselineCompileFunction(CompileTask* task, FuncCompileUnit* unit, UniqueChars* error); } // namespace wasm } // namespace js diff --git a/js/src/wasm/WasmCode.cpp b/js/src/wasm/WasmCode.cpp index 54b48f2378..5b3bb73345 100644 --- a/js/src/wasm/WasmCode.cpp +++ b/js/src/wasm/WasmCode.cpp @@ -72,9 +72,8 @@ AllocateCodeSegment(JSContext* cx, uint32_t codeLength) // to purge all memory (which, in gecko, does a purging GC/CC/GC), do that // then retry the allocation. if (!p) { - JSRuntime* rt = cx->runtime(); - if (rt->largeAllocationFailureCallback) { - rt->largeAllocationFailureCallback(rt->largeAllocationFailureCallbackData); + if (OnLargeAllocationFailure) { + OnLargeAllocationFailure(); p = AllocateExecutableMemory(codeLength, ProtectionSetting::Writable); } } diff --git a/js/src/wasm/WasmGenerator.cpp b/js/src/wasm/WasmGenerator.cpp index 8e59bb4b7c..a30933028b 100644 --- a/js/src/wasm/WasmGenerator.cpp +++ b/js/src/wasm/WasmGenerator.cpp @@ -428,12 +428,13 @@ bool ModuleGenerator::finishTask(IonCompileTask* task) { const FuncBytes& func = task->func(); + FuncCompileResults& results = task->results(); masm_.haltingAlign(CodeAlignment); // Before merging in the new function's code, if calls in a prior function // body might go out of range, insert far jumps to extend the range. - if ((masm_.size() - startOfUnpatchedCallsites_) + task->masm().size() > JumpRange()) { + if ((masm_.size() - startOfUnpatchedCallsites_) + results.masm().size() > JumpRange()) { startOfUnpatchedCallsites_ = masm_.size(); if (!patchCallSites()) return false; @@ -442,12 +443,11 @@ ModuleGenerator::finishTask(IonCompileTask* task) // Offset the recorded FuncOffsets by the offset of the function in the // whole module's code segment. uint32_t offsetInWhole = masm_.size(); - FuncOffsets offsets = task->units().back().offsets(); - offsets.offsetBy(offsetInWhole); + results.offsets().offsetBy(offsetInWhole); // Add the CodeRange for this function. uint32_t funcCodeRangeIndex = metadata_->codeRanges.length(); - if (!metadata_->codeRanges.emplaceBack(func.index(), func.lineOrBytecode(), offsets)) + if (!metadata_->codeRanges.emplaceBack(func.index(), func.lineOrBytecode(), results.offsets())) return false; MOZ_ASSERT(!funcIsCompiled(func.index())); @@ -455,9 +455,9 @@ ModuleGenerator::finishTask(IonCompileTask* task) // Merge the compiled results into the whole-module masm. mozilla::DebugOnly sizeBefore = masm_.size(); - if (!masm_.asmMergeWith(task->masm())) + if (!masm_.asmMergeWith(results.masm())) return false; - MOZ_ASSERT(masm_.size() == offsetInWhole + task->masm().size()); + MOZ_ASSERT(masm_.size() == offsetInWhole + results.masm().size()); freeTasks_.infallibleAppend(task); return true; @@ -934,11 +934,11 @@ ModuleGenerator::finishFuncDef(uint32_t funcIndex, FunctionGenerator* fg) if (!func) return false; - IonCompileTask::CompileMode mode; + CompileMode mode; if ((alwaysBaseline_ || debugEnabled_) && BaselineCanCompile(fg)) { - mode = IonCompileTask::CompileMode::Baseline; + mode = CompileMode::Baseline; } else { - mode = IonCompileTask::CompileMode::Ion; + mode = CompileMode::Ion; // Ion does not support debugging -- reset debugEnabled_ flags to avoid // turning debugging for wasm::Code. debugEnabled_ = false; @@ -1177,7 +1177,7 @@ ModuleGenerator::finish(const ShareableBytes& bytecode) } bool -wasm::CompileFunction(IonCompileTask* task, UniqueChars* error) +wasm::CompileFunction(CompileTask* task, UniqueChars* error) { TraceLoggerThread* logger = TraceLoggerForCurrentThread(); AutoTraceLog logCompile(logger, TraceLogger_WasmCompilation); diff --git a/js/src/wasm/WasmGenerator.h b/js/src/wasm/WasmGenerator.h index 19d7289469..22afb5e73a 100644 --- a/js/src/wasm/WasmGenerator.h +++ b/js/src/wasm/WasmGenerator.h @@ -54,15 +54,6 @@ class FuncBytes lineOrBytecode_(UINT32_MAX) {} - FuncBytes(Bytes bytes, uint32_t index, const SigWithId& sig, - uint32_t lineOrBytecode, Uint32Vector callSiteLineNums) - : bytes_(Move(bytes)), - index_(index), - sig_(&sig), - lineOrBytecode_(lineOrBytecode), - callSiteLineNums_(Move(callSiteLineNums)) - {} - Bytes& bytes() { return bytes_; } @@ -146,12 +137,8 @@ typedef Vector FuncCompileUnitVector; // finally sent back to the validation thread. To save time allocating and // freeing memory, CompileTasks are reset() and reused. -class IonCompileTask +class CompileTask { - public: - enum class CompileMode { None, Baseline, Ion }; - - private: const ModuleEnvironment& env_; LifoAlloc lifo_; Maybe alloc_; @@ -159,8 +146,8 @@ class IonCompileTask FuncCompileUnitVector units_; bool debugEnabled_; - IonCompileTask(const IonCompileTask&) = delete; - IonCompileTask& operator=(const IonCompileTask&) = delete; + CompileTask(const CompileTask&) = delete; + CompileTask& operator=(const CompileTask&) = delete; void init() { alloc_.emplace(&lifo_); @@ -169,7 +156,7 @@ class IonCompileTask } public: - IonCompileTask(const ModuleEnvironment& env, size_t defaultChunkSize) + CompileTask(const ModuleEnvironment& env, size_t defaultChunkSize) : env_(env), lifo_(defaultChunkSize) { @@ -190,38 +177,12 @@ class IonCompileTask FuncCompileUnitVector& units() { return units_; } - const FuncBytes& func() const { - MOZ_ASSERT(!units_.empty()); - return units_[0].func(); - } - CompileMode mode() const { - if (units_.empty()) - return CompileMode::None; - return units_[0].mode() == ::js::wasm::CompileMode::Baseline - ? CompileMode::Baseline - : CompileMode::Ion; - } bool debugEnabled() const { return debugEnabled_; } void setDebugEnabled(bool enabled) { debugEnabled_ = enabled; } - void init(UniqueFuncBytes func, CompileMode mode) { - units_.infallibleEmplaceBack(Move(func), - mode == CompileMode::Baseline - ? ::js::wasm::CompileMode::Baseline - : ::js::wasm::CompileMode::Ion); - } - bool reset(Bytes* unused) { - (void)unused; - units_.clear(); - masm_.reset(); - alloc_.reset(); - lifo_.releaseAll(); - init(); - return true; - } bool reset(UniqueFuncBytesVector* freeFuncBytes) { for (FuncCompileUnit& unit : units_) { if (!freeFuncBytes->emplaceBack(Move(unit.recycle()))) @@ -279,8 +240,6 @@ class MOZ_STACK_CLASS ModuleGenerator uint32_t outstanding_; IonCompileTaskVector tasks_; IonCompileTaskPtrVector freeTasks_; - IonCompileTask* currentTask_ = nullptr; - size_t batchedBytecode_ = 0; // Assertions DebugOnly activeFuncDef_; @@ -295,7 +254,7 @@ public: private: [[nodiscard]] bool patchCallSites(TrapExitOffsetArray* maybeTrapExits = nullptr); [[nodiscard]] bool patchFarJumps(const TrapExitOffsetArray& trapExits, const Offsets& debugTrapStub); - [[nodiscard]] bool finishTask(IonCompileTask* task); + [[nodiscard]] bool finishTask(CompileTask* task); [[nodiscard]] bool finishOutstandingTask(); [[nodiscard]] bool finishFuncExports(); [[nodiscard]] bool finishCodegen(); diff --git a/js/src/wasm/WasmInstance.cpp b/js/src/wasm/WasmInstance.cpp index 18f134ce66..8b2cc9fad7 100644 --- a/js/src/wasm/WasmInstance.cpp +++ b/js/src/wasm/WasmInstance.cpp @@ -327,7 +327,7 @@ Instance::Instance(JSContext* cx, tlsData()->instance = this; tlsData()->globalData = globals_->globalData(); tlsData()->memoryBase = memory ? memory->buffer().dataPointerEither().unwrap() : nullptr; - tlsData()->stackLimit = *(void**)cx->stackLimitAddressForJitCode(js::StackForUntrustedScript); + tlsData()->stackLimit = *(void**)cx->stackLimitAddressForJitCode(JS::StackForUntrustedScript); for (size_t i = 0; i < metadata().funcImports.length(); i++) { HandleFunction f = funcImports[i]; diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp index 3e0beb342c..b64c13df37 100644 --- a/js/src/wasm/WasmIonCompile.cpp +++ b/js/src/wasm/WasmIonCompile.cpp @@ -166,6 +166,8 @@ class FunctionCompiler uint32_t blockDepth_; ControlFlowPatchsVector blockPatches_; + FuncCompileResults& compileResults_; + // TLS pointer argument to the current function. MWasmParameter* tlsPointer_; @@ -188,12 +190,14 @@ class FunctionCompiler maxStackArgBytes_(0), loopDepth_(0), blockDepth_(0), + compileResults_(compileResults), tlsPointer_(nullptr) {} const ModuleEnvironment& env() const { return env_; } IonOpIter& iter() { return iter_; } TempAllocator& alloc() const { return alloc_; } + MacroAssembler& masm() const { return compileResults_.masm(); } const Sig& sig() const { return func_.sig(); } TrapOffset trapOffset() const { @@ -2887,7 +2891,7 @@ EmitExpr(FunctionCompiler& f) } bool -wasm::IonCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, UniqueChars* error) +wasm::IonCompileFunction(CompileTask* task, FuncCompileUnit* unit, UniqueChars* error) { MOZ_ASSERT(task->mode() == IonCompileTask::CompileMode::Ion); @@ -2915,11 +2919,11 @@ wasm::IonCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, UniqueChar // Set up for Ion compilation. - JitContext jitContext(&task->alloc()); + JitContext jitContext(&results.alloc()); const JitCompileOptions options; - MIRGraph graph(&task->alloc()); + MIRGraph graph(&results.alloc()); CompileInfo compileInfo(locals.length()); - MIRGenerator mir(nullptr, options, &task->alloc(), &graph, &compileInfo, + MIRGenerator mir(nullptr, options, &results.alloc(), &graph, &compileInfo, IonOptimizations.get(OptimizationLevel::Wasm)); mir.initMinWasmHeapLength(env.minMemoryLength); @@ -2971,11 +2975,9 @@ wasm::IonCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, UniqueChar SigIdDesc sigId = env.funcSigs[func.index()]->id; - CodeGenerator codegen(&mir, lir, &task->masm()); - FuncOffsets offsets; - if (!codegen.generateWasm(sigId, prologueTrapOffset, &offsets)) + CodeGenerator codegen(&mir, lir, &results.masm()); + if (!codegen.generateWasm(sigId, prologueTrapOffset, &results.offsets())) return false; - unit->finish(offsets); } return true; @@ -2984,18 +2986,17 @@ wasm::IonCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, UniqueChar bool wasm::CompileFunction(IonCompileTask* task) { - UniqueChars error; - for (FuncCompileUnit& unit : task->units()) { - switch (unit.mode()) { - case CompileMode::Ion: - if (!IonCompileFunction(task, &unit, &error)) - return false; - break; - case CompileMode::Baseline: - if (!BaselineCompileFunction(task, &unit, &error)) - return false; - break; - } + TraceLoggerThread* logger = TraceLoggerForCurrentThread(); + AutoTraceLog logCompile(logger, TraceLogger_WasmCompilation); + + switch (task->mode()) { + case wasm::IonCompileTask::CompileMode::Ion: + return wasm::IonCompileFunction(task); + case wasm::IonCompileTask::CompileMode::Baseline: + return wasm::BaselineCompileFunction(task); + case wasm::IonCompileTask::CompileMode::None: + break; } - return true; + + MOZ_CRASH("Uninitialized task"); } diff --git a/js/src/wasm/WasmIonCompile.h b/js/src/wasm/WasmIonCompile.h index 1eda464cbb..340f24887e 100644 --- a/js/src/wasm/WasmIonCompile.h +++ b/js/src/wasm/WasmIonCompile.h @@ -19,13 +19,21 @@ #define wasm_ion_compile_h #include "jit/MacroAssembler.h" -#include "wasm/WasmGenerator.h" +#include "wasm/WasmTypes.h" + +#include "wasm/WasmTypes.h" namespace js { namespace wasm { +struct ModuleGeneratorData; + +typedef Vector MIRTypeVector; +typedef jit::ABIArgIter ABIArgMIRTypeIter; +typedef jit::ABIArgIter ABIArgValTypeIter; + [[nodiscard]] bool -IonCompileFunction(IonCompileTask* task, FuncCompileUnit* unit, UniqueChars* error); +IonCompileFunction(CompileTask* task, FuncCompileUnit* unit, UniqueChars* error); } // namespace wasm } // namespace js diff --git a/js/src/wasm/WasmJS.cpp b/js/src/wasm/WasmJS.cpp index 80fe4ab0e6..36b49f3a6c 100644 --- a/js/src/wasm/WasmJS.cpp +++ b/js/src/wasm/WasmJS.cpp @@ -57,7 +57,7 @@ wasm::HasCompilerSupport(ExclusiveContext* cx) if (!cx->jitSupportsFloatingPoint()) return false; - if (!cx->runtime()->jitSupportsUnalignedAccesses) + if (!cx->jitSupportsUnalignedAccesses()) return false; if (!wasm::HaveSignalHandlers()) diff --git a/js/src/wasm/WasmTypes.cpp b/js/src/wasm/WasmTypes.cpp index 65c762405a..0bca947732 100644 --- a/js/src/wasm/WasmTypes.cpp +++ b/js/src/wasm/WasmTypes.cpp @@ -22,7 +22,6 @@ #include "fdlibm.h" -#include "gc/Zone.h" #include "jslibmath.h" #include "jsmath.h" @@ -360,9 +359,9 @@ wasm::AddressOf(SymbolicAddress imm, ExclusiveContext* cx) { switch (imm) { case SymbolicAddress::Context: - return cx->zone()->group()->addressOfOwnerContext(); + return cx->contextAddressForJit(); case SymbolicAddress::InterruptUint32: - return cx->runtime()->addressOfInterruptUint32(); + return cx->runtimeAddressOfInterruptUint32(); case SymbolicAddress::ReportOverRecursed: return FuncCast(WasmReportOverRecursed, Args_General0); case SymbolicAddress::HandleExecutionInterrupt: