diff --git a/js/public/Class.h b/js/public/Class.h index 40885e6082..d7aeffbc21 100644 --- a/js/public/Class.h +++ b/js/public/Class.h @@ -495,12 +495,6 @@ typedef bool */ typedef void (* JSFinalizeOp)(JSFreeOp* fop, JSObject* obj); - -/** Finalizes external strings created by JS_NewExternalString. */ -struct JSStringFinalizer { - void (*finalize)(JS::Zone* zone, const JSStringFinalizer* fin, char16_t* chars); -}; - /** * Check whether v is an instance of obj. Return false on error or exception, * true on success with true in *bp if v is an instance of obj, false in diff --git a/js/public/GCAPI.h b/js/public/GCAPI.h index 26574c1738..5b1c9a5f3f 100644 --- a/js/public/GCAPI.h +++ b/js/public/GCAPI.h @@ -9,9 +9,26 @@ #include "mozilla/Vector.h" #include "js/GCAnnotations.h" -#include "js/HeapAPI.h" #include "js/UniquePtr.h" +struct JSCompartment; +struct JSContext; +struct JSFreeOp; +class JSObject; +struct JSRuntime; +class JSString; + +#ifdef JS_BROKEN_GCC_ATTRIBUTE_WARNING +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wattributes" +#endif // JS_BROKEN_GCC_ATTRIBUTE_WARNING + +class JS_PUBLIC_API(JSTracer); + +#ifdef JS_BROKEN_GCC_ATTRIBUTE_WARNING +#pragma GCC diagnostic pop +#endif // JS_BROKEN_GCC_ATTRIBUTE_WARNING + namespace js { namespace gc { class GCRuntime; @@ -46,8 +63,273 @@ typedef enum JSGCInvocationKind { GC_SHRINK = 1 } JSGCInvocationKind; +typedef enum JSGCParamKey { + /** + * Maximum nominal heap before last ditch GC. + * + * Soft limit on the number of bytes we are allowed to allocate in the GC + * heap. Attempts to allocate gcthings over this limit will return null and + * subsequently invoke the standard OOM machinery, independent of available + * physical memory. + * + * Pref: javascript.options.mem.max + * Default: 0xffffffff + */ + JSGC_MAX_BYTES = 0, + + /** + * Initial value for the malloc bytes threshold. + * + * Pref: javascript.options.mem.high_water_mark + * Default: TuningDefaults::MaxMallocBytes + */ + JSGC_MAX_MALLOC_BYTES = 1, + + /** + * Maximum size of the generational GC nurseries. + * + * Pref: javascript.options.mem.nursery.max_kb + * Default: JS::DefaultNurseryBytes + */ + JSGC_MAX_NURSERY_BYTES = 2, + + /** Amount of bytes allocated by the GC. */ + JSGC_BYTES = 3, + + /** Number of times GC has been invoked. Includes both major and minor GC. */ + JSGC_NUMBER = 4, + + /** + * Select GC mode. + * + * See: JSGCMode in GCAPI.h + * prefs: javascript.options.mem.gc_per_zone and + * javascript.options.mem.gc_incremental. + * Default: JSGC_MODE_INCREMENTAL + */ + JSGC_MODE = 6, + + /** Number of cached empty GC chunks. */ + JSGC_UNUSED_CHUNKS = 7, + + /** Total number of allocated GC chunks. */ + JSGC_TOTAL_CHUNKS = 8, + + /** + * Max milliseconds to spend in an incremental GC slice. + * + * Pref: javascript.options.mem.gc_incremental_slice_ms + * Default: DefaultTimeBudget. + */ + JSGC_SLICE_TIME_BUDGET = 9, + + /** + * Maximum size the GC mark stack can grow to. + * + * Pref: none + * Default: MarkStack::DefaultCapacity + */ + JSGC_MARK_STACK_LIMIT = 10, + + /** + * GCs less than this far apart in time will be considered 'high-frequency + * GCs'. + * + * See setGCLastBytes in jsgc.cpp. + * + * Pref: javascript.options.mem.gc_high_frequency_time_limit_ms + * Default: HighFrequencyThresholdUsec + */ + JSGC_HIGH_FREQUENCY_TIME_LIMIT = 11, + + /** + * Start of dynamic heap growth. + * + * Pref: javascript.options.mem.gc_high_frequency_low_limit_mb + * Default: HighFrequencyLowLimitBytes + */ + JSGC_HIGH_FREQUENCY_LOW_LIMIT = 12, + + /** + * End of dynamic heap growth. + * + * Pref: javascript.options.mem.gc_high_frequency_high_limit_mb + * Default: HighFrequencyHighLimitBytes + */ + JSGC_HIGH_FREQUENCY_HIGH_LIMIT = 13, + + /** + * Upper bound of heap growth. + * + * Pref: javascript.options.mem.gc_high_frequency_heap_growth_max + * Default: HighFrequencyHeapGrowthMax + */ + JSGC_HIGH_FREQUENCY_HEAP_GROWTH_MAX = 14, + + /** + * Lower bound of heap growth. + * + * Pref: javascript.options.mem.gc_high_frequency_heap_growth_min + * Default: HighFrequencyHeapGrowthMin + */ + JSGC_HIGH_FREQUENCY_HEAP_GROWTH_MIN = 15, + + /** + * Heap growth for low frequency GCs. + * + * Pref: javascript.options.mem.gc_low_frequency_heap_growth + * Default: LowFrequencyHeapGrowth + */ + JSGC_LOW_FREQUENCY_HEAP_GROWTH = 16, + + /** + * If false, the heap growth factor is fixed at 3. If true, it is determined + * based on whether GCs are high- or low- frequency. + * + * Pref: javascript.options.mem.gc_dynamic_heap_growth + * Default: DynamicHeapGrowthEnabled + */ + JSGC_DYNAMIC_HEAP_GROWTH = 17, + + /** + * If true, high-frequency GCs will use a longer mark slice. + * + * Pref: javascript.options.mem.gc_dynamic_mark_slice + * Default: DynamicMarkSliceEnabled + */ + JSGC_DYNAMIC_MARK_SLICE = 18, + + /** + * Lower limit after which we limit the heap growth. + * + * The base value used to compute zone->threshold.gcTriggerBytes(). When + * usage.gcBytes() surpasses threshold.gcTriggerBytes() for a zone, the + * zone may be scheduled for a GC, depending on the exact circumstances. + * + * Pref: javascript.options.mem.gc_allocation_threshold_mb + * Default GCZoneAllocThresholdBase + */ + JSGC_ALLOCATION_THRESHOLD = 19, + + /** + * We try to keep at least this many unused chunks in the free chunk pool at + * all times, even after a shrinking GC. + * + * Pref: javascript.options.mem.gc_min_empty_chunk_count + * Default: MinEmptyChunkCount + */ + JSGC_MIN_EMPTY_CHUNK_COUNT = 21, + + /** + * We never keep more than this many unused chunks in the free chunk + * pool. + * + * Pref: javascript.options.mem.gc_min_empty_chunk_count + * Default: MinEmptyChunkCount + */ + JSGC_MAX_EMPTY_CHUNK_COUNT = 22, + + /** + * Whether compacting GC is enabled. + * + * Pref: javascript.options.mem.gc_compacting + * Default: CompactingEnabled + */ + JSGC_COMPACTING_ENABLED = 23, + + /** + * If true, painting can trigger IGC slices. + * + * Pref: javascript.options.mem.gc_refresh_frame_slices_enabled + * Default: RefreshFrameSlicesEnabled + */ + JSGC_REFRESH_FRAME_SLICES_ENABLED = 24, + + /** + * Factor for triggering a GC based on JSGC_ALLOCATION_THRESHOLD + * + * Default: ZoneAllocThresholdFactorDefault + * Pref: None + */ + JSGC_ALLOCATION_THRESHOLD_FACTOR = 25, + + /** + * Factor for triggering a GC based on JSGC_ALLOCATION_THRESHOLD. + * Used if another GC (in different zones) is already running. + * + * Default: ZoneAllocThresholdFactorAvoidInterruptDefault + * Pref: None + */ + JSGC_ALLOCATION_THRESHOLD_FACTOR_AVOID_INTERRUPT = 26, +} JSGCParamKey; + +/* + * Generic trace operation that calls JS::TraceEdge on each traceable thing's + * location reachable from data. + */ +typedef void +(* JSTraceDataOp)(JSTracer* trc, void* data); + +typedef enum JSGCStatus { + JSGC_BEGIN, + JSGC_END +} JSGCStatus; + +typedef void +(* JSGCCallback)(JSContext* cx, JSGCStatus status, void* data); + +typedef void +(* JSObjectsTenuredCallback)(JSContext* cx, void* data); + +typedef enum JSFinalizeStatus { + /** + * Called when preparing to sweep a group of zones, before anything has been + * swept. The collector will not yield to the mutator before calling the + * callback with JSFINALIZE_GROUP_START status. + */ + JSFINALIZE_GROUP_PREPARE, + + /** + * Called after preparing to sweep a group of zones. Weak references to + * unmarked things have been removed at this point, but no GC things have + * been swept. The collector may yield to the mutator after this point. + */ + JSFINALIZE_GROUP_START, + + /** + * Called after sweeping a group of zones. All dead GC things have been + * swept at this point. + */ + JSFINALIZE_GROUP_END, + + /** + * Called at the end of collection when everything has been swept. + */ + JSFINALIZE_COLLECTION_END +} JSFinalizeStatus; + +typedef void +(* JSFinalizeCallback)(JSFreeOp* fop, JSFinalizeStatus status, bool isZoneGC, void* data); + +typedef void +(* JSWeakPointerZonesCallback)(JSContext* cx, void* data); + +typedef void +(* JSWeakPointerCompartmentCallback)(JSContext* cx, JSCompartment* comp, void* data); + +/** + * Finalizes external strings created by JS_NewExternalString. The finalizer + * can be called off the main thread. + */ +struct JSStringFinalizer { + void (*finalize)(const JSStringFinalizer* fin, char16_t* chars); +}; + + namespace JS { +struct Zone; + #define GCREASONS(D) \ /* Reasons internal to the JS engine */ \ D(API) \ @@ -414,27 +696,6 @@ IsIncrementalGCEnabled(JSContext* cx); extern JS_PUBLIC_API(bool) IsIncrementalGCInProgress(JSContext* cx); -/* - * Returns true when writes to GC things must call an incremental (pre) barrier. - * This is generally only true when running mutator code in-between GC slices. - * At other times, the barrier may be elided for performance. - */ -extern JS_PUBLIC_API(bool) -IsIncrementalBarrierNeeded(JSContext* cx); - -/* - * Notify the GC that a reference to a GC thing is about to be overwritten. - * These methods must be called if IsIncrementalBarrierNeeded. - */ -extern JS_PUBLIC_API(void) -IncrementalReferenceBarrier(GCCellPtr thing); - -extern JS_PUBLIC_API(void) -IncrementalValueBarrier(const Value& v); - -extern JS_PUBLIC_API(void) -IncrementalObjectBarrier(JSObject* obj); - /** * Returns true if the most recent GC ran incrementally. */ @@ -605,86 +866,6 @@ class JS_PUBLIC_API(AutoCheckCannotGC) : public AutoRequireNoGC } JS_HAZ_GC_INVALIDATED; #endif -/** - * Unsets the gray bit for anything reachable from |thing|. |kind| should not be - * JS::TraceKind::Shape. |thing| should be non-null. The return value indicates - * if anything was unmarked. - */ -extern JS_FRIEND_API(bool) -UnmarkGrayGCThingRecursively(GCCellPtr thing); - -} /* namespace JS */ - -namespace js { -namespace gc { - -static MOZ_ALWAYS_INLINE void -ExposeGCThingToActiveJS(JS::GCCellPtr thing) -{ - // GC things residing in the nursery cannot be gray: they have no mark bits. - // All live objects in the nursery are moved to tenured at the beginning of - // each GC slice, so the gray marker never sees nursery things. - if (IsInsideNursery(thing.asCell())) - return; - - // There's nothing to do for permanent GC things that might be owned by - // another runtime. - if (thing.mayBeOwnedByOtherRuntime()) - return; - - JS::shadow::Runtime* rt = detail::GetCellRuntime(thing.asCell()); - MOZ_DIAGNOSTIC_ASSERT(rt->allowGCBarriers()); - - if (IsIncrementalBarrierNeededOnTenuredGCThing(rt, thing)) - JS::IncrementalReferenceBarrier(thing); - else if (!thing.mayBeOwnedByOtherRuntime() && js::gc::detail::CellIsMarkedGray(thing.asCell())) - JS::UnmarkGrayGCThingRecursively(thing); -} - -static MOZ_ALWAYS_INLINE void -MarkGCThingAsLive(JSRuntime* aRt, JS::GCCellPtr thing) -{ - // Any object in the nursery will not be freed during any GC running at that - // time. - if (IsInsideNursery(thing.asCell())) - return; - - // There's nothing to do for permanent GC things that might be owned by - // another runtime. - if (thing.mayBeOwnedByOtherRuntime()) - return; - - JS::shadow::Runtime* rt = JS::shadow::Runtime::asShadowRuntime(aRt); - MOZ_DIAGNOSTIC_ASSERT(rt->allowGCBarriers()); - - if (IsIncrementalBarrierNeededOnTenuredGCThing(rt, thing)) - JS::IncrementalReferenceBarrier(thing); -} - -} /* namespace gc */ -} /* namespace js */ - -namespace JS { - -/* - * This should be called when an object that is marked gray is exposed to the JS - * engine (by handing it to running JS code or writing it into live JS - * data). During incremental GC, since the gray bits haven't been computed yet, - * we conservatively mark the object black. - */ -static MOZ_ALWAYS_INLINE void -ExposeObjectToActiveJS(JSObject* obj) -{ - MOZ_ASSERT(obj); - js::gc::ExposeGCThingToActiveJS(GCCellPtr(obj)); -} - -static MOZ_ALWAYS_INLINE void -ExposeScriptToActiveJS(JSScript* script) -{ - js::gc::ExposeGCThingToActiveJS(GCCellPtr(script)); -} - /* * If a GC is currently marking, mark the string black. */ @@ -711,4 +892,162 @@ NotifyDidPaint(JSContext* cx); } /* namespace JS */ +/** + * Register externally maintained GC roots. + * + * traceOp: the trace operation. For each root the implementation should call + * JS::TraceEdge whenever the root contains a traceable thing. + * data: the data argument to pass to each invocation of traceOp. + */ +extern JS_PUBLIC_API(bool) +JS_AddExtraGCRootsTracer(JSContext* cx, JSTraceDataOp traceOp, void* data); + +/** Undo a call to JS_AddExtraGCRootsTracer. */ +extern JS_PUBLIC_API(void) +JS_RemoveExtraGCRootsTracer(JSContext* cx, JSTraceDataOp traceOp, void* data); + +extern JS_PUBLIC_API(void) +JS_GC(JSContext* cx); + +extern JS_PUBLIC_API(void) +JS_MaybeGC(JSContext* cx); + +extern JS_PUBLIC_API(void) +JS_SetGCCallback(JSContext* cx, JSGCCallback cb, void* data); + +extern JS_PUBLIC_API(void) +JS_SetObjectsTenuredCallback(JSContext* cx, JSObjectsTenuredCallback cb, + void* data); + +extern JS_PUBLIC_API(bool) +JS_AddFinalizeCallback(JSContext* cx, JSFinalizeCallback cb, void* data); + +extern JS_PUBLIC_API(void) +JS_RemoveFinalizeCallback(JSContext* cx, JSFinalizeCallback cb); + +/* + * Weak pointers and garbage collection + * + * Weak pointers are by their nature not marked as part of garbage collection, + * but they may need to be updated in two cases after a GC: + * + * 1) Their referent was found not to be live and is about to be finalized + * 2) Their referent has been moved by a compacting GC + * + * To handle this, any part of the system that maintain weak pointers to + * JavaScript GC things must register a callback with + * JS_(Add,Remove)WeakPointer{ZoneGroup,Compartment}Callback(). This callback + * must then call JS_UpdateWeakPointerAfterGC() on all weak pointers it knows + * about. + * + * Since sweeping is incremental, we have several callbacks to avoid repeatedly + * having to visit all embedder structures. The WeakPointerZonesCallback is + * called once for each strongly connected group of zones, whereas the + * WeakPointerCompartmentCallback is called once for each compartment that is + * visited while sweeping. Structures that cannot contain references in more + * than one compartment should sweep the relevant per-compartment structures + * using the latter callback to minimizer per-slice overhead. + * + * The argument to JS_UpdateWeakPointerAfterGC() is an in-out param. If the + * referent is about to be finalized the pointer will be set to null. If the + * referent has been moved then the pointer will be updated to point to the new + * location. + * + * Callers of this method are responsible for updating any state that is + * dependent on the object's address. For example, if the object's address is + * used as a key in a hashtable, then the object must be removed and + * re-inserted with the correct hash. + */ + +extern JS_PUBLIC_API(bool) +JS_AddWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb, void* data); + +extern JS_PUBLIC_API(void) +JS_RemoveWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb); + +extern JS_PUBLIC_API(bool) +JS_AddWeakPointerCompartmentCallback(JSContext* cx, JSWeakPointerCompartmentCallback cb, + void* data); + +extern JS_PUBLIC_API(void) +JS_RemoveWeakPointerCompartmentCallback(JSContext* cx, JSWeakPointerCompartmentCallback cb); + +namespace JS { +template class Heap; +} + +extern JS_PUBLIC_API(void) +JS_UpdateWeakPointerAfterGC(JS::Heap* objp); + +extern JS_PUBLIC_API(void) +JS_UpdateWeakPointerAfterGCUnbarriered(JSObject** objp); + +extern JS_PUBLIC_API(void) +JS_SetGCParameter(JSContext* cx, JSGCParamKey key, uint32_t value); + +extern JS_PUBLIC_API(void) +JS_ResetGCParameter(JSContext* cx, JSGCParamKey key); + +extern JS_PUBLIC_API(uint32_t) +JS_GetGCParameter(JSContext* cx, JSGCParamKey key); + +extern JS_PUBLIC_API(void) +JS_SetGCParametersBasedOnAvailableMemory(JSContext* cx, uint32_t availMem); + +/** + * Create a new JSString whose chars member refers to external memory, i.e., + * memory requiring application-specific finalization. + */ +extern JS_PUBLIC_API(JSString*) +JS_NewExternalString(JSContext* cx, const char16_t* chars, size_t length, + const JSStringFinalizer* fin); + +/** + * Create a new JSString whose chars member may refer to external memory. + * If a new external string is allocated, |*allocatedExternal| is set to true. + * Otherwise the returned string is either not an external string or an + * external string allocated by a previous call and |*allocatedExternal| is set + * to false. If |*allocatedExternal| is false, |fin| won't be called. + */ +extern JS_PUBLIC_API(JSString*) +JS_NewMaybeExternalString(JSContext* cx, const char16_t* chars, size_t length, + const JSStringFinalizer* fin, bool* allocatedExternal); + +/** + * Return whether 'str' was created with JS_NewExternalString or + * JS_NewExternalStringWithClosure. + */ +extern JS_PUBLIC_API(bool) +JS_IsExternalString(JSString* str); + +/** + * Return the 'fin' arg passed to JS_NewExternalString. + */ +extern JS_PUBLIC_API(const JSStringFinalizer*) +JS_GetExternalStringFinalizer(JSString* str); + +namespace JS { + +extern JS_PUBLIC_API(bool) +IsIdleGCTaskNeeded(JSRuntime* rt); + +extern JS_PUBLIC_API(void) +RunIdleTimeGCTask(JSRuntime* rt); + +} // namespace JS + +namespace js { +namespace gc { + +/** + * Create an object providing access to the garbage collector's internal notion + * of the current state of memory (both GC heap memory and GCthing-controlled + * malloc memory. + */ +extern JS_PUBLIC_API(JSObject*) +NewMemoryInfoObject(JSContext* cx); + +} /* namespace gc */ +} /* namespace js */ + #endif /* js_GCAPI_h */ diff --git a/js/public/HeapAPI.h b/js/public/HeapAPI.h index 5912a6e535..cda27e935f 100644 --- a/js/public/HeapAPI.h +++ b/js/public/HeapAPI.h @@ -39,6 +39,14 @@ const size_t CellShift = 3; const size_t CellSize = size_t(1) << CellShift; const size_t CellMask = CellSize - 1; +/* + * We sometimes use an index to refer to a cell in an arena. The index for a + * cell is found by dividing by the cell alignment so not all indicies refer to + * valid cells. + */ +const size_t ArenaCellIndexBytes = CellAlignBytes; +const size_t MaxArenaCellIndex = ArenaSize / CellAlignBytes; + /* These are magic constants derived from actual offsets in gc/Heap.h. */ #ifdef JS_GC_SMALL_CHUNK_SIZE const size_t ChunkMarkBitmapOffset = 258104; @@ -423,7 +431,38 @@ GCThingIsMarkedGray(GCCellPtr thing) extern JS_PUBLIC_API(JS::TraceKind) GCThingTraceKind(void* thing); -} /* namespace JS */ +/* + * Returns true when writes to GC thing pointers (and reads from weak pointers) + * must call an incremental barrier. This is generally only true when running + * mutator code in-between GC slices. At other times, the barrier may be elided + * for performance. + */ +extern JS_PUBLIC_API(bool) +IsIncrementalBarrierNeeded(JSContext* cx); + +/* + * Notify the GC that a reference to a JSObject is about to be overwritten. + * This method must be called if IsIncrementalBarrierNeeded. + */ +extern JS_PUBLIC_API(void) +IncrementalPreWriteBarrier(JSObject* obj); + +/* + * Notify the GC that a weak reference to a GC thing has been read. + * This method must be called if IsIncrementalBarrierNeeded. + */ +extern JS_PUBLIC_API(void) +IncrementalReadBarrier(GCCellPtr thing); + +/** + * Unsets the gray bit for anything reachable from |thing|. |kind| should not be + * JS::TraceKind::Shape. |thing| should be non-null. The return value indicates + * if anything was unmarked. + */ +extern JS_FRIEND_API(bool) +UnmarkGrayGCThingRecursively(GCCellPtr thing); + +} // namespace JS namespace js { namespace gc { @@ -443,16 +482,76 @@ IsIncrementalBarrierNeededOnTenuredGCThing(JS::shadow::Runtime* rt, const JS::GC } #ifdef MOZ_DEVTOOLS_SERVER -/** - * Create an object providing access to the garbage collector's internal notion - * of the current state of memory (both GC heap memory and GCthing-controlled - * malloc memory. +static MOZ_ALWAYS_INLINE void +ExposeGCThingToActiveJS(JS::GCCellPtr thing) +{ + // GC things residing in the nursery cannot be gray: they have no mark bits. + // All live objects in the nursery are moved to tenured at the beginning of + // each GC slice, so the gray marker never sees nursery things. + if (IsInsideNursery(thing.asCell())) + return; + + // There's nothing to do for permanent GC things that might be owned by + // another runtime. + if (thing.mayBeOwnedByOtherRuntime()) + return; + + if (IsIncrementalBarrierNeededOnTenuredGCThing(thing)) + JS::IncrementalReadBarrier(thing); + else if (js::gc::detail::TenuredCellIsMarkedGray(thing.asCell())) + JS::UnmarkGrayGCThingRecursively(thing); + + MOZ_ASSERT(!js::gc::detail::TenuredCellIsMarkedGray(thing.asCell())); +} + +template +extern JS_PUBLIC_API(bool) +EdgeNeedsSweepUnbarrieredSlow(T* thingp); + +static MOZ_ALWAYS_INLINE bool +EdgeNeedsSweepUnbarriered(JSObject** objp) +{ + // This function does not handle updating nursery pointers. Raw JSObject + // pointers should be updated separately or replaced with + // JS::Heap which handles this automatically. + MOZ_ASSERT(!JS::CurrentThreadIsHeapMinorCollecting()); + if (IsInsideNursery(reinterpret_cast(*objp))) + return false; + + auto zone = JS::shadow::Zone::asShadowZone(detail::GetGCThingZone(uintptr_t(*objp))); + if (!zone->isGCSweepingOrCompacting()) + return false; + + return EdgeNeedsSweepUnbarrieredSlow(objp); +} + +} // namespace gc +} // namesapce js + +namespace JS { + +/* + * This should be called when an object that is marked gray is exposed to the JS + * engine (by handing it to running JS code or writing it into live JS + * data). During incremental GC, since the gray bits haven't been computed yet, + * we conservatively mark the object black. */ -extern JS_PUBLIC_API(JSObject*) -NewMemoryInfoObject(JSContext* cx); +static MOZ_ALWAYS_INLINE void +ExposeObjectToActiveJS(JSObject* obj) +{ + MOZ_ASSERT(obj); + MOZ_ASSERT(!js::gc::EdgeNeedsSweepUnbarrieredSlow(&obj)); + js::gc::ExposeGCThingToActiveJS(GCCellPtr(obj)); +} #endif -} /* namespace gc */ -} /* namespace js */ +static MOZ_ALWAYS_INLINE void +ExposeScriptToActiveJS(JSScript* script) +{ + MOZ_ASSERT(!js::gc::EdgeNeedsSweepUnbarrieredSlow(&script)); + js::gc::ExposeGCThingToActiveJS(GCCellPtr(script)); +} + +} /* namespace JS */ #endif /* js_HeapAPI_h */ diff --git a/js/public/RootingAPI.h b/js/public/RootingAPI.h index 37f67caa14..7b37ab1c4c 100644 --- a/js/public/RootingAPI.h +++ b/js/public/RootingAPI.h @@ -19,7 +19,6 @@ #include "jspubtd.h" #include "js/GCAnnotations.h" -#include "js/GCAPI.h" #include "js/GCPolicyAPI.h" #include "js/HeapAPI.h" #include "js/TypeDecls.h" diff --git a/js/public/SliceBudget.h b/js/public/SliceBudget.h index 4334a704a8..e170ea2e42 100644 --- a/js/public/SliceBudget.h +++ b/js/public/SliceBudget.h @@ -8,6 +8,8 @@ #include +#include "jstypes.h" + namespace js { struct JS_PUBLIC_API(TimeBudget) diff --git a/js/src/builtin/MapObject.cpp b/js/src/builtin/MapObject.cpp index 91dbb22282..7ba1ef6cd2 100644 --- a/js/src/builtin/MapObject.cpp +++ b/js/src/builtin/MapObject.cpp @@ -10,7 +10,6 @@ #include "jsobj.h" #include "ds/OrderedHashTable.h" -#include "gc/Marking.h" #include "js/Utility.h" #include "vm/EqualityOperations.h" // js::SameValue #include "vm/GlobalObject.h" @@ -18,8 +17,7 @@ #include "vm/SelfHosting.h" #include "vm/Symbol.h" -#include "jsobjinlines.h" - +#include "gc/Marking-inl.h" #include "vm/Interpreter-inl.h" #include "vm/NativeObject-inl.h" diff --git a/js/src/builtin/TypedObject-inl.h b/js/src/builtin/TypedObject-inl.h new file mode 100644 index 0000000000..9970d31561 --- /dev/null +++ b/js/src/builtin/TypedObject-inl.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef builtin_TypedObject_inl_h +#define builtin_TypedObject_inl_h + +#include "builtin/TypedObject.h" + +#include "gc/ObjectKind-inl.h" + +/* static */ +js::gc::AllocKind +js::InlineTypedObject::allocKindForTypeDescriptor(TypeDescr* descr) +{ + size_t nbytes = descr->size(); + MOZ_ASSERT(nbytes <= MaximumSize); + + return gc::GetGCObjectKindForBytes(nbytes + sizeof(TypedObject)); +} + +#endif // builtin_TypedObject_inl_h \ No newline at end of file diff --git a/js/src/builtin/TypedObject.cpp b/js/src/builtin/TypedObject.cpp index c79e8b90c6..28e3f23e66 100644 --- a/js/src/builtin/TypedObject.cpp +++ b/js/src/builtin/TypedObject.cpp @@ -3,7 +3,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "builtin/TypedObject.h" +#include "builtin/TypedObject-inl.h" #include "mozilla/Casting.h" #include "mozilla/CheckedInt.h" diff --git a/js/src/builtin/TypedObject.h b/js/src/builtin/TypedObject.h index cceff0c638..4dbe8c7f8f 100644 --- a/js/src/builtin/TypedObject.h +++ b/js/src/builtin/TypedObject.h @@ -679,12 +679,7 @@ class InlineTypedObject : public TypedObject public: static const size_t MaximumSize = JSObject::MAX_BYTE_SIZE - sizeof(TypedObject); - static gc::AllocKind allocKindForTypeDescriptor(TypeDescr* descr) { - size_t nbytes = descr->size(); - MOZ_ASSERT(nbytes <= MaximumSize); - - return gc::GetGCObjectKindForBytes(nbytes + sizeof(TypedObject)); - } + static inline gc::AllocKind allocKindForTypeDescriptor(TypeDescr* descr); uint8_t* inlineTypedMem(const JS::AutoRequireNoGC&) const { return inlineTypedMem(); diff --git a/js/src/builtin/intl/LanguageTag.h b/js/src/builtin/intl/LanguageTag.h index 3c2ecb1553..1e2a584536 100644 --- a/js/src/builtin/intl/LanguageTag.h +++ b/js/src/builtin/intl/LanguageTag.h @@ -23,7 +23,6 @@ #include "jsalloc.h" #include "js/Result.h" -#include "js/GCAPI.h" #include "js/Utility.h" #include "js/Vector.h" diff --git a/js/src/builtin/intl/SharedIntlData.h b/js/src/builtin/intl/SharedIntlData.h index 81834804a1..e068bdeae8 100644 --- a/js/src/builtin/intl/SharedIntlData.h +++ b/js/src/builtin/intl/SharedIntlData.h @@ -1,285 +1,284 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef builtin_intl_SharedIntlData_h -#define builtin_intl_SharedIntlData_h - -#include "mozilla/MemoryReporting.h" - -#include - -#include "jsalloc.h" -#include "js/CharacterEncoding.h" -#include "js/GCAPI.h" -#include "js/GCHashTable.h" -#include "js/RootingAPI.h" -#include "js/Utility.h" -#include "vm/String.h" - -namespace js { - -namespace intl { - -/** - * Stores Intl data which can be shared across compartments (but not contexts). - * - * Used for data which is expensive when computed repeatedly or is not - * available through ICU. - */ -class SharedIntlData -{ - struct LinearStringLookup - { - union { - const JS::Latin1Char* latin1Chars; - const char16_t* twoByteChars; - }; - bool isLatin1; - size_t length; - JS::AutoCheckCannotGC nogc; - HashNumber hash = 0; - - explicit LinearStringLookup(JSLinearString* string) - : isLatin1(string->hasLatin1Chars()), length(string->length()) - { - if (isLatin1) - latin1Chars = string->latin1Chars(nogc); - else - twoByteChars = string->twoByteChars(nogc); - } - - LinearStringLookup(const char* chars, size_t length) - : isLatin1(true), length(length) - { - latin1Chars = reinterpret_cast(chars); - } - }; - - private: - /** - * Information tracking the set of the supported time zone names, derived - * from the IANA time zone database . - * - * There are two kinds of IANA time zone names: Zone and Link (denoted as - * such in database source files). Zone names are the canonical, preferred - * name for a time zone, e.g. Asia/Kolkata. Link names simply refer to - * target Zone names for their meaning, e.g. Asia/Calcutta targets - * Asia/Kolkata. That a name is a Link doesn't *necessarily* reflect a - * sense of deprecation: some Link names also exist partly for convenience, - * e.g. UTC and GMT as Link names targeting the Zone name Etc/UTC. - * - * Two data sources determine the time zone names we support: those ICU - * supports and IANA's zone information. - * - * Unfortunately the names ICU and IANA support, and their Link - * relationships from name to target, aren't identical, so we can't simply - * implicitly trust ICU's name handling. We must perform various - * preprocessing of user-provided zone names and post-processing of - * ICU-provided zone names to implement ECMA-402's IANA-consistent behavior. - * - * Also see and - * . - */ - - using TimeZoneName = JSAtom*; - - struct TimeZoneHasher - { - struct Lookup : LinearStringLookup - { - explicit Lookup(JSFlatString* timeZone); - }; - - static js::HashNumber hash(const Lookup& lookup) { return lookup.hash; } - static bool match(TimeZoneName key, const Lookup& lookup); - }; - - using TimeZoneSet = js::GCHashSet; - - using TimeZoneMap = js::GCHashMap; - - /** - * As a threshold matter, available time zones are those time zones ICU - * supports, via ucal_openTimeZones. But ICU supports additional non-IANA - * time zones described in intl/icu/source/tools/tzcode/icuzones (listed in - * IntlTimeZoneData.cpp's |legacyICUTimeZones|) for its own backwards - * compatibility purposes. This set consists of ICU's supported time zones, - * minus all backwards-compatibility time zones. - */ - TimeZoneSet availableTimeZones; - - /** - * IANA treats some time zone names as Zones, that ICU instead treats as - * Links. For example, IANA considers "America/Indiana/Indianapolis" to be - * a Zone and "America/Fort_Wayne" a Link that targets it, but ICU - * considers the former a Link that targets "America/Indianapolis" (which - * IANA treats as a Link). - * - * ECMA-402 requires that we respect IANA data, so if we're asked to - * canonicalize a time zone name in this set, we must *not* return ICU's - * canonicalization. - */ - TimeZoneSet ianaZonesTreatedAsLinksByICU; - - /** - * IANA treats some time zone names as Links to one target, that ICU - * instead treats as either Zones, or Links to different targets. An - * example of the former is "Asia/Calcutta, which IANA assigns the target - * "Asia/Kolkata" but ICU considers its own Zone. An example of the latter - * is "America/Virgin", which IANA assigns the target - * "America/Port_of_Spain" but ICU assigns the target "America/St_Thomas". - * - * ECMA-402 requires that we respect IANA data, so if we're asked to - * canonicalize a time zone name that's a key in this map, we *must* return - * the corresponding value and *must not* return ICU's canonicalization. - */ - TimeZoneMap ianaLinksCanonicalizedDifferentlyByICU; - - bool timeZoneDataInitialized = false; - - /** - * Precomputes the available time zone names, because it's too expensive to - * call ucal_openTimeZones() repeatedly. - */ - bool ensureTimeZones(JSContext* cx); - - public: - /** - * Returns the validated time zone name in |result|. If the input time zone - * isn't a valid IANA time zone name, |result| remains unchanged. - */ - bool validateTimeZoneName(JSContext* cx, JS::HandleString timeZone, - JS::MutableHandleString result); - - /** - * Returns the canonical time zone name in |result|. If no canonical name - * was found, |result| remains unchanged. - * - * This method only handles time zones which are canonicalized differently - * by ICU when compared to IANA. - */ - bool tryCanonicalizeTimeZoneConsistentWithIANA(JSContext* cx, JS::HandleString timeZone, - JS::MutableHandleString result); - private: - using Locale = JSAtom*; - - struct LocaleHasher - { - struct Lookup : LinearStringLookup - { - explicit Lookup(JSLinearString* locale); - Lookup(const char* chars, size_t length); - }; - - static js::HashNumber hash(const Lookup& lookup) { return lookup.hash; } - static bool match(Locale key, const Lookup& lookup); - }; - - using LocaleSet = GCHashSet; - - // Set of supported locales for all Intl service constructors except Collator, - // which uses its own set. - // - // UDateFormat: - // udat_[count,get]Available() return the same results as their - // uloc_[count,get]Available() counterparts. - // - // UNumberFormatter: - // unum_[count,get]Available() return the same results as their - // uloc_[count,get]Available() counterparts. - // - // UPluralRules and URelativeDateTimeFormatter: - // We're going to use ULocale availableLocales as per ICU recommendation: - // https://unicode-org.atlassian.net/browse/ICU-12756 - LocaleSet supportedLocales; - - // ucol_[count,get]Available() return different results compared to - // uloc_[count,get]Available(), we can't use |supportedLocales| here. - LocaleSet collatorSupportedLocales; - - bool supportedLocalesInitialized = false; - - // CountAvailable and GetAvailable describe the signatures used for ICU API - // to determine available locales for various functionality. - using CountAvailable = int32_t (*)(); - using GetAvailable = const char* (*)(int32_t localeIndex); - - static bool getAvailableLocales(JSContext* cx, LocaleSet& locales, - CountAvailable countAvailable, - GetAvailable getAvailable); - - /** - * Precomputes the available locales sets. - */ - bool ensureSupportedLocales(JSContext* cx); - - public: - enum class SupportedLocaleKind { - Collator, - DateTimeFormat, - NumberFormat, - PluralRules, - RelativeTimeFormat - }; - - /** - * Sets |supported| to true if |locale| is supported by the requested Intl - * service constructor. Otherwise sets |supported| to false. - */ - MOZ_MUST_USE bool isSupportedLocale(JSContext* cx, SupportedLocaleKind kind, - JS::Handle locale, - bool* supported); - - private: - /** - * The case first parameter (BCP47 key "kf") allows to switch the order of - * upper- and lower-case characters. ICU doesn't directly provide an API - * to query the default case first value of a given locale, but instead - * requires to instantiate a collator object and then query the case first - * attribute (UCOL_CASE_FIRST). - * To avoid instantiating an additional collator object whenever we need - * to retrieve the default case first value of a specific locale, we - * compute the default case first value for every supported locale only - * once and then keep a list of all locales which don't use the default - * case first setting. - * There is almost no difference between lower-case first and when case - * first is disabled (UCOL_LOWER_FIRST resp. UCOL_OFF), so we only need to - * track locales which use upper-case first as their default setting. - */ - - LocaleSet upperCaseFirstLocales; - - bool upperCaseFirstInitialized = false; - - /** - * Precomputes the available locales which use upper-case first sorting. - */ - bool ensureUpperCaseFirstLocales(JSContext* cx); - - public: - /** - * Sets |isUpperFirst| to true if |locale| sorts upper-case characters - * before lower-case characters. - */ - bool isUpperCaseFirst(JSContext* cx, JS::HandleString locale, bool* isUpperFirst); - - public: - void destroyInstance(); - - void trace(JSTracer* trc); - - size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; -}; - -} // namespace intl - -} // namespace js - +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef builtin_intl_SharedIntlData_h +#define builtin_intl_SharedIntlData_h + +#include "mozilla/MemoryReporting.h" + +#include + +#include "jsalloc.h" +#include "js/CharacterEncoding.h" +#include "js/GCHashTable.h" +#include "js/RootingAPI.h" +#include "js/Utility.h" +#include "vm/String.h" + +namespace js { + +namespace intl { + +/** + * Stores Intl data which can be shared across compartments (but not contexts). + * + * Used for data which is expensive when computed repeatedly or is not + * available through ICU. + */ +class SharedIntlData +{ + struct LinearStringLookup + { + union { + const JS::Latin1Char* latin1Chars; + const char16_t* twoByteChars; + }; + bool isLatin1; + size_t length; + JS::AutoCheckCannotGC nogc; + HashNumber hash = 0; + + explicit LinearStringLookup(JSLinearString* string) + : isLatin1(string->hasLatin1Chars()), length(string->length()) + { + if (isLatin1) + latin1Chars = string->latin1Chars(nogc); + else + twoByteChars = string->twoByteChars(nogc); + } + + LinearStringLookup(const char* chars, size_t length) + : isLatin1(true), length(length) + { + latin1Chars = reinterpret_cast(chars); + } + }; + + private: + /** + * Information tracking the set of the supported time zone names, derived + * from the IANA time zone database . + * + * There are two kinds of IANA time zone names: Zone and Link (denoted as + * such in database source files). Zone names are the canonical, preferred + * name for a time zone, e.g. Asia/Kolkata. Link names simply refer to + * target Zone names for their meaning, e.g. Asia/Calcutta targets + * Asia/Kolkata. That a name is a Link doesn't *necessarily* reflect a + * sense of deprecation: some Link names also exist partly for convenience, + * e.g. UTC and GMT as Link names targeting the Zone name Etc/UTC. + * + * Two data sources determine the time zone names we support: those ICU + * supports and IANA's zone information. + * + * Unfortunately the names ICU and IANA support, and their Link + * relationships from name to target, aren't identical, so we can't simply + * implicitly trust ICU's name handling. We must perform various + * preprocessing of user-provided zone names and post-processing of + * ICU-provided zone names to implement ECMA-402's IANA-consistent behavior. + * + * Also see and + * . + */ + + using TimeZoneName = JSAtom*; + + struct TimeZoneHasher + { + struct Lookup : LinearStringLookup + { + explicit Lookup(JSFlatString* timeZone); + }; + + static js::HashNumber hash(const Lookup& lookup) { return lookup.hash; } + static bool match(TimeZoneName key, const Lookup& lookup); + }; + + using TimeZoneSet = js::GCHashSet; + + using TimeZoneMap = js::GCHashMap; + + /** + * As a threshold matter, available time zones are those time zones ICU + * supports, via ucal_openTimeZones. But ICU supports additional non-IANA + * time zones described in intl/icu/source/tools/tzcode/icuzones (listed in + * IntlTimeZoneData.cpp's |legacyICUTimeZones|) for its own backwards + * compatibility purposes. This set consists of ICU's supported time zones, + * minus all backwards-compatibility time zones. + */ + TimeZoneSet availableTimeZones; + + /** + * IANA treats some time zone names as Zones, that ICU instead treats as + * Links. For example, IANA considers "America/Indiana/Indianapolis" to be + * a Zone and "America/Fort_Wayne" a Link that targets it, but ICU + * considers the former a Link that targets "America/Indianapolis" (which + * IANA treats as a Link). + * + * ECMA-402 requires that we respect IANA data, so if we're asked to + * canonicalize a time zone name in this set, we must *not* return ICU's + * canonicalization. + */ + TimeZoneSet ianaZonesTreatedAsLinksByICU; + + /** + * IANA treats some time zone names as Links to one target, that ICU + * instead treats as either Zones, or Links to different targets. An + * example of the former is "Asia/Calcutta, which IANA assigns the target + * "Asia/Kolkata" but ICU considers its own Zone. An example of the latter + * is "America/Virgin", which IANA assigns the target + * "America/Port_of_Spain" but ICU assigns the target "America/St_Thomas". + * + * ECMA-402 requires that we respect IANA data, so if we're asked to + * canonicalize a time zone name that's a key in this map, we *must* return + * the corresponding value and *must not* return ICU's canonicalization. + */ + TimeZoneMap ianaLinksCanonicalizedDifferentlyByICU; + + bool timeZoneDataInitialized = false; + + /** + * Precomputes the available time zone names, because it's too expensive to + * call ucal_openTimeZones() repeatedly. + */ + bool ensureTimeZones(JSContext* cx); + + public: + /** + * Returns the validated time zone name in |result|. If the input time zone + * isn't a valid IANA time zone name, |result| remains unchanged. + */ + bool validateTimeZoneName(JSContext* cx, JS::HandleString timeZone, + MutableHandleAtom result); + + /** + * Returns the canonical time zone name in |result|. If no canonical name + * was found, |result| remains unchanged. + * + * This method only handles time zones which are canonicalized differently + * by ICU when compared to IANA. + */ + bool tryCanonicalizeTimeZoneConsistentWithIANA(JSContext* cx, JS::HandleString timeZone, + MutableHandleAtom result); + private: + using Locale = JSAtom*; + + struct LocaleHasher + { + struct Lookup : LinearStringLookup + { + explicit Lookup(JSLinearString* locale); + Lookup(const char* chars, size_t length); + }; + + static js::HashNumber hash(const Lookup& lookup) { return lookup.hash; } + static bool match(Locale key, const Lookup& lookup); + }; + + using LocaleSet = GCHashSet; + + // Set of supported locales for all Intl service constructors except Collator, + // which uses its own set. + // + // UDateFormat: + // udat_[count,get]Available() return the same results as their + // uloc_[count,get]Available() counterparts. + // + // UNumberFormatter: + // unum_[count,get]Available() return the same results as their + // uloc_[count,get]Available() counterparts. + // + // UPluralRules and URelativeDateTimeFormatter: + // We're going to use ULocale availableLocales as per ICU recommendation: + // https://unicode-org.atlassian.net/browse/ICU-12756 + LocaleSet supportedLocales; + + // ucol_[count,get]Available() return different results compared to + // uloc_[count,get]Available(), we can't use |supportedLocales| here. + LocaleSet collatorSupportedLocales; + + bool supportedLocalesInitialized = false; + + // CountAvailable and GetAvailable describe the signatures used for ICU API + // to determine available locales for various functionality. + using CountAvailable = int32_t (*)(); + using GetAvailable = const char* (*)(int32_t localeIndex); + + static bool getAvailableLocales(JSContext* cx, LocaleSet& locales, + CountAvailable countAvailable, + GetAvailable getAvailable); + + /** + * Precomputes the available locales sets. + */ + bool ensureSupportedLocales(JSContext* cx); + + public: + enum class SupportedLocaleKind { + Collator, + DateTimeFormat, + NumberFormat, + PluralRules, + RelativeTimeFormat + }; + + /** + * Sets |supported| to true if |locale| is supported by the requested Intl + * service constructor. Otherwise sets |supported| to false. + */ + [[nodiscard]] bool isSupportedLocale(JSContext* cx, SupportedLocaleKind kind, + JS::Handle locale, + bool* supported); + + private: + /** + * The case first parameter (BCP47 key "kf") allows to switch the order of + * upper- and lower-case characters. ICU doesn't directly provide an API + * to query the default case first value of a given locale, but instead + * requires to instantiate a collator object and then query the case first + * attribute (UCOL_CASE_FIRST). + * To avoid instantiating an additional collator object whenever we need + * to retrieve the default case first value of a specific locale, we + * compute the default case first value for every supported locale only + * once and then keep a list of all locales which don't use the default + * case first setting. + * There is almost no difference between lower-case first and when case + * first is disabled (UCOL_LOWER_FIRST resp. UCOL_OFF), so we only need to + * track locales which use upper-case first as their default setting. + */ + + LocaleSet upperCaseFirstLocales; + + bool upperCaseFirstInitialized = false; + + /** + * Precomputes the available locales which use upper-case first sorting. + */ + bool ensureUpperCaseFirstLocales(JSContext* cx); + + public: + /** + * Sets |isUpperFirst| to true if |locale| sorts upper-case characters + * before lower-case characters. + */ + bool isUpperCaseFirst(JSContext* cx, JS::HandleString locale, bool* isUpperFirst); + + public: + void destroyInstance(); + + void trace(JSTracer* trc); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +} // namespace intl + +} // namespace js + #endif /* builtin_intl_SharedIntlData_h */ \ No newline at end of file diff --git a/js/src/frontend/BytecodeEmitter.cpp b/js/src/frontend/BytecodeEmitter.cpp index 27fbfec3a3..2b3a22ea93 100644 --- a/js/src/frontend/BytecodeEmitter.cpp +++ b/js/src/frontend/BytecodeEmitter.cpp @@ -52,7 +52,6 @@ #include "wasm/AsmJS.h" #include "jsatominlines.h" -#include "jsobjinlines.h" #include "jsscriptinlines.h" #include "frontend/ParseNode-inl.h" diff --git a/js/src/gc/Cell.h b/js/src/gc/Cell.h index 57e5fe16aa..25690e5983 100644 --- a/js/src/gc/Cell.h +++ b/js/src/gc/Cell.h @@ -10,6 +10,7 @@ #include "gc/GCEnum.h" #include "gc/Heap.h" #include "js/GCAnnotations.h" +#include "js/TraceKind.h" namespace JS { @@ -75,9 +76,26 @@ struct Cell static MOZ_ALWAYS_INLINE bool needWriteBarrierPre(JS::Zone* zone); + template + inline bool is() const { + return getTraceKind() == JS::MapTypeToTraceKind::kind; + } + + template + inline T* as() { + MOZ_ASSERT(is()); + return static_cast(this); + } + + template + inline const T* as() const { + MOZ_ASSERT(is()); + return static_cast(this); + } + #ifdef DEBUG inline bool isAligned() const; - void dump(GenericPrinter& out) const; + void dump(FILE* fp) const; void dump() const; #endif diff --git a/js/src/gc/GCMarker.h b/js/src/gc/GCMarker.h index f3222a6d1e..d910085ece 100644 --- a/js/src/gc/GCMarker.h +++ b/js/src/gc/GCMarker.h @@ -136,22 +136,22 @@ class MarkStack void setStack(TaggedPtr* stack, size_t tosIndex, size_t capacity); - MOZ_MUST_USE bool init(JSGCMode gcMode); + [[nodiscard]] bool init(JSGCMode gcMode); void setBaseCapacity(JSGCMode mode); size_t maxCapacity() const { return maxCapacity_; } void setMaxCapacity(size_t maxCapacity); template - MOZ_MUST_USE bool push(T* ptr); + [[nodiscard]] bool push(T* ptr); - MOZ_MUST_USE bool push(JSObject* obj, HeapSlot* start, HeapSlot* end); - MOZ_MUST_USE bool push(const ValueArray& array); - MOZ_MUST_USE bool push(const SavedValueArray& array); + [[nodiscard]] bool push(JSObject* obj, HeapSlot* start, HeapSlot* end); + [[nodiscard]] bool push(const ValueArray& array); + [[nodiscard]] bool push(const SavedValueArray& array); // GCMarker::eagerlyMarkChildren uses unused marking stack as temporary // storage to hold rope pointers. - MOZ_MUST_USE bool pushTempRope(JSRope* ptr); + [[nodiscard]] bool pushTempRope(JSRope* ptr); bool isEmpty() const { return tos_ == stack_; @@ -169,13 +169,13 @@ class MarkStack size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; private: - MOZ_MUST_USE bool ensureSpace(size_t count); + [[nodiscard]] bool ensureSpace(size_t count); /* Grow the stack, ensuring there is space for at least count elements. */ - MOZ_MUST_USE bool enlarge(size_t count); + [[nodiscard]] bool enlarge(size_t count); const TaggedPtr& peekPtr() const; - MOZ_MUST_USE bool pushTaggedPtr(Tag tag, Cell* ptr); + [[nodiscard]] bool pushTaggedPtr(Tag tag, Cell* ptr); ActiveThreadData stack_; ActiveThreadData tos_; @@ -222,7 +222,7 @@ class GCMarker : public JSTracer { public: explicit GCMarker(JSRuntime* rt); - MOZ_MUST_USE bool init(JSGCMode gcMode); + [[nodiscard]] bool init(JSGCMode gcMode); void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); } size_t maxCapacity() const { return stack.maxCapacity(); } @@ -270,7 +270,7 @@ class GCMarker : public JSTracer void delayMarkingArena(gc::Arena* arena); void delayMarkingChildren(const void* thing); void markDelayedChildren(gc::Arena* arena); - MOZ_MUST_USE bool markDelayedChildren(SliceBudget& budget); + [[nodiscard]] bool markDelayedChildren(SliceBudget& budget); bool hasDelayedChildren() const { return !!unmarkedArenaStackTop; } @@ -279,7 +279,7 @@ class GCMarker : public JSTracer return isMarkStackEmpty() && !unmarkedArenaStackTop; } - MOZ_MUST_USE bool drainMarkStack(SliceBudget& budget); + [[nodiscard]] bool drainMarkStack(SliceBudget& budget); void setGCMode(JSGCMode mode) { stack.setGCMode(mode); } @@ -331,7 +331,7 @@ class GCMarker : public JSTracer // Mark the given GC thing, but do not trace its children. Return true // if the thing became marked. template - MOZ_MUST_USE bool mark(T* thing); + [[nodiscard]] bool mark(T* thing); template inline void pushTaggedPtr(T* ptr); @@ -342,7 +342,7 @@ class GCMarker : public JSTracer return stack.isEmpty(); } - MOZ_MUST_USE bool restoreValueArray(const gc::MarkStack::SavedValueArray& array, + [[nodiscard]] bool restoreValueArray(const gc::MarkStack::SavedValueArray& array, HeapSlot** vpp, HeapSlot** endp); void saveValueRanges(); inline void processMarkStackTop(SliceBudget& budget); diff --git a/js/src/gc/Iteration-inl.h b/js/src/gc/Iteration-inl.h index 7f52acecb1..f0e0654b53 100644 --- a/js/src/gc/Iteration-inl.h +++ b/js/src/gc/Iteration-inl.h @@ -88,11 +88,11 @@ class GCZonesIter typedef CompartmentsIterT GCCompartmentsIter; /* Iterates over all zones in the current sweep group. */ -class SweepGroupZonesIter { +class GCSweepGroupIter { JS::Zone* current; public: - explicit SweepGroupZonesIter(JSRuntime* rt) { + explicit GCSweepGroupIter (JSRuntime* rt) { MOZ_ASSERT(CurrentThreadIsPerformingGC()); current = rt->gc.getCurrentSweepGroup(); } @@ -113,7 +113,7 @@ class SweepGroupZonesIter { JS::Zone* operator->() const { return get(); } }; -typedef CompartmentsIterT SweepGroupCompartmentsIter; +typedef CompartmentsIterT SweepGroupCompartmentsIter; } // namespace gc } // namespace js diff --git a/js/src/gc/Statistics.h b/js/src/gc/Statistics.h index 6cfe4d776e..e61660ab4e 100644 --- a/js/src/gc/Statistics.h +++ b/js/src/gc/Statistics.h @@ -11,10 +11,13 @@ #include "mozilla/Maybe.h" #include "jsalloc.h" -#include "jsgc.h" #include "jspubtd.h" +#include "NamespaceImports.h" +#include "gc/GCEnum.h" #include "js/GCAPI.h" +#include "js/SliceBudget.h" +#include "js/UniquePtr.h" #include "js/Vector.h" using mozilla::Maybe; diff --git a/js/src/jit/BaselineJIT.cpp b/js/src/jit/BaselineJIT.cpp index f6ed9fe7c1..49804d3978 100644 --- a/js/src/jit/BaselineJIT.cpp +++ b/js/src/jit/BaselineJIT.cpp @@ -23,6 +23,7 @@ #include "jsopcodeinlines.h" #include "jsscriptinlines.h" +#include "gc/Iteration-inl.h" #include "jit/JitFrames-inl.h" #include "jit/MacroAssembler-inl.h" #include "vm/Stack-inl.h" diff --git a/js/src/jit/ExecutableAllocator.h b/js/src/jit/ExecutableAllocator.h index eb53505879..10880567c3 100644 --- a/js/src/jit/ExecutableAllocator.h +++ b/js/src/jit/ExecutableAllocator.h @@ -42,7 +42,6 @@ #include "jit/mips32/Simulator-mips32.h" #include "jit/mips64/Simulator-mips64.h" #include "jit/ProcessExecutableMemory.h" -#include "js/GCAPI.h" #include "js/HashTable.h" #include "js/Vector.h" @@ -69,6 +68,8 @@ extern "C" void sync_instruction_memory(caddr_t v, u_int len); #include #endif +struct JSRuntime; + namespace JS { struct CodeSizes; } // namespace JS diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp index 938e233a1c..cf6b7400bb 100644 --- a/js/src/jit/Ion.cpp +++ b/js/src/jit/Ion.cpp @@ -53,6 +53,7 @@ #include "jsobjinlines.h" #include "jsscriptinlines.h" +#include "gc/Iteration-inl.h" #include "jit/JitFrames-inl.h" #include "jit/shared/Lowering-shared-inl.h" #include "vm/Debugger-inl.h" diff --git a/js/src/jit/IonBuilder.cpp b/js/src/jit/IonBuilder.cpp index a5adda4379..5cbbd3ca26 100644 --- a/js/src/jit/IonBuilder.cpp +++ b/js/src/jit/IonBuilder.cpp @@ -27,6 +27,7 @@ #include "jsopcodeinlines.h" #include "jsscriptinlines.h" +#include "gc/Nursery-inl.h" #include "jit/CompileInfo-inl.h" #include "jit/shared/Lowering-shared-inl.h" #include "vm/EnvironmentObject-inl.h" diff --git a/js/src/jit/IonCode.h b/js/src/jit/IonCode.h index e749738c82..509009b954 100644 --- a/js/src/jit/IonCode.h +++ b/js/src/jit/IonCode.h @@ -436,7 +436,7 @@ struct IonScript } MOZ_MUST_USE bool addTraceLoggerEvent(TraceLoggerEvent& event) { MOZ_ASSERT(event.hasPayload()); - return traceLoggerEvents_.append(Move(event)); + return traceLoggerEvents_.append(mozilla::Move(event)); } const uint8_t* snapshots() const { return reinterpret_cast(this) + snapshots_; diff --git a/js/src/jit/IonTypes.h b/js/src/jit/IonTypes.h index f24f2a230c..03f8456f0b 100644 --- a/js/src/jit/IonTypes.h +++ b/js/src/jit/IonTypes.h @@ -13,7 +13,6 @@ #include "jsfriendapi.h" #include "jstypes.h" -#include "js/GCAPI.h" #include "js/Value.h" #include "vm/String.h" diff --git a/js/src/jit/MIR.cpp b/js/src/jit/MIR.cpp index b6b879a595..4491e8ba4e 100644 --- a/js/src/jit/MIR.cpp +++ b/js/src/jit/MIR.cpp @@ -28,6 +28,7 @@ #include "jsboolinlines.h" #include "jsobjinlines.h" #include "jsscriptinlines.h" +#include "vm/UnboxedObject-inl.h" using namespace js; using namespace js::jit; diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp index 185ea2b68e..8a594ad2cf 100644 --- a/js/src/jit/MacroAssembler.cpp +++ b/js/src/jit/MacroAssembler.cpp @@ -20,7 +20,6 @@ #include "jit/Lowering.h" #include "jit/MIR.h" #include "js/Conversions.h" -#include "js/GCAPI.h" #include "vm/TraceLogging.h" #include "jsobjinlines.h" diff --git a/js/src/jsapi.h b/js/src/jsapi.h index 41aa126090..652d730ab0 100644 --- a/js/src/jsapi.h +++ b/js/src/jsapi.h @@ -562,48 +562,6 @@ struct JSFreeOp { /************************************************************************/ -typedef enum JSGCStatus { - JSGC_BEGIN, - JSGC_END -} JSGCStatus; - -typedef void -(* JSGCCallback)(JSContext* cx, JSGCStatus status, void* data); - -typedef void -(* JSObjectsTenuredCallback)(JSContext* cx, void* data); - -typedef enum JSFinalizeStatus { - /** - * Called when preparing to sweep a group of zones, before anything has been - * swept. The collector will not yield to the mutator before calling the - * callback with JSFINALIZE_GROUP_END status. - */ - JSFINALIZE_GROUP_START, - - /** - * Called when preparing to sweep a group of zones. Weak references to - * unmarked things have been removed and things that are not swept - * incrementally have been finalized at this point. The collector may yield - * to the mutator after this point. - */ - JSFINALIZE_GROUP_END, - - /** - * Called at the end of collection when everything has been swept. - */ - JSFINALIZE_COLLECTION_END -} JSFinalizeStatus; - -typedef void -(* JSFinalizeCallback)(JSFreeOp* fop, JSFinalizeStatus status, bool isZoneGC, void* data); - -typedef void -(* JSWeakPointerZonesCallback)(JSContext* cx, void* data); - -typedef void -(* JSWeakPointerCompartmentCallback)(JSContext* cx, JSCompartment* comp, void* data); - typedef bool (* JSInterruptCallback)(JSContext* cx); @@ -1572,205 +1530,6 @@ JS_updateMallocCounter(JSContext* cx, size_t nbytes); extern JS_PUBLIC_API(char*) JS_strdup(JSContext* cx, const char* s); -/** - * Register externally maintained GC roots. - * - * traceOp: the trace operation. For each root the implementation should call - * JS::TraceEdge whenever the root contains a traceable thing. - * data: the data argument to pass to each invocation of traceOp. - */ -extern JS_PUBLIC_API(bool) -JS_AddExtraGCRootsTracer(JSContext* cx, JSTraceDataOp traceOp, void* data); - -/** Undo a call to JS_AddExtraGCRootsTracer. */ -extern JS_PUBLIC_API(void) -JS_RemoveExtraGCRootsTracer(JSContext* cx, JSTraceDataOp traceOp, void* data); - -/* - * Garbage collector API. - */ -extern JS_PUBLIC_API(void) -JS_GC(JSContext* cx); - -extern JS_PUBLIC_API(void) -JS_MaybeGC(JSContext* cx); - -extern JS_PUBLIC_API(void) -JS_SetGCCallback(JSContext* cx, JSGCCallback cb, void* data); - -extern JS_PUBLIC_API(void) -JS_SetObjectsTenuredCallback(JSContext* cx, JSObjectsTenuredCallback cb, - void* data); - -extern JS_PUBLIC_API(bool) -JS_AddFinalizeCallback(JSContext* cx, JSFinalizeCallback cb, void* data); - -extern JS_PUBLIC_API(void) -JS_RemoveFinalizeCallback(JSContext* cx, JSFinalizeCallback cb); - -/* - * Weak pointers and garbage collection - * - * Weak pointers are by their nature not marked as part of garbage collection, - * but they may need to be updated in two cases after a GC: - * - * 1) Their referent was found not to be live and is about to be finalized - * 2) Their referent has been moved by a compacting GC - * - * To handle this, any part of the system that maintain weak pointers to - * JavaScript GC things must register a callback with - * JS_(Add,Remove)WeakPointer{ZoneGroup,Compartment}Callback(). This callback - * must then call JS_UpdateWeakPointerAfterGC() on all weak pointers it knows - * about. - * - * Since sweeping is incremental, we have several callbacks to avoid repeatedly - * having to visit all embedder structures. The WeakPointerZoneGroupCallback is - * called once for each strongly connected group of zones, whereas the - * WeakPointerCompartmentCallback is called once for each compartment that is - * visited while sweeping. Structures that cannot contain references in more - * than one compartment should sweep the relevant per-compartment structures - * using the latter callback to minimizer per-slice overhead. - * - * The argument to JS_UpdateWeakPointerAfterGC() is an in-out param. If the - * referent is about to be finalized the pointer will be set to null. If the - * referent has been moved then the pointer will be updated to point to the new - * location. - * - * Callers of this method are responsible for updating any state that is - * dependent on the object's address. For example, if the object's address is - * used as a key in a hashtable, then the object must be removed and - * re-inserted with the correct hash. - */ - -extern JS_PUBLIC_API(bool) -JS_AddWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb, void* data); - -extern JS_PUBLIC_API(void) -JS_RemoveWeakPointerZonesCallback(JSContext* cx, JSWeakPointerZonesCallback cb); - -extern JS_PUBLIC_API(bool) -JS_AddWeakPointerCompartmentCallback(JSContext* cx, JSWeakPointerCompartmentCallback cb, - void* data); - -extern JS_PUBLIC_API(void) -JS_RemoveWeakPointerCompartmentCallback(JSContext* cx, JSWeakPointerCompartmentCallback cb); - -extern JS_PUBLIC_API(void) -JS_UpdateWeakPointerAfterGC(JS::Heap* objp); - -extern JS_PUBLIC_API(void) -JS_UpdateWeakPointerAfterGCUnbarriered(JSObject** objp); - -typedef enum JSGCParamKey { - /** Maximum nominal heap before last ditch GC. */ - JSGC_MAX_BYTES = 0, - - /** Number of JS_malloc bytes before last ditch GC. */ - JSGC_MAX_MALLOC_BYTES = 1, - - /** Amount of bytes allocated by the GC. */ - JSGC_BYTES = 3, - - /** Number of times GC has been invoked. Includes both major and minor GC. */ - JSGC_NUMBER = 4, - - /** Select GC mode. */ - JSGC_MODE = 6, - - /** Number of cached empty GC chunks. */ - JSGC_UNUSED_CHUNKS = 7, - - /** Total number of allocated GC chunks. */ - JSGC_TOTAL_CHUNKS = 8, - - /** Max milliseconds to spend in an incremental GC slice. */ - JSGC_SLICE_TIME_BUDGET = 9, - - /** Maximum size the GC mark stack can grow to. */ - JSGC_MARK_STACK_LIMIT = 10, - - /** - * GCs less than this far apart in time will be considered 'high-frequency GCs'. - * See setGCLastBytes in jsgc.cpp. - */ - JSGC_HIGH_FREQUENCY_TIME_LIMIT = 11, - - /** Start of dynamic heap growth. */ - JSGC_HIGH_FREQUENCY_LOW_LIMIT = 12, - - /** End of dynamic heap growth. */ - JSGC_HIGH_FREQUENCY_HIGH_LIMIT = 13, - - /** Upper bound of heap growth. */ - JSGC_HIGH_FREQUENCY_HEAP_GROWTH_MAX = 14, - - /** Lower bound of heap growth. */ - JSGC_HIGH_FREQUENCY_HEAP_GROWTH_MIN = 15, - - /** Heap growth for low frequency GCs. */ - JSGC_LOW_FREQUENCY_HEAP_GROWTH = 16, - - /** - * If false, the heap growth factor is fixed at 3. If true, it is determined - * based on whether GCs are high- or low- frequency. - */ - JSGC_DYNAMIC_HEAP_GROWTH = 17, - - /** If true, high-frequency GCs will use a longer mark slice. */ - JSGC_DYNAMIC_MARK_SLICE = 18, - - /** Lower limit after which we limit the heap growth. */ - JSGC_ALLOCATION_THRESHOLD = 19, - - /** - * We try to keep at least this many unused chunks in the free chunk pool at - * all times, even after a shrinking GC. - */ - JSGC_MIN_EMPTY_CHUNK_COUNT = 21, - - /** We never keep more than this many unused chunks in the free chunk pool. */ - JSGC_MAX_EMPTY_CHUNK_COUNT = 22, - - /** Whether compacting GC is enabled. */ - JSGC_COMPACTING_ENABLED = 23, - - /** If true, painting can trigger IGC slices. */ - JSGC_REFRESH_FRAME_SLICES_ENABLED = 24, -} JSGCParamKey; - -extern JS_PUBLIC_API(void) -JS_SetGCParameter(JSContext* cx, JSGCParamKey key, uint32_t value); - -extern JS_PUBLIC_API(void) -JS_SetGGCMode(JSContext* cx, bool enabled); - -extern JS_PUBLIC_API(uint32_t) -JS_GetGCParameter(JSContext* cx, JSGCParamKey key); - -extern JS_PUBLIC_API(void) -JS_SetGCParametersBasedOnAvailableMemory(JSContext* cx, uint32_t availMem); - -/** - * Create a new JSString whose chars member refers to external memory, i.e., - * memory requiring application-specific finalization. - */ -extern JS_PUBLIC_API(JSString*) -JS_NewExternalString(JSContext* cx, const char16_t* chars, size_t length, - const JSStringFinalizer* fin); - -/** - * Return whether 'str' was created with JS_NewExternalString or - * JS_NewExternalStringWithClosure. - */ -extern JS_PUBLIC_API(bool) -JS_IsExternalString(JSString* str); - -/** - * Return the 'fin' arg passed to JS_NewExternalString. - */ -extern JS_PUBLIC_API(const JSStringFinalizer*) -JS_GetExternalStringFinalizer(JSString* str); - /** * Set the size of the native stack that should not be exceed. To disable * stack size checking pass 0. diff --git a/js/src/jsatom.h b/js/src/jsatom.h index eb43442e63..2a0a85a139 100644 --- a/js/src/jsatom.h +++ b/js/src/jsatom.h @@ -67,34 +67,9 @@ class AtomStateEntry struct AtomHasher { - struct Lookup - { - union { - const JS::Latin1Char* latin1Chars; - const char16_t* twoByteChars; - }; - bool isLatin1; - size_t length; - const JSAtom* atom; /* Optional. */ - JS::AutoCheckCannotGC nogc; - - HashNumber hash; - - Lookup(const char16_t* chars, size_t length) - : twoByteChars(chars), isLatin1(false), length(length), atom(nullptr) - { - hash = mozilla::HashString(chars, length); - } - Lookup(const JS::Latin1Char* chars, size_t length) - : latin1Chars(chars), isLatin1(true), length(length), atom(nullptr) - { - hash = mozilla::HashString(chars, length); - } - inline explicit Lookup(const JSAtom* atom); - }; - - static HashNumber hash(const Lookup& l) { return l.hash; } - static inline bool match(const AtomStateEntry& entry, const Lookup& lookup); + struct Lookup; + static inline HashNumber hash(const Lookup& l); + static MOZ_ALWAYS_INLINE bool match(const AtomStateEntry& entry, const Lookup& lookup); static void rekey(AtomStateEntry& k, const AtomStateEntry& newKey) { k = newKey; } }; diff --git a/js/src/jsatominlines.h b/js/src/jsatominlines.h index 0d8c5a9c9e..befb9875ad 100644 --- a/js/src/jsatominlines.h +++ b/js/src/jsatominlines.h @@ -34,6 +34,38 @@ js::AtomStateEntry::asPtrUnbarriered() const namespace js { +struct AtomHasher::Lookup +{ + union { + const JS::Latin1Char* latin1Chars; + const char16_t* twoByteChars; + }; + bool isLatin1; + size_t length; + const JSAtom* atom; /* Optional. */ + JS::AutoCheckCannotGC nogc; + + HashNumber hash; + + MOZ_ALWAYS_INLINE Lookup(const char16_t* chars, size_t length) + : twoByteChars(chars), isLatin1(false), length(length), atom(nullptr) + { + hash = mozilla::HashString(chars, length); + } + MOZ_ALWAYS_INLINE Lookup(const JS::Latin1Char* chars, size_t length) + : latin1Chars(chars), isLatin1(true), length(length), atom(nullptr) + { + hash = mozilla::HashString(chars, length); + } + inline explicit Lookup(const JSAtom* atom); +}; + +inline HashNumber +AtomHasher::hash(const Lookup& l) +{ + return l.hash; +} + inline jsid AtomToId(JSAtom* atom) { diff --git a/js/src/jscompartment.cpp b/js/src/jscompartment.cpp index a7dc6bc5f0..ef8404d74e 100644 --- a/js/src/jscompartment.cpp +++ b/js/src/jscompartment.cpp @@ -14,7 +14,6 @@ #include "jsiter.h" #include "jswrapper.h" -#include "gc/Marking.h" #include "gc/Policy.h" #include "jit/JitCompartment.h" #include "jit/JitOptions.h" @@ -32,6 +31,7 @@ #include "jsobjinlines.h" #include "jsscriptinlines.h" +#include "gc/Marking-inl.h" #include "vm/NativeObject-inl.h" using namespace js; diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index d296b971ed..0819ea128e 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -214,7 +214,6 @@ #include "gc/FindSCCs.h" #include "gc/GCInternals.h" #include "gc/GCTrace.h" -#include "gc/Marking.h" #include "gc/Memory.h" #include "gc/Policy.h" #include "jit/BaselineJIT.h" @@ -236,6 +235,11 @@ #include "jsobjinlines.h" #include "jsscriptinlines.h" +#include "gc/Heap-inl.h" +#include "gc/Iteration-inl.h" +#include "gc/Marking-inl.h" +#include "gc/Nursery-inl.h" +#include "vm/SPSProfiler-inl.h" #include "vm/Stack-inl.h" #include "vm/String-inl.h" @@ -264,7 +268,7 @@ const AllocKind gc::slotsToThingKind[] = { static_assert(JS_ARRAY_LENGTH(slotsToThingKind) == SLOTS_TO_THING_KIND_LIMIT, "We have defined a slot count for each kind."); -#define CHECK_THING_SIZE(allocKind, traceKind, type, sizedType) \ +#define CHECK_THING_SIZE(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ static_assert(sizeof(sizedType) >= SortedArenaList::MinThingSize, \ #sizedType " is smaller than SortedArenaList::MinThingSize!"); \ static_assert(sizeof(sizedType) >= sizeof(FreeSpan), \ @@ -275,7 +279,7 @@ FOR_EACH_ALLOCKIND(CHECK_THING_SIZE); #undef CHECK_THING_SIZE const uint32_t Arena::ThingSizes[] = { -#define EXPAND_THING_SIZE(allocKind, traceKind, type, sizedType) \ +#define EXPAND_THING_SIZE(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ sizeof(sizedType), FOR_EACH_ALLOCKIND(EXPAND_THING_SIZE) #undef EXPAND_THING_SIZE @@ -289,7 +293,7 @@ FreeSpan ArenaLists::placeholder; #define OFFSET(type) uint32_t(ArenaHeaderSize + (ArenaSize - ArenaHeaderSize) % sizeof(type)) const uint32_t Arena::FirstThingOffsets[] = { -#define EXPAND_FIRST_THING_OFFSET(allocKind, traceKind, type, sizedType) \ +#define EXPAND_FIRST_THING_OFFSET(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ OFFSET(sizedType), FOR_EACH_ALLOCKIND(EXPAND_FIRST_THING_OFFSET) #undef EXPAND_FIRST_THING_OFFSET @@ -300,7 +304,7 @@ FOR_EACH_ALLOCKIND(EXPAND_FIRST_THING_OFFSET) #define COUNT(type) uint32_t((ArenaSize - ArenaHeaderSize) / sizeof(type)) const uint32_t Arena::ThingsPerArena[] = { -#define EXPAND_THINGS_PER_ARENA(allocKind, traceKind, type, sizedType) \ +#define EXPAND_THINGS_PER_ARENA(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ COUNT(sizedType), FOR_EACH_ALLOCKIND(EXPAND_THINGS_PER_ARENA) #undef EXPAND_THINGS_PER_ARENA @@ -584,7 +588,7 @@ FinalizeArenas(FreeOp* fop, ArenaLists::KeepArenasEnum keepArenas) { switch (thingKind) { -#define EXPAND_CASE(allocKind, traceKind, type, sizedType) \ +#define EXPAND_CASE(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ case AllocKind::allocKind: \ return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); FOR_EACH_ALLOCKIND(EXPAND_CASE) @@ -893,8 +897,7 @@ GCRuntime::GCRuntime(JSRuntime* rt) : objectsMarkedInDeadZones(0), poked(false), fullCompartmentChecks(false), - mallocBytesUntilGC(0), - mallocGCTriggered(false), + gcBeginCallbackDepth(0), alwaysPreserveCode(false), inUnsafeRegion(0), #ifdef DEBUG @@ -910,6 +913,194 @@ GCRuntime::GCRuntime(JSRuntime* rt) : setGCMode(JSGC_MODE_GLOBAL); } +#ifdef JS_GC_ZEAL + +void +GCRuntime::getZealBits(uint32_t* zealBits, uint32_t* frequency, uint32_t* scheduled) +{ + *zealBits = zealModeBits; + *frequency = zealFrequency; + *scheduled = nextScheduled; +} + +const char* gc::ZealModeHelpText = + " Specifies how zealous the garbage collector should be. Some of these modes can\n" + " be set simultaneously, by passing multiple level options, e.g. \"2;4\" will activate\n" + " both modes 2 and 4. Modes can be specified by name or number.\n" + " \n" + " Values:\n" + " 0: (None) Normal amount of collection (resets all modes)\n" + " 1: (RootsChange) Collect when roots are added or removed\n" + " 2: (Alloc) Collect when every N allocations (default: 100)\n" + " 3: (FrameGC) Collect when the window paints (browser only)\n" + " 4: (VerifierPre) Verify pre write barriers between instructions\n" + " 5: (FrameVerifierPre) Verify pre write barriers between paints\n" + " 6: (StackRooting) Verify stack rooting\n" + " 7: (GenerationalGC) Collect the nursery every N nursery allocations\n" + " 8: (IncrementalRootsThenFinish) Incremental GC in two slices: 1) mark roots 2) finish collection\n" + " 9: (IncrementalMarkAllThenFinish) Incremental GC in two slices: 1) mark all 2) new marking and finish\n" + " 10: (IncrementalMultipleSlices) Incremental GC in multiple slices\n" + " 11: (IncrementalMarkingValidator) Verify incremental marking\n" + " 12: (ElementsBarrier) Always use the individual element post-write barrier, regardless of elements size\n" + " 13: (CheckHashTablesOnMinorGC) Check internal hashtables on minor GC\n" + " 14: (Compact) Perform a shrinking collection every N allocations\n" + " 15: (CheckHeapAfterGC) Walk the heap to check its integrity after every GC\n" + " 16: (CheckNursery) Check nursery integrity on minor GC\n" + " 17: (IncrementalSweepThenFinish) Incremental GC in two slices: 1) start sweeping 2) finish collection\n"; + +// The set of zeal modes that control incremental slices. These modes are +// mutually exclusive. +static const mozilla::EnumSet IncrementalSliceZealModes = { + ZealMode::IncrementalRootsThenFinish, + ZealMode::IncrementalMarkAllThenFinish, + ZealMode::IncrementalMultipleSlices, + ZealMode::IncrementalSweepThenFinish +}; + +void +GCRuntime::setZeal(uint8_t zeal, uint32_t frequency) +{ + MOZ_ASSERT(zeal <= unsigned(ZealMode::Limit)); + + if (verifyPreData) + VerifyBarriers(rt, PreBarrierVerifier); + + if (zeal == 0) { + if (hasZealMode(ZealMode::GenerationalGC)) { + evictNursery(JS::gcreason::DEBUG_GC); + nursery().leaveZealMode(); + } + + if (isIncrementalGCInProgress()) + finishGC(JS::gcreason::DEBUG_GC); + } + + ZealMode zealMode = ZealMode(zeal); + if (zealMode == ZealMode::GenerationalGC) { + for (ZoneGroupsIter group(rt); !group.done(); group.next()) + group->nursery().enterZealMode(); + } + + // Some modes are mutually exclusive. If we're setting one of those, we + // first reset all of them. + if (IncrementalSliceZealModes.contains(zealMode)) { + for (auto mode : IncrementalSliceZealModes) + clearZealMode(mode); + } + + bool schedule = zealMode >= ZealMode::Alloc; + if (zeal != 0) + zealModeBits |= 1 << unsigned(zeal); + else + zealModeBits = 0; + zealFrequency = frequency; + nextScheduled = schedule ? frequency : 0; +} + +void +GCRuntime::setNextScheduled(uint32_t count) +{ + nextScheduled = count; +} + +bool +GCRuntime::parseAndSetZeal(const char* str) +{ + int frequency = -1; + bool foundFrequency = false; + mozilla::Vector zeals; + + static const struct { + const char* const zealMode; + size_t length; + uint32_t zeal; + } zealModes[] = { +#define ZEAL_MODE(name, value) {#name, sizeof(#name) - 1, value}, + JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE) +#undef ZEAL_MODE + {"None", 4, 0} + }; + + do { + int zeal = -1; + + const char* p = nullptr; + if (isdigit(str[0])) { + zeal = atoi(str); + + size_t offset = strspn(str, "0123456789"); + p = str + offset; + } else { + for (auto z : zealModes) { + if (!strncmp(str, z.zealMode, z.length)) { + zeal = z.zeal; + p = str + z.length; + break; + } + } + } + if (p) { + if (!*p || *p == ';') { + frequency = JS_DEFAULT_ZEAL_FREQ; + } else if (*p == ',') { + frequency = atoi(p + 1); + foundFrequency = true; + } + } + + if (zeal < 0 || zeal > int(ZealMode::Limit) || frequency <= 0) { + fprintf(stderr, "Format: JS_GC_ZEAL=level(;level)*[,N]\n"); + fputs(ZealModeHelpText, stderr); + return false; + } + + if (!zeals.emplaceBack(zeal)) { + return false; + } + } while (!foundFrequency && + (str = strchr(str, ';')) != nullptr && + str++); + + for (auto z : zeals) + setZeal(z, frequency); + return true; +} + +static const char* +AllocKindName(AllocKind kind) +{ + static const char* names[] = { +#define EXPAND_THING_NAME(allocKind, _1, _2, _3, _4, _5) \ + #allocKind, +FOR_EACH_ALLOCKIND(EXPAND_THING_NAME) +#undef EXPAND_THING_NAME + }; + static_assert(ArrayLength(names) == size_t(AllocKind::LIMIT), + "names array should have an entry for every AllocKind"); + + size_t i = size_t(kind); + MOZ_ASSERT(i < ArrayLength(names)); + return names[i]; +} + +void +js::gc::DumpArenaInfo() +{ + fprintf(stderr, "Arena header size: %" PRIuSIZE "\n\n", ArenaHeaderSize); + + fprintf(stderr, "GC thing kinds:\n"); + fprintf(stderr, "%25s %8s %8s %8s\n", "AllocKind:", "Size:", "Count:", "Padding:"); + for (auto kind : AllAllocKinds()) { + fprintf(stderr, + "%25s %8" PRIuSIZE " %8" PRIuSIZE " %8" PRIuSIZE "\n", + AllocKindName(kind), + Arena::thingSize(kind), + Arena::thingsPerArena(kind), + Arena::firstThingOffset(kind) - ArenaHeaderSize); + } +} + +#endif // JS_GC_ZEAL /* * Lifetime in number of major GCs for type sets attached to scripts containing * observed types. @@ -1251,29 +1442,6 @@ GCRuntime::callObjectsTenuredCallback() tenuredCallback.op(rt->contextFromMainThread(), tenuredCallback.data); } -namespace { - -class AutoNotifyGCActivity { - public: - explicit AutoNotifyGCActivity(GCRuntime& gc) : gc_(gc) { - if (!gc_.isIncrementalGCInProgress()) { - gcstats::AutoPhase ap(gc_.stats, gcstats::PHASE_GC_BEGIN); - gc_.callGCCallback(JSGC_BEGIN); - } - } - ~AutoNotifyGCActivity() { - if (!gc_.isIncrementalGCInProgress()) { - gcstats::AutoPhase ap(gc_.stats, gcstats::PHASE_GC_END); - gc_.callGCCallback(JSGC_END); - } - } - - private: - GCRuntime& gc_; -}; - -} // (anon) - bool GCRuntime::addFinalizeCallback(JSFinalizeCallback callback, void* data) { @@ -2064,7 +2232,7 @@ UpdateArenaPointers(MovingTracer* trc, Arena* arena) AllocKind kind = arena->getAllocKind(); switch (kind) { -#define EXPAND_CASE(allocKind, traceKind, type, sizedType) \ +#define EXPAND_CASE(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ case AllocKind::allocKind: \ UpdateArenaPointersTyped(trc, arena, JS::TraceKind::traceKind); \ return; @@ -3380,7 +3548,7 @@ static const char* AllocKindToAscii(AllocKind kind) { switch(kind) { -#define MAKE_CASE(allocKind, traceKind, type, sizedType) \ +#define MAKE_CASE(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ case AllocKind:: allocKind: return #allocKind; FOR_EACH_ALLOCKIND(MAKE_CASE) #undef MAKE_CASE @@ -4570,7 +4738,7 @@ NextIncomingCrossCompartmentPointer(JSObject* prev, bool unlink) } void -js::DelayCrossCompartmentGrayMarking(JSObject* src) +js::gc::DelayCrossCompartmentGrayMarking(JSObject* src) { MOZ_ASSERT(IsGrayListObject(src)); @@ -6725,6 +6893,53 @@ class AutoExposeLiveCrossZoneEdges } /* anonymous namespace */ +class js::gc::AutoCallGCCallbacks { + GCRuntime& gc_; + + public: + explicit AutoCallGCCallbacks(GCRuntime& gc) : gc_(gc) { + gc_.maybeCallBeginCallback(); + } + ~AutoCallGCCallbacks() { + gc_.maybeCallEndCallback(); + } +}; + +void +GCRuntime::maybeCallBeginCallback() +{ + if (isIncrementalGCInProgress()) + return; + + if (gcBeginCallbackDepth == 0) { + // Save scheduled zone information in case the callback changes it. + for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) + zone->gcScheduledSaved_ = zone->gcScheduled_; + } + + gcBeginCallbackDepth++; + + callGCCallback(JSGC_BEGIN); + + MOZ_ASSERT(gcBeginCallbackDepth != 0); + gcBeginCallbackDepth--; + + if (gcBeginCallbackDepth == 0) { + // Restore scheduled zone information again. + for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) + zone->gcScheduled_ = zone->gcScheduledSaved_; + } +} + +void +GCRuntime::maybeCallEndCallback() +{ + if (isIncrementalGCInProgress()) + return; + + callGCCallback(JSGC_END); +} + /* * Run one GC "cycle" (either a slice of incremental GC or an entire * non-incremental GC. We disable inlining to ensure that the bottom of the @@ -6737,8 +6952,8 @@ class AutoExposeLiveCrossZoneEdges MOZ_NEVER_INLINE bool GCRuntime::gcCycle(bool nonincrementalByAPI, SliceBudget& budget, JS::gcreason::Reason reason) { - // Note that the following is allowed to re-enter GC in the finalizer. - AutoNotifyGCActivity notify(*this); + // Note that GC callbacks are allowed to re-enter GC. + AutoCallGCCallbacks callCallbacks(*this); gcstats::AutoGCSlice agc(stats, scanZonesBeforeGC(), invocationKind, budget, reason); @@ -8126,14 +8341,7 @@ StateName(State state) } void -AutoAssertHeapBusy::checkCondition(JSRuntime *rt) -{ - this->rt = rt; - MOZ_ASSERT(rt->isHeapBusy()); -} - -void -AutoAssertEmptyNursery::checkCondition(JSRuntime *rt) { +AutoAssertEmptyNursery::checkCondition(JSContext* cx) { if (!noAlloc) noAlloc.emplace(rt); this->rt = rt; diff --git a/js/src/jsgc.h b/js/src/jsgc.h index c463eb6237..25609f643d 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -24,9 +24,6 @@ namespace js { -class AutoLockHelperThreadState; -unsigned GetCPUCount(); - namespace gcstats { struct Statistics; } // namespace gcstats @@ -35,40 +32,6 @@ class Nursery; namespace gc { -struct FinalizePhase; - -#define GCSTATES(D) \ - D(NotActive) \ - D(MarkRoots) \ - D(Mark) \ - D(Sweep) \ - D(Finalize) \ - D(Compact) \ - D(Decommit) -enum class State { -#define MAKE_STATE(name) name, - GCSTATES(MAKE_STATE) -#undef MAKE_STATE -}; - -// Reasons we reset an ongoing incremental GC or perform a non-incremental GC. -#define GC_ABORT_REASONS(D) \ - D(None) \ - D(NonIncrementalRequested) \ - D(AbortRequested) \ - D(KeepAtomsSet) \ - D(IncrementalDisabled) \ - D(ModeChange) \ - D(MallocBytesTrigger) \ - D(GCBytesTrigger) \ - D(ZoneChange) \ - D(CompartmentRevived) -enum class AbortReason { -#define MAKE_REASON(name) name, - GC_ABORT_REASONS(MAKE_REASON) -#undef MAKE_REASON -}; - /* * Map from C++ type to alloc kind for non-object types. JSObject does not have * a 1:1 mapping, so must use Arena::thingSize. @@ -76,771 +39,15 @@ enum class AbortReason { * The AllocKind is available as MapTypeToFinalizeKind::kind. */ template struct MapTypeToFinalizeKind {}; -#define EXPAND_MAPTYPETOFINALIZEKIND(allocKind, traceKind, type, sizedType) \ +#define EXPAND_MAPTYPETOFINALIZEKIND(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ template <> struct MapTypeToFinalizeKind { \ static const AllocKind kind = AllocKind::allocKind; \ }; FOR_EACH_NONOBJECT_ALLOCKIND(EXPAND_MAPTYPETOFINALIZEKIND) #undef EXPAND_MAPTYPETOFINALIZEKIND -template struct ParticipatesInCC {}; -#define EXPAND_PARTICIPATES_IN_CC(_, type, addToCCKind) \ - template <> struct ParticipatesInCC { static const bool value = addToCCKind; }; -JS_FOR_EACH_TRACEKIND(EXPAND_PARTICIPATES_IN_CC) -#undef EXPAND_PARTICIPATES_IN_CC - -static inline bool -IsNurseryAllocable(AllocKind kind) -{ - MOZ_ASSERT(IsValidAllocKind(kind)); - static const bool map[] = { - true, /* AllocKind::FUNCTION */ - true, /* AllocKind::FUNCTION_EXTENDED */ - false, /* AllocKind::OBJECT0 */ - true, /* AllocKind::OBJECT0_BACKGROUND */ - false, /* AllocKind::OBJECT2 */ - true, /* AllocKind::OBJECT2_BACKGROUND */ - false, /* AllocKind::OBJECT4 */ - true, /* AllocKind::OBJECT4_BACKGROUND */ - false, /* AllocKind::OBJECT8 */ - true, /* AllocKind::OBJECT8_BACKGROUND */ - false, /* AllocKind::OBJECT12 */ - true, /* AllocKind::OBJECT12_BACKGROUND */ - false, /* AllocKind::OBJECT16 */ - true, /* AllocKind::OBJECT16_BACKGROUND */ - false, /* AllocKind::SCRIPT */ - false, /* AllocKind::LAZY_SCRIPT */ - false, /* AllocKind::SHAPE */ - false, /* AllocKind::ACCESSOR_SHAPE */ - false, /* AllocKind::BASE_SHAPE */ - false, /* AllocKind::OBJECT_GROUP */ - false, /* AllocKind::FAT_INLINE_STRING */ - false, /* AllocKind::STRING */ - false, /* AllocKind::EXTERNAL_STRING */ - false, /* AllocKind::FAT_INLINE_ATOM */ - false, /* AllocKind::ATOM */ - false, /* AllocKind::SYMBOL */ - false, /* AllocKind::BIGINT */ - false, /* AllocKind::JITCODE */ - false, /* AllocKind::SCOPE */ - false, /* AllocKind::REGEXP_SHARED */ - }; - JS_STATIC_ASSERT(JS_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT)); - return map[size_t(kind)]; -} - -static inline bool -IsBackgroundFinalized(AllocKind kind) -{ - MOZ_ASSERT(IsValidAllocKind(kind)); - static const bool map[] = { - true, /* AllocKind::FUNCTION */ - true, /* AllocKind::FUNCTION_EXTENDED */ - false, /* AllocKind::OBJECT0 */ - true, /* AllocKind::OBJECT0_BACKGROUND */ - false, /* AllocKind::OBJECT2 */ - true, /* AllocKind::OBJECT2_BACKGROUND */ - false, /* AllocKind::OBJECT4 */ - true, /* AllocKind::OBJECT4_BACKGROUND */ - false, /* AllocKind::OBJECT8 */ - true, /* AllocKind::OBJECT8_BACKGROUND */ - false, /* AllocKind::OBJECT12 */ - true, /* AllocKind::OBJECT12_BACKGROUND */ - false, /* AllocKind::OBJECT16 */ - true, /* AllocKind::OBJECT16_BACKGROUND */ - false, /* AllocKind::SCRIPT */ - true, /* AllocKind::LAZY_SCRIPT */ - true, /* AllocKind::SHAPE */ - true, /* AllocKind::ACCESSOR_SHAPE */ - true, /* AllocKind::BASE_SHAPE */ - true, /* AllocKind::OBJECT_GROUP */ - true, /* AllocKind::FAT_INLINE_STRING */ - true, /* AllocKind::STRING */ - false, /* AllocKind::EXTERNAL_STRING */ - true, /* AllocKind::FAT_INLINE_ATOM */ - true, /* AllocKind::ATOM */ - true, /* AllocKind::SYMBOL */ - true, /* AllocKind::BIGINT */ - false, /* AllocKind::JITCODE */ - true, /* AllocKind::SCOPE */ - true, /* AllocKind::REGEXP_SHARED */ - }; - JS_STATIC_ASSERT(JS_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT)); - return map[size_t(kind)]; -} - -static inline bool -CanBeFinalizedInBackground(AllocKind kind, const Class* clasp) -{ - MOZ_ASSERT(IsObjectAllocKind(kind)); - /* If the class has no finalizer or a finalizer that is safe to call on - * a different thread, we change the alloc kind. For example, - * AllocKind::OBJECT0 calls the finalizer on the main thread, - * AllocKind::OBJECT0_BACKGROUND calls the finalizer on the gcHelperThread. - * IsBackgroundFinalized is called to prevent recursively incrementing - * the alloc kind; kind may already be a background finalize kind. - */ - return (!IsBackgroundFinalized(kind) && - (!clasp->hasFinalize() || (clasp->flags & JSCLASS_BACKGROUND_FINALIZE))); -} - -/* Capacity for slotsToThingKind */ -const size_t SLOTS_TO_THING_KIND_LIMIT = 17; - -extern const AllocKind slotsToThingKind[]; - -/* Get the best kind to use when making an object with the given slot count. */ -static inline AllocKind -GetGCObjectKind(size_t numSlots) -{ - if (numSlots >= SLOTS_TO_THING_KIND_LIMIT) - return AllocKind::OBJECT16; - return slotsToThingKind[numSlots]; -} - -/* As for GetGCObjectKind, but for dense array allocation. */ -static inline AllocKind -GetGCArrayKind(size_t numElements) -{ - /* - * Dense arrays can use their fixed slots to hold their elements array - * (less two Values worth of ObjectElements header), but if more than the - * maximum number of fixed slots is needed then the fixed slots will be - * unused. - */ - JS_STATIC_ASSERT(ObjectElements::VALUES_PER_HEADER == 2); - if (numElements > NativeObject::MAX_DENSE_ELEMENTS_COUNT || - numElements + ObjectElements::VALUES_PER_HEADER >= SLOTS_TO_THING_KIND_LIMIT) - { - return AllocKind::OBJECT2; - } - return slotsToThingKind[numElements + ObjectElements::VALUES_PER_HEADER]; -} - -static inline AllocKind -GetGCObjectFixedSlotsKind(size_t numFixedSlots) -{ - MOZ_ASSERT(numFixedSlots < SLOTS_TO_THING_KIND_LIMIT); - return slotsToThingKind[numFixedSlots]; -} - -// Get the best kind to use when allocating an object that needs a specific -// number of bytes. -static inline AllocKind -GetGCObjectKindForBytes(size_t nbytes) -{ - MOZ_ASSERT(nbytes <= JSObject::MAX_BYTE_SIZE); - - if (nbytes <= sizeof(NativeObject)) - return AllocKind::OBJECT0; - nbytes -= sizeof(NativeObject); - - size_t dataSlots = AlignBytes(nbytes, sizeof(Value)) / sizeof(Value); - MOZ_ASSERT(nbytes <= dataSlots * sizeof(Value)); - return GetGCObjectKind(dataSlots); -} - -static inline AllocKind -GetBackgroundAllocKind(AllocKind kind) -{ - MOZ_ASSERT(!IsBackgroundFinalized(kind)); - MOZ_ASSERT(IsObjectAllocKind(kind)); - return AllocKind(size_t(kind) + 1); -} - -/* Get the number of fixed slots and initial capacity associated with a kind. */ -static inline size_t -GetGCKindSlots(AllocKind thingKind) -{ - /* Using a switch in hopes that thingKind will usually be a compile-time constant. */ - switch (thingKind) { - case AllocKind::FUNCTION: - case AllocKind::OBJECT0: - case AllocKind::OBJECT0_BACKGROUND: - return 0; - case AllocKind::FUNCTION_EXTENDED: - case AllocKind::OBJECT2: - case AllocKind::OBJECT2_BACKGROUND: - return 2; - case AllocKind::OBJECT4: - case AllocKind::OBJECT4_BACKGROUND: - return 4; - case AllocKind::OBJECT8: - case AllocKind::OBJECT8_BACKGROUND: - return 8; - case AllocKind::OBJECT12: - case AllocKind::OBJECT12_BACKGROUND: - return 12; - case AllocKind::OBJECT16: - case AllocKind::OBJECT16_BACKGROUND: - return 16; - default: - MOZ_CRASH("Bad object alloc kind"); - } -} - -static inline size_t -GetGCKindSlots(AllocKind thingKind, const Class* clasp) -{ - size_t nslots = GetGCKindSlots(thingKind); - - /* An object's private data uses the space taken by its last fixed slot. */ - if (clasp->flags & JSCLASS_HAS_PRIVATE) { - MOZ_ASSERT(nslots > 0); - nslots--; - } - - /* - * Functions have a larger alloc kind than AllocKind::OBJECT to reserve - * space for the extra fields in JSFunction, but have no fixed slots. - */ - if (clasp == FunctionClassPtr) - nslots = 0; - - return nslots; -} - -static inline size_t -GetGCKindBytes(AllocKind thingKind) -{ - return sizeof(JSObject_Slots0) + GetGCKindSlots(thingKind) * sizeof(Value); -} - -/* - * A single segment of a SortedArenaList. Each segment has a head and a tail, - * which track the start and end of a segment for O(1) append and concatenation. - */ -struct SortedArenaListSegment -{ - Arena* head; - Arena** tailp; - - void clear() { - head = nullptr; - tailp = &head; - } - - bool isEmpty() const { - return tailp == &head; - } - - // Appends |arena| to this segment. - void append(Arena* arena) { - MOZ_ASSERT(arena); - MOZ_ASSERT_IF(head, head->getAllocKind() == arena->getAllocKind()); - *tailp = arena; - tailp = &arena->next; - } - - // Points the tail of this segment at |arena|, which may be null. Note - // that this does not change the tail itself, but merely which arena - // follows it. This essentially turns the tail into a cursor (see also the - // description of ArenaList), but from the perspective of a SortedArenaList - // this makes no difference. - void linkTo(Arena* arena) { - *tailp = arena; - } -}; - -/* - * Arena lists have a head and a cursor. The cursor conceptually lies on arena - * boundaries, i.e. before the first arena, between two arenas, or after the - * last arena. - * - * Arenas are usually sorted in order of increasing free space, with the cursor - * following the Arena currently being allocated from. This ordering should not - * be treated as an invariant, however, as the free lists may be cleared, - * leaving arenas previously used for allocation partially full. Sorting order - * is restored during sweeping. - - * Arenas following the cursor should not be full. - */ -class ArenaList { - // The cursor is implemented via an indirect pointer, |cursorp_|, to allow - // for efficient list insertion at the cursor point and other list - // manipulations. - // - // - If the list is empty: |head| is null, |cursorp_| points to |head|, and - // therefore |*cursorp_| is null. - // - // - If the list is not empty: |head| is non-null, and... - // - // - If the cursor is at the start of the list: |cursorp_| points to - // |head|, and therefore |*cursorp_| points to the first arena. - // - // - If cursor is at the end of the list: |cursorp_| points to the |next| - // field of the last arena, and therefore |*cursorp_| is null. - // - // - If the cursor is at neither the start nor the end of the list: - // |cursorp_| points to the |next| field of the arena preceding the - // cursor, and therefore |*cursorp_| points to the arena following the - // cursor. - // - // |cursorp_| is never null. - // - Arena* head_; - Arena** cursorp_; - - void copy(const ArenaList& other) { - other.check(); - head_ = other.head_; - cursorp_ = other.isCursorAtHead() ? &head_ : other.cursorp_; - check(); - } - - public: - ArenaList() { - clear(); - } - - ArenaList(const ArenaList& other) { - copy(other); - } - - ArenaList& operator=(const ArenaList& other) { - copy(other); - return *this; - } - - explicit ArenaList(const SortedArenaListSegment& segment) { - head_ = segment.head; - cursorp_ = segment.isEmpty() ? &head_ : segment.tailp; - check(); - } - - // This does checking just of |head_| and |cursorp_|. - void check() const { -#ifdef DEBUG - // If the list is empty, it must have this form. - MOZ_ASSERT_IF(!head_, cursorp_ == &head_); - - // If there's an arena following the cursor, it must not be full. - Arena* cursor = *cursorp_; - MOZ_ASSERT_IF(cursor, cursor->hasFreeThings()); -#endif - } - - void clear() { - head_ = nullptr; - cursorp_ = &head_; - check(); - } - - ArenaList copyAndClear() { - ArenaList result = *this; - clear(); - return result; - } - - bool isEmpty() const { - check(); - return !head_; - } - - // This returns nullptr if the list is empty. - Arena* head() const { - check(); - return head_; - } - - bool isCursorAtHead() const { - check(); - return cursorp_ == &head_; - } - - bool isCursorAtEnd() const { - check(); - return !*cursorp_; - } - - void moveCursorToEnd() { - while (!isCursorAtEnd()) { - cursorp_ = &(*cursorp_)->next; - } - } - - // This can return nullptr. - Arena* arenaAfterCursor() const { - check(); - return *cursorp_; - } - - // This returns the arena after the cursor and moves the cursor past it. - Arena* takeNextArena() { - check(); - Arena* arena = *cursorp_; - if (!arena) - return nullptr; - cursorp_ = &arena->next; - check(); - return arena; - } - - // This does two things. - // - Inserts |a| at the cursor. - // - Leaves the cursor sitting just before |a|, if |a| is not full, or just - // after |a|, if |a| is full. - void insertAtCursor(Arena* a) { - check(); - a->next = *cursorp_; - *cursorp_ = a; - // At this point, the cursor is sitting before |a|. Move it after |a| - // if necessary. - if (!a->hasFreeThings()) - cursorp_ = &a->next; - check(); - } - - // Inserts |a| at the cursor, then moves the cursor past it. - void insertBeforeCursor(Arena* a) { - check(); - a->next = *cursorp_; - *cursorp_ = a; - cursorp_ = &a->next; - check(); - } - - // This inserts |other|, which must be full, at the cursor of |this|. - ArenaList& insertListWithCursorAtEnd(const ArenaList& other) { - check(); - other.check(); - MOZ_ASSERT(other.isCursorAtEnd()); - if (other.isCursorAtHead()) - return *this; - // Insert the full arenas of |other| after those of |this|. - *other.cursorp_ = *cursorp_; - *cursorp_ = other.head_; - cursorp_ = other.cursorp_; - check(); - return *this; - } - - Arena* removeRemainingArenas(Arena** arenap); - Arena** pickArenasToRelocate(size_t& arenaTotalOut, size_t& relocTotalOut); - Arena* relocateArenas(Arena* toRelocate, Arena* relocated, - SliceBudget& sliceBudget, gcstats::Statistics& stats); -}; - -/* - * A class that holds arenas in sorted order by appending arenas to specific - * segments. Each segment has a head and a tail, which can be linked up to - * other segments to create a contiguous ArenaList. - */ -class SortedArenaList -{ - public: - // The minimum size, in bytes, of a GC thing. - static const size_t MinThingSize = 16; - - static_assert(ArenaSize <= 4096, "When increasing the Arena size, please consider how"\ - " this will affect the size of a SortedArenaList."); - - static_assert(MinThingSize >= 16, "When decreasing the minimum thing size, please consider"\ - " how this will affect the size of a SortedArenaList."); - - private: - // The maximum number of GC things that an arena can hold. - static const size_t MaxThingsPerArena = (ArenaSize - ArenaHeaderSize) / MinThingSize; - - size_t thingsPerArena_; - SortedArenaListSegment segments[MaxThingsPerArena + 1]; - - // Convenience functions to get the nth head and tail. - Arena* headAt(size_t n) { return segments[n].head; } - Arena** tailAt(size_t n) { return segments[n].tailp; } - - public: - explicit SortedArenaList(size_t thingsPerArena = MaxThingsPerArena) { - reset(thingsPerArena); - } - - void setThingsPerArena(size_t thingsPerArena) { - MOZ_ASSERT(thingsPerArena && thingsPerArena <= MaxThingsPerArena); - thingsPerArena_ = thingsPerArena; - } - - // Resets the first |thingsPerArena| segments of this list for further use. - void reset(size_t thingsPerArena = MaxThingsPerArena) { - setThingsPerArena(thingsPerArena); - // Initialize the segments. - for (size_t i = 0; i <= thingsPerArena; ++i) - segments[i].clear(); - } - - // Inserts an arena, which has room for |nfree| more things, in its segment. - void insertAt(Arena* arena, size_t nfree) { - MOZ_ASSERT(nfree <= thingsPerArena_); - segments[nfree].append(arena); - } - - // Remove all empty arenas, inserting them as a linked list. - void extractEmpty(Arena** empty) { - SortedArenaListSegment& segment = segments[thingsPerArena_]; - if (segment.head) { - *segment.tailp = *empty; - *empty = segment.head; - segment.clear(); - } - } - - // Links up the tail of each non-empty segment to the head of the next - // non-empty segment, creating a contiguous list that is returned as an - // ArenaList. This is not a destructive operation: neither the head nor tail - // of any segment is modified. However, note that the Arenas in the - // resulting ArenaList should be treated as read-only unless the - // SortedArenaList is no longer needed: inserting or removing arenas would - // invalidate the SortedArenaList. - ArenaList toArenaList() { - // Link the non-empty segment tails up to the non-empty segment heads. - size_t tailIndex = 0; - for (size_t headIndex = 1; headIndex <= thingsPerArena_; ++headIndex) { - if (headAt(headIndex)) { - segments[tailIndex].linkTo(headAt(headIndex)); - tailIndex = headIndex; - } - } - // Point the tail of the final non-empty segment at null. Note that if - // the list is empty, this will just set segments[0].head to null. - segments[tailIndex].linkTo(nullptr); - // Create an ArenaList with head and cursor set to the head and tail of - // the first segment (if that segment is empty, only the head is used). - return ArenaList(segments[0]); - } -}; - -enum ShouldCheckThresholds -{ - DontCheckThresholds = 0, - CheckThresholds = 1 -}; - -class ArenaLists -{ - JSRuntime* runtime_; - - /* - * For each arena kind its free list is represented as the first span with - * free things. Initially all the spans are initialized as empty. After we - * find a new arena with available things we move its first free span into - * the list and set the arena as fully allocated. way we do not need to - * update the arena after the initial allocation. When starting the - * GC we only move the head of the of the list of spans back to the arena - * only for the arena that was not fully allocated. - */ - AllAllocKindArray freeLists; - - // Because the JITs can allocate from the free lists, they cannot be null. - // We use a placeholder FreeSpan that is empty (and wihout an associated - // Arena) so the JITs can fall back gracefully. - static FreeSpan placeholder; - - AllAllocKindArray arenaLists; - - enum BackgroundFinalizeStateEnum { BFS_DONE, BFS_RUN }; - - typedef mozilla::Atomic - BackgroundFinalizeState; - - /* The current background finalization state, accessed atomically. */ - AllAllocKindArray backgroundFinalizeState; - - /* For each arena kind, a list of arenas remaining to be swept. */ - AllAllocKindArray arenaListsToSweep; - - /* During incremental sweeping, a list of the arenas already swept. */ - AllocKind incrementalSweptArenaKind; - ArenaList incrementalSweptArenas; - - // Arena lists which have yet to be swept, but need additional foreground - // processing before they are swept. - Arena* gcShapeArenasToUpdate; - Arena* gcAccessorShapeArenasToUpdate; - Arena* gcScriptArenasToUpdate; - Arena* gcObjectGroupArenasToUpdate; - - // While sweeping type information, these lists save the arenas for the - // objects which have already been finalized in the foreground (which must - // happen at the beginning of the GC), so that type sweeping can determine - // which of the object pointers are marked. - ObjectAllocKindArray savedObjectArenas; - Arena* savedEmptyObjectArenas; - - public: - explicit ArenaLists(JSRuntime* rt) : runtime_(rt) { - for (auto i : AllAllocKinds()) - freeLists[i] = &placeholder; - for (auto i : AllAllocKinds()) - backgroundFinalizeState[i] = BFS_DONE; - for (auto i : AllAllocKinds()) - arenaListsToSweep[i] = nullptr; - incrementalSweptArenaKind = AllocKind::LIMIT; - gcShapeArenasToUpdate = nullptr; - gcAccessorShapeArenasToUpdate = nullptr; - gcScriptArenasToUpdate = nullptr; - gcObjectGroupArenasToUpdate = nullptr; - savedEmptyObjectArenas = nullptr; - } - - ~ArenaLists(); - - const void* addressOfFreeList(AllocKind thingKind) const { - return reinterpret_cast(&freeLists[thingKind]); - } - - Arena* getFirstArena(AllocKind thingKind) const { - return arenaLists[thingKind].head(); - } - - Arena* getFirstArenaToSweep(AllocKind thingKind) const { - return arenaListsToSweep[thingKind]; - } - - Arena* getFirstSweptArena(AllocKind thingKind) const { - if (thingKind != incrementalSweptArenaKind) - return nullptr; - return incrementalSweptArenas.head(); - } - - Arena* getArenaAfterCursor(AllocKind thingKind) const { - return arenaLists[thingKind].arenaAfterCursor(); - } - - bool arenaListsAreEmpty() const { - for (auto i : AllAllocKinds()) { - /* - * The arena cannot be empty if the background finalization is not yet - * done. - */ - if (backgroundFinalizeState[i] != BFS_DONE) - return false; - if (!arenaLists[i].isEmpty()) - return false; - } - return true; - } - - void unmarkAll() { - for (auto i : AllAllocKinds()) { - /* The background finalization must have stopped at this point. */ - MOZ_ASSERT(backgroundFinalizeState[i] == BFS_DONE); - for (Arena* arena = arenaLists[i].head(); arena; arena = arena->next) - arena->unmarkAll(); - } - } - - bool doneBackgroundFinalize(AllocKind kind) const { - return backgroundFinalizeState[kind] == BFS_DONE; - } - - bool needBackgroundFinalizeWait(AllocKind kind) const { - return backgroundFinalizeState[kind] != BFS_DONE; - } - - /* - * Clear the free lists so we won't try to allocate from swept arenas. - */ - void purge() { - for (auto i : AllAllocKinds()) - freeLists[i] = &placeholder; - } - - inline void prepareForIncrementalGC(); - - /* Check if this arena is in use. */ - bool arenaIsInUse(Arena* arena, AllocKind kind) const { - MOZ_ASSERT(arena); - return arena == freeLists[kind]->getArenaUnchecked(); - } - - MOZ_ALWAYS_INLINE TenuredCell* allocateFromFreeList(AllocKind thingKind, size_t thingSize) { - return freeLists[thingKind]->allocate(thingSize); - } - - /* - * Moves all arenas from |fromArenaLists| into |this|. - */ - void adoptArenas(JSRuntime* runtime, ArenaLists* fromArenaLists); - - /* True if the Arena in question is found in this ArenaLists */ - bool containsArena(JSRuntime* runtime, Arena* arena); - - void checkEmptyFreeLists() { -#ifdef DEBUG - for (auto i : AllAllocKinds()) - checkEmptyFreeList(i); -#endif - } - - bool checkEmptyArenaLists() { - bool empty = true; -#ifdef DEBUG - for (auto i : AllAllocKinds()) { - if (!checkEmptyArenaList(i)) - empty = false; - } -#endif - return empty; - } - - void checkEmptyFreeList(AllocKind kind) { - MOZ_ASSERT(freeLists[kind]->isEmpty()); - } - - bool checkEmptyArenaList(AllocKind kind); - - bool relocateArenas(Zone* zone, Arena*& relocatedListOut, JS::gcreason::Reason reason, - SliceBudget& sliceBudget, gcstats::Statistics& stats); - - void queueForegroundObjectsForSweep(FreeOp* fop); - void queueForegroundThingsForSweep(FreeOp* fop); - - void mergeForegroundSweptObjectArenas(); - - bool foregroundFinalize(FreeOp* fop, AllocKind thingKind, SliceBudget& sliceBudget, - SortedArenaList& sweepList); - static void backgroundFinalize(FreeOp* fop, Arena* listHead, Arena** empty); - - // When finalizing arenas, whether to keep empty arenas on the list or - // release them immediately. - enum KeepArenasEnum { - RELEASE_ARENAS, - KEEP_ARENAS - }; - - private: - inline void finalizeNow(FreeOp* fop, const FinalizePhase& phase); - inline void queueForForegroundSweep(FreeOp* fop, const FinalizePhase& phase); - inline void queueForBackgroundSweep(FreeOp* fop, const FinalizePhase& phase); - - inline void finalizeNow(FreeOp* fop, AllocKind thingKind, - KeepArenasEnum keepArenas, Arena** empty = nullptr); - inline void forceFinalizeNow(FreeOp* fop, AllocKind thingKind, - KeepArenasEnum keepArenas, Arena** empty = nullptr); - inline void queueForForegroundSweep(FreeOp* fop, AllocKind thingKind); - inline void queueForBackgroundSweep(FreeOp* fop, AllocKind thingKind); - inline void mergeSweptArenas(AllocKind thingKind); - - TenuredCell* allocateFromArena(JS::Zone* zone, AllocKind thingKind, - ShouldCheckThresholds checkThresholds); - inline TenuredCell* allocateFromArenaInner(JS::Zone* zone, Arena* arena, AllocKind kind); - - inline void normalizeBackgroundFinalizeState(AllocKind thingKind); - - friend class GCRuntime; - friend class js::Nursery; - friend class js::TenuringTracer; -}; - -/* The number of GC cycles an empty chunk can survive before been released. */ -const size_t MAX_EMPTY_CHUNK_AGE = 4; - -extern bool -InitializeStaticData(); - } /* namespace gc */ -class InterpreterFrame; - extern void MarkCompartmentActive(js::InterpreterFrame* fp); @@ -855,9 +62,6 @@ PrepareForDebugGC(JSRuntime* rt); /* Functions for managing cross compartment gray pointers. */ -extern void -DelayCrossCompartmentGrayMarking(JSObject* src); - extern void NotifyGCNukeWrapper(JSObject* o); @@ -867,186 +71,6 @@ NotifyGCPreSwap(JSObject* a, JSObject* b); extern void NotifyGCPostSwap(JSObject* a, JSObject* b, unsigned preResult); -/* - * Helper state for use when JS helper threads sweep and allocate GC thing kinds - * that can be swept and allocated off the main thread. - * - * In non-threadsafe builds, all actual sweeping and allocation is performed - * on the main thread, but GCHelperState encapsulates this from clients as - * much as possible. - */ -class GCHelperState -{ - enum State { - IDLE, - SWEEPING - }; - - // Associated runtime. - JSRuntime* const rt; - - // Condvar for notifying the main thread when work has finished. This is - // associated with the runtime's GC lock --- the worker thread state - // condvars can't be used here due to lock ordering issues. - js::ConditionVariable done; - - // Activity for the helper to do, protected by the GC lock. - State state_; - - // Thread which work is being performed on, if any. - mozilla::Maybe thread; - - void startBackgroundThread(State newState, const AutoLockGC& lock, - const AutoLockHelperThreadState& helperLock); - void waitForBackgroundThread(js::AutoLockGC& lock); - - State state(const AutoLockGC&); - void setState(State state, const AutoLockGC&); - - friend class js::gc::ArenaLists; - - static void freeElementsAndArray(void** array, void** end) { - MOZ_ASSERT(array <= end); - for (void** p = array; p != end; ++p) - js_free(*p); - js_free(array); - } - - void doSweep(AutoLockGC& lock); - - public: - explicit GCHelperState(JSRuntime* rt) - : rt(rt), - done(), - state_(IDLE) - { } - - void finish(); - - void work(); - - void maybeStartBackgroundSweep(const AutoLockGC& lock, - const AutoLockHelperThreadState& helperLock); - void startBackgroundShrink(const AutoLockGC& lock); - - /* Must be called without the GC lock taken. */ - void waitBackgroundSweepEnd(); - - bool onBackgroundThread(); - - /* - * Outside the GC lock may give true answer when in fact the sweeping has - * been done. - */ - bool isBackgroundSweeping() const { - return state_ == SWEEPING; - } -}; - -// A generic task used to dispatch work to the helper thread system. -// Users supply a function pointer to call. -// -// Note that we don't use virtual functions here because destructors can write -// the vtable pointer on entry, which can causes races if synchronization -// happens there. -class GCParallelTask -{ - public: - using TaskFunc = void (*)(GCParallelTask*); - - private: - TaskFunc func_; - - // The state of the parallel computation. - enum TaskState { - NotStarted, - Dispatched, - Finished, - } state; - - // Amount of time this task took to execute. - uint64_t duration_; - - explicit GCParallelTask(const GCParallelTask&) = delete; - - protected: - // A flag to signal a request for early completion of the off-thread task. - mozilla::Atomic cancel_; - - public: - explicit GCParallelTask(TaskFunc func) - : func_(func), - state(NotStarted), - duration_(0), - cancel_(false) - {} - - GCParallelTask(GCParallelTask&& other) - : func_(other.func_), - state(other.state), - duration_(0), - cancel_(false) - {} - - // Derived classes must override this to ensure that join() gets called - // before members get destructed. - ~GCParallelTask(); - - // Time spent in the most recent invocation of this task. - int64_t duration() const { return duration_; } - - // The simple interface to a parallel task works exactly like pthreads. - bool start(); - void join(); - - // If multiple tasks are to be started or joined at once, it is more - // efficient to take the helper thread lock once and use these methods. - bool startWithLockHeld(AutoLockHelperThreadState& locked); - void joinWithLockHeld(AutoLockHelperThreadState& locked); - - // Instead of dispatching to a helper, run the task on the main thread. - void runFromMainThread(JSRuntime* rt); - - // Dispatch a cancelation request. - enum CancelMode { CancelNoWait, CancelAndWait}; - void cancel(CancelMode mode = CancelNoWait) { - cancel_ = true; - if (mode == CancelAndWait) - join(); - } - - // Check if a task is actively running. - bool isRunningWithLockHeld(const AutoLockHelperThreadState& locked) const; - bool isRunning() const; - - void runTask() { - func_(this); - } - - // This should be friended to HelperThread, but cannot be because it - // would introduce several circular dependencies. - public: - void runFromHelperThread(AutoLockHelperThreadState& locked); -}; - -// CRTP template to handle cast to derived type when calling run(). -template -class GCParallelTaskHelper : public GCParallelTask -{ - public: - GCParallelTaskHelper() - : GCParallelTask(&runTaskTyped) - {} - GCParallelTaskHelper(GCParallelTaskHelper&& other) - : GCParallelTask(mozilla::Move(other)) - {} - - private: - static void runTaskTyped(GCParallelTask* task) { - static_cast(task)->run(); - } -}; - typedef void (*IterateChunkCallback)(JSRuntime* rt, void* data, gc::Chunk* chunk); typedef void (*IterateZoneCallback)(JSRuntime* rt, void* data, JS::Zone* zone); typedef void (*IterateArenaCallback)(JSRuntime* rt, void* data, gc::Arena* arena, @@ -1109,125 +133,32 @@ namespace gc { void MergeCompartments(JSCompartment* source, JSCompartment* target); -/* - * This structure overlays a Cell in the Nursery and re-purposes its memory - * for managing the Nursery collection process. - */ -class RelocationOverlay -{ - /* The low bit is set so this should never equal a normal pointer. */ - static const uintptr_t Relocated = uintptr_t(0xbad0bad1); - - /* Set to Relocated when moved. */ - uintptr_t magic_; - - /* The location |this| was moved to. */ - Cell* newLocation_; - - /* A list entry to track all relocated things. */ - RelocationOverlay* next_; - - public: - static RelocationOverlay* fromCell(Cell* cell) { - return reinterpret_cast(cell); - } - - bool isForwarded() const { - return magic_ == Relocated; - } - - Cell* forwardingAddress() const { - MOZ_ASSERT(isForwarded()); - return newLocation_; - } - - void forwardTo(Cell* cell); - - RelocationOverlay*& nextRef() { - MOZ_ASSERT(isForwarded()); - return next_; - } - - RelocationOverlay* next() const { - MOZ_ASSERT(isForwarded()); - return next_; - } - - static bool isCellForwarded(Cell* cell) { - return fromCell(cell)->isForwarded(); - } -}; - -// Functions for checking and updating GC thing pointers that might have been -// moved by compacting GC. Overloads are also provided that work with Values. -// -// IsForwarded - check whether a pointer refers to an GC thing that has been -// moved. -// -// Forwarded - return a pointer to the new location of a GC thing given a -// pointer to old location. -// -// MaybeForwarded - used before dereferencing a pointer that may refer to a -// moved GC thing without updating it. For JSObjects this will -// also update the object's shape pointer if it has been moved -// to allow slots to be accessed. - -template -inline bool IsForwarded(T* t); -inline bool IsForwarded(const JS::Value& value); - -template -inline T* Forwarded(T* t); - -inline Value Forwarded(const JS::Value& value); - -template -inline T MaybeForwarded(T t); - -#ifdef JSGC_HASH_TABLE_CHECKS - -template -inline bool IsGCThingValidAfterMovingGC(T* t); - -template -inline void CheckGCThingAfterMovingGC(T* t); - -template -inline void CheckGCThingAfterMovingGC(const ReadBarriered& t); - -inline void CheckValueAfterMovingGC(const JS::Value& value); - -#endif // JSGC_HASH_TABLE_CHECKS - -#define JS_FOR_EACH_ZEAL_MODE(D) \ - D(Poke, 1) \ - D(Alloc, 2) \ - D(FrameGC, 3) \ - D(VerifierPre, 4) \ - D(FrameVerifierPre, 5) \ - D(StackRooting, 6) \ - D(GenerationalGC, 7) \ - D(IncrementalRootsThenFinish, 8) \ - D(IncrementalMarkAllThenFinish, 9) \ - D(IncrementalMultipleSlices, 10) \ - D(IncrementalMarkingValidator, 11) \ - D(ElementsBarrier, 12) \ - D(CheckHashTablesOnMinorGC, 13) \ - D(Compact, 14) \ - D(CheckHeapAfterGC, 15) \ - D(CheckNursery, 16) - -enum class ZealMode { -#define ZEAL_MODE(name, value) name = value, - JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE) -#undef ZEAL_MODE - Limit = 16 -}; - enum VerifierType { PreBarrierVerifier }; +#ifdef JS_GC_ZEAL + +extern const char* ZealModeHelpText; + +/* Check that write barriers have been used correctly. See jsgc.cpp. */ +void +VerifyBarriers(JSRuntime* rt, VerifierType type); + +void +MaybeVerifyBarriers(JSContext* cx, bool always = false); + +void DumpArenaInfo(); + +#else + +static inline void +VerifyBarriers(JSRuntime* rt, VerifierType type) {} + +static inline void +MaybeVerifyBarriers(JSContext* cx, bool always = false) {} + +#endif /* * Instances of this class set the |JSRuntime::suppressGC| flag for the duration * that they are live. Use of this class is highly discouraged. Please carefully @@ -1249,153 +180,14 @@ class MOZ_RAII JS_HAZ_GC_SUPPRESSED AutoSuppressGC } }; -// A singly linked list of zones. -class ZoneList -{ - static Zone * const End; - - Zone* head; - Zone* tail; - - public: - ZoneList(); - ~ZoneList(); - - bool isEmpty() const; - Zone* front() const; - - void append(Zone* zone); - void transferFrom(ZoneList& other); - void removeFront(); - void clear(); - - private: - explicit ZoneList(Zone* singleZone); - void check() const; - - ZoneList(const ZoneList& other) = delete; - ZoneList& operator=(const ZoneList& other) = delete; -}; - #ifdef MOZ_DEVTOOLS_SERVER JSObject* NewMemoryStatisticsObject(JSContext* cx); #endif -struct MOZ_RAII AutoAssertNoNurseryAlloc -{ -#ifdef DEBUG - explicit AutoAssertNoNurseryAlloc(JSRuntime* rt); - ~AutoAssertNoNurseryAlloc(); - - private: - gc::GCRuntime& gc; -#else - explicit AutoAssertNoNurseryAlloc(JSRuntime* rt) {} -#endif -}; - -/* - * There are a couple of classes here that serve mostly as "tokens" indicating - * that a condition holds. Some functions force the caller to possess such a - * token because they would misbehave if the condition were false, and it is - * far more clear to make the condition visible at the point where it can be - * affected rather than just crashing in an assertion down in the place where - * it is relied upon. - */ - -/* - * Token meaning that the heap is busy and no allocations will be made. - * - * This class may be instantiated directly if it is known that the condition is - * already true, or it can be used as a base class for another RAII class that - * causes the condition to become true. Such base classes will use the no-arg - * constructor, establish the condition, then call checkCondition() to assert - * it and possibly record data needed to re-check the condition during - * destruction. - * - * Ordinarily, you would do something like this with a Maybe<> member that is - * emplaced during the constructor, but token-requiring functions want to - * require a reference to a base class instance. That said, you can always pass - * in the Maybe<> field as the token. - */ -class MOZ_RAII AutoAssertHeapBusy { - protected: - JSRuntime* rt; - - // Check that the heap really is busy, and record the rt for the check in - // the destructor. - void checkCondition(JSRuntime *rt); - - AutoAssertHeapBusy() : rt(nullptr) { - } - - public: - explicit AutoAssertHeapBusy(JSRuntime* rt) { - checkCondition(rt); - } - - ~AutoAssertHeapBusy() { - MOZ_ASSERT(rt); // checkCondition must always be called. - checkCondition(rt); - } -}; - -/* - * A class that serves as a token that the nursery is empty. It descends from - * AutoAssertHeapBusy, which means that it additionally requires the heap to be - * busy (which is not necessarily linked, but turns out to be true in practice - * for all users and simplifies the usage of these classes.) - */ -class MOZ_RAII AutoAssertEmptyNursery -{ - protected: - JSRuntime* rt; - - mozilla::Maybe noAlloc; - - // Check that the nursery is empty. - void checkCondition(JSRuntime *rt); - - // For subclasses that need to empty the nursery in their constructors. - AutoAssertEmptyNursery() : rt(nullptr) { - } - - public: - explicit AutoAssertEmptyNursery(JSRuntime* rt) : rt(nullptr) { - checkCondition(rt); - } - - AutoAssertEmptyNursery(const AutoAssertEmptyNursery& other) : AutoAssertEmptyNursery(other.rt) - { - } -}; - -/* - * Evict the nursery upon construction. Serves as a token indicating that the - * nursery is empty. (See AutoAssertEmptyNursery, above.) - * - * Note that this is very improper subclass of AutoAssertHeapBusy, in that the - * heap is *not* busy within the scope of an AutoEmptyNursery. I will most - * likely fix this by removing AutoAssertHeapBusy, but that is currently - * waiting on jonco's review. - */ -class MOZ_RAII AutoEmptyNursery : public AutoAssertEmptyNursery -{ - public: - explicit AutoEmptyNursery(JSRuntime *rt); -}; - const char* StateName(State state); -inline bool -IsOOMReason(JS::gcreason::Reason reason) -{ - return reason == JS::gcreason::LAST_DITCH || - reason == JS::gcreason::MEM_PRESSURE; -} - } /* namespace gc */ #ifdef DEBUG diff --git a/js/src/jsgcinlines.h b/js/src/jsgcinlines.h index dfea46278b..af29a9d5cb 100644 --- a/js/src/jsgcinlines.h +++ b/js/src/jsgcinlines.h @@ -6,36 +6,18 @@ #ifndef jsgcinlines_h #define jsgcinlines_h -#include "jsgc.h" - #include "mozilla/DebugOnly.h" #include "mozilla/Maybe.h" #include "gc/GCTrace.h" #include "gc/Zone.h" +#include "gc/ArenaList-inl.h" + namespace js { namespace gc { -inline void -MakeAccessibleAfterMovingGC(void* anyp) {} - -inline void -MakeAccessibleAfterMovingGC(JSObject* obj) { - if (obj->isNative()) - obj->as().updateShapeAfterMovingGC(); -} - -static inline AllocKind -GetGCObjectKind(const Class* clasp) -{ - if (clasp == FunctionClassPtr) - return AllocKind::FUNCTION; - uint32_t nslots = JSCLASS_RESERVED_SLOTS(clasp); - if (clasp->flags & JSCLASS_HAS_PRIVATE) - nslots++; - return GetGCObjectKind(nslots); -} +class AutoAssertEmptyNursery; inline void GCRuntime::poke() @@ -216,26 +198,6 @@ class ArenaCellIter : public ArenaCellIterImpl } }; -class ArenaCellIterUnderGC : public ArenaCellIterImpl -{ - public: - explicit ArenaCellIterUnderGC(Arena* arena) - : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier) - { - MOZ_ASSERT(CurrentThreadIsPerformingGC()); - } -}; - -class ArenaCellIterUnderFinalize : public ArenaCellIterImpl -{ - public: - explicit ArenaCellIterUnderFinalize(Arena* arena) - : ArenaCellIterImpl(arena, CellIterDoesntNeedBarrier) - { - MOZ_ASSERT(CurrentThreadIsGCSweeping()); - } -}; - template class ZoneCellIter; @@ -391,199 +353,6 @@ class ZoneCellIter : public ZoneCellIter { GCType* operator ->() const { return get(); } }; -class GrayObjectIter : public ZoneCellIter { - public: - explicit GrayObjectIter(JS::Zone* zone, AllocKind kind) : ZoneCellIter() { - initForTenuredIteration(zone, kind); - } - - JSObject* get() const { return ZoneCellIter::get(); } - operator JSObject*() const { return get(); } - JSObject* operator ->() const { return get(); } -}; - -class GCZonesIter -{ - private: - ZonesIter zone; - - public: - explicit GCZonesIter(JSRuntime* rt, ZoneSelector selector = WithAtoms) : zone(rt, selector) { - MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt) && rt->isHeapBusy()); - if (!zone->isCollecting()) - next(); - } - - bool done() const { return zone.done(); } - - void next() { - MOZ_ASSERT(!done()); - do { - zone.next(); - } while (!zone.done() && !zone->isCollectingFromAnyThread()); - } - - JS::Zone* get() const { - MOZ_ASSERT(!done()); - return zone; - } - - operator JS::Zone*() const { return get(); } - JS::Zone* operator->() const { return get(); } -}; - -typedef CompartmentsIterT GCCompartmentsIter; - -/* Iterates over all zones in the current sweep group. */ -class GCSweepGroupIter { - JS::Zone* current; - - public: - explicit GCSweepGroupIter(JSRuntime* rt) { - MOZ_ASSERT(CurrentThreadIsPerformingGC()); - current = rt->gc.getCurrentSweepGroup(); - } - - bool done() const { return !current; } - - void next() { - MOZ_ASSERT(!done()); - current = current->nextNodeInGroup(); - } - - JS::Zone* get() const { - MOZ_ASSERT(!done()); - return current; - } - - operator JS::Zone*() const { return get(); } - JS::Zone* operator->() const { return get(); } -}; - -typedef CompartmentsIterT GCCompartmentGroupIter; - -inline void -RelocationOverlay::forwardTo(Cell* cell) -{ - MOZ_ASSERT(!isForwarded()); - // The location of magic_ is important because it must never be valid to see - // the value Relocated there in a GC thing that has not been moved. - static_assert(offsetof(RelocationOverlay, magic_) == offsetof(JSObject, group_) && - offsetof(RelocationOverlay, magic_) == offsetof(js::Shape, base_) && - offsetof(RelocationOverlay, magic_) == offsetof(JSString, d.u1.flags), - "RelocationOverlay::magic_ is in the wrong location"); - magic_ = Relocated; - newLocation_ = cell; -} - -template -struct MightBeForwarded -{ - static_assert(mozilla::IsBaseOf::value, - "T must derive from Cell"); - static_assert(!mozilla::IsSame::value && !mozilla::IsSame::value, - "T must not be Cell or TenuredCell"); - - static const bool value = mozilla::IsBaseOf::value || - mozilla::IsBaseOf::value || - mozilla::IsBaseOf::value || - mozilla::IsBaseOf::value || - mozilla::IsBaseOf::value || - mozilla::IsBaseOf::value || - mozilla::IsBaseOf::value || - mozilla::IsBaseOf::value; -}; - -template -inline bool -IsForwarded(T* t) -{ - RelocationOverlay* overlay = RelocationOverlay::fromCell(t); - if (!MightBeForwarded::value) { - MOZ_ASSERT(!overlay->isForwarded()); - return false; - } - - return overlay->isForwarded(); -} - -struct IsForwardedFunctor : public BoolDefaultAdaptor { - template bool operator()(T* t) { return IsForwarded(t); } -}; - -inline bool -IsForwarded(const JS::Value& value) -{ - return DispatchTyped(IsForwardedFunctor(), value); -} - -template -inline T* -Forwarded(T* t) -{ - RelocationOverlay* overlay = RelocationOverlay::fromCell(t); - MOZ_ASSERT(overlay->isForwarded()); - return reinterpret_cast(overlay->forwardingAddress()); -} - -struct ForwardedFunctor : public IdentityDefaultAdaptor { - template inline Value operator()(T* t) { - return js::gc::RewrapTaggedPointer::wrap(Forwarded(t)); - } -}; - -inline Value -Forwarded(const JS::Value& value) -{ - return DispatchTyped(ForwardedFunctor(), value); -} - -template -inline T -MaybeForwarded(T t) -{ - if (IsForwarded(t)) - t = Forwarded(t); - MakeAccessibleAfterMovingGC(t); - return t; -} - -#ifdef JSGC_HASH_TABLE_CHECKS - -template -inline bool -IsGCThingValidAfterMovingGC(T* t) -{ - return !IsInsideNursery(t) && !RelocationOverlay::isCellForwarded(t); -} - -template -inline void -CheckGCThingAfterMovingGC(T* t) -{ - if (t) - MOZ_RELEASE_ASSERT(IsGCThingValidAfterMovingGC(t)); -} - -template -inline void -CheckGCThingAfterMovingGC(const ReadBarriered& t) -{ - CheckGCThingAfterMovingGC(t.unbarrieredGet()); -} - -struct CheckValueAfterMovingGCFunctor : public VoidDefaultAdaptor { - template void operator()(T* t) { CheckGCThingAfterMovingGC(t); } -}; - -inline void -CheckValueAfterMovingGC(const JS::Value& value) -{ - DispatchTyped(CheckValueAfterMovingGCFunctor(), value); -} - -#endif // JSGC_HASH_TABLE_CHECKS - } /* namespace gc */ } /* namespace js */ diff --git a/js/src/jsobj.cpp b/js/src/jsobj.cpp index be0633ac9f..bfea788df9 100644 --- a/js/src/jsobj.cpp +++ b/js/src/jsobj.cpp @@ -40,7 +40,6 @@ #include "builtin/Object.h" #include "builtin/SymbolObject.h" #include "frontend/BytecodeCompiler.h" -#include "gc/Marking.h" #include "gc/Policy.h" #include "jit/BaselineJIT.h" #include "js/MemoryMetrics.h" @@ -59,6 +58,8 @@ #include "jscntxtinlines.h" #include "jscompartmentinlines.h" +#include "gc/Marking-inl.h" +#include "builtin/TypedObject-inl.h" #include "vm/ArrayObject-inl.h" #include "vm/BooleanObject-inl.h" #include "vm/Caches-inl.h" @@ -67,6 +68,8 @@ #include "vm/NumberObject-inl.h" #include "vm/Shape-inl.h" #include "vm/StringObject-inl.h" +#include "vm/TypedArrayObject-inl.h" +#include "vm/UnboxedObject-inl.h" using namespace js; using namespace js::gc; diff --git a/js/src/jsobj.h b/js/src/jsobj.h index d41598635b..c470633524 100644 --- a/js/src/jsobj.h +++ b/js/src/jsobj.h @@ -20,7 +20,6 @@ #include "gc/Barrier.h" #include "gc/Marking.h" #include "js/Conversions.h" -#include "js/GCAPI.h" #include "js/GCVector.h" #include "js/HeapAPI.h" #include "vm/Shape.h" diff --git a/js/src/jsobjinlines.h b/js/src/jsobjinlines.h index a27a13fd6c..b3493f14b7 100644 --- a/js/src/jsobjinlines.h +++ b/js/src/jsobjinlines.h @@ -26,8 +26,9 @@ #include "jsatominlines.h" #include "jscompartmentinlines.h" -#include "jsgcinlines.h" +#include "gc/Marking-inl.h" +#include "gc/ObjectKind-inl.h" #include "vm/ShapedObject-inl.h" #include "vm/TypeInference-inl.h" @@ -119,6 +120,24 @@ js::NativeObject::sweepDictionaryListPointer() shape_->listp = nullptr; } +MOZ_ALWAYS_INLINE void +js::NativeObject::updateDictionaryListPointerAfterMinorGC(NativeObject* old) +{ + MOZ_ASSERT(this == Forwarded(old)); + + // Dictionary objects can be allocated in the nursery and when they are + // tenured the shape's pointer into the object needs to be updated. + if (shape_->listp == &old->shape_) + shape_->listp = &shape_; +} + +inline void +js::gc::MakeAccessibleAfterMovingGC(JSObject* obj) +{ + if (obj->isNative()) + obj->as().updateShapeAfterMovingGC(); +} + /* static */ inline bool JSObject::setSingleton(js::ExclusiveContext* cx, js::HandleObject obj) { diff --git a/js/src/jsopcode.cpp b/js/src/jsopcode.cpp index c9ec240e7f..99d9b0c72f 100644 --- a/js/src/jsopcode.cpp +++ b/js/src/jsopcode.cpp @@ -49,6 +49,8 @@ #include "jsobjinlines.h" #include "jsscriptinlines.h" +#include "gc/Iteration-inl.h" + using namespace js; using namespace js::gc; diff --git a/js/src/jspubtd.h b/js/src/jspubtd.h index d956ae4bd9..8d08c3c54c 100644 --- a/js/src/jspubtd.h +++ b/js/src/jspubtd.h @@ -115,13 +115,6 @@ template struct JSConstScalarSpec; typedef JSConstScalarSpec JSConstDoubleSpec; typedef JSConstScalarSpec JSConstIntegerSpec; -/* - * Generic trace operation that calls JS::TraceEdge on each traceable thing's - * location reachable from data. - */ -typedef void -(* JSTraceDataOp)(JSTracer* trc, void* data); - namespace js { namespace gc { class AutoTraceSession; diff --git a/js/src/jsscript.cpp b/js/src/jsscript.cpp index 857c4170a1..5a0e39d018 100644 --- a/js/src/jsscript.cpp +++ b/js/src/jsscript.cpp @@ -36,7 +36,6 @@ #include "frontend/BytecodeCompiler.h" #include "frontend/BytecodeEmitter.h" #include "frontend/SharedContext.h" -#include "gc/Marking.h" #include "jit/BaselineJIT.h" #include "jit/Ion.h" #include "jit/IonCode.h" @@ -55,6 +54,7 @@ #include "jsfuninlines.h" #include "jsobjinlines.h" +#include "gc/Marking-inl.h" #include "vm/EnvironmentObject-inl.h" #include "vm/NativeObject-inl.h" #include "vm/SharedImmutableStringsCache-inl.h" diff --git a/js/src/jsweakmap.cpp b/js/src/jsweakmap.cpp index 2fdc0448b7..14ac3549bc 100644 --- a/js/src/jsweakmap.cpp +++ b/js/src/jsweakmap.cpp @@ -13,7 +13,6 @@ #include "jsobj.h" #include "jswrapper.h" -#include "js/GCAPI.h" #include "vm/GlobalObject.h" #include "jsobjinlines.h" diff --git a/js/src/proxy/Proxy.cpp b/js/src/proxy/Proxy.cpp index 984e1f411d..6cdbd207e9 100644 --- a/js/src/proxy/Proxy.cpp +++ b/js/src/proxy/Proxy.cpp @@ -13,7 +13,6 @@ #include "jsgc.h" #include "jswrapper.h" -#include "gc/Marking.h" #include "proxy/DeadObjectProxy.h" #include "proxy/ScriptedProxyHandler.h" #include "vm/WrapperObject.h" @@ -21,6 +20,7 @@ #include "jsatominlines.h" #include "jsobjinlines.h" +#include "gc/Marking-inl.h" #include "vm/NativeObject-inl.h" using namespace js; diff --git a/js/src/proxy/Wrapper.cpp b/js/src/proxy/Wrapper.cpp index 7de6d1f623..0121b4f496 100644 --- a/js/src/proxy/Wrapper.cpp +++ b/js/src/proxy/Wrapper.cpp @@ -15,6 +15,7 @@ #include "jsobjinlines.h" +#include "gc/Marking-inl.h" #include "vm/NativeObject-inl.h" using namespace js; diff --git a/js/src/shell/js.cpp b/js/src/shell/js.cpp index 7b63a78ef1..6b260e94b7 100644 --- a/js/src/shell/js.cpp +++ b/js/src/shell/js.cpp @@ -73,7 +73,6 @@ #include "js/CompileOptions.h" #include "js/Debug.h" #include "js/Equality.h" // JS::SameValue -#include "js/GCAPI.h" #include "js/Initialization.h" #include "js/SourceBufferHolder.h" #include "js/StructuredClone.h" diff --git a/js/src/vm/ArrayBufferObject.cpp b/js/src/vm/ArrayBufferObject.cpp index da51abec63..7bb44c20f6 100644 --- a/js/src/vm/ArrayBufferObject.cpp +++ b/js/src/vm/ArrayBufferObject.cpp @@ -37,7 +37,6 @@ #include "jswrapper.h" #include "gc/Barrier.h" -#include "gc/Marking.h" #include "gc/Memory.h" #include "js/Conversions.h" #include "js/MemoryMetrics.h" @@ -51,6 +50,8 @@ #include "jsatominlines.h" +#include "gc/Marking-inl.h" +#include "gc/Nursery-inl.h" #include "vm/NativeObject-inl.h" #include "vm/Shape-inl.h" diff --git a/js/src/vm/ArrayObject-inl.h b/js/src/vm/ArrayObject-inl.h index e1ece2a172..cab239d1a4 100644 --- a/js/src/vm/ArrayObject-inl.h +++ b/js/src/vm/ArrayObject-inl.h @@ -11,7 +11,6 @@ #include "gc/GCTrace.h" #include "vm/String.h" -#include "jsgcinlines.h" #include "jsobjinlines.h" #include "vm/TypeInference-inl.h" diff --git a/js/src/vm/Debugger.cpp b/js/src/vm/Debugger.cpp index 7a421b567d..dd5e02eb81 100644 --- a/js/src/vm/Debugger.cpp +++ b/js/src/vm/Debugger.cpp @@ -26,7 +26,6 @@ #include "jit/BaselineDebugModeOSR.h" #include "jit/BaselineJIT.h" #include "js/Date.h" -#include "js/GCAPI.h" #include "js/SourceBufferHolder.h" #include "js/UbiNodeBreadthFirst.h" #include "js/Vector.h" diff --git a/js/src/vm/EnvironmentObject.cpp b/js/src/vm/EnvironmentObject.cpp index 0ec095b70a..8fed188a63 100644 --- a/js/src/vm/EnvironmentObject.cpp +++ b/js/src/vm/EnvironmentObject.cpp @@ -24,7 +24,6 @@ #include "wasm/WasmInstance.h" #include "jsatominlines.h" -#include "jsobjinlines.h" #include "jsscriptinlines.h" #include "vm/Stack-inl.h" diff --git a/js/src/vm/HelperThreads.cpp b/js/src/vm/HelperThreads.cpp index b8329160e6..d41372d08f 100644 --- a/js/src/vm/HelperThreads.cpp +++ b/js/src/vm/HelperThreads.cpp @@ -27,6 +27,9 @@ #include "jsobjinlines.h" #include "jsscriptinlines.h" +#include "gc/Iteration-inl.h" +#include "vm/NativeObject-inl.h" + using namespace js; using mozilla::ArrayLength; diff --git a/js/src/vm/NativeObject-inl.h b/js/src/vm/NativeObject-inl.h index bb93ff43da..7a229b01bf 100644 --- a/js/src/vm/NativeObject-inl.h +++ b/js/src/vm/NativeObject-inl.h @@ -11,12 +11,17 @@ #include "jscntxt.h" #include "builtin/TypedObject.h" +#include "gc/GCTrace.h" #include "proxy/Proxy.h" #include "vm/ProxyObject.h" #include "vm/TypedArrayObject.h" #include "jsobjinlines.h" +#include "gc/Heap-inl.h" +#include "gc/Marking-inl.h" +#include "gc/ObjectKind-inl.h" + namespace js { inline uint8_t* diff --git a/js/src/vm/ObjectGroup.cpp b/js/src/vm/ObjectGroup.cpp index 56039661bd..11bb490653 100644 --- a/js/src/vm/ObjectGroup.cpp +++ b/js/src/vm/ObjectGroup.cpp @@ -9,7 +9,6 @@ #include "jshashutil.h" #include "jsobj.h" -#include "gc/Marking.h" #include "gc/Policy.h" #include "gc/StoreBuffer.h" #include "gc/Zone.h" @@ -17,10 +16,8 @@ #include "vm/ArrayObject.h" #include "vm/Shape.h" #include "vm/TaggedProto.h" -#include "vm/UnboxedObject.h" - -#include "jsobjinlines.h" +#include "gc/Marking-inl.h" #include "vm/UnboxedObject-inl.h" using namespace js; diff --git a/js/src/vm/ProxyObject.cpp b/js/src/vm/ProxyObject.cpp index f241f16faf..08159045e3 100644 --- a/js/src/vm/ProxyObject.cpp +++ b/js/src/vm/ProxyObject.cpp @@ -7,11 +7,17 @@ #include "jscompartment.h" +#include "gc/Allocator.h" +#include "gc/GCTrace.h" + #include "proxy/DeadObjectProxy.h" #include "proxy/ScriptedProxyHandler.h" #include "jsobjinlines.h" +#include "gc/ObjectKind-inl.h" +#include "vm/TypeInference-inl.h" + using namespace js; /* static */ ProxyObject* diff --git a/js/src/vm/SPSProfiler.cpp b/js/src/vm/SPSProfiler.cpp index 2ee9241a1e..c043247066 100644 --- a/js/src/vm/SPSProfiler.cpp +++ b/js/src/vm/SPSProfiler.cpp @@ -18,7 +18,7 @@ #include "jit/JitFrames.h" #include "vm/StringBuffer.h" -#include "jsgcinlines.h" +#include "gc/Marking-inl.h" using namespace js; diff --git a/js/src/vm/Scope.cpp b/js/src/vm/Scope.cpp index bad4b474c5..6e68718e66 100644 --- a/js/src/vm/Scope.cpp +++ b/js/src/vm/Scope.cpp @@ -13,6 +13,7 @@ #include "vm/EnvironmentObject.h" #include "vm/Runtime.h" +#include "gc/ObjectKind-inl.h" #include "vm/Shape-inl.h" using namespace js; diff --git a/js/src/vm/SelfHosting.cpp b/js/src/vm/SelfHosting.cpp index 26fd6cc9e1..35f2892e50 100644 --- a/js/src/vm/SelfHosting.cpp +++ b/js/src/vm/SelfHosting.cpp @@ -60,6 +60,7 @@ #include "jsobjinlines.h" #include "jsscriptinlines.h" +#include "gc/Iteration-inl.h" #include "vm/BooleanObject-inl.h" #include "vm/NativeObject-inl.h" #include "vm/NumberObject-inl.h" diff --git a/js/src/vm/Shape-inl.h b/js/src/vm/Shape-inl.h index 34ac3b3d66..8a0be71149 100644 --- a/js/src/vm/Shape-inl.h +++ b/js/src/vm/Shape-inl.h @@ -18,7 +18,7 @@ #include "jsatominlines.h" #include "jscntxtinlines.h" -#include "jsgcinlines.h" +#include "gc/Marking-inl.h" namespace js { diff --git a/js/src/vm/Stack.cpp b/js/src/vm/Stack.cpp index 5934c68651..ab5426f6dc 100644 --- a/js/src/vm/Stack.cpp +++ b/js/src/vm/Stack.cpp @@ -13,7 +13,6 @@ #include "jit/BaselineFrame.h" #include "jit/JitcodeMap.h" #include "jit/JitCompartment.h" -#include "js/GCAPI.h" #include "vm/Debugger.h" #include "vm/Opcodes.h" #include "wasm/WasmDebugFrame.h" diff --git a/js/src/vm/String.h b/js/src/vm/String.h index 5eaf9e0c2e..0c763cf14a 100644 --- a/js/src/vm/String.h +++ b/js/src/vm/String.h @@ -18,7 +18,6 @@ #include "gc/Marking.h" #include "gc/Rooting.h" #include "js/CharacterEncoding.h" -#include "js/GCAPI.h" #include "js/RootingAPI.h" class JSDependentString; diff --git a/js/src/vm/TypeInference.cpp b/js/src/vm/TypeInference.cpp index 589fd888f1..6fd5188d03 100644 --- a/js/src/vm/TypeInference.cpp +++ b/js/src/vm/TypeInference.cpp @@ -22,7 +22,6 @@ #include "jsscript.h" #include "jsstr.h" -#include "gc/Marking.h" #include "jit/BaselineJIT.h" #include "jit/CompileInfo.h" #include "jit/Ion.h" @@ -39,6 +38,8 @@ #include "jsatominlines.h" #include "jsscriptinlines.h" +#include "gc/Iteration-inl.h" +#include "gc/Marking-inl.h" #include "vm/NativeObject-inl.h" using namespace js; diff --git a/js/src/vm/TypedArrayObject-inl.h b/js/src/vm/TypedArrayObject-inl.h new file mode 100644 index 0000000000..299d5975a8 --- /dev/null +++ b/js/src/vm/TypedArrayObject-inl.h @@ -0,0 +1,761 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef vm_TypedArrayObject_inl_h +#define vm_TypedArrayObject_inl_h + +/* Utilities and common inline code for TypedArray */ + +#include "vm/TypedArrayObject.h" + +#include "mozilla/Assertions.h" +#include "mozilla/FloatingPoint.h" + +#include + +#include "jsarray.h" +#include "jscntxt.h" +#include "jsnum.h" + +#include "gc/Zone.h" + +#include "jit/AtomicOperations.h" + +#include "js/Conversions.h" +#include "js/Value.h" + +#include "vm/NativeObject.h" + +#include "gc/ObjectKind-inl.h" + +namespace js { + +// ValueIsLength happens not to be according to ES6, which mandates +// the use of ToLength, which in turn includes ToNumber, ToInteger, +// and clamping. ValueIsLength is used in the current TypedArray code +// but will disappear when that code is made spec-compliant. + +inline bool +ValueIsLength(const Value& v, uint32_t* len) +{ + if (v.isInt32()) { + int32_t i = v.toInt32(); + if (i < 0) + return false; + *len = i; + return true; + } + + if (v.isDouble()) { + double d = v.toDouble(); + if (mozilla::IsNaN(d)) + return false; + + uint32_t length = uint32_t(d); + if (d != double(length)) + return false; + + *len = length; + return true; + } + + return false; +} + +template +inline To +ConvertNumber(From src); + +template<> +inline int8_t +ConvertNumber(float src) +{ + return JS::ToInt8(src); +} + +template<> +inline uint8_t +ConvertNumber(float src) +{ + return JS::ToUint8(src); +} + +template<> +inline uint8_clamped +ConvertNumber(float src) +{ + return uint8_clamped(src); +} + +template<> +inline int16_t +ConvertNumber(float src) +{ + return JS::ToInt16(src); +} + +template<> +inline uint16_t +ConvertNumber(float src) +{ + return JS::ToUint16(src); +} + +template<> +inline int32_t +ConvertNumber(float src) +{ + return JS::ToInt32(src); +} + +template<> +inline uint32_t +ConvertNumber(float src) +{ + return JS::ToUint32(src); +} + +template <> +inline int64_t +ConvertNumber(float src) +{ + return JS::ToInt64(src); +} + +template <> +inline uint64_t +ConvertNumber(float src) +{ + return JS::ToUint64(src); +} + +template<> inline int8_t +ConvertNumber(double src) +{ + return JS::ToInt8(src); +} + +template<> +inline uint8_t +ConvertNumber(double src) +{ + return JS::ToUint8(src); +} + +template<> +inline uint8_clamped +ConvertNumber(double src) +{ + return uint8_clamped(src); +} + +template<> +inline int16_t +ConvertNumber(double src) +{ + return JS::ToInt16(src); +} + +template<> +inline uint16_t +ConvertNumber(double src) +{ + return JS::ToUint16(src); +} + +template<> +inline int32_t +ConvertNumber(double src) +{ + return JS::ToInt32(src); +} + +template<> +inline uint32_t +ConvertNumber(double src) +{ + return JS::ToUint32(src); +} + +template <> +inline int64_t +ConvertNumber(double src) +{ + return JS::ToInt64(src); +} + +template <> +inline uint64_t +ConvertNumber(double src) +{ + return JS::ToUint64(src); +} + +template +inline To +ConvertNumber(From src) +{ + static_assert(!mozilla::IsFloatingPoint::value || + (mozilla::IsFloatingPoint::value && mozilla::IsFloatingPoint::value), + "conversion from floating point to int should have been handled by " + "specializations above"); + return To(src); +} + +template struct TypeIDOfType; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Int8; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Uint8; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Int16; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Uint16; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Int32; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Uint32; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::BigInt64; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::BigUint64; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Float32; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Float64; }; +template<> struct TypeIDOfType { static const Scalar::Type id = Scalar::Uint8Clamped; }; + +class SharedOps +{ + public: + template + static T load(SharedMem addr) { + return js::jit::AtomicOperations::loadSafeWhenRacy(addr); + } + + template + static void store(SharedMem addr, T value) { + js::jit::AtomicOperations::storeSafeWhenRacy(addr, value); + } + + template + static void memcpy(SharedMem dest, SharedMem src, size_t size) { + js::jit::AtomicOperations::memcpySafeWhenRacy(dest, src, size); + } + + template + static void memmove(SharedMem dest, SharedMem src, size_t size) { + js::jit::AtomicOperations::memmoveSafeWhenRacy(dest, src, size); + } + + template + static void podCopy(SharedMem dest, SharedMem src, size_t nelem) { + js::jit::AtomicOperations::podCopySafeWhenRacy(dest, src, nelem); + } + + template + static void podMove(SharedMem dest, SharedMem src, size_t nelem) { + js::jit::AtomicOperations::podMoveSafeWhenRacy(dest, src, nelem); + } + + static SharedMem extract(TypedArrayObject* obj) { + return obj->viewDataEither(); + } +}; + +class UnsharedOps +{ + public: + template + static T load(SharedMem addr) { + return *addr.unwrapUnshared(); + } + + template + static void store(SharedMem addr, T value) { + *addr.unwrapUnshared() = value; + } + + template + static void memcpy(SharedMem dest, SharedMem src, size_t size) { + ::memcpy(dest.unwrapUnshared(), src.unwrapUnshared(), size); + } + + template + static void memmove(SharedMem dest, SharedMem src, size_t size) { + ::memmove(dest.unwrapUnshared(), src.unwrapUnshared(), size); + } + + template + static void podCopy(SharedMem dest, SharedMem src, size_t nelem) { + // std::copy_n better matches the argument values/types of this + // function, but as noted below it allows the input/output ranges to + // overlap. std::copy does not, so use it so the compiler has extra + // ability to optimize. + const auto* first = src.unwrapUnshared(); + const auto* last = first + nelem; + auto* result = dest.unwrapUnshared(); + std::copy(first, last, result); + } + + template + static void podMove(SharedMem dest, SharedMem src, size_t n) { + // std::copy_n copies from |src| to |dest| starting from |src|, so + // input/output ranges *may* permissibly overlap, as this function + // allows. + const auto* start = src.unwrapUnshared(); + auto* result = dest.unwrapUnshared(); + std::copy_n(start, n, result); + } + + static SharedMem extract(TypedArrayObject* obj) { + return SharedMem::unshared(obj->viewDataUnshared()); + } +}; + +template +class ElementSpecific +{ + + public: + /* + * Copy |source|'s elements into |target|, starting at |target[offset]|. + * Act as if the assignments occurred from a fresh copy of |source|, in + * case the two memory ranges overlap. + */ + static bool + setFromTypedArray(JSContext* cx, + Handle target, Handle source, + uint32_t offset) + { + MOZ_ASSERT(TypeIDOfType::id == target->type(), + "calling wrong setFromTypedArray specialization"); + + MOZ_ASSERT(offset <= target->length()); + MOZ_ASSERT(source->length() <= target->length() - offset); + + if (TypedArrayObject::sameBuffer(target, source)) + return setFromOverlappingTypedArray(cx, target, source, offset); + + SharedMem dest = target->viewDataEither().template cast() + offset; + uint32_t count = source->length(); + + if (source->type() == target->type()) { + Ops::podCopy(dest, source->viewDataEither().template cast(), count); + return true; + } + + // Inhibit unaligned accesses on ARM (bug 1097253, a compiler bug). +#ifdef __arm__ +# define JS_VOLATILE_ARM volatile +#else +# define JS_VOLATILE_ARM +#endif + + SharedMem data = Ops::extract(source); + switch (source->type()) { + case Scalar::Int8: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::Uint8: + case Scalar::Uint8Clamped: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::Int16: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::Uint16: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::Int32: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::Uint32: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::BigInt64: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::BigUint64: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::Float32: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + case Scalar::Float64: { + SharedMem src = data.cast(); + for (uint32_t i = 0; i < count; ++i) + Ops::store(dest++, ConvertNumber(Ops::load(src++))); + break; + } + default: + MOZ_CRASH("setFromTypedArray with a typed array with bogus type"); + } + +#undef JS_VOLATILE_ARM + + return true; + } + + /* + * Copy |source[0]| to |source[len]| (exclusive) elements into the typed + * array |target|, starting at index |offset|. |source| must not be a + * typed array. + */ + static bool + setFromNonTypedArray(JSContext* cx, Handle target, HandleObject source, + uint32_t len, uint32_t offset = 0) + { + MOZ_ASSERT(target->type() == TypeIDOfType::id, + "target type and NativeType must match"); + MOZ_ASSERT(!source->is(), + "use setFromTypedArray instead of this method"); + + uint32_t i = 0; + if (source->isNative()) { + // Attempt fast-path infallible conversion of dense elements up to + // the first potentially side-effectful lookup or conversion. + uint32_t bound = Min(source->as().getDenseInitializedLength(), len); + + SharedMem dest = target->viewDataEither().template cast() + offset; + + MOZ_ASSERT(!canConvertInfallibly(MagicValue(JS_ELEMENTS_HOLE), target->type()), + "the following loop must abort on holes"); + + const Value* srcValues = source->as().getDenseElements(); + for (; i < bound; i++) { + if (!canConvertInfallibly(srcValues[i], target->type())) + break; + Ops::store(dest + i, infallibleValueToNative(srcValues[i])); + } + if (i == len) + return true; + } + + // Convert and copy any remaining elements generically. + RootedValue v(cx); + for (; i < len; i++) { + if (!GetElement(cx, source, source, i, &v)) + return false; + + T n; + if (!valueToNative(cx, v, &n)) + return false; + + len = Min(len, target->length()); + if (i >= len) + break; + + // Compute every iteration in case getElement/valueToNative is wacky. + SharedMem dest = target->viewDataEither().template cast() + offset + i; + Ops::store(dest, n); + } + + return true; + } + + /* + * Copy |source| into the typed array |target|. + */ + static bool + initFromIterablePackedArray(JSContext* cx, Handle target, + HandleArrayObject source) + { + MOZ_ASSERT(target->type() == TypeIDOfType::id, + "target type and NativeType must match"); + MOZ_ASSERT(IsPackedArray(source), "source array must be packed"); + MOZ_ASSERT(source->getDenseInitializedLength() <= target->length()); + + uint32_t len = source->getDenseInitializedLength(); + uint32_t i = 0; + + // Attempt fast-path infallible conversion of dense elements up to the + // first potentially side-effectful conversion. + + SharedMem dest = target->viewDataEither().template cast(); + + const Value* srcValues = source->getDenseElements(); + for (; i < len; i++) { + if (!canConvertInfallibly(srcValues[i], target->type())) + break; + Ops::store(dest + i, infallibleValueToNative(srcValues[i])); + } + if (i == len) + return true; + + // Convert any remaining elements by first collecting them into a + // temporary list, and then copying them into the typed array. + AutoValueVector values(cx); + if (!values.append(srcValues + i, len - i)) + return false; + + RootedValue v(cx); + for (uint32_t j = 0; j < values.length(); i++, j++) { + v = values[j]; + + T n; + if (!valueToNative(cx, v, &n)) + return false; + + // |target| is a newly allocated typed array and not yet visible to + // content script, so valueToNative can't detach the underlying + // buffer. + MOZ_ASSERT(i < target->length()); + + // Compute every iteration in case GC moves the data. + SharedMem newDest = target->viewDataEither().template cast(); + Ops::store(newDest + i, n); + } + + return true; + } + + private: + static bool + setFromOverlappingTypedArray(JSContext* cx, + Handle target, + Handle source, + uint32_t offset) + { + MOZ_ASSERT(TypeIDOfType::id == target->type(), + "calling wrong setFromTypedArray specialization"); + MOZ_ASSERT(TypedArrayObject::sameBuffer(target, source), + "the provided arrays don't actually overlap, so it's " + "undesirable to use this method"); + + MOZ_ASSERT(offset <= target->length()); + MOZ_ASSERT(source->length() <= target->length() - offset); + + SharedMem dest = target->viewDataEither().template cast() + offset; + uint32_t len = source->length(); + + if (source->type() == target->type()) { + SharedMem src = source->viewDataEither().template cast(); + Ops::podMove(dest, src, len); + return true; + } + + // Copy |source| in case it overlaps the target elements being set. + size_t sourceByteLen = len * source->bytesPerElement(); + void* data = target->zone()->template pod_malloc(sourceByteLen); + if (!data) + return false; + Ops::memcpy(SharedMem::unshared(data), + source->viewDataEither(), + sourceByteLen); + + switch (source->type()) { + case Scalar::Int8: { + int8_t* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::Uint8: + case Scalar::Uint8Clamped: { + uint8_t* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::Int16: { + int16_t* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::Uint16: { + uint16_t* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::Int32: { + int32_t* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::Uint32: { + uint32_t* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::BigInt64: { + int64_t* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::BigUint64: { + uint64_t* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::Float32: { + float* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + case Scalar::Float64: { + double* src = static_cast(data); + for (uint32_t i = 0; i < len; ++i) + Ops::store(dest++, ConvertNumber(*src++)); + break; + } + default: + MOZ_CRASH("setFromOverlappingTypedArray with a typed array with bogus type"); + } + + js_free(data); + return true; + } + + static bool + canConvertInfallibly(const Value& v, Scalar::Type type) + { + if (type == Scalar::BigInt64 || type == Scalar::BigUint64) { + return false; + } + return v.isNumber() || v.isBoolean() || v.isNull() || v.isUndefined(); + } + + static T + infallibleValueToNative(const Value& v) + { + if (v.isInt32()) + return T(v.toInt32()); + if (v.isDouble()) + return doubleToNative(v.toDouble()); + if (v.isBoolean()) + return T(v.toBoolean()); + if (v.isNull()) + return T(0); + + MOZ_ASSERT(v.isUndefined()); + return TypeIsFloatingPoint() ? T(JS::GenericNaN()) : T(0); + } + + static bool + valueToNative(JSContext* cx, HandleValue v, T* result) + { + MOZ_ASSERT(!v.isMagic()); + + if (MOZ_LIKELY(canConvertInfallibly(v, TypeIDOfType::id))) { + *result = infallibleValueToNative(v); + return true; + } + + if (std::is_same::value) { + JS_TRY_VAR_OR_RETURN_FALSE(cx, *result, ToBigInt64(cx, v)); + return true; + } + + if (std::is_same::value) { + JS_TRY_VAR_OR_RETURN_FALSE(cx, *result, ToBigUint64(cx, v)); + return true; + } + + double d; + MOZ_ASSERT(v.isString() || v.isObject() || v.isSymbol()); + if (!(v.isString() ? StringToNumber(cx, v.toString(), &d) : ToNumber(cx, v, &d))) + return false; + + *result = doubleToNative(d); + return true; + } + + static T + doubleToNative(double d) + { + if (TypeIsFloatingPoint()) { +#ifdef JS_MORE_DETERMINISTIC + // The JS spec doesn't distinguish among different NaN values, and + // it deliberately doesn't specify the bit pattern written to a + // typed array when NaN is written into it. This bit-pattern + // inconsistency could confuse deterministic testing, so always + // canonicalize NaN values in more-deterministic builds. + d = JS::CanonicalizeNaN(d); +#endif + return T(d); + } + if (MOZ_UNLIKELY(mozilla::IsNaN(d))) + return T(0); + if (TypeIDOfType::id == Scalar::Uint8Clamped) + return T(d); + if (TypeIsUnsigned()) + return T(JS::ToUint32(d)); + return T(JS::ToInt32(d)); + } +}; + +/* static */ gc::AllocKind +js::TypedArrayObject::AllocKindForLazyBuffer(size_t nbytes) +{ + MOZ_ASSERT(nbytes <= INLINE_BUFFER_LIMIT); + if (nbytes == 0) + nbytes += sizeof(uint8_t); + size_t dataSlots = AlignBytes(nbytes, sizeof(Value)) / sizeof(Value); + MOZ_ASSERT(nbytes <= dataSlots * sizeof(Value)); + return gc::GetGCObjectKind(FIXED_DATA_START + dataSlots); +} + +template +class TypedArrayMethods +{ + public: + static bool + setFromTypedArray(JSContext* cx, Handle target, + Handle source, uint32_t offset = 0) + { + if (target->isSharedMemory() || source->isSharedMemory()) + return ElementSpecific::setFromTypedArray(cx, target, source, offset); + return ElementSpecific::setFromTypedArray(cx, target, source, offset); + } + + static bool + setFromNonTypedArray(JSContext* cx, Handle target, HandleObject source, + uint32_t len, uint32_t offset = 0) + { + MOZ_ASSERT(!source->is(), "use setFromTypedArray"); + + if (target->isSharedMemory()) + return ElementSpecific::setFromNonTypedArray(cx, target, source, len, offset); + return ElementSpecific::setFromNonTypedArray(cx, target, source, len, offset); + } + + static bool + initFromIterablePackedArray(JSContext* cx, Handle target, + HandleArrayObject source) + { + if (target->isSharedMemory()) + return ElementSpecific::initFromIterablePackedArray(cx, target, source); + return ElementSpecific::initFromIterablePackedArray(cx, target, source); + } +}; + +} // namespace js + +#endif // vm_TypedArrayObject_inl_h diff --git a/js/src/vm/TypedArrayObject.h b/js/src/vm/TypedArrayObject.h index 8b4b0b5092..9427c14a01 100644 --- a/js/src/vm/TypedArrayObject.h +++ b/js/src/vm/TypedArrayObject.h @@ -121,16 +121,7 @@ class TypedArrayObject : public NativeObject static const uint32_t INLINE_BUFFER_LIMIT = (NativeObject::MAX_FIXED_SLOTS - FIXED_DATA_START) * sizeof(Value); - static gc::AllocKind - AllocKindForLazyBuffer(size_t nbytes) - { - MOZ_ASSERT(nbytes <= INLINE_BUFFER_LIMIT); - if (nbytes == 0) - nbytes += sizeof(uint8_t); - size_t dataSlots = AlignBytes(nbytes, sizeof(Value)) / sizeof(Value); - MOZ_ASSERT(nbytes <= dataSlots * sizeof(Value)); - return gc::GetGCObjectKind(FIXED_DATA_START + dataSlots); - } + static inline gc::AllocKind AllocKindForLazyBuffer(size_t nbytes); inline Scalar::Type type() const; inline size_t bytesPerElement() const; diff --git a/js/src/vm/UnboxedObject-inl.h b/js/src/vm/UnboxedObject-inl.h index fa986a7575..2d355c6946 100644 --- a/js/src/vm/UnboxedObject-inl.h +++ b/js/src/vm/UnboxedObject-inl.h @@ -171,6 +171,17 @@ UnboxedPlainObject::layout() const return group()->unboxedLayout(); } +///////////////////////////////////////////////////////////////////// +// UnboxedLayout +///////////////////////////////////////////////////////////////////// + +gc::AllocKind +js::UnboxedLayout::getAllocKind() const +{ + MOZ_ASSERT(size()); + return gc::GetGCObjectKindForBytes(UnboxedPlainObject::offsetOfData() + size()); +} + ///////////////////////////////////////////////////////////////////// // UnboxedArrayObject ///////////////////////////////////////////////////////////////////// diff --git a/js/src/vm/UnboxedObject.h b/js/src/vm/UnboxedObject.h index 6fc482ec71..42e08443e4 100644 --- a/js/src/vm/UnboxedObject.h +++ b/js/src/vm/UnboxedObject.h @@ -323,13 +323,6 @@ bool TryConvertToUnboxedLayout(ExclusiveContext* cx, AutoEnterAnalysis& enter, Shape* templateShape, ObjectGroup* group, PreliminaryObjectArray* objects); -inline gc::AllocKind -UnboxedLayout::getAllocKind() const -{ - MOZ_ASSERT(size()); - return gc::GetGCObjectKindForBytes(UnboxedPlainObject::offsetOfData() + size()); -} - // Class for an array object using an unboxed representation. class UnboxedArrayObject : public JSObject { diff --git a/js/src/wasm/WasmInstance.h b/js/src/wasm/WasmInstance.h index f5c294e4e0..75e56c6431 100644 --- a/js/src/wasm/WasmInstance.h +++ b/js/src/wasm/WasmInstance.h @@ -19,6 +19,7 @@ #define wasm_instance_h #include "gc/Barrier.h" +#include "vm/SharedMem.h" #include "wasm/WasmCode.h" #include "wasm/WasmTable.h"