diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp index c973908003..91fc4e89fa 100644 --- a/js/src/gc/Allocator.cpp +++ b/js/src/gc/Allocator.cpp @@ -226,7 +226,7 @@ js::Allocate(ExclusiveContext* cx) #define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType, bgFinal, nursery) \ template type* js::Allocate(JSContext* cx);\ template type* js::Allocate(JSContext* cx); -FOR_EACH_NONOBJECT_ALLOCKIND(DECL_ALLOCATOR_INSTANCES) +FOR_EACH_NONOBJECT_NONNURSERY_ALLOCKIND(DECL_ALLOCATOR_INSTANCES) #undef DECL_ALLOCATOR_INSTANCES template diff --git a/js/src/gc/Allocator.h b/js/src/gc/Allocator.h index 3f7cdf00a7..eb664ca98c 100644 --- a/js/src/gc/Allocator.h +++ b/js/src/gc/Allocator.h @@ -10,6 +10,8 @@ #include "gc/Heap.h" #include "js/RootingAPI.h" +class JSFatInlineString; + namespace js { struct Class; @@ -18,19 +20,49 @@ struct Class; // fully initialize the thing before calling any function that can potentially // trigger GC. This will ensure that GC tracing never sees junk values stored // in the partially initialized thing. -// -// Note that JSObject allocation must use the longer signature below that -// includes slot, heap, and finalizer information in support of various -// object-specific optimizations. template T* Allocate(ExclusiveContext* cx); +// Note that JSObject allocation must use the longer signature below that +// includes slot, heap, and finalizer information in support of various +// object-specific optimizations. template JSObject* Allocate(ExclusiveContext* cx, gc::AllocKind kind, size_t nDynamicSlots, gc::InitialHeap heap, const Class* clasp); +// Internal function used for nursery-allocatable strings. +template +StringAllocT* +AllocateString(JSContext* cx, gc::InitialHeap heap); + +// Use for nursery-allocatable strings. Returns a value cast to the correct +// type. +template +StringT* +Allocate(JSContext* cx, gc::InitialHeap heap) +{ + return static_cast(js::AllocateString(cx, heap)); +} + +// Specialization for JSFatInlineString that must use a different allocation +// type. Note that we have to explicitly specialize for both values of AllowGC +// because partial function specialization is not allowed. +template <> +inline JSFatInlineString* +Allocate(JSContext* cx, gc::InitialHeap heap) +{ + return static_cast(js::AllocateString(cx, heap)); +} + +template <> +inline JSFatInlineString* +Allocate(JSContext* cx, gc::InitialHeap heap) +{ + return static_cast(js::AllocateString(cx, heap)); +} + } // namespace js #endif // gc_Allocator_h diff --git a/js/src/gc/AtomMarking-inl.h b/js/src/gc/AtomMarking-inl.h new file mode 100644 index 0000000000..e0d897504b --- /dev/null +++ b/js/src/gc/AtomMarking-inl.h @@ -0,0 +1,77 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/AtomMarking.h" + +#include "jscompartment.h" + +#include "gc/Heap-inl.h" + +namespace js { +namespace gc { + +inline size_t +GetAtomBit(TenuredCell* thing) +{ + MOZ_ASSERT(thing->zoneFromAnyThread()->isAtomsZone()); + Arena* arena = thing->arena(); + size_t arenaBit = (reinterpret_cast(thing) - arena->address()) / CellBytesPerMarkBit; + return arena->atomBitmapStart() * JS_BITS_PER_WORD + arenaBit; +} + +inline bool +ThingIsPermanent(JSAtom* atom) +{ + return atom->isPermanentAtom(); +} + +inline bool +ThingIsPermanent(JS::Symbol* symbol) +{ + return symbol->isWellKnownSymbol(); +} + +template +MOZ_ALWAYS_INLINE void +AtomMarkingRuntime::inlinedMarkAtom(JSContext* cx, T* thing) +{ + static_assert(mozilla::IsSame::value || + mozilla::IsSame::value, + "Should only be called with JSAtom* or JS::Symbol* argument"); + + MOZ_ASSERT(thing); + js::gc::TenuredCell* cell = &thing->asTenured(); + MOZ_ASSERT(cell->zoneFromAnyThread()->isAtomsZone()); + + // The context's zone will be null during initialization of the runtime. + if (!cx->zone()) + return; + MOZ_ASSERT(!cx->zone()->isAtomsZone()); + + if (ThingIsPermanent(thing)) + return; + + size_t bit = GetAtomBit(cell); + MOZ_ASSERT(bit / JS_BITS_PER_WORD < allocatedWords); + + cx->zone()->markedAtoms().setBit(bit); + + if (!cx->helperThread()) { + // Trigger a read barrier on the atom, in case there is an incremental + // GC in progress. This is necessary if the atom is being marked + // because a reference to it was obtained from another zone which is + // not being collected by the incremental GC. + T::readBarrier(thing); + } + + // Children of the thing also need to be marked in the context's zone. + // We don't have a JSTracer for this so manually handle the cases in which + // an atom can reference other atoms. + markChildren(cx, thing); +} + +} // namespace gc +} // namespace js diff --git a/js/src/gc/AtomMarking.cpp b/js/src/gc/AtomMarking.cpp index cc84ab4333..d688257b54 100644 --- a/js/src/gc/AtomMarking.cpp +++ b/js/src/gc/AtomMarking.cpp @@ -225,7 +225,7 @@ AtomMarkingRuntime::atomIsMarked(Zone* zone, T* thing) return true; } - size_t bit = GetAtomBit(thing); + size_t bit = GetAtomBit(&thing->asTenured()); return zone->markedAtoms().getBit(bit); } diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index c6e06d1731..dff65809e4 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -1028,7 +1028,9 @@ class GCRuntime static JSObject* tryNewTenuredObject(ExclusiveContext* cx, AllocKind kind, size_t thingSize, size_t nDynamicSlots); template - static T* tryNewTenuredThing(ExclusiveContext* cx, AllocKind kind, size_t thingSize); + static T* tryNewTenuredThing(JSContext* cx, AllocKind kind, size_t thingSize); + template + JSString* tryNewNurseryString(JSContext* cx, size_t thingSize, AllocKind kind); static TenuredCell* refillFreeListInGC(Zone* zone, AllocKind thingKind); void bufferGrayRoots(); diff --git a/js/src/gc/GCTrace.cpp b/js/src/gc/GCTrace.cpp index 6f856cf563..8cc18019be 100644 --- a/js/src/gc/GCTrace.cpp +++ b/js/src/gc/GCTrace.cpp @@ -132,6 +132,13 @@ js::gc::TraceNurseryAlloc(Cell* thing, size_t size) } } +void +js::gc::TraceNurseryAlloc(Cell* thing, AllocKind kind) +{ + if (thing) + TraceEvent(TraceEventNurseryAlloc, uint64_t(thing), kind); +} + void js::gc::TraceTenuredAlloc(Cell* thing, AllocKind kind) { diff --git a/js/src/gc/GCTrace.h b/js/src/gc/GCTrace.h index 4ebb74ee6f..8e72811e26 100644 --- a/js/src/gc/GCTrace.h +++ b/js/src/gc/GCTrace.h @@ -20,6 +20,7 @@ extern MOZ_MUST_USE bool InitTrace(GCRuntime& gc); extern void FinishTrace(); extern bool TraceEnabled(); extern void TraceNurseryAlloc(Cell* thing, size_t size); +extern void TraceNurseryAlloc(Cell* thing, AllocKind kind); extern void TraceTenuredAlloc(Cell* thing, AllocKind kind); extern void TraceCreateObject(JSObject* object); extern void TraceMinorGCStart(); @@ -36,6 +37,7 @@ inline MOZ_MUST_USE bool InitTrace(GCRuntime& gc) { return true; } inline void FinishTrace() {} inline bool TraceEnabled() { return false; } inline void TraceNurseryAlloc(Cell* thing, size_t size) {} +inline void TraceNurseryAlloc(Cell* thing, AllocKind kind) {} inline void TraceTenuredAlloc(Cell* thing, AllocKind kind) {} inline void TraceCreateObject(JSObject* object) {} inline void TraceMinorGCStart() {} diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp index 63dc92d2ba..8dafcb6461 100644 --- a/js/src/gc/Marking.cpp +++ b/js/src/gc/Marking.cpp @@ -624,7 +624,7 @@ js::TraceProcessGlobalRoot(JSTracer* trc, T* thing, const char* name) // permanent atoms, so likewise require no subsquent marking. CheckTracedThing(trc, *ConvertToBase(&thing)); if (trc->isMarkingTracer()) - thing->markIfUnmarked(gc::MarkColor::Black); + thing->asTenured().markIfUnmarked(gc::MarkColor::Black); else DoCallback(trc->asCallbackTracer(), ConvertToBase(&thing), name); } diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index 663a23c07c..bdde70a261 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -321,6 +321,23 @@ js::Nursery::allocateObject(JSContext* cx, size_t size, size_t numDynamic, const return obj; } +Cell* +js::Nursery::allocateString(JSContext* cx, Zone* zone, size_t size, AllocKind kind) +{ + /* Ensure there's enough space to replace the contents with a RelocationOverlay. */ + MOZ_ASSERT(size >= sizeof(RelocationOverlay)); + + size_t allocSize = JS_ROUNDUP(sizeof(StringLayout) - 1 + size, CellAlignBytes); + auto header = static_cast(allocate(allocSize)); + if (!header) + return nullptr; + header->zone = zone; + + auto cell = reinterpret_cast(&header->cell); + TraceNurseryAlloc(cell, kind); + return cell; +} + void* js::Nursery::allocate(size_t size) { diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 6ca66fd09b..67c2fed982 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -88,7 +88,6 @@ class TenuringTracer : public JSTracer public: Nursery& nursery() { return nursery_; } - // Returns true if the pointer was updated. template void traverse(T** thingp); template void traverse(T* thingp); @@ -136,6 +135,15 @@ class Nursery static const size_t Alignment = gc::ChunkSize; static const size_t ChunkShift = gc::ChunkShift; + struct alignas(gc::CellAlignBytes) CellAlignedByte { + char byte; + }; + + struct StringLayout { + JS::Zone* zone; + CellAlignedByte cell; + }; + explicit Nursery(JSRuntime* rt); ~Nursery(); @@ -187,6 +195,29 @@ class Nursery */ JSObject* allocateObject(JSContext* cx, size_t size, size_t numDynamic, const js::Class* clasp); + /* + * Allocate and return a pointer to a new string. Returns nullptr if the + * Nursery is full. + */ + gc::Cell* allocateString(JSContext* cx, JS::Zone* zone, size_t size, gc::AllocKind kind); + + /* + * String zones are stored just before the string in nursery memory. + */ + static JS::Zone* getStringZone(const JSString* str) { +#ifdef DEBUG + auto cell = reinterpret_cast(str); // JSString type is incomplete here + MOZ_ASSERT(js::gc::IsInsideNursery(cell), "getStringZone must be passed a nursery string"); +#endif + + auto layout = reinterpret_cast(str) - offsetof(StringLayout, cell); + return reinterpret_cast(layout)->zone; + } + + static size_t stringHeaderSize() { + return offsetof(StringLayout, cell); + } + /* Allocate a buffer for a given zone, using the nursery if possible. */ void* allocateBuffer(JS::Zone* zone, size_t nbytes); diff --git a/js/src/gdb/mozilla/JSString.py b/js/src/gdb/mozilla/JSString.py index 99d5ce987d..c9baf36f6e 100644 --- a/js/src/gdb/mozilla/JSString.py +++ b/js/src/gdb/mozilla/JSString.py @@ -17,7 +17,7 @@ class JSStringTypeCache(object): def __init__(self, cache): dummy = gdb.Value(0).cast(cache.JSString_ptr_t) self.ROPE_FLAGS = dummy['ROPE_FLAGS'] - self.ATOM_BIT = dummy['ATOM_BIT'] + self.NON_ATOM_BIT = dummy['NON_ATOM_BIT'] self.INLINE_CHARS_BIT = dummy['INLINE_CHARS_BIT'] self.TYPE_FLAGS_MASK = dummy['TYPE_FLAGS_MASK'] self.LATIN1_CHARS_BIT = dummy['LATIN1_CHARS_BIT'] diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index 8d713f4b07..35d027bcc8 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -1591,13 +1591,13 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names, static void* AllocateString(JSContext* cx) { - return js::Allocate(cx); + return js::Allocate(cx, js::gc::TenuredHeap); } static void* AllocateFatInlineString(JSContext* cx) { - return js::Allocate(cx); + return js::Allocate(cx, js::gc::TenuredHeap); } void @@ -7700,10 +7700,13 @@ JitCompartment::generateStringConcatStub(JSContext* cx) masm.newGCString(output, temp3, &failure, stringsCanBeInNursery); // Store rope length and flags. temp1 still holds the result of AND'ing the - // lhs and rhs flags, so we just have to clear the other flags to get our - // rope flags (Latin1 if both lhs and rhs are Latin1). - static_assert(JSString::ROPE_FLAGS == 0, "Rope flags must be 0"); + // lhs and rhs flags, so we just have to clear the other flags and set + // NON_ATOM_BIT to get our rope flags (Latin1 if both lhs and rhs are + // Latin1). + static_assert(JSString::INIT_ROPE_FLAGS == JSString::NON_ATOM_BIT, + "Rope type flags must be NON_ATOM_BIT only"); masm.and32(Imm32(JSString::LATIN1_CHARS_BIT), temp1); + masm.or32(Imm32(JSString::NON_ATOM_BIT), temp1); masm.store32(temp1, Address(output, JSString::offsetOfFlags())); masm.store32(temp2, Address(output, JSString::offsetOfLength())); diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp index 8a594ad2cf..2da9b2dee8 100644 --- a/js/src/jit/MacroAssembler.cpp +++ b/js/src/jit/MacroAssembler.cpp @@ -1276,9 +1276,9 @@ MacroAssembler::compareStrings(JSOp op, Register left, Register right, Register Label notAtom; // Optimize the equality operation to a pointer compare for two atoms. - Imm32 atomBit(JSString::ATOM_BIT); - branchTest32(Assembler::Zero, Address(left, JSString::offsetOfFlags()), atomBit, ¬Atom); - branchTest32(Assembler::Zero, Address(right, JSString::offsetOfFlags()), atomBit, ¬Atom); + Imm32 nonAtomBit(JSString::NON_ATOM_BIT); + branchTest32(Assembler::NonZero, Address(left, JSString::offsetOfFlags()), nonAtomBit, ¬Atom); + branchTest32(Assembler::NonZero, Address(right, JSString::offsetOfFlags()), nonAtomBit, ¬Atom); cmpPtrSet(JSOpToCondition(MCompare::Compare_String, op), left, right, result); jump(&done); diff --git a/js/src/jsfriendapi.h b/js/src/jsfriendapi.h index 3013802113..5db137e50e 100644 --- a/js/src/jsfriendapi.h +++ b/js/src/jsfriendapi.h @@ -585,7 +585,8 @@ struct Function { struct String { - static const uint32_t INLINE_CHARS_BIT = JS_BIT(2); + static const uint32_t NON_ATOM_BIT = JS_BIT(0); + static const uint32_t INLINE_CHARS_BIT = JS_BIT(3); static const uint32_t LATIN1_CHARS_BIT = JS_BIT(6); static const uint32_t ROPE_FLAGS = NON_ATOM_BIT; static const uint32_t TYPE_FLAGS_MASK = JS_BIT(6) - 1; @@ -597,6 +598,11 @@ struct String JS::Latin1Char inlineStorageLatin1[1]; char16_t inlineStorageTwoByte[1]; }; + + static bool nurseryCellIsString(const js::gc::Cell* cell) { + MOZ_ASSERT(IsInsideNursery(cell)); + return reinterpret_cast(cell)->flags & NON_ATOM_BIT; + } }; } /* namespace shadow */ diff --git a/js/src/jsgcinlines.h b/js/src/jsgcinlines.h index af29a9d5cb..955517fc7c 100644 --- a/js/src/jsgcinlines.h +++ b/js/src/jsgcinlines.h @@ -173,7 +173,7 @@ class ArenaCellIterImpl template T* get() const { MOZ_ASSERT(!done()); MOZ_ASSERT(JS::MapTypeToTraceKind::kind == traceKind); - return static_cast(getCell()); + return reinterpret_cast(getCell()); } void next() { diff --git a/js/src/vm/String-inl.h b/js/src/vm/String-inl.h index 70f05fbd62..32009cccf4 100644 --- a/js/src/vm/String-inl.h +++ b/js/src/vm/String-inl.h @@ -121,7 +121,7 @@ JSRope::new_(js::ExclusiveContext* cx, { if (!validateLength(cx, length)) return nullptr; - JSRope* str = static_cast(js::Allocate(cx)); + JSRope* str = js::Allocate(cx, js::gc::TenuredHeap); if (!str) return nullptr; str->init(cx, left, right, length); @@ -181,7 +181,7 @@ JSDependentString::new_(js::ExclusiveContext* cx, JSLinearString* baseArg, size_ if (baseArg->isExternal() && !baseArg->ensureFlat(cx->asJSContext())) return nullptr; - JSDependentString* str = static_cast(js::Allocate(cx)); + JSDependentString* str = js::Allocate(cx, js::gc::TenuredHeap); if (str) { str->init(cx, baseArg, start, length); return str; @@ -189,7 +189,7 @@ JSDependentString::new_(js::ExclusiveContext* cx, JSLinearString* baseArg, size_ js::RootedLinearString base(cx, baseArg); - str = static_cast(js::Allocate(cx)); + str = js::Allocate(cx, js::gc::TenuredHeap); if (!str) return nullptr; str->init(cx, base, start, length); @@ -200,7 +200,7 @@ MOZ_ALWAYS_INLINE void JSFlatString::init(const char16_t* chars, size_t length) { d.u1.length = length; - d.u1.flags = FLAT_BIT; + d.u1.flags = INIT_FLAT_FLAGS; d.s.u2.nonInlineCharsTwoByte = chars; } @@ -208,7 +208,7 @@ MOZ_ALWAYS_INLINE void JSFlatString::init(const JS::Latin1Char* chars, size_t length) { d.u1.length = length; - d.u1.flags = FLAT_BIT | LATIN1_CHARS_BIT; + d.u1.flags = INIT_FLAT_FLAGS | LATIN1_CHARS_BIT; d.s.u2.nonInlineCharsLatin1 = chars; } @@ -225,7 +225,7 @@ JSFlatString::new_(js::ExclusiveContext* cx, const CharT* chars, size_t length) if (cx->compartment()->isAtomsCompartment()) str = js::Allocate(cx); else - str = static_cast(js::Allocate(cx)); + str = js::Allocate(cx, js::gc::TenuredHeap); if (!str) return nullptr; @@ -269,7 +269,7 @@ JSThinInlineString::new_(js::ExclusiveContext* cx) if (cx->compartment()->isAtomsCompartment()) return (JSThinInlineString*)(js::Allocate(cx)); - return static_cast(js::Allocate(cx)); + return js::Allocate(cx, js::gc::TenuredHeap); } template @@ -279,7 +279,7 @@ JSFatInlineString::new_(js::ExclusiveContext* cx) if (cx->compartment()->isAtomsCompartment()) return (JSFatInlineString*)(js::Allocate(cx)); - return js::Allocate(cx); + return js::Allocate(cx, js::gc::TenuredHeap); } template<> diff --git a/js/src/vm/String.cpp b/js/src/vm/String.cpp index 1cf020eaca..af46079c2d 100644 --- a/js/src/vm/String.cpp +++ b/js/src/vm/String.cpp @@ -196,9 +196,10 @@ JSString::dumpRepresentationHeader(FILE* fp, int indent, const char* subclass) c if (flags & FLAT_BIT) fputs(" FLAT", fp); if (flags & HAS_BASE_BIT) fputs(" HAS_BASE", fp); if (flags & INLINE_CHARS_BIT) fputs(" INLINE_CHARS", fp); - if (flags & ATOM_BIT) fputs(" ATOM", fp); + if (flags & NON_ATOM_BIT) fputs(" NON_ATOM", fp); if (isPermanentAtom()) fputs(" PERMANENT", fp); if (flags & LATIN1_CHARS_BIT) fputs(" LATIN1", fp); + if (!isTenured()) fputs(" NURSERY", fp); fputc('\n', fp); } @@ -1092,9 +1093,11 @@ JSExternalString::ensureFlat(JSContext* cx) // Release the external chars. finalize(cx->runtime()->defaultFreeOp()); - // Transform the string into a non-external, flat string. + // Transform the string into a non-external, flat string. Note that the + // resulting string will still be in an AllocKind::EXTERNAL_STRING arena, + // but will no longer be an external string. setNonInlineChars(s); - d.u1.flags = FLAT_BIT; + d.u1.flags = INIT_FLAT_FLAGS; return &this->asFlat(); } diff --git a/js/src/vm/String.h b/js/src/vm/String.h index 1250959a3a..61dbad688b 100644 --- a/js/src/vm/String.h +++ b/js/src/vm/String.h @@ -16,6 +16,7 @@ #include "gc/Barrier.h" #include "gc/Heap.h" #include "gc/Nursery.h" +#include "gc/Cell.h" #include "gc/Marking.h" #include "gc/Rooting.h" #include "js/CharacterEncoding.h" @@ -148,7 +149,7 @@ static const size_t UINT32_CHAR_BUFFER_LENGTH = sizeof("4294967295") - 1; * at least X (e.g., ensureLinear will change a JSRope to be a JSFlatString). */ -class JSString : public js::gc::TenuredCell +class JSString : public js::gc::Cell { protected: static const size_t NUM_INLINE_CHARS_LATIN1 = 2 * sizeof(void*) / sizeof(JS::Latin1Char); @@ -215,41 +216,49 @@ class JSString : public js::gc::TenuredCell * String Instance Subtype * type encoding predicate * ------------------------------------ - * Rope 000000 000000 - * Linear - !000000 - * HasBase - xxxx1x - * Dependent 000010 000010 - * External 100000 100000 + * Rope 000001 000000 + * Linear - !000010 + * HasBase - xxx1xx + * Dependent 000110 000110 + * External 100010 100010 * Flat - xxxxx1 - * Undepended 000011 000011 - * Extensible 010001 010001 - * Inline 000101 xxx1xx - * FatInline 010101 x1x1xx - * Atom 001001 xx1xxx - * PermanentAtom 101001 1x1xxx - * InlineAtom - xx11xx - * FatInlineAtom - x111xx + * Undepended 000111 000111 + * Extensible 010011 010011 + * Inline 001011 xx1xxx + * FatInline 011011 x11xxx + * NormalAtom 000010 xxxxx0 + * PermanentAtom 100010 1xxxx0 + * InlineAtom - xx1xx0 + * FatInlineAtom - x11xx0 * * Note that the first 4 flag bits (from right to left in the previous table) * have the following meaning and can be used for some hot queries: * - * Bit 0: IsFlat - * Bit 1: HasBase (Dependent, Undepended) - * Bit 2: IsInline (Inline, FatInline) - * Bit 3: IsAtom (Atom, PermanentAtom) + * Bit 0: !IsAtom (Atom, PermanentAtom) + * Bit 1: IsFlat + * Bit 2: HasBase (Dependent, Undepended) + * Bit 3: IsInline (Inline, FatInline) * * "HasBase" here refers to the two string types that have a 'base' field: * JSDependentString and JSUndependedString. * A JSUndependedString is a JSDependentString which has been 'fixed' (by ensureFixed) * to be null-terminated. In such cases, the string must keep marking its base since * there may be any number of *other* JSDependentStrings transitively depending on it. + * The atom bit (NON_ATOM_BIT) is inverted so that objects and strings can + * + * be differentiated in the nursery: atoms are never in the nursery, so + * this bit is always 1 for a nursery string. For an object on a + * little-endian architecture, this is the low-order bit of the ObjectGroup + * pointer in a JSObject, which will always be zero. A 64-bit big-endian + * architecture will need to do something else (the ObjectGroup* is in the + * same place as a string's struct { uint32_t flags; uint32_t length; }). * */ - static const uint32_t FLAT_BIT = JS_BIT(0); - static const uint32_t HAS_BASE_BIT = JS_BIT(1); - static const uint32_t INLINE_CHARS_BIT = JS_BIT(2); - static const uint32_t ATOM_BIT = JS_BIT(3); + static const uint32_t NON_ATOM_BIT = JS_BIT(0); + static const uint32_t FLAT_BIT = JS_BIT(1); + static const uint32_t HAS_BASE_BIT = JS_BIT(2); + static const uint32_t INLINE_CHARS_BIT = JS_BIT(3); static const uint32_t ROPE_FLAGS = NON_ATOM_BIT; static const uint32_t DEPENDENT_FLAGS = NON_ATOM_BIT | HAS_BASE_BIT; @@ -258,11 +267,13 @@ class JSString : public js::gc::TenuredCell static const uint32_t EXTERNAL_FLAGS = NON_ATOM_BIT | JS_BIT(5); static const uint32_t FAT_INLINE_MASK = INLINE_CHARS_BIT | JS_BIT(4); - static const uint32_t PERMANENT_ATOM_MASK = ATOM_BIT | JS_BIT(5); + static const uint32_t PERMANENT_ATOM_MASK = NON_ATOM_BIT | JS_BIT(5); + static const uint32_t PERMANENT_ATOM_FLAGS = JS_BIT(5); /* Initial flags for thin inline and fat inline strings. */ - static const uint32_t INIT_THIN_INLINE_FLAGS = FLAT_BIT | INLINE_CHARS_BIT; - static const uint32_t INIT_FAT_INLINE_FLAGS = FLAT_BIT | FAT_INLINE_MASK; + static const uint32_t INIT_THIN_INLINE_FLAGS = NON_ATOM_BIT | FLAT_BIT | INLINE_CHARS_BIT; + static const uint32_t INIT_FAT_INLINE_FLAGS = NON_ATOM_BIT | FLAT_BIT | FAT_INLINE_MASK; + static const uint32_t INIT_FLAT_FLAGS = NON_ATOM_BIT | FLAT_BIT; static const uint32_t TYPE_FLAGS_MASK = JS_BIT(6) - 1; @@ -304,6 +315,8 @@ class JSString : public js::gc::TenuredCell "shadow::String inlineStorage offset must match JSString"); static_assert(offsetof(JSString, d.inlineStorageTwoByte) == offsetof(String, inlineStorageTwoByte), "shadow::String inlineStorage offset must match JSString"); + static_assert(NON_ATOM_BIT == String::NON_ATOM_BIT, + "shadow::String::NON_ATOM_BIT must match JSString::NON_ATOM_BIT"); static_assert(INLINE_CHARS_BIT == String::INLINE_CHARS_BIT, "shadow::String::INLINE_CHARS_BIT must match JSString::INLINE_CHARS_BIT"); static_assert(LATIN1_CHARS_BIT == String::LATIN1_CHARS_BIT, @@ -447,12 +460,12 @@ class JSString : public js::gc::TenuredCell MOZ_ALWAYS_INLINE bool isAtom() const { - return d.u1.flags & ATOM_BIT; + return !(d.u1.flags & NON_ATOM_BIT); } MOZ_ALWAYS_INLINE bool isPermanentAtom() const { - return (d.u1.flags & PERMANENT_ATOM_MASK) == PERMANENT_ATOM_MASK; + return (d.u1.flags & PERMANENT_ATOM_MASK) == PERMANENT_ATOM_FLAGS; } MOZ_ALWAYS_INLINE @@ -461,8 +474,8 @@ class JSString : public js::gc::TenuredCell return *(JSAtom*)this; } - // Used for distinguishing strings from objects in the nursery. The caller - // must ensure that cell is in the nursery (and not forwarded). + // Used for distinguishing strings from objects in the nursery. 'cell' must + // be in the nursery. MOZ_ALWAYS_INLINE static bool nurseryCellIsString(js::gc::Cell* cell) { MOZ_ASSERT(!cell->isTenured()); @@ -573,14 +586,14 @@ class JSString : public js::gc::TenuredCell if (thing->isPermanentAtom() || js::gc::IsInsideNursery(thing)) return; - TenuredCell::readBarrier(thing); + js::gc::TenuredCell::readBarrier(&thing->asTenured()); } static MOZ_ALWAYS_INLINE void writeBarrierPre(JSString* thing) { if (!thing || thing->isPermanentAtom() || js::gc::IsInsideNursery(thing)) return; - TenuredCell::writeBarrierPre(thing); + js::gc::TenuredCell::writeBarrierPre(&thing->asTenured()); } static void writeBarrierPost(void* cellp, JSString* prev, JSString* next) { @@ -1015,6 +1028,11 @@ class JSExternalString : public JSLinearString inline void finalize(js::FreeOp* fop); + /* + * Free the external chars and allocate a new buffer, converting this to a + * flat string (which still lives in an AllocKind::EXTERNAL_STRING + * arena). + */ JSFlatString* ensureFlat(JSContext* cx); #ifdef DEBUG @@ -1057,7 +1075,8 @@ class JSAtom : public JSFlatString // Transform this atom into a permanent atom. This is only done during // initialization of the runtime. MOZ_ALWAYS_INLINE void morphIntoPermanentAtom() { - d.u1.flags |= PERMANENT_ATOM_MASK; + MOZ_ASSERT(static_cast(this)->isAtom()); + d.u1.flags |= PERMANENT_ATOM_FLAGS; } inline js::HashNumber hash() const; @@ -1134,7 +1153,8 @@ JSAtom::initHash(js::HashNumber hash) MOZ_ALWAYS_INLINE JSAtom* JSFlatString::morphAtomizedStringIntoAtom(js::HashNumber hash) { - d.u1.flags |= ATOM_BIT; + MOZ_ASSERT(!isAtom()); + d.u1.flags &= ~NON_ATOM_BIT; JSAtom* atom = &asAtom(); atom->initHash(hash); return atom; @@ -1143,7 +1163,9 @@ JSFlatString::morphAtomizedStringIntoAtom(js::HashNumber hash) MOZ_ALWAYS_INLINE JSAtom* JSFlatString::morphAtomizedStringIntoPermanentAtom(js::HashNumber hash) { - d.u1.flags |= PERMANENT_ATOM_MASK; + MOZ_ASSERT(!isAtom()); + d.u1.flags |= PERMANENT_ATOM_FLAGS; + d.u1.flags &= ~NON_ATOM_BIT; JSAtom* atom = &asAtom(); atom->initHash(hash); return atom; @@ -1597,6 +1619,13 @@ Cell::as() { MOZ_ASSERT(is()); return reinterpret_cast(this); } + +template<> +inline JSString* +TenuredCell::as() { + MOZ_ASSERT(is()); + return reinterpret_cast(this); +} } }