From 5eb6f53b0db455e789a9534331236ae363399c50 Mon Sep 17 00:00:00 2001 From: win7-7 Date: Mon, 22 Dec 2025 11:30:26 +0200 Subject: [PATCH] Performance improvements jit 1322724: IonMonkey - Add the hit count information on the extra false branch blocks 1322932: IonMonkey - Only iterate the backedge of the inner-loop when it has already be visited 1329901 - Remove expensive isObservableOperand() loop guards. 1330667: IonMonkey - Create a new constant for every optimized arguments use. 1342016 - Fast-path for isObservableSlot(). 1388045 - Branch Pruning: Check the compile info associated with the resume point. --- js/src/jit/CacheIR.cpp | 2323 +++++++++++++++++++++++++++ js/src/jit/CacheIRCompiler.cpp | 2463 +++++++++++++++++++++++++++++ js/src/jit/CacheIRCompiler.h | 752 +++++++++ js/src/jit/CodeGenerator.cpp | 3 +- js/src/jit/IonCacheIRCompiler.cpp | 2185 +++++++++++++++++++++++++ js/src/jit/MacroAssembler-inl.h | 8 + js/src/jit/MacroAssembler.cpp | 27 +- js/src/jit/MacroAssembler.h | 4 +- 8 files changed, 7759 insertions(+), 6 deletions(-) create mode 100644 js/src/jit/CacheIRCompiler.cpp create mode 100644 js/src/jit/CacheIRCompiler.h create mode 100644 js/src/jit/IonCacheIRCompiler.cpp diff --git a/js/src/jit/CacheIR.cpp b/js/src/jit/CacheIR.cpp index 4da9b7539f..3f9a71249e 100644 --- a/js/src/jit/CacheIR.cpp +++ b/js/src/jit/CacheIR.cpp @@ -475,3 +475,2326 @@ GetPropIRGenerator::tryAttachPrimitive(CacheIRWriter& writer, ValOperandId valId emitted_ = true; return true; } + +bool +GetPropIRGenerator::tryAttachStringLength(ValOperandId valId, HandleId id) +{ + if (!val_.isString() || !JSID_IS_ATOM(id, cx_->names().length)) + return false; + + StringOperandId strId = writer.guardIsString(valId); + maybeEmitIdGuard(id); + writer.loadStringLengthResult(strId); + writer.returnFromIC(); + + trackAttached("StringLength"); + return true; +} + +bool +GetPropIRGenerator::tryAttachStringChar(ValOperandId valId, ValOperandId indexId) +{ + MOZ_ASSERT(idVal_.isInt32()); + + if (!val_.isString()) + return false; + + int32_t index = idVal_.toInt32(); + if (index < 0) + return false; + + JSString* str = val_.toString(); + if (size_t(index) >= str->length()) + return false; + + // This follows JSString::getChar, otherwise we fail to attach getChar in a lot of cases. + if (str->isRope()) { + JSRope* rope = &str->asRope(); + + // Make sure the left side contains the index. + if (size_t(index) >= rope->leftChild()->length()) + return false; + + str = rope->leftChild(); + } + + if (!str->isLinear() || + str->asLinear().latin1OrTwoByteChar(index) >= StaticStrings::UNIT_STATIC_LIMIT) + { + return false; + } + + StringOperandId strId = writer.guardIsString(valId); + Int32OperandId int32IndexId = writer.guardIsInt32Index(indexId); + writer.loadStringCharResult(strId, int32IndexId); + writer.returnFromIC(); + + trackAttached("StringChar"); + return true; +} + +bool +GetPropIRGenerator::tryAttachMagicArgumentsName(ValOperandId valId, HandleId id) +{ + if (!val_.isMagic(JS_OPTIMIZED_ARGUMENTS)) + return false; + + if (!JSID_IS_ATOM(id, cx_->names().length) && !JSID_IS_ATOM(id, cx_->names().callee)) + return false; + + maybeEmitIdGuard(id); + writer.guardMagicValue(valId, JS_OPTIMIZED_ARGUMENTS); + writer.guardFrameHasNoArgumentsObject(); + + if (JSID_IS_ATOM(id, cx_->names().length)) { + writer.loadFrameNumActualArgsResult(); + writer.returnFromIC(); + } else { + MOZ_ASSERT(JSID_IS_ATOM(id, cx_->names().callee)); + writer.loadFrameCalleeResult(); + writer.typeMonitorResult(); + } + + trackAttached("MagicArgumentsName"); + return true; +} + +bool +GetPropIRGenerator::tryAttachMagicArgument(ValOperandId valId, ValOperandId indexId) +{ + MOZ_ASSERT(idVal_.isInt32()); + + if (!val_.isMagic(JS_OPTIMIZED_ARGUMENTS)) + return false; + + writer.guardMagicValue(valId, JS_OPTIMIZED_ARGUMENTS); + writer.guardFrameHasNoArgumentsObject(); + + Int32OperandId int32IndexId = writer.guardIsInt32Index(indexId); + writer.loadFrameArgumentResult(int32IndexId); + writer.typeMonitorResult(); + + trackAttached("MagicArgument"); + return true; +} + +bool +GetPropIRGenerator::tryAttachArgumentsObjectArg(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId) +{ + if (!obj->is() || obj->as().hasOverriddenElement()) + return false; + + if (obj->is()) { + writer.guardClass(objId, GuardClassKind::MappedArguments); + } else { + MOZ_ASSERT(obj->is()); + writer.guardClass(objId, GuardClassKind::UnmappedArguments); + } + + writer.loadArgumentsObjectArgResult(objId, indexId); + writer.typeMonitorResult(); + + trackAttached("ArgumentsObjectArg"); + return true; +} + +bool +GetPropIRGenerator::tryAttachDenseElement(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId) +{ + if (!obj->isNative()) + return false; + + if (!obj->as().containsDenseElement(index)) + return false; + + writer.guardShape(objId, obj->as().lastProperty()); + + writer.loadDenseElementResult(objId, indexId); + writer.typeMonitorResult(); + + trackAttached("DenseElement"); + return true; +} + +static bool +CanAttachDenseElementHole(JSObject* obj, bool ownProp) +{ + // Make sure the objects on the prototype don't have any indexed properties + // or that such properties can't appear without a shape change. + // Otherwise returning undefined for holes would obviously be incorrect, + // because we would have to lookup a property on the prototype instead. + do { + // The first two checks are also relevant to the receiver object. + if (obj->isIndexed()) + return false; + + if (ClassCanHaveExtraProperties(obj->getClass())) + return false; + + // Don't need to check prototype for OwnProperty checks + if (ownProp) + return true; + + JSObject* proto = obj->staticPrototype(); + if (!proto) + break; + + if (!proto->isNative()) + return false; + + // Make sure objects on the prototype don't have dense elements. + if (proto->as().getDenseInitializedLength() != 0) + return false; + + obj = proto; + } while (true); + + return true; +} + +bool +GetPropIRGenerator::tryAttachDenseElementHole(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId) +{ + if (!obj->isNative()) + return false; + + if (obj->as().containsDenseElement(index)) + return false; + + if (!CanAttachDenseElementHole(obj, false)) + return false; + + // Guard on the shape, to prevent non-dense elements from appearing. + writer.guardShape(objId, obj->as().lastProperty()); + + GeneratePrototypeHoleGuards(writer, obj, objId); + + writer.loadDenseElementHoleResult(objId, indexId); + writer.typeMonitorResult(); + trackAttached("DenseElementHole"); + return true; +} + +bool +GetPropIRGenerator::tryAttachUnboxedArrayElement(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId) +{ + if (!obj->is()) + return false; + + if (index >= obj->as().initializedLength()) + return false; + + writer.guardGroup(objId, obj->group()); + + JSValueType elementType = obj->group()->unboxedLayoutDontCheckGeneration().elementType(); + writer.loadUnboxedArrayElementResult(objId, indexId, elementType); + + // Only monitor the result if its type might change. + if (elementType == JSVAL_TYPE_OBJECT) + writer.typeMonitorResult(); + else + writer.returnFromIC(); + + trackAttached("UnboxedArrayElement"); + return true; +} + +bool +GetPropIRGenerator::tryAttachTypedElement(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId) +{ + if (!obj->is() && !IsPrimitiveArrayTypedObject(obj)) + return false; + + if (!cx_->runtime()->jitSupportsFloatingPoint && TypedThingRequiresFloatingPoint(obj)) + return false; + + // Ensure the index is in-bounds so the element type gets monitored. + if (obj->is() && index >= obj->as().length()) + return false; + + // Don't attach typed object stubs if the underlying storage could be + // detached, as the stub will always bail out. + if (IsPrimitiveArrayTypedObject(obj) && cx_->compartment()->detachedTypedObjects) + return false; + + TypedThingLayout layout = GetTypedThingLayout(obj->getClass()); + if (layout != Layout_TypedArray) + writer.guardNoDetachedTypedObjects(); + + writer.guardShape(objId, obj->as().shape()); + + writer.loadTypedElementResult(objId, indexId, layout, TypedThingElementType(obj)); + + // Reading from Uint32Array may produce an int32 now but a double value + // later, so ensure we monitor the result. + if (TypedThingElementType(obj) == Scalar::Type::Uint32) + writer.typeMonitorResult(); + else + writer.returnFromIC(); + + trackAttached("TypedElement"); + return true; +} + +bool +GetPropIRGenerator::tryAttachProxyElement(HandleObject obj, ObjOperandId objId) +{ + if (!obj->is()) + return false; + + // The proxy stubs don't currently support |super| access. + if (isSuper()) + return false; + + writer.guardIsProxy(objId); + + // We are not guarding against DOM proxies here, because there is no other + // specialized DOM IC we could attach. + // We could call maybeEmitIdGuard here and then emit CallProxyGetResult, + // but for GetElem we prefer to attach a stub that can handle any Value + // so we don't attach a new stub for every id. + MOZ_ASSERT(cacheKind_ == CacheKind::GetElem); + MOZ_ASSERT(!isSuper()); + writer.callProxyGetByValueResult(objId, getElemKeyValueId()); + writer.typeMonitorResult(); + + trackAttached("ProxyElement"); + return true; +} + +void +GetPropIRGenerator::trackAttached(const char* name) +{ +#ifdef JS_CACHEIR_SPEW + CacheIRSpewer& sp = CacheIRSpewer::singleton(); + if (sp.enabled()) { + LockGuard guard(sp.lock()); + sp.beginCache(guard, *this); + sp.valueProperty(guard, "base", val_); + sp.valueProperty(guard, "property", idVal_); + sp.attached(guard, name); + sp.endCache(guard); + } +#endif +} + +void +GetPropIRGenerator::trackNotAttached() +{ +#ifdef JS_CACHEIR_SPEW + CacheIRSpewer& sp = CacheIRSpewer::singleton(); + if (sp.enabled()) { + LockGuard guard(sp.lock()); + sp.beginCache(guard, *this); + sp.valueProperty(guard, "base", val_); + sp.valueProperty(guard, "property", idVal_); + sp.endCache(guard); + } +#endif +} + +void +IRGenerator::emitIdGuard(ValOperandId valId, jsid id) +{ + if (JSID_IS_SYMBOL(id)) { + SymbolOperandId symId = writer.guardIsSymbol(valId); + writer.guardSpecificSymbol(symId, JSID_TO_SYMBOL(id)); + } else { + MOZ_ASSERT(JSID_IS_ATOM(id)); + StringOperandId strId = writer.guardIsString(valId); + writer.guardSpecificAtom(strId, JSID_TO_ATOM(id)); + } +} + +void +GetPropIRGenerator::maybeEmitIdGuard(jsid id) +{ + if (cacheKind_ == CacheKind::GetProp || cacheKind_ == CacheKind::GetPropSuper) { + // Constant PropertyName, no guards necessary. + MOZ_ASSERT(&idVal_.toString()->asAtom() == JSID_TO_ATOM(id)); + return; + } + + MOZ_ASSERT(cacheKind_ == CacheKind::GetElem || cacheKind_ == CacheKind::GetElemSuper); + emitIdGuard(getElemKeyValueId(), id); +} + +void +SetPropIRGenerator::maybeEmitIdGuard(jsid id) +{ + if (cacheKind_ == CacheKind::SetProp) { + // Constant PropertyName, no guards necessary. + MOZ_ASSERT(&idVal_.toString()->asAtom() == JSID_TO_ATOM(id)); + return; + } + MOZ_ASSERT(cacheKind_ == CacheKind::SetElem); + emitIdGuard(setElemKeyValueId(), id); +} + +GetNameIRGenerator::GetNameIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc, + ICState::Mode mode, HandleObject env, + HandlePropertyName name) + : IRGenerator(cx, script, pc, CacheKind::GetName, mode), + env_(env), + name_(name) +{} + +bool +GetNameIRGenerator::tryAttachStub() +{ + MOZ_ASSERT(cacheKind_ == CacheKind::GetName); + + AutoAssertNoPendingException aanpe(cx_); + + ObjOperandId envId(writer.setInputOperandId(0)); + RootedId id(cx_, NameToId(name_)); + + if (tryAttachGlobalNameValue(envId, id)) + return true; + if (tryAttachGlobalNameGetter(envId, id)) + return true; + if (tryAttachEnvironmentName(envId, id)) + return true; + + return false; +} + +bool +CanAttachGlobalName(JSContext* cx, Handle globalLexical, HandleId id, + MutableHandleNativeObject holder, MutableHandleShape shape) +{ + // The property must be found, and it must be found as a normal data property. + RootedNativeObject current(cx, globalLexical); + while (true) { + shape.set(current->lookup(cx, id)); + if (shape) + break; + + if (current == globalLexical) { + current = &globalLexical->global(); + } else { + // In the browser the global prototype chain should be immutable. + if (!current->staticPrototypeIsImmutable()) + return false; + + JSObject* proto = current->staticPrototype(); + if (!proto || !proto->is()) + return false; + + current = &proto->as(); + } + } + + holder.set(current); + return true; +} + +bool +GetNameIRGenerator::tryAttachGlobalNameValue(ObjOperandId objId, HandleId id) +{ + if (!IsGlobalOp(JSOp(*pc_)) || script_->hasNonSyntacticScope()) + return false; + + Handle globalLexical = env_.as(); + MOZ_ASSERT(globalLexical->isGlobal()); + + RootedNativeObject holder(cx_); + RootedShape shape(cx_); + if (!CanAttachGlobalName(cx_, globalLexical, id, &holder, &shape)) + return false; + + // The property must be found, and it must be found as a normal data property. + if (!shape->hasDefaultGetter() || !shape->hasSlot()) + return false; + + // This might still be an uninitialized lexical. + if (holder->getSlot(shape->slot()).isMagic()) + return false; + + // Instantiate this global property, for use during Ion compilation. + if (IsIonEnabled(cx_)) + EnsureTrackPropertyTypes(cx_, holder, id); + + if (holder == globalLexical) { + // There is no need to guard on the shape. Lexical bindings are + // non-configurable, and this stub cannot be shared across globals. + size_t dynamicSlotOffset = holder->dynamicSlotIndex(shape->slot()) * sizeof(Value); + writer.loadDynamicSlotResult(objId, dynamicSlotOffset); + } else { + // Check the prototype chain from the global to the holder + // prototype. Ignore the global lexical scope as it doesn't figure + // into the prototype chain. We guard on the global lexical + // scope's shape independently. + if (!IsCacheableGetPropReadSlotForIonOrCacheIR(&globalLexical->global(), holder, PropertyResult(shape))) + return false; + + // Shape guard for global lexical. + writer.guardShape(objId, globalLexical->lastProperty()); + + // Guard on the shape of the GlobalObject. + ObjOperandId globalId = writer.loadEnclosingEnvironment(objId); + writer.guardShape(globalId, globalLexical->global().lastProperty()); + + ObjOperandId holderId = globalId; + if (holder != &globalLexical->global()) { + // Shape guard holder. + holderId = writer.loadObject(holder); + writer.guardShape(holderId, holder->lastProperty()); + } + + EmitLoadSlotResult(writer, holderId, holder, shape); + } + + writer.typeMonitorResult(); + return true; +} + +bool +GetNameIRGenerator::tryAttachGlobalNameGetter(ObjOperandId objId, HandleId id) +{ + if (!IsGlobalOp(JSOp(*pc_)) || script_->hasNonSyntacticScope()) + return false; + + Handle globalLexical = env_.as(); + MOZ_ASSERT(globalLexical->isGlobal()); + + RootedNativeObject holder(cx_); + RootedShape shape(cx_); + if (!CanAttachGlobalName(cx_, globalLexical, id, &holder, &shape)) + return false; + + if (holder == globalLexical) + return false; + + if (!IsCacheableGetPropCallNative(&globalLexical->global(), holder, shape)) + return false; + + if (IsIonEnabled(cx_)) + EnsureTrackPropertyTypes(cx_, holder, id); + + // Shape guard for global lexical. + writer.guardShape(objId, globalLexical->lastProperty()); + + // Guard on the shape of the GlobalObject. + ObjOperandId globalId = writer.loadEnclosingEnvironment(objId); + writer.guardShape(globalId, globalLexical->global().lastProperty()); + + if (holder != &globalLexical->global()) { + // Shape guard holder. + ObjOperandId holderId = writer.loadObject(holder); + writer.guardShape(holderId, holder->lastProperty()); + } + + EmitCallGetterResultNoGuards(writer, &globalLexical->global(), holder, shape, globalId); + return true; +} + +static bool +NeedEnvironmentShapeGuard(JSObject* envObj) +{ + if (!envObj->is()) + return true; + + // We can skip a guard on the call object if the script's bindings are + // guaranteed to be immutable (and thus cannot introduce shadowing + // variables). The function might have been relazified under rare + // conditions. In that case, we pessimistically create the guard. + CallObject* callObj = &envObj->as(); + JSFunction* fun = &callObj->callee(); + if (!fun->hasScript() || fun->nonLazyScript()->funHasExtensibleScope()) + return true; + + return false; +} + +bool +GetNameIRGenerator::tryAttachEnvironmentName(ObjOperandId objId, HandleId id) +{ + if (IsGlobalOp(JSOp(*pc_)) || script_->hasNonSyntacticScope()) + return false; + + RootedObject env(cx_, env_); + RootedShape shape(cx_); + RootedNativeObject holder(cx_); + + while (env) { + if (env->is()) { + shape = env->as().lookup(cx_, id); + if (shape) + break; + return false; + } + + if (!env->is() || env->is()) + return false; + + MOZ_ASSERT(!env->hasUncacheableProto()); + + // Check for an 'own' property on the env. There is no need to + // check the prototype as non-with scopes do not inherit properties + // from any prototype. + shape = env->as().lookup(cx_, id); + if (shape) + break; + + env = env->enclosingEnvironment(); + } + + holder = &env->as(); + if (!IsCacheableGetPropReadSlotForIonOrCacheIR(holder, holder, PropertyResult(shape))) + return false; + if (holder->getSlot(shape->slot()).isMagic()) + return false; + + ObjOperandId lastObjId = objId; + env = env_; + while (env) { + if (NeedEnvironmentShapeGuard(env)) + writer.guardShape(lastObjId, env->maybeShape()); + + if (env == holder) + break; + + lastObjId = writer.loadEnclosingEnvironment(lastObjId); + env = env->enclosingEnvironment(); + } + + if (holder->isFixedSlot(shape->slot())) { + writer.loadEnvironmentFixedSlotResult(lastObjId, NativeObject::getFixedSlotOffset(shape->slot())); + } else { + size_t dynamicSlotOffset = holder->dynamicSlotIndex(shape->slot()) * sizeof(Value); + writer.loadEnvironmentDynamicSlotResult(lastObjId, dynamicSlotOffset); + } + + writer.typeMonitorResult(); + return true; +} + +BindNameIRGenerator::BindNameIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc, + ICState::Mode mode, HandleObject env, + HandlePropertyName name) + : IRGenerator(cx, script, pc, CacheKind::BindName, mode), + env_(env), + name_(name) +{} + +bool +BindNameIRGenerator::tryAttachStub() +{ + MOZ_ASSERT(cacheKind_ == CacheKind::BindName); + + AutoAssertNoPendingException aanpe(cx_); + + ObjOperandId envId(writer.setInputOperandId(0)); + RootedId id(cx_, NameToId(name_)); + + if (tryAttachGlobalName(envId, id)) + return true; + if (tryAttachEnvironmentName(envId, id)) + return true; + + return false; +} + +bool +BindNameIRGenerator::tryAttachGlobalName(ObjOperandId objId, HandleId id) +{ + if (!IsGlobalOp(JSOp(*pc_)) || script_->hasNonSyntacticScope()) + return false; + + Handle globalLexical = env_.as(); + MOZ_ASSERT(globalLexical->isGlobal()); + + JSObject* result = nullptr; + if (Shape* shape = globalLexical->lookup(cx_, id)) { + // If this is an uninitialized lexical or a const, we need to return a + // RuntimeLexicalErrorObject. + if (globalLexical->getSlot(shape->slot()).isMagic() || !shape->writable()) + return false; + result = globalLexical; + } else { + result = &globalLexical->global(); + } + + if (result == globalLexical) { + // Lexical bindings are non-configurable so we can just return the + // global lexical. + writer.loadObjectResult(objId); + } else { + // If the property exists on the global and is non-configurable, it cannot be + // shadowed by the lexical scope so we can just return the global without a + // shape guard. + Shape* shape = result->as().lookup(cx_, id); + if (!shape || shape->configurable()) + writer.guardShape(objId, globalLexical->lastProperty()); + ObjOperandId globalId = writer.loadEnclosingEnvironment(objId); + writer.loadObjectResult(globalId); + } + writer.returnFromIC(); + + return true; +} + +bool +BindNameIRGenerator::tryAttachEnvironmentName(ObjOperandId objId, HandleId id) +{ + if (IsGlobalOp(JSOp(*pc_)) || script_->hasNonSyntacticScope()) + return false; + + RootedObject env(cx_, env_); + RootedShape shape(cx_); + while (true) { + if (!env->is() && !env->is()) + return false; + if (env->is()) + return false; + + MOZ_ASSERT(!env->hasUncacheableProto()); + + // When we reach an unqualified variables object (like the global) we + // have to stop looking and return that object. + if (env->isUnqualifiedVarObj()) + break; + + // Check for an 'own' property on the env. There is no need to + // check the prototype as non-with scopes do not inherit properties + // from any prototype. + shape = env->as().lookup(cx_, id); + if (shape) + break; + + env = env->enclosingEnvironment(); + } + + // If this is an uninitialized lexical or a const, we need to return a + // RuntimeLexicalErrorObject. + RootedNativeObject holder(cx_, &env->as()); + if (shape && + holder->is() && + (holder->getSlot(shape->slot()).isMagic() || !shape->writable())) + { + return false; + } + + ObjOperandId lastObjId = objId; + env = env_; + while (env) { + if (NeedEnvironmentShapeGuard(env) && !env->is()) + writer.guardShape(lastObjId, env->maybeShape()); + + if (env == holder) + break; + + lastObjId = writer.loadEnclosingEnvironment(lastObjId); + env = env->enclosingEnvironment(); + } + writer.loadObjectResult(lastObjId); + writer.returnFromIC(); + + return true; +} + +HasPropIRGenerator::HasPropIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc, + CacheKind cacheKind, ICState::Mode mode, + HandleValue idVal, HandleValue val) + : IRGenerator(cx, script, pc, cacheKind, mode), + val_(val), + idVal_(idVal) +{ } + +bool +HasPropIRGenerator::tryAttachDense(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId) +{ + if (!obj->isNative()) + return false; + if (!obj->as().containsDenseElement(index)) + return false; + + // Guard shape to ensure object class is NativeObject. + writer.guardShape(objId, obj->as().lastProperty()); + writer.loadDenseElementExistsResult(objId, indexId); + writer.returnFromIC(); + + trackAttached("DenseHasProp"); + return true; +} + +bool +HasPropIRGenerator::tryAttachDenseHole(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId) +{ + bool hasOwn = (cacheKind_ == CacheKind::HasOwn); + if (!obj->isNative()) + return false; + + if (obj->as().containsDenseElement(index)) + return false; + + if (!CanAttachDenseElementHole(obj, hasOwn)) + return false; + + // Guard shape to ensure class is NativeObject and to prevent non-dense + // elements being added. Also ensures prototype doesn't change if dynamic + // checks aren't emitted. + writer.guardShape(objId, obj->as().lastProperty()); + + // Generate prototype guards if needed. This includes monitoring that + // properties were not added in the chain. + if (!hasOwn) + GeneratePrototypeHoleGuards(writer, obj, objId); + writer.loadDenseElementHoleExistsResult(objId, indexId); + writer.returnFromIC(); + + trackAttached("DenseHasPropHole"); + return true; +} + +bool +HasPropIRGenerator::tryAttachNative(HandleObject obj, ObjOperandId objId, + HandleId key, ValOperandId keyId) +{ + bool hasOwn = (cacheKind_ == CacheKind::HasOwn); + + JSObject* holder = nullptr; + PropertyResult prop; + + if (hasOwn) { + if (!LookupOwnPropertyPure(cx_, obj, key, &prop)) + return false; + + holder = obj; + } else { + if (!LookupPropertyPure(cx_, obj, key, &holder, &prop)) + return false; + } + if (!prop.isFound()) + return false; + + // Use MegamorphicHasOwnResult if applicable + if (hasOwn && mode_ == ICState::Mode::Megamorphic) { + writer.megamorphicHasOwnResult(objId, keyId); + writer.returnFromIC(); + trackAttached("MegamorphicHasProp"); + return true; + } + + Maybe tempId; + emitIdGuard(keyId, key); + EmitReadSlotGuard(writer, obj, holder, objId, &tempId); + writer.loadBooleanResult(true); + writer.returnFromIC(); + + trackAttached("NativeHasProp"); + return true; +} + +bool +HasPropIRGenerator::tryAttachNativeDoesNotExist(HandleObject obj, ObjOperandId objId, + HandleId key, ValOperandId keyId) +{ + bool hasOwn = (cacheKind_ == CacheKind::HasOwn); + + if (hasOwn) { + if (!CheckHasNoSuchOwnProperty(cx_, obj, key)) + return false; + } else { + if (!CheckHasNoSuchProperty(cx_, obj, key)) + return false; + } + + // Use MegamorphicHasOwnResult if applicable + if (hasOwn && mode_ == ICState::Mode::Megamorphic) { + writer.megamorphicHasOwnResult(objId, keyId); + writer.returnFromIC(); + trackAttached("MegamorphicHasOwn"); + return true; + } + + Maybe tempId; + emitIdGuard(keyId, key); + if (hasOwn) { + TestMatchingReceiver(writer, obj, objId, &tempId); + } else { + EmitReadSlotGuard(writer, obj, nullptr, objId, &tempId); + } + writer.loadBooleanResult(false); + writer.returnFromIC(); + + trackAttached("NativeDoesNotExist"); + return true; +} + +bool +HasPropIRGenerator::tryAttachProxyElement(HandleObject obj, ObjOperandId objId, + ValOperandId keyId) +{ + MOZ_ASSERT(cacheKind_ == CacheKind::HasOwn); + + if (!obj->is()) + return false; + + writer.guardIsProxy(objId); + writer.callProxyHasOwnResult(objId, keyId); + writer.returnFromIC(); + + trackAttached("ProxyHasProp"); + return true; +} + +bool +HasPropIRGenerator::tryAttachStub() +{ + MOZ_ASSERT(cacheKind_ == CacheKind::In || + cacheKind_ == CacheKind::HasOwn); + + AutoAssertNoPendingException aanpe(cx_); + + // NOTE: Argument order is PROPERTY, OBJECT + ValOperandId keyId(writer.setInputOperandId(0)); + ValOperandId valId(writer.setInputOperandId(1)); + + if (!val_.isObject()) { + trackNotAttached(); + return false; + } + RootedObject obj(cx_, &val_.toObject()); + ObjOperandId objId = writer.guardIsObject(valId); + + // Optimize DOM Proxies for JSOP_HASOWN + if (cacheKind_ == CacheKind::HasOwn) { + if (tryAttachProxyElement(obj, objId, keyId)) + return true; + } + + RootedId id(cx_); + bool nameOrSymbol; + if (!ValueToNameOrSymbolId(cx_, idVal_, &id, &nameOrSymbol)) { + cx_->clearPendingException(); + return false; + } + + if (nameOrSymbol) { + if (tryAttachNative(obj, objId, id, keyId)) + return true; + if (tryAttachNativeDoesNotExist(obj, objId, id, keyId)) + return true; + + trackNotAttached(); + return false; + } + + uint32_t index; + Int32OperandId indexId; + if (maybeGuardInt32Index(idVal_, keyId, &index, &indexId)) { + if (tryAttachDense(obj, objId, index, indexId)) + return true; + if (tryAttachDenseHole(obj, objId, index, indexId)) + return true; + + trackNotAttached(); + return false; + } + + trackNotAttached(); + return false; +} + +void +HasPropIRGenerator::trackAttached(const char* name) +{ +#ifdef JS_CACHEIR_SPEW + CacheIRSpewer& sp = CacheIRSpewer::singleton(); + if (sp.enabled()) { + LockGuard guard(sp.lock()); + sp.beginCache(guard, *this); + RootedValue objV(cx_, ObjectValue(*obj_)); + sp.valueProperty(guard, "base", val_); + sp.valueProperty(guard, "property", idVal_); + sp.attached(guard, name); + sp.endCache(guard); + } +#endif +} + +void +HasPropIRGenerator::trackNotAttached() +{ +#ifdef JS_CACHEIR_SPEW + CacheIRSpewer& sp = CacheIRSpewer::singleton(); + if (sp.enabled()) { + LockGuard guard(sp.lock()); + sp.beginCache(guard, *this); + sp.valueProperty(guard, "base", val_); + sp.valueProperty(guard, "property", idVal_); + sp.endCache(guard); + } +#endif +} + +bool +IRGenerator::maybeGuardInt32Index(const Value& index, ValOperandId indexId, + uint32_t* int32Index, Int32OperandId* int32IndexId) +{ + if (index.isNumber()) { + int32_t indexSigned; + if (index.isInt32()) { + indexSigned = index.toInt32(); + } else { + // We allow negative zero here. + if (!mozilla::NumberEqualsInt32(index.toDouble(), &indexSigned)) + return false; + if (!cx_->runtime()->jitSupportsFloatingPoint) + return false; + } + + if (indexSigned < 0) + return false; + + *int32Index = uint32_t(indexSigned); + *int32IndexId = writer.guardIsInt32Index(indexId); + return true; + } + + if (index.isString()) { + int32_t indexSigned = GetIndexFromString(index.toString()); + if (indexSigned < 0) + return false; + + StringOperandId strId = writer.guardIsString(indexId); + *int32Index = uint32_t(indexSigned); + *int32IndexId = writer.guardAndGetIndexFromString(strId); + return true; + } + + return false; +} + +SetPropIRGenerator::SetPropIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc, + CacheKind cacheKind, ICState::Mode mode, + bool* isTemporarilyUnoptimizable, + HandleValue lhsVal, HandleValue idVal, HandleValue rhsVal, + bool needsTypeBarrier, bool maybeHasExtraIndexedProps) + : IRGenerator(cx, script, pc, cacheKind, mode), + lhsVal_(lhsVal), + idVal_(idVal), + rhsVal_(rhsVal), + isTemporarilyUnoptimizable_(isTemporarilyUnoptimizable), + typeCheckInfo_(cx, needsTypeBarrier), + preliminaryObjectAction_(PreliminaryObjectAction::None), + attachedTypedArrayOOBStub_(false), + maybeHasExtraIndexedProps_(maybeHasExtraIndexedProps) +{} + +bool +SetPropIRGenerator::tryAttachStub() +{ + AutoAssertNoPendingException aanpe(cx_); + + ValOperandId objValId(writer.setInputOperandId(0)); + ValOperandId rhsValId; + if (cacheKind_ == CacheKind::SetProp) { + rhsValId = ValOperandId(writer.setInputOperandId(1)); + } else { + MOZ_ASSERT(cacheKind_ == CacheKind::SetElem); + MOZ_ASSERT(setElemKeyValueId().id() == 1); + writer.setInputOperandId(1); + rhsValId = ValOperandId(writer.setInputOperandId(2)); + } + + RootedId id(cx_); + bool nameOrSymbol; + if (!ValueToNameOrSymbolId(cx_, idVal_, &id, &nameOrSymbol)) { + cx_->clearPendingException(); + return false; + } + + if (lhsVal_.isObject()) { + RootedObject obj(cx_, &lhsVal_.toObject()); + + ObjOperandId objId = writer.guardIsObject(objValId); + if (nameOrSymbol) { + if (tryAttachNativeSetSlot(obj, objId, id, rhsValId)) + return true; + if (tryAttachUnboxedExpandoSetSlot(obj, objId, id, rhsValId)) + return true; + if (tryAttachUnboxedProperty(obj, objId, id, rhsValId)) + return true; + if (tryAttachTypedObjectProperty(obj, objId, id, rhsValId)) + return true; + if (IsPropertySetOp(JSOp(*pc_))) { + if (tryAttachSetArrayLength(obj, objId, id, rhsValId)) + return true; + if (tryAttachSetter(obj, objId, id, rhsValId)) + return true; + if (tryAttachWindowProxy(obj, objId, id, rhsValId)) + return true; + if (tryAttachProxy(obj, objId, id, rhsValId)) + return true; + } + return false; + } + + if (IsPropertySetOp(JSOp(*pc_))) { + if (tryAttachProxyElement(obj, objId, rhsValId)) + return true; + } + uint32_t index; + Int32OperandId indexId; + if (maybeGuardInt32Index(idVal_, setElemKeyValueId(), &index, &indexId)) { + if (tryAttachSetDenseElement(obj, objId, index, indexId, rhsValId)) + return true; + if (tryAttachSetDenseElementHole(obj, objId, index, indexId, rhsValId)) + return true; + if (tryAttachSetUnboxedArrayElement(obj, objId, index, indexId, rhsValId)) + return true; + if (tryAttachSetUnboxedArrayElementHole(obj, objId, index, indexId, rhsValId)) + return true; + if (tryAttachSetTypedElement(obj, objId, index, indexId, rhsValId)) + return true; + return false; + } + return false; + } + + return false; +} + +static void +EmitStoreSlotAndReturn(CacheIRWriter& writer, ObjOperandId objId, NativeObject* nobj, Shape* shape, + ValOperandId rhsId) +{ + if (nobj->isFixedSlot(shape->slot())) { + size_t offset = NativeObject::getFixedSlotOffset(shape->slot()); + writer.storeFixedSlot(objId, offset, rhsId); + } else { + size_t offset = nobj->dynamicSlotIndex(shape->slot()) * sizeof(Value); + writer.storeDynamicSlot(objId, offset, rhsId); + } + writer.returnFromIC(); +} + +static Shape* +LookupShapeForSetSlot(JSOp op, NativeObject* obj, jsid id) +{ + Shape* shape = obj->lookupPure(id); + if (!shape || !shape->hasSlot() || !shape->hasDefaultSetter() || !shape->writable()) + return nullptr; + + // If this is an op like JSOP_INITELEM / [[DefineOwnProperty]], the + // property's attributes may have to be changed too, so make sure it's a + // simple data property. + if (IsPropertyInitOp(op) && (!shape->configurable() || + !shape->enumerable() || + !shape->hasDefaultGetter())) + { + return nullptr; + } + + return shape; +} + +static bool +CanAttachNativeSetSlot(JSContext* cx, JSOp op, HandleObject obj, HandleId id, + bool* isTemporarilyUnoptimizable, MutableHandleShape propShape) +{ + if (!obj->isNative()) + return false; + + propShape.set(LookupShapeForSetSlot(op, &obj->as(), id)); + if (!propShape) + return false; + + ObjectGroup* group = JSObject::getGroup(cx, obj); + if (!group) { + cx->recoverFromOutOfMemory(); + return false; + } + + // For some property writes, such as the initial overwrite of global + // properties, TI will not mark the property as having been + // overwritten. Don't attach a stub in this case, so that we don't + // execute another write to the property without TI seeing that write. + EnsureTrackPropertyTypes(cx, obj, id); + if (!PropertyHasBeenMarkedNonConstant(obj, id)) { + *isTemporarilyUnoptimizable = true; + return false; + } + + return true; +} + +bool +SetPropIRGenerator::tryAttachNativeSetSlot(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + RootedShape propShape(cx_); + if (!CanAttachNativeSetSlot(cx_, JSOp(*pc_), obj, id, isTemporarilyUnoptimizable_, &propShape)) + return false; + + if (mode_ == ICState::Mode::Megamorphic && cacheKind_ == CacheKind::SetProp) { + writer.megamorphicStoreSlot(objId, JSID_TO_ATOM(id)->asPropertyName(), rhsId, + typeCheckInfo_.needsTypeBarrier()); + writer.returnFromIC(); + trackAttached("MegamorphicNativeSlot"); + return true; + } + + maybeEmitIdGuard(id); + + // If we need a property type barrier (always in Baseline, sometimes in + // Ion), guard on both the shape and the group. If Ion knows the property + // types match, we don't need the group guard. + NativeObject* nobj = &obj->as(); + if (typeCheckInfo_.needsTypeBarrier()) + writer.guardGroup(objId, nobj->group()); + writer.guardShape(objId, nobj->lastProperty()); + + if (IsPreliminaryObject(obj)) + preliminaryObjectAction_ = PreliminaryObjectAction::NotePreliminary; + else + preliminaryObjectAction_ = PreliminaryObjectAction::Unlink; + + typeCheckInfo_.set(nobj->group(), id); + EmitStoreSlotAndReturn(writer, objId, nobj, propShape, rhsId); + + trackAttached("NativeSlot"); + return true; +} + +bool +SetPropIRGenerator::tryAttachUnboxedExpandoSetSlot(HandleObject obj, ObjOperandId objId, + HandleId id, ValOperandId rhsId) +{ + if (!obj->is()) + return false; + + UnboxedExpandoObject* expando = obj->as().maybeExpando(); + if (!expando) + return false; + + Shape* propShape = LookupShapeForSetSlot(JSOp(*pc_), expando, id); + if (!propShape) + return false; + + maybeEmitIdGuard(id); + writer.guardGroup(objId, obj->group()); + ObjOperandId expandoId = writer.guardAndLoadUnboxedExpando(objId); + writer.guardShape(expandoId, expando->lastProperty()); + + // Property types must be added to the unboxed object's group, not the + // expando's group (it has unknown properties). + typeCheckInfo_.set(obj->group(), id); + EmitStoreSlotAndReturn(writer, expandoId, expando, propShape, rhsId); + + trackAttached("UnboxedExpando"); + return true; +} + +static void +EmitGuardUnboxedPropertyType(CacheIRWriter& writer, JSValueType propType, ValOperandId valId) +{ + if (propType == JSVAL_TYPE_OBJECT) { + // Unboxed objects store NullValue as nullptr object. + writer.guardIsObjectOrNull(valId); + } else { + writer.guardType(valId, propType); + } +} + +bool +SetPropIRGenerator::tryAttachUnboxedProperty(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + if (!obj->is() || !cx_->runtime()->jitSupportsFloatingPoint) + return false; + + const UnboxedLayout::Property* property = obj->as().layout().lookup(id); + if (!property) + return false; + + maybeEmitIdGuard(id); + writer.guardGroup(objId, obj->group()); + EmitGuardUnboxedPropertyType(writer, property->type, rhsId); + writer.storeUnboxedProperty(objId, property->type, + UnboxedPlainObject::offsetOfData() + property->offset, + rhsId); + writer.returnFromIC(); + + typeCheckInfo_.set(obj->group(), id); + preliminaryObjectAction_ = PreliminaryObjectAction::Unlink; + + trackAttached("Unboxed"); + return true; +} + +bool +SetPropIRGenerator::tryAttachTypedObjectProperty(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + if (!obj->is() || !cx_->runtime()->jitSupportsFloatingPoint) + return false; + + if (cx_->compartment()->detachedTypedObjects) + return false; + + if (!obj->as().typeDescr().is()) + return false; + + StructTypeDescr* structDescr = &obj->as().typeDescr().as(); + size_t fieldIndex; + if (!structDescr->fieldIndex(id, &fieldIndex)) + return false; + + TypeDescr* fieldDescr = &structDescr->fieldDescr(fieldIndex); + if (!fieldDescr->is()) + return false; + + uint32_t fieldOffset = structDescr->fieldOffset(fieldIndex); + TypedThingLayout layout = GetTypedThingLayout(obj->getClass()); + + maybeEmitIdGuard(id); + writer.guardNoDetachedTypedObjects(); + writer.guardShape(objId, obj->as().shape()); + writer.guardGroup(objId, obj->group()); + + typeCheckInfo_.set(obj->group(), id); + + // Scalar types can always be stored without a type update stub. + if (fieldDescr->is()) { + Scalar::Type type = fieldDescr->as().type(); + writer.storeTypedObjectScalarProperty(objId, fieldOffset, layout, type, rhsId); + writer.returnFromIC(); + + trackAttached("TypedObject"); + return true; + } + + // For reference types, guard on the RHS type first, so that + // StoreTypedObjectReferenceProperty is infallible. + ReferenceTypeDescr::Type type = fieldDescr->as().type(); + switch (type) { + case ReferenceTypeDescr::TYPE_ANY: + break; + case ReferenceTypeDescr::TYPE_OBJECT: + writer.guardIsObjectOrNull(rhsId); + break; + case ReferenceTypeDescr::TYPE_STRING: + writer.guardType(rhsId, JSVAL_TYPE_STRING); + break; + } + + writer.storeTypedObjectReferenceProperty(objId, fieldOffset, layout, type, rhsId); + writer.returnFromIC(); + + trackAttached("TypedObject"); + return true; +} + +void +SetPropIRGenerator::trackAttached(const char* name) +{ +#ifdef JS_CACHEIR_SPEW + CacheIRSpewer& sp = CacheIRSpewer::singleton(); + if (sp.enabled()) { + LockGuard guard(sp.lock()); + sp.beginCache(guard, *this); + sp.valueProperty(guard, "base", lhsVal_); + sp.valueProperty(guard, "property", idVal_); + sp.valueProperty(guard, "value", rhsVal_); + sp.attached(guard, name); + sp.endCache(guard); + } +#endif +} + +void +SetPropIRGenerator::trackNotAttached() +{ +#ifdef JS_CACHEIR_SPEW + CacheIRSpewer& sp = CacheIRSpewer::singleton(); + if (sp.enabled()) { + LockGuard guard(sp.lock()); + sp.beginCache(guard, *this); + sp.valueProperty(guard, "base", lhsVal_); + sp.valueProperty(guard, "property", idVal_); + sp.valueProperty(guard, "value", rhsVal_); + sp.endCache(guard); + } +#endif +} + +static bool +CanAttachSetter(JSContext* cx, jsbytecode* pc, HandleObject obj, HandleId id, + MutableHandleObject holder, MutableHandleShape propShape, + bool* isTemporarilyUnoptimizable) +{ + // Don't attach a setter stub for ops like JSOP_INITELEM. + MOZ_ASSERT(IsPropertySetOp(JSOp(*pc))); + + PropertyResult prop; + if (!LookupPropertyPure(cx, obj, id, holder.address(), &prop)) + return false; + + if (prop.isNonNativeProperty()) + return false; + + propShape.set(prop.maybeShape()); + if (!IsCacheableSetPropCallScripted(obj, holder, propShape, isTemporarilyUnoptimizable) && + !IsCacheableSetPropCallNative(obj, holder, propShape)) + { + return false; + } + + return true; +} + +static void +EmitCallSetterNoGuards(CacheIRWriter& writer, JSObject* obj, JSObject* holder, + Shape* shape, ObjOperandId objId, ValOperandId rhsId) +{ + if (IsCacheableSetPropCallNative(obj, holder, shape)) { + JSFunction* target = &shape->setterValue().toObject().as(); + MOZ_ASSERT(target->isNative()); + writer.callNativeSetter(objId, target, rhsId); + writer.returnFromIC(); + return; + } + + MOZ_ASSERT(IsCacheableSetPropCallScripted(obj, holder, shape)); + + JSFunction* target = &shape->setterValue().toObject().as(); + MOZ_ASSERT(target->hasJITCode()); + writer.callScriptedSetter(objId, target, rhsId); + writer.returnFromIC(); +} + +bool +SetPropIRGenerator::tryAttachSetter(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + RootedObject holder(cx_); + RootedShape propShape(cx_); + if (!CanAttachSetter(cx_, pc_, obj, id, &holder, &propShape, isTemporarilyUnoptimizable_)) + return false; + + maybeEmitIdGuard(id); + + // Use the megamorphic guard if we're in megamorphic mode, except if |obj| + // is a Window as GuardHasGetterSetter doesn't support this yet (Window may + // require outerizing). + if (mode_ == ICState::Mode::Specialized || IsWindow(obj)) { + Maybe expandoId; + TestMatchingReceiver(writer, obj, objId, &expandoId); + + if (obj != holder) { + GeneratePrototypeGuards(writer, obj, holder, objId); + + // Guard on the holder's shape. + ObjOperandId holderId = writer.loadObject(holder); + writer.guardShape(holderId, holder->as().lastProperty()); + } + } else { + writer.guardHasGetterSetter(objId, propShape); + } + + EmitCallSetterNoGuards(writer, obj, holder, propShape, objId, rhsId); + + trackAttached("Setter"); + return true; +} + +bool +SetPropIRGenerator::tryAttachSetArrayLength(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + // Don't attach an array length stub for ops like JSOP_INITELEM. + MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_))); + + if (!obj->is() || + !JSID_IS_ATOM(id, cx_->names().length) || + !obj->as().lengthIsWritable()) + { + return false; + } + + maybeEmitIdGuard(id); + writer.guardClass(objId, GuardClassKind::Array); + writer.callSetArrayLength(objId, IsStrictSetPC(pc_), rhsId); + writer.returnFromIC(); + + trackAttached("SetArrayLength"); + return true; +} + +bool +SetPropIRGenerator::tryAttachSetDenseElement(HandleObject obj, ObjOperandId objId, uint32_t index, + Int32OperandId indexId, ValOperandId rhsId) +{ + if (!obj->isNative()) + return false; + + NativeObject* nobj = &obj->as(); + if (!nobj->containsDenseElement(index) || nobj->getElementsHeader()->isFrozen()) + return false; + + if (typeCheckInfo_.needsTypeBarrier()) + writer.guardGroup(objId, nobj->group()); + writer.guardShape(objId, nobj->shape()); + + writer.storeDenseElement(objId, indexId, rhsId); + writer.returnFromIC(); + + // Type inference uses JSID_VOID for the element types. + typeCheckInfo_.set(nobj->group(), JSID_VOID); + + trackAttached("SetDenseElement"); + return true; +} + +static bool +CanAttachAddElement(JSObject* obj, bool isInit) +{ + // Make sure the objects on the prototype don't have any indexed properties + // or that such properties can't appear without a shape change. + do { + // The first two checks are also relevant to the receiver object. + if (obj->isIndexed()) + return false; + + const Class* clasp = obj->getClass(); + if ((clasp != &ArrayObject::class_ && clasp != &UnboxedArrayObject::class_) && + (clasp->getAddProperty() || + clasp->getResolve() || + clasp->getOpsLookupProperty() || + clasp->getSetProperty() || + clasp->getOpsSetProperty())) + { + return false; + } + + // If we're initializing a property instead of setting one, the objects + // on the prototype are not relevant. + if (isInit) + break; + + JSObject* proto = obj->staticPrototype(); + if (!proto) + break; + + if (!proto->isNative()) + return false; + + obj = proto; + } while (true); + + return true; +} + +static void +ShapeGuardProtoChain(CacheIRWriter& writer, JSObject* obj, ObjOperandId objId) +{ + while (true) { + // Guard on the proto if the shape does not imply the proto. Singleton + // objects always trigger a shape change when the proto changes, so we + // don't need a guard in that case. + bool guardProto = obj->hasUncacheableProto() && !obj->isSingleton(); + + obj = obj->staticPrototype(); + if (!obj) + return; + + objId = writer.loadProto(objId); + if (guardProto) + writer.guardSpecificObject(objId, obj); + writer.guardShape(objId, obj->as().shape()); + } +} + + +bool +SetPropIRGenerator::tryAttachSetDenseElementHole(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId, + ValOperandId rhsId) +{ + if (!obj->isNative() || rhsVal_.isMagic(JS_ELEMENTS_HOLE)) + return false; + + JSOp op = JSOp(*pc_); + MOZ_ASSERT(IsPropertySetOp(op) || IsPropertyInitOp(op)); + + if (op == JSOP_INITHIDDENELEM) + return false; + + NativeObject* nobj = &obj->as(); + if (!nobj->nonProxyIsExtensible()) + return false; + + MOZ_ASSERT(!nobj->getElementsHeader()->isFrozen(), + "Extensible objects should not have frozen elements"); + + uint32_t initLength = nobj->getDenseInitializedLength(); + + // Optimize if we're adding an element at initLength or writing to a hole. + // Don't handle the adding case if the current accesss is in bounds, to + // ensure we always call noteArrayWriteHole. + bool isAdd = index == initLength; + bool isHoleInBounds = index < initLength && !nobj->containsDenseElement(index); + if (!isAdd && !isHoleInBounds) + return false; + + // Can't add new elements to arrays with non-writable length. + if (isAdd && nobj->is() && !nobj->as().lengthIsWritable()) + return false; + + // Typed arrays don't have dense elements. + if (nobj->is()) + return false; + + // Check for other indexed properties or class hooks. + if (!CanAttachAddElement(nobj, IsPropertyInitOp(op))) + return false; + + if (typeCheckInfo_.needsTypeBarrier()) + writer.guardGroup(objId, nobj->group()); + writer.guardShape(objId, nobj->shape()); + + // Also shape guard the proto chain, unless this is an INITELEM. + if (IsPropertySetOp(op)) + ShapeGuardProtoChain(writer, obj, objId); + + writer.storeDenseElementHole(objId, indexId, rhsId, isAdd); + writer.returnFromIC(); + + // Type inference uses JSID_VOID for the element types. + typeCheckInfo_.set(nobj->group(), JSID_VOID); + + trackAttached(isAdd ? "AddDenseElement" : "StoreDenseElementHole"); + return true; +} + +bool +SetPropIRGenerator::tryAttachSetUnboxedArrayElement(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId, + ValOperandId rhsId) +{ + if (!obj->is()) + return false; + + if (!cx_->runtime()->jitSupportsFloatingPoint) + return false; + + if (index >= obj->as().initializedLength()) + return false; + + writer.guardGroup(objId, obj->group()); + + JSValueType elementType = obj->group()->unboxedLayoutDontCheckGeneration().elementType(); + EmitGuardUnboxedPropertyType(writer, elementType, rhsId); + + writer.storeUnboxedArrayElement(objId, indexId, rhsId, elementType); + writer.returnFromIC(); + + // Type inference uses JSID_VOID for the element types. + typeCheckInfo_.set(obj->group(), JSID_VOID); + + trackAttached("SetUnboxedArrayElement"); + return true; +} + +bool +SetPropIRGenerator::tryAttachSetTypedElement(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId, + ValOperandId rhsId) +{ + if (!obj->is() && !IsPrimitiveArrayTypedObject(obj)) + return false; + + if (!rhsVal_.isNumber()) + return false; + + if (!cx_->runtime()->jitSupportsFloatingPoint && TypedThingRequiresFloatingPoint(obj)) + return false; + + bool handleOutOfBounds = false; + if (obj->is()) { + handleOutOfBounds = (index >= obj->as().length()); + } else { + // Typed objects throw on out of bounds accesses. Don't attach + // a stub in this case. + if (index >= obj->as().length()) + return false; + + // Don't attach stubs if the underlying storage for typed objects + // in the compartment could be detached, as the stub will always + // bail out. + if (cx_->compartment()->detachedTypedObjects) + return false; + } + + Scalar::Type elementType = TypedThingElementType(obj); + TypedThingLayout layout = GetTypedThingLayout(obj->getClass()); + + if (!obj->is()) + writer.guardNoDetachedTypedObjects(); + + writer.guardShape(objId, obj->as().shape()); + writer.storeTypedElement(objId, indexId, rhsId, layout, elementType, handleOutOfBounds); + writer.returnFromIC(); + + if (handleOutOfBounds) + attachedTypedArrayOOBStub_ = true; + + trackAttached(handleOutOfBounds ? "SetTypedElementOOB" : "SetTypedElement"); + return true; +} + +bool +SetPropIRGenerator::tryAttachSetUnboxedArrayElementHole(HandleObject obj, ObjOperandId objId, + uint32_t index, Int32OperandId indexId, + ValOperandId rhsId) +{ + if (!obj->is() || rhsVal_.isMagic(JS_ELEMENTS_HOLE)) + return false; + + if (!cx_->runtime()->jitSupportsFloatingPoint) + return false; + + JSOp op = JSOp(*pc_); + MOZ_ASSERT(IsPropertySetOp(op) || IsPropertyInitOp(op)); + + if (op == JSOP_INITHIDDENELEM) + return false; + + // Optimize if we're adding an element at initLength. Unboxed arrays don't + // have holes at indexes < initLength. + UnboxedArrayObject* aobj = &obj->as(); + if (index != aobj->initializedLength() || index >= aobj->capacity()) + return false; + + // Check for other indexed properties or class hooks. + if (!CanAttachAddElement(aobj, IsPropertyInitOp(op))) + return false; + + writer.guardGroup(objId, aobj->group()); + + JSValueType elementType = aobj->group()->unboxedLayoutDontCheckGeneration().elementType(); + EmitGuardUnboxedPropertyType(writer, elementType, rhsId); + + // Also shape guard the proto chain, unless this is an INITELEM. + if (IsPropertySetOp(op)) + ShapeGuardProtoChain(writer, aobj, objId); + + writer.storeUnboxedArrayElementHole(objId, indexId, rhsId, elementType); + writer.returnFromIC(); + + // Type inference uses JSID_VOID for the element types. + typeCheckInfo_.set(aobj->group(), JSID_VOID); + + trackAttached("StoreUnboxedArrayElementHole"); + return true; +} + +bool +SetPropIRGenerator::tryAttachGenericProxy(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId, bool handleDOMProxies) +{ + MOZ_ASSERT(obj->is()); + + writer.guardIsProxy(objId); + + if (!handleDOMProxies) { + // Ensure that the incoming object is not a DOM proxy, so that we can + // get to the specialized stubs. If handleDOMProxies is true, we were + // unable to attach a specialized DOM stub, so we just handle all + // proxies here. + writer.guardNotDOMProxy(objId); + } + + if (cacheKind_ == CacheKind::SetProp) { + writer.callProxySet(objId, id, rhsId, IsStrictSetPC(pc_)); + } else { + // We could call maybeEmitIdGuard here and then emit CallProxySet, but + // for SetElem we prefer to attach a stub that can handle any Value + // so we don't attach a new stub for every id. + MOZ_ASSERT(cacheKind_ == CacheKind::SetElem); + writer.callProxySetByValue(objId, setElemKeyValueId(), rhsId, IsStrictSetPC(pc_)); + } + + writer.returnFromIC(); + + trackAttached("GenericProxy"); + return true; +} + +bool +SetPropIRGenerator::tryAttachDOMProxyShadowed(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + MOZ_ASSERT(IsCacheableDOMProxy(obj)); + + maybeEmitIdGuard(id); + writer.guardShape(objId, obj->maybeShape()); + + // No need for more guards: we know this is a DOM proxy, since the shape + // guard enforces a given JSClass, so just go ahead and emit the call to + // ProxySet. + writer.callProxySet(objId, id, rhsId, IsStrictSetPC(pc_)); + writer.returnFromIC(); + + trackAttached("DOMProxyShadowed"); + return true; +} + +bool +SetPropIRGenerator::tryAttachDOMProxyUnshadowed(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + MOZ_ASSERT(IsCacheableDOMProxy(obj)); + + RootedObject proto(cx_, obj->staticPrototype()); + if (!proto) + return false; + + RootedObject holder(cx_); + RootedShape propShape(cx_); + if (!CanAttachSetter(cx_, pc_, proto, id, &holder, &propShape, isTemporarilyUnoptimizable_)) + return false; + + maybeEmitIdGuard(id); + writer.guardShape(objId, obj->maybeShape()); + + // Guard that our expando object hasn't started shadowing this property. + CheckDOMProxyExpandoDoesNotShadow(writer, obj, id, objId); + + GeneratePrototypeGuards(writer, obj, holder, objId); + + // Guard on the holder of the property. + ObjOperandId holderId = writer.loadObject(holder); + writer.guardShape(holderId, holder->as().lastProperty()); + + // EmitCallSetterNoGuards expects |obj| to be the object the property is + // on to do some checks. Since we actually looked at proto, and no extra + // guards will be generated, we can just pass that instead. + EmitCallSetterNoGuards(writer, proto, holder, propShape, objId, rhsId); + + trackAttached("DOMProxyUnshadowed"); + return true; +} + +bool +SetPropIRGenerator::tryAttachDOMProxyExpando(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + MOZ_ASSERT(IsCacheableDOMProxy(obj)); + + RootedValue expandoVal(cx_, GetProxyPrivate(obj)); + RootedObject expandoObj(cx_); + if (expandoVal.isObject()) { + expandoObj = &expandoVal.toObject(); + } else { + MOZ_ASSERT(!expandoVal.isUndefined(), + "How did a missing expando manage to shadow things?"); + auto expandoAndGeneration = static_cast(expandoVal.toPrivate()); + MOZ_ASSERT(expandoAndGeneration); + expandoObj = &expandoAndGeneration->expando.toObject(); + } + + RootedShape propShape(cx_); + if (CanAttachNativeSetSlot(cx_, JSOp(*pc_), expandoObj, id, isTemporarilyUnoptimizable_, + &propShape)) + { + maybeEmitIdGuard(id); + ObjOperandId expandoObjId = + guardDOMProxyExpandoObjectAndShape(obj, objId, expandoVal, expandoObj); + + NativeObject* nativeExpandoObj = &expandoObj->as(); + writer.guardGroup(expandoObjId, nativeExpandoObj->group()); + typeCheckInfo_.set(nativeExpandoObj->group(), id); + + EmitStoreSlotAndReturn(writer, expandoObjId, nativeExpandoObj, propShape, rhsId); + trackAttached("DOMProxyExpandoSlot"); + return true; + } + + RootedObject holder(cx_); + if (CanAttachSetter(cx_, pc_, expandoObj, id, &holder, &propShape, + isTemporarilyUnoptimizable_)) + { + // Note that we don't actually use the expandoObjId here after the + // shape guard. The DOM proxy (objId) is passed to the setter as + // |this|. + maybeEmitIdGuard(id); + guardDOMProxyExpandoObjectAndShape(obj, objId, expandoVal, expandoObj); + + MOZ_ASSERT(holder == expandoObj); + EmitCallSetterNoGuards(writer, expandoObj, expandoObj, propShape, objId, rhsId); + trackAttached("DOMProxyExpandoSetter"); + return true; + } + + return false; +} + +bool +SetPropIRGenerator::tryAttachProxy(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + // Don't attach a proxy stub for ops like JSOP_INITELEM. + MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_))); + + switch (GetProxyStubType(cx_, obj, id)) { + case ProxyStubType::None: + return false; + case ProxyStubType::DOMExpando: + if (tryAttachDOMProxyExpando(obj, objId, id, rhsId)) + return true; + if (*isTemporarilyUnoptimizable_) { + // Scripted setter without JIT code. Just wait. + return false; + } + [[fallthrough]]; // Fall through to the generic shadowed case. + case ProxyStubType::DOMShadowed: + return tryAttachDOMProxyShadowed(obj, objId, id, rhsId); + case ProxyStubType::DOMUnshadowed: + if (tryAttachDOMProxyUnshadowed(obj, objId, id, rhsId)) + return true; + return tryAttachGenericProxy(obj, objId, id, rhsId, /* handleDOMProxies = */ true); + case ProxyStubType::Generic: + return tryAttachGenericProxy(obj, objId, id, rhsId, /* handleDOMProxies = */ false); + } + + MOZ_CRASH("Unexpected ProxyStubType"); +} + +bool +SetPropIRGenerator::tryAttachProxyElement(HandleObject obj, ObjOperandId objId, ValOperandId rhsId) +{ + // Don't attach a proxy stub for ops like JSOP_INITELEM. + MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_))); + + if (!obj->is()) + return false; + + writer.guardIsProxy(objId); + + // Like GetPropIRGenerator::tryAttachProxyElement, don't check for DOM + // proxies here as we don't have specialized DOM stubs for this. + MOZ_ASSERT(cacheKind_ == CacheKind::SetElem); + writer.callProxySetByValue(objId, setElemKeyValueId(), rhsId, IsStrictSetPC(pc_)); + writer.returnFromIC(); + + trackAttached("ProxyElement"); + return true; +} + +bool +SetPropIRGenerator::tryAttachWindowProxy(HandleObject obj, ObjOperandId objId, HandleId id, + ValOperandId rhsId) +{ + // Attach a stub when the receiver is a WindowProxy and we can do the set + // on the Window (the global object). + + if (!IsWindowProxy(obj)) + return false; + + // If we're megamorphic prefer a generic proxy stub that handles a lot more + // cases. + if (mode_ == ICState::Mode::Megamorphic) + return false; + + // This must be a WindowProxy for the current Window/global. Else it would + // be a cross-compartment wrapper and IsWindowProxy returns false for + // those. + MOZ_ASSERT(obj->getClass() == cx_->runtime()->maybeWindowProxyClass()); + MOZ_ASSERT(ToWindowIfWindowProxy(obj) == cx_->global()); + + // Now try to do the set on the Window (the current global). + Handle windowObj = cx_->global(); + + RootedShape propShape(cx_); + if (!CanAttachNativeSetSlot(cx_, JSOp(*pc_), windowObj, id, isTemporarilyUnoptimizable_, + &propShape)) + { + return false; + } + + maybeEmitIdGuard(id); + + writer.guardClass(objId, GuardClassKind::WindowProxy); + ObjOperandId windowObjId = writer.loadObject(windowObj); + + writer.guardShape(windowObjId, windowObj->lastProperty()); + writer.guardGroup(windowObjId, windowObj->group()); + typeCheckInfo_.set(windowObj->group(), id); + + EmitStoreSlotAndReturn(writer, windowObjId, windowObj, propShape, rhsId); + + trackAttached("WindowProxySlot"); + return true; +} + +bool +SetPropIRGenerator::tryAttachAddSlotStub(HandleObjectGroup oldGroup, HandleShape oldShape) +{ + AutoAssertNoPendingException aanpe(cx_); + + ValOperandId objValId(writer.setInputOperandId(0)); + ValOperandId rhsValId; + if (cacheKind_ == CacheKind::SetProp) { + rhsValId = ValOperandId(writer.setInputOperandId(1)); + } else { + MOZ_ASSERT(cacheKind_ == CacheKind::SetElem); + MOZ_ASSERT(setElemKeyValueId().id() == 1); + writer.setInputOperandId(1); + rhsValId = ValOperandId(writer.setInputOperandId(2)); + } + + RootedId id(cx_); + bool nameOrSymbol; + if (!ValueToNameOrSymbolId(cx_, idVal_, &id, &nameOrSymbol)) { + cx_->clearPendingException(); + return false; + } + + if (!lhsVal_.isObject() || !nameOrSymbol) + return false; + + RootedObject obj(cx_, &lhsVal_.toObject()); + + PropertyResult prop; + JSObject* holder; + if (!LookupPropertyPure(cx_, obj, id, &holder, &prop)) + return false; + if (obj != holder) + return false; + + Shape* propShape = nullptr; + NativeObject* holderOrExpando = nullptr; + + if (obj->isNative()) { + propShape = prop.shape(); + holderOrExpando = &obj->as(); + } else { + if (!obj->is()) + return false; + UnboxedExpandoObject* expando = obj->as().maybeExpando(); + if (!expando) + return false; + propShape = expando->lookupPure(id); + if (!propShape) + return false; + holderOrExpando = expando; + } + + MOZ_ASSERT(propShape); + + // The property must be the last added property of the object. + if (holderOrExpando->lastProperty() != propShape) + return false; + + // Object must be extensible, oldShape must be immediate parent of + // current shape. + if (!obj->nonProxyIsExtensible() || propShape->previous() != oldShape) + return false; + + // Basic shape checks. + if (propShape->inDictionary() || + !propShape->hasSlot() || + !propShape->hasDefaultSetter() || + !propShape->writable()) + { + return false; + } + + // Watch out for resolve hooks. + if (ClassMayResolveId(cx_->names(), obj->getClass(), id, obj)) { + // The JSFunction resolve hook defines a (non-configurable and + // non-enumerable) |prototype| property on certain functions. Scripts + // often assign a custom |prototype| object and we want to optimize + // this |prototype| set and eliminate the default object allocation. + // + // We check group->maybeInterpretedFunction() here and guard on the + // group. The group is unique for a particular function so this ensures + // we don't add the default prototype property to functions that don't + // have it. + if (!obj->is() || + !JSID_IS_ATOM(id, cx_->names().prototype) || + !oldGroup->maybeInterpretedFunction() || + !obj->as().needsPrototypeProperty()) + { + return false; + } + MOZ_ASSERT(!propShape->configurable()); + MOZ_ASSERT(!propShape->enumerable()); + } + + // Also watch out for addProperty hooks. Ignore the Array addProperty hook, + // because it doesn't do anything for non-index properties. + DebugOnly index; + MOZ_ASSERT_IF(obj->is(), !IdIsIndex(id, &index)); + if (!obj->is() && obj->getClass()->getAddProperty()) + return false; + + // Walk up the object prototype chain and ensure that all prototypes are + // native, and that all prototypes have no setter defined on the property. + for (JSObject* proto = obj->staticPrototype(); proto; proto = proto->staticPrototype()) { + if (!proto->isNative()) + return false; + + // If prototype defines this property in a non-plain way, don't optimize. + Shape* protoShape = proto->as().lookup(cx_, id); + if (protoShape && !protoShape->hasDefaultSetter()) + return false; + + // Otherwise, if there's no such property, watch out for a resolve hook + // that would need to be invoked and thus prevent inlining of property + // addition. Allow the JSFunction resolve hook as it only defines plain + // data properties and we don't need to invoke it for objects on the + // proto chain. + if (ClassMayResolveId(cx_->names(), proto->getClass(), id, proto) && + !proto->is()) + { + return false; + } + } + + ObjOperandId objId = writer.guardIsObject(objValId); + maybeEmitIdGuard(id); + writer.guardGroup(objId, oldGroup); + + // If we are adding a property to an object for which the new script + // properties analysis hasn't been performed yet, make sure the stub fails + // after we run the analysis as a group change may be required here. The + // group change is not required for correctness but improves type + // information elsewhere. + if (oldGroup->newScript() && !oldGroup->newScript()->analyzed()) { + writer.guardGroupHasUnanalyzedNewScript(oldGroup); + MOZ_ASSERT(IsPreliminaryObject(obj)); + preliminaryObjectAction_ = PreliminaryObjectAction::NotePreliminary; + } else { + preliminaryObjectAction_ = PreliminaryObjectAction::Unlink; + } + + // Shape guard the holder. + ObjOperandId holderId = objId; + if (!obj->isNative()) { + MOZ_ASSERT(obj->as().maybeExpando()); + holderId = writer.guardAndLoadUnboxedExpando(objId); + } + writer.guardShape(holderId, oldShape); + + ShapeGuardProtoChain(writer, obj, objId); + + ObjectGroup* newGroup = obj->group(); + + // Check if we have to change the object's group. If we're adding an + // unboxed expando property, we pass the expando object to AddAndStore*Slot. + // That's okay because we only have to do a group change if the object is a + // PlainObject. + bool changeGroup = oldGroup != newGroup; + MOZ_ASSERT_IF(changeGroup, obj->is()); + + if (holderOrExpando->isFixedSlot(propShape->slot())) { + size_t offset = NativeObject::getFixedSlotOffset(propShape->slot()); + writer.addAndStoreFixedSlot(holderId, offset, rhsValId, propShape, + changeGroup, newGroup); + trackAttached("AddSlot"); + } else { + size_t offset = holderOrExpando->dynamicSlotIndex(propShape->slot()) * sizeof(Value); + uint32_t numOldSlots = NativeObject::dynamicSlotsCount(oldShape); + uint32_t numNewSlots = NativeObject::dynamicSlotsCount(propShape); + if (numOldSlots == numNewSlots) { + writer.addAndStoreDynamicSlot(holderId, offset, rhsValId, propShape, + changeGroup, newGroup); + trackAttached("AddSlot"); + } else { + MOZ_ASSERT(numNewSlots > numOldSlots); + writer.allocateAndStoreDynamicSlot(holderId, offset, rhsValId, propShape, + changeGroup, newGroup, numNewSlots); + trackAttached("AllocateSlot"); + } + } + writer.returnFromIC(); + + typeCheckInfo_.set(oldGroup, id); + return true; +} + +TypeOfIRGenerator::TypeOfIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc, + ICState::Mode mode, HandleValue value) + : IRGenerator(cx, script, pc, CacheKind::TypeOf, mode), + val_(value) +{ } + +bool +TypeOfIRGenerator::tryAttachStub() +{ + MOZ_ASSERT(cacheKind_ == CacheKind::TypeOf); + + AutoAssertNoPendingException aanpe(cx_); + + ValOperandId valId(writer.setInputOperandId(0)); + + if (tryAttachPrimitive(valId)) + return true; + + MOZ_ALWAYS_TRUE(tryAttachObject(valId)); + return true; +} + +bool +TypeOfIRGenerator::tryAttachPrimitive(ValOperandId valId) +{ + if (!val_.isPrimitive()) + return false; + + writer.guardType(valId, val_.isNumber() ? JSVAL_TYPE_DOUBLE : val_.extractNonDoubleType()); + writer.loadStringResult(TypeName(js::TypeOfValue(val_), cx_->names())); + writer.returnFromIC(); + + return true; +} + +bool +TypeOfIRGenerator::tryAttachObject(ValOperandId valId) +{ + if (!val_.isObject()) + return false; + + ObjOperandId objId = writer.guardIsObject(valId); + writer.loadTypeOfObjectResult(objId); + writer.returnFromIC(); + + return true; +} + +CallIRGenerator::CallIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc, + ICState::Mode mode, uint32_t argc, + HandleValue callee, HandleValue thisval, HandleValueArray args) + : IRGenerator(cx, script, pc, CacheKind::Call, mode), + argc_(argc), + callee_(callee), + thisval_(thisval), + args_(args), + cachedStrategy_() +{ } + +CallIRGenerator::OptStrategy +CallIRGenerator::canOptimize() +{ + // Ensure callee is a function. + if (!callee_.isObject() || !callee_.toObject().is()) + return OptStrategy::None; + + RootedFunction calleeFunc(cx_, &callee_.toObject().as()); + + OptStrategy strategy; + if ((strategy = canOptimizeStringSplit(calleeFunc)) != OptStrategy::None) { + return strategy; + } + + return OptStrategy::None; +} + +CallIRGenerator::OptStrategy +CallIRGenerator::canOptimizeStringSplit(HandleFunction calleeFunc) +{ + if (argc_ != 2 || !args_[0].isString() || !args_[1].isString()) + return OptStrategy::None; + + // Just for now: if they're both atoms, then do not optimize using + // CacheIR and allow the legacy "ConstStringSplit" BaselineIC optimization + // to proceed. + if (args_[0].toString()->isAtom() && args_[1].toString()->isAtom()) + return OptStrategy::None; + + if (!calleeFunc->isNative()) + return OptStrategy::None; + + if (calleeFunc->native() != js::intrinsic_StringSplitString) + return OptStrategy::None; + + return OptStrategy::StringSplit; +} + +bool +CallIRGenerator::tryAttachStringSplit() +{ + // Get the object group to use for this location. + RootedObjectGroup group(cx_, ObjectGroupCompartment::getStringSplitStringGroup(cx_)); + if (!group) { + return false; + } + + AutoAssertNoPendingException aanpe(cx_); + Int32OperandId argcId(writer.setInputOperandId(0)); + + // Ensure argc == 1. + writer.guardSpecificInt32Immediate(argcId, 2); + + // 1 argument only. Stack-layout here is (bottom to top): + // + // 3: Callee + // 2: ThisValue + // 1: Arg0 + // 0: Arg1 <-- Top of stack + + // Ensure callee is an object and is the function that matches the callee optimized + // against during stub generation (i.e. the String_split function object). + ValOperandId calleeValId = writer.loadStackValue(3); + ObjOperandId calleeObjId = writer.guardIsObject(calleeValId); + writer.guardIsNativeFunction(calleeObjId, js::intrinsic_StringSplitString); + + // Ensure arg0 is a string. + ValOperandId arg0ValId = writer.loadStackValue(1); + StringOperandId arg0StrId = writer.guardIsString(arg0ValId); + + // Ensure arg1 is a string. + ValOperandId arg1ValId = writer.loadStackValue(0); + StringOperandId arg1StrId = writer.guardIsString(arg1ValId); + + // Call custom string splitter VM-function. + writer.callStringSplitResult(arg0StrId, arg1StrId, group); + writer.typeMonitorResult(); + + return true; +} + +CallIRGenerator::OptStrategy +CallIRGenerator::getOptStrategy(bool* optimizeAfterCall) +{ + if (!cachedStrategy_) { + cachedStrategy_ = mozilla::Some(canOptimize()); + } + if (optimizeAfterCall != nullptr) { + MOZ_ASSERT(cachedStrategy_.isSome()); + switch (cachedStrategy_.value()) { + case OptStrategy::StringSplit: + *optimizeAfterCall = true; + break; + + default: + *optimizeAfterCall = false; + } + } + return cachedStrategy_.value(); +} + +bool +CallIRGenerator::tryAttachStub() +{ + OptStrategy strategy = getOptStrategy(); + + if (strategy == OptStrategy::StringSplit) { + return tryAttachStringSplit(); + } + + MOZ_ASSERT(strategy == OptStrategy::None); + return false; +} diff --git a/js/src/jit/CacheIRCompiler.cpp b/js/src/jit/CacheIRCompiler.cpp new file mode 100644 index 0000000000..41b1aba3c4 --- /dev/null +++ b/js/src/jit/CacheIRCompiler.cpp @@ -0,0 +1,2463 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/CacheIRCompiler.h" + +#include "jit/IonIC.h" +#include "jit/SharedICHelpers.h" + +#include "jscompartmentinlines.h" + +#include "jit/MacroAssembler-inl.h" +#include "vm/TypeInference-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::Maybe; + +ValueOperand +CacheRegisterAllocator::useValueRegister(MacroAssembler& masm, ValOperandId op) +{ + OperandLocation& loc = operandLocations_[op.id()]; + + switch (loc.kind()) { + case OperandLocation::ValueReg: + currentOpRegs_.add(loc.valueReg()); + return loc.valueReg(); + + case OperandLocation::ValueStack: { + ValueOperand reg = allocateValueRegister(masm); + popValue(masm, &loc, reg); + return reg; + } + + case OperandLocation::BaselineFrame: { + ValueOperand reg = allocateValueRegister(masm); + Address addr = addressOf(masm, loc.baselineFrameSlot()); + masm.loadValue(addr, reg); + loc.setValueReg(reg); + return reg; + } + + case OperandLocation::Constant: { + ValueOperand reg = allocateValueRegister(masm); + masm.moveValue(loc.constant(), reg); + loc.setValueReg(reg); + return reg; + } + + case OperandLocation::PayloadReg: { + // Temporarily add the payload register to currentOpRegs_ so + // allocateValueRegister will stay away from it. + currentOpRegs_.add(loc.payloadReg()); + ValueOperand reg = allocateValueRegister(masm); + masm.tagValue(loc.payloadType(), loc.payloadReg(), reg); + loc.setValueReg(reg); + return reg; + } + + case OperandLocation::PayloadStack: { + ValueOperand reg = allocateValueRegister(masm); + popPayload(masm, &loc, reg.scratchReg()); + masm.tagValue(loc.payloadType(), reg.scratchReg(), reg); + loc.setValueReg(reg); + return reg; + } + + case OperandLocation::DoubleReg: { + ValueOperand reg = allocateValueRegister(masm); + masm.boxDouble(loc.doubleReg(), reg); + loc.setValueReg(reg); + return reg; + } + + case OperandLocation::Uninitialized: + break; + } + + MOZ_CRASH(); +} + +ValueOperand +CacheRegisterAllocator::useFixedValueRegister(MacroAssembler& masm, ValOperandId valId, + ValueOperand reg) +{ + allocateFixedValueRegister(masm, reg); + + OperandLocation& loc = operandLocations_[valId.id()]; + switch (loc.kind()) { + case OperandLocation::ValueReg: + masm.moveValue(loc.valueReg(), reg); + MOZ_ASSERT(!currentOpRegs_.aliases(loc.valueReg()), "Register shouldn't be in use"); + availableRegs_.add(loc.valueReg()); + break; + case OperandLocation::ValueStack: + popValue(masm, &loc, reg); + break; + case OperandLocation::BaselineFrame: { + Address addr = addressOf(masm, loc.baselineFrameSlot()); + masm.loadValue(addr, reg); + break; + } + case OperandLocation::Constant: + masm.moveValue(loc.constant(), reg); + break; + case OperandLocation::PayloadReg: + masm.tagValue(loc.payloadType(), loc.payloadReg(), reg); + MOZ_ASSERT(!currentOpRegs_.has(loc.payloadReg()), "Register shouldn't be in use"); + availableRegs_.add(loc.payloadReg()); + break; + case OperandLocation::PayloadStack: + popPayload(masm, &loc, reg.scratchReg()); + masm.tagValue(loc.payloadType(), reg.scratchReg(), reg); + break; + case OperandLocation::DoubleReg: + masm.boxDouble(loc.doubleReg(), reg); + break; + case OperandLocation::Uninitialized: + MOZ_CRASH(); + } + + loc.setValueReg(reg); + return reg; +} + +Register +CacheRegisterAllocator::useRegister(MacroAssembler& masm, TypedOperandId typedId) +{ + OperandLocation& loc = operandLocations_[typedId.id()]; + switch (loc.kind()) { + case OperandLocation::PayloadReg: + currentOpRegs_.add(loc.payloadReg()); + return loc.payloadReg(); + + case OperandLocation::ValueReg: { + // It's possible the value is still boxed: as an optimization, we unbox + // the first time we use a value as object. + ValueOperand val = loc.valueReg(); + availableRegs_.add(val); + Register reg = val.scratchReg(); + availableRegs_.take(reg); + masm.unboxObject(val, reg); + loc.setPayloadReg(reg, typedId.type()); + currentOpRegs_.add(reg); + return reg; + } + + case OperandLocation::PayloadStack: { + Register reg = allocateRegister(masm); + popPayload(masm, &loc, reg); + return reg; + } + + case OperandLocation::ValueStack: { + // The value is on the stack, but boxed. If it's on top of the stack we + // unbox it and then remove it from the stack, else we just unbox. + Register reg = allocateRegister(masm); + if (loc.valueStack() == stackPushed_) { + masm.unboxObject(Address(masm.getStackPointer(), 0), reg); + masm.addToStackPtr(Imm32(sizeof(js::Value))); + MOZ_ASSERT(stackPushed_ >= sizeof(js::Value)); + stackPushed_ -= sizeof(js::Value); + } else { + MOZ_ASSERT(loc.valueStack() < stackPushed_); + masm.unboxObject(Address(masm.getStackPointer(), stackPushed_ - loc.valueStack()), + reg); + } + loc.setPayloadReg(reg, typedId.type()); + return reg; + } + + case OperandLocation::BaselineFrame: { + Register reg = allocateRegister(masm); + Address addr = addressOf(masm, loc.baselineFrameSlot()); + masm.unboxNonDouble(addr, reg); + loc.setPayloadReg(reg, typedId.type()); + return reg; + }; + + + case OperandLocation::Constant: { + Value v = loc.constant(); + Register reg = allocateRegister(masm); + if (v.isString()) + masm.movePtr(ImmGCPtr(v.toString()), reg); + else if (v.isSymbol()) + masm.movePtr(ImmGCPtr(v.toSymbol()), reg); + else + MOZ_CRASH("Unexpected Value"); + loc.setPayloadReg(reg, v.extractNonDoubleType()); + return reg; + } + + case OperandLocation::DoubleReg: + case OperandLocation::Uninitialized: + break; + } + + MOZ_CRASH(); +} + +ConstantOrRegister +CacheRegisterAllocator::useConstantOrRegister(MacroAssembler& masm, ValOperandId val) +{ + OperandLocation& loc = operandLocations_[val.id()]; + switch (loc.kind()) { + case OperandLocation::Constant: + return loc.constant(); + + case OperandLocation::PayloadReg: + case OperandLocation::PayloadStack: { + JSValueType payloadType = loc.payloadType(); + Register reg = useRegister(masm, TypedOperandId(val, payloadType)); + return TypedOrValueRegister(MIRTypeFromValueType(payloadType), AnyRegister(reg)); + } + + case OperandLocation::ValueReg: + case OperandLocation::ValueStack: + case OperandLocation::BaselineFrame: + return TypedOrValueRegister(useValueRegister(masm, val)); + + case OperandLocation::DoubleReg: + return TypedOrValueRegister(MIRType::Double, AnyRegister(loc.doubleReg())); + + case OperandLocation::Uninitialized: + break; + } + + MOZ_CRASH(); +} + +Register +CacheRegisterAllocator::defineRegister(MacroAssembler& masm, TypedOperandId typedId) +{ + OperandLocation& loc = operandLocations_[typedId.id()]; + MOZ_ASSERT(loc.kind() == OperandLocation::Uninitialized); + + Register reg = allocateRegister(masm); + loc.setPayloadReg(reg, typedId.type()); + return reg; +} + +ValueOperand +CacheRegisterAllocator::defineValueRegister(MacroAssembler& masm, ValOperandId val) +{ + OperandLocation& loc = operandLocations_[val.id()]; + MOZ_ASSERT(loc.kind() == OperandLocation::Uninitialized); + + ValueOperand reg = allocateValueRegister(masm); + loc.setValueReg(reg); + return reg; +} + +void +CacheRegisterAllocator::freeDeadOperandLocations(MacroAssembler& masm) +{ + // See if any operands are dead so we can reuse their registers. Note that + // we skip the input operands, as those are also used by failure paths, and + // we currently don't track those uses. + for (size_t i = writer_.numInputOperands(); i < operandLocations_.length(); i++) { + if (!writer_.operandIsDead(i, currentInstruction_)) + continue; + + OperandLocation& loc = operandLocations_[i]; + switch (loc.kind()) { + case OperandLocation::PayloadReg: + availableRegs_.add(loc.payloadReg()); + break; + case OperandLocation::ValueReg: + availableRegs_.add(loc.valueReg()); + break; + case OperandLocation::PayloadStack: + masm.propagateOOM(freePayloadSlots_.append(loc.payloadStack())); + break; + case OperandLocation::ValueStack: + masm.propagateOOM(freeValueSlots_.append(loc.valueStack())); + break; + case OperandLocation::Uninitialized: + case OperandLocation::BaselineFrame: + case OperandLocation::Constant: + case OperandLocation::DoubleReg: + break; + } + loc.setUninitialized(); + } +} + +void +CacheRegisterAllocator::discardStack(MacroAssembler& masm) +{ + // This should only be called when we are no longer using the operands, + // as we're discarding everything from the native stack. Set all operand + // locations to Uninitialized to catch bugs. + for (size_t i = 0; i < operandLocations_.length(); i++) + operandLocations_[i].setUninitialized(); + + if (stackPushed_ > 0) { + masm.addToStackPtr(Imm32(stackPushed_)); + stackPushed_ = 0; + } + freePayloadSlots_.clear(); + freeValueSlots_.clear(); +} + +Register +CacheRegisterAllocator::allocateRegister(MacroAssembler& masm) +{ + if (availableRegs_.empty()) + freeDeadOperandLocations(masm); + + if (availableRegs_.empty()) { + // Still no registers available, try to spill unused operands to + // the stack. + for (size_t i = 0; i < operandLocations_.length(); i++) { + OperandLocation& loc = operandLocations_[i]; + if (loc.kind() == OperandLocation::PayloadReg) { + Register reg = loc.payloadReg(); + if (currentOpRegs_.has(reg)) + continue; + + spillOperandToStack(masm, &loc); + availableRegs_.add(reg); + break; // We got a register, so break out of the loop. + } + if (loc.kind() == OperandLocation::ValueReg) { + ValueOperand reg = loc.valueReg(); + if (currentOpRegs_.aliases(reg)) + continue; + + spillOperandToStack(masm, &loc); + availableRegs_.add(reg); + break; // Break out of the loop. + } + } + } + + if (availableRegs_.empty() && !availableRegsAfterSpill_.empty()) { + Register reg = availableRegsAfterSpill_.takeAny(); + masm.push(reg); + stackPushed_ += sizeof(uintptr_t); + + masm.propagateOOM(spilledRegs_.append(SpilledRegister(reg, stackPushed_))); + + availableRegs_.add(reg); + } + + // At this point, there must be a free register. + MOZ_RELEASE_ASSERT(!availableRegs_.empty()); + + Register reg = availableRegs_.takeAny(); + currentOpRegs_.add(reg); + return reg; +} + +void +CacheRegisterAllocator::allocateFixedRegister(MacroAssembler& masm, Register reg) +{ + // Fixed registers should be allocated first, to ensure they're + // still available. + MOZ_ASSERT(!currentOpRegs_.has(reg), "Register is in use"); + + freeDeadOperandLocations(masm); + + if (availableRegs_.has(reg)) { + availableRegs_.take(reg); + currentOpRegs_.add(reg); + return; + } + + // The register must be used by some operand. Spill it to the stack. + for (size_t i = 0; i < operandLocations_.length(); i++) { + OperandLocation& loc = operandLocations_[i]; + if (loc.kind() == OperandLocation::PayloadReg) { + if (loc.payloadReg() != reg) + continue; + + spillOperandToStackOrRegister(masm, &loc); + currentOpRegs_.add(reg); + return; + } + if (loc.kind() == OperandLocation::ValueReg) { + if (!loc.valueReg().aliases(reg)) + continue; + + ValueOperand valueReg = loc.valueReg(); + spillOperandToStackOrRegister(masm, &loc); + + availableRegs_.add(valueReg); + availableRegs_.take(reg); + currentOpRegs_.add(reg); + return; + } + } + + MOZ_CRASH("Invalid register"); +} + +void +CacheRegisterAllocator::allocateFixedValueRegister(MacroAssembler& masm, ValueOperand reg) +{ +#ifdef JS_NUNBOX32 + allocateFixedRegister(masm, reg.payloadReg()); + allocateFixedRegister(masm, reg.typeReg()); +#else + allocateFixedRegister(masm, reg.valueReg()); +#endif +} + +ValueOperand +CacheRegisterAllocator::allocateValueRegister(MacroAssembler& masm) +{ +#ifdef JS_NUNBOX32 + Register reg1 = allocateRegister(masm); + Register reg2 = allocateRegister(masm); + return ValueOperand(reg1, reg2); +#else + Register reg = allocateRegister(masm); + return ValueOperand(reg); +#endif +} + +bool +CacheRegisterAllocator::init() +{ + if (!origInputLocations_.resize(writer_.numInputOperands())) + return false; + if (!operandLocations_.resize(writer_.numOperandIds())) + return false; + return true; +} + +void +CacheRegisterAllocator::initAvailableRegsAfterSpill() +{ + // Registers not in availableRegs_ and not used by input operands are + // available after being spilled. + availableRegsAfterSpill_.set() = + GeneralRegisterSet::Intersect(GeneralRegisterSet::Not(availableRegs_.set()), + GeneralRegisterSet::Not(inputRegisterSet())); +} + +void +CacheRegisterAllocator::fixupAliasedInputs(MacroAssembler& masm) +{ + // If IC inputs alias each other, make sure they are stored in different + // locations so we don't have to deal with this complexity in the rest of + // the allocator. + // + // Note that this can happen in IonMonkey with something like |o.foo = o| + // or |o[i] = i|. + + size_t numInputs = writer_.numInputOperands(); + MOZ_ASSERT(origInputLocations_.length() == numInputs); + + for (size_t i = 1; i < numInputs; i++) { + OperandLocation& loc1 = operandLocations_[i]; + if (!loc1.isInRegister()) + continue; + + for (size_t j = 0; j < i; j++) { + OperandLocation& loc2 = operandLocations_[j]; + if (!loc1.aliasesReg(loc2)) + continue; + + // loc1 and loc2 alias so we spill one of them. If one is a + // ValueReg and the other is a PayloadReg, we have to spill the + // PayloadReg: spilling the ValueReg instead would leave its type + // register unallocated on 32-bit platforms. + if (loc1.kind() == OperandLocation::ValueReg) { + MOZ_ASSERT_IF(loc2.kind() == OperandLocation::ValueReg, + loc1 == loc2); + spillOperandToStack(masm, &loc2); + } else { + MOZ_ASSERT(loc1.kind() == OperandLocation::PayloadReg); + spillOperandToStack(masm, &loc1); + break; // Spilled loc1, so nothing else will alias it. + } + } + } +} + +GeneralRegisterSet +CacheRegisterAllocator::inputRegisterSet() const +{ + MOZ_ASSERT(origInputLocations_.length() == writer_.numInputOperands()); + + AllocatableGeneralRegisterSet result; + for (size_t i = 0; i < writer_.numInputOperands(); i++) { + const OperandLocation& loc = operandLocations_[i]; + MOZ_ASSERT(loc == origInputLocations_[i]); + + switch (loc.kind()) { + case OperandLocation::PayloadReg: + result.addUnchecked(loc.payloadReg()); + continue; + case OperandLocation::ValueReg: + result.addUnchecked(loc.valueReg()); + continue; + case OperandLocation::PayloadStack: + case OperandLocation::ValueStack: + case OperandLocation::BaselineFrame: + case OperandLocation::Constant: + case OperandLocation::DoubleReg: + continue; + case OperandLocation::Uninitialized: + break; + } + MOZ_CRASH("Invalid kind"); + } + + return result.set(); +} + +JSValueType +CacheRegisterAllocator::knownType(ValOperandId val) const +{ + const OperandLocation& loc = operandLocations_[val.id()]; + + switch (loc.kind()) { + case OperandLocation::ValueReg: + case OperandLocation::ValueStack: + case OperandLocation::BaselineFrame: + return JSVAL_TYPE_UNKNOWN; + + case OperandLocation::PayloadStack: + case OperandLocation::PayloadReg: + return loc.payloadType(); + + case OperandLocation::Constant: + return loc.constant().isDouble() + ? JSVAL_TYPE_DOUBLE + : loc.constant().extractNonDoubleType(); + + case OperandLocation::DoubleReg: + return JSVAL_TYPE_DOUBLE; + + case OperandLocation::Uninitialized: + break; + } + + MOZ_CRASH("Invalid kind"); +} + +void +CacheRegisterAllocator::initInputLocation(size_t i, const TypedOrValueRegister& reg) +{ + if (reg.hasValue()) { + initInputLocation(i, reg.valueReg()); + } else if (reg.typedReg().isFloat()) { + MOZ_ASSERT(reg.type() == MIRType::Double); + initInputLocation(i, reg.typedReg().fpu()); + } else { + initInputLocation(i, reg.typedReg().gpr(), ValueTypeFromMIRType(reg.type())); + } +} + +void +CacheRegisterAllocator::initInputLocation(size_t i, const ConstantOrRegister& value) +{ + if (value.constant()) + initInputLocation(i, value.value()); + else + initInputLocation(i, value.reg()); +} + +void +CacheRegisterAllocator::spillOperandToStack(MacroAssembler& masm, OperandLocation* loc) +{ + MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end()); + + if (loc->kind() == OperandLocation::ValueReg) { + if (!freeValueSlots_.empty()) { + uint32_t stackPos = freeValueSlots_.popCopy(); + MOZ_ASSERT(stackPos <= stackPushed_); + masm.storeValue(loc->valueReg(), Address(masm.getStackPointer(), + stackPushed_ - stackPos)); + loc->setValueStack(stackPos); + return; + } + stackPushed_ += sizeof(js::Value); + masm.pushValue(loc->valueReg()); + loc->setValueStack(stackPushed_); + return; + } + + MOZ_ASSERT(loc->kind() == OperandLocation::PayloadReg); + + if (!freePayloadSlots_.empty()) { + uint32_t stackPos = freePayloadSlots_.popCopy(); + MOZ_ASSERT(stackPos <= stackPushed_); + masm.storePtr(loc->payloadReg(), Address(masm.getStackPointer(), + stackPushed_ - stackPos)); + loc->setPayloadStack(stackPos, loc->payloadType()); + return; + } + stackPushed_ += sizeof(uintptr_t); + masm.push(loc->payloadReg()); + loc->setPayloadStack(stackPushed_, loc->payloadType()); +} + +void +CacheRegisterAllocator::spillOperandToStackOrRegister(MacroAssembler& masm, OperandLocation* loc) +{ + MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end()); + + // If enough registers are available, use them. + if (loc->kind() == OperandLocation::ValueReg) { + static const size_t BoxPieces = sizeof(Value) / sizeof(uintptr_t); + if (availableRegs_.set().size() >= BoxPieces) { + ValueOperand reg = availableRegs_.takeAnyValue(); + masm.moveValue(loc->valueReg(), reg); + loc->setValueReg(reg); + return; + } + } else { + MOZ_ASSERT(loc->kind() == OperandLocation::PayloadReg); + if (!availableRegs_.empty()) { + Register reg = availableRegs_.takeAny(); + masm.movePtr(loc->payloadReg(), reg); + loc->setPayloadReg(reg, loc->payloadType()); + return; + } + } + + // Not enough registers available, spill to the stack. + spillOperandToStack(masm, loc); +} + +void +CacheRegisterAllocator::popPayload(MacroAssembler& masm, OperandLocation* loc, Register dest) +{ + MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end()); + MOZ_ASSERT(stackPushed_ >= sizeof(uintptr_t)); + + // The payload is on the stack. If it's on top of the stack we can just + // pop it, else we emit a load. + if (loc->payloadStack() == stackPushed_) { + masm.pop(dest); + stackPushed_ -= sizeof(uintptr_t); + } else { + MOZ_ASSERT(loc->payloadStack() < stackPushed_); + masm.loadPtr(Address(masm.getStackPointer(), stackPushed_ - loc->payloadStack()), dest); + masm.propagateOOM(freePayloadSlots_.append(loc->payloadStack())); + } + + loc->setPayloadReg(dest, loc->payloadType()); +} + +void +CacheRegisterAllocator::popValue(MacroAssembler& masm, OperandLocation* loc, ValueOperand dest) +{ + MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end()); + MOZ_ASSERT(stackPushed_ >= sizeof(js::Value)); + + // The Value is on the stack. If it's on top of the stack we can just + // pop it, else we emit a load. + if (loc->valueStack() == stackPushed_) { + masm.popValue(dest); + stackPushed_ -= sizeof(js::Value); + } else { + MOZ_ASSERT(loc->valueStack() < stackPushed_); + masm.loadValue(Address(masm.getStackPointer(), stackPushed_ - loc->valueStack()), dest); + masm.propagateOOM(freeValueSlots_.append(loc->valueStack())); + } + + loc->setValueReg(dest); +} + +bool +OperandLocation::aliasesReg(const OperandLocation& other) const +{ + MOZ_ASSERT(&other != this); + + switch (other.kind_) { + case PayloadReg: + return aliasesReg(other.payloadReg()); + case ValueReg: + return aliasesReg(other.valueReg()); + case PayloadStack: + case ValueStack: + case BaselineFrame: + case Constant: + case DoubleReg: + return false; + case Uninitialized: + break; + } + + MOZ_CRASH("Invalid kind"); +} + +void +CacheRegisterAllocator::restoreInputState(MacroAssembler& masm, bool shouldDiscardStack) +{ + size_t numInputOperands = origInputLocations_.length(); + MOZ_ASSERT(writer_.numInputOperands() == numInputOperands); + + for (size_t j = 0; j < numInputOperands; j++) { + const OperandLocation& dest = origInputLocations_[j]; + OperandLocation& cur = operandLocations_[j]; + if (dest == cur) + continue; + + auto autoAssign = mozilla::MakeScopeExit([&] { cur = dest; }); + + // We have a cycle if a destination register will be used later + // as source register. If that happens, just push the current value + // on the stack and later get it from there. + for (size_t k = j + 1; k < numInputOperands; k++) { + OperandLocation& laterSource = operandLocations_[k]; + if (dest.aliasesReg(laterSource)) + spillOperandToStack(masm, &laterSource); + } + + if (dest.kind() == OperandLocation::ValueReg) { + // We have to restore a Value register. + switch (cur.kind()) { + case OperandLocation::ValueReg: + masm.moveValue(cur.valueReg(), dest.valueReg()); + continue; + case OperandLocation::PayloadReg: + masm.tagValue(cur.payloadType(), cur.payloadReg(), dest.valueReg()); + continue; + case OperandLocation::PayloadStack: { + Register scratch = dest.valueReg().scratchReg(); + popPayload(masm, &cur, scratch); + masm.tagValue(cur.payloadType(), scratch, dest.valueReg()); + continue; + } + case OperandLocation::ValueStack: + popValue(masm, &cur, dest.valueReg()); + continue; + case OperandLocation::Constant: + case OperandLocation::BaselineFrame: + case OperandLocation::DoubleReg: + case OperandLocation::Uninitialized: + break; + } + } else if (dest.kind() == OperandLocation::PayloadReg) { + // We have to restore a payload register. + switch (cur.kind()) { + case OperandLocation::ValueReg: + MOZ_ASSERT(dest.payloadType() != JSVAL_TYPE_DOUBLE); + masm.unboxNonDouble(cur.valueReg(), dest.payloadReg()); + continue; + case OperandLocation::PayloadReg: + MOZ_ASSERT(cur.payloadType() == dest.payloadType()); + masm.mov(cur.payloadReg(), dest.payloadReg()); + continue; + case OperandLocation::PayloadStack: { + MOZ_ASSERT(cur.payloadType() == dest.payloadType()); + popPayload(masm, &cur, dest.payloadReg()); + continue; + } + case OperandLocation::ValueStack: + MOZ_ASSERT(stackPushed_ >= sizeof(js::Value)); + MOZ_ASSERT(cur.valueStack() <= stackPushed_); + MOZ_ASSERT(dest.payloadType() != JSVAL_TYPE_DOUBLE); + masm.unboxNonDouble(Address(masm.getStackPointer(), stackPushed_ - cur.valueStack()), + dest.payloadReg()); + continue; + case OperandLocation::Constant: + case OperandLocation::BaselineFrame: + case OperandLocation::DoubleReg: + case OperandLocation::Uninitialized: + break; + } + } else if (dest.kind() == OperandLocation::Constant || + dest.kind() == OperandLocation::BaselineFrame || + dest.kind() == OperandLocation::DoubleReg) + { + // Nothing to do. + continue; + } + MOZ_CRASH("Invalid kind"); + } + + for (const SpilledRegister& spill : spilledRegs_) { + MOZ_ASSERT(stackPushed_ >= sizeof(uintptr_t)); + + if (spill.stackPushed == stackPushed_) { + masm.pop(spill.reg); + stackPushed_ -= sizeof(uintptr_t); + } else { + MOZ_ASSERT(spill.stackPushed < stackPushed_); + masm.loadPtr(Address(masm.getStackPointer(), stackPushed_ - spill.stackPushed), + spill.reg); + } + } + + if (shouldDiscardStack) + discardStack(masm); +} + +size_t +CacheIRStubInfo::stubDataSize() const +{ + size_t field = 0; + size_t size = 0; + while (true) { + StubField::Type type = fieldType(field++); + if (type == StubField::Type::Limit) + return size; + size += StubField::sizeInBytes(type); + } +} + +void +CacheIRStubInfo::copyStubData(ICStub* src, ICStub* dest) const +{ + uint8_t* srcBytes = reinterpret_cast(src); + uint8_t* destBytes = reinterpret_cast(dest); + + size_t field = 0; + size_t offset = 0; + while (true) { + StubField::Type type = fieldType(field); + switch (type) { + case StubField::Type::RawWord: + *reinterpret_cast(destBytes + offset) = + *reinterpret_cast(srcBytes + offset); + break; + case StubField::Type::RawInt64: + case StubField::Type::DOMExpandoGeneration: + *reinterpret_cast(destBytes + offset) = + *reinterpret_cast(srcBytes + offset); + break; + case StubField::Type::Shape: + getStubField(dest, offset).init(getStubField(src, offset)); + break; + case StubField::Type::JSObject: + getStubField(dest, offset).init(getStubField(src, offset)); + break; + case StubField::Type::ObjectGroup: + getStubField(dest, offset).init(getStubField(src, offset)); + break; + case StubField::Type::Symbol: + getStubField(dest, offset).init(getStubField(src, offset)); + break; + case StubField::Type::String: + getStubField(dest, offset).init(getStubField(src, offset)); + break; + case StubField::Type::Id: + getStubField(dest, offset).init(getStubField(src, offset)); + break; + case StubField::Type::Value: + getStubField(dest, offset).init(getStubField(src, offset)); + break; + case StubField::Type::Limit: + return; // Done. + } + field++; + offset += StubField::sizeInBytes(type); + } +} + +template +static GCPtr* +AsGCPtr(uintptr_t* ptr) +{ + return reinterpret_cast*>(ptr); +} + +template +GCPtr& +CacheIRStubInfo::getStubField(Stub* stub, uint32_t offset) const +{ + uint8_t* stubData = (uint8_t*)stub + stubDataOffset_; + MOZ_ASSERT(uintptr_t(stubData) % sizeof(uintptr_t) == 0); + + return *AsGCPtr((uintptr_t*)(stubData + offset)); +} + +template GCPtr& CacheIRStubInfo::getStubField(ICStub* stub, uint32_t offset) const; +template GCPtr& CacheIRStubInfo::getStubField(ICStub* stub, uint32_t offset) const; +template GCPtr& CacheIRStubInfo::getStubField(ICStub* stub, uint32_t offset) const; +template GCPtr& CacheIRStubInfo::getStubField(ICStub* stub, uint32_t offset) const; +template GCPtr& CacheIRStubInfo::getStubField(ICStub* stub, uint32_t offset) const; +template GCPtr& CacheIRStubInfo::getStubField(ICStub* stub, uint32_t offset) const; +template GCPtr& CacheIRStubInfo::getStubField(ICStub* stub, uint32_t offset) const; + +template +static void +InitGCPtr(uintptr_t* ptr, V val) +{ + AsGCPtr(ptr)->init(mozilla::BitwiseCast(val)); +} + +void +CacheIRWriter::copyStubData(uint8_t* dest) const +{ + MOZ_ASSERT(!failed()); + + uintptr_t* destWords = reinterpret_cast(dest); + + for (const StubField& field : stubFields_) { + switch (field.type()) { + case StubField::Type::RawWord: + *destWords = field.asWord(); + break; + case StubField::Type::Shape: + InitGCPtr(destWords, field.asWord()); + break; + case StubField::Type::JSObject: + InitGCPtr(destWords, field.asWord()); + break; + case StubField::Type::ObjectGroup: + InitGCPtr(destWords, field.asWord()); + break; + case StubField::Type::Symbol: + InitGCPtr(destWords, field.asWord()); + break; + case StubField::Type::String: + InitGCPtr(destWords, field.asWord()); + break; + case StubField::Type::Id: + InitGCPtr(destWords, field.asWord()); + break; + case StubField::Type::RawInt64: + case StubField::Type::DOMExpandoGeneration: + *reinterpret_cast(destWords) = field.asInt64(); + break; + case StubField::Type::Value: + AsGCPtr(destWords)->init(Value::fromRawBits(uint64_t(field.asInt64()))); + break; + case StubField::Type::Limit: + MOZ_CRASH("Invalid type"); + } + destWords += StubField::sizeInBytes(field.type()) / sizeof(uintptr_t); + } +} + +template +void +jit::TraceCacheIRStub(JSTracer* trc, T* stub, const CacheIRStubInfo* stubInfo) +{ + uint32_t field = 0; + size_t offset = 0; + while (true) { + StubField::Type fieldType = stubInfo->fieldType(field); + switch (fieldType) { + case StubField::Type::RawWord: + case StubField::Type::RawInt64: + case StubField::Type::DOMExpandoGeneration: + break; + case StubField::Type::Shape: + TraceNullableEdge(trc, &stubInfo->getStubField(stub, offset), + "cacheir-shape"); + break; + case StubField::Type::ObjectGroup: + TraceNullableEdge(trc, &stubInfo->getStubField(stub, offset), + "cacheir-group"); + break; + case StubField::Type::JSObject: + TraceNullableEdge(trc, &stubInfo->getStubField(stub, offset), + "cacheir-object"); + break; + case StubField::Type::Symbol: + TraceNullableEdge(trc, &stubInfo->getStubField(stub, offset), + "cacheir-symbol"); + break; + case StubField::Type::String: + TraceNullableEdge(trc, &stubInfo->getStubField(stub, offset), + "cacheir-string"); + break; + case StubField::Type::Id: + TraceEdge(trc, &stubInfo->getStubField(stub, offset), "cacheir-id"); + break; + case StubField::Type::Value: + TraceEdge(trc, &stubInfo->getStubField(stub, offset), + "cacheir-value"); + break; + case StubField::Type::Limit: + return; // Done. + } + field++; + offset += StubField::sizeInBytes(fieldType); + } +} + +template +void jit::TraceCacheIRStub(JSTracer* trc, ICStub* stub, const CacheIRStubInfo* stubInfo); + +template +void jit::TraceCacheIRStub(JSTracer* trc, IonICStub* stub, const CacheIRStubInfo* stubInfo); + +bool +CacheIRWriter::stubDataEqualsMaybeUpdate(uint8_t* stubData, bool* updated) const +{ + MOZ_ASSERT(!failed()); + + *updated = false; + const uintptr_t* stubDataWords = reinterpret_cast(stubData); + + // If DOMExpandoGeneration fields are different but all other stub fields + // are exactly the same, we overwrite the old stub data instead of attaching + // a new stub, as the old stub is never going to succeed. This works because + // even Ion stubs read the DOMExpandoGeneration field from the stub instead + // of baking it in. + bool expandoGenerationIsDifferent = false; + + for (const StubField& field : stubFields_) { + if (field.sizeIsWord()) { + if (field.asWord() != *stubDataWords) + return false; + stubDataWords++; + continue; + } + + if (field.asInt64() != *reinterpret_cast(stubDataWords)) { + if (field.type() != StubField::Type::DOMExpandoGeneration) + return false; + expandoGenerationIsDifferent = true; + } + stubDataWords += sizeof(uint64_t) / sizeof(uintptr_t); + } + + if (expandoGenerationIsDifferent) { + copyStubData(stubData); + *updated = true; + } + + return true; +} + +HashNumber +CacheIRStubKey::hash(const CacheIRStubKey::Lookup& l) +{ + HashNumber hash = mozilla::HashBytes(l.code, l.length); + hash = mozilla::AddToHash(hash, uint32_t(l.kind)); + hash = mozilla::AddToHash(hash, uint32_t(l.engine)); + return hash; +} + +bool +CacheIRStubKey::match(const CacheIRStubKey& entry, const CacheIRStubKey::Lookup& l) +{ + if (entry.stubInfo->kind() != l.kind) + return false; + + if (entry.stubInfo->engine() != l.engine) + return false; + + if (entry.stubInfo->codeLength() != l.length) + return false; + + if (!mozilla::PodEqual(entry.stubInfo->code(), l.code, l.length)) + return false; + + return true; +} + +CacheIRReader::CacheIRReader(const CacheIRStubInfo* stubInfo) + : CacheIRReader(stubInfo->code(), stubInfo->code() + stubInfo->codeLength()) +{} + +CacheIRStubInfo* +CacheIRStubInfo::New(CacheKind kind, ICStubEngine engine, bool makesGCCalls, + uint32_t stubDataOffset, const CacheIRWriter& writer) +{ + size_t numStubFields = writer.numStubFields(); + size_t bytesNeeded = sizeof(CacheIRStubInfo) + + writer.codeLength() + + (numStubFields + 1); // +1 for the GCType::Limit terminator. + uint8_t* p = js_pod_malloc(bytesNeeded); + if (!p) + return nullptr; + + // Copy the CacheIR code. + uint8_t* codeStart = p + sizeof(CacheIRStubInfo); + mozilla::PodCopy(codeStart, writer.codeStart(), writer.codeLength()); + + static_assert(sizeof(StubField::Type) == sizeof(uint8_t), + "StubField::Type must fit in uint8_t"); + + // Copy the stub field types. + uint8_t* fieldTypes = codeStart + writer.codeLength(); + for (size_t i = 0; i < numStubFields; i++) + fieldTypes[i] = uint8_t(writer.stubFieldType(i)); + fieldTypes[numStubFields] = uint8_t(StubField::Type::Limit); + + return new(p) CacheIRStubInfo(kind, engine, makesGCCalls, stubDataOffset, codeStart, + writer.codeLength(), fieldTypes); +} + +bool +OperandLocation::operator==(const OperandLocation& other) const +{ + if (kind_ != other.kind_) + return false; + + switch (kind()) { + case Uninitialized: + return true; + case PayloadReg: + return payloadReg() == other.payloadReg() && payloadType() == other.payloadType(); + case ValueReg: + return valueReg() == other.valueReg(); + case PayloadStack: + return payloadStack() == other.payloadStack() && payloadType() == other.payloadType(); + case ValueStack: + return valueStack() == other.valueStack(); + case BaselineFrame: + return baselineFrameSlot() == other.baselineFrameSlot(); + case Constant: + return constant() == other.constant(); + case DoubleReg: + return doubleReg() == other.doubleReg(); + } + + MOZ_CRASH("Invalid OperandLocation kind"); +} + +AutoOutputRegister::AutoOutputRegister(CacheIRCompiler& compiler) + : output_(compiler.outputUnchecked_.ref()), + alloc_(compiler.allocator) +{ + if (output_.hasValue()) + alloc_.allocateFixedValueRegister(compiler.masm, output_.valueReg()); + else if (!output_.typedReg().isFloat()) + alloc_.allocateFixedRegister(compiler.masm, output_.typedReg().gpr()); +} + +AutoOutputRegister::~AutoOutputRegister() +{ + if (output_.hasValue()) + alloc_.releaseValueRegister(output_.valueReg()); + else if (!output_.typedReg().isFloat()) + alloc_.releaseRegister(output_.typedReg().gpr()); +} + +bool +FailurePath::canShareFailurePath(const FailurePath& other) const +{ + if (stackPushed_ != other.stackPushed_) + return false; + + if (spilledRegs_.length() != other.spilledRegs_.length()) + return false; + + for (size_t i = 0; i < spilledRegs_.length(); i++) { + if (spilledRegs_[i] != other.spilledRegs_[i]) + return false; + } + + MOZ_ASSERT(inputs_.length() == other.inputs_.length()); + + for (size_t i = 0; i < inputs_.length(); i++) { + if (inputs_[i] != other.inputs_[i]) + return false; + } + return true; +} + +bool +CacheIRCompiler::addFailurePath(FailurePath** failure) +{ + FailurePath newFailure; + for (size_t i = 0; i < writer_.numInputOperands(); i++) { + if (!newFailure.appendInput(allocator.operandLocation(i))) + return false; + } + if (!newFailure.setSpilledRegs(allocator.spilledRegs())) + return false; + newFailure.setStackPushed(allocator.stackPushed()); + + // Reuse the previous failure path if the current one is the same, to + // avoid emitting duplicate code. + if (failurePaths.length() > 0 && failurePaths.back().canShareFailurePath(newFailure)) { + *failure = &failurePaths.back(); + return true; + } + + if (!failurePaths.append(Move(newFailure))) + return false; + + *failure = &failurePaths.back(); + return true; +} + +bool +CacheIRCompiler::emitFailurePath(size_t index) +{ + FailurePath& failure = failurePaths[index]; + + allocator.setStackPushed(failure.stackPushed()); + + for (size_t i = 0; i < writer_.numInputOperands(); i++) + allocator.setOperandLocation(i, failure.input(i)); + + if (!allocator.setSpilledRegs(failure.spilledRegs())) + return false; + + masm.bind(failure.label()); + allocator.restoreInputState(masm); + return true; +} + +bool +CacheIRCompiler::emitGuardIsObject() +{ + ValOperandId inputId = reader.valOperandId(); + if (allocator.knownType(inputId) == JSVAL_TYPE_OBJECT) + return true; + + ValueOperand input = allocator.useValueRegister(masm, inputId); + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + masm.branchTestObject(Assembler::NotEqual, input, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardIsObjectOrNull() +{ + ValOperandId inputId = reader.valOperandId(); + JSValueType knownType = allocator.knownType(inputId); + if (knownType == JSVAL_TYPE_OBJECT || knownType == JSVAL_TYPE_NULL) + return true; + + ValueOperand input = allocator.useValueRegister(masm, inputId); + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + Label done; + masm.branchTestObject(Assembler::Equal, input, &done); + masm.branchTestNull(Assembler::NotEqual, input, failure->label()); + masm.bind(&done); + return true; +} + +bool +CacheIRCompiler::emitGuardIsString() +{ + ValOperandId inputId = reader.valOperandId(); + if (allocator.knownType(inputId) == JSVAL_TYPE_STRING) + return true; + + ValueOperand input = allocator.useValueRegister(masm, inputId); + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + masm.branchTestString(Assembler::NotEqual, input, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardIsSymbol() +{ + ValOperandId inputId = reader.valOperandId(); + if (allocator.knownType(inputId) == JSVAL_TYPE_SYMBOL) + return true; + + ValueOperand input = allocator.useValueRegister(masm, inputId); + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + masm.branchTestSymbol(Assembler::NotEqual, input, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardIsInt32Index() +{ + ValOperandId inputId = reader.valOperandId(); + Register output = allocator.defineRegister(masm, reader.int32OperandId()); + + if (allocator.knownType(inputId) == JSVAL_TYPE_INT32) { + Register input = allocator.useRegister(masm, Int32OperandId(inputId.id())); + masm.move32(input, output); + return true; + } + + ValueOperand input = allocator.useValueRegister(masm, inputId); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + Label notInt32, done; + masm.branchTestInt32(Assembler::NotEqual, input, ¬Int32); + masm.unboxInt32(input, output); + masm.jump(&done); + + masm.bind(¬Int32); + + if (cx_->runtime()->jitSupportsFloatingPoint) { + masm.branchTestDouble(Assembler::NotEqual, input, failure->label()); + + // If we're compiling a Baseline IC, FloatReg0 is always available. + Label failurePopReg; + if (mode_ != Mode::Baseline) + masm.push(FloatReg0); + + masm.unboxDouble(input, FloatReg0); + // ToPropertyKey(-0.0) is "0", so we can truncate -0.0 to 0 here. + masm.convertDoubleToInt32(FloatReg0, output, + (mode_ == Mode::Baseline) ? failure->label() : &failurePopReg, + false); + if (mode_ != Mode::Baseline) { + masm.pop(FloatReg0); + masm.jump(&done); + + masm.bind(&failurePopReg); + masm.pop(FloatReg0); + masm.jump(failure->label()); + } + } else { + masm.jump(failure->label()); + } + + masm.bind(&done); + return true; +} + +bool +CacheIRCompiler::emitGuardType() +{ + ValOperandId inputId = reader.valOperandId(); + JSValueType type = reader.valueType(); + + if (allocator.knownType(inputId) == type) + return true; + + ValueOperand input = allocator.useValueRegister(masm, inputId); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + switch (type) { + case JSVAL_TYPE_STRING: + masm.branchTestString(Assembler::NotEqual, input, failure->label()); + break; + case JSVAL_TYPE_SYMBOL: + masm.branchTestSymbol(Assembler::NotEqual, input, failure->label()); + break; + case JSVAL_TYPE_BIGINT: + masm.branchTestBigInt(Assembler::NotEqual, input, failure->label()); + break; + case JSVAL_TYPE_INT32: + masm.branchTestInt32(Assembler::NotEqual, input, failure->label()); + break; + case JSVAL_TYPE_DOUBLE: + masm.branchTestNumber(Assembler::NotEqual, input, failure->label()); + break; + case JSVAL_TYPE_BOOLEAN: + masm.branchTestBoolean(Assembler::NotEqual, input, failure->label()); + break; + case JSVAL_TYPE_UNDEFINED: + masm.branchTestUndefined(Assembler::NotEqual, input, failure->label()); + break; + case JSVAL_TYPE_NULL: + masm.branchTestNull(Assembler::NotEqual, input, failure->label()); + break; + default: + MOZ_CRASH("Unexpected type"); + } + + return true; +} + +bool +CacheIRCompiler::emitGuardClass() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + const Class* clasp = nullptr; + switch (reader.guardClassKind()) { + case GuardClassKind::Array: + clasp = &ArrayObject::class_; + break; + case GuardClassKind::UnboxedArray: + clasp = &UnboxedArrayObject::class_; + break; + case GuardClassKind::MappedArguments: + clasp = &MappedArgumentsObject::class_; + break; + case GuardClassKind::UnmappedArguments: + clasp = &UnmappedArgumentsObject::class_; + break; + case GuardClassKind::WindowProxy: + clasp = cx_->runtime()->maybeWindowProxyClass(); + break; + case GuardClassKind::JSFunction: + clasp = &JSFunction::class_; + break; + } + + MOZ_ASSERT(clasp); + masm.branchTestObjClass(Assembler::NotEqual, obj, scratch, clasp, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardIsNativeFunction() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + JSNative nativeFunc = reinterpret_cast(reader.pointer()); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Ensure obj is a function. + const Class* clasp = &JSFunction::class_; + masm.branchTestObjClass(Assembler::NotEqual, obj, scratch, clasp, failure->label()); + + // Ensure function native matches. + masm.branchPtr(Assembler::NotEqual, Address(obj, JSFunction::offsetOfNativeOrScript()), + ImmPtr(nativeFunc), failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardIsProxy() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.branchTestObjectIsProxy(false, obj, scratch, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardIsCrossCompartmentWrapper() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + Address handlerAddr(obj, ProxyObject::offsetOfHandler()); + masm.branchPtr(Assembler::NotEqual, handlerAddr, ImmPtr(&CrossCompartmentWrapper::singleton), + failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardNotDOMProxy() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.branchTestProxyHandlerFamily(Assembler::Equal, obj, scratch, + GetDOMProxyHandlerFamily(), failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardSpecificInt32Immediate() +{ + Register reg = allocator.useRegister(masm, reader.int32OperandId()); + int32_t ival = reader.int32Immediate(); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.branch32(Assembler::NotEqual, reg, Imm32(ival), failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardMagicValue() +{ + ValueOperand val = allocator.useValueRegister(masm, reader.valOperandId()); + JSWhyMagic magic = reader.whyMagic(); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.branchTestMagicValue(Assembler::NotEqual, val, magic, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardNoUnboxedExpando() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + Address expandoAddr(obj, UnboxedPlainObject::offsetOfExpando()); + masm.branchPtr(Assembler::NotEqual, expandoAddr, ImmWord(0), failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardAndLoadUnboxedExpando() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register output = allocator.defineRegister(masm, reader.objOperandId()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + Address expandoAddr(obj, UnboxedPlainObject::offsetOfExpando()); + masm.loadPtr(expandoAddr, output); + masm.branchTestPtr(Assembler::Zero, output, output, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardNoDetachedTypedObjects() +{ + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + CheckForTypedObjectWithDetachedStorage(cx_, masm, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardNoDenseElements() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Load obj->elements. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + + // Make sure there are no dense elements. + Address initLength(scratch, ObjectElements::offsetOfInitializedLength()); + masm.branch32(Assembler::NotEqual, initLength, Imm32(0), failure->label()); + return true; +} + +bool +CacheIRCompiler::emitGuardAndGetIndexFromString() +{ + Register str = allocator.useRegister(masm, reader.stringOperandId()); + Register output = allocator.defineRegister(masm, reader.int32OperandId()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + masm.PushRegsInMask(save); + + masm.setupUnalignedABICall(output); + masm.passABIArg(str); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, GetIndexFromString)); + masm.mov(ReturnReg, output); + + LiveRegisterSet ignore; + ignore.add(output); + masm.PopRegsInMaskIgnore(save, ignore); + + // GetIndexFromString returns a negative value on failure. + masm.branchTest32(Assembler::Signed, output, output, failure->label()); + return true; +} + +bool +CacheIRCompiler::emitLoadProto() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register reg = allocator.defineRegister(masm, reader.objOperandId()); + masm.loadObjProto(obj, reg); + return true; +} + +bool +CacheIRCompiler::emitLoadEnclosingEnvironment() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register reg = allocator.defineRegister(masm, reader.objOperandId()); + masm.extractObject(Address(obj, EnvironmentObject::offsetOfEnclosingEnvironment()), reg); + return true; +} + +bool +CacheIRCompiler::emitLoadWrapperTarget() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register reg = allocator.defineRegister(masm, reader.objOperandId()); + + masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), reg); + masm.unboxObject(Address(reg, detail::ProxyReservedSlots::offsetOfPrivateSlot()), reg); + return true; +} + +bool +CacheIRCompiler::emitLoadDOMExpandoValue() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ValueOperand val = allocator.defineValueRegister(masm, reader.valOperandId()); + + masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), val.scratchReg()); + masm.loadValue(Address(val.scratchReg(), + detail::ProxyReservedSlots::offsetOfPrivateSlot()), + val); + return true; +} + +bool +CacheIRCompiler::emitLoadDOMExpandoValueIgnoreGeneration() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ValueOperand output = allocator.defineValueRegister(masm, reader.valOperandId()); + + // Determine the expando's Address. + Register scratch = output.scratchReg(); + masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), scratch); + Address expandoAddr(scratch, detail::ProxyReservedSlots::offsetOfPrivateSlot()); + +#ifdef DEBUG + // Private values are stored as doubles, so assert we have a double. + Label ok; + masm.branchTestDouble(Assembler::Equal, expandoAddr, &ok); + masm.assumeUnreachable("DOM expando is not a PrivateValue!"); + masm.bind(&ok); +#endif + + // Load the ExpandoAndGeneration* from the PrivateValue. + masm.loadPrivate(expandoAddr, scratch); + + // Load expandoAndGeneration->expando into the output Value register. + masm.loadValue(Address(scratch, ExpandoAndGeneration::offsetOfExpando()), output); + return true; +} + +bool +CacheIRCompiler::emitLoadUndefinedResult() +{ + AutoOutputRegister output(*this); + if (output.hasValue()) + masm.moveValue(UndefinedValue(), output.valueReg()); + else + masm.assumeUnreachable("Should have monitored undefined result"); + return true; +} + +static void +EmitStoreBoolean(MacroAssembler& masm, bool b, const AutoOutputRegister& output) +{ + if (output.hasValue()) { + Value val = BooleanValue(b); + masm.moveValue(val, output.valueReg()); + } else { + MOZ_ASSERT(output.type() == JSVAL_TYPE_BOOLEAN); + masm.movePtr(ImmWord(b), output.typedReg().gpr()); + } +} + +bool +CacheIRCompiler::emitLoadBooleanResult() +{ + AutoOutputRegister output(*this); + bool b = reader.readBool(); + EmitStoreBoolean(masm, b, output); + + return true; +} + +static void +EmitStoreResult(MacroAssembler& masm, Register reg, JSValueType type, + const AutoOutputRegister& output) +{ + if (output.hasValue()) { + masm.tagValue(type, reg, output.valueReg()); + return; + } + if (type == JSVAL_TYPE_INT32 && output.typedReg().isFloat()) { + masm.convertInt32ToDouble(reg, output.typedReg().fpu()); + return; + } + if (type == output.type()) { + masm.mov(reg, output.typedReg().gpr()); + return; + } + masm.assumeUnreachable("Should have monitored result"); +} + +bool +CacheIRCompiler::emitLoadInt32ArrayLengthResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + masm.load32(Address(scratch, ObjectElements::offsetOfLength()), scratch); + + // Guard length fits in an int32. + masm.branchTest32(Assembler::Signed, scratch, scratch, failure->label()); + EmitStoreResult(masm, scratch, JSVAL_TYPE_INT32, output); + return true; +} + +bool +CacheIRCompiler::emitLoadUnboxedArrayLengthResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + masm.load32(Address(obj, UnboxedArrayObject::offsetOfLength()), scratch); + EmitStoreResult(masm, scratch, JSVAL_TYPE_INT32, output); + return true; +} + +bool +CacheIRCompiler::emitLoadArgumentsObjectLengthResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Get initial length value. + masm.unboxInt32(Address(obj, ArgumentsObject::getInitialLengthSlotOffset()), scratch); + + // Test if length has been overridden. + masm.branchTest32(Assembler::NonZero, + scratch, + Imm32(ArgumentsObject::LENGTH_OVERRIDDEN_BIT), + failure->label()); + + // Shift out arguments length and return it. No need to type monitor + // because this stub always returns int32. + masm.rshiftPtr(Imm32(ArgumentsObject::PACKED_BITS_COUNT), scratch); + EmitStoreResult(masm, scratch, JSVAL_TYPE_INT32, output); + return true; +} + +bool +CacheIRCompiler::emitLoadFunctionLengthResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Get the JSFunction flags. + masm.load16ZeroExtend(Address(obj, JSFunction::offsetOfFlags()), scratch); + + // Functions with lazy scripts don't store their length. + // If the length was resolved before the length property might be shadowed. + masm.branchTest32(Assembler::NonZero, + scratch, + Imm32(JSFunction::INTERPRETED_LAZY | + JSFunction::RESOLVED_LENGTH), + failure->label()); + + Label boundFunction; + masm.branchTest32(Assembler::NonZero, scratch, Imm32(JSFunction::BOUND_FUN), &boundFunction); + Label interpreted; + masm.branchTest32(Assembler::NonZero, scratch, Imm32(JSFunction::INTERPRETED), &interpreted); + + // Load the length of the native function. + masm.load16ZeroExtend(Address(obj, JSFunction::offsetOfNargs()), scratch); + Label done; + masm.jump(&done); + + masm.bind(&boundFunction); + // Bound functions might have a non-int32 length. + Address boundLength(obj, FunctionExtended::offsetOfExtendedSlot(BOUND_FUN_LENGTH_SLOT)); + masm.branchTestInt32(Assembler::NotEqual, boundLength, failure->label()); + masm.unboxInt32(boundLength, scratch); + masm.jump(&done); + + masm.bind(&interpreted); + // Load the length from the function's script. + masm.loadPtr(Address(obj, JSFunction::offsetOfNativeOrScript()), scratch); + masm.load16ZeroExtend(Address(scratch, JSScript::offsetOfFunLength()), scratch); + + masm.bind(&done); + EmitStoreResult(masm, scratch, JSVAL_TYPE_INT32, output); + return true; +} + +bool +CacheIRCompiler::emitLoadStringLengthResult() +{ + AutoOutputRegister output(*this); + Register str = allocator.useRegister(masm, reader.stringOperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + masm.loadStringLength(str, scratch); + EmitStoreResult(masm, scratch, JSVAL_TYPE_INT32, output); + return true; +} + +bool +CacheIRCompiler::emitLoadStringCharResult() +{ + AutoOutputRegister output(*this); + Register str = allocator.useRegister(masm, reader.stringOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output); + AutoScratchRegister scratch2(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Bounds check, load string char. + masm.branch32(Assembler::BelowOrEqual, Address(str, JSString::offsetOfLength()), + index, failure->label()); + masm.loadStringChar(str, index, scratch1, failure->label()); + + // Load StaticString for this char. + masm.branch32(Assembler::AboveOrEqual, scratch1, Imm32(StaticStrings::UNIT_STATIC_LIMIT), + failure->label()); + masm.movePtr(ImmPtr(&cx_->staticStrings().unitStaticTable), scratch2); + masm.loadPtr(BaseIndex(scratch2, scratch1, ScalePointer), scratch2); + + EmitStoreResult(masm, scratch2, JSVAL_TYPE_STRING, output); + return true; +} + +bool +CacheIRCompiler::emitLoadArgumentsObjectArgResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Get initial length value. + masm.unboxInt32(Address(obj, ArgumentsObject::getInitialLengthSlotOffset()), scratch); + + // Ensure no overridden length/element. + masm.branchTest32(Assembler::NonZero, + scratch, + Imm32(ArgumentsObject::LENGTH_OVERRIDDEN_BIT | + ArgumentsObject::ELEMENT_OVERRIDDEN_BIT), + failure->label()); + + // Bounds check. + masm.rshift32(Imm32(ArgumentsObject::PACKED_BITS_COUNT), scratch); + masm.branch32(Assembler::AboveOrEqual, index, scratch, failure->label()); + + // Load ArgumentsData. + masm.loadPrivate(Address(obj, ArgumentsObject::getDataSlotOffset()), scratch); + + // Fail if we have a RareArgumentsData (elements were deleted). + masm.branchPtr(Assembler::NotEqual, + Address(scratch, offsetof(ArgumentsData, rareData)), + ImmWord(0), + failure->label()); + + // Guard the argument is not a FORWARD_TO_CALL_SLOT MagicValue. + BaseValueIndex argValue(scratch, index, ArgumentsData::offsetOfArgs()); + masm.branchTestMagic(Assembler::Equal, argValue, failure->label()); + masm.loadValue(argValue, output.valueReg()); + return true; +} + +bool +CacheIRCompiler::emitLoadDenseElementResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Load obj->elements. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + + // Bounds check. + Address initLength(scratch, ObjectElements::offsetOfInitializedLength()); + masm.branch32(Assembler::BelowOrEqual, initLength, index, failure->label()); + + // Hole check. + BaseObjectElementIndex element(scratch, index); + masm.branchTestMagic(Assembler::Equal, element, failure->label()); + masm.loadTypedOrValue(element, output); + return true; +} + +bool +CacheIRCompiler::emitLoadDenseElementHoleResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + if (!output.hasValue()) { + masm.assumeUnreachable("Should have monitored undefined value after attaching stub"); + return true; + } + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Make sure the index is nonnegative. + masm.branch32(Assembler::LessThan, index, Imm32(0), failure->label()); + + // Load obj->elements. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + + // Guard on the initialized length. + Label hole; + Address initLength(scratch, ObjectElements::offsetOfInitializedLength()); + masm.branch32(Assembler::BelowOrEqual, initLength, index, &hole); + + // Load the value. + Label done; + masm.loadValue(BaseObjectElementIndex(scratch, index), output.valueReg()); + masm.branchTestMagic(Assembler::NotEqual, output.valueReg(), &done); + + // Load undefined for the hole. + masm.bind(&hole); + masm.moveValue(UndefinedValue(), output.valueReg()); + + masm.bind(&done); + return true; +} + +bool +CacheIRCompiler::emitLoadDenseElementExistsResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Load obj->elements. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + + // Bounds check. Unsigned compare sends negative indices to next IC. + Address initLength(scratch, ObjectElements::offsetOfInitializedLength()); + masm.branch32(Assembler::BelowOrEqual, initLength, index, failure->label()); + + // Hole check. + BaseObjectElementIndex element(scratch, index); + masm.branchTestMagic(Assembler::Equal, element, failure->label()); + + EmitStoreBoolean(masm, true, output); + return true; +} + +bool +CacheIRCompiler::emitLoadDenseElementHoleExistsResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Make sure the index is nonnegative. + masm.branch32(Assembler::LessThan, index, Imm32(0), failure->label()); + + // Load obj->elements. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + + // Guard on the initialized length. + Label hole; + Address initLength(scratch, ObjectElements::offsetOfInitializedLength()); + masm.branch32(Assembler::BelowOrEqual, initLength, index, &hole); + + // Load value and replace with true. + Label done; + BaseObjectElementIndex element(scratch, index); + masm.branchTestMagic(Assembler::Equal, element, &hole); + EmitStoreBoolean(masm, true, output); + masm.jump(&done); + + // Load false for the hole. + masm.bind(&hole); + EmitStoreBoolean(masm, false, output); + + masm.bind(&done); + return true; +} + +bool +CacheIRCompiler::emitLoadUnboxedArrayElementResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + JSValueType elementType = reader.valueType(); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + if (!output.hasValue() && + elementType != output.type() && + !(elementType == JSVAL_TYPE_INT32 && output.type() == JSVAL_TYPE_DOUBLE)) + { + masm.assumeUnreachable("Should have monitored unboxed property type"); + return true; + } + + // Bounds check. + masm.load32(Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()), + scratch); + masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), scratch); + masm.branch32(Assembler::BelowOrEqual, scratch, index, failure->label()); + + // Load obj->elements. + masm.loadPtr(Address(obj, UnboxedArrayObject::offsetOfElements()), scratch); + + // Load value. + size_t width = UnboxedTypeSize(elementType); + BaseIndex addr(scratch, index, ScaleFromElemWidth(width)); + masm.loadUnboxedProperty(addr, elementType, output); + return true; +} + +bool +CacheIRCompiler::emitLoadTypedElementResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + TypedThingLayout layout = reader.typedThingLayout(); + Scalar::Type type = reader.scalarType(); + + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + if (!output.hasValue()) { + if (type == Scalar::Float32 || type == Scalar::Float64) { + if (output.type() != JSVAL_TYPE_DOUBLE) { + masm.assumeUnreachable("Should have monitored double after attaching stub"); + return true; + } + } else { + if (output.type() != JSVAL_TYPE_INT32 && output.type() != JSVAL_TYPE_DOUBLE) { + masm.assumeUnreachable("Should have monitored int32 after attaching stub"); + return true; + } + } + } + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Bounds check. + LoadTypedThingLength(masm, layout, obj, scratch); + masm.branch32(Assembler::BelowOrEqual, scratch, index, failure->label()); + + // Load the elements vector. + LoadTypedThingData(masm, layout, obj, scratch); + + // Load the value. + BaseIndex source(scratch, index, ScaleFromElemWidth(Scalar::byteSize(type))); + if (output.hasValue()) { + masm.loadFromTypedArray(type, source, output.valueReg(), *allowDoubleResult_, scratch, + failure->label()); + } else { + bool needGpr = (type == Scalar::Int8 || type == Scalar::Uint8 || + type == Scalar::Int16 || type == Scalar::Uint16 || + type == Scalar::Uint8Clamped || type == Scalar::Int32); + if (needGpr && output.type() == JSVAL_TYPE_DOUBLE) { + // Load the element as integer, then convert it to double. + masm.loadFromTypedArray(type, source, AnyRegister(scratch), scratch, failure->label()); + masm.convertInt32ToDouble(source, output.typedReg().fpu()); + } else { + masm.loadFromTypedArray(type, source, output.typedReg(), scratch, failure->label()); + } + } + return true; +} + +void +CacheIRCompiler::emitLoadTypedObjectResultShared(const Address& fieldAddr, Register scratch, + TypedThingLayout layout, uint32_t typeDescr, + const AutoOutputRegister& output) +{ + MOZ_ASSERT(output.hasValue()); + + if (SimpleTypeDescrKeyIsScalar(typeDescr)) { + Scalar::Type type = ScalarTypeFromSimpleTypeDescrKey(typeDescr); + masm.loadFromTypedArray(type, fieldAddr, output.valueReg(), + /* allowDouble = */ true, scratch, nullptr); + } else { + ReferenceTypeDescr::Type type = ReferenceTypeFromSimpleTypeDescrKey(typeDescr); + switch (type) { + case ReferenceTypeDescr::TYPE_ANY: + masm.loadValue(fieldAddr, output.valueReg()); + break; + + case ReferenceTypeDescr::TYPE_OBJECT: { + Label notNull, done; + masm.loadPtr(fieldAddr, scratch); + masm.branchTestPtr(Assembler::NonZero, scratch, scratch, ¬Null); + masm.moveValue(NullValue(), output.valueReg()); + masm.jump(&done); + masm.bind(¬Null); + masm.tagValue(JSVAL_TYPE_OBJECT, scratch, output.valueReg()); + masm.bind(&done); + break; + } + + case ReferenceTypeDescr::TYPE_STRING: + masm.loadPtr(fieldAddr, scratch); + masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg()); + break; + + default: + MOZ_CRASH("Invalid ReferenceTypeDescr"); + } + } +} + +bool +CacheIRCompiler::emitLoadObjectResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + + if (output.hasValue()) + masm.tagValue(JSVAL_TYPE_OBJECT, obj, output.valueReg()); + else + masm.mov(obj, output.typedReg().gpr()); + + return true; +} + +bool +CacheIRCompiler::emitLoadTypeOfObjectResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + Label slowCheck, isObject, isCallable, isUndefined, done; + masm.typeOfObject(obj, scratch, &slowCheck, &isObject, &isCallable, &isUndefined); + + masm.bind(&isCallable); + masm.moveValue(StringValue(cx_->names().function), output.valueReg()); + masm.jump(&done); + + masm.bind(&isUndefined); + masm.moveValue(StringValue(cx_->names().undefined), output.valueReg()); + masm.jump(&done); + + masm.bind(&isObject); + masm.moveValue(StringValue(cx_->names().object), output.valueReg()); + masm.jump(&done); + + { + masm.bind(&slowCheck); + LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + masm.PushRegsInMask(save); + + masm.setupUnalignedABICall(scratch); + masm.passABIArg(obj); + masm.movePtr(ImmPtr(cx_->runtime()), scratch); + masm.passABIArg(scratch); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, TypeOfObject)); + masm.mov(ReturnReg, scratch); + + LiveRegisterSet ignore; + ignore.add(scratch); + masm.PopRegsInMaskIgnore(save, ignore); + + masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg()); + } + + masm.bind(&done); + return true; +} + +bool +CacheIRCompiler::emitCallPrintString() +{ + const char* str = reinterpret_cast(reader.pointer()); + masm.printf(str); + return true; +} + +bool +CacheIRCompiler::emitBreakpoint() +{ + masm.breakpoint(); + return true; +} + +void +CacheIRCompiler::emitStoreTypedObjectReferenceProp(ValueOperand val, ReferenceTypeDescr::Type type, + const Address& dest, Register scratch) +{ + switch (type) { + case ReferenceTypeDescr::TYPE_ANY: + EmitPreBarrier(masm, dest, MIRType::Value); + masm.storeValue(val, dest); + break; + + case ReferenceTypeDescr::TYPE_OBJECT: { + EmitPreBarrier(masm, dest, MIRType::Object); + Label isNull, done; + masm.branchTestObject(Assembler::NotEqual, val, &isNull); + masm.unboxObject(val, scratch); + masm.storePtr(scratch, dest); + masm.jump(&done); + masm.bind(&isNull); + masm.storePtr(ImmWord(0), dest); + masm.bind(&done); + break; + } + + case ReferenceTypeDescr::TYPE_STRING: + EmitPreBarrier(masm, dest, MIRType::String); + masm.unboxString(val, scratch); + masm.storePtr(scratch, dest); + break; + } +} + +void +CacheIRCompiler::emitPostBarrierShared(Register obj, const ConstantOrRegister& val, + Register scratch, Register maybeIndex) +{ + if (!cx_->nursery().exists()) + return; + + if (val.constant()) { + MOZ_ASSERT_IF(val.value().isObject(), !IsInsideNursery(&val.value().toObject())); + return; + } + + TypedOrValueRegister reg = val.reg(); + if (reg.hasTyped() && reg.type() != MIRType::Object) + return; + + Label skipBarrier; + if (reg.hasValue()) { + masm.branchValueIsNurseryObject(Assembler::NotEqual, reg.valueReg(), scratch, + &skipBarrier); + } else { + masm.branchPtrInNurseryChunk(Assembler::NotEqual, reg.typedReg().gpr(), scratch, + &skipBarrier); + } + masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch, &skipBarrier); + + // Call one of these, depending on maybeIndex: + // + // void PostWriteBarrier(JSRuntime* rt, JSObject* obj); + // void PostWriteElementBarrier(JSRuntime* rt, JSObject* obj, + // int32_t index); + LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + masm.PushRegsInMask(save); + masm.setupUnalignedABICall(scratch); + masm.movePtr(ImmPtr(cx_->runtime()), scratch); + masm.passABIArg(scratch); + masm.passABIArg(obj); + if (maybeIndex != InvalidReg) { + masm.passABIArg(maybeIndex); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, + (PostWriteElementBarrier))); + } else { + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, PostWriteBarrier)); + } + masm.PopRegsInMask(save); + + masm.bind(&skipBarrier); +} + + +bool +CacheIRCompiler::emitWrapResult() +{ + AutoOutputRegister output(*this); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + Label done; + // We only have to wrap objects, because we are in the same zone. + masm.branchTestObject(Assembler::NotEqual, output.valueReg(), &done); + + Register obj = output.valueReg().scratchReg(); + masm.unboxObject(output.valueReg(), obj); + + AllocatableRegisterSet regs(RegisterSet::Volatile()); + LiveRegisterSet save(regs.asLiveSet()); + masm.PushRegsInMask(save); + + masm.setupUnalignedABICall(scratch); + masm.loadJSContext(scratch); + masm.passABIArg(scratch); + masm.passABIArg(obj); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, WrapObjectPure)); + masm.mov(ReturnReg, obj); + + LiveRegisterSet ignore; + ignore.add(obj); + masm.PopRegsInMaskIgnore(save, ignore); + + // We could not get a wrapper for this object. + masm.branchTestPtr(Assembler::Zero, obj, obj, failure->label()); + + // We clobbered the output register, so we have to retag. + masm.tagValue(JSVAL_TYPE_OBJECT, obj, output.valueReg()); + + masm.bind(&done); + return true; +} + +bool +CacheIRCompiler::emitMegamorphicLoadSlotByValueResult() +{ + AutoOutputRegister output(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ValueOperand idVal = allocator.useValueRegister(masm, reader.valOperandId()); + bool handleMissing = reader.readBool(); + + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // The object must be Native. + masm.loadObjClass(obj, scratch); + masm.branchTest32(Assembler::NonZero, Address(scratch, Class::offsetOfFlags()), + Imm32(Class::NON_NATIVE), failure->label()); + + // idVal will be in vp[0], result will be stored in vp[1]. + masm.reserveStack(sizeof(Value)); + masm.Push(idVal); + masm.moveStackPtrTo(idVal.scratchReg()); + + LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + volatileRegs.takeUnchecked(scratch); + volatileRegs.takeUnchecked(idVal); + masm.PushRegsInMask(volatileRegs); + + masm.setupUnalignedABICall(scratch); + masm.loadJSContext(scratch); + masm.passABIArg(scratch); + masm.passABIArg(obj); + masm.passABIArg(idVal.scratchReg()); + if (handleMissing) + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, (GetNativeDataPropertyByValuePure))); + else + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, (GetNativeDataPropertyByValuePure))); + masm.mov(ReturnReg, scratch); + masm.PopRegsInMask(volatileRegs); + + masm.Pop(idVal); + + Label ok; + uint32_t framePushed = masm.framePushed(); + masm.branchIfTrueBool(scratch, &ok); + masm.adjustStack(sizeof(Value)); + masm.jump(failure->label()); + + masm.bind(&ok); + masm.setFramePushed(framePushed); + masm.loadTypedOrValue(Address(masm.getStackPointer(), 0), output); + masm.adjustStack(sizeof(Value)); + return true; +} + +bool +CacheIRCompiler::emitMegamorphicHasOwnResult() +{ + AutoOutputRegister output(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ValueOperand idVal = allocator.useValueRegister(masm, reader.valOperandId()); + + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // idVal will be in vp[0], result will be stored in vp[1]. + masm.reserveStack(sizeof(Value)); + masm.Push(idVal); + masm.moveStackPtrTo(idVal.scratchReg()); + + LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + volatileRegs.takeUnchecked(scratch); + volatileRegs.takeUnchecked(idVal); + masm.PushRegsInMask(volatileRegs); + + masm.setupUnalignedABICall(scratch); + masm.loadJSContext(scratch); + masm.passABIArg(scratch); + masm.passABIArg(obj); + masm.passABIArg(idVal.scratchReg()); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, HasNativeDataPropertyPure)); + masm.mov(ReturnReg, scratch); + masm.PopRegsInMask(volatileRegs); + + masm.Pop(idVal); + + Label ok; + uint32_t framePushed = masm.framePushed(); + masm.branchIfTrueBool(scratch, &ok); + masm.adjustStack(sizeof(Value)); + masm.jump(failure->label()); + + masm.bind(&ok); + masm.setFramePushed(framePushed); + masm.loadTypedOrValue(Address(masm.getStackPointer(), 0), output); + masm.adjustStack(sizeof(Value)); + return true; +} \ No newline at end of file diff --git a/js/src/jit/CacheIRCompiler.h b/js/src/jit/CacheIRCompiler.h new file mode 100644 index 0000000000..0c6578f003 --- /dev/null +++ b/js/src/jit/CacheIRCompiler.h @@ -0,0 +1,752 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_CacheIRCompiler_h +#define jit_CacheIRCompiler_h + +#include "jit/CacheIR.h" + +namespace js { +namespace jit { + +// The ops below are defined in CacheIRCompiler and codegen is shared between +// BaselineCacheIRCompiler and IonCacheIRCompiler. +#define CACHE_IR_SHARED_OPS(_) \ + _(GuardIsObject) \ + _(GuardIsObjectOrNull) \ + _(GuardIsString) \ + _(GuardIsSymbol) \ + _(GuardIsInt32Index) \ + _(GuardType) \ + _(GuardClass) \ + _(GuardIsNativeFunction) \ + _(GuardIsProxy) \ + _(GuardIsCrossCompartmentWrapper) \ + _(GuardNotDOMProxy) \ + _(GuardSpecificInt32Immediate) \ + _(GuardMagicValue) \ + _(GuardNoUnboxedExpando) \ + _(GuardAndLoadUnboxedExpando) \ + _(GuardNoDetachedTypedObjects) \ + _(GuardNoDenseElements) \ + _(GuardAndGetIndexFromString) \ + _(LoadProto) \ + _(LoadEnclosingEnvironment) \ + _(LoadWrapperTarget) \ + _(LoadDOMExpandoValue) \ + _(LoadDOMExpandoValueIgnoreGeneration)\ + _(LoadUndefinedResult) \ + _(LoadBooleanResult) \ + _(LoadInt32ArrayLengthResult) \ + _(LoadUnboxedArrayLengthResult) \ + _(LoadArgumentsObjectLengthResult) \ + _(LoadFunctionLengthResult) \ + _(LoadStringLengthResult) \ + _(LoadStringCharResult) \ + _(LoadArgumentsObjectArgResult) \ + _(LoadDenseElementResult) \ + _(LoadDenseElementHoleResult) \ + _(LoadDenseElementExistsResult) \ + _(LoadDenseElementHoleExistsResult) \ + _(LoadUnboxedArrayElementResult) \ + _(LoadTypedElementResult) \ + _(LoadObjectResult) \ + _(LoadTypeOfObjectResult) \ + _(CallPrintString) \ + _(Breakpoint) \ + _(MegamorphicLoadSlotByValueResult) \ + _(MegamorphicHasOwnResult) \ + _(WrapResult) + +// Represents a Value on the Baseline frame's expression stack. Slot 0 is the +// value on top of the stack (the most recently pushed value), slot 1 is the +// value pushed before that, etc. +class BaselineFrameSlot +{ + uint32_t slot_; + + public: + explicit BaselineFrameSlot(uint32_t slot) : slot_(slot) {} + uint32_t slot() const { return slot_; } + + bool operator==(const BaselineFrameSlot& other) const { return slot_ == other.slot_; } + bool operator!=(const BaselineFrameSlot& other) const { return slot_ != other.slot_; } +}; + +// OperandLocation represents the location of an OperandId. The operand is +// either in a register or on the stack, and is either boxed or unboxed. +class OperandLocation +{ + public: + enum Kind { + Uninitialized = 0, + PayloadReg, + DoubleReg, + ValueReg, + PayloadStack, + ValueStack, + BaselineFrame, + Constant, + }; + + private: + Kind kind_; + + union Data { + struct { + Register reg; + JSValueType type; + } payloadReg; + FloatRegister doubleReg; + ValueOperand valueReg; + struct { + uint32_t stackPushed; + JSValueType type; + } payloadStack; + uint32_t valueStackPushed; + BaselineFrameSlot baselineFrameSlot; + Value constant; + + Data() : valueStackPushed(0) {} + }; + Data data_; + + public: + OperandLocation() : kind_(Uninitialized) {} + + Kind kind() const { return kind_; } + + void setUninitialized() { + kind_ = Uninitialized; + } + + ValueOperand valueReg() const { + MOZ_ASSERT(kind_ == ValueReg); + return data_.valueReg; + } + Register payloadReg() const { + MOZ_ASSERT(kind_ == PayloadReg); + return data_.payloadReg.reg; + } + FloatRegister doubleReg() const { + MOZ_ASSERT(kind_ == DoubleReg); + return data_.doubleReg; + } + uint32_t payloadStack() const { + MOZ_ASSERT(kind_ == PayloadStack); + return data_.payloadStack.stackPushed; + } + uint32_t valueStack() const { + MOZ_ASSERT(kind_ == ValueStack); + return data_.valueStackPushed; + } + JSValueType payloadType() const { + if (kind_ == PayloadReg) + return data_.payloadReg.type; + MOZ_ASSERT(kind_ == PayloadStack); + return data_.payloadStack.type; + } + Value constant() const { + MOZ_ASSERT(kind_ == Constant); + return data_.constant; + } + + BaselineFrameSlot baselineFrameSlot() const { + MOZ_ASSERT(kind_ == BaselineFrame); + return data_.baselineFrameSlot; + } + + void setPayloadReg(Register reg, JSValueType type) { + kind_ = PayloadReg; + data_.payloadReg.reg = reg; + data_.payloadReg.type = type; + } + void setDoubleReg(FloatRegister reg) { + kind_ = DoubleReg; + data_.doubleReg = reg; + } + void setValueReg(ValueOperand reg) { + kind_ = ValueReg; + data_.valueReg = reg; + } + void setPayloadStack(uint32_t stackPushed, JSValueType type) { + kind_ = PayloadStack; + data_.payloadStack.stackPushed = stackPushed; + data_.payloadStack.type = type; + } + void setValueStack(uint32_t stackPushed) { + kind_ = ValueStack; + data_.valueStackPushed = stackPushed; + } + void setConstant(const Value& v) { + kind_ = Constant; + data_.constant = v; + } + + void setBaselineFrame(BaselineFrameSlot slot) { + kind_ = BaselineFrame; + data_.baselineFrameSlot = slot; + } + + bool isInRegister() const { return kind_ == PayloadReg || kind_ == ValueReg; } + bool isOnStack() const { return kind_ == PayloadStack || kind_ == ValueStack; } + + size_t stackPushed() const { + if (kind_ == PayloadStack) + return data_.payloadStack.stackPushed; + MOZ_ASSERT(kind_ == ValueStack); + return data_.valueStackPushed; + } + size_t stackSizeInBytes() const { + if (kind_ == PayloadStack) + return sizeof(uintptr_t); + MOZ_ASSERT(kind_ == ValueStack); + return sizeof(js::Value); + } + void adjustStackPushed(int32_t diff) { + if (kind_ == PayloadStack) { + data_.payloadStack.stackPushed += diff; + return; + } + MOZ_ASSERT(kind_ == ValueStack); + data_.valueStackPushed += diff; + } + + bool aliasesReg(Register reg) const { + if (kind_ == PayloadReg) + return payloadReg() == reg; + if (kind_ == ValueReg) + return valueReg().aliases(reg); + return false; + } + bool aliasesReg(ValueOperand reg) const { +#if defined(JS_NUNBOX32) + return aliasesReg(reg.typeReg()) || aliasesReg(reg.payloadReg()); +#else + return aliasesReg(reg.valueReg()); +#endif + } + + bool aliasesReg(const OperandLocation& other) const; + + bool operator==(const OperandLocation& other) const; + bool operator!=(const OperandLocation& other) const { return !operator==(other); } +}; + +struct SpilledRegister +{ + Register reg; + uint32_t stackPushed; + + SpilledRegister(Register reg, uint32_t stackPushed) + : reg(reg), stackPushed(stackPushed) + {} + bool operator==(const SpilledRegister& other) const { + return reg == other.reg && stackPushed == other.stackPushed; + } + bool operator!=(const SpilledRegister& other) const { return !(*this == other); } +}; + +using SpilledRegisterVector = Vector; + +// Class to track and allocate registers while emitting IC code. +class MOZ_RAII CacheRegisterAllocator +{ + // The original location of the inputs to the cache. + Vector origInputLocations_; + + // The current location of each operand. + Vector operandLocations_; + + // Free lists for value- and payload-slots on stack + Vector freeValueSlots_; + Vector freePayloadSlots_; + + // The registers allocated while emitting the current CacheIR op. + // This prevents us from allocating a register and then immediately + // clobbering it for something else, while we're still holding on to it. + LiveGeneralRegisterSet currentOpRegs_; + + const AllocatableGeneralRegisterSet allocatableRegs_; + + // Registers that are currently unused and available. + AllocatableGeneralRegisterSet availableRegs_; + + // Registers that are available, but before use they must be saved and + // then restored when returning from the stub. + AllocatableGeneralRegisterSet availableRegsAfterSpill_; + + // Registers we took from availableRegsAfterSpill_ and spilled to the stack. + SpilledRegisterVector spilledRegs_; + + // The number of bytes pushed on the native stack. + uint32_t stackPushed_; + + // The index of the CacheIR instruction we're currently emitting. + uint32_t currentInstruction_; + + const CacheIRWriter& writer_; + + CacheRegisterAllocator(const CacheRegisterAllocator&) = delete; + CacheRegisterAllocator& operator=(const CacheRegisterAllocator&) = delete; + + void freeDeadOperandLocations(MacroAssembler& masm); + + void spillOperandToStack(MacroAssembler& masm, OperandLocation* loc); + void spillOperandToStackOrRegister(MacroAssembler& masm, OperandLocation* loc); + + void popPayload(MacroAssembler& masm, OperandLocation* loc, Register dest); + void popValue(MacroAssembler& masm, OperandLocation* loc, ValueOperand dest); + + public: + friend class AutoScratchRegister; + friend class AutoScratchRegisterExcluding; + + explicit CacheRegisterAllocator(const CacheIRWriter& writer) + : allocatableRegs_(GeneralRegisterSet::All()), + stackPushed_(0), + currentInstruction_(0), + writer_(writer) + {} + + [[nodiscard]] bool init(); + + void initAvailableRegs(const AllocatableGeneralRegisterSet& available) { + availableRegs_ = available; + } + + void initAvailableRegsAfterSpill(); + + void fixupAliasedInputs(MacroAssembler& masm); + + OperandLocation operandLocation(size_t i) const { + return operandLocations_[i]; + } + void setOperandLocation(size_t i, const OperandLocation& loc) { + operandLocations_[i] = loc; + } + + OperandLocation origInputLocation(size_t i) const { + return origInputLocations_[i]; + } + void initInputLocation(size_t i, ValueOperand reg) { + origInputLocations_[i].setValueReg(reg); + operandLocations_[i].setValueReg(reg); + } + void initInputLocation(size_t i, Register reg, JSValueType type) { + origInputLocations_[i].setPayloadReg(reg, type); + operandLocations_[i].setPayloadReg(reg, type); + } + + void initInputLocation(size_t i, FloatRegister reg) { + origInputLocations_[i].setDoubleReg(reg); + operandLocations_[i].setDoubleReg(reg); + } + void initInputLocation(size_t i, const Value& v) { + origInputLocations_[i].setConstant(v); + operandLocations_[i].setConstant(v); + } + + void initInputLocation(size_t i, BaselineFrameSlot slot) { + origInputLocations_[i].setBaselineFrame(slot); + operandLocations_[i].setBaselineFrame(slot); + } + + void initInputLocation(size_t i, const TypedOrValueRegister& reg); + void initInputLocation(size_t i, const ConstantOrRegister& value); + + const SpilledRegisterVector& spilledRegs() const { return spilledRegs_; } + + [[nodiscard]] bool setSpilledRegs(const SpilledRegisterVector& regs) { + spilledRegs_.clear(); + return spilledRegs_.appendAll(regs); + } + + void nextOp() { + currentOpRegs_.clear(); + currentInstruction_++; + } + + uint32_t stackPushed() const { + return stackPushed_; + } + + void setStackPushed(uint32_t pushed) { + stackPushed_ = pushed; + } + + bool isAllocatable(Register reg) const { + return allocatableRegs_.has(reg); + } + + // Allocates a new register. + Register allocateRegister(MacroAssembler& masm); + ValueOperand allocateValueRegister(MacroAssembler& masm); + + void allocateFixedRegister(MacroAssembler& masm, Register reg); + void allocateFixedValueRegister(MacroAssembler& masm, ValueOperand reg); + + // Releases a register so it can be reused later. + void releaseRegister(Register reg) { + MOZ_ASSERT(currentOpRegs_.has(reg)); + availableRegs_.add(reg); + currentOpRegs_.take(reg); + } + void releaseValueRegister(ValueOperand reg) { +#ifdef JS_NUNBOX32 + releaseRegister(reg.payloadReg()); + releaseRegister(reg.typeReg()); +#else + releaseRegister(reg.valueReg()); +#endif + } + + // Removes spilled values from the native stack. This should only be + // called after all registers have been allocated. + void discardStack(MacroAssembler& masm); + + Address addressOf(MacroAssembler& masm, BaselineFrameSlot slot) const; + + // Returns the register for the given operand. If the operand is currently + // not in a register, it will load it into one. + ValueOperand useValueRegister(MacroAssembler& masm, ValOperandId val); + ValueOperand useFixedValueRegister(MacroAssembler& masm, ValOperandId valId, ValueOperand reg); + Register useRegister(MacroAssembler& masm, TypedOperandId typedId); + + ConstantOrRegister useConstantOrRegister(MacroAssembler& masm, ValOperandId val); + + // Allocates an output register for the given operand. + Register defineRegister(MacroAssembler& masm, TypedOperandId typedId); + ValueOperand defineValueRegister(MacroAssembler& masm, ValOperandId val); + + // Returns |val|'s JSValueType or JSVAL_TYPE_UNKNOWN. + JSValueType knownType(ValOperandId val) const; + + // Emits code to restore registers and stack to the state at the start of + // the stub. + void restoreInputState(MacroAssembler& masm, bool discardStack = true); + + // Returns the set of registers storing the IC input operands. + GeneralRegisterSet inputRegisterSet() const; + + void saveIonLiveRegisters(MacroAssembler& masm, LiveRegisterSet liveRegs, + Register scratch, IonScript* ionScript); + void restoreIonLiveRegisters(MacroAssembler& masm, LiveRegisterSet liveRegs); +}; + +// RAII class to allocate a scratch register and release it when we're done +// with it. +class MOZ_RAII AutoScratchRegister +{ + CacheRegisterAllocator& alloc_; + Register reg_; + + AutoScratchRegister(const AutoScratchRegister&) = delete; + void operator=(const AutoScratchRegister&) = delete; + + public: + AutoScratchRegister(CacheRegisterAllocator& alloc, MacroAssembler& masm, + Register reg = InvalidReg) + : alloc_(alloc) + { + if (reg != InvalidReg) { + alloc.allocateFixedRegister(masm, reg); + reg_ = reg; + } else { + reg_ = alloc.allocateRegister(masm); + } + MOZ_ASSERT(alloc_.currentOpRegs_.has(reg_)); + } + ~AutoScratchRegister() { + alloc_.releaseRegister(reg_); + } + + Register get() const { return reg_; } + operator Register() const { return reg_; } +}; + +// Like AutoScratchRegister, but lets the caller specify a register that should +// not be allocated here. +class MOZ_RAII AutoScratchRegisterExcluding +{ + CacheRegisterAllocator& alloc_; + Register reg_; + + public: + AutoScratchRegisterExcluding(CacheRegisterAllocator& alloc, MacroAssembler& masm, + Register excluding) + : alloc_(alloc) + { + MOZ_ASSERT(excluding != InvalidReg); + + reg_ = alloc.allocateRegister(masm); + + if (reg_ == excluding) { + // We need a different register, so try again. + reg_ = alloc.allocateRegister(masm); + MOZ_ASSERT(reg_ != excluding); + alloc_.releaseRegister(excluding); + } + + MOZ_ASSERT(alloc_.currentOpRegs_.has(reg_)); + } + ~AutoScratchRegisterExcluding() { + alloc_.releaseRegister(reg_); + } + operator Register() const { return reg_; } +}; + +// The FailurePath class stores everything we need to generate a failure path +// at the end of the IC code. The failure path restores the input registers, if +// needed, and jumps to the next stub. +class FailurePath +{ + Vector inputs_; + SpilledRegisterVector spilledRegs_; + NonAssertingLabel label_; + uint32_t stackPushed_; + + public: + FailurePath() = default; + + FailurePath(FailurePath&& other) + : inputs_(Move(other.inputs_)), + spilledRegs_(Move(other.spilledRegs_)), + label_(other.label_), + stackPushed_(other.stackPushed_) + {} + + Label* label() { return &label_; } + + void setStackPushed(uint32_t i) { stackPushed_ = i; } + uint32_t stackPushed() const { return stackPushed_; } + + [[nodiscard]] bool appendInput(const OperandLocation& loc) { + return inputs_.append(loc); + } + OperandLocation input(size_t i) const { + return inputs_[i]; + } + + const SpilledRegisterVector& spilledRegs() const { return spilledRegs_; } + + [[nodiscard]] bool setSpilledRegs(const SpilledRegisterVector& regs) { + MOZ_ASSERT(spilledRegs_.empty()); + return spilledRegs_.appendAll(regs); + } + + // If canShareFailurePath(other) returns true, the same machine code will + // be emitted for two failure paths, so we can share them. + bool canShareFailurePath(const FailurePath& other) const; +}; + +class AutoOutputRegister; + +// Base class for BaselineCacheIRCompiler and IonCacheIRCompiler. +class MOZ_RAII CacheIRCompiler +{ + protected: + friend class AutoOutputRegister; + + enum class Mode { Baseline, Ion }; + + JSContext* cx_; + CacheIRReader reader; + const CacheIRWriter& writer_; + MacroAssembler masm; + + CacheRegisterAllocator allocator; + Vector failurePaths; + + // Float registers that are live. Registers not in this set can be + // clobbered and don't need to be saved before performing a VM call. + // Doing this for non-float registers is a bit more complicated because + // the IC register allocator allocates GPRs. + LiveFloatRegisterSet liveFloatRegs_; + + Maybe outputUnchecked_; + Mode mode_; + + // Whether this IC may read double values from uint32 arrays. + Maybe allowDoubleResult_; + + CacheIRCompiler(JSContext* cx, const CacheIRWriter& writer, Mode mode) + : cx_(cx), + reader(writer), + writer_(writer), + allocator(writer_), + liveFloatRegs_(FloatRegisterSet::All()), + mode_(mode) + { + MOZ_ASSERT(!writer.failed()); + } + + [[nodiscard]] bool addFailurePath(FailurePath** failure); + [[nodiscard]] bool emitFailurePath(size_t i); + + // Returns the set of volatile float registers that are live. These + // registers need to be saved when making non-GC calls with callWithABI. + FloatRegisterSet liveVolatileFloatRegs() const { + return FloatRegisterSet::Intersect(liveFloatRegs_.set(), FloatRegisterSet::Volatile()); + } + + void emitLoadTypedObjectResultShared(const Address& fieldAddr, Register scratch, + TypedThingLayout layout, uint32_t typeDescr, + const AutoOutputRegister& output); + + void emitStoreTypedObjectReferenceProp(ValueOperand val, ReferenceTypeDescr::Type type, + const Address& dest, Register scratch); + + private: + void emitPostBarrierShared(Register obj, const ConstantOrRegister& val, Register scratch, + Register maybeIndex); + + void emitPostBarrierShared(Register obj, ValueOperand val, Register scratch, + Register maybeIndex) { + emitPostBarrierShared(obj, ConstantOrRegister(val), scratch, maybeIndex); + } + + protected: + template + void emitPostBarrierSlot(Register obj, const T& val, Register scratch) { + emitPostBarrierShared(obj, val, scratch, InvalidReg); + } + + template + void emitPostBarrierElement(Register obj, const T& val, Register scratch, Register index) { + MOZ_ASSERT(index != InvalidReg); + emitPostBarrierShared(obj, val, scratch, index); + } + +#define DEFINE_SHARED_OP(op) [[nodiscard]] bool emit##op(); + CACHE_IR_SHARED_OPS(DEFINE_SHARED_OP) +#undef DEFINE_SHARED_OP +}; + +// Ensures the IC's output register is available for writing. +class MOZ_RAII AutoOutputRegister +{ + TypedOrValueRegister output_; + CacheRegisterAllocator& alloc_; + + AutoOutputRegister(const AutoOutputRegister&) = delete; + void operator=(const AutoOutputRegister&) = delete; + + public: + explicit AutoOutputRegister(CacheIRCompiler& compiler); + ~AutoOutputRegister(); + + Register maybeReg() const { + if (output_.hasValue()) + return output_.valueReg().scratchReg(); + if (!output_.typedReg().isFloat()) + return output_.typedReg().gpr(); + return InvalidReg; + } + + bool hasValue() const { return output_.hasValue(); } + ValueOperand valueReg() const { return output_.valueReg(); } + AnyRegister typedReg() const { return output_.typedReg(); } + + JSValueType type() const { + MOZ_ASSERT(!hasValue()); + return ValueTypeFromMIRType(output_.type()); + } + + operator TypedOrValueRegister() const { return output_; } +}; + +// Like AutoScratchRegister, but reuse a register of |output| if possible. +class MOZ_RAII AutoScratchRegisterMaybeOutput +{ + mozilla::Maybe scratch_; + Register scratchReg_; + + AutoScratchRegisterMaybeOutput(const AutoScratchRegisterMaybeOutput&) = delete; + void operator=(const AutoScratchRegisterMaybeOutput&) = delete; + + public: + AutoScratchRegisterMaybeOutput(CacheRegisterAllocator& alloc, MacroAssembler& masm, + const AutoOutputRegister& output) + { + scratchReg_ = output.maybeReg(); + if (scratchReg_ == InvalidReg) { + scratch_.emplace(alloc, masm); + scratchReg_ = scratch_.ref(); + } + } + + operator Register() const { return scratchReg_; } +}; + +// See the 'Sharing Baseline stub code' comment in CacheIR.h for a description +// of this class. +class CacheIRStubInfo +{ + // These fields don't require 8 bits, but GCC complains if these fields are + // smaller than the size of the enums. + CacheKind kind_ : 8; + ICStubEngine engine_ : 8; + bool makesGCCalls_ : 1; + uint8_t stubDataOffset_; + + const uint8_t* code_; + uint32_t length_; + const uint8_t* fieldTypes_; + + CacheIRStubInfo(CacheKind kind, ICStubEngine engine, bool makesGCCalls, + uint32_t stubDataOffset, const uint8_t* code, uint32_t codeLength, + const uint8_t* fieldTypes) + : kind_(kind), + engine_(engine), + makesGCCalls_(makesGCCalls), + stubDataOffset_(stubDataOffset), + code_(code), + length_(codeLength), + fieldTypes_(fieldTypes) + { + MOZ_ASSERT(kind_ == kind, "Kind must fit in bitfield"); + MOZ_ASSERT(engine_ == engine, "Engine must fit in bitfield"); + MOZ_ASSERT(stubDataOffset_ == stubDataOffset, "stubDataOffset must fit in uint8_t"); + } + + CacheIRStubInfo(const CacheIRStubInfo&) = delete; + CacheIRStubInfo& operator=(const CacheIRStubInfo&) = delete; + + public: + CacheKind kind() const { return kind_; } + ICStubEngine engine() const { return engine_; } + bool makesGCCalls() const { return makesGCCalls_; } + + const uint8_t* code() const { return code_; } + uint32_t codeLength() const { return length_; } + uint32_t stubDataOffset() const { return stubDataOffset_; } + + size_t stubDataSize() const; + + StubField::Type fieldType(uint32_t i) const { return (StubField::Type)fieldTypes_[i]; } + + static CacheIRStubInfo* New(CacheKind kind, ICStubEngine engine, bool canMakeCalls, + uint32_t stubDataOffset, const CacheIRWriter& writer); + + template + js::GCPtr& getStubField(Stub* stub, uint32_t field) const; + + template + js::GCPtr& getStubField(ICStub* stub, uint32_t field) const { + return getStubField(stub, field); + } + + void copyStubData(ICStub* src, ICStub* dest) const; +}; + +template +void TraceCacheIRStub(JSTracer* trc, T* stub, const CacheIRStubInfo* stubInfo); + +} // namespace jit +} // namespace js + +#endif /* jit_CacheIRCompiler_h */ \ No newline at end of file diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index ff72f7060c..63528a42f2 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -7725,8 +7725,7 @@ CodeGenerator::visitCharCodeAt(LCharCodeAt* lir) OutOfLineCode* ool = oolCallVM(CharCodeAtInfo, lir, ArgList(str, index), StoreRegisterTo(output)); - masm.branchIfRope(str, ool->entry()); - masm.loadStringChar(str, index, output); + masm.loadStringChar(str, index, output, ool->entry()); masm.bind(ool->rejoin()); } diff --git a/js/src/jit/IonCacheIRCompiler.cpp b/js/src/jit/IonCacheIRCompiler.cpp new file mode 100644 index 0000000000..06e736cb36 --- /dev/null +++ b/js/src/jit/IonCacheIRCompiler.cpp @@ -0,0 +1,2185 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/DebugOnly.h" + +#include "jit/CacheIRCompiler.h" +#include "jit/IonCaches.h" +#include "jit/IonIC.h" + +#include "jit/Linker.h" +#include "jit/SharedICHelpers.h" +#include "proxy/Proxy.h" + +#include "jscompartmentinlines.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::DebugOnly; + +namespace js { +namespace jit { + +// IonCacheIRCompiler compiles CacheIR to IonIC native code. +class MOZ_RAII IonCacheIRCompiler : public CacheIRCompiler +{ + public: + friend class AutoSaveLiveRegisters; + + IonCacheIRCompiler(JSContext* cx, const CacheIRWriter& writer, IonIC* ic, IonScript* ionScript, + IonICStub* stub, const PropertyTypeCheckInfo* typeCheckInfo) + : CacheIRCompiler(cx, writer, Mode::Ion), + writer_(writer), + ic_(ic), + ionScript_(ionScript), + stub_(stub), + typeCheckInfo_(typeCheckInfo), + nextStubField_(0), +#ifdef DEBUG + calledPrepareVMCall_(false), +#endif + savedLiveRegs_(false) + { + MOZ_ASSERT(ic_); + MOZ_ASSERT(ionScript_); + } + + [[nodiscard]] bool init(); + JitCode* compile(); + + private: + const CacheIRWriter& writer_; + IonIC* ic_; + IonScript* ionScript_; + + // The stub we're generating code for. + IonICStub* stub_; + + // Information necessary to generate property type checks. Non-null iff + // this is a SetProp/SetElem stub. + const PropertyTypeCheckInfo* typeCheckInfo_; + + CodeOffsetJump rejoinOffset_; + Vector nextCodeOffsets_; + Maybe liveRegs_; + Maybe stubJitCodeOffset_; + uint32_t nextStubField_; + +#ifdef DEBUG + bool calledPrepareVMCall_; +#endif + bool savedLiveRegs_; + + uintptr_t readStubWord(uint32_t offset, StubField::Type type) { + MOZ_ASSERT((offset % sizeof(uintptr_t)) == 0); + return writer_.readStubFieldForIon(nextStubField_++, type).asWord(); + } + uint64_t readStubInt64(uint32_t offset, StubField::Type type) { + MOZ_ASSERT((offset % sizeof(uintptr_t)) == 0); + return writer_.readStubFieldForIon(nextStubField_++, type).asInt64(); + } + int32_t int32StubField(uint32_t offset) { + return readStubWord(offset, StubField::Type::RawWord); + } + Shape* shapeStubField(uint32_t offset) { + return (Shape*)readStubWord(offset, StubField::Type::Shape); + } + JSObject* objectStubField(uint32_t offset) { + return (JSObject*)readStubWord(offset, StubField::Type::JSObject); + } + JSString* stringStubField(uint32_t offset) { + return (JSString*)readStubWord(offset, StubField::Type::String); + } + JS::Symbol* symbolStubField(uint32_t offset) { + return (JS::Symbol*)readStubWord(offset, StubField::Type::Symbol); + } + ObjectGroup* groupStubField(uint32_t offset) { + return (ObjectGroup*)readStubWord(offset, StubField::Type::ObjectGroup); + } + JSCompartment* compartmentStubField(uint32_t offset) { + return (JSCompartment*)readStubWord(offset, StubField::Type::RawWord); + } + jsid idStubField(uint32_t offset) { + return mozilla::BitwiseCast(readStubWord(offset, StubField::Type::Id)); + } + template + T rawWordStubField(uint32_t offset) { + static_assert(sizeof(T) == sizeof(uintptr_t), "T must have word size"); + return (T)readStubWord(offset, StubField::Type::RawWord); + } + template + T rawInt64StubField(uint32_t offset) { + static_assert(sizeof(T) == sizeof(int64_t), "T musthave int64 size"); + return (T)readStubInt64(offset, StubField::Type::RawInt64); + } + + uint64_t* expandoGenerationStubFieldPtr(uint32_t offset) { + DebugOnly generation = + readStubInt64(offset, StubField::Type::DOMExpandoGeneration); + uint64_t* ptr = reinterpret_cast(stub_->stubDataStart() + offset); + MOZ_ASSERT(*ptr == generation); + return ptr; + } + + void prepareVMCall(MacroAssembler& masm); + [[nodiscard]] bool callVM(MacroAssembler& masm, const VMFunction& fun); + + [[nodiscard]] bool emitAddAndStoreSlotShared(CacheOp op); + + bool needsPostBarrier() const { + return ic_->asSetPropertyIC()->needsPostBarrier(); + } + + void pushStubCodePointer() { + stubJitCodeOffset_.emplace(masm.PushWithPatch(ImmPtr((void*)-1))); + } + +#define DEFINE_OP(op) [[nodiscard]] bool emit##op(); + CACHE_IR_OPS(DEFINE_OP) +#undef DEFINE_OP +}; + +// AutoSaveLiveRegisters must be used when we make a call that can GC. The +// constructor ensures all live registers are stored on the stack (where the GC +// expects them) and the destructor restores these registers. +class MOZ_RAII AutoSaveLiveRegisters +{ + IonCacheIRCompiler& compiler_; + + AutoSaveLiveRegisters(const AutoSaveLiveRegisters&) = delete; + void operator=(const AutoSaveLiveRegisters&) = delete; + + public: + explicit AutoSaveLiveRegisters(IonCacheIRCompiler& compiler) + : compiler_(compiler) + { + MOZ_ASSERT(compiler_.liveRegs_.isSome()); + compiler_.allocator.saveIonLiveRegisters(compiler_.masm, + compiler_.liveRegs_.ref(), + compiler_.ic_->scratchRegisterForEntryJump(), + compiler_.ionScript_); + compiler_.savedLiveRegs_ = true; + } + ~AutoSaveLiveRegisters() { + MOZ_ASSERT(compiler_.stubJitCodeOffset_.isSome(), "Must have pushed JitCode* pointer"); + compiler_.allocator.restoreIonLiveRegisters(compiler_.masm, compiler_.liveRegs_.ref()); + MOZ_ASSERT(compiler_.masm.framePushed() == compiler_.ionScript_->frameSize()); + } +}; + +} // namespace jit +} // namespace js + +#define DEFINE_SHARED_OP(op) \ + bool IonCacheIRCompiler::emit##op() { return CacheIRCompiler::emit##op(); } + CACHE_IR_SHARED_OPS(DEFINE_SHARED_OP) +#undef DEFINE_SHARED_OP + +void +CacheRegisterAllocator::saveIonLiveRegisters(MacroAssembler& masm, LiveRegisterSet liveRegs, + Register scratch, IonScript* ionScript) +{ + // We have to push all registers in liveRegs on the stack. It's possible we + // stored other values in our live registers and stored operands on the + // stack (where our live registers should go), so this requires some careful + // work. Try to keep it simple by taking one small step at a time. + + // Step 1. Discard any dead operands so we can reuse their registers. + freeDeadOperandLocations(masm); + + // Step 2. Figure out the size of our live regs. + size_t sizeOfLiveRegsInBytes = + liveRegs.gprs().size() * sizeof(intptr_t) + + liveRegs.fpus().getPushSizeInBytes(); + + MOZ_ASSERT(sizeOfLiveRegsInBytes > 0); + + // Step 3. Ensure all non-input operands are on the stack. + size_t numInputs = writer_.numInputOperands(); + for (size_t i = numInputs; i < operandLocations_.length(); i++) { + OperandLocation& loc = operandLocations_[i]; + if (loc.isInRegister()) + spillOperandToStack(masm, &loc); + } + + // Step 4. Restore the register state, but don't discard the stack as + // non-input operands are stored there. + restoreInputState(masm, /* shouldDiscardStack = */ false); + + // We just restored the input state, so no input operands should be stored + // on the stack. +#ifdef DEBUG + for (size_t i = 0; i < numInputs; i++) { + const OperandLocation& loc = operandLocations_[i]; + MOZ_ASSERT(!loc.isOnStack()); + } +#endif + + // Step 5. At this point our register state is correct. Stack values, + // however, may cover the space where we have to store the live registers. + // Move them out of the way. + + bool hasOperandOnStack = false; + for (size_t i = numInputs; i < operandLocations_.length(); i++) { + OperandLocation& loc = operandLocations_[i]; + if (!loc.isOnStack()) + continue; + + hasOperandOnStack = true; + + size_t operandSize = loc.stackSizeInBytes(); + size_t operandStackPushed = loc.stackPushed(); + MOZ_ASSERT(operandSize > 0); + MOZ_ASSERT(stackPushed_ >= operandStackPushed); + MOZ_ASSERT(operandStackPushed >= operandSize); + + // If this operand doesn't cover the live register space, there's + // nothing to do. + if (operandStackPushed - operandSize >= sizeOfLiveRegsInBytes) { + MOZ_ASSERT(stackPushed_ > sizeOfLiveRegsInBytes); + continue; + } + + // Reserve stack space for the live registers if needed. + if (sizeOfLiveRegsInBytes > stackPushed_) { + size_t extraBytes = sizeOfLiveRegsInBytes - stackPushed_; + MOZ_ASSERT((extraBytes % sizeof(uintptr_t)) == 0); + masm.subFromStackPtr(Imm32(extraBytes)); + stackPushed_ += extraBytes; + } + + // Push the operand below the live register space. + if (loc.kind() == OperandLocation::PayloadStack) { + masm.push(Address(masm.getStackPointer(), stackPushed_ - operandStackPushed)); + stackPushed_ += operandSize; + loc.setPayloadStack(stackPushed_, loc.payloadType()); + continue; + } + MOZ_ASSERT(loc.kind() == OperandLocation::ValueStack); + masm.pushValue(Address(masm.getStackPointer(), stackPushed_ - operandStackPushed)); + stackPushed_ += operandSize; + loc.setValueStack(stackPushed_); + } + + // Step 6. If we have any operands on the stack, adjust their stackPushed + // values to not include sizeOfLiveRegsInBytes (this simplifies code down + // the line). Then push/store the live registers. + if (hasOperandOnStack) { + MOZ_ASSERT(stackPushed_ > sizeOfLiveRegsInBytes); + stackPushed_ -= sizeOfLiveRegsInBytes; + + for (size_t i = numInputs; i < operandLocations_.length(); i++) { + OperandLocation& loc = operandLocations_[i]; + if (loc.isOnStack()) + loc.adjustStackPushed(-int32_t(sizeOfLiveRegsInBytes)); + } + + size_t stackBottom = stackPushed_ + sizeOfLiveRegsInBytes; + masm.storeRegsInMask(liveRegs, Address(masm.getStackPointer(), stackBottom), scratch); + masm.setFramePushed(masm.framePushed() + sizeOfLiveRegsInBytes); + } else { + // If no operands are on the stack, discard the unused stack space. + if (stackPushed_ > 0) { + masm.addToStackPtr(Imm32(stackPushed_)); + stackPushed_ = 0; + } + masm.PushRegsInMask(liveRegs); + } + freePayloadSlots_.clear(); + freeValueSlots_.clear(); + + MOZ_ASSERT(masm.framePushed() == ionScript->frameSize() + sizeOfLiveRegsInBytes); + + // Step 7. All live registers and non-input operands are stored on the stack + // now, so at this point all registers except for the input registers are + // available. + availableRegs_.set() = GeneralRegisterSet::Not(inputRegisterSet()); + availableRegsAfterSpill_.set() = GeneralRegisterSet(); + + // Step 8. We restored our input state, so we have to fix up aliased input + // registers again. + fixupAliasedInputs(masm); +} + +void +CacheRegisterAllocator::restoreIonLiveRegisters(MacroAssembler& masm, LiveRegisterSet liveRegs) +{ + masm.PopRegsInMask(liveRegs); + + availableRegs_.set() = GeneralRegisterSet(); + availableRegsAfterSpill_.set() = GeneralRegisterSet::All(); +} + +void +IonCacheIRCompiler::prepareVMCall(MacroAssembler& masm) +{ + uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS, + IonICCallFrameLayout::Size()); + pushStubCodePointer(); + masm.Push(Imm32(descriptor)); + masm.Push(ImmPtr(GetReturnAddressToIonCode(cx_))); + +#ifdef DEBUG + calledPrepareVMCall_ = true; +#endif +} + +bool +IonCacheIRCompiler::callVM(MacroAssembler& masm, const VMFunction& fun) +{ + MOZ_ASSERT(calledPrepareVMCall_); + + JitCode* code = cx_->runtime()->jitRuntime()->getVMWrapper(fun); + if (!code) + return false; + + uint32_t frameSize = fun.explicitStackSlots() * sizeof(void*); + uint32_t descriptor = MakeFrameDescriptor(frameSize, JitFrame_IonICCall, + ExitFrameLayout::Size()); + masm.Push(Imm32(descriptor)); + masm.callJit(code); + + // Remove rest of the frame left on the stack. We remove the return address + // which is implicitly poped when returning. + int framePop = sizeof(ExitFrameLayout) - sizeof(void*); + + // Pop arguments from framePushed. + masm.implicitPop(frameSize + framePop); + masm.freeStack(IonICCallFrameLayout::Size()); + return true; +} + +bool +IonCacheIRCompiler::init() +{ + if (!allocator.init()) + return false; + + size_t numInputs = writer_.numInputOperands(); + + AllocatableGeneralRegisterSet available; + + switch (ic_->kind()) { + case CacheKind::GetProp: + case CacheKind::GetElem: { + IonGetPropertyIC* ic = ic_->asGetPropertyIC(); + TypedOrValueRegister output = ic->output(); + + if (output.hasValue()) + available.add(output.valueReg()); + else if (!output.typedReg().isFloat()) + available.add(output.typedReg().gpr()); + + if (ic->maybeTemp() != InvalidReg) + available.add(ic->maybeTemp()); + + liveRegs_.emplace(ic->liveRegs()); + outputUnchecked_.emplace(output); + + allowDoubleResult_.emplace(ic->allowDoubleResult()); + + MOZ_ASSERT(numInputs == 1 || numInputs == 2); + + allocator.initInputLocation(0, ic->value()); + if (numInputs > 1) + allocator.initInputLocation(1, ic->id()); + break; + } + case CacheKind::GetPropSuper: + case CacheKind::GetElemSuper: { + IonGetPropSuperIC* ic = ic_->asGetPropSuperIC(); + TypedOrValueRegister output = ic->output(); + + available.add(output.valueReg()); + + liveRegs_.emplace(ic->liveRegs()); + outputUnchecked_.emplace(output); + + allowDoubleResult_.emplace(true); + + MOZ_ASSERT(numInputs == 2 || numInputs == 3); + + allocator.initInputLocation(0, ic->object(), JSVAL_TYPE_OBJECT); + + if (ic->kind() == CacheKind::GetPropSuper) { + MOZ_ASSERT(numInputs == 2); + allocator.initInputLocation(1, ic->receiver()); + } else { + MOZ_ASSERT(numInputs == 3); + allocator.initInputLocation(1, ic->id()); + allocator.initInputLocation(2, ic->receiver()); + } + break; + } + case CacheKind::SetProp: + case CacheKind::SetElem: { + IonSetPropertyIC* ic = ic_->asSetPropertyIC(); + + available.add(ic->temp()); + + liveRegs_.emplace(ic->liveRegs()); + + allocator.initInputLocation(0, ic->object(), JSVAL_TYPE_OBJECT); + + if (ic->kind() == CacheKind::SetProp) { + MOZ_ASSERT(numInputs == 2); + allocator.initInputLocation(1, ic->rhs()); + } else { + MOZ_ASSERT(numInputs == 3); + allocator.initInputLocation(1, ic->id()); + allocator.initInputLocation(2, ic->rhs()); + } + break; + } + case CacheKind::GetName: { + IonGetNameIC* ic = ic_->asGetNameIC(); + ValueOperand output = ic->output(); + + available.add(output); + available.add(ic->temp()); + + liveRegs_.emplace(ic->liveRegs()); + outputUnchecked_.emplace(output); + + MOZ_ASSERT(numInputs == 1); + allocator.initInputLocation(0, ic->environment(), JSVAL_TYPE_OBJECT); + break; + } + case CacheKind::BindName: { + IonBindNameIC* ic = ic_->asBindNameIC(); + Register output = ic->output(); + + available.add(output); + available.add(ic->temp()); + + liveRegs_.emplace(ic->liveRegs()); + outputUnchecked_.emplace(TypedOrValueRegister(MIRType::Object, AnyRegister(output))); + + MOZ_ASSERT(numInputs == 1); + allocator.initInputLocation(0, ic->environment(), JSVAL_TYPE_OBJECT); + break; + } + case CacheKind::In: { + IonInIC* ic = ic_->asInIC(); + Register output = ic->output(); + + available.add(output); + + liveRegs_.emplace(ic->liveRegs()); + outputUnchecked_.emplace(TypedOrValueRegister(MIRType::Boolean, AnyRegister(output))); + + MOZ_ASSERT(numInputs == 2); + allocator.initInputLocation(0, ic->key()); + allocator.initInputLocation(1, TypedOrValueRegister(MIRType::Object, + AnyRegister(ic->object()))); + break; + } + case CacheKind::HasOwn: { + IonHasOwnIC* ic = ic_->asHasOwnIC(); + Register output = ic->output(); + + available.add(output); + + liveRegs_.emplace(ic->liveRegs()); + outputUnchecked_.emplace(TypedOrValueRegister(MIRType::Boolean, AnyRegister(output))); + + MOZ_ASSERT(numInputs == 2); + allocator.initInputLocation(0, ic->id()); + allocator.initInputLocation(1, ic->value()); + break; + } + case CacheKind::Call: + case CacheKind::TypeOf: + MOZ_CRASH("Unsupported IC"); + } + + if (liveRegs_) + liveFloatRegs_ = LiveFloatRegisterSet(liveRegs_->fpus()); + + allocator.initAvailableRegs(available); + allocator.initAvailableRegsAfterSpill(); + return true; +} + +JitCode* +IonCacheIRCompiler::compile() +{ + masm.setFramePushed(ionScript_->frameSize()); + if (cx_->runtime()->spsProfiler().enabled()) + masm.enableProfilingInstrumentation(); + + allocator.fixupAliasedInputs(masm); + + do { + switch (reader.readOp()) { +#define DEFINE_OP(op) \ + case CacheOp::op: \ + if (!emit##op()) \ + return nullptr; \ + break; + CACHE_IR_OPS(DEFINE_OP) +#undef DEFINE_OP + + default: + MOZ_CRASH("Invalid op"); + } + + allocator.nextOp(); + } while (reader.more()); + + MOZ_ASSERT(nextStubField_ == writer_.numStubFields()); + + masm.assumeUnreachable("Should have returned from IC"); + + // Done emitting the main IC code. Now emit the failure paths. + for (size_t i = 0; i < failurePaths.length(); i++) { + if (!emitFailurePath(i)) + return nullptr; + Register scratch = ic_->scratchRegisterForEntryJump(); + CodeOffset offset = masm.movWithPatch(ImmWord(-1), scratch); + masm.jump(Address(scratch, 0)); + if (!nextCodeOffsets_.append(offset)) + return nullptr; + } + + Linker linker(masm); + AutoFlushICache afc("getStubCode"); + Rooted newStubCode(cx_, linker.newCode(cx_, ION_CODE)); + if (!newStubCode) { + cx_->recoverFromOutOfMemory(); + return nullptr; + } + + rejoinOffset_.fixup(&masm); + CodeLocationJump rejoinJump(newStubCode, rejoinOffset_); + PatchJump(rejoinJump, ic_->rejoinLabel()); + + for (CodeOffset offset : nextCodeOffsets_) { + Assembler::PatchDataWithValueCheck(CodeLocationLabel(newStubCode, offset), + ImmPtr(stub_->nextCodeRawPtr()), + ImmPtr((void*)-1)); + } + if (stubJitCodeOffset_) { + Assembler::PatchDataWithValueCheck(CodeLocationLabel(newStubCode, *stubJitCodeOffset_), + ImmPtr(newStubCode.get()), + ImmPtr((void*)-1)); + } + return newStubCode; +} + +bool +IonCacheIRCompiler::emitGuardShape() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Shape* shape = shapeStubField(reader.stubOffset()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.branchTestObjShape(Assembler::NotEqual, obj, shape, failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitGuardGroup() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ObjectGroup* group = groupStubField(reader.stubOffset()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.branchTestObjGroup(Assembler::NotEqual, obj, group, failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitGuardGroupHasUnanalyzedNewScript() +{ + ObjectGroup* group = groupStubField(reader.stubOffset()); + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.movePtr(ImmGCPtr(group), scratch1); + masm.guardGroupHasUnanalyzedNewScript(scratch1, scratch2, failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitGuardProto() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + JSObject* proto = objectStubField(reader.stubOffset()); + + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.loadObjProto(obj, scratch); + masm.branchPtr(Assembler::NotEqual, scratch, ImmGCPtr(proto), failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitGuardCompartment() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + objectStubField(reader.stubOffset()); // Read global wrapper. + JSCompartment* compartment = compartmentStubField(reader.stubOffset()); + + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.loadPtr(Address(obj, JSObject::offsetOfGroup()), scratch); + masm.loadPtr(Address(scratch, ObjectGroup::offsetOfCompartment()), scratch); + masm.branchPtr(Assembler::NotEqual, scratch, ImmPtr(compartment), failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitGuardSpecificObject() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + JSObject* expected = objectStubField(reader.stubOffset()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.branchPtr(Assembler::NotEqual, obj, ImmGCPtr(expected), failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitGuardSpecificAtom() +{ + Register str = allocator.useRegister(masm, reader.stringOperandId()); + AutoScratchRegister scratch(allocator, masm); + + JSAtom* atom = &stringStubField(reader.stubOffset())->asAtom(); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + Label done; + masm.branchPtr(Assembler::Equal, str, ImmGCPtr(atom), &done); + + // The pointers are not equal, so if the input string is also an atom it + // must be a different string. + masm.branchTest32(Assembler::NonZero, Address(str, JSString::offsetOfFlags()), + Imm32(JSString::ATOM_BIT), failure->label()); + + // Check the length. + masm.branch32(Assembler::NotEqual, Address(str, JSString::offsetOfLength()), + Imm32(atom->length()), failure->label()); + + // We have a non-atomized string with the same length. Call a helper + // function to do the comparison. + LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + masm.PushRegsInMask(volatileRegs); + + masm.setupUnalignedABICall(scratch); + masm.movePtr(ImmGCPtr(atom), scratch); + masm.passABIArg(scratch); + masm.passABIArg(str); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, EqualStringsHelper)); + masm.mov(ReturnReg, scratch); + + LiveRegisterSet ignore; + ignore.add(scratch); + masm.PopRegsInMaskIgnore(volatileRegs, ignore); + masm.branchIfFalseBool(scratch, failure->label()); + + masm.bind(&done); + return true; +} + +bool +IonCacheIRCompiler::emitGuardSpecificSymbol() +{ + Register sym = allocator.useRegister(masm, reader.symbolOperandId()); + JS::Symbol* expected = symbolStubField(reader.stubOffset()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.branchPtr(Assembler::NotEqual, sym, ImmGCPtr(expected), failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitLoadFixedSlotResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + masm.loadTypedOrValue(Address(obj, offset), output); + return true; +} + +bool +IonCacheIRCompiler::emitLoadDynamicSlotResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch); + masm.loadTypedOrValue(Address(scratch, offset), output); + return true; +} + +bool +IonCacheIRCompiler::emitMegamorphicLoadSlotResult() +{ + AutoOutputRegister output(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + PropertyName* name = stringStubField(reader.stubOffset())->asAtom().asPropertyName(); + bool handleMissing = reader.readBool(); + + AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output); + AutoScratchRegister scratch2(allocator, masm); + AutoScratchRegister scratch3(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // The object must be Native. + masm.loadObjClass(obj, scratch3); + masm.branchTest32(Assembler::NonZero, Address(scratch3, Class::offsetOfFlags()), + Imm32(Class::NON_NATIVE), failure->label()); + + masm.Push(UndefinedValue()); + masm.moveStackPtrTo(scratch3.get()); + + LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + volatileRegs.takeUnchecked(scratch1); + volatileRegs.takeUnchecked(scratch2); + volatileRegs.takeUnchecked(scratch3); + masm.PushRegsInMask(volatileRegs); + + masm.setupUnalignedABICall(scratch1); + masm.loadJSContext(scratch1); + masm.passABIArg(scratch1); + masm.passABIArg(obj); + masm.movePtr(ImmGCPtr(name), scratch2); + masm.passABIArg(scratch2); + masm.passABIArg(scratch3); + if (handleMissing) + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, (GetNativeDataProperty))); + else + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, (GetNativeDataProperty))); + masm.mov(ReturnReg, scratch2); + masm.PopRegsInMask(volatileRegs); + + masm.loadTypedOrValue(Address(masm.getStackPointer(), 0), output); + masm.adjustStack(sizeof(Value)); + + masm.branchIfFalseBool(scratch2, failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitMegamorphicStoreSlot() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + PropertyName* name = stringStubField(reader.stubOffset())->asAtom().asPropertyName(); + ValueOperand val = allocator.useValueRegister(masm, reader.valOperandId()); + bool needsTypeBarrier = reader.readBool(); + + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.Push(val); + masm.moveStackPtrTo(val.scratchReg()); + + LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + volatileRegs.takeUnchecked(scratch1); + volatileRegs.takeUnchecked(scratch2); + volatileRegs.takeUnchecked(val); + masm.PushRegsInMask(volatileRegs); + + masm.setupUnalignedABICall(scratch1); + masm.loadJSContext(scratch1); + masm.passABIArg(scratch1); + masm.passABIArg(obj); + masm.movePtr(ImmGCPtr(name), scratch2); + masm.passABIArg(scratch2); + masm.passABIArg(val.scratchReg()); + if (needsTypeBarrier) + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, (SetNativeDataProperty))); + else + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, (SetNativeDataProperty))); + masm.mov(ReturnReg, scratch1); + masm.PopRegsInMask(volatileRegs); + + masm.loadValue(Address(masm.getStackPointer(), 0), val); + masm.adjustStack(sizeof(Value)); + + masm.branchIfFalseBool(scratch1, failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitGuardHasGetterSetter() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Shape* shape = shapeStubField(reader.stubOffset()); + + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + volatileRegs.takeUnchecked(scratch1); + volatileRegs.takeUnchecked(scratch2); + masm.PushRegsInMask(volatileRegs); + + masm.setupUnalignedABICall(scratch1); + masm.loadJSContext(scratch1); + masm.passABIArg(scratch1); + masm.passABIArg(obj); + masm.movePtr(ImmGCPtr(shape), scratch2); + masm.passABIArg(scratch2); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ObjectHasGetterSetter)); + masm.mov(ReturnReg, scratch1); + masm.PopRegsInMask(volatileRegs); + + masm.branchIfFalseBool(scratch1, failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitCallScriptedGetterResult() +{ + AutoSaveLiveRegisters save(*this); + AutoOutputRegister output(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + JSFunction* target = &objectStubField(reader.stubOffset())->as(); + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + uint32_t framePushedBefore = masm.framePushed(); + + // Construct IonICCallFrameLayout. + uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS, + IonICCallFrameLayout::Size()); + pushStubCodePointer(); + masm.Push(Imm32(descriptor)); + masm.Push(ImmPtr(GetReturnAddressToIonCode(cx_))); + + // The JitFrameLayout pushed below will be aligned to JitStackAlignment, + // so we just have to make sure the stack is aligned after we push the + // |this| + argument Values. + uint32_t argSize = (target->nargs() + 1) * sizeof(Value); + uint32_t padding = ComputeByteAlignment(masm.framePushed() + argSize, JitStackAlignment); + MOZ_ASSERT(padding % sizeof(uintptr_t) == 0); + MOZ_ASSERT(padding < JitStackAlignment); + masm.reserveStack(padding); + + for (size_t i = 0; i < target->nargs(); i++) + masm.Push(UndefinedValue()); + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(obj))); + + masm.movePtr(ImmGCPtr(target), scratch); + + descriptor = MakeFrameDescriptor(argSize + padding, JitFrame_IonICCall, + JitFrameLayout::Size()); + masm.Push(Imm32(0)); // argc + masm.Push(scratch); + masm.Push(Imm32(descriptor)); + + // Check stack alignment. Add sizeof(uintptr_t) for the return address. + MOZ_ASSERT(((masm.framePushed() + sizeof(uintptr_t)) % JitStackAlignment) == 0); + + // The getter has JIT code now and we will only discard the getter's JIT + // code when discarding all JIT code in the Zone, so we can assume it'll + // still have JIT code. + MOZ_ASSERT(target->hasJITCode()); + masm.loadPtr(Address(scratch, JSFunction::offsetOfNativeOrScript()), scratch); + masm.loadBaselineOrIonRaw(scratch, scratch, nullptr); + masm.callJit(scratch); + masm.storeCallResultValue(output); + + masm.freeStack(masm.framePushed() - framePushedBefore); + return true; +} + +bool +IonCacheIRCompiler::emitCallNativeGetterResult() +{ + AutoSaveLiveRegisters save(*this); + AutoOutputRegister output(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + JSFunction* target = &objectStubField(reader.stubOffset())->as(); + MOZ_ASSERT(target->isNative()); + + AutoScratchRegister argJSContext(allocator, masm); + AutoScratchRegister argUintN(allocator, masm); + AutoScratchRegister argVp(allocator, masm); + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + // Native functions have the signature: + // bool (*)(JSContext*, unsigned, Value* vp) + // Where vp[0] is space for an outparam, vp[1] is |this|, and vp[2] onward + // are the function arguments. + + // Construct vp array: + // Push object value for |this| + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(obj))); + // Push callee/outparam. + masm.Push(ObjectValue(*target)); + + // Preload arguments into registers. + masm.loadJSContext(argJSContext); + masm.move32(Imm32(0), argUintN); + masm.moveStackPtrTo(argVp.get()); + + // Push marking data for later use. + masm.Push(argUintN); + pushStubCodePointer(); + + if (!masm.icBuildOOLFakeExitFrame(GetReturnAddressToIonCode(cx_), save)) + return false; + masm.enterFakeExitFrame(scratch, IonOOLNativeExitFrameLayoutToken); + + // Construct and execute call. + masm.setupUnalignedABICall(scratch); + masm.passABIArg(argJSContext); + masm.passABIArg(argUintN); + masm.passABIArg(argVp); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, target->native())); + + // Test for failure. + masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel()); + + // Load the outparam vp[0] into output register(s). + Address outparam(masm.getStackPointer(), IonOOLNativeExitFrameLayout::offsetOfResult()); + masm.loadValue(outparam, output.valueReg()); + + masm.adjustStack(IonOOLNativeExitFrameLayout::Size(0)); + return true; +} + +bool +IonCacheIRCompiler::emitCallProxyGetResult() +{ + AutoSaveLiveRegisters save(*this); + AutoOutputRegister output(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + jsid id = idStubField(reader.stubOffset()); + + // ProxyGetProperty(JSContext* cx, HandleObject proxy, HandleId id, + // MutableHandleValue vp) + AutoScratchRegisterMaybeOutput argJSContext(allocator, masm, output); + AutoScratchRegister argProxy(allocator, masm); + AutoScratchRegister argId(allocator, masm); + AutoScratchRegister argVp(allocator, masm); + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + // Push stubCode for marking. + pushStubCodePointer(); + + // Push args on stack first so we can take pointers to make handles. + masm.Push(UndefinedValue()); + masm.moveStackPtrTo(argVp.get()); + + masm.Push(id, scratch); + masm.moveStackPtrTo(argId.get()); + + // Push the proxy. Also used as receiver. + masm.Push(obj); + masm.moveStackPtrTo(argProxy.get()); + + masm.loadJSContext(argJSContext); + + if (!masm.icBuildOOLFakeExitFrame(GetReturnAddressToIonCode(cx_), save)) + return false; + masm.enterFakeExitFrame(scratch, IonOOLProxyExitFrameLayoutToken); + + // Make the call. + masm.setupUnalignedABICall(scratch); + masm.passABIArg(argJSContext); + masm.passABIArg(argProxy); + masm.passABIArg(argId); + masm.passABIArg(argVp); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ProxyGetProperty)); + + // Test for failure. + masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel()); + + // Load the outparam vp[0] into output register(s). + Address outparam(masm.getStackPointer(), IonOOLProxyExitFrameLayout::offsetOfResult()); + masm.loadValue(outparam, output.valueReg()); + + // masm.leaveExitFrame & pop locals + masm.adjustStack(IonOOLProxyExitFrameLayout::Size()); + return true; +} + +typedef bool (*ProxyGetPropertyByValueFn)(JSContext*, HandleObject, HandleValue, MutableHandleValue); +static const VMFunction ProxyGetPropertyByValueInfo = + FunctionInfo(ProxyGetPropertyByValue, "ProxyGetPropertyByValue"); + +bool +IonCacheIRCompiler::emitCallProxyGetByValueResult() +{ + AutoSaveLiveRegisters save(*this); + AutoOutputRegister output(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ValueOperand idVal = allocator.useValueRegister(masm, reader.valOperandId()); + + allocator.discardStack(masm); + + prepareVMCall(masm); + + masm.Push(idVal); + masm.Push(obj); + + if (!callVM(masm, ProxyGetPropertyByValueInfo)) + return false; + + masm.storeCallResultValue(output); + return true; +} + +typedef bool (*ProxyHasOwnFn)(JSContext*, HandleObject, HandleValue, MutableHandleValue); +static const VMFunction ProxyHasOwnInfo = FunctionInfo(ProxyHasOwn, "ProxyHasOwn"); + +bool +IonCacheIRCompiler::emitCallProxyHasOwnResult() +{ + AutoSaveLiveRegisters save(*this); + AutoOutputRegister output(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ValueOperand idVal = allocator.useValueRegister(masm, reader.valOperandId()); + + allocator.discardStack(masm); + + prepareVMCall(masm); + + masm.Push(idVal); + masm.Push(obj); + + if (!callVM(masm, ProxyHasOwnInfo)) + return false; + + masm.storeCallResultValue(output); + return true; +} + +bool +IonCacheIRCompiler::emitLoadUnboxedPropertyResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + + JSValueType fieldType = reader.valueType(); + int32_t fieldOffset = int32StubField(reader.stubOffset()); + masm.loadUnboxedProperty(Address(obj, fieldOffset), fieldType, output); + return true; +} + +bool +IonCacheIRCompiler::emitGuardFrameHasNoArgumentsObject() +{ + MOZ_CRASH("Baseline-specific op"); +} + +bool +IonCacheIRCompiler::emitLoadFrameCalleeResult() +{ + MOZ_CRASH("Baseline-specific op"); +} + +bool +IonCacheIRCompiler::emitLoadFrameNumActualArgsResult() +{ + MOZ_CRASH("Baseline-specific op"); +} + +bool +IonCacheIRCompiler::emitLoadFrameArgumentResult() +{ + MOZ_CRASH("Baseline-specific op"); +} + +bool +IonCacheIRCompiler::emitLoadEnvironmentFixedSlotResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Check for uninitialized lexicals. + Address slot(obj, offset); + masm.branchTestMagic(Assembler::Equal, slot, failure->label()); + + // Load the value. + masm.loadTypedOrValue(slot, output); + return true; +} + +bool +IonCacheIRCompiler::emitLoadEnvironmentDynamicSlotResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch); + + // Check for uninitialized lexicals. + Address slot(scratch, offset); + masm.branchTestMagic(Assembler::Equal, slot, failure->label()); + + // Load the value. + masm.loadTypedOrValue(slot, output); + return true; +} + +bool +IonCacheIRCompiler::emitLoadStringResult() +{ + MOZ_CRASH("not used in ion"); +} + +typedef bool (*StringSplitHelperFn)(JSContext*, HandleString, HandleString, HandleObjectGroup, + uint32_t limit, MutableHandleValue); +static const VMFunction StringSplitHelperInfo = + FunctionInfo(StringSplitHelper, "StringSplitHelper"); + +bool +IonCacheIRCompiler::emitCallStringSplitResult() +{ + AutoSaveLiveRegisters save(*this); + AutoOutputRegister output(*this); + + Register str = allocator.useRegister(masm, reader.stringOperandId()); + Register sep = allocator.useRegister(masm, reader.stringOperandId()); + ObjectGroup* group = groupStubField(reader.stubOffset()); + + allocator.discardStack(masm); + + prepareVMCall(masm); + + masm.Push(str); + masm.Push(sep); + masm.Push(ImmGCPtr(group)); + masm.Push(Imm32(INT32_MAX)); + + if (!callVM(masm, StringSplitHelperInfo)) + return false; + + masm.storeCallResultValue(output); + return true; +} + +static bool +GroupHasPropertyTypes(ObjectGroup* group, jsid* id, Value* v) +{ + if (group->unknownProperties()) + return true; + HeapTypeSet* propTypes = group->maybeGetProperty(*id); + if (!propTypes) + return true; + if (!propTypes->nonConstantProperty()) + return false; + return propTypes->hasType(TypeSet::GetValueType(*v)); +} + +static void +EmitCheckPropertyTypes(MacroAssembler& masm, const PropertyTypeCheckInfo* typeCheckInfo, + Register obj, const ConstantOrRegister& val, + const LiveRegisterSet& liveRegs, Label* failures) +{ + // Emit code to check |val| is part of the property's HeapTypeSet. + + if (!typeCheckInfo->isSet()) + return; + + ObjectGroup* group = typeCheckInfo->group(); + if (group->unknownProperties()) + return; + + jsid id = typeCheckInfo->id(); + HeapTypeSet* propTypes = group->maybeGetProperty(id); + if (propTypes && propTypes->unknown()) + return; + + // Use the object register as scratch, as we don't need it here. + masm.Push(obj); + Register scratch1 = obj; + + bool checkTypeSet = true; + Label failedFastPath; + + if (propTypes && !propTypes->nonConstantProperty()) + masm.jump(&failedFastPath); + + if (val.constant()) { + // If the input is a constant, then don't bother if the barrier will always fail. + if (!propTypes || !propTypes->hasType(TypeSet::GetValueType(val.value()))) + masm.jump(&failedFastPath); + checkTypeSet = false; + } else { + // We can do the same trick as above for primitive types of specialized + // registers. + TypedOrValueRegister reg = val.reg(); + if (reg.hasTyped() && reg.type() != MIRType::Object) { + JSValueType valType = ValueTypeFromMIRType(reg.type()); + if (!propTypes || !propTypes->hasType(TypeSet::PrimitiveType(valType))) + masm.jump(&failedFastPath); + checkTypeSet = false; + } + } + + Label done; + if (checkTypeSet) { + TypedOrValueRegister valReg = val.reg(); + if (propTypes) { + // guardTypeSet can read from type sets without triggering read barriers. + TypeSet::readBarrier(propTypes); + masm.guardTypeSet(valReg, propTypes, BarrierKind::TypeSet, scratch1, &failedFastPath); + masm.jump(&done); + } else { + masm.jump(&failedFastPath); + } + } + + if (failedFastPath.used()) { + // The inline type check failed. Do a callWithABI to check the current + // TypeSet in case the type was added after we generated this stub. + masm.bind(&failedFastPath); + + AllocatableRegisterSet regs(GeneralRegisterSet::Volatile(), liveRegs.fpus()); + LiveRegisterSet save(regs.asLiveSet()); + masm.PushRegsInMask(save); + + regs.takeUnchecked(scratch1); + + // Push |val| first to make sure everything is fine if |val| aliases + // scratch2. + Register scratch2 = regs.takeAnyGeneral(); + masm.Push(val); + masm.moveStackPtrTo(scratch2); + + Register scratch3 = regs.takeAnyGeneral(); + masm.Push(id, scratch3); + masm.moveStackPtrTo(scratch3); + + masm.setupUnalignedABICall(scratch1); + masm.movePtr(ImmGCPtr(group), scratch1); + masm.passABIArg(scratch1); + masm.passABIArg(scratch3); + masm.passABIArg(scratch2); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, GroupHasPropertyTypes)); + masm.mov(ReturnReg, scratch1); + + masm.adjustStack(sizeof(Value) + sizeof(jsid)); + + LiveRegisterSet ignore; + ignore.add(scratch1); + masm.PopRegsInMaskIgnore(save, ignore); + + masm.branchIfTrueBool(scratch1, &done); + masm.pop(obj); + masm.jump(failures); + } + + masm.bind(&done); + masm.Pop(obj); +} + +bool +IonCacheIRCompiler::emitStoreFixedSlot() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + Maybe scratch; + if (needsPostBarrier()) + scratch.emplace(allocator, masm); + + if (typeCheckInfo_->isSet()) { + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + EmitCheckPropertyTypes(masm, typeCheckInfo_, obj, val, *liveRegs_, failure->label()); + } + + Address slot(obj, offset); + EmitPreBarrier(masm, slot, MIRType::Value); + masm.storeConstantOrRegister(val, slot); + if (needsPostBarrier()) + emitPostBarrierSlot(obj, val, scratch.ref()); + return true; +} + +bool +IonCacheIRCompiler::emitStoreDynamicSlot() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + AutoScratchRegister scratch(allocator, masm); + + if (typeCheckInfo_->isSet()) { + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + EmitCheckPropertyTypes(masm, typeCheckInfo_, obj, val, *liveRegs_, failure->label()); + } + + masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch); + Address slot(scratch, offset); + EmitPreBarrier(masm, slot, MIRType::Value); + masm.storeConstantOrRegister(val, slot); + if (needsPostBarrier()) + emitPostBarrierSlot(obj, val, scratch); + return true; +} + +bool +IonCacheIRCompiler::emitAddAndStoreSlotShared(CacheOp op) +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + AutoScratchRegister scratch1(allocator, masm); + + Maybe scratch2; + if (op == CacheOp::AllocateAndStoreDynamicSlot) + scratch2.emplace(allocator, masm); + + bool changeGroup = reader.readBool(); + ObjectGroup* newGroup = groupStubField(reader.stubOffset()); + Shape* newShape = shapeStubField(reader.stubOffset()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + EmitCheckPropertyTypes(masm, typeCheckInfo_, obj, val, *liveRegs_, failure->label()); + + if (op == CacheOp::AllocateAndStoreDynamicSlot) { + // We have to (re)allocate dynamic slots. Do this first, as it's the + // only fallible operation here. This simplifies the callTypeUpdateIC + // call below: it does not have to worry about saving registers used by + // failure paths. + int32_t numNewSlots = int32StubField(reader.stubOffset()); + MOZ_ASSERT(numNewSlots > 0); + + AllocatableRegisterSet regs(RegisterSet::Volatile()); + LiveRegisterSet save(regs.asLiveSet()); + + masm.PushRegsInMask(save); + + masm.setupUnalignedABICall(scratch1); + masm.loadJSContext(scratch1); + masm.passABIArg(scratch1); + masm.passABIArg(obj); + masm.move32(Imm32(numNewSlots), scratch2.ref()); + masm.passABIArg(scratch2.ref()); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, NativeObject::growSlotsDontReportOOM)); + masm.mov(ReturnReg, scratch1); + + LiveRegisterSet ignore; + ignore.add(scratch1); + masm.PopRegsInMaskIgnore(save, ignore); + + masm.branchIfFalseBool(scratch1, failure->label()); + } + + if (changeGroup) { + // Changing object's group from a partially to fully initialized group, + // per the acquired properties analysis. Only change the group if the + // old group still has a newScript. This only applies to PlainObjects. + Label noGroupChange; + masm.loadPtr(Address(obj, JSObject::offsetOfGroup()), scratch1); + masm.branchPtr(Assembler::Equal, + Address(scratch1, ObjectGroup::offsetOfAddendum()), + ImmWord(0), + &noGroupChange); + + Address groupAddr(obj, JSObject::offsetOfGroup()); + EmitPreBarrier(masm, groupAddr, MIRType::ObjectGroup); + masm.storePtr(ImmGCPtr(newGroup), groupAddr); + + masm.bind(&noGroupChange); + } + + // Update the object's shape. + Address shapeAddr(obj, ShapedObject::offsetOfShape()); + EmitPreBarrier(masm, shapeAddr, MIRType::Shape); + masm.storePtr(ImmGCPtr(newShape), shapeAddr); + + // Perform the store. No pre-barrier required since this is a new + // initialization. + if (op == CacheOp::AddAndStoreFixedSlot) { + Address slot(obj, offset); + masm.storeConstantOrRegister(val, slot); + } else { + MOZ_ASSERT(op == CacheOp::AddAndStoreDynamicSlot || + op == CacheOp::AllocateAndStoreDynamicSlot); + masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch1); + Address slot(scratch1, offset); + masm.storeConstantOrRegister(val, slot); + } + + if (needsPostBarrier()) + emitPostBarrierSlot(obj, val, scratch1); + + return true; +} + +bool +IonCacheIRCompiler::emitAddAndStoreFixedSlot() +{ + return emitAddAndStoreSlotShared(CacheOp::AddAndStoreFixedSlot); +} + +bool +IonCacheIRCompiler::emitAddAndStoreDynamicSlot() +{ + return emitAddAndStoreSlotShared(CacheOp::AddAndStoreDynamicSlot); +} + +bool +IonCacheIRCompiler::emitAllocateAndStoreDynamicSlot() +{ + return emitAddAndStoreSlotShared(CacheOp::AllocateAndStoreDynamicSlot); +} + +bool +IonCacheIRCompiler::emitStoreUnboxedProperty() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + JSValueType fieldType = reader.valueType(); + int32_t offset = int32StubField(reader.stubOffset()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + Maybe scratch; + if (needsPostBarrier() && UnboxedTypeNeedsPostBarrier(fieldType)) + scratch.emplace(allocator, masm); + + if (fieldType == JSVAL_TYPE_OBJECT && typeCheckInfo_->isSet()) { + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + EmitCheckPropertyTypes(masm, typeCheckInfo_, obj, val, *liveRegs_, failure->label()); + } + + // Note that the storeUnboxedProperty call here is infallible, as the + // IR emitter is responsible for guarding on |val|'s type. + Address fieldAddr(obj, offset); + EmitICUnboxedPreBarrier(masm, fieldAddr, fieldType); + masm.storeUnboxedProperty(fieldAddr, fieldType, val, /* failure = */ nullptr); + if (needsPostBarrier() && UnboxedTypeNeedsPostBarrier(fieldType)) + emitPostBarrierSlot(obj, val, scratch.ref()); + return true; +} + +bool +IonCacheIRCompiler::emitStoreTypedObjectReferenceProperty() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + TypedThingLayout layout = reader.typedThingLayout(); + ReferenceTypeDescr::Type type = reader.referenceTypeDescrType(); + + ValueOperand val = allocator.useValueRegister(masm, reader.valOperandId()); + + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + + // We don't need to check property types if the property is always a + // string. + if (type != ReferenceTypeDescr::TYPE_STRING) { + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + EmitCheckPropertyTypes(masm, typeCheckInfo_, obj, TypedOrValueRegister(val), + *liveRegs_, failure->label()); + } + + // Compute the address being written to. + LoadTypedThingData(masm, layout, obj, scratch1); + Address dest(scratch1, offset); + + emitStoreTypedObjectReferenceProp(val, type, dest, scratch2); + + if (needsPostBarrier() && type != ReferenceTypeDescr::TYPE_STRING) + emitPostBarrierSlot(obj, val, scratch1); + return true; +} + +bool +IonCacheIRCompiler::emitStoreTypedObjectScalarProperty() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + int32_t offset = int32StubField(reader.stubOffset()); + TypedThingLayout layout = reader.typedThingLayout(); + Scalar::Type type = reader.scalarType(); + ValueOperand val = allocator.useValueRegister(masm, reader.valOperandId()); + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Compute the address being written to. + LoadTypedThingData(masm, layout, obj, scratch1); + Address dest(scratch1, offset); + + StoreToTypedArray(cx_, masm, type, val, dest, scratch2, failure->label()); + return true; +} + +bool +IonCacheIRCompiler::emitStoreDenseElement() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + EmitCheckPropertyTypes(masm, typeCheckInfo_, obj, val, *liveRegs_, failure->label()); + + // Load obj->elements in scratch. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + + // Bounds check. + Address initLength(scratch, ObjectElements::offsetOfInitializedLength()); + masm.branch32(Assembler::BelowOrEqual, initLength, index, failure->label()); + + // Hole check. + BaseObjectElementIndex element(scratch, index); + masm.branchTestMagic(Assembler::Equal, element, failure->label()); + + EmitPreBarrier(masm, element, MIRType::Value); + EmitIonStoreDenseElement(masm, val, scratch, element); + if (needsPostBarrier()) + emitPostBarrierElement(obj, val, scratch, index); + return true; +} + +bool +IonCacheIRCompiler::emitStoreDenseElementHole() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + // handleAdd boolean is only relevant for Baseline. Ion ICs can always + // handle adds as we don't have to set any flags on the fallback stub to + // track this. + reader.readBool(); + + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + EmitCheckPropertyTypes(masm, typeCheckInfo_, obj, val, *liveRegs_, failure->label()); + + // Load obj->elements in scratch. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + + Address initLength(scratch, ObjectElements::offsetOfInitializedLength()); + BaseObjectElementIndex element(scratch, index); + + Label inBounds, doStore; + masm.branch32(Assembler::Above, initLength, index, &inBounds); + masm.branch32(Assembler::NotEqual, initLength, index, failure->label()); + + // If index < capacity, we can add a dense element inline. If not we + // need to allocate more elements. + Label capacityOk; + Address capacity(scratch, ObjectElements::offsetOfCapacity()); + masm.branch32(Assembler::Above, capacity, index, &capacityOk); + + // Check for non-writable array length. We only have to do this if + // index >= capacity. + Address elementsFlags(scratch, ObjectElements::offsetOfFlags()); + masm.branchTest32(Assembler::NonZero, elementsFlags, + Imm32(ObjectElements::NONWRITABLE_ARRAY_LENGTH), + failure->label()); + + LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs()); + save.takeUnchecked(scratch); + masm.PushRegsInMask(save); + + masm.setupUnalignedABICall(scratch); + masm.loadJSContext(scratch); + masm.passABIArg(scratch); + masm.passABIArg(obj); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, NativeObject::addDenseElementDontReportOOM)); + masm.mov(ReturnReg, scratch); + + masm.PopRegsInMask(save); + masm.branchIfFalseBool(scratch, failure->label()); + + // Load the reallocated elements pointer. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + + masm.bind(&capacityOk); + + // Increment initLength. + masm.add32(Imm32(1), initLength); + + // If length is now <= index, increment length too. + Label skipIncrementLength; + Address length(scratch, ObjectElements::offsetOfLength()); + masm.branch32(Assembler::Above, length, index, &skipIncrementLength); + masm.add32(Imm32(1), length); + masm.bind(&skipIncrementLength); + + // Skip EmitPreBarrier as the memory is uninitialized. + masm.jump(&doStore); + + masm.bind(&inBounds); + + EmitPreBarrier(masm, element, MIRType::Value); + + masm.bind(&doStore); + EmitIonStoreDenseElement(masm, val, scratch, element); + if (needsPostBarrier()) + emitPostBarrierElement(obj, val, scratch, index); + return true; +} + +bool +IonCacheIRCompiler::emitStoreTypedElement() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + Register index = allocator.useRegister(masm, reader.int32OperandId()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + TypedThingLayout layout = reader.typedThingLayout(); + Scalar::Type arrayType = reader.scalarType(); + bool handleOOB = reader.readBool(); + + AutoScratchRegister scratch1(allocator, masm); + + Maybe scratch2; + if (arrayType != Scalar::Float32 && arrayType != Scalar::Float64) + scratch2.emplace(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + // Bounds check. + Label done; + LoadTypedThingLength(masm, layout, obj, scratch1); + masm.branch32(Assembler::BelowOrEqual, scratch1, index, handleOOB ? &done : failure->label()); + + // Load the elements vector. + LoadTypedThingData(masm, layout, obj, scratch1); + + BaseIndex dest(scratch1, index, ScaleFromElemWidth(Scalar::byteSize(arrayType))); + + FloatRegister maybeTempDouble = ic_->asSetPropertyIC()->maybeTempDouble(); + FloatRegister maybeTempFloat32 = ic_->asSetPropertyIC()->maybeTempFloat32(); + MOZ_ASSERT(maybeTempDouble != InvalidFloatReg); + MOZ_ASSERT_IF(jit::hasUnaliasedDouble(), maybeTempFloat32 != InvalidFloatReg); + + if (arrayType == Scalar::Float32) { + FloatRegister tempFloat = hasUnaliasedDouble() ? maybeTempFloat32 : maybeTempDouble; + if (!masm.convertConstantOrRegisterToFloat(cx_, val, tempFloat, failure->label())) + return false; + masm.storeToTypedFloatArray(arrayType, tempFloat, dest); + } else if (arrayType == Scalar::Float64) { + if (!masm.convertConstantOrRegisterToDouble(cx_, val, maybeTempDouble, failure->label())) + return false; + masm.storeToTypedFloatArray(arrayType, maybeTempDouble, dest); + } else { + Register valueToStore = scratch2.ref(); + if (arrayType == Scalar::Uint8Clamped) { + if (!masm.clampConstantOrRegisterToUint8(cx_, val, maybeTempDouble, valueToStore, + failure->label())) + { + return false; + } + } else { + if (!masm.truncateConstantOrRegisterToInt32(cx_, val, maybeTempDouble, valueToStore, + failure->label())) + { + return false; + } + } + masm.storeToTypedIntArray(arrayType, valueToStore, dest); + } + + masm.bind(&done); + return true; +} + +bool +IonCacheIRCompiler::emitStoreUnboxedArrayElement() +{ + MOZ_CRASH("Baseline-specific op"); +} + +bool +IonCacheIRCompiler::emitStoreUnboxedArrayElementHole() +{ + MOZ_CRASH("Baseline-specific op"); +} + +bool +IonCacheIRCompiler::emitCallNativeSetter() +{ + AutoSaveLiveRegisters save(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + JSFunction* target = &objectStubField(reader.stubOffset())->as(); + MOZ_ASSERT(target->isNative()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + AutoScratchRegister argJSContext(allocator, masm); + AutoScratchRegister argVp(allocator, masm); + AutoScratchRegister argUintN(allocator, masm); + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + // Set up the call: + // bool (*)(JSContext*, unsigned, Value* vp) + // vp[0] is callee/outparam + // vp[1] is |this| + // vp[2] is the value + + // Build vp and move the base into argVpReg. + masm.Push(val); + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(obj))); + masm.Push(ObjectValue(*target)); + masm.moveStackPtrTo(argVp.get()); + + // Preload other regs. + masm.loadJSContext(argJSContext); + masm.move32(Imm32(1), argUintN); + + // Push marking data for later use. + masm.Push(argUintN); + pushStubCodePointer(); + + if (!masm.icBuildOOLFakeExitFrame(GetReturnAddressToIonCode(cx_), save)) + return false; + masm.enterFakeExitFrame(scratch, IonOOLNativeExitFrameLayoutToken); + + // Make the call. + masm.setupUnalignedABICall(scratch); + masm.passABIArg(argJSContext); + masm.passABIArg(argUintN); + masm.passABIArg(argVp); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, target->native())); + + // Test for failure. + masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel()); + + masm.adjustStack(IonOOLNativeExitFrameLayout::Size(1)); + return true; +} + +bool +IonCacheIRCompiler::emitCallScriptedSetter() +{ + AutoSaveLiveRegisters save(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + JSFunction* target = &objectStubField(reader.stubOffset())->as(); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + uint32_t framePushedBefore = masm.framePushed(); + + // Construct IonICCallFrameLayout. + uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS, + IonICCallFrameLayout::Size()); + pushStubCodePointer(); + masm.Push(Imm32(descriptor)); + masm.Push(ImmPtr(GetReturnAddressToIonCode(cx_))); + + // The JitFrameLayout pushed below will be aligned to JitStackAlignment, + // so we just have to make sure the stack is aligned after we push the + // |this| + argument Values. + size_t numArgs = Max(1, target->nargs()); + uint32_t argSize = (numArgs + 1) * sizeof(Value); + uint32_t padding = ComputeByteAlignment(masm.framePushed() + argSize, JitStackAlignment); + MOZ_ASSERT(padding % sizeof(uintptr_t) == 0); + MOZ_ASSERT(padding < JitStackAlignment); + masm.reserveStack(padding); + + for (size_t i = 1; i < target->nargs(); i++) + masm.Push(UndefinedValue()); + masm.Push(val); + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(obj))); + + masm.movePtr(ImmGCPtr(target), scratch); + + descriptor = MakeFrameDescriptor(argSize + padding, JitFrame_IonICCall, + JitFrameLayout::Size()); + masm.Push(Imm32(1)); // argc + masm.Push(scratch); + masm.Push(Imm32(descriptor)); + + // Check stack alignment. Add sizeof(uintptr_t) for the return address. + MOZ_ASSERT(((masm.framePushed() + sizeof(uintptr_t)) % JitStackAlignment) == 0); + + // The setter has JIT code now and we will only discard the setter's JIT + // code when discarding all JIT code in the Zone, so we can assume it'll + // still have JIT code. + MOZ_ASSERT(target->hasJITCode()); + masm.loadPtr(Address(scratch, JSFunction::offsetOfNativeOrScript()), scratch); + masm.loadBaselineOrIonRaw(scratch, scratch, nullptr); + masm.callJit(scratch); + + masm.freeStack(masm.framePushed() - framePushedBefore); + return true; +} + +typedef bool (*SetArrayLengthFn)(JSContext*, HandleObject, HandleValue, bool); +static const VMFunction SetArrayLengthInfo = + FunctionInfo(SetArrayLength, "SetArrayLength"); + +bool +IonCacheIRCompiler::emitCallSetArrayLength() +{ + AutoSaveLiveRegisters save(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + bool strict = reader.readBool(); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + + allocator.discardStack(masm); + prepareVMCall(masm); + + masm.Push(Imm32(strict)); + masm.Push(val); + masm.Push(obj); + + return callVM(masm, SetArrayLengthInfo); +} + +typedef bool (*ProxySetPropertyFn)(JSContext*, HandleObject, HandleId, HandleValue, bool); +static const VMFunction ProxySetPropertyInfo = + FunctionInfo(ProxySetProperty, "ProxySetProperty"); + +bool +IonCacheIRCompiler::emitCallProxySet() +{ + AutoSaveLiveRegisters save(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + jsid id = idStubField(reader.stubOffset()); + bool strict = reader.readBool(); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + prepareVMCall(masm); + + masm.Push(Imm32(strict)); + masm.Push(val); + masm.Push(id, scratch); + masm.Push(obj); + + return callVM(masm, ProxySetPropertyInfo); +} + +typedef bool (*ProxySetPropertyByValueFn)(JSContext*, HandleObject, HandleValue, HandleValue, bool); +static const VMFunction ProxySetPropertyByValueInfo = + FunctionInfo(ProxySetPropertyByValue, "ProxySetPropertyByValue"); + +bool +IonCacheIRCompiler::emitCallProxySetByValue() +{ + AutoSaveLiveRegisters save(*this); + + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ConstantOrRegister idVal = allocator.useConstantOrRegister(masm, reader.valOperandId()); + ConstantOrRegister val = allocator.useConstantOrRegister(masm, reader.valOperandId()); + bool strict = reader.readBool(); + + allocator.discardStack(masm); + prepareVMCall(masm); + + masm.Push(Imm32(strict)); + masm.Push(val); + masm.Push(idVal); + masm.Push(obj); + + return callVM(masm, ProxySetPropertyByValueInfo); +} + +bool +IonCacheIRCompiler::emitLoadTypedObjectResult() +{ + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, reader.objOperandId()); + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + + TypedThingLayout layout = reader.typedThingLayout(); + uint32_t typeDescr = reader.typeDescrKey(); + uint32_t fieldOffset = int32StubField(reader.stubOffset()); + + // Get the object's data pointer. + LoadTypedThingData(masm, layout, obj, scratch1); + + Address fieldAddr(scratch1, fieldOffset); + emitLoadTypedObjectResultShared(fieldAddr, scratch2, layout, typeDescr, output); + return true; +} + +bool +IonCacheIRCompiler::emitTypeMonitorResult() +{ + return emitReturnFromIC(); +} + +bool +IonCacheIRCompiler::emitReturnFromIC() +{ + if (!savedLiveRegs_) + allocator.restoreInputState(masm); + + RepatchLabel rejoin; + rejoinOffset_ = masm.jumpWithPatch(&rejoin); + masm.bind(&rejoin); + return true; +} + +bool +IonCacheIRCompiler::emitLoadObject() +{ + Register reg = allocator.defineRegister(masm, reader.objOperandId()); + JSObject* obj = objectStubField(reader.stubOffset()); + masm.movePtr(ImmGCPtr(obj), reg); + return true; +} + +bool +IonCacheIRCompiler::emitLoadStackValue() +{ + MOZ_ASSERT_UNREACHABLE("emitLoadStackValue not supported for IonCaches."); + return false; +} + +bool +IonCacheIRCompiler::emitGuardDOMExpandoMissingOrGuardShape() +{ + ValueOperand val = allocator.useValueRegister(masm, reader.valOperandId()); + Shape* shape = shapeStubField(reader.stubOffset()); + + AutoScratchRegister objScratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + Label done; + masm.branchTestUndefined(Assembler::Equal, val, &done); + + masm.debugAssertIsObject(val); + masm.unboxObject(val, objScratch); + masm.branchTestObjShape(Assembler::NotEqual, objScratch, shape, failure->label()); + + masm.bind(&done); + return true; +} + +bool +IonCacheIRCompiler::emitLoadDOMExpandoValueGuardGeneration() +{ + Register obj = allocator.useRegister(masm, reader.objOperandId()); + ExpandoAndGeneration* expandoAndGeneration = + rawWordStubField(reader.stubOffset()); + uint64_t* generationFieldPtr = expandoGenerationStubFieldPtr(reader.stubOffset()); + + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + ValueOperand output = allocator.defineValueRegister(masm, reader.valOperandId()); + + FailurePath* failure; + if (!addFailurePath(&failure)) + return false; + + masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), scratch1); + Address expandoAddr(scratch1, detail::ProxyReservedSlots::offsetOfPrivateSlot()); + + // Guard the ExpandoAndGeneration* matches the proxy's ExpandoAndGeneration. + masm.loadValue(expandoAddr, output); + masm.branchTestValue(Assembler::NotEqual, output, PrivateValue(expandoAndGeneration), + failure->label()); + + // Guard expandoAndGeneration->generation matches the expected generation. + masm.movePtr(ImmPtr(expandoAndGeneration), output.scratchReg()); + masm.movePtr(ImmPtr(generationFieldPtr), scratch1); + masm.branch64(Assembler::NotEqual, + Address(output.scratchReg(), ExpandoAndGeneration::offsetOfGeneration()), + Address(scratch1, 0), + scratch2, + failure->label()); + + // Load expandoAndGeneration->expando into the output Value register. + masm.loadValue(Address(output.scratchReg(), ExpandoAndGeneration::offsetOfExpando()), output); + return true; +} + +void +IonIC::attachCacheIRStub(JSContext* cx, const CacheIRWriter& writer, CacheKind kind, + IonScript* ionScript, bool* attached, + const PropertyTypeCheckInfo* typeCheckInfo) +{ + // We shouldn't GC or report OOM (or any other exception) here. + AutoAssertNoPendingException aanpe(cx); + JS::AutoCheckCannotGC nogc; + + MOZ_ASSERT(!*attached); + + // SetProp/SetElem stubs must have non-null typeCheckInfo. + MOZ_ASSERT(!!typeCheckInfo == (kind == CacheKind::SetProp || kind == CacheKind::SetElem)); + + // Do nothing if the IR generator failed or triggered a GC that invalidated + // the script. + if (writer.failed() || ionScript->invalidated()) + return; + + JitZone* jitZone = cx->zone()->jitZone(); + uint32_t stubDataOffset = sizeof(IonICStub); + + // Try to reuse a previously-allocated CacheIRStubInfo. + CacheIRStubKey::Lookup lookup(kind, ICStubEngine::IonIC, + writer.codeStart(), writer.codeLength()); + CacheIRStubInfo* stubInfo = jitZone->getIonCacheIRStubInfo(lookup); + if (!stubInfo) { + // Allocate the shared CacheIRStubInfo. Note that the + // putIonCacheIRStubInfo call below will transfer ownership to + // the stub info HashSet, so we don't have to worry about freeing + // it below. + + // For Ion ICs, we don't track/use the makesGCCalls flag, so just pass true. + bool makesGCCalls = true; + stubInfo = CacheIRStubInfo::New(kind, ICStubEngine::IonIC, makesGCCalls, + stubDataOffset, writer); + if (!stubInfo) + return; + + CacheIRStubKey key(stubInfo); + if (!jitZone->putIonCacheIRStubInfo(lookup, key)) + return; + } + + MOZ_ASSERT(stubInfo); + + // Ensure we don't attach duplicate stubs. This can happen if a stub failed + // for some reason and the IR generator doesn't check for exactly the same + // conditions. + for (IonICStub* stub = firstStub_; stub; stub = stub->next()) { + if (stub->stubInfo() != stubInfo) + continue; + bool updated = false; + if (!writer.stubDataEqualsMaybeUpdate(stub->stubDataStart(), &updated)) + continue; + if (updated || (typeCheckInfo && typeCheckInfo->needsTypeBarrier())) { + // We updated a stub or have a stub that requires property type + // checks. In this case the stub will likely handle more cases in + // the future and we shouldn't deoptimize. + *attached = true; + } + return; + } + + size_t bytesNeeded = stubInfo->stubDataOffset() + stubInfo->stubDataSize(); + + // Allocate the IonICStub in the optimized stub space. Ion stubs and + // CacheIRStubInfo instances for Ion stubs can be purged on GC. That's okay + // because the stub code is rooted separately when we make a VM call, and + // stub code should never access the IonICStub after making a VM call. The + // IonICStub::poison method poisons the stub to catch bugs in this area. + ICStubSpace* stubSpace = cx->zone()->jitZone()->optimizedStubSpace(); + void* newStubMem = stubSpace->alloc(bytesNeeded); + if (!newStubMem) + return; + + IonICStub* newStub = new(newStubMem) IonICStub(fallbackLabel_.raw(), stubInfo); + writer.copyStubData(newStub->stubDataStart()); + + JitContext jctx(cx, nullptr); + IonCacheIRCompiler compiler(cx, writer, this, ionScript, newStub, typeCheckInfo); + if (!compiler.init()) + return; + + JitCode* code = compiler.compile(); + if (!code) + return; + + attachStub(newStub, code); + *attached = true; +} diff --git a/js/src/jit/MacroAssembler-inl.h b/js/src/jit/MacroAssembler-inl.h index 536b71399b..9ffffff117 100644 --- a/js/src/jit/MacroAssembler-inl.h +++ b/js/src/jit/MacroAssembler-inl.h @@ -409,6 +409,14 @@ MacroAssembler::branchIfRopeOrExternal(Register str, Register temp, Label* label branch32(Assembler::Equal, temp, Imm32(JSString::EXTERNAL_FLAGS), label); } +void +MacroAssembler::branchIfNotRope(Register str, Label* label) +{ + Address flags(str, JSString::offsetOfFlags()); + static_assert(JSString::ROPE_FLAGS == 0, "Rope type flags must be 0"); + branchTest32(Assembler::NonZero, flags, Imm32(JSString::TYPE_FLAGS_MASK), label); +} + void MacroAssembler::branchLatin1String(Register string, Label* label) { diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp index cab426d708..185ea2b68e 100644 --- a/js/src/jit/MacroAssembler.cpp +++ b/js/src/jit/MacroAssembler.cpp @@ -1310,19 +1310,40 @@ MacroAssembler::loadStringChars(Register str, Register dest) } void -MacroAssembler::loadStringChar(Register str, Register index, Register output) +MacroAssembler::loadStringChar(Register str, Register index, Register output, Label* fail) { MOZ_ASSERT(str != output); MOZ_ASSERT(index != output); - loadStringChars(str, output); + movePtr(str, output); + + // This follows JSString::getChar. + Label notRope; + branchIfNotRope(str, ¬Rope); + + // Load leftChild. + loadPtr(Address(str, JSRope::offsetOfLeft()), output); + + // Check if the index is contained in the leftChild. + // Todo: Handle index in the rightChild. + branch32(Assembler::BelowOrEqual, Address(output, JSString::offsetOfLength()), index, fail); + + // If the left side is another rope, give up. + branchIfRope(output, fail); + + bind(¬Rope); Label isLatin1, done; - branchLatin1String(str, &isLatin1); + // We have to check the left/right side for ropes, + // because a TwoByte rope might have a Latin1 child. + branchLatin1String(output, &isLatin1); + + loadStringChars(output, output); load16ZeroExtend(BaseIndex(output, index, TimesTwo), output); jump(&done); bind(&isLatin1); + loadStringChars(output, output); load8ZeroExtend(BaseIndex(output, index, TimesOne), output); bind(&done); diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h index af7de1c16a..9524951772 100644 --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -1096,6 +1096,8 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branchIfRope(Register str, Label* label); inline void branchIfRopeOrExternal(Register str, Register temp, Label* label); + inline void branchIfNotRope(Register str, Label* label); + inline void branchLatin1String(Register string, Label* label); inline void branchTwoByteString(Register string, Label* label); @@ -1445,7 +1447,7 @@ class MacroAssembler : public MacroAssemblerSpecific } void loadStringChars(Register str, Register dest); - void loadStringChar(Register str, Register index, Register output); + void loadStringChar(Register str, Register index, Register output, Label* fail); void loadJSContext(Register dest) { movePtr(ImmPtr(GetJitContext()->runtime->getJSContext()), dest);