From 38ebcb17c99a0c5f5e87cf1565d9c1780009017f Mon Sep 17 00:00:00 2001 From: Basilisk-Dev Date: Thu, 23 Apr 2026 20:03:41 -0400 Subject: [PATCH] Issue #3049 - Adapt loongarch64 baseline macroassembler to old JIT APIs --- .../jit/loongarch64/Assembler-loongarch64.h | 1 + .../MacroAssembler-loongarch64-inl.h | 76 + .../MacroAssembler-loongarch64.cpp | 1906 +---------------- .../loongarch64/MacroAssembler-loongarch64.h | 5 +- .../jit/loongarch64/SharedIC-loongarch64.cpp | 31 +- .../loongarch64/Trampoline-loongarch64.cpp | 154 +- 6 files changed, 248 insertions(+), 1925 deletions(-) diff --git a/js/src/jit/loongarch64/Assembler-loongarch64.h b/js/src/jit/loongarch64/Assembler-loongarch64.h index e7e7c5fe6e..12bdd87027 100644 --- a/js/src/jit/loongarch64/Assembler-loongarch64.h +++ b/js/src/jit/loongarch64/Assembler-loongarch64.h @@ -1431,6 +1431,7 @@ class AssemblerLOONGARCH64 : public AssemblerShared { // label operations void bind(Label* label, BufferOffset boff = BufferOffset()); virtual void bind(InstImm* inst, uintptr_t branch, uintptr_t target) = 0; + void bind(CodeOffset* label) { label->bind(currentOffset()); } void bind(CodeLabel* label) { label->target()->bind(currentOffset()); } uint32_t currentOffset() { return nextOffset().getOffset(); } void retarget(Label* label, Label* target); diff --git a/js/src/jit/loongarch64/MacroAssembler-loongarch64-inl.h b/js/src/jit/loongarch64/MacroAssembler-loongarch64-inl.h index ce515d5172..944baf6c9a 100644 --- a/js/src/jit/loongarch64/MacroAssembler-loongarch64-inl.h +++ b/js/src/jit/loongarch64/MacroAssembler-loongarch64-inl.h @@ -849,6 +849,32 @@ void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs, branchPtr(cond, scratch2, rhs, label); } +template +CodeOffsetJump MacroAssembler::branchPtrWithPatch(Condition cond, Register lhs, + T rhs, + RepatchLabel* label) { + movePtr(rhs, ScratchRegister); + Label skipJump; + ma_b(lhs, ScratchRegister, &skipJump, InvertCondition(cond), ShortJump); + CodeOffsetJump off = jumpWithPatch(label); + bind(&skipJump); + return off; +} + +template +CodeOffsetJump MacroAssembler::branchPtrWithPatch(Condition cond, Address lhs, + T rhs, + RepatchLabel* label) { + loadPtr(lhs, SecondScratchReg); + movePtr(rhs, ScratchRegister); + Label skipJump; + ma_b(SecondScratchReg, ScratchRegister, &skipJump, InvertCondition(cond), + ShortJump); + CodeOffsetJump off = jumpWithPatch(label); + bind(&skipJump); + return off; +} + void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs, Register rhs, Label* label) { branchPtr(cond, lhs, rhs, label); @@ -1373,6 +1399,56 @@ void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, ma_b(scratch, ImmWord(magic), label, cond); } +void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src, + const Address& addr) { + ma_fst_d(src, addr); +} + +void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src, + const BaseIndex& addr) { + MOZ_ASSERT(addr.offset == 0); + ma_fst_d(src, addr); +} + +void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src, + const Address& addr) { + ma_fst_s(src, addr); +} + +void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src, + const BaseIndex& addr) { + MOZ_ASSERT(addr.offset == 0); + ma_fst_s(src, addr); +} + +void MacroAssembler::clampIntToUint8(Register reg) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(reg != scratch && reg != scratch2); + + as_slt(scratch, reg, zero); + moveIfNotZero(reg, zero, scratch); + + ma_li(scratch2, Imm32(255)); + as_sltui(scratch, reg, 255); + moveIfZero(reg, scratch2, scratch); +} + +template +void MacroAssembler::wasmBoundsCheck(Condition cond, Register index, L label) { + (void)cond; + (void)index; + (void)label; + MOZ_CRASH("wasm bounds checks are not supported on loongarch64 yet"); +} + +void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) { + if (barrier == MembarNobits) { + return; + } + as_dbar(0); +} + //}}} check_macroassembler_style // =============================================================== diff --git a/js/src/jit/loongarch64/MacroAssembler-loongarch64.cpp b/js/src/jit/loongarch64/MacroAssembler-loongarch64.cpp index 0be6e066f1..3cd5fb5d91 100644 --- a/js/src/jit/loongarch64/MacroAssembler-loongarch64.cpp +++ b/js/src/jit/loongarch64/MacroAssembler-loongarch64.cpp @@ -39,7 +39,7 @@ void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) { bool MacroAssemblerLOONGARCH64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr) { uint32_t descriptor = MakeFrameDescriptor( - asMasm().framePushed(), FrameType::IonJS, ExitFrameLayout::Size()); + asMasm().framePushed(), JitFrame_IonJS, ExitFrameLayout::Size()); asMasm().Push(Imm32(descriptor)); // descriptor_ asMasm().Push(ImmPtr(fakeReturnAddr)); @@ -51,7 +51,8 @@ void MacroAssemblerLOONGARCH64Compat::convertUInt32ToDouble(Register src, FloatRegister dest) { ScratchRegisterScope scratch(asMasm()); as_bstrpick_d(scratch, src, 31, 0); - asMasm().convertInt64ToDouble(Register64(scratch), dest); + as_movgr2fr_d(dest, scratch); + as_ffint_d_l(dest, dest); } void MacroAssemblerLOONGARCH64Compat::convertUInt64ToDouble(Register src, @@ -79,11 +80,22 @@ void MacroAssemblerLOONGARCH64Compat::convertUInt64ToDouble(Register src, bind(&done); } +bool MacroAssemblerLOONGARCH64Compat::convertUInt64ToDoubleNeedsTemp() { + return false; +} + +void MacroAssemblerLOONGARCH64Compat::convertUInt64ToDouble( + Register64 src, FloatRegister dest, Register temp) { + MOZ_ASSERT(temp == InvalidReg); + convertUInt64ToDouble(src.reg, dest); +} + void MacroAssemblerLOONGARCH64Compat::convertUInt32ToFloat32(Register src, FloatRegister dest) { ScratchRegisterScope scratch(asMasm()); as_bstrpick_d(scratch, src, 31, 0); - asMasm().convertInt64ToFloat32(Register64(scratch), dest); + as_movgr2fr_d(dest, scratch); + as_ffint_s_l(dest, dest); } void MacroAssemblerLOONGARCH64Compat::convertDoubleToFloat32(FloatRegister src, @@ -186,10 +198,10 @@ void MacroAssemblerLOONGARCH64Compat::movq(Register rj, Register rd) { as_or(rd, rj, zero); } -void MacroAssemblerLOONGARCH64::ma_li(Register dest, CodeLabel* label) { +void MacroAssemblerLOONGARCH64::ma_li(Register dest, CodeOffset* label) { BufferOffset bo = m_buffer.nextOffset(); ma_liPatchable(dest, ImmWord(/* placeholder */ 0)); - label->patchAt()->bind(bo.getOffset()); + label->bind(bo.getOffset()); } void MacroAssemblerLOONGARCH64::ma_li(Register dest, ImmWord imm) { @@ -2040,7 +2052,7 @@ void MacroAssemblerLOONGARCH64::wasmLoadImpl(const wasm::MemoryAccessDesc& acces ptr = ptrScratch; } - asMasm().memoryBarrierBefore(access.sync()); + asMasm().memoryBarrier(access.barrierBefore()); switch (access.type()) { case Scalar::Int8: @@ -2069,8 +2081,8 @@ void MacroAssemblerLOONGARCH64::wasmLoadImpl(const wasm::MemoryAccessDesc& acces MOZ_CRASH("unexpected array type"); } - asMasm().append(access, asMasm().size() - 4); - asMasm().memoryBarrierAfter(access.sync()); + asMasm().append(access, asMasm().size() - 4, asMasm().framePushed()); + asMasm().memoryBarrier(access.barrierAfter()); } void MacroAssemblerLOONGARCH64::wasmStoreImpl(const wasm::MemoryAccessDesc& access, @@ -2087,7 +2099,7 @@ void MacroAssemblerLOONGARCH64::wasmStoreImpl(const wasm::MemoryAccessDesc& acce ptr = ptrScratch; } - asMasm().memoryBarrierBefore(access.sync()); + asMasm().memoryBarrier(access.barrierBefore()); switch (access.type()) { case Scalar::Int8: @@ -2116,8 +2128,8 @@ void MacroAssemblerLOONGARCH64::wasmStoreImpl(const wasm::MemoryAccessDesc& acce } // Only the last emitted instruction is a memory access. - asMasm().append(access, asMasm().size() - 4); - asMasm().memoryBarrierAfter(access.sync()); + asMasm().append(access, asMasm().size() - 4, asMasm().framePushed()); + asMasm().memoryBarrier(access.barrierAfter()); } void MacroAssemblerLOONGARCH64Compat::wasmLoadI64Impl( @@ -2133,7 +2145,7 @@ void MacroAssemblerLOONGARCH64Compat::wasmLoadI64Impl( ptr = ptrScratch; } - asMasm().memoryBarrierBefore(access.sync()); + asMasm().memoryBarrier(access.barrierBefore()); switch (access.type()) { case Scalar::Int8: @@ -2162,8 +2174,8 @@ void MacroAssemblerLOONGARCH64Compat::wasmLoadI64Impl( MOZ_CRASH("unexpected array type"); } - asMasm().append(access, asMasm().size() - 4); - asMasm().memoryBarrierAfter(access.sync()); + asMasm().append(access, asMasm().size() - 4, asMasm().framePushed()); + asMasm().memoryBarrier(access.barrierAfter()); } void MacroAssemblerLOONGARCH64Compat::wasmStoreI64Impl( @@ -2179,7 +2191,7 @@ void MacroAssemblerLOONGARCH64Compat::wasmStoreI64Impl( ptr = ptrScratch; } - asMasm().memoryBarrierBefore(access.sync()); + asMasm().memoryBarrier(access.barrierBefore()); switch (access.type()) { case Scalar::Int8: @@ -2201,14 +2213,14 @@ void MacroAssemblerLOONGARCH64Compat::wasmStoreI64Impl( MOZ_CRASH("unexpected array type"); } - asMasm().append(access, asMasm().size() - 4); - asMasm().memoryBarrierAfter(access.sync()); + asMasm().append(access, asMasm().size() - 4, asMasm().framePushed()); + asMasm().memoryBarrier(access.barrierAfter()); } void MacroAssemblerLOONGARCH64Compat::profilerEnterFrame(Register framePtr, Register scratch) { - asMasm().loadJSContext(scratch); - loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch); + AbsoluteAddress activation(GetJitContext()->runtime->addressOfProfilingActivation()); + loadPtr(activation, scratch); storePtr(framePtr, Address(scratch, JitActivation::offsetOfLastProfilingFrame())); storePtr(ImmPtr(nullptr), @@ -2242,10 +2254,6 @@ void MacroAssembler::flush() {} // =============================================================== // Stack manipulation functions. -size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) { - return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); -} - void MacroAssembler::PushRegsInMask(LiveRegisterSet set) { int32_t diff = set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); @@ -2297,44 +2305,6 @@ void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set, freeStack(reserved); } -void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest, - Register) { - FloatRegisterSet fpuSet(set.fpus().reduceSetForPush()); - mozilla::DebugOnly numFpu = fpuSet.size(); - int32_t diffF = fpuSet.getPushSizeInBytes(); - mozilla::DebugOnly diffG = set.gprs().size() * sizeof(intptr_t); - - MOZ_ASSERT(dest.offset >= diffG + diffF); - - for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { - diffG -= sizeof(intptr_t); - dest.offset -= sizeof(intptr_t); - storePtr(*iter, dest); - } - MOZ_ASSERT(diffG == 0); - -#ifdef ENABLE_WASM_SIMD -# error "Needs more careful logic if SIMD is enabled" -#endif - - for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) { - FloatRegister reg = *iter; - diffF -= reg.size(); - numFpu -= 1; - dest.offset -= reg.size(); - if (reg.isDouble()) { - storeDouble(reg, dest); - } else if (reg.isSingle()) { - storeFloat32(reg, dest); - } else { - MOZ_CRASH("Unknown register type."); - } - } - MOZ_ASSERT(numFpu == 0); - diffF -= diffF % sizeof(uintptr_t); - MOZ_ASSERT(diffF == 0); -} - void MacroAssembler::Push(Register reg) { ma_push(reg); adjustFrame(int32_t(sizeof(intptr_t))); @@ -2370,12 +2340,6 @@ void MacroAssembler::Push(FloatRegister f) { adjustFrame(int32_t(sizeof(double))); } -void MacroAssembler::PushBoxed(FloatRegister reg) { - subFromStackPtr(Imm32(sizeof(double))); - boxDouble(reg, Address(getStackPointer(), 0)); - adjustFrame(sizeof(double)); -} - void MacroAssembler::Pop(Register reg) { ma_pop(reg); adjustFrame(-int32_t(sizeof(intptr_t))); @@ -2391,11 +2355,6 @@ void MacroAssembler::Pop(const ValueOperand& val) { adjustFrame(-int32_t(sizeof(Value))); } -void MacroAssembler::PopStackPtr() { - loadPtr(Address(StackPointer, 0), StackPointer); - adjustFrame(-int32_t(sizeof(intptr_t))); -} - // =============================================================== // Simple call functions. @@ -2450,13 +2409,8 @@ void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) { *u32 = targetOffset - farJump.offset(); } -CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) { +void MacroAssembler::call(wasm::SymbolicAddress target) { movePtr(target, CallReg); - return call(CallReg); -} - -void MacroAssembler::call(const Address& addr) { - loadPtr(addr, CallReg); call(CallReg); } @@ -2476,13 +2430,13 @@ void MacroAssembler::call(JitCode* c) { callJitNoProfiler(scratch); } -CodeOffsetJump MacroAssembler::backedgeJump(RepatchLabel* label, - Label* documentation) { +CodeOffsetJump MacroAssemblerLOONGARCH64Compat::backedgeJump( + RepatchLabel* label, Label* documentation) { return jumpWithPatch(label, documentation); } -CodeOffsetJump MacroAssembler::jumpWithPatch(RepatchLabel* label, - Label* documentation) { +CodeOffsetJump MacroAssemblerLOONGARCH64Compat::jumpWithPatch( + RepatchLabel* label, Label* documentation) { MOZ_ASSERT(!label->used()); BufferOffset bo = nextOffset(); @@ -2496,29 +2450,6 @@ CodeOffsetJump MacroAssembler::jumpWithPatch(RepatchLabel* label, return CodeOffsetJump(bo.getOffset()); } -CodeOffset MacroAssembler::nopPatchableToCall() { - // LOONGARCH64 - as_nop(); // lu12i_w - as_nop(); // ori - as_nop(); // lu32i_d - as_nop(); // jirl - return CodeOffset(currentOffset()); -} - -void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) { - Instruction* inst = (Instruction*)call - 4 /* four nops */; - Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)target); - inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); -} - -void MacroAssembler::patchCallToNop(uint8_t* call) { - Instruction* inst = (Instruction*)call - 4 /* four nops */; - inst[0].makeNop(); // lu12i_w - inst[1].makeNop(); // ori - inst[2].makeNop(); // lu32i_d - inst[3].makeNop(); // jirl -} - void MacroAssembler::pushReturnAddress() { push(ra); } void MacroAssembler::popReturnAddress() { pop(ra); } @@ -2528,7 +2459,7 @@ void MacroAssembler::popReturnAddress() { pop(ra); } void MacroAssembler::setupUnalignedABICall(Register scratch) { MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls"); - setupNativeABICall(); + setupABICall(); dynamicAlignment_ = true; as_or(scratch, StackPointer, zero); @@ -2577,8 +2508,7 @@ void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { assertStackAlignment(ABIStackAlignment); } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result, - bool callFromWasm) { +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result) { // Restore ra value (as stored in callWithABIPre()). loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra); @@ -2640,7 +2570,7 @@ uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) { // =============================================================== // Move instructions -void MacroAssembler::moveValue(const Value& src, Register dest) { +void MacroAssemblerLOONGARCH64Compat::moveValue(const Value& src, Register dest) { if (!src.isGCThing()) { ma_li(dest, ImmWord(src.asRawBits())); return; @@ -2650,7 +2580,7 @@ void MacroAssembler::moveValue(const Value& src, Register dest) { movWithPatch(ImmWord(src.asRawBits()), dest); } -void MacroAssembler::moveValue(const TypedOrValueRegister& src, +void MacroAssemblerLOONGARCH64Compat::moveValue(const TypedOrValueRegister& src, const ValueOperand& dest) { if (src.hasValue()) { moveValue(src.valueReg(), dest); @@ -2675,7 +2605,7 @@ void MacroAssembler::moveValue(const TypedOrValueRegister& src, boxDouble(freg, dest, scratch); } -void MacroAssembler::moveValue(const ValueOperand& src, +void MacroAssemblerLOONGARCH64Compat::moveValue(const ValueOperand& src, const ValueOperand& dest) { if (src == dest) { return; @@ -2683,62 +2613,55 @@ void MacroAssembler::moveValue(const ValueOperand& src, movePtr(src.valueReg(), dest.valueReg()); } -void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) { +void MacroAssemblerLOONGARCH64Compat::moveValue(const Value& src, const ValueOperand& dest) { moveValue(src, dest.valueReg()); } // =============================================================== // Branch functions -void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) { - if (ptr != buffer) { - movePtr(ptr, buffer); - } - orPtr(Imm32(gc::ChunkMask), buffer); - loadPtr(Address(buffer, gc::ChunkStoreBufferOffsetFromLastByte), buffer); -} - void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp, Label* label) { MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); MOZ_ASSERT(ptr != temp); - MOZ_ASSERT(ptr != ScratchRegister && - ptr != SecondScratchReg); // Both may be used internally. - MOZ_ASSERT(temp != ScratchRegister && temp != SecondScratchReg); - MOZ_ASSERT(temp != InvalidReg); + MOZ_ASSERT(ptr != SecondScratchReg); - movePtr(ptr, temp); - orPtr(Imm32(gc::ChunkMask), temp); - branchPtr(InvertCondition(cond), - Address(temp, gc::ChunkStoreBufferOffsetFromLastByte), zero, label); + movePtr(ptr, SecondScratchReg); + orPtr(Imm32(gc::ChunkMask), SecondScratchReg); + branch32(cond, Address(SecondScratchReg, gc::ChunkLocationOffsetFromLastByte), + Imm32(int32_t(gc::ChunkLocation::Nursery)), label); } -void MacroAssembler::branchValueIsNurseryCell(Condition cond, - const Address& address, - Register temp, Label* label) { - branchValueIsNurseryCellImpl(cond, address, temp, label); -} - -void MacroAssembler::branchValueIsNurseryCell(Condition cond, - ValueOperand value, Register temp, - Label* label) { - branchValueIsNurseryCellImpl(cond, value, temp, label); -} - -template -void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, - const T& value, Register temp, - Label* label) { +void MacroAssembler::branchValueIsNurseryObject(Condition cond, + const Address& address, + Register temp, Label* label) { MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); - MOZ_ASSERT(temp != InvalidReg); - Label done; - branchTestGCThing(Assembler::NotEqual, value, - cond == Assembler::Equal ? &done : label); - unboxGCThingForGCBarrier(value, temp); + Label done; + branchTestObject(Assembler::NotEqual, address, + cond == Assembler::Equal ? &done : label); + + extractObject(address, temp); orPtr(Imm32(gc::ChunkMask), temp); - loadPtr(Address(temp, gc::ChunkStoreBufferOffsetFromLastByte), temp); - branchPtr(InvertCondition(cond), temp, zero, label); + branch32(cond, Address(temp, gc::ChunkLocationOffsetFromLastByte), + Imm32(int32_t(gc::ChunkLocation::Nursery)), label); + + bind(&done); +} + +void MacroAssembler::branchValueIsNurseryObject(Condition cond, + ValueOperand value, + Register temp, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label done; + branchTestObject(Assembler::NotEqual, value, + cond == Assembler::Equal ? &done : label); + + extractObject(value, temp); + orPtr(Imm32(gc::ChunkMask), temp); + branch32(cond, Address(temp, gc::ChunkLocationOffsetFromLastByte), + Imm32(int32_t(gc::ChunkLocation::Nursery)), label); bind(&done); } @@ -2760,7 +2683,7 @@ void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, MIRType valueType, const T& dest, MIRType slotType) { if (valueType == MIRType::Double) { - boxDouble(value.reg().typedReg().fpu(), dest); + storeDouble(value.reg().typedReg().fpu(), dest); return; } @@ -2799,1614 +2722,6 @@ template void MacroAssembler::storeUnboxedValue( void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); } -// =============================================================== -// WebAssembly - -CodeOffset MacroAssembler::wasmTrapInstruction() { - CodeOffset offset(currentOffset()); - as_break(WASM_TRAP); // TODO: as_teq(zero, zero, WASM_TRAP) - return offset; -} - -void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, - Register boundsCheckLimit, Label* ok) { - ma_b(index, boundsCheckLimit, ok, cond); -} - -void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, - Address boundsCheckLimit, Label* ok) { - SecondScratchRegisterScope scratch2(asMasm()); - load32(boundsCheckLimit, scratch2); - ma_b(index, Register(scratch2), ok, cond); -} - -void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, - Register64 boundsCheckLimit, Label* ok) { - ma_b(index.reg, boundsCheckLimit.reg, ok, cond); -} - -void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, - Address boundsCheckLimit, Label* ok) { - SecondScratchRegisterScope scratch2(asMasm()); - loadPtr(boundsCheckLimit, scratch2); - ma_b(index.reg, scratch2, ok, cond); -} - -// FTINTRZ behaves as follows: -// -// on NaN it produces zero -// on too large it produces INT_MAX (for appropriate type) -// on too small it produces INT_MIN (ditto) - -void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, - Register output, - bool isSaturating, - Label* oolEntry) { - ScratchRegisterScope scratch(asMasm()); - ScratchDoubleScope fpscratch(asMasm()); - if (!isSaturating) { - ma_bc_d(input, input, oolEntry, Assembler::DoubleUnordered); - } - as_ftintrz_l_d(fpscratch, input); - moveFromDouble(fpscratch, output); - as_srli_d(scratch, output, 32); - as_slli_w(output, output, 0); - ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); -} - -void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, - Register output, - bool isSaturating, - Label* oolEntry) { - ScratchRegisterScope scratch(asMasm()); - ScratchDoubleScope fpscratch(asMasm()); - if (!isSaturating) { - ma_bc_s(input, input, oolEntry, Assembler::DoubleUnordered); - } - as_ftintrz_l_s(fpscratch, input); - moveFromDouble(fpscratch, output); - as_srli_d(scratch, output, 32); - as_slli_w(output, output, 0); - ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); -} - -// Assembler::CauseV is a enum,called FCSRBit. Assembler::CauseV == 16 -void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input, - Register output, - bool isSaturating, - Label* oolEntry) { - ScratchRegisterScope scratch(asMasm()); - ScratchFloat32Scope fpscratch(asMasm()); - as_ftintrz_w_d(fpscratch, input); - as_movfcsr2gr(scratch); - moveFromFloat32(fpscratch, output); - MOZ_ASSERT(Assembler::CauseV < 32); - as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); - ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); -} - -void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, - Register output, - bool isSaturating, - Label* oolEntry) { - ScratchRegisterScope scratch(asMasm()); - ScratchFloat32Scope fpscratch(asMasm()); - as_ftintrz_w_s(fpscratch, input); - as_movfcsr2gr(scratch); - moveFromFloat32(fpscratch, output); - MOZ_ASSERT(Assembler::CauseV < 32); - as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); - ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); -} - -void MacroAssembler::wasmTruncateDoubleToUInt64( - FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry, - Label* oolRejoin, FloatRegister tempDouble) { - MOZ_ASSERT(tempDouble.isInvalid()); - ScratchDoubleScope fpscratch(asMasm()); - Register output = output_.reg; - - Label done; - - if (!isSaturating) { - ma_bc_d(input, input, oolEntry, Assembler::DoubleUnordered); - } - as_ftintrz_l_d(fpscratch, input); - moveFromDouble(fpscratch, output); - loadConstantDouble(double(INT64_MAX + 1ULL), fpscratch); - - ScratchRegisterScope scratch(asMasm()); - SecondScratchRegisterScope scratch2(asMasm()); - ma_li(scratch2, ImmWord(INT64_MAX)); - // For numbers in -1.[ : ]INT64_MAX range do nothing more - ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump); - - ma_li(scratch2, ImmWord(INT64_MIN)); - as_fsub_d(fpscratch, input, fpscratch); - as_ftintrz_l_d(fpscratch, fpscratch); - as_movfcsr2gr(scratch); - moveFromDouble(fpscratch, output); - as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); - as_add_d(output, output, scratch2); - - // Guard against negative values that result in 0 due the precision loss. - as_sltui(scratch2, output, 1); - as_or(scratch, scratch, scratch2); - - ma_b(scratch, zero, oolEntry, Assembler::NotEqual); - - bind(&done); - - if (isSaturating) { - bind(oolRejoin); - } -} - -void MacroAssembler::wasmTruncateFloat32ToUInt64( - FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry, - Label* oolRejoin, FloatRegister tempFloat) { - MOZ_ASSERT(tempFloat.isInvalid()); - ScratchDoubleScope fpscratch(asMasm()); - Register output = output_.reg; - - Label done; - - if (!isSaturating) { - ma_bc_s(input, input, oolEntry, Assembler::DoubleUnordered); - } - as_ftintrz_l_s(fpscratch, input); - moveFromDouble(fpscratch, output); - loadConstantFloat32(float(INT64_MAX + 1ULL), fpscratch); - - ScratchRegisterScope scratch(asMasm()); - SecondScratchRegisterScope scratch2(asMasm()); - ma_li(scratch2, ImmWord(INT64_MAX)); - // For numbers in -1.[ : ]INT64_MAX range do nothing more - ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump); - - ma_li(scratch2, ImmWord(INT64_MIN)); - as_fsub_s(fpscratch, input, fpscratch); - as_ftintrz_l_s(fpscratch, fpscratch); - as_movfcsr2gr(scratch); - moveFromDouble(fpscratch, output); - as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); - as_add_d(output, output, scratch2); - - // Guard against negative values that result in 0 due the precision loss. - as_sltui(scratch2, output, 1); - as_or(scratch, scratch, scratch2); - - ma_b(scratch, zero, oolEntry, Assembler::NotEqual); - - bind(&done); - - if (isSaturating) { - bind(oolRejoin); - } -} - -void MacroAssembler::wasmTruncateDoubleToInt64( - FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, - Label* oolRejoin, FloatRegister tempDouble) { - MOZ_ASSERT(tempDouble.isInvalid()); - ScratchRegisterScope scratch(asMasm()); - ScratchDoubleScope fpscratch(asMasm()); - - as_ftintrz_l_d(fpscratch, input); - as_movfcsr2gr(scratch); - moveFromDouble(fpscratch, output.reg); - as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); - ma_b(scratch, zero, oolEntry, Assembler::NotEqual); - - if (isSaturating) { - bind(oolRejoin); - } -} - -void MacroAssembler::wasmTruncateFloat32ToInt64( - FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, - Label* oolRejoin, FloatRegister tempFloat) { - MOZ_ASSERT(tempFloat.isInvalid()); - ScratchRegisterScope scratch(asMasm()); - ScratchDoubleScope fpscratch(asMasm()); - - as_ftintrz_l_s(fpscratch, input); - as_movfcsr2gr(scratch); - moveFromDouble(fpscratch, output.reg); - as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); - ma_b(scratch, zero, oolEntry, Assembler::NotEqual); - - if (isSaturating) { - bind(oolRejoin); - } -} - -void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, - Register memoryBase, Register ptr, - Register ptrScratch, AnyRegister output) { - wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg); -} - -void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, - Register memoryBase, Register ptr, - Register ptrScratch, Register64 output) { - wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg); -} - -void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, - AnyRegister value, Register memoryBase, - Register ptr, Register ptrScratch) { - wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg); -} - -void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, - Register64 value, Register memoryBase, - Register ptr, Register ptrScratch) { - wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg); -} - -void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch, - ExitFrameType type) { - enterFakeExitFrame(cxreg, scratch, type); -} - -// TODO(loongarch64): widenInt32 should be nop? -void MacroAssembler::widenInt32(Register r) { - // Sign-extend the low 32 bits in-place for pointer-sized consumers. - as_slli_w(r, r, 0); -} - -// ======================================================================== -// Convert floating point. - -void MacroAssembler::convertUInt64ToFloat32(Register64 src_, FloatRegister dest, - Register temp) { - MOZ_ASSERT(temp == Register::Invalid()); - ScratchRegisterScope scratch(asMasm()); - SecondScratchRegisterScope scratch2(asMasm()); - - Register src = src_.reg; - Label positive, done; - ma_b(src, src, &positive, NotSigned, ShortJump); - - MOZ_ASSERT(src != scratch); - MOZ_ASSERT(src != scratch2); - - ma_and(scratch, src, Imm32(1)); - as_srli_d(scratch2, src, 1); - as_or(scratch, scratch, scratch2); - as_movgr2fr_d(dest, scratch); - as_ffint_s_l(dest, dest); - addFloat32(dest, dest); - ma_b(&done, ShortJump); - - bind(&positive); - as_movgr2fr_d(dest, src); - as_ffint_s_l(dest, dest); - - bind(&done); -} - -void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) { - as_movgr2fr_d(dest, src.reg); - as_ffint_s_l(dest, dest); -} - -bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; } - -void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, - Register temp) { - MOZ_ASSERT(temp == Register::Invalid()); - MacroAssemblerSpecific::convertUInt64ToDouble(src.reg, dest); -} - -void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) { - as_movgr2fr_d(dest, src.reg); - as_ffint_d_l(dest, dest); -} - -void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) { - convertInt64ToDouble(Register64(src), dest); -} - -// ======================================================================== -// Primitive atomic operations. - -template -static void CompareExchange(MacroAssembler& masm, - const wasm::MemoryAccessDesc* access, - Scalar::Type type, const Synchronization& sync, - const T& mem, Register oldval, Register newval, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - ScratchRegisterScope scratch(masm); - SecondScratchRegisterScope scratch2(masm); - bool signExtend = Scalar::isSignedIntType(type); - unsigned nbytes = Scalar::byteSize(type); - - switch (nbytes) { - case 1: - case 2: - break; - case 4: - MOZ_ASSERT(valueTemp == InvalidReg); - MOZ_ASSERT(offsetTemp == InvalidReg); - MOZ_ASSERT(maskTemp == InvalidReg); - break; - default: - MOZ_CRASH(); - } - - Label again, end; - - masm.computeEffectiveAddress(mem, scratch); - - if (nbytes == 4) { - masm.memoryBarrierBefore(sync); - masm.bind(&again); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_w(output, scratch, 0); - masm.ma_b(output, oldval, &end, Assembler::NotEqual, ShortJump); - masm.as_or(scratch2, newval, zero); - masm.as_sc_w(scratch2, scratch, 0); - masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); - - masm.memoryBarrierAfter(sync); - masm.bind(&end); - - return; - } - - masm.as_andi(offsetTemp, scratch, 3); - masm.subPtr(offsetTemp, scratch); - masm.as_slli_w(offsetTemp, offsetTemp, 3); - masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); - masm.as_sll_w(maskTemp, maskTemp, offsetTemp); - masm.as_nor(maskTemp, zero, maskTemp); - - masm.memoryBarrierBefore(sync); - - masm.bind(&again); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_w(scratch2, scratch, 0); - - masm.as_srl_w(output, scratch2, offsetTemp); - - switch (nbytes) { - case 1: - if (signExtend) { - masm.as_ext_w_b(valueTemp, oldval); - masm.as_ext_w_b(output, output); - } else { - masm.as_andi(valueTemp, oldval, 0xff); - masm.as_andi(output, output, 0xff); - } - break; - case 2: - if (signExtend) { - masm.as_ext_w_h(valueTemp, oldval); - masm.as_ext_w_h(output, output); - } else { - masm.as_bstrpick_d(valueTemp, oldval, 15, 0); - masm.as_bstrpick_d(output, output, 15, 0); - } - break; - } - - masm.ma_b(output, valueTemp, &end, Assembler::NotEqual, ShortJump); - - masm.as_sll_w(valueTemp, newval, offsetTemp); - masm.as_and(scratch2, scratch2, maskTemp); - masm.as_or(scratch2, scratch2, valueTemp); - - masm.as_sc_w(scratch2, scratch, 0); - - masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); - - masm.memoryBarrierAfter(sync); - - masm.bind(&end); -} - -template -static void CompareExchange64(MacroAssembler& masm, - const wasm::MemoryAccessDesc* access, - const Synchronization& sync, const T& mem, - Register64 expect, Register64 replace, - Register64 output) { - MOZ_ASSERT(expect != output && replace != output); - ScratchRegisterScope scratch(masm); - SecondScratchRegisterScope scratch2(masm); - masm.computeEffectiveAddress(mem, scratch); - - Label tryAgain; - Label exit; - - masm.memoryBarrierBefore(sync); - - masm.bind(&tryAgain); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_d(output.reg, scratch, 0); - - masm.ma_b(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump); - masm.movePtr(replace.reg, scratch2); - masm.as_sc_d(scratch2, scratch, 0); - masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero, - ShortJump); - - masm.memoryBarrierAfter(sync); - - masm.bind(&exit); -} - -template -static void AtomicExchange(MacroAssembler& masm, - const wasm::MemoryAccessDesc* access, - Scalar::Type type, const Synchronization& sync, - const T& mem, Register value, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - ScratchRegisterScope scratch(masm); - SecondScratchRegisterScope scratch2(masm); - bool signExtend = Scalar::isSignedIntType(type); - unsigned nbytes = Scalar::byteSize(type); - - switch (nbytes) { - case 1: - case 2: - break; - case 4: - MOZ_ASSERT(valueTemp == InvalidReg); - MOZ_ASSERT(offsetTemp == InvalidReg); - MOZ_ASSERT(maskTemp == InvalidReg); - break; - default: - MOZ_CRASH(); - } - - Label again; - - masm.computeEffectiveAddress(mem, scratch); - - if (nbytes == 4) { - masm.memoryBarrierBefore(sync); - masm.bind(&again); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_w(output, scratch, 0); - masm.as_or(scratch2, value, zero); - masm.as_sc_w(scratch2, scratch, 0); - masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); - - masm.memoryBarrierAfter(sync); - - return; - } - - masm.as_andi(offsetTemp, scratch, 3); - masm.subPtr(offsetTemp, scratch); - masm.as_slli_w(offsetTemp, offsetTemp, 3); - masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); - masm.as_sll_w(maskTemp, maskTemp, offsetTemp); - masm.as_nor(maskTemp, zero, maskTemp); - switch (nbytes) { - case 1: - masm.as_andi(valueTemp, value, 0xff); - break; - case 2: - masm.as_bstrpick_d(valueTemp, value, 15, 0); - break; - } - masm.as_sll_w(valueTemp, valueTemp, offsetTemp); - - masm.memoryBarrierBefore(sync); - - masm.bind(&again); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_w(output, scratch, 0); - masm.as_and(scratch2, output, maskTemp); - masm.as_or(scratch2, scratch2, valueTemp); - - masm.as_sc_w(scratch2, scratch, 0); - - masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); - - masm.as_srl_w(output, output, offsetTemp); - - switch (nbytes) { - case 1: - if (signExtend) { - masm.as_ext_w_b(output, output); - } else { - masm.as_andi(output, output, 0xff); - } - break; - case 2: - if (signExtend) { - masm.as_ext_w_h(output, output); - } else { - masm.as_bstrpick_d(output, output, 15, 0); - } - break; - } - - masm.memoryBarrierAfter(sync); -} - -template -static void AtomicExchange64(MacroAssembler& masm, - const wasm::MemoryAccessDesc* access, - const Synchronization& sync, const T& mem, - Register64 value, Register64 output) { - MOZ_ASSERT(value != output); - ScratchRegisterScope scratch(masm); - SecondScratchRegisterScope scratch2(masm); - masm.computeEffectiveAddress(mem, scratch); - - Label tryAgain; - - masm.memoryBarrierBefore(sync); - - masm.bind(&tryAgain); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_d(output.reg, scratch, 0); - - masm.movePtr(value.reg, scratch2); - masm.as_sc_d(scratch2, scratch, 0); - masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero, - ShortJump); - - masm.memoryBarrierAfter(sync); -} - -template -static void AtomicFetchOp(MacroAssembler& masm, - const wasm::MemoryAccessDesc* access, - Scalar::Type type, const Synchronization& sync, - AtomicOp op, const T& mem, Register value, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - ScratchRegisterScope scratch(masm); - SecondScratchRegisterScope scratch2(masm); - bool signExtend = Scalar::isSignedIntType(type); - unsigned nbytes = Scalar::byteSize(type); - - switch (nbytes) { - case 1: - case 2: - break; - case 4: - MOZ_ASSERT(valueTemp == InvalidReg); - MOZ_ASSERT(offsetTemp == InvalidReg); - MOZ_ASSERT(maskTemp == InvalidReg); - break; - default: - MOZ_CRASH(); - } - - Label again; - - masm.computeEffectiveAddress(mem, scratch); - - if (nbytes == 4) { - masm.memoryBarrierBefore(sync); - masm.bind(&again); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_w(output, scratch, 0); - - switch (op) { - case AtomicFetchAddOp: - masm.as_add_w(scratch2, output, value); - break; - case AtomicFetchSubOp: - masm.as_sub_w(scratch2, output, value); - break; - case AtomicFetchAndOp: - masm.as_and(scratch2, output, value); - break; - case AtomicFetchOrOp: - masm.as_or(scratch2, output, value); - break; - case AtomicFetchXorOp: - masm.as_xor(scratch2, output, value); - break; - default: - MOZ_CRASH(); - } - - masm.as_sc_w(scratch2, scratch, 0); - masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); - - masm.memoryBarrierAfter(sync); - - return; - } - - masm.as_andi(offsetTemp, scratch, 3); - masm.subPtr(offsetTemp, scratch); - masm.as_slli_w(offsetTemp, offsetTemp, 3); - masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); - masm.as_sll_w(maskTemp, maskTemp, offsetTemp); - masm.as_nor(maskTemp, zero, maskTemp); - - masm.memoryBarrierBefore(sync); - - masm.bind(&again); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_w(scratch2, scratch, 0); - masm.as_srl_w(output, scratch2, offsetTemp); - - switch (op) { - case AtomicFetchAddOp: - masm.as_add_w(valueTemp, output, value); - break; - case AtomicFetchSubOp: - masm.as_sub_w(valueTemp, output, value); - break; - case AtomicFetchAndOp: - masm.as_and(valueTemp, output, value); - break; - case AtomicFetchOrOp: - masm.as_or(valueTemp, output, value); - break; - case AtomicFetchXorOp: - masm.as_xor(valueTemp, output, value); - break; - default: - MOZ_CRASH(); - } - - switch (nbytes) { - case 1: - masm.as_andi(valueTemp, valueTemp, 0xff); - break; - case 2: - masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0); - break; - } - - masm.as_sll_w(valueTemp, valueTemp, offsetTemp); - - masm.as_and(scratch2, scratch2, maskTemp); - masm.as_or(scratch2, scratch2, valueTemp); - - masm.as_sc_w(scratch2, scratch, 0); - - masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); - - switch (nbytes) { - case 1: - if (signExtend) { - masm.as_ext_w_b(output, output); - } else { - masm.as_andi(output, output, 0xff); - } - break; - case 2: - if (signExtend) { - masm.as_ext_w_h(output, output); - } else { - masm.as_bstrpick_d(output, output, 15, 0); - } - break; - } - - masm.memoryBarrierAfter(sync); -} - -template -static void AtomicFetchOp64(MacroAssembler& masm, - const wasm::MemoryAccessDesc* access, - const Synchronization& sync, AtomicOp op, - Register64 value, const T& mem, Register64 temp, - Register64 output) { - MOZ_ASSERT(value != output); - MOZ_ASSERT(value != temp); - ScratchRegisterScope scratch(masm); - SecondScratchRegisterScope scratch2(masm); - masm.computeEffectiveAddress(mem, scratch); - - Label tryAgain; - - masm.memoryBarrierBefore(sync); - - masm.bind(&tryAgain); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_d(output.reg, scratch, 0); - - switch (op) { - case AtomicFetchAddOp: - masm.as_add_d(temp.reg, output.reg, value.reg); - break; - case AtomicFetchSubOp: - masm.as_sub_d(temp.reg, output.reg, value.reg); - break; - case AtomicFetchAndOp: - masm.as_and(temp.reg, output.reg, value.reg); - break; - case AtomicFetchOrOp: - masm.as_or(temp.reg, output.reg, value.reg); - break; - case AtomicFetchXorOp: - masm.as_xor(temp.reg, output.reg, value.reg); - break; - default: - MOZ_CRASH(); - } - - masm.as_sc_d(temp.reg, scratch, 0); - masm.ma_b(temp.reg, temp.reg, &tryAgain, Assembler::Zero, ShortJump); - - masm.memoryBarrierAfter(sync); -} - -void MacroAssembler::compareExchange(Scalar::Type type, - const Synchronization& sync, - const Address& mem, Register oldval, - Register newval, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, - offsetTemp, maskTemp, output); -} - -void MacroAssembler::compareExchange(Scalar::Type type, - const Synchronization& sync, - const BaseIndex& mem, Register oldval, - Register newval, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, - offsetTemp, maskTemp, output); -} - -void MacroAssembler::compareExchange64(const Synchronization& sync, - const Address& mem, Register64 expect, - Register64 replace, Register64 output) { - CompareExchange64(*this, nullptr, sync, mem, expect, replace, output); -} - -void MacroAssembler::compareExchange64(const Synchronization& sync, - const BaseIndex& mem, Register64 expect, - Register64 replace, Register64 output) { - CompareExchange64(*this, nullptr, sync, mem, expect, replace, output); -} - -void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, - const Address& mem, Register oldval, - Register newval, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, - newval, valueTemp, offsetTemp, maskTemp, output); -} - -void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, - const BaseIndex& mem, Register oldval, - Register newval, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, - newval, valueTemp, offsetTemp, maskTemp, output); -} - -void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, - const Address& mem, - Register64 expect, - Register64 replace, - Register64 output) { - CompareExchange64(*this, &access, access.sync(), mem, expect, replace, - output); -} - -void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, - const BaseIndex& mem, - Register64 expect, - Register64 replace, - Register64 output) { - CompareExchange64(*this, &access, access.sync(), mem, expect, replace, - output); -} - -void MacroAssembler::atomicExchange(Scalar::Type type, - const Synchronization& sync, - const Address& mem, Register value, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, - maskTemp, output); -} - -void MacroAssembler::atomicExchange(Scalar::Type type, - const Synchronization& sync, - const BaseIndex& mem, Register value, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, - maskTemp, output); -} - -void MacroAssembler::atomicExchange64(const Synchronization& sync, - const Address& mem, Register64 value, - Register64 output) { - AtomicExchange64(*this, nullptr, sync, mem, value, output); -} - -void MacroAssembler::atomicExchange64(const Synchronization& sync, - const BaseIndex& mem, Register64 value, - Register64 output) { - AtomicExchange64(*this, nullptr, sync, mem, value, output); -} - -void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, - const Address& mem, Register value, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, - valueTemp, offsetTemp, maskTemp, output); -} - -void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, - const BaseIndex& mem, Register value, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, - valueTemp, offsetTemp, maskTemp, output); -} - -void MacroAssembler::atomicFetchOp(Scalar::Type type, - const Synchronization& sync, AtomicOp op, - Register value, const Address& mem, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, - offsetTemp, maskTemp, output); -} - -void MacroAssembler::atomicFetchOp(Scalar::Type type, - const Synchronization& sync, AtomicOp op, - Register value, const BaseIndex& mem, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, - offsetTemp, maskTemp, output); -} - -void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, - Register64 value, const Address& mem, - Register64 temp, Register64 output) { - AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output); -} - -void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, - Register64 value, const BaseIndex& mem, - Register64 temp, Register64 output) { - AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output); -} - -void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op, - Register64 value, const Address& mem, - Register64 temp) { - AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp); -} - -void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op, - Register64 value, const BaseIndex& mem, - Register64 temp) { - AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp); -} - -void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, - AtomicOp op, Register value, - const Address& mem, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, - valueTemp, offsetTemp, maskTemp, output); -} - -void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, - AtomicOp op, Register value, - const BaseIndex& mem, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, - valueTemp, offsetTemp, maskTemp, output); -} - -template -static void AtomicEffectOp(MacroAssembler& masm, - const wasm::MemoryAccessDesc* access, - Scalar::Type type, const Synchronization& sync, - AtomicOp op, const T& mem, Register value, - Register valueTemp, Register offsetTemp, - Register maskTemp) { - ScratchRegisterScope scratch(masm); - SecondScratchRegisterScope scratch2(masm); - unsigned nbytes = Scalar::byteSize(type); - - switch (nbytes) { - case 1: - case 2: - break; - case 4: - MOZ_ASSERT(valueTemp == InvalidReg); - MOZ_ASSERT(offsetTemp == InvalidReg); - MOZ_ASSERT(maskTemp == InvalidReg); - break; - default: - MOZ_CRASH(); - } - - Label again; - - masm.computeEffectiveAddress(mem, scratch); - - if (nbytes == 4) { - masm.memoryBarrierBefore(sync); - masm.bind(&again); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_w(scratch2, scratch, 0); - - switch (op) { - case AtomicFetchAddOp: - masm.as_add_w(scratch2, scratch2, value); - break; - case AtomicFetchSubOp: - masm.as_sub_w(scratch2, scratch2, value); - break; - case AtomicFetchAndOp: - masm.as_and(scratch2, scratch2, value); - break; - case AtomicFetchOrOp: - masm.as_or(scratch2, scratch2, value); - break; - case AtomicFetchXorOp: - masm.as_xor(scratch2, scratch2, value); - break; - default: - MOZ_CRASH(); - } - - masm.as_sc_w(scratch2, scratch, 0); - masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); - - masm.memoryBarrierAfter(sync); - - return; - } - - masm.as_andi(offsetTemp, scratch, 3); - masm.subPtr(offsetTemp, scratch); - masm.as_slli_w(offsetTemp, offsetTemp, 3); - masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); - masm.as_sll_w(maskTemp, maskTemp, offsetTemp); - masm.as_nor(maskTemp, zero, maskTemp); - - masm.memoryBarrierBefore(sync); - - masm.bind(&again); - - if (access) { - masm.append(*access, masm.size()); - } - - masm.as_ll_w(scratch2, scratch, 0); - masm.as_srl_w(valueTemp, scratch2, offsetTemp); - - switch (op) { - case AtomicFetchAddOp: - masm.as_add_w(valueTemp, valueTemp, value); - break; - case AtomicFetchSubOp: - masm.as_sub_w(valueTemp, valueTemp, value); - break; - case AtomicFetchAndOp: - masm.as_and(valueTemp, valueTemp, value); - break; - case AtomicFetchOrOp: - masm.as_or(valueTemp, valueTemp, value); - break; - case AtomicFetchXorOp: - masm.as_xor(valueTemp, valueTemp, value); - break; - default: - MOZ_CRASH(); - } - - switch (nbytes) { - case 1: - masm.as_andi(valueTemp, valueTemp, 0xff); - break; - case 2: - masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0); - break; - } - - masm.as_sll_w(valueTemp, valueTemp, offsetTemp); - - masm.as_and(scratch2, scratch2, maskTemp); - masm.as_or(scratch2, scratch2, valueTemp); - - masm.as_sc_w(scratch2, scratch, 0); - - masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); - - masm.memoryBarrierAfter(sync); -} - -void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, - AtomicOp op, Register value, - const Address& mem, Register valueTemp, - Register offsetTemp, - Register maskTemp) { - AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value, - valueTemp, offsetTemp, maskTemp); -} - -void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, - AtomicOp op, Register value, - const BaseIndex& mem, - Register valueTemp, Register offsetTemp, - Register maskTemp) { - AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value, - valueTemp, offsetTemp, maskTemp); -} - -template -static void WasmAtomicExchange64(MacroAssembler& masm, - const wasm::MemoryAccessDesc& access, - const T& mem, Register64 value, - Register64 output) { - AtomicExchange64(masm, &access, access.sync(), mem, value, output); -} - -void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, - const Address& mem, Register64 src, - Register64 output) { - WasmAtomicExchange64(*this, access, mem, src, output); -} - -void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, - const BaseIndex& mem, Register64 src, - Register64 output) { - WasmAtomicExchange64(*this, access, mem, src, output); -} - -void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, - AtomicOp op, Register64 value, - const Address& mem, Register64 temp, - Register64 output) { - AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output); -} - -void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, - AtomicOp op, Register64 value, - const BaseIndex& mem, Register64 temp, - Register64 output) { - AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output); -} - -// ======================================================================== -// JS atomic operations. - -template -static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, - const Synchronization& sync, const T& mem, - Register oldval, Register newval, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register temp, - AnyRegister output) { - if (arrayType == Scalar::Uint32) { - masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, - offsetTemp, maskTemp, temp); - masm.convertUInt32ToDouble(temp, output.fpu()); - } else { - masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, - offsetTemp, maskTemp, output.gpr()); - } -} - -template -static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, - const Synchronization& sync, const T& mem, - Register value, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register temp, AnyRegister output) { - if (arrayType == Scalar::Uint32) { - masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, - maskTemp, temp); - masm.convertUInt32ToDouble(temp, output.fpu()); - } else { - masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, - maskTemp, output.gpr()); - } -} - -template -static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType, - const Synchronization& sync, AtomicOp op, - Register value, const T& mem, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register temp, AnyRegister output) { - if (arrayType == Scalar::Uint32) { - masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, - maskTemp, temp); - masm.convertUInt32ToDouble(temp, output.fpu()); - } else { - masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, - maskTemp, output.gpr()); - } -} - -void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, - const Synchronization& sync, - const Address& mem, Register oldval, - Register newval, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register temp, AnyRegister output) { - CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, - offsetTemp, maskTemp, temp, output); -} - -void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, - const Synchronization& sync, - const BaseIndex& mem, Register oldval, - Register newval, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register temp, AnyRegister output) { - CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, - offsetTemp, maskTemp, temp, output); -} - -void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, - const Synchronization& sync, - const Address& mem, Register value, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register temp, - AnyRegister output) { - AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, - maskTemp, temp, output); -} - -void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, - const Synchronization& sync, - const BaseIndex& mem, Register value, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register temp, - AnyRegister output) { - AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, - maskTemp, temp, output); -} - -void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, - const Synchronization& sync, AtomicOp op, - Register value, const Address& mem, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register temp, - AnyRegister output) { - AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, - maskTemp, temp, output); -} - -void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, - const Synchronization& sync, AtomicOp op, - Register value, const BaseIndex& mem, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register temp, - AnyRegister output) { - AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, - maskTemp, temp, output); -} - -void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, - const Synchronization& sync, AtomicOp op, - Register value, const BaseIndex& mem, - Register valueTemp, Register offsetTemp, - Register maskTemp) { - AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp, - offsetTemp, maskTemp); -} - -void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, - const Synchronization& sync, AtomicOp op, - Register value, const Address& mem, - Register valueTemp, Register offsetTemp, - Register maskTemp) { - AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp, - offsetTemp, maskTemp); -} - -void MacroAssembler::flexibleQuotient32(Register rhs, Register srcDest, - bool isUnsigned, - const LiveRegisterSet&) { - quotient32(rhs, srcDest, isUnsigned); -} - -void MacroAssembler::flexibleRemainder32(Register rhs, Register srcDest, - bool isUnsigned, - const LiveRegisterSet&) { - remainder32(rhs, srcDest, isUnsigned); -} - -void MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest, - Register remOutput, bool isUnsigned, - const LiveRegisterSet&) { - if (isUnsigned) { - as_mod_wu(remOutput, srcDest, rhs); - as_div_wu(srcDest, srcDest, rhs); - } else { - as_mod_w(remOutput, srcDest, rhs); - as_div_w(srcDest, srcDest, rhs); - } -} - -CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) { - return movWithPatch(ImmPtr(nullptr), dest); -} - -void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc, - CodeLocationLabel target) { - PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr)); -} - -// ======================================================================== -// Spectre Mitigations. - -void MacroAssembler::speculationBarrier() { MOZ_CRASH(); } - -void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest, - Label* fail) { - ScratchFloat32Scope fpscratch(asMasm()); - FloatRegister scratch = fpscratch; - Label skipCheck, done; - - // If Nan, 0 or -0 check for bailout - loadConstantFloat32(0.0f, scratch); - ma_bc_s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); - - // If high part is not zero, it is NaN or -0, so we bail. - { - ScratchRegisterScope scratch(asMasm()); - moveFromDoubleLo(src, scratch); - branch32(Assembler::NotEqual, scratch, zero, fail); - } - - // Input was zero, so return zero. - move32(Imm32(0), dest); - ma_b(&done, ShortJump); - - bind(&skipCheck); - as_ftintrm_w_s(scratch, src); - moveFromDoubleLo(scratch, dest); - - branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); - branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); - - bind(&done); -} - -void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest, - Label* fail) { - ScratchDoubleScope fpscratch(asMasm()); - FloatRegister scratch = fpscratch; - Label skipCheck, done; - - // If Nan, 0 or -0 check for bailout - loadConstantDouble(0.0, scratch); - ma_bc_d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); - - // If high part is not zero, it is NaN or -0, so we bail. - { - ScratchRegisterScope scratch(asMasm()); - moveFromDoubleHi(src, scratch); - branch32(Assembler::NotEqual, scratch, zero, fail); - } - - // Input was zero, so return zero. - move32(Imm32(0), dest); - ma_b(&done, ShortJump); - - bind(&skipCheck); - as_ftintrm_w_d(scratch, src); - moveFromDoubleLo(scratch, dest); - - branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); - branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); - - bind(&done); -} - -void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest, - Label* fail) { - ScratchFloat32Scope fpscratch(asMasm()); - FloatRegister scratch = fpscratch; - Label performCeil, done; - - // If x < -1 or x > 0 then perform ceil. - loadConstantFloat32(0.0f, scratch); - branchFloat(Assembler::DoubleGreaterThan, src, scratch, &performCeil); - loadConstantFloat32(-1.0f, scratch); - branchFloat(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil); - - // If binary value is not zero, the input was not 0, so we bail. - { - ScratchRegisterScope scratch(asMasm()); - moveFromFloat32(src, scratch); - branch32(Assembler::NotEqual, scratch, zero, fail); - } - - // Input was zero, so return zero. - move32(Imm32(0), dest); - ma_b(&done, ShortJump); - - bind(&performCeil); - as_ftintrp_w_s(scratch, src); - moveFromFloat32(scratch, dest); - - branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); - branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); - - bind(&done); -} - -void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest, - Label* fail) { - ScratchDoubleScope fpscratch(asMasm()); - FloatRegister scratch = fpscratch; - Label performCeil, done; - - // If x < -1 or x > 0 then perform ceil. - loadConstantDouble(0, scratch); - branchDouble(Assembler::DoubleGreaterThan, src, scratch, &performCeil); - loadConstantDouble(-1.0, scratch); - branchDouble(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil); - - // If binary value is not zero, the input was not 0, so we bail. - { - ScratchRegisterScope scratch(asMasm()); - moveFromDoubleHi(src, scratch); - branch32(Assembler::NotEqual, scratch, zero, fail); - } - - // Input was zero, so return zero. - move32(Imm32(0), dest); - ma_b(&done, ShortJump); - - bind(&performCeil); - as_ftintrp_w_d(scratch, src); - moveFromDoubleLo(scratch, dest); - - branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); - branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); - - bind(&done); -} - -void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest, - FloatRegister temp, Label* fail) { - ScratchFloat32Scope scratch(*this); - - Label negative, end, skipCheck; - - // Load biggest number less than 0.5 in the temp register. - loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp); - - // Branch to a slow path for negative inputs. Doesn't catch NaN or -0. - loadConstantFloat32(0.0f, scratch); - ma_bc_s(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump); - - // If Nan, 0 or -0 check for bailout - ma_bc_s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); - - // If binary value is not zero, it is NaN or -0, so we bail. - { - ScratchRegisterScope scratch(asMasm()); - moveFromFloat32(src, scratch); - branch32(Assembler::NotEqual, scratch, zero, fail); - } - - // Input was zero, so return zero. - move32(Imm32(0), dest); - ma_b(&end, ShortJump); - - bind(&skipCheck); - as_fadd_s(scratch, src, temp); - as_ftintrm_w_s(scratch, scratch); - - moveFromFloat32(scratch, dest); - - branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); - branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); - - jump(&end); - - // Input is negative, but isn't -0. - bind(&negative); - - // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to - // be added the biggest double less than 0.5. - Label loadJoin; - loadConstantFloat32(-0.5f, scratch); - branchFloat(Assembler::DoubleLessThan, src, scratch, &loadJoin); - loadConstantFloat32(0.5f, temp); - bind(&loadJoin); - - as_fadd_s(temp, src, temp); - - // If input + 0.5 >= 0, input is a negative number >= -0.5 and the - // result is -0. - branchFloat(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail); - - // Truncate and round toward zero. - // This is off-by-one for everything but integer-valued inputs. - as_ftintrm_w_s(scratch, temp); - moveFromFloat32(scratch, dest); - - branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); - - bind(&end); -} - -void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest, - FloatRegister temp, Label* fail) { - ScratchDoubleScope scratch(*this); - - Label negative, end, skipCheck; - - // Load biggest number less than 0.5 in the temp register. - loadConstantDouble(GetBiggestNumberLessThan(0.5), temp); - - // Branch to a slow path for negative inputs. Doesn't catch NaN or -0. - loadConstantDouble(0.0, scratch); - ma_bc_d(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump); - - // If Nan, 0 or -0 check for bailout - ma_bc_d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); - - // If high part is not zero, it is NaN or -0, so we bail. - { - ScratchRegisterScope scratch(asMasm()); - moveFromDoubleHi(src, scratch); - branch32(Assembler::NotEqual, scratch, zero, fail); - } - - // Input was zero, so return zero. - move32(Imm32(0), dest); - ma_b(&end, ShortJump); - - bind(&skipCheck); - as_fadd_d(scratch, src, temp); - as_ftintrm_w_d(scratch, scratch); - - moveFromDoubleLo(scratch, dest); - - branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); - branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); - - jump(&end); - - // Input is negative, but isn't -0. - bind(&negative); - - // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to - // be added the biggest double less than 0.5. - Label loadJoin; - loadConstantDouble(-0.5, scratch); - branchDouble(Assembler::DoubleLessThan, src, scratch, &loadJoin); - loadConstantDouble(0.5, temp); - bind(&loadJoin); - - addDouble(src, temp); - - // If input + 0.5 >= 0, input is a negative number >= -0.5 and the - // result is -0. - branchDouble(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail); - - // Truncate and round toward zero. - // This is off-by-one for everything but integer-valued inputs. - as_ftintrm_w_d(scratch, temp); - moveFromDoubleLo(scratch, dest); - - branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); - - bind(&end); -} - -void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest, - Label* fail) { - ScratchRegisterScope scratch(asMasm()); - ScratchFloat32Scope fpscratch(asMasm()); - - Label notZero; - as_ftintrz_w_s(fpscratch, src); - as_movfcsr2gr(scratch); - moveFromFloat32(fpscratch, dest); - as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); - ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); - - { - // dest == zero - SecondScratchRegisterScope scratch2(asMasm()); - moveFromFloat32(src, scratch2); - // Check if input is in ]-1; -0] range by checking the sign bit. - as_slt(scratch2, scratch2, zero); - as_add_d(scratch, scratch, scratch2); - } - - bind(¬Zero); - branch32(Assembler::NotEqual, Register(scratch), zero, fail); -} - -void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest, - Label* fail) { - ScratchRegisterScope scratch(asMasm()); - ScratchFloat32Scope fpscratch(asMasm()); - - Label notZero; - as_ftintrz_w_d(fpscratch, src); - as_movfcsr2gr(scratch); - moveFromFloat32(fpscratch, dest); - as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); - ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); - - { - // dest == zero - SecondScratchRegisterScope scratch2(asMasm()); - moveFromDoubleHi(src, scratch2); - // Check if input is in ]-1; -0] range by checking the sign bit. - as_slt(scratch2, scratch2, zero); - as_add_d(scratch, scratch, scratch2); - } - - bind(¬Zero); - branch32(Assembler::NotEqual, Register(scratch), zero, fail); -} - -void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs, - FloatRegister output) { - MOZ_CRASH("not supported on this platform"); -} - void MacroAssemblerLOONGARCH64Compat::move32(Imm32 imm, Register dest) { ma_li(dest, imm); } @@ -5109,28 +3424,23 @@ void MacroAssemblerLOONGARCH64Compat::breakpoint(uint32_t value) { as_break(value); } -void MacroAssemblerLOONGARCH64Compat::handleFailureWithHandlerTail( - Label* profilerExitTail) { +void MacroAssemblerLOONGARCH64Compat::handleFailureWithHandlerTail(void* handler) { // Reserve space for exception information. int size = (sizeof(ResumeFromException) + ABIStackAlignment) & ~(ABIStackAlignment - 1); asMasm().subPtr(Imm32(size), StackPointer); - mov(StackPointer, a0); // Use a0 since it is a first function argument + mov(StackPointer, a0); // Use a0 since it is the first function argument. // Call the handler. - using Fn = void (*)(ResumeFromException * rfe); asMasm().setupUnalignedABICall(a1); asMasm().passABIArg(a0); - asMasm().callWithABI( - MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + asMasm().callWithABI(handler); Label entryFrame; Label catch_; Label finally; Label return_; Label bailout; - Label wasm; - Label wasmCatch; // Already clobbered a0, so use it... load32(Address(StackPointer, offsetof(ResumeFromException, kind)), a0); @@ -5142,13 +3452,10 @@ void MacroAssemblerLOONGARCH64Compat::handleFailureWithHandlerTail( asMasm().branch32(Assembler::Equal, a0, Imm32(ResumeFromException::RESUME_FINALLY), &finally); asMasm().branch32(Assembler::Equal, a0, - Imm32(ResumeFromException::RESUME_FORCED_RETURN), &return_); + Imm32(ResumeFromException::RESUME_FORCED_RETURN), + &return_); asMasm().branch32(Assembler::Equal, a0, Imm32(ResumeFromException::RESUME_BAILOUT), &bailout); - asMasm().branch32(Assembler::Equal, a0, - Imm32(ResumeFromException::RESUME_WASM), &wasm); - asMasm().branch32(Assembler::Equal, a0, - Imm32(ResumeFromException::RESUME_WASM_CATCH), &wasmCatch); breakpoint(); // Invalid kind. @@ -5158,10 +3465,9 @@ void MacroAssemblerLOONGARCH64Compat::handleFailureWithHandlerTail( asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), StackPointer); - - // We're going to be returning by the ion calling convention - ma_pop(ra); - as_jirl(zero, ra, BOffImm16(0)); + loadPtr(Address(StackPointer, 0), ra); + asMasm().addPtr(Imm32(sizeof(intptr_t)), StackPointer); + ret(); // If we found a catch handler, this must be a baseline frame. Restore // state and jump to the catch block. @@ -5174,7 +3480,8 @@ void MacroAssemblerLOONGARCH64Compat::handleFailureWithHandlerTail( jump(a0); // If we found a finally block, this must be a baseline frame. Push - // two values expected by JSOp::Retsub: the exception and BooleanValue(true). + // two values expected by JSOP_RETSUB: BooleanValue(true) and the + // exception. bind(&finally); ValueOperand exception = ValueOperand(a1); loadValue(Address(sp, offsetof(ResumeFromException, exception)), exception); @@ -5184,8 +3491,8 @@ void MacroAssemblerLOONGARCH64Compat::handleFailureWithHandlerTail( BaselineFrameReg); loadPtr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp); - pushValue(exception); pushValue(BooleanValue(true)); + pushValue(exception); jump(a0); // Only used in debug mode. Return BaselineFrame->returnValue() to the @@ -5198,50 +3505,30 @@ void MacroAssemblerLOONGARCH64Compat::handleFailureWithHandlerTail( loadValue( Address(BaselineFrameReg, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); - as_or(StackPointer, BaselineFrameReg, zero); + movePtr(BaselineFrameReg, StackPointer); pop(BaselineFrameReg); // If profiling is enabled, then update the lastProfilingFrame to refer to // caller frame before returning. { Label skipProfilingInstrumentation; - // Test if profiler enabled. AbsoluteAddress addressOfEnabled( - GetJitContext()->runtime->geckoProfiler().addressOfEnabled()); + GetJitContext()->runtime->spsProfiler().addressOfEnabled()); asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); - jump(profilerExitTail); + profilerExitFrame(); bind(&skipProfilingInstrumentation); } ret(); // If we are bailing out to baseline to handle an exception, jump to - // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success. + // the bailout tail stub. bind(&bailout); loadPtr(Address(sp, offsetof(ResumeFromException, bailoutInfo)), a2); - ma_li(ReturnReg, Imm32(1)); + ma_li(ReturnReg, Imm32(BAILOUT_RETURN_OK)); loadPtr(Address(sp, offsetof(ResumeFromException, target)), a1); jump(a1); - - // If we are throwing and the innermost frame was a wasm frame, reset SP and - // FP; SP is pointing to the unwound return address to the wasm entry, so - // we can just ret(). - bind(&wasm); - loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), - FramePointer); - loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), - StackPointer); - ret(); - - // Found a wasm catch handler, restore state and jump to it. - bind(&wasmCatch); - loadPtr(Address(sp, offsetof(ResumeFromException, target)), a1); - loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), - FramePointer); - loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), - StackPointer); - jump(a1); } CodeOffset MacroAssemblerLOONGARCH64Compat::toggledJump(Label* label) { @@ -5267,17 +3554,6 @@ CodeOffset MacroAssemblerLOONGARCH64Compat::toggledCall(JitCode* target, return offset; // location of first instruction of call instr sequence. } -void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift, - Register pointer) { - if (IsShiftInScaleRange(shift)) { - computeEffectiveAddress( - BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer); - return; - } - lshift32(Imm32(shift), indexTemp32); - addPtr(indexTemp32, pointer); -} - //}}} check_macroassembler_style } // namespace jit diff --git a/js/src/jit/loongarch64/MacroAssembler-loongarch64.h b/js/src/jit/loongarch64/MacroAssembler-loongarch64.h index 2b84262e64..3e3048c5cc 100644 --- a/js/src/jit/loongarch64/MacroAssembler-loongarch64.h +++ b/js/src/jit/loongarch64/MacroAssembler-loongarch64.h @@ -84,7 +84,8 @@ class MacroAssemblerLOONGARCH64 : public Assembler { FPConditionBit fcc = FCC0); public: - void ma_li(Register dest, CodeLabel* label); + void ma_li(Register dest, CodeOffset* label); + void ma_li(Register dest, CodeLabel* label) { ma_li(dest, label->patchAt()); } void ma_li(Register dest, ImmWord imm); void ma_liPatchable(Register dest, ImmPtr imm); void ma_liPatchable(Register dest, ImmWord imm, LiFlags flags = Li48); @@ -878,7 +879,7 @@ class MacroAssemblerLOONGARCH64Compat : public MacroAssemblerLOONGARCH64 { } void pushValue(const Address& addr); - void handleFailureWithHandlerTail(Label* profilerExitTail); + void handleFailureWithHandlerTail(void* handler); ///////////////////////////////////////////////////////////////// // Common interface. diff --git a/js/src/jit/loongarch64/SharedIC-loongarch64.cpp b/js/src/jit/loongarch64/SharedIC-loongarch64.cpp index 6903f1958b..29b37f6420 100644 --- a/js/src/jit/loongarch64/SharedIC-loongarch64.cpp +++ b/js/src/jit/loongarch64/SharedIC-loongarch64.cpp @@ -37,19 +37,19 @@ ICBinaryArith_Int32::Compiler::generateStubCode(MacroAssembler& masm) case JSOP_ADD: masm.unboxInt32(R0, ExtractTemp0); masm.unboxInt32(R1, ExtractTemp1); - masm.ma_addTestOverflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); + masm.ma_add32TestOverflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); masm.boxValue(JSVAL_TYPE_INT32, scratchReg, R0.valueReg()); break; case JSOP_SUB: masm.unboxInt32(R0, ExtractTemp0); masm.unboxInt32(R1, ExtractTemp1); - masm.ma_subTestOverflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); + masm.ma_sub32TestOverflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); masm.boxValue(JSVAL_TYPE_INT32, scratchReg, R0.valueReg()); break; case JSOP_MUL: { masm.unboxInt32(R0, ExtractTemp0); masm.unboxInt32(R1, ExtractTemp1); - masm.ma_mul_branch_overflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); + masm.ma_mul32TestOverflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); masm.ma_b(scratchReg, Imm32(0), &goodMul, Assembler::NotEqual, ShortJump); @@ -70,7 +70,7 @@ ICBinaryArith_Int32::Compiler::generateStubCode(MacroAssembler& masm) masm.ma_b(ExtractTemp1, Imm32(-1), &failure, Assembler::Equal, ShortJump); masm.bind(&divTest1); - // Check for division by zero + // Check for division by zero. masm.ma_b(ExtractTemp1, Imm32(0), &failure, Assembler::Equal, ShortJump); // Check for 0 / X with X < 0 (results in -0). @@ -78,18 +78,16 @@ ICBinaryArith_Int32::Compiler::generateStubCode(MacroAssembler& masm) masm.ma_b(ExtractTemp1, Imm32(0), &failure, Assembler::LessThan, ShortJump); masm.bind(&divTest2); - masm.as_div(ExtractTemp0, ExtractTemp1); + masm.as_mod_w(scratchReg, ExtractTemp0, ExtractTemp1); if (op_ == JSOP_DIV) { // Result is a double if the remainder != 0. - masm.as_mfhi(scratchReg); masm.ma_b(scratchReg, Imm32(0), &failure, Assembler::NotEqual, ShortJump); - masm.as_mflo(scratchReg); + masm.as_div_w(scratchReg, ExtractTemp0, ExtractTemp1); masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); } else { Label done; // If X % Y == 0 and X < 0, the result is -0. - masm.as_mfhi(scratchReg); masm.ma_b(scratchReg, Imm32(0), &done, Assembler::NotEqual, ShortJump); masm.ma_b(ExtractTemp0, Imm32(0), &failure, Assembler::LessThan, ShortJump); masm.bind(&done); @@ -98,35 +96,36 @@ ICBinaryArith_Int32::Compiler::generateStubCode(MacroAssembler& masm) break; } case JSOP_BITOR: - masm.as_or(R0.valueReg() , R0.valueReg(), R1.valueReg()); + masm.as_or(R0.valueReg(), R0.valueReg(), R1.valueReg()); break; case JSOP_BITXOR: masm.as_xor(scratchReg, R0.valueReg(), R1.valueReg()); masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); break; case JSOP_BITAND: - masm.as_and(R0.valueReg() , R0.valueReg(), R1.valueReg()); + masm.as_and(R0.valueReg(), R0.valueReg(), R1.valueReg()); break; case JSOP_LSH: masm.unboxInt32(R0, ExtractTemp0); masm.unboxInt32(R1, ExtractTemp1); - // MIPS will only use 5 lowest bits in R1 as shift offset. - masm.ma_sll(scratchReg, ExtractTemp0, ExtractTemp1); + // LoongArch only uses the low five bits of the shift count here. + masm.as_sll_w(scratchReg, ExtractTemp0, ExtractTemp1); masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); break; case JSOP_RSH: masm.unboxInt32(R0, ExtractTemp0); masm.unboxInt32(R1, ExtractTemp1); - masm.ma_sra(scratchReg, ExtractTemp0, ExtractTemp1); + masm.as_sra_w(scratchReg, ExtractTemp0, ExtractTemp1); masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); break; case JSOP_URSH: masm.unboxInt32(R0, ExtractTemp0); masm.unboxInt32(R1, ExtractTemp1); - masm.ma_srl(scratchReg, ExtractTemp0, ExtractTemp1); + masm.as_srl_w(scratchReg, ExtractTemp0, ExtractTemp1); + masm.as_bstrpick_d(ExtractTemp1, scratchReg, 31, 31); if (allowDouble_) { Label toUint; - masm.ma_b(scratchReg, Imm32(0), &toUint, Assembler::LessThan, ShortJump); + masm.ma_b(ExtractTemp1, Imm32(0), &toUint, Assembler::NotEqual, ShortJump); // Move result and box for return. masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); @@ -136,7 +135,7 @@ ICBinaryArith_Int32::Compiler::generateStubCode(MacroAssembler& masm) masm.convertUInt32ToDouble(scratchReg, FloatReg1); masm.boxDouble(FloatReg1, R0); } else { - masm.ma_b(scratchReg, Imm32(0), &failure, Assembler::LessThan, ShortJump); + masm.ma_b(ExtractTemp1, Imm32(0), &failure, Assembler::NotEqual, ShortJump); // Move result for return. masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); } diff --git a/js/src/jit/loongarch64/Trampoline-loongarch64.cpp b/js/src/jit/loongarch64/Trampoline-loongarch64.cpp index 246052a4ef..c061542f78 100644 --- a/js/src/jit/loongarch64/Trampoline-loongarch64.cpp +++ b/js/src/jit/loongarch64/Trampoline-loongarch64.cpp @@ -62,41 +62,26 @@ GenerateReturn(MacroAssembler& masm, int returnCode) { MOZ_ASSERT(masm.framePushed() == sizeof(EnterJITRegs)); - if (isLoongson()) { - // Restore non-volatile registers - masm.as_ld(s0, StackPointer, offsetof(EnterJITRegs, s0)); - masm.as_gslq(s1, s2, StackPointer, offsetof(EnterJITRegs, s2)); - masm.as_gslq(s3, s4, StackPointer, offsetof(EnterJITRegs, s4)); - masm.as_gslq(s5, s6, StackPointer, offsetof(EnterJITRegs, s6)); - masm.as_gslq(s7, ra, StackPointer, offsetof(EnterJITRegs, ra)); + // Restore non-volatile registers. + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, s0)), s0); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, s1)), s1); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, s2)), s2); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, s3)), s3); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, s4)), s4); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, s5)), s5); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, s6)), s6); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, s7)), s7); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, ra)), ra); - // Restore non-volatile floating point registers - masm.as_gslq(f24, f25, StackPointer, offsetof(EnterJITRegs, f25)); - masm.as_gslq(f26, f27, StackPointer, offsetof(EnterJITRegs, f27)); - masm.as_gslq(f28, f29, StackPointer, offsetof(EnterJITRegs, f29)); - masm.as_gslq(f30, f31, StackPointer, offsetof(EnterJITRegs, f31)); - } else { - // Restore non-volatile registers - masm.as_ld(s0, StackPointer, offsetof(EnterJITRegs, s0)); - masm.as_ld(s1, StackPointer, offsetof(EnterJITRegs, s1)); - masm.as_ld(s2, StackPointer, offsetof(EnterJITRegs, s2)); - masm.as_ld(s3, StackPointer, offsetof(EnterJITRegs, s3)); - masm.as_ld(s4, StackPointer, offsetof(EnterJITRegs, s4)); - masm.as_ld(s5, StackPointer, offsetof(EnterJITRegs, s5)); - masm.as_ld(s6, StackPointer, offsetof(EnterJITRegs, s6)); - masm.as_ld(s7, StackPointer, offsetof(EnterJITRegs, s7)); - masm.as_ld(ra, StackPointer, offsetof(EnterJITRegs, ra)); - - // Restore non-volatile floating point registers - masm.as_ld(f24, StackPointer, offsetof(EnterJITRegs, f24)); - masm.as_ld(f25, StackPointer, offsetof(EnterJITRegs, f25)); - masm.as_ld(f26, StackPointer, offsetof(EnterJITRegs, f26)); - masm.as_ld(f27, StackPointer, offsetof(EnterJITRegs, f27)); - masm.as_ld(f28, StackPointer, offsetof(EnterJITRegs, f28)); - masm.as_ld(f29, StackPointer, offsetof(EnterJITRegs, f29)); - masm.as_ld(f30, StackPointer, offsetof(EnterJITRegs, f30)); - masm.as_ld(f31, StackPointer, offsetof(EnterJITRegs, f31)); - } + // Restore non-volatile floating point registers. + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f24)), f24); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f25)), f25); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f26)), f26); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f27)), f27); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f28)), f28); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f29)), f29); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f30)), f30); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f31)), f31); masm.freeStack(sizeof(EnterJITRegs)); @@ -108,42 +93,27 @@ GeneratePrologue(MacroAssembler& masm) { masm.reserveStack(sizeof(EnterJITRegs)); - if (isLoongson()) { - masm.as_gssq(a7, s0, StackPointer, offsetof(EnterJITRegs, s0)); - masm.as_gssq(s1, s2, StackPointer, offsetof(EnterJITRegs, s2)); - masm.as_gssq(s3, s4, StackPointer, offsetof(EnterJITRegs, s4)); - masm.as_gssq(s5, s6, StackPointer, offsetof(EnterJITRegs, s6)); - masm.as_gssq(s7, ra, StackPointer, offsetof(EnterJITRegs, ra)); + masm.storePtr(s0, Address(StackPointer, offsetof(EnterJITRegs, s0))); + masm.storePtr(s1, Address(StackPointer, offsetof(EnterJITRegs, s1))); + masm.storePtr(s2, Address(StackPointer, offsetof(EnterJITRegs, s2))); + masm.storePtr(s3, Address(StackPointer, offsetof(EnterJITRegs, s3))); + masm.storePtr(s4, Address(StackPointer, offsetof(EnterJITRegs, s4))); + masm.storePtr(s5, Address(StackPointer, offsetof(EnterJITRegs, s5))); + masm.storePtr(s6, Address(StackPointer, offsetof(EnterJITRegs, s6))); + masm.storePtr(s7, Address(StackPointer, offsetof(EnterJITRegs, s7))); + masm.storePtr(ra, Address(StackPointer, offsetof(EnterJITRegs, ra))); + masm.storePtr(a7, Address(StackPointer, offsetof(EnterJITRegs, a7))); - masm.as_gssq(f24, f25, StackPointer, offsetof(EnterJITRegs, f25)); - masm.as_gssq(f26, f27, StackPointer, offsetof(EnterJITRegs, f27)); - masm.as_gssq(f28, f29, StackPointer, offsetof(EnterJITRegs, f29)); - masm.as_gssq(f30, f31, StackPointer, offsetof(EnterJITRegs, f31)); - return; - } - - masm.as_sd(s0, StackPointer, offsetof(EnterJITRegs, s0)); - masm.as_sd(s1, StackPointer, offsetof(EnterJITRegs, s1)); - masm.as_sd(s2, StackPointer, offsetof(EnterJITRegs, s2)); - masm.as_sd(s3, StackPointer, offsetof(EnterJITRegs, s3)); - masm.as_sd(s4, StackPointer, offsetof(EnterJITRegs, s4)); - masm.as_sd(s5, StackPointer, offsetof(EnterJITRegs, s5)); - masm.as_sd(s6, StackPointer, offsetof(EnterJITRegs, s6)); - masm.as_sd(s7, StackPointer, offsetof(EnterJITRegs, s7)); - masm.as_sd(ra, StackPointer, offsetof(EnterJITRegs, ra)); - masm.as_sd(a7, StackPointer, offsetof(EnterJITRegs, a7)); - - masm.as_sd(f24, StackPointer, offsetof(EnterJITRegs, f24)); - masm.as_sd(f25, StackPointer, offsetof(EnterJITRegs, f25)); - masm.as_sd(f26, StackPointer, offsetof(EnterJITRegs, f26)); - masm.as_sd(f27, StackPointer, offsetof(EnterJITRegs, f27)); - masm.as_sd(f28, StackPointer, offsetof(EnterJITRegs, f28)); - masm.as_sd(f29, StackPointer, offsetof(EnterJITRegs, f29)); - masm.as_sd(f30, StackPointer, offsetof(EnterJITRegs, f30)); - masm.as_sd(f31, StackPointer, offsetof(EnterJITRegs, f31)); + masm.storeDouble(f24, Address(StackPointer, offsetof(EnterJITRegs, f24))); + masm.storeDouble(f25, Address(StackPointer, offsetof(EnterJITRegs, f25))); + masm.storeDouble(f26, Address(StackPointer, offsetof(EnterJITRegs, f26))); + masm.storeDouble(f27, Address(StackPointer, offsetof(EnterJITRegs, f27))); + masm.storeDouble(f28, Address(StackPointer, offsetof(EnterJITRegs, f28))); + masm.storeDouble(f29, Address(StackPointer, offsetof(EnterJITRegs, f29))); + masm.storeDouble(f30, Address(StackPointer, offsetof(EnterJITRegs, f30))); + masm.storeDouble(f31, Address(StackPointer, offsetof(EnterJITRegs, f31))); } - // Generates a trampoline for calling Jit compiled code from a C++ function. // The trampoline use the EnterJitCode signature, with the standard x64 fastcall // calling convention. @@ -191,10 +161,10 @@ JitRuntime::generateEnterJIT(JSContext* cx, EnterJitType type) // Make stack algined masm.ma_and(s0, reg_argc, Imm32(1)); - masm.ma_dsubu(s1, StackPointer, Imm32(sizeof(Value))); - masm.as_movn(StackPointer, s1, s0); + masm.ma_sub_d(s1, StackPointer, Imm32(sizeof(Value))); + masm.moveIfNotZero(StackPointer, s1, s0); - masm.as_dsll(s0, reg_argc, 3); // Value* argv + masm.as_slli_d(s0, reg_argc, 3); // Value* argv masm.addPtr(reg_argv, s0); // s0 = &argv[argc] // Loop over arguments, copying them from an unknown buffer onto the Ion @@ -257,7 +227,7 @@ JitRuntime::generateEnterJIT(JSContext* cx, EnterJitType type) masm.movePtr(StackPointer, framePtr); // Reserve space for locals and stack values. - masm.ma_dsll(scratch, numStackValues, Imm32(3)); + masm.as_slli_d(scratch, numStackValues, 3); masm.subPtr(scratch, StackPointer); // Enter exit frame. @@ -301,7 +271,7 @@ JitRuntime::generateEnterJIT(JSContext* cx, EnterJitType type) AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); - masm.ma_daddu(realFramePtr, framePtr, Imm32(sizeof(void*))); + masm.ma_add_d(realFramePtr, framePtr, Imm32(sizeof(void*))); masm.profilerEnterFrame(realFramePtr, scratch); masm.bind(&skipProfilingInstrumentation); } @@ -320,7 +290,7 @@ JitRuntime::generateEnterJIT(JSContext* cx, EnterJitType type) masm.bind(¬Osr); // Load the scope chain in R1. MOZ_ASSERT(R1.scratchReg() != reg_code); - masm.ma_move(R1.scratchReg(), reg_chain); + masm.movePtr(reg_chain, R1.scratchReg()); } // The call will push the return address on the stack, thus we check that @@ -345,7 +315,7 @@ JitRuntime::generateEnterJIT(JSContext* cx, EnterJitType type) masm.addPtr(s0, StackPointer); // Store the returned value into the vp - masm.as_ld(reg_vp, StackPointer, offsetof(EnterJITRegs, a7)); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, a7)), reg_vp); masm.storeValue(JSReturnOperand, Address(reg_vp, 0)); // Restore non-volatile registers and return. @@ -379,7 +349,7 @@ JitRuntime::generateInvalidator(JSContext* cx) // Reserve place for return value and BailoutInfo pointer masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); // Pass pointer to return value. - masm.ma_daddu(a1, StackPointer, Imm32(sizeof(uintptr_t))); + masm.ma_add_d(a1, StackPointer, Imm32(sizeof(uintptr_t))); // Pass pointer to BailoutInfo masm.movePtr(StackPointer, a2); @@ -451,7 +421,7 @@ JitRuntime::generateArgumentsRectifier(JSContext* cx, void** returnAddrOut) static_assert(CalleeToken_FunctionConstructing == 1, "Ensure that we can use the constructing bit to count the value"); masm.mov(calleeTokenReg, t2); - masm.ma_and(t2, Imm32(uint32_t(CalleeToken_FunctionConstructing))); + masm.ma_and(t2, t2, Imm32(uint32_t(CalleeToken_FunctionConstructing))); // Including |this|, and |new.target|, there are (|nformals| + 1 + isConstructing) // arguments to push to the stack. Then we push a JitFrameLayout. We @@ -468,7 +438,7 @@ JitRuntime::generateArgumentsRectifier(JSContext* cx, void** returnAddrOut) masm.and32(Imm32(~(JitStackValueAlignment - 1)), numArgsReg); // Load the number of |undefined|s to push into t1. - masm.as_dsubu(t1, numArgsReg, s3); + masm.as_sub_d(t1, numArgsReg, s3); // Caller: // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- sp <- t2 @@ -504,8 +474,8 @@ JitRuntime::generateArgumentsRectifier(JSContext* cx, void** returnAddrOut) // | - sizeof(Value)| is used to put rcx such that we can read the last // argument, and not the value which is after. - masm.ma_dsll(t0, s3, Imm32(3)); // t0 <- nargs * 8 - masm.as_daddu(t1, t2, t0); // t1 <- t2(saved sp) + nargs * 8 + masm.as_slli_d(t0, s3, 3); // t0 <- nargs * 8 + masm.as_add_d(t1, t2, t0); // t1 <- t2(saved sp) + nargs * 8 masm.addPtr(Imm32(sizeof(RectifierFrameLayout) - sizeof(Value)), t1); // Copy & Push arguments, |nargs| + 1 times (to include |this|). @@ -713,7 +683,7 @@ JitRuntime::generateVMWrapper(JSContext* cx, const VMFunction& f) if (f.explicitArgs) { argsBase = t1; // Use temporary register. regs.take(argsBase); - masm.ma_daddu(argsBase, StackPointer, Imm32(ExitFrameLayout::SizeWithFooter())); + masm.ma_add_d(argsBase, StackPointer, Imm32(ExitFrameLayout::SizeWithFooter())); } // Reserve space for the outparameter. @@ -799,11 +769,11 @@ JitRuntime::generateVMWrapper(JSContext* cx, const VMFunction& f) // Test for failure. switch (f.failType()) { case Type_Object: - masm.branchTestPtr(Assembler::Zero, v0, v0, masm.failureLabel()); + masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, masm.failureLabel()); break; case Type_Bool: // Called functions return bools, which are 0/false and non-zero/true - masm.branchIfFalseBool(v0, masm.failureLabel()); + masm.branchIfFalseBool(ReturnReg, masm.failureLabel()); break; default: MOZ_CRASH("unknown failure kind"); @@ -837,7 +807,7 @@ JitRuntime::generateVMWrapper(JSContext* cx, const VMFunction& f) case Type_Double: if (cx->runtime()->jitSupportsFloatingPoint) { - masm.as_ld(ReturnDoubleReg, StackPointer, 0); + masm.loadDouble(Address(StackPointer, 0), ReturnDoubleReg); } else { masm.assumeUnreachable("Unable to load into float reg, with no FP support."); } @@ -1148,8 +1118,8 @@ JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) // Store return frame in lastProfilingFrame. // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size(); - masm.as_daddu(scratch2, StackPointer, scratch1); - masm.ma_daddu(scratch2, scratch2, Imm32(JitFrameLayout::Size())); + masm.as_add_d(scratch2, StackPointer, scratch1); + masm.ma_add_d(scratch2, scratch2, Imm32(JitFrameLayout::Size())); masm.storePtr(scratch2, lastProfilingFrame); masm.ret(); } @@ -1182,7 +1152,7 @@ JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) // masm.bind(&handle_BaselineStub); { - masm.as_daddu(scratch3, StackPointer, scratch1); + masm.as_add_d(scratch3, StackPointer, scratch1); Address stubFrameReturnAddr(scratch3, JitFrameLayout::Size() + BaselineStubFrameLayout::offsetOfReturnAddress()); @@ -1239,10 +1209,10 @@ JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) masm.bind(&handle_Rectifier); { // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size(); - masm.as_daddu(scratch2, StackPointer, scratch1); + masm.as_add_d(scratch2, StackPointer, scratch1); masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2); masm.loadPtr(Address(scratch2, RectifierFrameLayout::offsetOfDescriptor()), scratch3); - masm.ma_dsrl(scratch1, scratch3, Imm32(FRAMESIZE_SHIFT)); + masm.as_srli_d(scratch1, scratch3, FRAMESIZE_SHIFT); masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch3); // Now |scratch1| contains Rect-Descriptor.Size @@ -1260,7 +1230,7 @@ JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) masm.storePtr(scratch3, lastProfilingCallSite); // scratch3 := RectFrame + Rect-Descriptor.Size + RectifierFrameLayout::Size() - masm.as_daddu(scratch3, scratch2, scratch1); + masm.as_add_d(scratch3, scratch2, scratch1); masm.addPtr(Imm32(RectifierFrameLayout::Size()), scratch3); masm.storePtr(scratch3, lastProfilingFrame); masm.ret(); @@ -1275,7 +1245,7 @@ JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) masm.bind(&checkOk); } #endif - masm.as_daddu(scratch3, scratch2, scratch1); + masm.as_add_d(scratch3, scratch2, scratch1); Address stubFrameReturnAddr(scratch3, RectifierFrameLayout::Size() + BaselineStubFrameLayout::offsetOfReturnAddress()); masm.loadPtr(stubFrameReturnAddr, scratch2); @@ -1307,7 +1277,7 @@ JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) masm.bind(&handle_IonAccessorIC); { // scratch2 := StackPointer + Descriptor.size + JitFrameLayout::Size() - masm.as_daddu(scratch2, StackPointer, scratch1); + masm.as_add_d(scratch2, StackPointer, scratch1); masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2); // scratch3 := AccFrame-Descriptor.Size @@ -1331,7 +1301,7 @@ JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) // lastProfilingFrame := AccessorFrame + AccFrame-Descriptor.Size + // IonAccessorICFrameLayout::Size() - masm.as_daddu(scratch1, scratch2, scratch3); + masm.as_add_d(scratch1, scratch2, scratch3); masm.addPtr(Imm32(IonAccessorICFrameLayout::Size()), scratch1); masm.storePtr(scratch1, lastProfilingFrame); masm.ret();