diff --git a/js/src/jit/BaselineBailouts.cpp b/js/src/jit/BaselineBailouts.cpp index 4bcaadee47..be69f3f1ee 100644 --- a/js/src/jit/BaselineBailouts.cpp +++ b/js/src/jit/BaselineBailouts.cpp @@ -385,7 +385,7 @@ struct BaselineStackBuilder return virtualPointerAtStackOffset(priorOffset); #elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ - defined(JS_CODEGEN_X64) + defined(JS_CODEGEN_LOONGARCH64) || defined(JS_CODEGEN_X64) // On X64, ARM, ARM64, and MIPS, the frame pointer save location depends on // the caller of the rectifier frame. BufferPointer priorFrame = diff --git a/js/src/jit/BaselineCompiler.h b/js/src/jit/BaselineCompiler.h index 30da13d9c1..516be80609 100644 --- a/js/src/jit/BaselineCompiler.h +++ b/js/src/jit/BaselineCompiler.h @@ -15,6 +15,8 @@ # include "jit/arm/BaselineCompiler-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/BaselineCompiler-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/loongarch64/BaselineCompiler-loongarch64.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/BaselineCompiler-mips32.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/CodeGenerator.h b/js/src/jit/CodeGenerator.h index b0fd03538d..ed50c4c402 100644 --- a/js/src/jit/CodeGenerator.h +++ b/js/src/jit/CodeGenerator.h @@ -19,6 +19,8 @@ # include "jit/arm/CodeGenerator-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/CodeGenerator-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/none/CodeGenerator-none.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/CodeGenerator-mips32.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/ExecutableAllocator.h b/js/src/jit/ExecutableAllocator.h index 8042ae7a98..eb53505879 100644 --- a/js/src/jit/ExecutableAllocator.h +++ b/js/src/jit/ExecutableAllocator.h @@ -64,6 +64,7 @@ extern "C" void sync_instruction_memory(caddr_t v, u_int len); #if defined(__linux__) && \ (defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)) && \ + !defined(JS_CODEGEN_LOONGARCH64) && \ (!defined(JS_SIMULATOR_MIPS32) && !defined(JS_SIMULATOR_MIPS64)) #include #endif @@ -220,6 +221,12 @@ class ExecutableAllocator { js::jit::Simulator::FlushICache(code, size); } +#elif defined(JS_CODEGEN_LOONGARCH64) + static void cacheFlush(void* code, size_t size) + { + intptr_t end = reinterpret_cast(code) + size; + __builtin___clear_cache(reinterpret_cast(code), reinterpret_cast(end)); + } #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) static void cacheFlush(void* code, size_t size) { diff --git a/js/src/jit/Ion.h b/js/src/jit/Ion.h index 59a7823499..97ac2d9610 100644 --- a/js/src/jit/Ion.h +++ b/js/src/jit/Ion.h @@ -162,7 +162,7 @@ static inline bool IsIonEnabled(JSContext* cx) { // The ARM64 Ion engine is not yet implemented. -#if defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_ARM64) +#if defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_LOONGARCH64) return false; #else return cx->options().ion() && diff --git a/js/src/jit/IonCaches.h b/js/src/jit/IonCaches.h index 914965055b..ef6965312a 100644 --- a/js/src/jit/IonCaches.h +++ b/js/src/jit/IonCaches.h @@ -10,6 +10,8 @@ # include "jit/arm/Assembler-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/Assembler-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/loongarch64/Assembler-loongarch64.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/Assembler-mips32.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/LIR.h b/js/src/jit/LIR.h index bee3b3ebea..7205bd8b8c 100644 --- a/js/src/jit/LIR.h +++ b/js/src/jit/LIR.h @@ -1927,6 +1927,8 @@ LAllocation::toRegister() const # include "jit/mips64/LIR-mips64.h" # endif # include "jit/mips-shared/LIR-mips-shared.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/none/LIR-none.h" #elif defined(JS_CODEGEN_NONE) # include "jit/none/LIR-none.h" #else diff --git a/js/src/jit/LOpcodes.h b/js/src/jit/LOpcodes.h index 135c616bd1..70d874f690 100644 --- a/js/src/jit/LOpcodes.h +++ b/js/src/jit/LOpcodes.h @@ -14,6 +14,8 @@ # include "jit/arm/LOpcodes-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/LOpcodes-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/none/LOpcodes-none.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/LOpcodes-mips32.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/Lowering.h b/js/src/jit/Lowering.h index 8c380dcdef..f0db465f6e 100644 --- a/js/src/jit/Lowering.h +++ b/js/src/jit/Lowering.h @@ -18,6 +18,8 @@ # include "jit/arm/Lowering-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/Lowering-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/none/Lowering-none.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/Lowering-mips32.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/MacroAssembler-inl.h b/js/src/jit/MacroAssembler-inl.h index 98316eb643..536b71399b 100644 --- a/js/src/jit/MacroAssembler-inl.h +++ b/js/src/jit/MacroAssembler-inl.h @@ -18,6 +18,8 @@ # include "jit/arm/MacroAssembler-arm-inl.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/MacroAssembler-arm64-inl.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/loongarch64/MacroAssembler-loongarch64-inl.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/MacroAssembler-mips32-inl.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h index e5efb475c6..53e0c6bb6d 100644 --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -19,6 +19,8 @@ # include "jit/arm/MacroAssembler-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/MacroAssembler-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/loongarch64/MacroAssembler-loongarch64.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/MacroAssembler-mips32.h" #elif defined(JS_CODEGEN_MIPS64) @@ -66,8 +68,8 @@ using mozilla::FloatingPoint; // architectures on each method declaration, such as PER_ARCH and // PER_SHARED_ARCH. -# define ALL_ARCH mips32, mips64, arm, arm64, x86, x64 -# define ALL_SHARED_ARCH arm, arm64, x86_shared, mips_shared +# define ALL_ARCH mips32, mips64, loongarch64, arm, arm64, x86, x64 +# define ALL_SHARED_ARCH arm, arm64, loongarch64, x86_shared, mips_shared // * How this macro works: // @@ -110,6 +112,7 @@ using mozilla::FloatingPoint; # define DEFINED_ON_x86_shared # define DEFINED_ON_arm # define DEFINED_ON_arm64 +# define DEFINED_ON_loongarch64 # define DEFINED_ON_mips32 # define DEFINED_ON_mips64 # define DEFINED_ON_mips_shared @@ -132,6 +135,9 @@ using mozilla::FloatingPoint; #elif defined(JS_CODEGEN_ARM64) # undef DEFINED_ON_arm64 # define DEFINED_ON_arm64 define +#elif defined(JS_CODEGEN_LOONGARCH64) +# undef DEFINED_ON_loongarch64 +# define DEFINED_ON_loongarch64 define #elif defined(JS_CODEGEN_MIPS32) # undef DEFINED_ON_mips32 # define DEFINED_ON_mips32 define @@ -419,13 +425,13 @@ class MacroAssembler : public MacroAssemblerSpecific // Stack manipulation functions. void PushRegsInMask(LiveRegisterSet set) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); void PushRegsInMask(LiveGeneralRegisterSet set); void PopRegsInMask(LiveRegisterSet set); void PopRegsInMask(LiveGeneralRegisterSet set); void PopRegsInMaskIgnore(LiveRegisterSet set, LiveRegisterSet ignore) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); void Push(const Operand op) DEFINED_ON(x86_shared); void Push(Register reg) PER_SHARED_ARCH; @@ -497,8 +503,8 @@ class MacroAssembler : public MacroAssemblerSpecific void callAndPushReturnAddress(Register reg) DEFINED_ON(x86_shared); void callAndPushReturnAddress(Label* label) DEFINED_ON(x86_shared); - void pushReturnAddress() DEFINED_ON(mips_shared, arm, arm64); - void popReturnAddress() DEFINED_ON(mips_shared, arm, arm64); + void pushReturnAddress() DEFINED_ON(mips_shared, loongarch64, arm, arm64); + void popReturnAddress() DEFINED_ON(mips_shared, loongarch64, arm, arm64); public: // =============================================================== @@ -741,9 +747,9 @@ class MacroAssembler : public MacroAssemblerSpecific inline void xorPtr(Register src, Register dest) PER_ARCH; inline void xorPtr(Imm32 imm, Register dest) PER_ARCH; - inline void and64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64); - inline void or64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64); - inline void xor64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64); + inline void and64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64, loongarch64); + inline void or64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64, loongarch64); + inline void xor64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64, loongarch64); // =============================================================== // Arithmetic functions @@ -759,14 +765,14 @@ class MacroAssembler : public MacroAssemblerSpecific inline void addPtr(Imm32 imm, Register src, Register dest) DEFINED_ON(arm64); inline void addPtr(ImmWord imm, Register dest) PER_ARCH; inline void addPtr(ImmPtr imm, Register dest); - inline void addPtr(Imm32 imm, const Address& dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64); + inline void addPtr(Imm32 imm, const Address& dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64, loongarch64); inline void addPtr(Imm32 imm, const AbsoluteAddress& dest) DEFINED_ON(x86, x64); - inline void addPtr(const Address& src, Register dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64); + inline void addPtr(const Address& src, Register dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64, loongarch64); inline void add64(Register64 src, Register64 dest) PER_ARCH; inline void add64(Imm32 imm, Register64 dest) PER_ARCH; - inline void add64(Imm64 imm, Register64 dest) DEFINED_ON(x86, x64, arm, mips32, mips64); - inline void add64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64); + inline void add64(Imm64 imm, Register64 dest) DEFINED_ON(x86, x64, arm, mips32, mips64, loongarch64); + inline void add64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64, loongarch64); inline void addFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; @@ -778,14 +784,14 @@ class MacroAssembler : public MacroAssemblerSpecific inline void sub32(Imm32 imm, Register dest) PER_SHARED_ARCH; inline void subPtr(Register src, Register dest) PER_ARCH; - inline void subPtr(Register src, const Address& dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64); + inline void subPtr(Register src, const Address& dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64, loongarch64); inline void subPtr(Imm32 imm, Register dest) PER_ARCH; inline void subPtr(ImmWord imm, Register dest) DEFINED_ON(x64); - inline void subPtr(const Address& addr, Register dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64); + inline void subPtr(const Address& addr, Register dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64, loongarch64); inline void sub64(Register64 src, Register64 dest) PER_ARCH; - inline void sub64(Imm64 imm, Register64 dest) DEFINED_ON(x86, x64, arm, mips32, mips64); - inline void sub64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64); + inline void sub64(Imm64 imm, Register64 dest) DEFINED_ON(x86, x64, arm, mips32, mips64, loongarch64); + inline void sub64(const Operand& src, Register64 dest) DEFINED_ON(x64, mips64, loongarch64); inline void subFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; @@ -798,10 +804,10 @@ class MacroAssembler : public MacroAssemblerSpecific inline void mul64(const Operand& src, const Register64& dest) DEFINED_ON(x64); inline void mul64(const Operand& src, const Register64& dest, const Register temp) - DEFINED_ON(x64, mips64); + DEFINED_ON(x64, mips64, loongarch64); inline void mul64(Imm64 imm, const Register64& dest) PER_ARCH; inline void mul64(Imm64 imm, const Register64& dest, const Register temp) - DEFINED_ON(x86, x64, arm, mips32, mips64); + DEFINED_ON(x86, x64, arm, mips32, mips64, loongarch64); inline void mul64(const Register64& src, const Register64& dest, const Register temp) PER_ARCH; @@ -810,7 +816,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline void mulFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; inline void mulDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; - inline void mulDoublePtr(ImmPtr imm, Register temp, FloatRegister dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64); + inline void mulDoublePtr(ImmPtr imm, Register temp, FloatRegister dest) DEFINED_ON(mips_shared, arm, arm64, x86, x64, loongarch64); // Perform an integer division, returning the integer part rounded toward zero. // rhs must not be zero, and the division must not overflow. @@ -835,7 +841,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline void dec32(RegisterOrInt32Constant* key); inline void neg32(Register reg) PER_SHARED_ARCH; - inline void neg64(Register64 reg) DEFINED_ON(x86, x64, arm, mips32, mips64); + inline void neg64(Register64 reg) DEFINED_ON(x86, x64, arm, mips32, mips64, loongarch64); inline void negateFloat(FloatRegister reg) PER_SHARED_ARCH; @@ -899,7 +905,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest) DEFINED_ON(x64); inline void rotateLeft64(Register count, Register64 input, Register64 dest) DEFINED_ON(x64); inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest, Register temp) - DEFINED_ON(x86, x64, arm, mips32, mips64); + DEFINED_ON(x86, x64, arm, mips32, mips64, loongarch64); inline void rotateLeft64(Register count, Register64 input, Register64 dest, Register temp) PER_ARCH; @@ -908,7 +914,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline void rotateRight64(Imm32 count, Register64 input, Register64 dest) DEFINED_ON(x64); inline void rotateRight64(Register count, Register64 input, Register64 dest) DEFINED_ON(x64); inline void rotateRight64(Imm32 count, Register64 input, Register64 dest, Register temp) - DEFINED_ON(x86, x64, arm, mips32, mips64); + DEFINED_ON(x86, x64, arm, mips32, mips64, loongarch64); inline void rotateRight64(Register count, Register64 input, Register64 dest, Register temp) PER_ARCH; @@ -934,7 +940,7 @@ class MacroAssembler : public MacroAssemblerSpecific template inline void cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loongarch64); template inline void cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) @@ -956,9 +962,9 @@ class MacroAssembler : public MacroAssemblerSpecific Label* label); inline void branch32(Condition cond, const AbsoluteAddress& lhs, Register rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); inline void branch32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); inline void branch32(Condition cond, const BaseIndex& lhs, Register rhs, Label* label) DEFINED_ON(x86_shared); @@ -968,7 +974,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branch32(Condition cond, const Operand& lhs, Imm32 rhs, Label* label) DEFINED_ON(x86_shared); inline void branch32(Condition cond, wasm::SymbolicAddress lhs, Imm32 rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); // The supported condition are Equal, NotEqual, LessThan(orEqual), GreaterThan(orEqual), // Below(orEqual) and Above(orEqual). @@ -977,7 +983,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branch64(Condition cond, Register64 lhs, Imm64 val, Label* success, Label* fail = nullptr) PER_ARCH; inline void branch64(Condition cond, Register64 lhs, Register64 rhs, Label* success, - Label* fail = nullptr) DEFINED_ON(x86, x64, arm, mips32, mips64); + Label* fail = nullptr) DEFINED_ON(x86, x64, arm, mips32, mips64, loongarch64); // On x86 and x64 NotEqual and Equal conditions are allowed for the branch64 variants // with Address as lhs. On others only the NotEqual condition. inline void branch64(Condition cond, const Address& lhs, Imm64 val, Label* label) PER_ARCH; @@ -1001,12 +1007,12 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branchPtr(Condition cond, const Address& lhs, ImmWord rhs, Label* label) PER_SHARED_ARCH; inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, Register rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, ImmWord rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); inline void branchPtr(Condition cond, wasm::SymbolicAddress lhs, Register rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); template inline CodeOffsetJump branchPtrWithPatch(Condition cond, Register lhs, T rhs, RepatchLabel* label) PER_SHARED_ARCH; @@ -1014,7 +1020,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline CodeOffsetJump branchPtrWithPatch(Condition cond, Address lhs, T rhs, RepatchLabel* label) PER_SHARED_ARCH; void branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); void branchPtrInNurseryChunk(Condition cond, const Address& address, Register temp, Label* label) DEFINED_ON(x86); void branchValueIsNurseryObject(Condition cond, const Address& address, Register temp, Label* label) PER_ARCH; @@ -1032,9 +1038,9 @@ class MacroAssembler : public MacroAssemblerSpecific // which isn't implemented on all architectures. // E.g. the x64 variants will do this only in the int64_t range. inline void branchTruncateFloat32MaybeModUint32(FloatRegister src, Register dest, Label* fail) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); inline void branchTruncateDoubleMaybeModUint32(FloatRegister src, Register dest, Label* fail) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); // Truncate a double/float32 to intptr and when it doesn't fit jump to the failure label. inline void branchTruncateFloat32ToPtr(FloatRegister src, Register dest, Label* fail) @@ -1044,9 +1050,9 @@ class MacroAssembler : public MacroAssemblerSpecific // Truncate a double/float32 to int32 and when it doesn't fit jump to the failure label. inline void branchTruncateFloat32ToInt32(FloatRegister src, Register dest, Label* fail) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); inline void branchTruncateDoubleToInt32(FloatRegister src, Register dest, Label* fail) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); inline void branchDouble(DoubleCondition cond, FloatRegister lhs, FloatRegister rhs, Label* label) PER_SHARED_ARCH; @@ -1069,7 +1075,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branchTest32(Condition cond, Register lhs, Imm32 rhs, L label) PER_SHARED_ARCH; inline void branchTest32(Condition cond, const Address& lhs, Imm32 rhh, Label* label) PER_SHARED_ARCH; inline void branchTest32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs, Label* label) - DEFINED_ON(arm, arm64, mips_shared, x86, x64); + DEFINED_ON(arm, arm64, mips_shared, x86, x64, loongarch64); template inline void branchTestPtr(Condition cond, Register lhs, Register rhs, L label) PER_SHARED_ARCH; @@ -1132,7 +1138,7 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branchTestUndefined(Condition cond, Register tag, Label* label) PER_SHARED_ARCH; inline void branchTestInt32(Condition cond, Register tag, Label* label) PER_SHARED_ARCH; inline void branchTestDouble(Condition cond, Register tag, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestNumber(Condition cond, Register tag, Label* label) PER_SHARED_ARCH; inline void branchTestBoolean(Condition cond, Register tag, Label* label) PER_SHARED_ARCH; inline void branchTestString(Condition cond, Register tag, Label* label) PER_SHARED_ARCH; @@ -1150,60 +1156,60 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branchTestUndefined(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; inline void branchTestUndefined(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestUndefined(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestInt32(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; inline void branchTestInt32(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestInt32(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestDouble(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; inline void branchTestDouble(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestDouble(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestNumber(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestBoolean(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; inline void branchTestBoolean(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestBoolean(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestString(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestString(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestSymbol(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestSymbol(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestBigInt(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestBigInt(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestNull(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; inline void branchTestNull(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestNull(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); // Clobbers the ScratchReg on x64. inline void branchTestObject(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; inline void branchTestObject(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestObject(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestGCThing(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; inline void branchTestGCThing(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; inline void branchTestPrimitive(Condition cond, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestMagic(Condition cond, const Address& address, Label* label) PER_SHARED_ARCH; inline void branchTestMagic(Condition cond, const BaseIndex& address, Label* label) PER_SHARED_ARCH; template inline void branchTestMagic(Condition cond, const ValueOperand& value, L label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) PER_ARCH; @@ -1216,13 +1222,13 @@ class MacroAssembler : public MacroAssemblerSpecific // Checks if given Value is evaluated to true or false in a condition. // The type of the value should match the type of the method. inline void branchTestInt32Truthy(bool truthy, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestDoubleTruthy(bool truthy, FloatRegister reg, Label* label) PER_SHARED_ARCH; inline void branchTestBooleanTruthy(bool truthy, const ValueOperand& value, Label* label) PER_ARCH; inline void branchTestStringTruthy(bool truthy, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); inline void branchTestBigIntTruthy(bool truthy, const ValueOperand& value, Label* label) - DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + DEFINED_ON(arm, arm64, mips32, mips64, loongarch64, x86_shared); private: @@ -1294,9 +1300,9 @@ class MacroAssembler : public MacroAssemblerSpecific // ======================================================================== // Memory access primitives. inline void storeUncanonicalizedDouble(FloatRegister src, const Address& dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loongarch64); inline void storeUncanonicalizedDouble(FloatRegister src, const BaseIndex& dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loongarch64); inline void storeUncanonicalizedDouble(FloatRegister src, const Operand& dest) DEFINED_ON(x86_shared); @@ -1304,9 +1310,9 @@ class MacroAssembler : public MacroAssemblerSpecific inline void storeDouble(FloatRegister src, const T& dest); inline void storeUncanonicalizedFloat32(FloatRegister src, const Address& dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loongarch64); inline void storeUncanonicalizedFloat32(FloatRegister src, const BaseIndex& dest) - DEFINED_ON(x86_shared, arm, arm64, mips32, mips64); + DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loongarch64); inline void storeUncanonicalizedFloat32(FloatRegister src, const Operand& dest) DEFINED_ON(x86_shared); @@ -1747,7 +1753,7 @@ class MacroAssembler : public MacroAssemblerSpecific template inline void addToStackPtr(T t); template inline void addStackPtrTo(T t); - void subFromStackPtr(Imm32 imm32) DEFINED_ON(mips32, mips64, arm, x86, x64); + void subFromStackPtr(Imm32 imm32) DEFINED_ON(mips32, mips64, loongarch64, arm, x86, x64); void subFromStackPtr(Register reg); template diff --git a/js/src/jit/MoveEmitter.h b/js/src/jit/MoveEmitter.h index d8c808f0e2..eadd9d27bf 100644 --- a/js/src/jit/MoveEmitter.h +++ b/js/src/jit/MoveEmitter.h @@ -12,6 +12,8 @@ # include "jit/arm/MoveEmitter-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/MoveEmitter-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/mips64/MoveEmitter-mips64.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/MoveEmitter-mips32.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/Registers.h b/js/src/jit/Registers.h index 1016c18379..1ed928a510 100644 --- a/js/src/jit/Registers.h +++ b/js/src/jit/Registers.h @@ -15,6 +15,8 @@ # include "jit/arm/Architecture-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/Architecture-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/loongarch64/Architecture-loongarch64.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/Architecture-mips32.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/SharedICHelpers.h b/js/src/jit/SharedICHelpers.h index 6093eb4ad8..aa73e795f7 100644 --- a/js/src/jit/SharedICHelpers.h +++ b/js/src/jit/SharedICHelpers.h @@ -14,6 +14,8 @@ # include "jit/arm/SharedICHelpers-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/SharedICHelpers-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/loongarch64/SharedICHelpers-loongarch64.h" #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) #include "jit/mips-shared/SharedICHelpers-mips-shared.h" #elif defined(JS_CODEGEN_NONE) diff --git a/js/src/jit/SharedICRegisters.h b/js/src/jit/SharedICRegisters.h index f8ad817e1a..ef6dde36ca 100644 --- a/js/src/jit/SharedICRegisters.h +++ b/js/src/jit/SharedICRegisters.h @@ -14,6 +14,8 @@ # include "jit/arm/SharedICRegisters-arm.h" #elif defined(JS_CODEGEN_ARM64) # include "jit/arm64/SharedICRegisters-arm64.h" +#elif defined(JS_CODEGEN_LOONGARCH64) +# include "jit/loongarch64/SharedICRegisters-loongarch64.h" #elif defined(JS_CODEGEN_MIPS32) # include "jit/mips32/SharedICRegisters-mips32.h" #elif defined(JS_CODEGEN_MIPS64) diff --git a/js/src/jit/loongarch64/Architecture-loongarch64.cpp b/js/src/jit/loongarch64/Architecture-loongarch64.cpp new file mode 100644 index 0000000000..41321265b8 --- /dev/null +++ b/js/src/jit/loongarch64/Architecture-loongarch64.cpp @@ -0,0 +1,98 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loongarch64/Architecture-loongarch64.h" + +#include + +#ifdef JS_SIMULATOR +# include "jit/loongarch64/Simulator-loongarch64.h" +#endif +#include "jit/RegisterSets.h" + +namespace js { +namespace jit { + +Registers::Code Registers::FromName(const char* name) { + for (size_t i = 0; i < TotalPhys; i++) { + if (strcmp(GetName(i), name) == 0) { + return Code(i); + } + } + + return Invalid; +} + +FloatRegisters::Code FloatRegisters::FromName(const char* name) { + for (size_t i = 0; i < TotalPhys; i++) { + if (strcmp(GetName(i), name) == 0) { + return Code(i); + } + } + + return Invalid; +} + +FloatRegisterSet FloatRegister::ReduceSetForPush(const FloatRegisterSet& s) { +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + LiveFloatRegisterSet ret; + for (FloatRegisterIterator iter(s); iter.more(); ++iter) { + ret.addUnchecked(FromCode((*iter).encoding())); + } + return ret.set(); +} + +uint32_t FloatRegister::GetSizeInBytes(const FloatRegisterSet& s) { +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + return s.size() * sizeof(double); +} + +uint32_t FloatRegister::GetPushSizeInBytes(const FloatRegisterSet& s) { +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + return s.size() * sizeof(double); +} + +uint32_t FloatRegister::getRegisterDumpOffsetInBytes() { +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + return encoding() * sizeof(double); +} + +bool CPUFlagsHaveBeenComputed() { + // TODO(loongarch64): Add CPU flags support. + return true; +} + +uint32_t GetLOONGARCH64Flags() { return 0; } + +void FlushICache(void* code, size_t size, bool codeIsThreadLocal) { +#if defined(JS_SIMULATOR) + js::jit::SimulatorProcess::FlushICache(code, size); + +#elif defined(__GNUC__) + intptr_t end = reinterpret_cast(code) + size; + __builtin___clear_cache(reinterpret_cast(code), + reinterpret_cast(end)); + +#else + _flush_cache(reinterpret_cast(code), size, BCACHE); + +#endif +} + +} // namespace jit +} // namespace js diff --git a/js/src/jit/loongarch64/Architecture-loongarch64.h b/js/src/jit/loongarch64/Architecture-loongarch64.h new file mode 100644 index 0000000000..779183444b --- /dev/null +++ b/js/src/jit/loongarch64/Architecture-loongarch64.h @@ -0,0 +1,510 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loongarch64_Architecture_loongarch64_h +#define jit_loongarch64_Architecture_loongarch64_h + +#include "mozilla/MathAlgorithms.h" + +#include +#include +#include + +#include "js/Utility.h" + +namespace js { +namespace jit { + +// LoongArch64 has 32 64-bit integer registers, r0 though r31. +// The program counter is not accessible as a register. +// +// SIMD and scalar floating-point registers share a register bank. +// Floating-point registers are f0 through f31. +// 128 bit SIMD registers are vr0 through vr31. +// e.g., f0 is the bottom 64 bits of vr0. + +// LoongArch64 INT Register Convention: +// Name Alias Usage +// $r0 $zero Constant zero +// $r1 $ra Return address +// $r2 $tp TLS +// $r3 $sp Stack pointer +// $r4-$r11 $a0-$a7 Argument registers +// $r4-$r5 $v0-$v1 Return values +// $r12-$r20 $t0-$t8 Temporary registers +// $r21 $x Reserved +// $r22 $fp Frame pointer +// $r23-$r31 $s0-$s8 Callee-saved registers + +// LoongArch64 FP Register Convention: +// Name Alias Usage +// $f0-$f7 $fa0-$fa7 Argument registers +// $f0-$f1 $fv0-$fv1 Return values +// $f8-f23 $ft0-$ft15 Temporary registers +// $f24-$f31 $fs0-$fs7 Callee-saved registers + +class Registers { + public: + enum RegisterID { + r0 = 0, + r1, + r2, + r3, + r4, + r5, + r6, + r7, + r8, + r9, + r10, + r11, + r12, + r13, + r14, + r15, + r16, + r17, + r18, + r19, + r20, + r21, + r22, + r23, + r24, + r25, + r26, + r27, + r28, + r29, + r30, + r31, + zero = r0, + ra = r1, + tp = r2, + sp = r3, + a0 = r4, + a1 = r5, + a2 = r6, + a3 = r7, + a4 = r8, + a5 = r9, + a6 = r10, + a7 = r11, + t0 = r12, + t1 = r13, + t2 = r14, + t3 = r15, + t4 = r16, + t5 = r17, + t6 = r18, + t7 = r19, + t8 = r20, + rx = r21, + fp = r22, + s0 = r23, + s1 = r24, + s2 = r25, + s3 = r26, + s4 = r27, + s5 = r28, + s6 = r29, + s7 = r30, + s8 = r31, + invalid_reg, + }; + typedef uint8_t Code; + typedef RegisterID Encoding; + typedef uint32_t SetType; + + static const Encoding StackPointer = sp; + static const Encoding Invalid = invalid_reg; + + // Content spilled during bailouts. + union RegisterContent { + uintptr_t r; + }; + + static uint32_t SetSize(SetType x) { + static_assert(sizeof(SetType) == 4, "SetType must be 32 bits"); + return mozilla::CountPopulation32(x); + } + static uint32_t FirstBit(SetType x) { + return mozilla::CountTrailingZeroes32(x); + } + static uint32_t LastBit(SetType x) { + return 31 - mozilla::CountLeadingZeroes32(x); + } + + static const char* GetName(uint32_t code) { + static const char* const Names[] = { + "zero", "ra", "tp", "sp", "a0", "a1", "a2", "a3", "a4", "a5", "a6", + "a7", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "rx", + "fp", "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8"}; + static_assert(Total == sizeof(Names) / sizeof(Names[0]), + "Table is the correct size"); + if ((code & 31) >= TotalPhys) { + return "invalid"; + } + return Names[code & 31]; + } + + static Code FromName(const char* name); + + static const uint32_t Total = 32; + static const uint32_t TotalPhys = 32; + static const uint32_t Allocatable = + 23; // No named special-function registers. + + static const SetType AllMask = 0xFFFFFFFF; + static const SetType NoneMask = 0x0; + + static const SetType ArgRegMask = + (1 << Registers::a0) | (1 << Registers::a1) | (1 << Registers::a2) | + (1 << Registers::a3) | (1 << Registers::a4) | (1 << Registers::a5) | + (1 << Registers::a6) | (1 << Registers::a7); + + static const SetType VolatileMask = + (1 << Registers::a0) | (1 << Registers::a1) | (1 << Registers::a2) | + (1 << Registers::a3) | (1 << Registers::a4) | (1 << Registers::a5) | + (1 << Registers::a6) | (1 << Registers::a7) | (1 << Registers::t0) | + (1 << Registers::t1) | (1 << Registers::t2) | (1 << Registers::t3) | + (1 << Registers::t4) | (1 << Registers::t5) | (1 << Registers::t6); + + // We use this constant to save registers when entering functions. This + // is why $ra is added here even though it is not "Non Volatile". + static const SetType NonVolatileMask = + (1 << Registers::ra) | (1 << Registers::fp) | (1 << Registers::s0) | + (1 << Registers::s1) | (1 << Registers::s2) | (1 << Registers::s3) | + (1 << Registers::s4) | (1 << Registers::s5) | (1 << Registers::s6) | + (1 << Registers::s7) | (1 << Registers::s8); + + static const SetType SingleByteRegs = VolatileMask | NonVolatileMask; + + static const SetType NonAllocatableMask = + (1 << Registers::zero) | // Always be zero. + (1 << Registers::t7) | // First scratch register. + (1 << Registers::t8) | // Second scratch register. + (1 << Registers::rx) | // Reserved Register. + (1 << Registers::ra) | (1 << Registers::tp) | (1 << Registers::sp); + + static const SetType TempMask = VolatileMask & ~NonAllocatableMask; + + static const SetType WrapperMask = VolatileMask; + + // Registers returned from a JS -> JS call. + static const SetType JSCallMask = (1 << Registers::a2); + + // Registers returned from a JS -> C call. + static const SetType CallMask = (1 << Registers::a0); + + static const SetType AllocatableMask = AllMask & ~NonAllocatableMask; +}; + +// Smallest integer type that can hold a register bitmask. +typedef uint32_t PackedRegisterMask; + +template +class TypedRegisterSet; + +class FloatRegisters { + public: + enum FPRegisterID { + f0 = 0, + f1, + f2, + f3, + f4, + f5, + f6, + f7, + f8, + f9, + f10, + f11, + f12, + f13, + f14, + f15, + f16, + f17, + f18, + f19, + f20, + f21, + f22, + f23, // Scratch register. + f24, + f25, + f26, + f27, + f28, + f29, + f30, + f31, + }; + + // Eight bits: (invalid << 7) | (kind << 5) | encoding + typedef uint8_t Code; + typedef FPRegisterID Encoding; + typedef uint64_t SetType; + + enum Kind : uint8_t { Double, Single, NumTypes }; + + static constexpr Code Invalid = 0x80; + + static const char* GetName(uint32_t code) { + static const char* const Names[] = { + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"}; + static_assert(TotalPhys == sizeof(Names) / sizeof(Names[0]), + "Table is the correct size"); + if ((code & 31) >= TotalPhys) { + return "invalid"; + } + return Names[code & 31]; + } + + static Code FromName(const char* name); + + static const uint32_t TotalPhys = 32; + static const uint32_t Total = TotalPhys * NumTypes; + static const uint32_t Allocatable = 31; // Without f23, the scratch register. + + static_assert(sizeof(SetType) * 8 >= Total, + "SetType should be large enough to enumerate all registers."); + + // Magic values which are used to duplicate a mask of physical register for + // a specific type of register. A multiplication is used to copy and shift + // the bits of the physical register mask. + static const SetType SpreadSingle = SetType(1) + << (uint32_t(Single) * TotalPhys); + static const SetType SpreadDouble = SetType(1) + << (uint32_t(Double) * TotalPhys); + static const SetType Spread = SpreadSingle | SpreadDouble; + + static const SetType AllPhysMask = ((SetType(1) << TotalPhys) - 1); + static const SetType AllMask = AllPhysMask * Spread; + static const SetType AllSingleMask = AllPhysMask * SpreadSingle; + static const SetType AllDoubleMask = AllPhysMask * SpreadDouble; + static const SetType NoneMask = SetType(0); + + // TODO(loongarch64): Much less than ARM64 here. + static const SetType NonVolatileMask = + SetType((1 << FloatRegisters::f24) | (1 << FloatRegisters::f25) | + (1 << FloatRegisters::f26) | (1 << FloatRegisters::f27) | + (1 << FloatRegisters::f28) | (1 << FloatRegisters::f29) | + (1 << FloatRegisters::f30) | (1 << FloatRegisters::f31)) * + Spread; + + static const SetType VolatileMask = AllMask & ~NonVolatileMask; + + static const SetType WrapperMask = VolatileMask; + + // f23 is the scratch register. + static const SetType NonAllocatableMask = + (SetType(1) << FloatRegisters::f23) * Spread; + + static const SetType TempMask = VolatileMask & ~NonAllocatableMask; + + static const SetType AllocatableMask = AllMask & ~NonAllocatableMask; + + // Content spilled during bailouts. + union RegisterContent { + float s; + double d; + }; + + static constexpr Encoding encoding(Code c) { + // assert() not available in constexpr function. + // assert(c < Total); + return Encoding(c & 31); + } + + static constexpr Kind kind(Code c) { + // assert() not available in constexpr function. + // assert(c < Total && ((c >> 5) & 3) < NumTypes); + return Kind((c >> 5) & 3); + } + + static constexpr Code fromParts(uint32_t encoding, uint32_t kind, + uint32_t invalid) { + return Code((invalid << 7) | (kind << 5) | encoding); + } +}; + +static const uint32_t SpillSlotSize = + std::max(sizeof(Registers::RegisterContent), + sizeof(FloatRegisters::RegisterContent)); + +static const uint32_t ION_FRAME_SLACK_SIZE = 24; +static const uint32_t ShadowStackSpace = 0; +static const uint32_t SizeOfReturnAddressAfterCall = 0; + +// When our only strategy for far jumps is to encode the offset directly, and +// not insert any jump islands during assembly for even further jumps, then the +// architecture restricts us to -2^27 .. 2^27-4, to fit into a signed 28-bit +// value. We further reduce this range to allow the far-jump inserting code to +// have some breathing room. +static const uint32_t JumpImmediateRange = ((1 << 27) - (20 * 1024 * 1024)); + +struct FloatRegister { + typedef FloatRegisters Codes; + typedef Codes::Code Code; + typedef Codes::Encoding Encoding; + typedef Codes::SetType SetType; + + static uint32_t SetSize(SetType x) { + static_assert(sizeof(SetType) == 8, "SetType must be 64 bits"); + x |= x >> FloatRegisters::TotalPhys; + x &= FloatRegisters::AllPhysMask; + return mozilla::CountPopulation32(x); + } + + static uint32_t FirstBit(SetType x) { + static_assert(sizeof(SetType) == 8, "SetType"); + return mozilla::CountTrailingZeroes64(x); + } + static uint32_t LastBit(SetType x) { + static_assert(sizeof(SetType) == 8, "SetType"); + return 63 - mozilla::CountLeadingZeroes64(x); + } + + private: + // These fields only hold valid values: an invalid register is always + // represented as a valid encoding and kind with the invalid_ bit set. + uint8_t encoding_; // 32 encodings + uint8_t kind_; // Double, Single; more later + bool invalid_; + + typedef Codes::Kind Kind; + + public: + union RegisterContent { + float s; + double d; + }; + + constexpr FloatRegister(Encoding encoding, Kind kind) + : encoding_(encoding), kind_(kind), invalid_(false) { + // assert(uint32_t(encoding) < Codes::TotalPhys); + } + + explicit constexpr FloatRegister(uint32_t code) + : encoding_(FloatRegisters::encoding(code)), + kind_(FloatRegisters::kind(code)), + invalid_(false) {} + + constexpr FloatRegister() + : encoding_(0), kind_(FloatRegisters::Double), invalid_(true) {} + + static FloatRegister FromCode(uint32_t i) { + MOZ_ASSERT(i < Codes::Total); + return FloatRegister(FloatRegisters::encoding(i), FloatRegisters::kind(i)); + } + + bool isSingle() const { + MOZ_ASSERT(!invalid_); + return kind_ == FloatRegisters::Single; + } + bool isDouble() const { + MOZ_ASSERT(!invalid_); + return kind_ == FloatRegisters::Double; + } + bool isSimd128() const { + MOZ_ASSERT(!invalid_); + return false; + } + bool isInvalid() const { return invalid_; } + + FloatRegister asSingle() const { + MOZ_ASSERT(!invalid_); + return FloatRegister(Encoding(encoding_), FloatRegisters::Single); + } + FloatRegister asDouble() const { + MOZ_ASSERT(!invalid_); + return FloatRegister(Encoding(encoding_), FloatRegisters::Double); + } + FloatRegister asSimd128() const { MOZ_CRASH(); } + + constexpr uint32_t size() const { + MOZ_ASSERT(!invalid_); + if (kind_ == FloatRegisters::Double) { + return sizeof(double); + } + MOZ_ASSERT(kind_ == FloatRegisters::Single); + return sizeof(float); + } + + constexpr Code code() const { + // assert(!invalid_); + return Codes::fromParts(encoding_, kind_, invalid_); + } + + constexpr Encoding encoding() const { + MOZ_ASSERT(!invalid_); + return Encoding(encoding_); + } + + const char* name() const { return FloatRegisters::GetName(uint32_t(encoding_)); } + bool volatile_() const { + MOZ_ASSERT(!invalid_); + return !!((SetType(1) << code()) & FloatRegisters::VolatileMask); + } + constexpr bool operator!=(FloatRegister other) const { + return code() != other.code(); + } + constexpr bool operator==(FloatRegister other) const { + return code() == other.code(); + } + + bool aliases(FloatRegister other) const { + return other.encoding_ == encoding_; + } + // Ensure that two floating point registers' types are equivalent. + bool equiv(FloatRegister other) const { + MOZ_ASSERT(!invalid_); + return kind_ == other.kind_; + } + + uint32_t numAliased() const { return Codes::NumTypes; } + uint32_t numAlignedAliased() const { return numAliased(); } + + void aliased(uint32_t aliasIdx, FloatRegister* ret) const { + MOZ_ASSERT(!invalid_); + MOZ_ASSERT(aliasIdx < numAliased()); + *ret = FloatRegister(Encoding(encoding_), + Kind((aliasIdx + kind_) % numAliased())); + } + void alignedAliased(uint32_t aliasIdx, FloatRegister* ret) const { + MOZ_ASSERT(aliasIdx < numAliased()); + aliased(aliasIdx, ret); + } + SetType alignedOrDominatedAliasedSet() const { + return Codes::Spread << encoding_; + } + + static Code FromName(const char* name) { return FloatRegisters::FromName(name); } + static TypedRegisterSet ReduceSetForPush( + const TypedRegisterSet& s); + static uint32_t GetSizeInBytes(const TypedRegisterSet& s); + static uint32_t GetPushSizeInBytes(const TypedRegisterSet& s); + uint32_t getRegisterDumpOffsetInBytes(); +}; + +// LoongArch doesn't have double registers that cannot be treated as float32. +inline bool hasUnaliasedDouble() { return false; } + +// LoongArch doesn't have double registers that alias multiple floats. +inline bool hasMultiAlias() { return false; } + +uint32_t GetLOONGARCH64Flags(); + +} // namespace jit +} // namespace js + +#endif /* jit_loongarch64_Architecture_loongarch64_h */ diff --git a/js/src/jit/loongarch64/Assembler-loongarch64.cpp b/js/src/jit/loongarch64/Assembler-loongarch64.cpp new file mode 100644 index 0000000000..ffa748e09b --- /dev/null +++ b/js/src/jit/loongarch64/Assembler-loongarch64.cpp @@ -0,0 +1,2491 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loongarch64/Assembler-loongarch64.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/Maybe.h" + +#include "gc/Marking.h" +#include "jit/JitCompartment.h" +#include "jit/ExecutableAllocator.h" +#include "vm/Realm.h" + +using mozilla::DebugOnly; + +using namespace js; +using namespace js::jit; + +// Note this is used for inter-wasm calls and may pass arguments and results +// in floating point registers even if the system ABI does not. + +// TODO(loongarch64): Inconsistent with LoongArch's calling convention. +// LoongArch floating-point parameters calling convention: +// The first eight floating-point parameters should be passed in f0-f7, and +// the other floating point parameters will be passed like integer parameters. +// But we just pass the other floating-point parameters on stack here. +ABIArg ABIArgGenerator::next(MIRType type) { + switch (type) { + case MIRType::Int32: + case MIRType::Int64: + case MIRType::Pointer: + case MIRType::RefOrNull: + case MIRType::StackResults: { + if (intRegIndex_ == NumIntArgRegs) { + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(uintptr_t); + break; + } + current_ = ABIArg(Register::FromCode(intRegIndex_ + a0.encoding())); + intRegIndex_++; + break; + } + case MIRType::Float32: + case MIRType::Double: { + if (floatRegIndex_ == NumFloatArgRegs) { + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(double); + break; + } + current_ = ABIArg(FloatRegister( + FloatRegisters::Encoding(floatRegIndex_ + f0.encoding()), + type == MIRType::Double ? FloatRegisters::Double + : FloatRegisters::Single)); + floatRegIndex_++; + break; + } + case MIRType::Simd128: { + MOZ_CRASH("LoongArch does not support simd yet."); + break; + } + default: + MOZ_CRASH("Unexpected argument type"); + } + return current_; +} + +// Encode a standard register when it is being used as rd, the rj, and +// an extra register(rk). These should never be called with an InvalidReg. +uint32_t js::jit::RJ(Register r) { + MOZ_ASSERT(r != InvalidReg); + return r.encoding() << RJShift; +} + +uint32_t js::jit::RK(Register r) { + MOZ_ASSERT(r != InvalidReg); + return r.encoding() << RKShift; +} + +uint32_t js::jit::RD(Register r) { + MOZ_ASSERT(r != InvalidReg); + return r.encoding() << RDShift; +} + +uint32_t js::jit::FJ(FloatRegister r) { return r.encoding() << RJShift; } + +uint32_t js::jit::FK(FloatRegister r) { return r.encoding() << RKShift; } + +uint32_t js::jit::FD(FloatRegister r) { return r.encoding() << RDShift; } + +uint32_t js::jit::FA(FloatRegister r) { return r.encoding() << FAShift; } + +uint32_t js::jit::SA2(uint32_t value) { + MOZ_ASSERT(value < 4); + return (value & SA2Mask) << SAShift; +} + +uint32_t js::jit::SA3(uint32_t value) { + MOZ_ASSERT(value < 8); + return (value & SA3Mask) << SAShift; +} + +Register js::jit::toRK(Instruction& i) { + return Register::FromCode((i.encode() & RKMask) >> RKShift); +} + +Register js::jit::toRJ(Instruction& i) { + return Register::FromCode((i.encode() & RJMask) >> RJShift); +} + +Register js::jit::toRD(Instruction& i) { + return Register::FromCode((i.encode() & RDMask) >> RDShift); +} + +Register js::jit::toR(Instruction& i) { + return Register::FromCode(i.encode() & RegMask); +} + +void InstImm::extractImm16(BOffImm16* dest) { *dest = BOffImm16(*this); } + +void AssemblerLOONGARCH64::finish() { + MOZ_ASSERT(!isFinished); + isFinished = true; +} + +bool AssemblerLOONGARCH64::appendRawCode(const uint8_t* code, size_t numBytes) { + return m_buffer.appendRawCode(code, numBytes); +} + +bool AssemblerLOONGARCH64::reserve(size_t size) { + // This buffer uses fixed-size chunks so there's no point in reserving + // now vs. on-demand. + return !oom(); +} + +bool AssemblerLOONGARCH64::swapBuffer(wasm::Bytes& bytes) { + // For now, specialize to the one use case. As long as wasm::Bytes is a + // Vector, not a linked-list of chunks, there's not much we can do other + // than copy. + MOZ_ASSERT(bytes.empty()); + if (!bytes.resize(bytesNeeded())) { + return false; + } + m_buffer.executableCopy(bytes.begin()); + return true; +} + +void AssemblerLOONGARCH64::copyJumpRelocationTable(uint8_t* dest) { + if (jumpRelocations_.length()) { + memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length()); + } +} + +void AssemblerLOONGARCH64::copyDataRelocationTable(uint8_t* dest) { + if (dataRelocations_.length()) { + memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length()); + } +} + +void AssemblerLOONGARCH64::copyPreBarrierTable(uint8_t* dest) { + if (preBarriers_.length()) { + memcpy(dest, preBarriers_.buffer(), preBarriers_.length()); + } +} + +AssemblerLOONGARCH64::Condition AssemblerLOONGARCH64::InvertCondition(Condition cond) { + switch (cond) { + case Equal: + return NotEqual; + case NotEqual: + return Equal; + case Zero: + return NonZero; + case NonZero: + return Zero; + case LessThan: + return GreaterThanOrEqual; + case LessThanOrEqual: + return GreaterThan; + case GreaterThan: + return LessThanOrEqual; + case GreaterThanOrEqual: + return LessThan; + case Above: + return BelowOrEqual; + case AboveOrEqual: + return Below; + case Below: + return AboveOrEqual; + case BelowOrEqual: + return Above; + case Signed: + return NotSigned; + case NotSigned: + return Signed; + default: + MOZ_CRASH("unexpected condition"); + } +} + +AssemblerLOONGARCH64::DoubleCondition AssemblerLOONGARCH64::InvertCondition( + DoubleCondition cond) { + switch (cond) { + case DoubleOrdered: + return DoubleUnordered; + case DoubleEqual: + return DoubleNotEqualOrUnordered; + case DoubleNotEqual: + return DoubleEqualOrUnordered; + case DoubleGreaterThan: + return DoubleLessThanOrEqualOrUnordered; + case DoubleGreaterThanOrEqual: + return DoubleLessThanOrUnordered; + case DoubleLessThan: + return DoubleGreaterThanOrEqualOrUnordered; + case DoubleLessThanOrEqual: + return DoubleGreaterThanOrUnordered; + case DoubleUnordered: + return DoubleOrdered; + case DoubleEqualOrUnordered: + return DoubleNotEqual; + case DoubleNotEqualOrUnordered: + return DoubleEqual; + case DoubleGreaterThanOrUnordered: + return DoubleLessThanOrEqual; + case DoubleGreaterThanOrEqualOrUnordered: + return DoubleLessThan; + case DoubleLessThanOrUnordered: + return DoubleGreaterThanOrEqual; + case DoubleLessThanOrEqualOrUnordered: + return DoubleGreaterThan; + default: + MOZ_CRASH("unexpected condition"); + } +} + +AssemblerLOONGARCH64::Condition AssemblerLOONGARCH64::InvertCmpCondition( + Condition cond) { + switch (cond) { + case Equal: + case NotEqual: + return cond; + case LessThan: + return GreaterThan; + case LessThanOrEqual: + return GreaterThanOrEqual; + case GreaterThan: + return LessThanOrEqual; + case GreaterThanOrEqual: + return LessThan; + case Above: + return Below; + case AboveOrEqual: + return BelowOrEqual; + case Below: + return Above; + case BelowOrEqual: + return AboveOrEqual; + default: + MOZ_CRASH("no meaningful swapped-operand condition"); + } +} + +BOffImm16::BOffImm16(InstImm inst) + : data((inst.encode() >> Imm16Shift) & Imm16Mask) {} + +Instruction* BOffImm16::getDest(Instruction* src) const { + return &src[(((int32_t)data << 16) >> 16) + 1]; +} + +bool AssemblerLOONGARCH64::oom() const { + return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() || + dataRelocations_.oom() || preBarriers_.oom(); +} + +// Size of the instruction stream, in bytes. +size_t AssemblerLOONGARCH64::size() const { return m_buffer.size(); } + +// Size of the relocation table, in bytes. +size_t AssemblerLOONGARCH64::jumpRelocationTableBytes() const { + return jumpRelocations_.length(); +} + +size_t AssemblerLOONGARCH64::dataRelocationTableBytes() const { + return dataRelocations_.length(); +} + +size_t AssemblerLOONGARCH64::preBarrierTableBytes() const { + return preBarriers_.length(); +} + +// Size of the data table, in bytes. +size_t AssemblerLOONGARCH64::bytesNeeded() const { + return size() + jumpRelocationTableBytes() + dataRelocationTableBytes() + + preBarrierTableBytes(); +} + +// write a blob of binary into the instruction stream +BufferOffset AssemblerLOONGARCH64::writeInst(uint32_t x, uint32_t* dest) { + MOZ_ASSERT(hasCreator()); + if (dest == nullptr) { + return m_buffer.putInt(x); + } + + WriteInstStatic(x, dest); + return BufferOffset(); +} + +void AssemblerLOONGARCH64::WriteInstStatic(uint32_t x, uint32_t* dest) { + MOZ_ASSERT(dest != nullptr); + *dest = x; +} + +BufferOffset AssemblerLOONGARCH64::haltingAlign(int alignment) { + // TODO(loongarch64): Implement a proper halting align. + return nopAlign(alignment); +} + +BufferOffset AssemblerLOONGARCH64::nopAlign(int alignment) { + BufferOffset ret; + MOZ_ASSERT(m_buffer.isAligned(4)); + if (alignment == 8) { + if (!m_buffer.isAligned(alignment)) { + BufferOffset tmp = as_nop(); + if (!ret.assigned()) { + ret = tmp; + } + } + } else { + MOZ_ASSERT((alignment & (alignment - 1)) == 0); + while (size() & (alignment - 1)) { + BufferOffset tmp = as_nop(); + if (!ret.assigned()) { + ret = tmp; + } + } + } + return ret; +} + +// Logical operations. +BufferOffset AssemblerLOONGARCH64::as_and(Register rd, Register rj, Register rk) { + spew("and %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_and, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_or(Register rd, Register rj, Register rk) { + spew("or %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_or, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_xor(Register rd, Register rj, Register rk) { + spew("xor %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_xor, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_nor(Register rd, Register rj, Register rk) { + spew("nor %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_nor, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_andn(Register rd, Register rj, Register rk) { + spew("andn %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_andn, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_orn(Register rd, Register rj, Register rk) { + spew("orn %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_orn, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_andi(Register rd, Register rj, int32_t ui12) { + MOZ_ASSERT(is_uintN(ui12, 12)); + spew("andi %3s,%3s,0x%x", rd.name(), rj.name(), ui12); + return writeInst(InstImm(op_andi, ui12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ori(Register rd, Register rj, int32_t ui12) { + MOZ_ASSERT(is_uintN(ui12, 12)); + spew("ori %3s,%3s,0x%x", rd.name(), rj.name(), ui12); + return writeInst(InstImm(op_ori, ui12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_xori(Register rd, Register rj, int32_t ui12) { + MOZ_ASSERT(is_uintN(ui12, 12)); + spew("xori %3s,%3s,0x%x", rd.name(), rj.name(), ui12); + return writeInst(InstImm(op_xori, ui12, rj, rd, 12).encode()); +} + +// Branch and jump instructions +BufferOffset AssemblerLOONGARCH64::as_b(JOffImm26 off) { + spew("b %d", off.decode()); + return writeInst(InstJump(op_b, off).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bl(JOffImm26 off) { + spew("bl %d", off.decode()); + return writeInst(InstJump(op_bl, off).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_jirl(Register rd, Register rj, + BOffImm16 off) { + spew("jirl %3s, %3s, %d", rd.name(), rj.name(), off.decode()); + return writeInst(InstImm(op_jirl, off, rj, rd).encode()); +} + +InstImm AssemblerLOONGARCH64::getBranchCode(JumpOrCall jumpOrCall) { + // jirl or beq + if (jumpOrCall == BranchIsCall) { + return InstImm(op_jirl, BOffImm16(0), zero, ra); + } + + return InstImm(op_beq, BOffImm16(0), zero, zero); +} + +InstImm AssemblerLOONGARCH64::getBranchCode(Register rj, Register rd, Condition c) { + // beq, bne + MOZ_ASSERT(c == AssemblerLOONGARCH64::Equal || c == AssemblerLOONGARCH64::NotEqual); + return InstImm(c == AssemblerLOONGARCH64::Equal ? op_beq : op_bne, BOffImm16(0), + rj, rd); +} + +InstImm AssemblerLOONGARCH64::getBranchCode(Register rj, Condition c) { + // beq, bne, blt, bge + switch (c) { + case AssemblerLOONGARCH64::Equal: + case AssemblerLOONGARCH64::Zero: + case AssemblerLOONGARCH64::BelowOrEqual: + return InstImm(op_beq, BOffImm16(0), rj, zero); + case AssemblerLOONGARCH64::NotEqual: + case AssemblerLOONGARCH64::NonZero: + case AssemblerLOONGARCH64::Above: + return InstImm(op_bne, BOffImm16(0), rj, zero); + case AssemblerLOONGARCH64::GreaterThan: + return InstImm(op_blt, BOffImm16(0), zero, rj); + case AssemblerLOONGARCH64::GreaterThanOrEqual: + case AssemblerLOONGARCH64::NotSigned: + return InstImm(op_bge, BOffImm16(0), rj, zero); + case AssemblerLOONGARCH64::LessThan: + case AssemblerLOONGARCH64::Signed: + return InstImm(op_blt, BOffImm16(0), rj, zero); + case AssemblerLOONGARCH64::LessThanOrEqual: + return InstImm(op_bge, BOffImm16(0), zero, rj); + default: + MOZ_CRASH("Condition not supported."); + } +} + +// Code semantics must conform to compareFloatingpoint +InstImm AssemblerLOONGARCH64::getBranchCode(FPConditionBit cj) { + return InstImm(op_bcz, 0, cj, true); // bcnez +} + +// Arithmetic instructions +BufferOffset AssemblerLOONGARCH64::as_add_w(Register rd, Register rj, Register rk) { + spew("add_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_add_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_add_d(Register rd, Register rj, Register rk) { + spew("add_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_add_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sub_w(Register rd, Register rj, Register rk) { + spew("sub_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sub_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sub_d(Register rd, Register rj, Register rk) { + spew("sub_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sub_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_addi_w(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("addi_w %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_addi_w, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_addi_d(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("addi_d %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_addi_d, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_addu16i_d(Register rd, Register rj, + int32_t si16) { + MOZ_ASSERT(Imm16::IsInSignedRange(si16)); + spew("addu16i_d %3s,%3s,0x%x", rd.name(), rj.name(), si16); + return writeInst(InstImm(op_addu16i_d, Imm16(si16), rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_alsl_w(Register rd, Register rj, Register rk, + uint32_t sa2) { + MOZ_ASSERT(sa2 < 4); + spew("alsl_w %3s,%3s,0x%x", rd.name(), rj.name(), sa2); + return writeInst(InstReg(op_alsl_w, sa2, rk, rj, rd, 2).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_alsl_wu(Register rd, Register rj, Register rk, + uint32_t sa2) { + MOZ_ASSERT(sa2 < 4); + spew("alsl_wu %3s,%3s,0x%x", rd.name(), rj.name(), sa2); + return writeInst(InstReg(op_alsl_wu, sa2, rk, rj, rd, 2).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_alsl_d(Register rd, Register rj, Register rk, + uint32_t sa2) { + MOZ_ASSERT(sa2 < 4); + spew("alsl_d %3s,%3s,%3s,0x%x", rd.name(), rj.name(), rk.name(), sa2); + return writeInst(InstReg(op_alsl_d, sa2, rk, rj, rd, 2).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_lu12i_w(Register rd, int32_t si20) { + spew("lu12i_w %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_lu12i_w, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_lu32i_d(Register rd, int32_t si20) { + spew("lu32i_d %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_lu32i_d, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_lu52i_d(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_uintN(si12, 12)); + spew("lu52i_d %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_lu52i_d, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_slt(Register rd, Register rj, Register rk) { + spew("slt %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_slt, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sltu(Register rd, Register rj, Register rk) { + spew("sltu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sltu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_slti(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("slti %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_slti, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sltui(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("sltui %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_sltui, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_pcaddi(Register rd, int32_t si20) { + spew("pcaddi %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_pcaddi, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_pcaddu12i(Register rd, int32_t si20) { + spew("pcaddu12i %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_pcaddu12i, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_pcaddu18i(Register rd, int32_t si20) { + spew("pcaddu18i %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_pcaddu18i, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_pcalau12i(Register rd, int32_t si20) { + spew("pcalau12i %3s,0x%x", rd.name(), si20); + return writeInst(InstImm(op_pcalau12i, si20, rd, false).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mul_w(Register rd, Register rj, Register rk) { + spew("mul_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mul_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mulh_w(Register rd, Register rj, + Register rk) { + spew("mulh_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulh_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mulh_wu(Register rd, Register rj, + Register rk) { + spew("mulh_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulh_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mul_d(Register rd, Register rj, Register rk) { + spew("mul_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mul_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mulh_d(Register rd, Register rj, + Register rk) { + spew("mulh_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulh_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mulh_du(Register rd, Register rj, + Register rk) { + spew("mulh_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulh_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mulw_d_w(Register rd, Register rj, + Register rk) { + spew("mulw_d_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulw_d_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mulw_d_wu(Register rd, Register rj, + Register rk) { + spew("mulw_d_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mulw_d_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_div_w(Register rd, Register rj, Register rk) { + spew("div_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_div_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mod_w(Register rd, Register rj, Register rk) { + spew("mod_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mod_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_div_wu(Register rd, Register rj, + Register rk) { + spew("div_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_div_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mod_wu(Register rd, Register rj, + Register rk) { + spew("mod_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mod_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_div_d(Register rd, Register rj, Register rk) { + spew("div_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_div_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mod_d(Register rd, Register rj, Register rk) { + spew("mod_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mod_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_div_du(Register rd, Register rj, + Register rk) { + spew("div_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_div_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_mod_du(Register rd, Register rj, + Register rk) { + spew("mod_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_mod_du, rk, rj, rd).encode()); +} + +// Shift instructions +BufferOffset AssemblerLOONGARCH64::as_sll_w(Register rd, Register rj, Register rk) { + spew("sll_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sll_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_srl_w(Register rd, Register rj, Register rk) { + spew("srl_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_srl_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sra_w(Register rd, Register rj, Register rk) { + spew("sra_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sra_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_rotr_w(Register rd, Register rj, + Register rk) { + spew("rotr_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_rotr_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_slli_w(Register rd, Register rj, + int32_t ui5) { + MOZ_ASSERT(is_uintN(ui5, 5)); + spew("slli_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5); + return writeInst(InstImm(op_slli_w, ui5, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_srli_w(Register rd, Register rj, + int32_t ui5) { + MOZ_ASSERT(is_uintN(ui5, 5)); + spew("srli_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5); + return writeInst(InstImm(op_srli_w, ui5, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_srai_w(Register rd, Register rj, + int32_t ui5) { + MOZ_ASSERT(is_uintN(ui5, 5)); + spew("srai_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5); + return writeInst(InstImm(op_srai_w, ui5, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_rotri_w(Register rd, Register rj, + int32_t ui5) { + MOZ_ASSERT(is_uintN(ui5, 5)); + spew("rotri_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5); + return writeInst(InstImm(op_rotri_w, ui5, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sll_d(Register rd, Register rj, Register rk) { + spew("sll_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sll_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_srl_d(Register rd, Register rj, Register rk) { + spew("srl_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_srl_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sra_d(Register rd, Register rj, Register rk) { + spew("sra_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_sra_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_rotr_d(Register rd, Register rj, + Register rk) { + spew("rotr_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_rotr_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_slli_d(Register rd, Register rj, + int32_t ui6) { + MOZ_ASSERT(is_uintN(ui6, 6)); + spew("slli_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6); + return writeInst(InstImm(op_slli_d, ui6, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_srli_d(Register rd, Register rj, + int32_t ui6) { + MOZ_ASSERT(is_uintN(ui6, 6)); + spew("srli_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6); + return writeInst(InstImm(op_srli_d, ui6, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_srai_d(Register rd, Register rj, + int32_t ui6) { + MOZ_ASSERT(is_uintN(ui6, 6)); + spew("srai_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6); + return writeInst(InstImm(op_srai_d, ui6, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_rotri_d(Register rd, Register rj, + int32_t ui6) { + MOZ_ASSERT(is_uintN(ui6, 6)); + spew("rotri_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6); + return writeInst(InstImm(op_rotri_d, ui6, rj, rd, 6).encode()); +} + +// Bit operation instrucitons +BufferOffset AssemblerLOONGARCH64::as_ext_w_b(Register rd, Register rj) { + spew("ext_w_b %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_ext_w_b, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ext_w_h(Register rd, Register rj) { + spew("ext_w_h %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_ext_w_h, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_clo_w(Register rd, Register rj) { + spew("clo_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_clo_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_clz_w(Register rd, Register rj) { + spew("clz_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_clz_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_cto_w(Register rd, Register rj) { + spew("cto_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_cto_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ctz_w(Register rd, Register rj) { + spew("ctz_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_ctz_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_clo_d(Register rd, Register rj) { + spew("clo_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_clo_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_clz_d(Register rd, Register rj) { + spew("clz_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_clz_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_cto_d(Register rd, Register rj) { + spew("cto_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_cto_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ctz_d(Register rd, Register rj) { + spew("ctz_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_ctz_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bytepick_w(Register rd, Register rj, + Register rk, int32_t sa2) { + MOZ_ASSERT(sa2 < 4); + spew("bytepick_w %3s,%3s,%3s, 0x%x", rd.name(), rj.name(), rk.name(), sa2); + return writeInst(InstReg(op_bytepick_w, sa2, rk, rj, rd, 2).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bytepick_d(Register rd, Register rj, + Register rk, int32_t sa3) { + MOZ_ASSERT(sa3 < 8); + spew("bytepick_d %3s,%3s,%3s, 0x%x", rd.name(), rj.name(), rk.name(), sa3); + return writeInst(InstReg(op_bytepick_d, sa3, rk, rj, rd, 3).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_revb_2h(Register rd, Register rj) { + spew("revb_2h %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revb_2h, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_revb_4h(Register rd, Register rj) { + spew("revb_4h %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revb_4h, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_revb_2w(Register rd, Register rj) { + spew("revb_2w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revb_2w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_revb_d(Register rd, Register rj) { + spew("revb_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revb_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_revh_2w(Register rd, Register rj) { + spew("revh_2w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revh_2w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_revh_d(Register rd, Register rj) { + spew("revh_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_revh_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bitrev_4b(Register rd, Register rj) { + spew("bitrev_4b %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_bitrev_4b, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bitrev_8b(Register rd, Register rj) { + spew("bitrev_8b %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_bitrev_8b, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bitrev_w(Register rd, Register rj) { + spew("bitrev_w %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_bitrev_w, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bitrev_d(Register rd, Register rj) { + spew("bitrev_d %3s,%3s", rd.name(), rj.name()); + return writeInst(InstReg(op_bitrev_d, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bstrins_w(Register rd, Register rj, + int32_t msbw, int32_t lsbw) { + MOZ_ASSERT(lsbw <= msbw); + spew("bstrins_w %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbw, lsbw); + return writeInst(InstImm(op_bstr_w, msbw, lsbw, rj, rd, 5).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bstrins_d(Register rd, Register rj, + int32_t msbd, int32_t lsbd) { + MOZ_ASSERT(lsbd <= msbd); + spew("bstrins_d %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbd, lsbd); + return writeInst(InstImm(op_bstrins_d, msbd, lsbd, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bstrpick_w(Register rd, Register rj, + int32_t msbw, int32_t lsbw) { + MOZ_ASSERT(lsbw <= msbw); + spew("bstrpick_w %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbw, lsbw); + return writeInst(InstImm(op_bstr_w, msbw, lsbw, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_bstrpick_d(Register rd, Register rj, + int32_t msbd, int32_t lsbd) { + MOZ_ASSERT(lsbd <= msbd); + spew("bstrpick_d %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbd, lsbd); + return writeInst(InstImm(op_bstrpick_d, msbd, lsbd, rj, rd, 6).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_maskeqz(Register rd, Register rj, + Register rk) { + spew("maskeqz %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_maskeqz, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_masknez(Register rd, Register rj, + Register rk) { + spew("masknez %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_masknez, rk, rj, rd).encode()); +} + +// Load and store instructions +BufferOffset AssemblerLOONGARCH64::as_ld_b(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_b %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_b, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ld_h(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_h %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_h, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ld_w(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_w %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_w, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ld_d(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_d %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_d, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ld_bu(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_bu %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_bu, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ld_hu(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_hu %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_hu, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ld_wu(Register rd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("ld_wu %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_ld_wu, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_st_b(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("st_b %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_st_b, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_st_h(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("st_h %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_st_h, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_st_w(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("st_w %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_st_w, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_st_d(Register rd, Register rj, int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("st_d %3s,%3s,0x%x", rd.name(), rj.name(), si12); + return writeInst(InstImm(op_st_d, si12, rj, rd, 12).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldx_b(Register rd, Register rj, Register rk) { + spew("ldx_b %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_b, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldx_h(Register rd, Register rj, Register rk) { + spew("ldx_h %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_h, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldx_w(Register rd, Register rj, Register rk) { + spew("ldx_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldx_d(Register rd, Register rj, Register rk) { + spew("ldx_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldx_bu(Register rd, Register rj, + Register rk) { + spew("ldx_bu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_bu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldx_hu(Register rd, Register rj, + Register rk) { + spew("ldx_hu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_hu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldx_wu(Register rd, Register rj, + Register rk) { + spew("ldx_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ldx_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_stx_b(Register rd, Register rj, Register rk) { + spew("stx_b %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_stx_b, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_stx_h(Register rd, Register rj, Register rk) { + spew("stx_h %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_stx_h, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_stx_w(Register rd, Register rj, Register rk) { + spew("stx_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_stx_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_stx_d(Register rd, Register rj, Register rk) { + spew("stx_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_stx_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldptr_w(Register rd, Register rj, + int32_t si14) { + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + spew("ldptr_w %3s,%3s,0x%x", rd.name(), rj.name(), si14); + return writeInst(InstImm(op_ldptr_w, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ldptr_d(Register rd, Register rj, + int32_t si14) { + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + spew("ldptr_d %3s,%3s,0x%x", rd.name(), rj.name(), si14); + return writeInst(InstImm(op_ldptr_d, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_stptr_w(Register rd, Register rj, + int32_t si14) { + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + spew("stptr_w %3s,%3s,0x%x", rd.name(), rj.name(), si14); + return writeInst(InstImm(op_stptr_w, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_stptr_d(Register rd, Register rj, + int32_t si14) { + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + spew("stptr_d %3s,%3s,0x%x", rd.name(), rj.name(), si14); + return writeInst(InstImm(op_stptr_d, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_preld(int32_t hint, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("preld 0x%x,%3s,0x%x", hint, rj.name(), si12); + return writeInst(InstImm(op_preld, si12, rj, hint).encode()); +} + +// Atomic instructions +BufferOffset AssemblerLOONGARCH64::as_amswap_w(Register rd, Register rj, + Register rk) { + spew("amswap_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amswap_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amswap_d(Register rd, Register rj, + Register rk) { + spew("amswap_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amswap_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amadd_w(Register rd, Register rj, + Register rk) { + spew("amadd_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amadd_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amadd_d(Register rd, Register rj, + Register rk) { + spew("amadd_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amadd_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amand_w(Register rd, Register rj, + Register rk) { + spew("amand_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amand_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amand_d(Register rd, Register rj, + Register rk) { + spew("amand_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amand_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amor_w(Register rd, Register rj, + Register rk) { + spew("amor_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amor_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amor_d(Register rd, Register rj, + Register rk) { + spew("amor_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amor_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amxor_w(Register rd, Register rj, + Register rk) { + spew("amxor_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amxor_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amxor_d(Register rd, Register rj, + Register rk) { + spew("amxor_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amxor_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammax_w(Register rd, Register rj, + Register rk) { + spew("ammax_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammax_d(Register rd, Register rj, + Register rk) { + spew("ammax_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammin_w(Register rd, Register rj, + Register rk) { + spew("ammin_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammin_d(Register rd, Register rj, + Register rk) { + spew("ammin_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammax_wu(Register rd, Register rj, + Register rk) { + spew("ammax_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammax_du(Register rd, Register rj, + Register rk) { + spew("ammax_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammin_wu(Register rd, Register rj, + Register rk) { + spew("ammin_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammin_du(Register rd, Register rj, + Register rk) { + spew("ammin_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amswap_db_w(Register rd, Register rj, + Register rk) { + spew("amswap_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amswap_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amswap_db_d(Register rd, Register rj, + Register rk) { + spew("amswap_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amswap_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amadd_db_w(Register rd, Register rj, + Register rk) { + spew("amadd_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amadd_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amadd_db_d(Register rd, Register rj, + Register rk) { + spew("amadd_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amadd_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amand_db_w(Register rd, Register rj, + Register rk) { + spew("amand_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amand_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amand_db_d(Register rd, Register rj, + Register rk) { + spew("amand_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amand_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amor_db_w(Register rd, Register rj, + Register rk) { + spew("amor_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amor_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amor_db_d(Register rd, Register rj, + Register rk) { + spew("amor_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amor_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amxor_db_w(Register rd, Register rj, + Register rk) { + spew("amxor_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amxor_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_amxor_db_d(Register rd, Register rj, + Register rk) { + spew("amxor_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_amxor_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammax_db_w(Register rd, Register rj, + Register rk) { + spew("ammax_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammax_db_d(Register rd, Register rj, + Register rk) { + spew("ammax_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammin_db_w(Register rd, Register rj, + Register rk) { + spew("ammin_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_db_w, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammin_db_d(Register rd, Register rj, + Register rk) { + spew("ammin_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_db_d, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammax_db_wu(Register rd, Register rj, + Register rk) { + spew("ammax_db_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_db_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammax_db_du(Register rd, Register rj, + Register rk) { + spew("ammax_db_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammax_db_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammin_db_wu(Register rd, Register rj, + Register rk) { + spew("ammin_db_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_db_wu, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ammin_db_du(Register rd, Register rj, + Register rk) { + spew("ammin_db_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_ammin_db_du, rk, rj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ll_w(Register rd, Register rj, int32_t si14) { + spew("ll_w %3s,%3s,0x%x", rd.name(), rj.name(), si14); + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + return writeInst(InstImm(op_ll_w, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ll_d(Register rd, Register rj, int32_t si14) { + spew("ll_d %3s,%3s,0x%x", rd.name(), rj.name(), si14); + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + return writeInst(InstImm(op_ll_d, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sc_w(Register rd, Register rj, int32_t si14) { + spew("sc_w %3s,%3s,0x%x", rd.name(), rj.name(), si14); + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + return writeInst(InstImm(op_sc_w, si14 >> 2, rj, rd, 14).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_sc_d(Register rd, Register rj, int32_t si14) { + spew("sc_d %3s,%3s,0x%x", rd.name(), rj.name(), si14); + MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0)); + return writeInst(InstImm(op_sc_d, si14 >> 2, rj, rd, 14).encode()); +} + +// Barrier instructions +BufferOffset AssemblerLOONGARCH64::as_dbar(int32_t hint) { + MOZ_ASSERT(is_uintN(hint, 15)); + spew("dbar 0x%x", hint); + return writeInst(InstImm(op_dbar, hint).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ibar(int32_t hint) { + MOZ_ASSERT(is_uintN(hint, 15)); + spew("ibar 0x%x", hint); + return writeInst(InstImm(op_ibar, hint).encode()); +} + +/* =============================================================== */ + +// FP Arithmetic instructions +BufferOffset AssemblerLOONGARCH64::as_fadd_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fadd_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fadd_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fadd_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fadd_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fadd_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fsub_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fsub_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fsub_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fsub_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fsub_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fsub_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmul_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmul_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmul_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmul_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmul_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmul_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fdiv_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fdiv_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fdiv_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fdiv_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fdiv_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fdiv_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmadd_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fmadd_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fmadd_s, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmadd_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fmadd_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fmadd_d, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmsub_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fmsub_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fmsub_s, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmsub_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fmsub_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fmsub_d, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fnmadd_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fnmadd_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fnmadd_s, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fnmadd_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fnmadd_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fnmadd_d, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fnmsub_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fnmsub_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fnmsub_s, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fnmsub_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FloatRegister fa) { + spew("fnmsub_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(), + fa.name()); + return writeInst(InstReg(op_fnmsub_d, fa, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmax_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmax_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmax_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmax_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmax_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmax_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmin_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmin_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmin_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmin_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmin_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmin_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmaxa_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmaxa_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmaxa_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmaxa_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmaxa_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmaxa_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmina_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmina_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmina_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmina_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk) { + spew("fmina_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fmina_d, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fabs_s(FloatRegister fd, FloatRegister fj) { + spew("fabs_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fabs_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fabs_d(FloatRegister fd, FloatRegister fj) { + spew("fabs_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fabs_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fneg_s(FloatRegister fd, FloatRegister fj) { + spew("fneg_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fneg_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fneg_d(FloatRegister fd, FloatRegister fj) { + spew("fneg_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fneg_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fsqrt_s(FloatRegister fd, FloatRegister fj) { + spew("fsqrt_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fsqrt_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fsqrt_d(FloatRegister fd, FloatRegister fj) { + spew("fsqrt_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fsqrt_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fcopysign_s(FloatRegister fd, + FloatRegister fj, + FloatRegister fk) { + spew("fcopysign_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fcopysign_s, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fcopysign_d(FloatRegister fd, + FloatRegister fj, + FloatRegister fk) { + spew("fcopysign_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name()); + return writeInst(InstReg(op_fcopysign_d, fk, fj, fd).encode()); +} + +// FP compare instructions +// fcmp.cond.s and fcmp.cond.d instructions +BufferOffset AssemblerLOONGARCH64::as_fcmp_cor(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cor_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, COR, fk, fj, cd).encode()); + } else { + spew("fcmp_cor_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, COR, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_ceq(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_ceq_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CEQ, fk, fj, cd).encode()); + } else { + spew("fcmp_ceq_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CEQ, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_cne(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cne_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CNE, fk, fj, cd).encode()); + } else { + spew("fcmp_cne_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CNE, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_cle(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cle_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CLE, fk, fj, cd).encode()); + } else { + spew("fcmp_cle_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CLE, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_clt(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_clt_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CLT, fk, fj, cd).encode()); + } else { + spew("fcmp_clt_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CLT, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_cun(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cun_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CUN, fk, fj, cd).encode()); + } else { + spew("fcmp_cun_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CUN, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_cueq(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cueq_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CUEQ, fk, fj, cd).encode()); + } else { + spew("fcmp_cueq_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CUEQ, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_cune(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cune_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CUNE, fk, fj, cd).encode()); + } else { + spew("fcmp_cune_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CUNE, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_cule(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cule_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CULE, fk, fj, cd).encode()); + } else { + spew("fcmp_cule_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CULE, fk, fj, cd).encode()); + } +} + +BufferOffset AssemblerLOONGARCH64::as_fcmp_cult(FloatFormat fmt, FloatRegister fj, + FloatRegister fk, + FPConditionBit cd) { + if (fmt == DoubleFloat) { + spew("fcmp_cult_d FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_d, CULT, fk, fj, cd).encode()); + } else { + spew("fcmp_cult_s FCC%d,%3s,%3s", cd, fj.name(), fk.name()); + return writeInst(InstReg(op_fcmp_cond_s, CULT, fk, fj, cd).encode()); + } +} + +// FP conversion instructions +BufferOffset AssemblerLOONGARCH64::as_fcvt_s_d(FloatRegister fd, FloatRegister fj) { + spew("fcvt_s_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fcvt_s_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fcvt_d_s(FloatRegister fd, FloatRegister fj) { + spew("fcvt_d_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fcvt_d_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ffint_s_w(FloatRegister fd, + FloatRegister fj) { + spew("ffint_s_w %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ffint_s_w, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ffint_s_l(FloatRegister fd, + FloatRegister fj) { + spew("ffint_s_l %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ffint_s_l, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ffint_d_w(FloatRegister fd, + FloatRegister fj) { + spew("ffint_d_w %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ffint_d_w, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ffint_d_l(FloatRegister fd, + FloatRegister fj) { + spew("ffint_d_l %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ffint_d_l, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftint_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftint_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftint_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftint_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftint_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftint_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftint_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftint_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftint_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftint_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftint_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftint_l_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrm_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrm_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrm_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrm_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrm_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrm_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrm_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrm_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrm_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrm_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrm_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrm_l_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrp_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrp_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrp_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrp_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrp_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrp_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrp_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrp_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrp_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrp_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrp_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrp_l_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrz_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrz_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrz_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrz_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrz_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrz_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrz_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrz_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrz_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrz_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrz_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrz_l_d, fj, fd).encode()); +} +BufferOffset AssemblerLOONGARCH64::as_ftintrne_w_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrne_w_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrne_w_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrne_w_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrne_w_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrne_w_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrne_l_s(FloatRegister fd, + FloatRegister fj) { + spew("ftintrne_l_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrne_l_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_ftintrne_l_d(FloatRegister fd, + FloatRegister fj) { + spew("ftintrne_l_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_ftintrne_l_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_frint_s(FloatRegister fd, FloatRegister fj) { + spew("frint_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_frint_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_frint_d(FloatRegister fd, FloatRegister fj) { + spew("frint_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_frint_d, fj, fd).encode()); +} + +// FP mov instructions +BufferOffset AssemblerLOONGARCH64::as_fmov_s(FloatRegister fd, FloatRegister fj) { + spew("fmov_s %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fmov_s, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fmov_d(FloatRegister fd, FloatRegister fj) { + spew("fmov_d %3s,%3s", fd.name(), fj.name()); + return writeInst(InstReg(op_fmov_d, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fsel(FloatRegister fd, FloatRegister fj, + FloatRegister fk, FPConditionBit ca) { + spew("fsel %3s,%3s,%3s,%d", fd.name(), fj.name(), fk.name(), ca); + return writeInst(InstReg(op_fsel, ca, fk, fj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movgr2fr_w(FloatRegister fd, Register rj) { + spew("movgr2fr_w %3s,%3s", fd.name(), rj.name()); + return writeInst(InstReg(op_movgr2fr_w, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movgr2fr_d(FloatRegister fd, Register rj) { + spew("movgr2fr_d %3s,%3s", fd.name(), rj.name()); + return writeInst(InstReg(op_movgr2fr_d, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movgr2frh_w(FloatRegister fd, Register rj) { + spew("movgr2frh_w %3s,%3s", fd.name(), rj.name()); + return writeInst(InstReg(op_movgr2frh_w, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movfr2gr_s(Register rd, FloatRegister fj) { + spew("movfr2gr_s %3s,%3s", rd.name(), fj.name()); + return writeInst(InstReg(op_movfr2gr_s, fj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movfr2gr_d(Register rd, FloatRegister fj) { + spew("movfr2gr_d %3s,%3s", rd.name(), fj.name()); + return writeInst(InstReg(op_movfr2gr_d, fj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movfrh2gr_s(Register rd, FloatRegister fj) { + spew("movfrh2gr_s %3s,%3s", rd.name(), fj.name()); + return writeInst(InstReg(op_movfrh2gr_s, fj, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movgr2fcsr(Register rj) { + spew("movgr2fcsr %3s", rj.name()); + return writeInst(InstReg(op_movgr2fcsr, rj, FCSR).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movfcsr2gr(Register rd) { + spew("movfcsr2gr %3s", rd.name()); + return writeInst(InstReg(op_movfcsr2gr, FCSR, rd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movfr2cf(FPConditionBit cd, + FloatRegister fj) { + spew("movfr2cf %d,%3s", cd, fj.name()); + return writeInst(InstReg(op_movfr2cf, fj, cd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movcf2fr(FloatRegister fd, + FPConditionBit cj) { + spew("movcf2fr %3s,%d", fd.name(), cj); + return writeInst(InstReg(op_movcf2fr, cj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movgr2cf(FPConditionBit cd, Register rj) { + spew("movgr2cf %d,%3s", cd, rj.name()); + return writeInst(InstReg(op_movgr2cf, rj, cd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_movcf2gr(Register rd, FPConditionBit cj) { + spew("movcf2gr %3s,%d", rd.name(), cj); + return writeInst(InstReg(op_movcf2gr, cj, rd).encode()); +} + +// FP load/store instructions +BufferOffset AssemblerLOONGARCH64::as_fld_s(FloatRegister fd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("fld_s %3s,%3s,0x%x", fd.name(), rj.name(), si12); + return writeInst(InstImm(op_fld_s, si12, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fld_d(FloatRegister fd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("fld_d %3s,%3s,0x%x", fd.name(), rj.name(), si12); + return writeInst(InstImm(op_fld_d, si12, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fst_s(FloatRegister fd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("fst_s %3s,%3s,0x%x", fd.name(), rj.name(), si12); + return writeInst(InstImm(op_fst_s, si12, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fst_d(FloatRegister fd, Register rj, + int32_t si12) { + MOZ_ASSERT(is_intN(si12, 12)); + spew("fst_d %3s,%3s,0x%x", fd.name(), rj.name(), si12); + return writeInst(InstImm(op_fst_d, si12, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fldx_s(FloatRegister fd, Register rj, + Register rk) { + spew("fldx_s %3s,%3s,%3s", fd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_fldx_s, rk, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fldx_d(FloatRegister fd, Register rj, + Register rk) { + spew("fldx_d %3s,%3s,%3s", fd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_fldx_d, rk, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fstx_s(FloatRegister fd, Register rj, + Register rk) { + spew("fstx_s %3s,%3s,%3s", fd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_fstx_s, rk, rj, fd).encode()); +} + +BufferOffset AssemblerLOONGARCH64::as_fstx_d(FloatRegister fd, Register rj, + Register rk) { + spew("fstx_d %3s,%3s,%3s", fd.name(), rj.name(), rk.name()); + return writeInst(InstReg(op_fstx_d, rk, rj, fd).encode()); +} + +/* ========================================================================= */ + +void AssemblerLOONGARCH64::bind(Label* label, BufferOffset boff) { + spew(".set Llabel %p", label); + // If our caller didn't give us an explicit target to bind to + // then we want to bind to the location of the next instruction + BufferOffset dest = boff.assigned() ? boff : nextOffset(); + if (label->used()) { + int32_t next; + + // A used label holds a link to branch that uses it. + BufferOffset b(label); + do { + // Even a 0 offset may be invalid if we're out of memory. + if (oom()) { + return; + } + + Instruction* inst = editSrc(b); + + // Second word holds a pointer to the next branch in label's chain. + next = inst[1].encode(); + bind(reinterpret_cast(inst), b.getOffset(), dest.getOffset()); + + b = BufferOffset(next); + } while (next != LabelBase::INVALID_OFFSET); + } + label->bind(dest.getOffset()); +} + +void AssemblerLOONGARCH64::retarget(Label* label, Label* target) { + spew("retarget %p -> %p", label, target); + if (label->used() && !oom()) { + if (target->bound()) { + bind(label, BufferOffset(target)); + } else if (target->used()) { + // The target is not bound but used. Prepend label's branch list + // onto target's. + int32_t next; + BufferOffset labelBranchOffset(label); + + // Find the head of the use chain for label. + do { + Instruction* inst = editSrc(labelBranchOffset); + + // Second word holds a pointer to the next branch in chain. + next = inst[1].encode(); + labelBranchOffset = BufferOffset(next); + } while (next != LabelBase::INVALID_OFFSET); + + // Then patch the head of label's use chain to the tail of + // target's use chain, prepending the entire use chain of target. + Instruction* inst = editSrc(labelBranchOffset); + int32_t prev = target->offset(); + target->use(label->offset()); + inst[1].setData(prev); + } else { + // The target is unbound and unused. We can just take the head of + // the list hanging off of label, and dump that into target. + target->use(label->offset()); + } + } + label->reset(); +} + +void dbg_break() {} + +void AssemblerLOONGARCH64::as_break(uint32_t code) { + MOZ_ASSERT(code <= MAX_BREAK_CODE); + spew("break %d", code); + writeInst(InstImm(op_break, code).encode()); +} + +// This just stomps over memory with 32 bits of raw data. Its purpose is to +// overwrite the call of JITed code with 32 bits worth of an offset. This will +// is only meant to function on code that has been invalidated, so it should +// be totally safe. Since that instruction will never be executed again, a +// ICache flush should not be necessary +void AssemblerLOONGARCH64::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) { + // Raw is going to be the return address. + uint32_t* raw = (uint32_t*)label.raw(); + // Overwrite the 4 bytes before the return address, which will + // end up being the call instruction. + *(raw - 1) = imm.value; +} + +uint8_t* AssemblerLOONGARCH64::NextInstruction(uint8_t* inst_, uint32_t* count) { + Instruction* inst = reinterpret_cast(inst_); + if (count != nullptr) { + *count += sizeof(Instruction); + } + return reinterpret_cast(inst->next()); +} + +void AssemblerLOONGARCH64::ToggleToJmp(CodeLocationLabel inst_) { + InstImm* inst = (InstImm*)inst_.raw(); + + MOZ_ASSERT(inst->extractBitField(31, 26) == (uint32_t)op_addu16i_d >> 26); + // We converted beq to addu16i_d, so now we restore it. + inst->setOpcode(op_beq, 6); +} + +void AssemblerLOONGARCH64::ToggleToCmp(CodeLocationLabel inst_) { + InstImm* inst = (InstImm*)inst_.raw(); + + // toggledJump is allways used for short jumps. + MOZ_ASSERT(inst->extractBitField(31, 26) == (uint32_t)op_beq >> 26); + // Replace "beq $zero, $zero, offset" with "addu16i_d $zero, $zero, offset" + inst->setOpcode(op_addu16i_d, 6); +} + +// Since there are no pools in LoongArch64 implementation, this should be +// simple. +Instruction* Instruction::next() { return this + 1; } + +InstImm AssemblerLOONGARCH64::invertBranch(InstImm branch, BOffImm16 skipOffset) { + uint32_t rj = 0; + OpcodeField opcode = (OpcodeField)((branch.extractBitField(31, 26)) << 26); + switch (opcode) { + case op_beq: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bne, 6); + return branch; + case op_bne: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_beq, 6); + return branch; + case op_bge: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_blt, 6); + return branch; + case op_bgeu: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bltu, 6); + return branch; + case op_blt: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bge, 6); + return branch; + case op_bltu: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bgeu, 6); + return branch; + case op_beqz: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_bnez, 6); + return branch; + case op_bnez: + branch.setBOffImm16(skipOffset); + branch.setOpcode(op_beqz, 6); + return branch; + case op_bcz: + branch.setBOffImm16(skipOffset); + rj = branch.extractRJ(); + if (rj & 0x8) { + branch.setRJ(rj & 0x17); + } else { + branch.setRJ(rj | 0x8); + } + return branch; + default: + MOZ_CRASH("Error creating long branch."); + } +} + +#ifdef JS_JITSPEW +void AssemblerLOONGARCH64::decodeBranchInstAndSpew(InstImm branch) { + OpcodeField opcode = (OpcodeField)((branch.extractBitField(31, 26)) << 26); + uint32_t rd_id; + uint32_t rj_id; + uint32_t cj_id; + uint32_t immi = branch.extractImm16Value(); + switch (opcode) { + case op_beq: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("beq 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_bne: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bne 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_bge: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bge 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_bgeu: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bgeu 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_blt: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("blt 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_bltu: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bltu 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + case op_beqz: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("beqz 0x%x,%3s,0x%x", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), rd_id); + break; + case op_bnez: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("bnez 0x%x,%3s,0x%x", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), rd_id); + break; + case op_bcz: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + cj_id = branch.extractBitField(CJShift + CJBits - 1, CJShift); + if (rj_id & 0x8) { + spew("bcnez 0x%x,FCC%d,0x%x", (int32_t(immi << 18) >> 16) + 4, cj_id, + rd_id); + } else { + spew("bceqz 0x%x,FCC%d,0x%x", (int32_t(immi << 18) >> 16) + 4, cj_id, + rd_id); + } + break; + case op_jirl: + rd_id = branch.extractRD(); + rj_id = branch.extractRJ(); + spew("beqz 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4, + Registers::GetName(rj_id), Registers::GetName(rd_id)); + break; + default: + MOZ_CRASH("Error disassemble branch."); + } +} +#endif + +void Assembler::executableCopy(uint8_t* buffer) { + MOZ_ASSERT(isFinished); + m_buffer.executableCopy(buffer); +} + +uintptr_t Assembler::GetPointer(uint8_t* instPtr) { + Instruction* inst = (Instruction*)instPtr; + return Assembler::ExtractLoad64Value(inst); +} + +static JitCode* CodeFromJump(Instruction* jump) { + uint8_t* target = (uint8_t*)Assembler::ExtractLoad64Value(jump); + return JitCode::FromExecutable(target); +} + +void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + while (reader.more()) { + JitCode* child = + CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned())); + TraceManuallyBarrieredEdge(trc, &child, "rel32"); + } +} + +static void TraceOneDataRelocation(JSTracer* trc, + mozilla::Maybe& awjc, + JitCode* code, Instruction* inst) { + void* ptr = (void*)Assembler::ExtractLoad64Value(inst); + void* prior = ptr; + + // Data relocations can be for Values or for raw pointers. If a Value is + // zero-tagged, we can trace it as if it were a raw pointer. If a Value + // is not zero-tagged, we have to interpret it as a Value to ensure that the + // tag bits are masked off to recover the actual pointer. + uintptr_t word = reinterpret_cast(ptr); + if (word >> JSVAL_TAG_SHIFT) { + // This relocation is a Value with a non-zero tag. + Value v = Value::fromRawBits(word); + TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value"); + ptr = (void*)v.bitsAsPunboxPointer(); + } else { + // This relocation is a raw pointer or a Value with a zero tag. + // No barrier needed since these are constants. + TraceManuallyBarrieredGenericPointerEdge( + trc, reinterpret_cast(&ptr), "jit-masm-ptr"); + } + + if (ptr != prior) { + if (awjc.isNothing()) { + awjc.emplace(code); + } + Assembler::UpdateLoad64Value(inst, uint64_t(ptr)); + } +} + +/* static */ +void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + mozilla::Maybe awjc; + while (reader.more()) { + size_t offset = reader.readUnsigned(); + Instruction* inst = (Instruction*)(code->raw() + offset); + TraceOneDataRelocation(trc, awjc, code, inst); + } +} + +void Assembler::bind(RepatchLabel* label) { + BufferOffset dest = nextOffset(); + if (label->used() && !oom()) { + BufferOffset branch(label->offset()); + InstImm* inst = reinterpret_cast(editSrc(branch)); + bind(inst, branch.getOffset(), dest.getOffset()); + } + label->bind(dest.getOffset()); +} + +void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) { + if (label.patchAt()->bound()) { + intptr_t offset = label.patchAt()->offset(); + intptr_t target = label.target()->offset(); + Instruction* inst = (Instruction*)(rawCode + offset); + Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + target)); + } +} + +void Assembler::bind(InstImm* inst, uintptr_t branch, uintptr_t target) { + int64_t offset = target - branch; + InstImm inst_jirl = InstImm(op_jirl, BOffImm16(0), zero, ra); + InstImm inst_beq = InstImm(op_beq, BOffImm16(0), zero, zero); + + // If encoded offset is 4, then the jump must be short + if (BOffImm16(inst[0]).decode() == 4) { + MOZ_ASSERT(BOffImm16::IsInRange(offset)); + inst[0].setBOffImm16(BOffImm16(offset)); + inst[1].makeNop(); // because before set INVALID_OFFSET + return; + } + + // Generate the long jump for calls because return address has to be the + // address after the reserved block. + if (inst[0].encode() == inst_jirl.encode()) { + addLongJump(BufferOffset(branch), BufferOffset(target)); + Assembler::WriteLoad64Instructions(inst, ScratchRegister, + LabelBase::INVALID_OFFSET); + inst[3].makeNop(); // There are 1 nop. + inst[4] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); + return; + } + + if (BOffImm16::IsInRange(offset)) { + // Skip trailing nops . + bool skipNops = (inst[0].encode() != inst_jirl.encode() && + inst[0].encode() != inst_beq.encode()); + + inst[0].setBOffImm16(BOffImm16(offset)); + inst[1].makeNop(); + + if (skipNops) { + inst[2] = InstImm(op_bge, BOffImm16(3 * sizeof(uint32_t)), zero, zero); + // There are 2 nops after this + } + return; + } + + if (inst[0].encode() == inst_beq.encode()) { + // Handle long unconditional jump. Only four 4 instruction. + addLongJump(BufferOffset(branch), BufferOffset(target)); + Assembler::WriteLoad64Instructions(inst, ScratchRegister, + LabelBase::INVALID_OFFSET); + inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, zero); + } else { + // Handle long conditional jump. + inst[0] = invertBranch(inst[0], BOffImm16(5 * sizeof(uint32_t))); + // No need for a "nop" here because we can clobber scratch. + addLongJump(BufferOffset(branch + sizeof(uint32_t)), BufferOffset(target)); + Assembler::WriteLoad64Instructions(&inst[1], ScratchRegister, + LabelBase::INVALID_OFFSET); + inst[4] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, zero); + } +} + +void Assembler::processCodeLabels(uint8_t* rawCode) { + for (const CodeLabel& label : codeLabels_) { + Bind(rawCode, label); + } +} + +uint32_t Assembler::PatchWrite_NearCallSize() { + // Load an address needs 3 instructions, and a jump. + return (3 + 1) * sizeof(uint32_t); +} + +void Assembler::PatchWrite_NearCall(CodeLocationLabel start, + CodeLocationLabel toCall) { + Instruction* inst = (Instruction*)start.raw(); + uint8_t* dest = toCall.raw(); + + // Overwrite whatever instruction used to be here with a call. + // Always use long jump for two reasons: + // - Jump has to be the same size because of PatchWrite_NearCallSize. + // - Return address has to be at the end of replaced block. + // Short jump wouldn't be more efficient. + Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)dest); + inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); +} + +uint64_t Assembler::ExtractLoad64Value(Instruction* inst0) { + InstImm* i0 = (InstImm*)inst0; + InstImm* i1 = (InstImm*)i0->next(); + InstImm* i2 = (InstImm*)i1->next(); + InstImm* i3 = (InstImm*)i2->next(); + + MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25)); + MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22)); + MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25)); + + if ((i3->extractBitField(31, 22)) == ((uint32_t)op_lu52i_d >> 22)) { + // Li64 + uint64_t value = + (uint64_t(i0->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift)) + << 12) | + (uint64_t( + i1->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift))) | + (uint64_t(i2->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift)) + << 32) | + (uint64_t(i3->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift)) + << 52); + return value; + } else { + // Li48 + uint64_t value = + (uint64_t(i0->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift)) + << 12) | + (uint64_t( + i1->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift))) | + (uint64_t(i2->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift)) + << 32); + + return uint64_t((int64_t(value) << 16) >> 16); + } +} + +void Assembler::UpdateLoad64Value(Instruction* inst0, uint64_t value) { + // Todo: with ma_liPatchable + InstImm* i0 = (InstImm*)inst0; + InstImm* i1 = (InstImm*)i0->next(); + InstImm* i2 = (InstImm*)i1->next(); + InstImm* i3 = (InstImm*)i2->next(); + + MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25)); + MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22)); + MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25)); + + if ((i3->extractBitField(31, 22)) == ((uint32_t)op_lu52i_d >> 22)) { + // Li64 + *i0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff), + Register::FromCode(i0->extractRD()), false); + *i1 = InstImm(op_ori, (int32_t)(value & 0xfff), + Register::FromCode(i1->extractRJ()), + Register::FromCode(i1->extractRD()), 12); + *i2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff), + Register::FromCode(i2->extractRD()), false); + *i3 = InstImm(op_lu52i_d, (int32_t)((value >> 52) & 0xfff), + Register::FromCode(i3->extractRJ()), + Register::FromCode(i3->extractRD()), 12); + } else { + // Li48 + *i0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff), + Register::FromCode(i0->extractRD()), false); + *i1 = InstImm(op_ori, (int32_t)(value & 0xfff), + Register::FromCode(i1->extractRJ()), + Register::FromCode(i1->extractRD()), 12); + *i2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff), + Register::FromCode(i2->extractRD()), false); + } +} + +void Assembler::WriteLoad64Instructions(Instruction* inst0, Register reg, + uint64_t value) { + Instruction* inst1 = inst0->next(); + Instruction* inst2 = inst1->next(); + *inst0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff), reg, false); + *inst1 = InstImm(op_ori, (int32_t)(value & 0xfff), reg, reg, 12); + *inst2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff), reg, false); +} + +void Assembler::PatchDataWithValueCheck(CodeLocationLabel label, + ImmPtr newValue, ImmPtr expectedValue) { + PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value), + PatchedImmPtr(expectedValue.value)); +} + +void Assembler::PatchDataWithValueCheck(CodeLocationLabel label, + PatchedImmPtr newValue, + PatchedImmPtr expectedValue) { + Instruction* inst = (Instruction*)label.raw(); + + // Extract old Value + DebugOnly value = Assembler::ExtractLoad64Value(inst); + MOZ_ASSERT(value == uint64_t(expectedValue.value)); + + // Replace with new value + Assembler::UpdateLoad64Value(inst, uint64_t(newValue.value)); +} + +uint64_t Assembler::ExtractInstructionImmediate(uint8_t* code) { + InstImm* inst = (InstImm*)code; + return Assembler::ExtractLoad64Value(inst); +} + +void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) { + Instruction* inst = (Instruction*)inst_.raw(); + InstImm* i0 = (InstImm*)inst; + InstImm* i1 = (InstImm*)i0->next(); + InstImm* i2 = (InstImm*)i1->next(); + Instruction* i3 = (Instruction*)i2->next(); + + MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25)); + MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22)); + MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25)); + + if (enabled) { + MOZ_ASSERT((i3->extractBitField(31, 25)) != ((uint32_t)op_lu12i_w >> 25)); + InstImm jirl = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); + *i3 = jirl; + } else { + InstNOP nop; + *i3 = nop; + } +} diff --git a/js/src/jit/loongarch64/Assembler-loongarch64.h b/js/src/jit/loongarch64/Assembler-loongarch64.h new file mode 100644 index 0000000000..ffb14e742f --- /dev/null +++ b/js/src/jit/loongarch64/Assembler-loongarch64.h @@ -0,0 +1,1888 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loongarch64_Assembler_loongarch64_h +#define jit_loongarch64_Assembler_loongarch64_h + +#include "mozilla/Sprintf.h" +#include + +#include "jit/CompactBuffer.h" +#include "jit/IonCode.h" +#include "jit/JitSpewer.h" +#include "jit/loongarch64/Architecture-loongarch64.h" +#include "jit/shared/Assembler-shared.h" + +namespace js { namespace jit { class Operand; } } + +#include "jit/Disassembler.h" +#include "jit/shared/IonAssemblerBuffer.h" +#include "wasm/WasmTypes.h" + +namespace js { +namespace jit { + +static constexpr Register zero{Registers::zero}; +static constexpr Register ra{Registers::ra}; +static constexpr Register tp{Registers::tp}; +static constexpr Register sp{Registers::sp}; +static constexpr Register a0{Registers::a0}; +static constexpr Register a1{Registers::a1}; +static constexpr Register a2{Registers::a2}; +static constexpr Register a3{Registers::a3}; +static constexpr Register a4{Registers::a4}; +static constexpr Register a5{Registers::a5}; +static constexpr Register a6{Registers::a6}; +static constexpr Register a7{Registers::a7}; +static constexpr Register t0{Registers::t0}; +static constexpr Register t1{Registers::t1}; +static constexpr Register t2{Registers::t2}; +static constexpr Register t3{Registers::t3}; +static constexpr Register t4{Registers::t4}; +static constexpr Register t5{Registers::t5}; +static constexpr Register t6{Registers::t6}; +static constexpr Register t7{Registers::t7}; +static constexpr Register t8{Registers::t8}; +static constexpr Register rx{Registers::rx}; +static constexpr Register fp{Registers::fp}; +static constexpr Register s0{Registers::s0}; +static constexpr Register s1{Registers::s1}; +static constexpr Register s2{Registers::s2}; +static constexpr Register s3{Registers::s3}; +static constexpr Register s4{Registers::s4}; +static constexpr Register s5{Registers::s5}; +static constexpr Register s6{Registers::s6}; +static constexpr Register s7{Registers::s7}; +static constexpr Register s8{Registers::s8}; + +static constexpr FloatRegister f0{FloatRegisters::f0, FloatRegisters::Double}; +static constexpr FloatRegister f1{FloatRegisters::f1, FloatRegisters::Double}; +static constexpr FloatRegister f2{FloatRegisters::f2, FloatRegisters::Double}; +static constexpr FloatRegister f3{FloatRegisters::f3, FloatRegisters::Double}; +static constexpr FloatRegister f4{FloatRegisters::f4, FloatRegisters::Double}; +static constexpr FloatRegister f5{FloatRegisters::f5, FloatRegisters::Double}; +static constexpr FloatRegister f6{FloatRegisters::f6, FloatRegisters::Double}; +static constexpr FloatRegister f7{FloatRegisters::f7, FloatRegisters::Double}; +static constexpr FloatRegister f8{FloatRegisters::f8, FloatRegisters::Double}; +static constexpr FloatRegister f9{FloatRegisters::f9, FloatRegisters::Double}; +static constexpr FloatRegister f10{FloatRegisters::f10, FloatRegisters::Double}; +static constexpr FloatRegister f11{FloatRegisters::f11, FloatRegisters::Double}; +static constexpr FloatRegister f12{FloatRegisters::f12, FloatRegisters::Double}; +static constexpr FloatRegister f13{FloatRegisters::f13, FloatRegisters::Double}; +static constexpr FloatRegister f14{FloatRegisters::f14, FloatRegisters::Double}; +static constexpr FloatRegister f15{FloatRegisters::f15, FloatRegisters::Double}; +static constexpr FloatRegister f16{FloatRegisters::f16, FloatRegisters::Double}; +static constexpr FloatRegister f17{FloatRegisters::f17, FloatRegisters::Double}; +static constexpr FloatRegister f18{FloatRegisters::f18, FloatRegisters::Double}; +static constexpr FloatRegister f19{FloatRegisters::f19, FloatRegisters::Double}; +static constexpr FloatRegister f20{FloatRegisters::f20, FloatRegisters::Double}; +static constexpr FloatRegister f21{FloatRegisters::f21, FloatRegisters::Double}; +static constexpr FloatRegister f22{FloatRegisters::f22, FloatRegisters::Double}; +static constexpr FloatRegister f23{FloatRegisters::f23, FloatRegisters::Double}; +static constexpr FloatRegister f24{FloatRegisters::f24, FloatRegisters::Double}; +static constexpr FloatRegister f25{FloatRegisters::f25, FloatRegisters::Double}; +static constexpr FloatRegister f26{FloatRegisters::f26, FloatRegisters::Double}; +static constexpr FloatRegister f27{FloatRegisters::f27, FloatRegisters::Double}; +static constexpr FloatRegister f28{FloatRegisters::f28, FloatRegisters::Double}; +static constexpr FloatRegister f29{FloatRegisters::f29, FloatRegisters::Double}; +static constexpr FloatRegister f30{FloatRegisters::f30, FloatRegisters::Double}; +static constexpr FloatRegister f31{FloatRegisters::f31, FloatRegisters::Double}; + +static constexpr Register InvalidReg{Registers::Invalid}; +static constexpr FloatRegister InvalidFloatReg; + +static constexpr Register StackPointer = sp; +static constexpr Register FramePointer = fp; +static constexpr Register ReturnReg = a0; +static constexpr Register64 ReturnReg64(ReturnReg); +static constexpr FloatRegister ReturnFloat32Reg{FloatRegisters::f0, + FloatRegisters::Single}; +static constexpr FloatRegister ReturnDoubleReg = f0; +static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg; + +static constexpr Register ScratchRegister = t7; +static constexpr Register SecondScratchReg = t8; + +// Helper classes for ScratchRegister usage. Asserts that only one piece +// of code thinks it has exclusive ownership of each scratch register. +struct ScratchRegisterScope : public AutoRegisterScope { + explicit ScratchRegisterScope(MacroAssembler& masm) + : AutoRegisterScope(masm, ScratchRegister) {} +}; + +struct SecondScratchRegisterScope : public AutoRegisterScope { + explicit SecondScratchRegisterScope(MacroAssembler& masm) + : AutoRegisterScope(masm, SecondScratchReg) {} +}; + +static constexpr FloatRegister ScratchFloat32Reg{FloatRegisters::f23, + FloatRegisters::Single}; +static constexpr FloatRegister ScratchDoubleReg = f23; +static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg; + +struct ScratchFloat32Scope : public AutoFloatRegisterScope { + explicit ScratchFloat32Scope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchFloat32Reg) {} +}; + +struct ScratchDoubleScope : public AutoFloatRegisterScope { + explicit ScratchDoubleScope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchDoubleReg) {} +}; + +// Use arg reg from EnterJIT function as OsrFrameReg. +static constexpr Register OsrFrameReg = a3; +static constexpr Register PreBarrierReg = a1; +static constexpr Register InterpreterPCReg = t0; +static constexpr Register CallTempReg0 = t0; +static constexpr Register CallTempReg1 = t1; +static constexpr Register CallTempReg2 = t2; +static constexpr Register CallTempReg3 = t3; +static constexpr Register CallTempReg4 = t4; +static constexpr Register CallTempReg5 = t5; +static constexpr Register CallTempNonArgRegs[] = {t0, t1, t2, t3}; +static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs); + +static constexpr Register IntArgReg0 = a0; +static constexpr Register IntArgReg1 = a1; +static constexpr Register IntArgReg2 = a2; +static constexpr Register IntArgReg3 = a3; +static constexpr Register IntArgReg4 = a4; +static constexpr Register IntArgReg5 = a5; +static constexpr Register IntArgReg6 = a6; +static constexpr Register IntArgReg7 = a7; +static constexpr Register HeapReg = s7; + +// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand). +static constexpr Register RegExpMatcherRegExpReg = CallTempReg0; +static constexpr Register RegExpMatcherStringReg = CallTempReg1; +static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2; + +// Registerd used in RegExpTester instruction (do not use ReturnReg). +static constexpr Register RegExpTesterRegExpReg = CallTempReg0; +static constexpr Register RegExpTesterStringReg = CallTempReg1; +static constexpr Register RegExpTesterLastIndexReg = CallTempReg2; + +static constexpr Register JSReturnReg_Type = a3; +static constexpr Register JSReturnReg_Data = a2; +static constexpr Register JSReturnReg = a2; +static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg); +static constexpr Register ArgumentsRectifierReg = s3; + +// These registers may be volatile or nonvolatile. +static constexpr Register ABINonArgReg0 = t0; +static constexpr Register ABINonArgReg1 = t1; +static constexpr Register ABINonArgReg2 = t2; +static constexpr Register ABINonArgReg3 = t3; + +// These registers may be volatile or nonvolatile. +// Note: these three registers are all guaranteed to be different +static constexpr Register ABINonArgReturnReg0 = t0; +static constexpr Register ABINonArgReturnReg1 = t1; +static constexpr Register ABINonVolatileReg = s0; + +// This register is guaranteed to be clobberable during the prologue and +// epilogue of an ABI call which must preserve both ABI argument, return +// and non-volatile registers. +static constexpr Register ABINonArgReturnVolatileReg = ra; + +// This register may be volatile or nonvolatile. +// Avoid f23 which is the scratch register. +static constexpr FloatRegister ABINonArgDoubleReg{FloatRegisters::f21, + FloatRegisters::Double}; + +// TLS pointer argument register for WebAssembly functions. This must not alias +// any other register used for passing function arguments or return values. +// Preserved by WebAssembly functions. Must be nonvolatile. +static constexpr Register InstanceReg = s4; +static constexpr Register WasmTlsReg = InstanceReg; + +// Registers used for wasm table calls. These registers must be disjoint +// from the ABI argument registers, InstanceReg and each other. +static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0; +static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1; +static constexpr Register WasmTableCallScratchReg = WasmTableCallScratchReg0; +static constexpr Register WasmTableCallSigReg = ABINonArgReg2; +static constexpr Register WasmTableCallIndexReg = ABINonArgReg3; + +// Register used as a scratch along the return path in the fast js -> wasm stub +// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg. +// It must be a volatile register. +static constexpr Register WasmJitEntryReturnScratch = t1; + +static constexpr uint32_t ABIStackAlignment = 16; +static constexpr uint32_t CodeAlignment = 16; +static constexpr uint32_t JitStackAlignment = 16; + +static constexpr uint32_t JitStackValueAlignment = + JitStackAlignment / sizeof(Value); +static_assert(JitStackAlignment % sizeof(Value) == 0 && + JitStackValueAlignment >= 1, + "Stack alignment should be a non-zero multiple of sizeof(Value)"); + +// TODO(loongarch64): this is just a filler to prevent a build failure. The +// LoongArch SIMD alignment requirements still need to be explored. +static constexpr uint32_t SimdMemoryAlignment = 16; + +static_assert(CodeAlignment % SimdMemoryAlignment == 0, + "Code alignment should be larger than any of the alignments " + "which are used for " + "the constant sections of the code buffer. Thus it should be " + "larger than the " + "alignment for SIMD constants."); + +static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment; +static const uint32_t WasmTrapInstructionLength = 4; + +// See comments in wasm::GenerateFunctionPrologue. The difference between these +// is the size of the largest callable prologue on the platform. +static constexpr uint32_t WasmCheckedCallEntryOffset = 0u; +static constexpr uint32_t WasmCheckedTailEntryOffset = 16u; + +static constexpr Scale ScalePointer = TimesEight; + +// TODO(loongarch64): Add LoongArch instruction types description. + +// LoongArch instruction encoding constants. +static const uint32_t RJShift = 5; +static const uint32_t RJBits = 5; +static const uint32_t RKShift = 10; +static const uint32_t RKBits = 5; +static const uint32_t RDShift = 0; +static const uint32_t RDBits = 5; +static const uint32_t FJShift = 5; +static const uint32_t FJBits = 5; +static const uint32_t FKShift = 10; +static const uint32_t FKBits = 5; +static const uint32_t FDShift = 0; +static const uint32_t FDBits = 5; +static const uint32_t FAShift = 15; +static const uint32_t FABits = 5; +static const uint32_t CJShift = 5; +static const uint32_t CJBits = 3; +static const uint32_t CDShift = 0; +static const uint32_t CDBits = 3; +static const uint32_t CAShift = 15; +static const uint32_t CABits = 3; +static const uint32_t CONDShift = 15; +static const uint32_t CONDBits = 5; + +static const uint32_t SAShift = 15; +static const uint32_t SA2Bits = 2; +static const uint32_t SA3Bits = 3; +static const uint32_t LSBWShift = 10; +static const uint32_t LSBWBits = 5; +static const uint32_t LSBDShift = 10; +static const uint32_t LSBDBits = 6; +static const uint32_t MSBWShift = 16; +static const uint32_t MSBWBits = 5; +static const uint32_t MSBDShift = 16; +static const uint32_t MSBDBits = 6; +static const uint32_t Imm5Shift = 10; +static const uint32_t Imm5Bits = 5; +static const uint32_t Imm6Shift = 10; +static const uint32_t Imm6Bits = 6; +static const uint32_t Imm12Shift = 10; +static const uint32_t Imm12Bits = 12; +static const uint32_t Imm14Shift = 10; +static const uint32_t Imm14Bits = 14; +static const uint32_t Imm15Shift = 0; +static const uint32_t Imm15Bits = 15; +static const uint32_t Imm16Shift = 10; +static const uint32_t Imm16Bits = 16; +static const uint32_t Imm20Shift = 5; +static const uint32_t Imm20Bits = 20; +static const uint32_t Imm21Shift = 0; +static const uint32_t Imm21Bits = 21; +static const uint32_t Imm26Shift = 0; +static const uint32_t Imm26Bits = 26; +static const uint32_t CODEShift = 0; +static const uint32_t CODEBits = 15; + +// LoongArch instruction field bit masks. +static const uint32_t RJMask = (1 << RJBits) - 1; +static const uint32_t RKMask = (1 << RKBits) - 1; +static const uint32_t RDMask = (1 << RDBits) - 1; +static const uint32_t SA2Mask = (1 << SA2Bits) - 1; +static const uint32_t SA3Mask = (1 << SA3Bits) - 1; +static const uint32_t CONDMask = (1 << CONDBits) - 1; +static const uint32_t LSBWMask = (1 << LSBWBits) - 1; +static const uint32_t LSBDMask = (1 << LSBDBits) - 1; +static const uint32_t MSBWMask = (1 << MSBWBits) - 1; +static const uint32_t MSBDMask = (1 << MSBDBits) - 1; +static const uint32_t CODEMask = (1 << CODEBits) - 1; +static const uint32_t Imm5Mask = (1 << Imm5Bits) - 1; +static const uint32_t Imm6Mask = (1 << Imm6Bits) - 1; +static const uint32_t Imm12Mask = (1 << Imm12Bits) - 1; +static const uint32_t Imm14Mask = (1 << Imm14Bits) - 1; +static const uint32_t Imm15Mask = (1 << Imm15Bits) - 1; +static const uint32_t Imm16Mask = (1 << Imm16Bits) - 1; +static const uint32_t Imm20Mask = (1 << Imm20Bits) - 1; +static const uint32_t Imm21Mask = (1 << Imm21Bits) - 1; +static const uint32_t Imm26Mask = (1 << Imm26Bits) - 1; +static const uint32_t BOffImm16Mask = ((1 << Imm16Bits) - 1) << Imm16Shift; +static const uint32_t BOffImm21Mask = ((1 << Imm21Bits) - 1) << Imm21Shift; +static const uint32_t BOffImm26Mask = ((1 << Imm26Bits) - 1) << Imm26Shift; +static const uint32_t RegMask = Registers::Total - 1; + +// TODO(loongarch64) Change to syscall? +static const uint32_t MAX_BREAK_CODE = 1024 - 1; +static const uint32_t WASM_TRAP = 6; // BRK_OVERFLOW + +// TODO(loongarch64) Change to LoongArch instruction type. +class Instruction; +class InstReg; +class InstImm; +class InstJump; + +uint32_t RJ(Register r); +uint32_t RK(Register r); +uint32_t RD(Register r); +uint32_t FJ(FloatRegister r); +uint32_t FK(FloatRegister r); +uint32_t FD(FloatRegister r); +uint32_t FA(FloatRegister r); +uint32_t SA2(uint32_t value); +uint32_t SA2(FloatRegister r); +uint32_t SA3(uint32_t value); +uint32_t SA3(FloatRegister r); + +Register toRK(Instruction& i); +Register toRJ(Instruction& i); +Register toRD(Instruction& i); +Register toR(Instruction& i); + +// LoongArch enums for instruction fields +enum OpcodeField { + op_beqz = 0x10U << 26, + op_bnez = 0x11U << 26, + op_bcz = 0x12U << 26, // bceqz & bcnez + op_jirl = 0x13U << 26, + op_b = 0x14U << 26, + op_bl = 0x15U << 26, + op_beq = 0x16U << 26, + op_bne = 0x17U << 26, + op_blt = 0x18U << 26, + op_bge = 0x19U << 26, + op_bltu = 0x1aU << 26, + op_bgeu = 0x1bU << 26, + + op_addu16i_d = 0x4U << 26, + + op_lu12i_w = 0xaU << 25, + op_lu32i_d = 0xbU << 25, + op_pcaddi = 0xcU << 25, + op_pcalau12i = 0xdU << 25, + op_pcaddu12i = 0xeU << 25, + op_pcaddu18i = 0xfU << 25, + op_ll_w = 0x20U << 24, + op_sc_w = 0x21U << 24, + op_ll_d = 0x22U << 24, + op_sc_d = 0x23U << 24, + op_ldptr_w = 0x24U << 24, + op_stptr_w = 0x25U << 24, + op_ldptr_d = 0x26U << 24, + op_stptr_d = 0x27U << 24, + op_bstrins_d = 0x2U << 22, + op_bstrpick_d = 0x3U << 22, + op_slti = 0x8U << 22, + op_sltui = 0x9U << 22, + op_addi_w = 0xaU << 22, + op_addi_d = 0xbU << 22, + op_lu52i_d = 0xcU << 22, + op_andi = 0xdU << 22, + op_ori = 0xeU << 22, + op_xori = 0xfU << 22, + op_ld_b = 0xa0U << 22, + op_ld_h = 0xa1U << 22, + op_ld_w = 0xa2U << 22, + op_ld_d = 0xa3U << 22, + op_st_b = 0xa4U << 22, + op_st_h = 0xa5U << 22, + op_st_w = 0xa6U << 22, + op_st_d = 0xa7U << 22, + op_ld_bu = 0xa8U << 22, + op_ld_hu = 0xa9U << 22, + op_ld_wu = 0xaaU << 22, + op_preld = 0xabU << 22, + op_fld_s = 0xacU << 22, + op_fst_s = 0xadU << 22, + op_fld_d = 0xaeU << 22, + op_fst_d = 0xafU << 22, + op_bstr_w = 0x3U << 21, // BSTRINS_W & BSTRPICK_W + op_fmadd_s = 0x81U << 20, + op_fmadd_d = 0x82U << 20, + op_fmsub_s = 0x85U << 20, + op_fmsub_d = 0x86U << 20, + op_fnmadd_s = 0x89U << 20, + op_fnmadd_d = 0x8aU << 20, + op_fnmsub_s = 0x8dU << 20, + op_fnmsub_d = 0x8eU << 20, + op_fcmp_cond_s = 0xc1U << 20, + op_fcmp_cond_d = 0xc2U << 20, + + op_bytepick_d = 0x3U << 18, + op_fsel = 0x340U << 18, + + op_bytepick_w = 0x4U << 17, + op_alsl_w = 0x2U << 17, + op_alsl_wu = 0x3U << 17, + op_alsl_d = 0x16U << 17, + + op_slli_d = 0x41U << 16, + op_srli_d = 0x45U << 16, + op_srai_d = 0x49U << 16, + + op_slli_w = 0x81U << 15, + op_srli_w = 0x89U << 15, + op_srai_w = 0x91U << 15, + op_add_w = 0x20U << 15, + op_add_d = 0x21U << 15, + op_sub_w = 0x22U << 15, + op_sub_d = 0x23U << 15, + op_slt = 0x24U << 15, + op_sltu = 0x25U << 15, + op_maskeqz = 0x26U << 15, + op_masknez = 0x27U << 15, + op_nor = 0x28U << 15, + op_and = 0x29U << 15, + op_or = 0x2aU << 15, + op_xor = 0x2bU << 15, + op_orn = 0x2cU << 15, + op_andn = 0x2dU << 15, + op_sll_w = 0x2eU << 15, + op_srl_w = 0x2fU << 15, + op_sra_w = 0x30U << 15, + op_sll_d = 0x31U << 15, + op_srl_d = 0x32U << 15, + op_sra_d = 0x33U << 15, + op_rotr_w = 0x36U << 15, + op_rotr_d = 0x37U << 15, + op_rotri_w = 0x99U << 15, + op_rotri_d = 0x4DU << 16, + op_mul_w = 0x38U << 15, + op_mulh_w = 0x39U << 15, + op_mulh_wu = 0x3aU << 15, + op_mul_d = 0x3bU << 15, + op_mulh_d = 0x3cU << 15, + op_mulh_du = 0x3dU << 15, + op_mulw_d_w = 0x3eU << 15, + op_mulw_d_wu = 0x3fU << 15, + op_div_w = 0x40U << 15, + op_mod_w = 0x41U << 15, + op_div_wu = 0x42U << 15, + op_mod_wu = 0x43U << 15, + op_div_d = 0x44U << 15, + op_mod_d = 0x45U << 15, + op_div_du = 0x46U << 15, + op_mod_du = 0x47U << 15, + op_break = 0x54U << 15, + op_syscall = 0x56U << 15, + op_fadd_s = 0x201U << 15, + op_fadd_d = 0x202U << 15, + op_fsub_s = 0x205U << 15, + op_fsub_d = 0x206U << 15, + op_fmul_s = 0x209U << 15, + op_fmul_d = 0x20aU << 15, + op_fdiv_s = 0x20dU << 15, + op_fdiv_d = 0x20eU << 15, + op_fmax_s = 0x211U << 15, + op_fmax_d = 0x212U << 15, + op_fmin_s = 0x215U << 15, + op_fmin_d = 0x216U << 15, + op_fmaxa_s = 0x219U << 15, + op_fmaxa_d = 0x21aU << 15, + op_fmina_s = 0x21dU << 15, + op_fmina_d = 0x21eU << 15, + op_fcopysign_s = 0x225U << 15, + op_fcopysign_d = 0x226U << 15, + op_ldx_b = 0x7000U << 15, + op_ldx_h = 0x7008U << 15, + op_ldx_w = 0x7010U << 15, + op_ldx_d = 0x7018U << 15, + op_stx_b = 0x7020U << 15, + op_stx_h = 0x7028U << 15, + op_stx_w = 0x7030U << 15, + op_stx_d = 0x7038U << 15, + op_ldx_bu = 0x7040U << 15, + op_ldx_hu = 0x7048U << 15, + op_ldx_wu = 0x7050U << 15, + op_fldx_s = 0x7060U << 15, + op_fldx_d = 0x7068U << 15, + op_fstx_s = 0x7070U << 15, + op_fstx_d = 0x7078U << 15, + op_amswap_w = 0x70c0U << 15, + op_amswap_d = 0x70c1U << 15, + op_amadd_w = 0x70c2U << 15, + op_amadd_d = 0x70c3U << 15, + op_amand_w = 0x70c4U << 15, + op_amand_d = 0x70c5U << 15, + op_amor_w = 0x70c6U << 15, + op_amor_d = 0x70c7U << 15, + op_amxor_w = 0x70c8U << 15, + op_amxor_d = 0x70c9U << 15, + op_ammax_w = 0x70caU << 15, + op_ammax_d = 0x70cbU << 15, + op_ammin_w = 0x70ccU << 15, + op_ammin_d = 0x70cdU << 15, + op_ammax_wu = 0x70ceU << 15, + op_ammax_du = 0x70cfU << 15, + op_ammin_wu = 0x70d0U << 15, + op_ammin_du = 0x70d1U << 15, + op_amswap_db_w = 0x70d2U << 15, + op_amswap_db_d = 0x70d3U << 15, + op_amadd_db_w = 0x70d4U << 15, + op_amadd_db_d = 0x70d5U << 15, + op_amand_db_w = 0x70d6U << 15, + op_amand_db_d = 0x70d7U << 15, + op_amor_db_w = 0x70d8U << 15, + op_amor_db_d = 0x70d9U << 15, + op_amxor_db_w = 0x70daU << 15, + op_amxor_db_d = 0x70dbU << 15, + op_ammax_db_w = 0x70dcU << 15, + op_ammax_db_d = 0x70ddU << 15, + op_ammin_db_w = 0x70deU << 15, + op_ammin_db_d = 0x70dfU << 15, + op_ammax_db_wu = 0x70e0U << 15, + op_ammax_db_du = 0x70e1U << 15, + op_ammin_db_wu = 0x70e2U << 15, + op_ammin_db_du = 0x70e3U << 15, + op_dbar = 0x70e4U << 15, + op_ibar = 0x70e5U << 15, + op_clo_w = 0x4U << 10, + op_clz_w = 0x5U << 10, + op_cto_w = 0x6U << 10, + op_ctz_w = 0x7U << 10, + op_clo_d = 0x8U << 10, + op_clz_d = 0x9U << 10, + op_cto_d = 0xaU << 10, + op_ctz_d = 0xbU << 10, + op_revb_2h = 0xcU << 10, + op_revb_4h = 0xdU << 10, + op_revb_2w = 0xeU << 10, + op_revb_d = 0xfU << 10, + op_revh_2w = 0x10U << 10, + op_revh_d = 0x11U << 10, + op_bitrev_4b = 0x12U << 10, + op_bitrev_8b = 0x13U << 10, + op_bitrev_w = 0x14U << 10, + op_bitrev_d = 0x15U << 10, + op_ext_w_h = 0x16U << 10, + op_ext_w_b = 0x17U << 10, + op_fabs_s = 0x4501U << 10, + op_fabs_d = 0x4502U << 10, + op_fneg_s = 0x4505U << 10, + op_fneg_d = 0x4506U << 10, + op_fsqrt_s = 0x4511U << 10, + op_fsqrt_d = 0x4512U << 10, + op_fmov_s = 0x4525U << 10, + op_fmov_d = 0x4526U << 10, + op_movgr2fr_w = 0x4529U << 10, + op_movgr2fr_d = 0x452aU << 10, + op_movgr2frh_w = 0x452bU << 10, + op_movfr2gr_s = 0x452dU << 10, + op_movfr2gr_d = 0x452eU << 10, + op_movfrh2gr_s = 0x452fU << 10, + op_movgr2fcsr = 0x4530U << 10, + op_movfcsr2gr = 0x4532U << 10, + op_movfr2cf = 0x4534U << 10, + op_movgr2cf = 0x4536U << 10, + op_fcvt_s_d = 0x4646U << 10, + op_fcvt_d_s = 0x4649U << 10, + op_ftintrm_w_s = 0x4681U << 10, + op_ftintrm_w_d = 0x4682U << 10, + op_ftintrm_l_s = 0x4689U << 10, + op_ftintrm_l_d = 0x468aU << 10, + op_ftintrp_w_s = 0x4691U << 10, + op_ftintrp_w_d = 0x4692U << 10, + op_ftintrp_l_s = 0x4699U << 10, + op_ftintrp_l_d = 0x469aU << 10, + op_ftintrz_w_s = 0x46a1U << 10, + op_ftintrz_w_d = 0x46a2U << 10, + op_ftintrz_l_s = 0x46a9U << 10, + op_ftintrz_l_d = 0x46aaU << 10, + op_ftintrne_w_s = 0x46b1U << 10, + op_ftintrne_w_d = 0x46b2U << 10, + op_ftintrne_l_s = 0x46b9U << 10, + op_ftintrne_l_d = 0x46baU << 10, + op_ftint_w_s = 0x46c1U << 10, + op_ftint_w_d = 0x46c2U << 10, + op_ftint_l_s = 0x46c9U << 10, + op_ftint_l_d = 0x46caU << 10, + op_ffint_s_w = 0x4744U << 10, + op_ffint_s_l = 0x4746U << 10, + op_ffint_d_w = 0x4748U << 10, + op_ffint_d_l = 0x474aU << 10, + op_frint_s = 0x4791U << 10, + op_frint_d = 0x4792U << 10, + op_movcf2fr = 0x114d4U << 8, + op_movcf2gr = 0x114dcU << 8, +}; + +class Operand; + +// A BOffImm16 is a 16 bit immediate that is used for branches. +class BOffImm16 { + uint32_t data; + + public: + uint32_t encode() { + MOZ_ASSERT(!isInvalid()); + return data; + } + int32_t decode() { + MOZ_ASSERT(!isInvalid()); + return (int32_t(data << 18) >> 16); + } + + explicit BOffImm16(int offset) : data((offset) >> 2 & Imm16Mask) { + MOZ_ASSERT((offset & 0x3) == 0); + MOZ_ASSERT(IsInRange(offset)); + } + static bool IsInRange(int offset) { + if ((offset) < int(unsigned(INT16_MIN) << 2)) { + return false; + } + if ((offset) > (INT16_MAX << 2)) { + return false; + } + return true; + } + static const uint32_t INVALID = 0x00020000; + BOffImm16() : data(INVALID) {} + + bool isInvalid() { return data == INVALID; } + Instruction* getDest(Instruction* src) const; + + BOffImm16(InstImm inst); +}; + +// A JOffImm26 is a 26 bit immediate that is used for unconditional jumps. +class JOffImm26 { + uint32_t data; + + public: + uint32_t encode() { + MOZ_ASSERT(!isInvalid()); + return data; + } + int32_t decode() { + MOZ_ASSERT(!isInvalid()); + return (int32_t(data << 8) >> 6); + } + + explicit JOffImm26(int offset) : data((offset) >> 2 & Imm26Mask) { + MOZ_ASSERT((offset & 0x3) == 0); + MOZ_ASSERT(IsInRange(offset)); + } + static bool IsInRange(int offset) { + if ((offset) < -536870912) { + return false; + } + if ((offset) > 536870908) { + return false; + } + return true; + } + static const uint32_t INVALID = 0x20000000; + JOffImm26() : data(INVALID) {} + + bool isInvalid() { return data == INVALID; } + Instruction* getDest(Instruction* src); +}; + +class Imm16 { + uint16_t value; + + public: + Imm16(); + Imm16(uint32_t imm) : value(imm) {} + uint32_t encode() { return value; } + int32_t decodeSigned() { return value; } + uint32_t decodeUnsigned() { return value; } + + static bool IsInSignedRange(int32_t imm) { + return imm >= INT16_MIN && imm <= INT16_MAX; + } + + static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT16_MAX; } +}; + +class Imm8 { + uint8_t value; + + public: + Imm8(); + Imm8(uint32_t imm) : value(imm) {} + uint32_t encode(uint32_t shift) { return value << shift; } + int32_t decodeSigned() { return value; } + uint32_t decodeUnsigned() { return value; } + static bool IsInSignedRange(int32_t imm) { + return imm >= INT8_MIN && imm <= INT8_MAX; + } + static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT8_MAX; } + static Imm8 Lower(Imm16 imm) { return Imm8(imm.decodeSigned() & 0xff); } + static Imm8 Upper(Imm16 imm) { + return Imm8((imm.decodeSigned() >> 8) & 0xff); + } +}; + +class Operand { + public: + enum Tag { REG, FREG, MEM }; + + private: + Tag tag : 3; + uint32_t reg : 5; + int32_t offset; + + public: + Operand(Register reg_) : tag(REG), reg(reg_.code()) {} + + Operand(FloatRegister freg) : tag(FREG), reg(freg.code()) {} + + Operand(Register base, Imm32 off) + : tag(MEM), reg(base.code()), offset(off.value) {} + + Operand(Register base, int32_t off) + : tag(MEM), reg(base.code()), offset(off) {} + + Operand(const Address& addr) + : tag(MEM), reg(addr.base.code()), offset(addr.offset) {} + + Tag getTag() const { return tag; } + + Register toReg() const { + MOZ_ASSERT(tag == REG); + return Register::FromCode(reg); + } + + FloatRegister toFReg() const { + MOZ_ASSERT(tag == FREG); + return FloatRegister::FromCode(reg); + } + + void toAddr(Register* r, Imm32* dest) const { + MOZ_ASSERT(tag == MEM); + *r = Register::FromCode(reg); + *dest = Imm32(offset); + } + Address toAddress() const { + MOZ_ASSERT(tag == MEM); + return Address(Register::FromCode(reg), offset); + } + int32_t disp() const { + MOZ_ASSERT(tag == MEM); + return offset; + } + + int32_t base() const { + MOZ_ASSERT(tag == MEM); + return reg; + } + Register baseReg() const { + MOZ_ASSERT(tag == MEM); + return Register::FromCode(reg); + } +}; + +// int check. +inline bool is_intN(int32_t x, unsigned n) { + MOZ_ASSERT((0 < n) && (n < 64)); + int32_t limit = static_cast(1) << (n - 1); + return (-limit <= x) && (x < limit); +} + +inline bool is_uintN(int32_t x, unsigned n) { + MOZ_ASSERT((0 < n) && (n < (sizeof(x) * 8))); + return !(x >> n); +} + +inline Imm32 Imm64::firstHalf() const { return low(); } + +inline Imm32 Imm64::secondHalf() const { return hi(); } + +static constexpr int32_t SliceSize = 1024; +typedef js::jit::AssemblerBuffer LOONGBuffer; + +class LOONGBufferWithExecutableCopy : public LOONGBuffer { + public: + void executableCopy(uint8_t* buffer) { + if (this->oom()) { + return; + } + + for (Slice* cur = head; cur != nullptr; cur = cur->getNext()) { + memcpy(buffer, &cur->instructions, cur->length()); + buffer += cur->length(); + } + } + + bool appendRawCode(const uint8_t* code, size_t numBytes) { + if (this->oom()) { + return false; + } + while (numBytes > SliceSize) { + this->putBytes(SliceSize, code); + numBytes -= SliceSize; + code += SliceSize; + } + this->putBytes(numBytes, code); + return !this->oom(); + } +}; + +class AssemblerLOONGARCH64 : public AssemblerShared { + public: + // TODO(loongarch64): Should we remove these conditions here? + enum Condition { + Equal, + NotEqual, + Above, + AboveOrEqual, + Below, + BelowOrEqual, + GreaterThan, + GreaterThanOrEqual, + GreaterThanOrEqual_Signed, + GreaterThanOrEqual_NotSigned, + LessThan, + LessThan_Signed, + LessThan_NotSigned, + LessThanOrEqual, + Overflow, + CarrySet, + CarryClear, + Signed, + NotSigned, + Zero, + NonZero, + Always, + }; + + enum DoubleCondition { + DoubleOrdered, + DoubleEqual, + DoubleNotEqual, + DoubleGreaterThan, + DoubleGreaterThanOrEqual, + DoubleLessThan, + DoubleLessThanOrEqual, + DoubleUnordered, + DoubleEqualOrUnordered, + DoubleNotEqualOrUnordered, + DoubleGreaterThanOrUnordered, + DoubleGreaterThanOrEqualOrUnordered, + DoubleLessThanOrUnordered, + DoubleLessThanOrEqualOrUnordered + }; + + enum FPUCondition { + kNoFPUCondition = -1, + + CAF = 0x00, + SAF = 0x01, + CLT = 0x02, + SLT = 0x03, + CEQ = 0x04, + SEQ = 0x05, + CLE = 0x06, + SLE = 0x07, + CUN = 0x08, + SUN = 0x09, + CULT = 0x0a, + SULT = 0x0b, + CUEQ = 0x0c, + SUEQ = 0x0d, + CULE = 0x0e, + SULE = 0x0f, + CNE = 0x10, + SNE = 0x11, + COR = 0x14, + SOR = 0x15, + CUNE = 0x18, + SUNE = 0x19, + }; + + enum FPConditionBit { FCC0 = 0, FCC1, FFC2, FCC3, FCC4, FCC5, FCC6, FCC7 }; + + enum FPControl { FCSR = 0 }; + + enum FCSRBit { CauseI = 24, CauseU, CauseO, CauseZ, CauseV }; + + enum FloatFormat { SingleFloat, DoubleFloat }; + + enum JumpOrCall { BranchIsJump, BranchIsCall }; + + enum FloatTestKind { TestForTrue, TestForFalse }; + + // :( this should be protected, but since CodeGenerator + // wants to use it, It needs to go out here :( + + BufferOffset nextOffset() { return m_buffer.nextOffset(); } + + protected: + Instruction* editSrc(BufferOffset bo) { return m_buffer.getInst(bo); } + + // structure for fixing up pc-relative loads/jumps when a the machine code + // gets moved (executable copy, gc, etc.) + struct RelativePatch { + // the offset within the code buffer where the value is loaded that + // we want to fix-up + BufferOffset offset; + void* target; + Relocation::Kind kind; + + RelativePatch(BufferOffset offset, void* target, Relocation::Kind kind) + : offset(offset), target(target), kind(kind) {} + }; + + js::Vector jumps_; + + CompactBufferWriter jumpRelocations_; + CompactBufferWriter dataRelocations_; + CompactBufferWriter preBarriers_; + + LOONGBufferWithExecutableCopy m_buffer; + +#ifdef JS_JITSPEW + Sprinter* printer; +#endif + + public: + AssemblerLOONGARCH64() + : m_buffer(), +#ifdef JS_JITSPEW + printer(nullptr), +#endif + isFinished(false) { + } + + static Condition InvertCondition(Condition cond); + static DoubleCondition InvertCondition(DoubleCondition cond); + // This is changing the condition codes for cmp a, b to the same codes for cmp + // b, a. + static Condition InvertCmpCondition(Condition cond); + + // As opposed to x86/x64 version, the data relocation has to be executed + // before to recover the pointer, and not after. + void writeDataRelocation(ImmGCPtr ptr) { + // Raw GC pointer relocations and Value relocations both end up in + // TraceOneDataRelocation. + if (ptr.value) { + if (gc::IsInsideNursery(ptr.value)) { + embedsNurseryPointers_ = true; + } + dataRelocations_.writeUnsigned(nextOffset().getOffset()); + } + } + + void writePrebarrierOffset(CodeOffset label) { + preBarriers_.writeUnsigned(label.offset()); + } + + void assertNoGCThings() const { +#ifdef DEBUG + MOZ_ASSERT(dataRelocations_.length() == 0); + for (auto& j : jumps_) { + MOZ_ASSERT(j.kind == Relocation::HARDCODED); + } +#endif + } + + public: + void setUnlimitedBuffer() {} + bool oom() const; + + void setPrinter(Sprinter* sp) { +#ifdef JS_JITSPEW + printer = sp; +#endif + } + +#ifdef JS_JITSPEW + inline void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) { + if (MOZ_UNLIKELY(printer || JitSpewEnabled(JitSpew_Codegen))) { + va_list va; + va_start(va, fmt); + spew(fmt, va); + va_end(va); + } + } + + void decodeBranchInstAndSpew(InstImm branch); +#else + MOZ_ALWAYS_INLINE void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) {} +#endif + +#ifdef JS_JITSPEW + MOZ_COLD void spew(const char* fmt, va_list va) MOZ_FORMAT_PRINTF(2, 0) { + // Buffer to hold the formatted string. Note that this may contain + // '%' characters, so do not pass it directly to printf functions. + char buf[200]; + + int i = VsprintfLiteral(buf, fmt, va); + if (i > -1) { + if (printer) { + printer->printf("%s\n", buf); + } + js::jit::JitSpew(js::jit::JitSpew_Codegen, "%s", buf); + } + } +#endif + + Register getStackPointer() const { return StackPointer; } + + protected: + bool isFinished; + + public: + void finish(); + bool appendRawCode(const uint8_t* code, size_t numBytes); + bool reserve(size_t size); + bool swapBuffer(wasm::Bytes& bytes); + void executableCopy(void* buffer); + void copyJumpRelocationTable(uint8_t* dest); + void copyDataRelocationTable(uint8_t* dest); + void copyPreBarrierTable(uint8_t* dest); + + // Size of the instruction stream, in bytes. + size_t size() const; + // Size of the jump relocation table, in bytes. + size_t jumpRelocationTableBytes() const; + size_t dataRelocationTableBytes() const; + size_t preBarrierTableBytes() const; + + // Size of the data table, in bytes. + size_t bytesNeeded() const; + + // Write a blob of binary into the instruction stream *OR* + // into a destination address. If dest is nullptr (the default), then the + // instruction gets written into the instruction stream. If dest is not null + // it is interpreted as a pointer to the location that we want the + // instruction to be written. + BufferOffset writeInst(uint32_t x, uint32_t* dest = nullptr); + // A static variant for the cases where we don't want to have an assembler + // object at all. Normally, you would use the dummy (nullptr) object. + static void WriteInstStatic(uint32_t x, uint32_t* dest); + + public: + BufferOffset haltingAlign(int alignment); + BufferOffset nopAlign(int alignment); + BufferOffset as_nop() { return as_andi(zero, zero, 0); } + + // Branch and jump instructions + BufferOffset as_b(JOffImm26 off); + BufferOffset as_bl(JOffImm26 off); + BufferOffset as_jirl(Register rd, Register rj, BOffImm16 off); + + InstImm getBranchCode(JumpOrCall jumpOrCall); // b, bl + InstImm getBranchCode(Register rd, Register rj, + Condition c); // beq, bne, bge, bgeu, blt, bltu + InstImm getBranchCode(Register rj, Condition c); // beqz, bnez + InstImm getBranchCode(FPConditionBit cj); // bceqz, bcnez + + // Arithmetic instructions + BufferOffset as_add_w(Register rd, Register rj, Register rk); + BufferOffset as_add_d(Register rd, Register rj, Register rk); + BufferOffset as_sub_w(Register rd, Register rj, Register rk); + BufferOffset as_sub_d(Register rd, Register rj, Register rk); + + BufferOffset as_addi_w(Register rd, Register rj, int32_t si12); + BufferOffset as_addi_d(Register rd, Register rj, int32_t si12); + BufferOffset as_addu16i_d(Register rd, Register rj, int32_t si16); + + BufferOffset as_alsl_w(Register rd, Register rj, Register rk, uint32_t sa2); + BufferOffset as_alsl_wu(Register rd, Register rj, Register rk, uint32_t sa2); + BufferOffset as_alsl_d(Register rd, Register rj, Register rk, uint32_t sa2); + + BufferOffset as_lu12i_w(Register rd, int32_t si20); + BufferOffset as_lu32i_d(Register rd, int32_t si20); + BufferOffset as_lu52i_d(Register rd, Register rj, int32_t si12); + + BufferOffset as_slt(Register rd, Register rj, Register rk); + BufferOffset as_sltu(Register rd, Register rj, Register rk); + BufferOffset as_slti(Register rd, Register rj, int32_t si12); + BufferOffset as_sltui(Register rd, Register rj, int32_t si12); + + BufferOffset as_pcaddi(Register rd, int32_t si20); + BufferOffset as_pcaddu12i(Register rd, int32_t si20); + BufferOffset as_pcaddu18i(Register rd, int32_t si20); + BufferOffset as_pcalau12i(Register rd, int32_t si20); + + BufferOffset as_mul_w(Register rd, Register rj, Register rk); + BufferOffset as_mulh_w(Register rd, Register rj, Register rk); + BufferOffset as_mulh_wu(Register rd, Register rj, Register rk); + BufferOffset as_mul_d(Register rd, Register rj, Register rk); + BufferOffset as_mulh_d(Register rd, Register rj, Register rk); + BufferOffset as_mulh_du(Register rd, Register rj, Register rk); + + BufferOffset as_mulw_d_w(Register rd, Register rj, Register rk); + BufferOffset as_mulw_d_wu(Register rd, Register rj, Register rk); + + BufferOffset as_div_w(Register rd, Register rj, Register rk); + BufferOffset as_mod_w(Register rd, Register rj, Register rk); + BufferOffset as_div_wu(Register rd, Register rj, Register rk); + BufferOffset as_mod_wu(Register rd, Register rj, Register rk); + BufferOffset as_div_d(Register rd, Register rj, Register rk); + BufferOffset as_mod_d(Register rd, Register rj, Register rk); + BufferOffset as_div_du(Register rd, Register rj, Register rk); + BufferOffset as_mod_du(Register rd, Register rj, Register rk); + + // Logical instructions + BufferOffset as_and(Register rd, Register rj, Register rk); + BufferOffset as_or(Register rd, Register rj, Register rk); + BufferOffset as_xor(Register rd, Register rj, Register rk); + BufferOffset as_nor(Register rd, Register rj, Register rk); + BufferOffset as_andn(Register rd, Register rj, Register rk); + BufferOffset as_orn(Register rd, Register rj, Register rk); + + BufferOffset as_andi(Register rd, Register rj, int32_t ui12); + BufferOffset as_ori(Register rd, Register rj, int32_t ui12); + BufferOffset as_xori(Register rd, Register rj, int32_t ui12); + + // Shift instructions + BufferOffset as_sll_w(Register rd, Register rj, Register rk); + BufferOffset as_srl_w(Register rd, Register rj, Register rk); + BufferOffset as_sra_w(Register rd, Register rj, Register rk); + BufferOffset as_rotr_w(Register rd, Register rj, Register rk); + + BufferOffset as_slli_w(Register rd, Register rj, int32_t ui5); + BufferOffset as_srli_w(Register rd, Register rj, int32_t ui5); + BufferOffset as_srai_w(Register rd, Register rj, int32_t ui5); + BufferOffset as_rotri_w(Register rd, Register rj, int32_t ui5); + + BufferOffset as_sll_d(Register rd, Register rj, Register rk); + BufferOffset as_srl_d(Register rd, Register rj, Register rk); + BufferOffset as_sra_d(Register rd, Register rj, Register rk); + BufferOffset as_rotr_d(Register rd, Register rj, Register rk); + + BufferOffset as_slli_d(Register rd, Register rj, int32_t ui6); + BufferOffset as_srli_d(Register rd, Register rj, int32_t ui6); + BufferOffset as_srai_d(Register rd, Register rj, int32_t ui6); + BufferOffset as_rotri_d(Register rd, Register rj, int32_t ui6); + + // Bit operation instrucitons + BufferOffset as_ext_w_b(Register rd, Register rj); + BufferOffset as_ext_w_h(Register rd, Register rj); + + BufferOffset as_clo_w(Register rd, Register rj); + BufferOffset as_clz_w(Register rd, Register rj); + BufferOffset as_cto_w(Register rd, Register rj); + BufferOffset as_ctz_w(Register rd, Register rj); + BufferOffset as_clo_d(Register rd, Register rj); + BufferOffset as_clz_d(Register rd, Register rj); + BufferOffset as_cto_d(Register rd, Register rj); + BufferOffset as_ctz_d(Register rd, Register rj); + + BufferOffset as_bytepick_w(Register rd, Register rj, Register rk, + int32_t sa2); + BufferOffset as_bytepick_d(Register rd, Register rj, Register rk, + int32_t sa3); + + BufferOffset as_revb_2h(Register rd, Register rj); + BufferOffset as_revb_4h(Register rd, Register rj); + BufferOffset as_revb_2w(Register rd, Register rj); + BufferOffset as_revb_d(Register rd, Register rj); + + BufferOffset as_revh_2w(Register rd, Register rj); + BufferOffset as_revh_d(Register rd, Register rj); + + BufferOffset as_bitrev_4b(Register rd, Register rj); + BufferOffset as_bitrev_8b(Register rd, Register rj); + + BufferOffset as_bitrev_w(Register rd, Register rj); + BufferOffset as_bitrev_d(Register rd, Register rj); + + BufferOffset as_bstrins_w(Register rd, Register rj, int32_t msbw, + int32_t lsbw); + BufferOffset as_bstrins_d(Register rd, Register rj, int32_t msbd, + int32_t lsbd); + BufferOffset as_bstrpick_w(Register rd, Register rj, int32_t msbw, + int32_t lsbw); + BufferOffset as_bstrpick_d(Register rd, Register rj, int32_t msbd, + int32_t lsbd); + + BufferOffset as_maskeqz(Register rd, Register rj, Register rk); + BufferOffset as_masknez(Register rd, Register rj, Register rk); + + // Load and store instructions + BufferOffset as_ld_b(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_h(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_w(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_d(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_bu(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_hu(Register rd, Register rj, int32_t si12); + BufferOffset as_ld_wu(Register rd, Register rj, int32_t si12); + BufferOffset as_st_b(Register rd, Register rj, int32_t si12); + BufferOffset as_st_h(Register rd, Register rj, int32_t si12); + BufferOffset as_st_w(Register rd, Register rj, int32_t si12); + BufferOffset as_st_d(Register rd, Register rj, int32_t si12); + + BufferOffset as_ldx_b(Register rd, Register rj, Register rk); + BufferOffset as_ldx_h(Register rd, Register rj, Register rk); + BufferOffset as_ldx_w(Register rd, Register rj, Register rk); + BufferOffset as_ldx_d(Register rd, Register rj, Register rk); + BufferOffset as_ldx_bu(Register rd, Register rj, Register rk); + BufferOffset as_ldx_hu(Register rd, Register rj, Register rk); + BufferOffset as_ldx_wu(Register rd, Register rj, Register rk); + BufferOffset as_stx_b(Register rd, Register rj, Register rk); + BufferOffset as_stx_h(Register rd, Register rj, Register rk); + BufferOffset as_stx_w(Register rd, Register rj, Register rk); + BufferOffset as_stx_d(Register rd, Register rj, Register rk); + + BufferOffset as_ldptr_w(Register rd, Register rj, int32_t si14); + BufferOffset as_ldptr_d(Register rd, Register rj, int32_t si14); + BufferOffset as_stptr_w(Register rd, Register rj, int32_t si14); + BufferOffset as_stptr_d(Register rd, Register rj, int32_t si14); + + BufferOffset as_preld(int32_t hint, Register rj, int32_t si12); + + // Atomic instructions + BufferOffset as_amswap_w(Register rd, Register rj, Register rk); + BufferOffset as_amswap_d(Register rd, Register rj, Register rk); + BufferOffset as_amadd_w(Register rd, Register rj, Register rk); + BufferOffset as_amadd_d(Register rd, Register rj, Register rk); + BufferOffset as_amand_w(Register rd, Register rj, Register rk); + BufferOffset as_amand_d(Register rd, Register rj, Register rk); + BufferOffset as_amor_w(Register rd, Register rj, Register rk); + BufferOffset as_amor_d(Register rd, Register rj, Register rk); + BufferOffset as_amxor_w(Register rd, Register rj, Register rk); + BufferOffset as_amxor_d(Register rd, Register rj, Register rk); + BufferOffset as_ammax_w(Register rd, Register rj, Register rk); + BufferOffset as_ammax_d(Register rd, Register rj, Register rk); + BufferOffset as_ammin_w(Register rd, Register rj, Register rk); + BufferOffset as_ammin_d(Register rd, Register rj, Register rk); + BufferOffset as_ammax_wu(Register rd, Register rj, Register rk); + BufferOffset as_ammax_du(Register rd, Register rj, Register rk); + BufferOffset as_ammin_wu(Register rd, Register rj, Register rk); + BufferOffset as_ammin_du(Register rd, Register rj, Register rk); + + BufferOffset as_amswap_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amswap_db_d(Register rd, Register rj, Register rk); + BufferOffset as_amadd_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amadd_db_d(Register rd, Register rj, Register rk); + BufferOffset as_amand_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amand_db_d(Register rd, Register rj, Register rk); + BufferOffset as_amor_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amor_db_d(Register rd, Register rj, Register rk); + BufferOffset as_amxor_db_w(Register rd, Register rj, Register rk); + BufferOffset as_amxor_db_d(Register rd, Register rj, Register rk); + BufferOffset as_ammax_db_w(Register rd, Register rj, Register rk); + BufferOffset as_ammax_db_d(Register rd, Register rj, Register rk); + BufferOffset as_ammin_db_w(Register rd, Register rj, Register rk); + BufferOffset as_ammin_db_d(Register rd, Register rj, Register rk); + BufferOffset as_ammax_db_wu(Register rd, Register rj, Register rk); + BufferOffset as_ammax_db_du(Register rd, Register rj, Register rk); + BufferOffset as_ammin_db_wu(Register rd, Register rj, Register rk); + BufferOffset as_ammin_db_du(Register rd, Register rj, Register rk); + + BufferOffset as_ll_w(Register rd, Register rj, int32_t si14); + BufferOffset as_ll_d(Register rd, Register rj, int32_t si14); + BufferOffset as_sc_w(Register rd, Register rj, int32_t si14); + BufferOffset as_sc_d(Register rd, Register rj, int32_t si14); + + // Barrier instructions + BufferOffset as_dbar(int32_t hint); + BufferOffset as_ibar(int32_t hint); + + // FP Arithmetic instructions + BufferOffset as_fadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmul_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmul_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fdiv_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fdiv_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + + BufferOffset as_fmadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fmadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fmsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fmsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fnmadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fnmadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fnmsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + BufferOffset as_fnmsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FloatRegister fa); + + BufferOffset as_fmax_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmax_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmin_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmin_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + + BufferOffset as_fmaxa_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmaxa_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmina_s(FloatRegister fd, FloatRegister fj, FloatRegister fk); + BufferOffset as_fmina_d(FloatRegister fd, FloatRegister fj, FloatRegister fk); + + BufferOffset as_fabs_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_fabs_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_fneg_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_fneg_d(FloatRegister fd, FloatRegister fj); + + BufferOffset as_fsqrt_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_fsqrt_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_fcopysign_s(FloatRegister fd, FloatRegister fj, + FloatRegister fk); + BufferOffset as_fcopysign_d(FloatRegister fd, FloatRegister fj, + FloatRegister fk); + + // FP compare instructions (fcmp.cond.s fcmp.cond.d) + BufferOffset as_fcmp_cor(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_ceq(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cne(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cle(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_clt(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cun(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cueq(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cune(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cule(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + BufferOffset as_fcmp_cult(FloatFormat fmt, FloatRegister fj, FloatRegister fk, + FPConditionBit cd); + + // FP conversion instructions + BufferOffset as_fcvt_s_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_fcvt_d_s(FloatRegister fd, FloatRegister fj); + + BufferOffset as_ffint_s_w(FloatRegister fd, FloatRegister fj); + BufferOffset as_ffint_s_l(FloatRegister fd, FloatRegister fj); + BufferOffset as_ffint_d_w(FloatRegister fd, FloatRegister fj); + BufferOffset as_ffint_d_l(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftint_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftint_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftint_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftint_l_d(FloatRegister fd, FloatRegister fj); + + BufferOffset as_ftintrm_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrm_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrm_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrm_l_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrp_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrp_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrp_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrp_l_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrz_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrz_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrz_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrz_l_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrne_w_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrne_w_d(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrne_l_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_ftintrne_l_d(FloatRegister fd, FloatRegister fj); + + BufferOffset as_frint_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_frint_d(FloatRegister fd, FloatRegister fj); + + // FP mov instructions + BufferOffset as_fmov_s(FloatRegister fd, FloatRegister fj); + BufferOffset as_fmov_d(FloatRegister fd, FloatRegister fj); + + BufferOffset as_fsel(FloatRegister fd, FloatRegister fj, FloatRegister fk, + FPConditionBit ca); + + BufferOffset as_movgr2fr_w(FloatRegister fd, Register rj); + BufferOffset as_movgr2fr_d(FloatRegister fd, Register rj); + BufferOffset as_movgr2frh_w(FloatRegister fd, Register rj); + + BufferOffset as_movfr2gr_s(Register rd, FloatRegister fj); + BufferOffset as_movfr2gr_d(Register rd, FloatRegister fj); + BufferOffset as_movfrh2gr_s(Register rd, FloatRegister fj); + + BufferOffset as_movgr2fcsr(Register rj); + BufferOffset as_movfcsr2gr(Register rd); + + BufferOffset as_movfr2cf(FPConditionBit cd, FloatRegister fj); + BufferOffset as_movcf2fr(FloatRegister fd, FPConditionBit cj); + + BufferOffset as_movgr2cf(FPConditionBit cd, Register rj); + BufferOffset as_movcf2gr(Register rd, FPConditionBit cj); + + // FP load/store instructions + BufferOffset as_fld_s(FloatRegister fd, Register rj, int32_t si12); + BufferOffset as_fld_d(FloatRegister fd, Register rj, int32_t si12); + BufferOffset as_fst_s(FloatRegister fd, Register rj, int32_t si12); + BufferOffset as_fst_d(FloatRegister fd, Register rj, int32_t si12); + + BufferOffset as_fldx_s(FloatRegister fd, Register rj, Register rk); + BufferOffset as_fldx_d(FloatRegister fd, Register rj, Register rk); + BufferOffset as_fstx_s(FloatRegister fd, Register rj, Register rk); + BufferOffset as_fstx_d(FloatRegister fd, Register rj, Register rk); + + // label operations + void bind(Label* label, BufferOffset boff = BufferOffset()); + virtual void bind(InstImm* inst, uintptr_t branch, uintptr_t target) = 0; + void bind(CodeLabel* label) { label->target()->bind(currentOffset()); } + uint32_t currentOffset() { return nextOffset().getOffset(); } + void retarget(Label* label, Label* target); + + void call(Label* label); + void call(void* target); + + void as_break(uint32_t code); + + public: + static bool SupportsFloatingPoint() { +#if defined(__loongarch_hard_float) || defined(JS_SIMULATOR_LOONGARCH64) + return true; +#else + return false; +#endif + } + static bool SupportsUnalignedAccesses() { return true; } + static bool SupportsFastUnalignedFPAccesses() { return true; } + + static bool HasRoundInstruction(int mode) { (void)mode; return false; } + + protected: + InstImm invertBranch(InstImm branch, BOffImm16 skipOffset); + void addPendingJump(BufferOffset src, ImmPtr target, Relocation::Kind kind) { + enoughMemory_ &= jumps_.append(RelativePatch(src, target.value, kind)); + if (kind == Relocation::JITCODE) { + jumpRelocations_.writeUnsigned(src.getOffset()); + } + } + + void addLongJump(BufferOffset src, BufferOffset dst) { + CodeLabel cl; + cl.patchAt()->bind(src.getOffset()); + cl.target()->bind(dst.getOffset()); + addCodeLabel(std::move(cl)); + } + + public: + void flushBuffer() {} + + void comment(const char* msg) { spew("; %s", msg); } + + static uint32_t NopSize() { return 4; } + + static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm); + + static uint8_t* NextInstruction(uint8_t* instruction, + uint32_t* count = nullptr); + + static void ToggleToJmp(CodeLocationLabel inst_); + static void ToggleToCmp(CodeLocationLabel inst_); + + void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, + const Disassembler::HeapAccess& heapAccess) { + // Implement this if we implement a disassembler. + } +}; // AssemblerLOONGARCH64 + +// andi r0, r0, 0 +const uint32_t NopInst = 0x03400000; + +// An Instruction is a structure for both encoding and decoding any and all +// LoongArch instructions. +class Instruction { + public: + uint32_t data; + + protected: + // Standard constructor + Instruction(uint32_t data_) : data(data_) {} + // You should never create an instruction directly. You should create a + // more specific instruction which will eventually call one of these + // constructors for you. + + public: + uint32_t encode() const { return data; } + + void makeNop() { data = NopInst; } + + void setData(uint32_t data) { this->data = data; } + + const Instruction& operator=(const Instruction& src) { + data = src.data; + return *this; + } + + // Extract the one particular bit. + uint32_t extractBit(uint32_t bit) { return (encode() >> bit) & 1; } + // Extract a bit field out of the instruction + uint32_t extractBitField(uint32_t hi, uint32_t lo) { + return (encode() >> lo) & ((2 << (hi - lo)) - 1); + } + + // Get the next instruction in the instruction stream. + // This does neat things like ignoreconstant pools and their guards. + Instruction* next(); + + // Sometimes, an api wants a uint32_t (or a pointer to it) rather than + // an instruction. raw() just coerces this into a pointer to a uint32_t + const uint32_t* raw() const { return &data; } + uint32_t size() const { return 4; } +}; // Instruction + +// make sure that it is the right size +static_assert(sizeof(Instruction) == 4, + "Size of Instruction class has to be 4 bytes."); + +class InstNOP : public Instruction { + public: + InstNOP() : Instruction(NopInst) {} +}; + +// Class for register type instructions. +class InstReg : public Instruction { + public: + InstReg(OpcodeField op, Register rj, Register rd) + : Instruction(op | RJ(rj) | RD(rd)) {} + InstReg(OpcodeField op, Register rk, Register rj, Register rd) + : Instruction(op | RK(rk) | RJ(rj) | RD(rd)) {} + InstReg(OpcodeField op, uint32_t sa, Register rk, Register rj, Register rd, + uint32_t sa_bit) + : Instruction(sa_bit == 2 ? op | SA2(sa) | RK(rk) | RJ(rj) | RD(rd) + : op | SA3(sa) | RK(rk) | RJ(rj) | RD(rd)) { + MOZ_ASSERT(sa_bit == 2 || sa_bit == 3); + } + InstReg(OpcodeField op, Register rj, Register rd, bool HasRd) + : Instruction(HasRd ? op | RJ(rj) | RD(rd) : op | RK(rj) | RJ(rd)) {} + + // For floating-point + InstReg(OpcodeField op, Register rj, FloatRegister fd) + : Instruction(op | RJ(rj) | FD(fd)) {} + InstReg(OpcodeField op, FloatRegister fj, FloatRegister fd) + : Instruction(op | FJ(fj) | FD(fd)) {} + InstReg(OpcodeField op, FloatRegister fk, FloatRegister fj, FloatRegister fd) + : Instruction(op | FK(fk) | FJ(fj) | FD(fd)) {} + InstReg(OpcodeField op, Register rk, Register rj, FloatRegister fd) + : Instruction(op | RK(rk) | RJ(rj) | FD(fd)) {} + InstReg(OpcodeField op, FloatRegister fa, FloatRegister fk, FloatRegister fj, + FloatRegister fd) + : Instruction(op | FA(fa) | FK(fk) | FJ(fj) | FD(fd)) {} + InstReg(OpcodeField op, AssemblerLOONGARCH64::FPConditionBit ca, FloatRegister fk, + FloatRegister fj, FloatRegister fd) + : Instruction(op | ca << CAShift | FK(fk) | FJ(fj) | FD(fd)) { + MOZ_ASSERT(op == op_fsel); + } + InstReg(OpcodeField op, FloatRegister fj, Register rd) + : Instruction(op | FJ(fj) | RD(rd)) { + MOZ_ASSERT((op == op_movfr2gr_s) || (op == op_movfr2gr_d) || + (op == op_movfrh2gr_s)); + } + InstReg(OpcodeField op, Register rj, uint32_t fd) + : Instruction(op | RJ(rj) | fd) { + MOZ_ASSERT(op == op_movgr2fcsr); + } + InstReg(OpcodeField op, uint32_t fj, Register rd) + : Instruction(op | (fj << FJShift) | RD(rd)) { + MOZ_ASSERT(op == op_movfcsr2gr); + } + InstReg(OpcodeField op, FloatRegister fj, AssemblerLOONGARCH64::FPConditionBit cd) + : Instruction(op | FJ(fj) | cd) { + MOZ_ASSERT(op == op_movfr2cf); + } + InstReg(OpcodeField op, AssemblerLOONGARCH64::FPConditionBit cj, FloatRegister fd) + : Instruction(op | (cj << CJShift) | FD(fd)) { + MOZ_ASSERT(op == op_movcf2fr); + } + InstReg(OpcodeField op, Register rj, AssemblerLOONGARCH64::FPConditionBit cd) + : Instruction(op | RJ(rj) | cd) { + MOZ_ASSERT(op == op_movgr2cf); + } + InstReg(OpcodeField op, AssemblerLOONGARCH64::FPConditionBit cj, Register rd) + : Instruction(op | (cj << CJShift) | RD(rd)) { + MOZ_ASSERT(op == op_movcf2gr); + } + InstReg(OpcodeField op, int32_t cond, FloatRegister fk, FloatRegister fj, + AssemblerLOONGARCH64::FPConditionBit cd) + : Instruction(op | (cond & CONDMask) << CONDShift | FK(fk) | FJ(fj) | + (cd & RDMask)) { + MOZ_ASSERT(is_uintN(cond, 5)); + } + + uint32_t extractRK() { + return extractBitField(RKShift + RKBits - 1, RKShift); + } + uint32_t extractRJ() { + return extractBitField(RJShift + RJBits - 1, RJShift); + } + uint32_t extractRD() { + return extractBitField(RDShift + RDBits - 1, RDShift); + } + uint32_t extractSA2() { + return extractBitField(SAShift + SA2Bits - 1, SAShift); + } + uint32_t extractSA3() { + return extractBitField(SAShift + SA3Bits - 1, SAShift); + } +}; + +// Class for branch, load and store instructions with immediate offset. +class InstImm : public Instruction { + public: + void extractImm16(BOffImm16* dest); + uint32_t genImm(int32_t value, uint32_t value_bits) { + uint32_t imm = value & Imm5Mask; + if (value_bits == 6) { + imm = value & Imm6Mask; + } else if (value_bits == 12) { + imm = value & Imm12Mask; + } else if (value_bits == 14) { + imm = value & Imm14Mask; + } + + return imm; + } + + InstImm(OpcodeField op, int32_t value, Register rj, Register rd, + uint32_t value_bits) + : Instruction(op | genImm(value, value_bits) << RKShift | RJ(rj) | + RD(rd)) { + MOZ_ASSERT(value_bits == 5 || value_bits == 6 || value_bits == 12 || + value_bits == 14); + } + InstImm(OpcodeField op, BOffImm16 off, Register rj, Register rd) + : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift | RJ(rj) | + RD(rd)) {} + InstImm(OpcodeField op, int32_t si21, Register rj, bool NotHasRd) + : Instruction(NotHasRd ? op | (si21 & Imm16Mask) << RKShift | RJ(rj) | + (si21 & Imm21Mask) >> 16 + : op | (si21 & Imm20Mask) << Imm20Shift | RD(rj)) { + if (NotHasRd) { + MOZ_ASSERT(op == op_beqz || op == op_bnez); + MOZ_ASSERT(is_intN(si21, 21)); + } else { + MOZ_ASSERT(op == op_lu12i_w || op == op_lu32i_d || op == op_pcaddi || + op == op_pcaddu12i || op == op_pcaddu18i || + op == op_pcalau12i); + // si20 + MOZ_ASSERT(is_intN(si21, 20) || is_uintN(si21, 20)); + } + } + InstImm(OpcodeField op, int32_t si21, AssemblerLOONGARCH64::FPConditionBit cj, + bool isNotEqual) + : Instruction(isNotEqual + ? op | (si21 & Imm16Mask) << RKShift | + (cj + 8) << CJShift | (si21 & Imm21Mask) >> 16 + : op | (si21 & Imm16Mask) << RKShift | cj << CJShift | + (si21 & Imm21Mask) >> 16) { + MOZ_ASSERT(is_intN(si21, 21)); + MOZ_ASSERT(op == op_bcz); + MOZ_ASSERT(cj >= 0 && cj <= 7); + } + InstImm(OpcodeField op, Imm16 off, Register rj, Register rd) + : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift | RJ(rj) | + RD(rd)) {} + InstImm(OpcodeField op, int32_t bit15) + : Instruction(op | (bit15 & Imm15Mask)) { + MOZ_ASSERT(is_uintN(bit15, 15)); + } + + InstImm(OpcodeField op, int32_t bit26, bool jump) + : Instruction(op | (bit26 & Imm16Mask) << Imm16Shift | + (bit26 & Imm26Mask) >> 16) { + MOZ_ASSERT(is_intN(bit26, 26)); + } + InstImm(OpcodeField op, int32_t si12, Register rj, int32_t hint) + : Instruction(op | (si12 & Imm12Mask) << Imm12Shift | RJ(rj) | + (hint & RDMask)) { + MOZ_ASSERT(op == op_preld); + } + InstImm(OpcodeField op, int32_t msb, int32_t lsb, Register rj, Register rd, + uint32_t sb_bits) + : Instruction((sb_bits == 5) + ? op | (msb & MSBWMask) << MSBWShift | + (lsb & LSBWMask) << LSBWShift | RJ(rj) | RD(rd) + : op | (msb & MSBDMask) << MSBDShift | + (lsb & LSBDMask) << LSBDShift | RJ(rj) | RD(rd)) { + MOZ_ASSERT(sb_bits == 5 || sb_bits == 6); + MOZ_ASSERT(op == op_bstr_w || op == op_bstrins_d || op == op_bstrpick_d); + } + InstImm(OpcodeField op, int32_t msb, int32_t lsb, Register rj, Register rd) + : Instruction(op | (msb & MSBWMask) << MSBWShift | + ((lsb + 0x20) & LSBDMask) << LSBWShift | RJ(rj) | RD(rd)) { + MOZ_ASSERT(op == op_bstr_w); + } + + // For floating-point loads and stores. + InstImm(OpcodeField op, int32_t si12, Register rj, FloatRegister fd) + : Instruction(op | (si12 & Imm12Mask) << Imm12Shift | RJ(rj) | FD(fd)) { + MOZ_ASSERT(is_intN(si12, 12)); + } + + void setOpcode(OpcodeField op, uint32_t opBits) { + // opBits not greater than 24. + MOZ_ASSERT(opBits < 25); + uint32_t OpcodeShift = 32 - opBits; + uint32_t OpcodeMask = ((1 << opBits) - 1) << OpcodeShift; + data = (data & ~OpcodeMask) | op; + } + uint32_t extractRK() { + return extractBitField(RKShift + RKBits - 1, RKShift); + } + uint32_t extractRJ() { + return extractBitField(RJShift + RJBits - 1, RJShift); + } + void setRJ(uint32_t rj) { data = (data & ~RJMask) | (rj << RJShift); } + uint32_t extractRD() { + return extractBitField(RDShift + RDBits - 1, RDShift); + } + uint32_t extractImm16Value() { + return extractBitField(Imm16Shift + Imm16Bits - 1, Imm16Shift); + } + void setBOffImm16(BOffImm16 off) { + // Reset immediate field and replace it + data = (data & ~BOffImm16Mask) | (off.encode() << Imm16Shift); + } + void setImm21(int32_t off) { + // Reset immediate field and replace it + uint32_t low16 = (off >> 2) & Imm16Mask; + int32_t high5 = (off >> 18) & Imm5Mask; + uint32_t fcc_info = (data >> 5) & 0x1F; + data = (data & ~BOffImm26Mask) | (low16 << Imm16Shift) | high5 | + (fcc_info << 5); + } +}; + +// Class for Jump type instructions. +class InstJump : public Instruction { + public: + InstJump(OpcodeField op, JOffImm26 off) + : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift | + (off.encode() & Imm26Mask) >> 16) { + MOZ_ASSERT(op == op_b || op == op_bl); + } + + void setJOffImm26(JOffImm26 off) { + // Reset immediate field and replace it + data = (data & ~BOffImm26Mask) | + ((off.encode() & Imm16Mask) << Imm16Shift) | + ((off.encode() >> 16) & 0x3ff); + } + uint32_t extractImm26Value() { + return extractBitField(Imm26Shift + Imm26Bits - 1, Imm26Shift); + } +}; + +class ABIArgGenerator { + public: + ABIArgGenerator() + : intRegIndex_(0), floatRegIndex_(0), stackOffset_(0), current_() {} + + ABIArg next(MIRType argType); + ABIArg& current() { return current_; } + uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; } + + protected: + unsigned intRegIndex_; + unsigned floatRegIndex_; + uint32_t stackOffset_; + ABIArg current_; +}; + +class Assembler : public AssemblerLOONGARCH64 { + public: + Assembler() : AssemblerLOONGARCH64() {} + + static uintptr_t GetPointer(uint8_t*); + + using AssemblerLOONGARCH64::bind; + + static void Bind(uint8_t* rawCode, const CodeLabel& label); + void bind(RepatchLabel* label); + + void processCodeLabels(uint8_t* rawCode); + + static void TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader); + static void TraceDataRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader); + + void bind(InstImm* inst, uintptr_t branch, uintptr_t target); + + // Copy the assembly code to the given buffer, and perform any pending + // relocations relying on the target address. + void executableCopy(uint8_t* buffer); + + static uint32_t PatchWrite_NearCallSize(); + + static uint64_t ExtractLoad64Value(Instruction* inst0); + static void UpdateLoad64Value(Instruction* inst0, uint64_t value); + static void WriteLoad64Instructions(Instruction* inst0, Register reg, + uint64_t value); + + static void PatchWrite_NearCall(CodeLocationLabel start, + CodeLocationLabel toCall); + static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue, + ImmPtr expectedValue); + static void PatchDataWithValueCheck(CodeLocationLabel label, + PatchedImmPtr newValue, + PatchedImmPtr expectedValue); + + static uint64_t ExtractInstructionImmediate(uint8_t* code); + + static void ToggleCall(CodeLocationLabel inst_, bool enabled); +}; // Assembler + +static const uint32_t NumIntArgRegs = 8; +static const uint32_t NumFloatArgRegs = 8; + +static inline bool GetIntArgReg(uint32_t usedIntArgs, Register* out) { + if (usedIntArgs < NumIntArgRegs) { + *out = Register::FromCode(a0.code() + usedIntArgs); + return true; + } + return false; +} + +static inline bool GetFloatArgReg(uint32_t usedFloatArgs, FloatRegister* out) { + if (usedFloatArgs < NumFloatArgRegs) { + *out = FloatRegister::FromCode(f0.code() + usedFloatArgs); + return true; + } + return false; +} + +// Get a register in which we plan to put a quantity that will be used as an +// integer argument. This differs from GetIntArgReg in that if we have no more +// actual argument registers to use we will fall back on using whatever +// CallTempReg* don't overlap the argument registers, and only fail once those +// run out too. +static inline bool GetTempRegForIntArg(uint32_t usedIntArgs, + uint32_t usedFloatArgs, Register* out) { + // NOTE: We can't properly determine which regs are used if there are + // float arguments. If this is needed, we will have to guess. + MOZ_ASSERT(usedFloatArgs == 0); + + if (GetIntArgReg(usedIntArgs, out)) { + return true; + } + // Unfortunately, we have to assume things about the point at which + // GetIntArgReg returns false, because we need to know how many registers it + // can allocate. + usedIntArgs -= NumIntArgRegs; + if (usedIntArgs >= NumCallTempNonArgRegs) { + return false; + } + *out = CallTempNonArgRegs[usedIntArgs]; + return true; +} + +} // namespace jit +} // namespace js + +#endif /* jit_loongarch64_Assembler_loongarch64_h */ diff --git a/js/src/jit/loongarch64/Bailouts-loongarch64.cpp b/js/src/jit/loongarch64/Bailouts-loongarch64.cpp new file mode 100644 index 0000000000..9fc484db97 --- /dev/null +++ b/js/src/jit/loongarch64/Bailouts-loongarch64.cpp @@ -0,0 +1,27 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loongarch64/Bailouts-loongarch64.h" + +#include "jscntxt.h" +#include "jscompartment.h" + +using namespace js; +using namespace js::jit; + +BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations, + BailoutStack* bailout) + : machine_(bailout->machineState()) +{ + uint8_t* sp = bailout->parentStackPointer(); + framePointer_ = sp + bailout->frameSize(); + topFrameSize_ = framePointer_ - sp; + + JSScript* script = ScriptFromCalleeToken(((JitFrameLayout*) framePointer_)->calleeToken()); + topIonScript_ = script->ionScript(); + + attachOnJitActivation(activations); + snapshotOffset_ = bailout->snapshotOffset(); +} diff --git a/js/src/jit/loongarch64/Bailouts-loongarch64.h b/js/src/jit/loongarch64/Bailouts-loongarch64.h new file mode 100644 index 0000000000..d180c6e92c --- /dev/null +++ b/js/src/jit/loongarch64/Bailouts-loongarch64.h @@ -0,0 +1,43 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loongarch64_Bailouts_loongarch64_h +#define jit_loongarch64_Bailouts_loongarch64_h + +#include "jit/Bailouts.h" +#include "jit/JitCompartment.h" + +namespace js { +namespace jit { + +class BailoutStack +{ + RegisterDump::FPUArray fpregs_; + RegisterDump::GPRArray regs_; + uintptr_t frameSize_; + uintptr_t snapshotOffset_; + + public: + MachineState machineState() { + return MachineState::FromBailout(regs_, fpregs_); + } + uint32_t snapshotOffset() const { + return snapshotOffset_; + } + uint32_t frameSize() const { + return frameSize_; + } + uint8_t* parentStackPointer() { + return (uint8_t*)this + sizeof(BailoutStack); + } + static size_t offsetOfFrameSize() { + return offsetof(BailoutStack, frameSize_); + } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_loongarch64_Bailouts_loongarch64_h */ diff --git a/js/src/jit/loongarch64/BaselineCompiler-loongarch64.cpp b/js/src/jit/loongarch64/BaselineCompiler-loongarch64.cpp new file mode 100644 index 0000000000..a6e04dc2a3 --- /dev/null +++ b/js/src/jit/loongarch64/BaselineCompiler-loongarch64.cpp @@ -0,0 +1,15 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loongarch64/BaselineCompiler-loongarch64.h" + +using namespace js; +using namespace js::jit; + +BaselineCompilerLOONGARCH64::BaselineCompilerLOONGARCH64(JSContext* cx, TempAllocator& alloc, + JSScript* script) + : BaselineCompilerMIPSShared(cx, alloc, script) +{ +} diff --git a/js/src/jit/loongarch64/BaselineCompiler-loongarch64.h b/js/src/jit/loongarch64/BaselineCompiler-loongarch64.h new file mode 100644 index 0000000000..42a9fa1d14 --- /dev/null +++ b/js/src/jit/loongarch64/BaselineCompiler-loongarch64.h @@ -0,0 +1,25 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loongarch64_BaselineCompiler_loongarch64_h +#define jit_loongarch64_BaselineCompiler_loongarch64_h + +#include "jit/mips-shared/BaselineCompiler-mips-shared.h" + +namespace js { +namespace jit { + +class BaselineCompilerLOONGARCH64 : public BaselineCompilerMIPSShared +{ + protected: + BaselineCompilerLOONGARCH64(JSContext* cx, TempAllocator& alloc, JSScript* script); +}; + +typedef BaselineCompilerLOONGARCH64 BaselineCompilerSpecific; + +} // namespace jit +} // namespace js + +#endif /* jit_loongarch64_BaselineCompiler_loongarch64_h */ diff --git a/js/src/jit/loongarch64/BaselineIC-loongarch64.cpp b/js/src/jit/loongarch64/BaselineIC-loongarch64.cpp new file mode 100644 index 0000000000..84ff1a8c78 --- /dev/null +++ b/js/src/jit/loongarch64/BaselineIC-loongarch64.cpp @@ -0,0 +1,46 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/BaselineCompiler.h" +#include "jit/BaselineIC.h" +#include "jit/BaselineJIT.h" +#include "jit/Linker.h" +#include "jit/SharedICHelpers.h" + +using namespace js; +using namespace js::jit; + +namespace js { +namespace jit { + +// ICCompare_Int32 + +bool +ICCompare_Int32::Compiler::generateStubCode(MacroAssembler& masm) +{ + // Guard that R0 is an integer and R1 is an integer. + Label failure; + Label conditionTrue; + masm.branchTestInt32(Assembler::NotEqual, R0, &failure); + masm.branchTestInt32(Assembler::NotEqual, R1, &failure); + + // Compare payload regs of R0 and R1. + masm.unboxInt32(R0, ExtractTemp0); + masm.unboxInt32(R1, ExtractTemp1); + Assembler::Condition cond = JSOpToCondition(op, /* signed = */true); + masm.ma_cmp_set(R0.valueReg(), ExtractTemp0, ExtractTemp1, cond); + + masm.tagValue(JSVAL_TYPE_BOOLEAN, R0.valueReg(), R0); + EmitReturnFromIC(masm); + + // Failure case - jump to next stub + masm.bind(&failure); + EmitStubGuardFailure(masm); + + return true; +} + +} // namespace jit +} // namespace js diff --git a/js/src/jit/loongarch64/MacroAssembler-loongarch64-inl.h b/js/src/jit/loongarch64/MacroAssembler-loongarch64-inl.h new file mode 100644 index 0000000000..fd87043ede --- /dev/null +++ b/js/src/jit/loongarch64/MacroAssembler-loongarch64-inl.h @@ -0,0 +1,1445 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loongarch64_MacroAssembler_loongarch64_inl_h +#define jit_loongarch64_MacroAssembler_loongarch64_inl_h + +#include "jit/loongarch64/MacroAssembler-loongarch64.h" + +namespace js { +namespace jit { + +//{{{ check_macroassembler_style + +void MacroAssembler::move64(Register64 src, Register64 dest) { + movePtr(src.reg, dest.reg); +} + +void MacroAssembler::move64(Imm64 imm, Register64 dest) { + movePtr(ImmWord(imm.value), dest.reg); +} +void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) { + moveFromFloat32(src, dest); +} + +void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) { + moveToFloat32(src, dest); +} + +void MacroAssembler::move8SignExtend(Register src, Register dest) { + as_ext_w_b(dest, src); +} + +void MacroAssembler::move16SignExtend(Register src, Register dest) { + as_ext_w_h(dest, src); +} +// =============================================================== +// Load instructions +// =============================================================== +// Logical instructions + +void MacroAssembler::not32(Register reg) { as_nor(reg, reg, zero); } +void MacroAssembler::andPtr(Register src, Register dest) { + as_and(dest, dest, src); +} + +void MacroAssembler::andPtr(Imm32 imm, Register dest) { + ma_and(dest, dest, imm); +} + +void MacroAssembler::and64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, ImmWord(imm.value)); + as_and(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::and64(Register64 src, Register64 dest) { + as_and(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::and64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(*this); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + and64(scratch64, dest); + } else { + and64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::and32(Register src, Register dest) { + as_and(dest, dest, src); +} + +void MacroAssembler::and32(Imm32 imm, Register dest) { + ma_and(dest, dest, imm); +} + +void MacroAssembler::and32(Imm32 imm, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(dest, scratch2); + and32(imm, scratch2); + store32(scratch2, dest); +} + +void MacroAssembler::and32(const Address& src, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(src, scratch2); + as_and(dest, dest, scratch2); +} + +void MacroAssembler::or64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, ImmWord(imm.value)); + as_or(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::or32(Register src, Register dest) { + as_or(dest, dest, src); +} + +void MacroAssembler::or32(Imm32 imm, Register dest) { ma_or(dest, dest, imm); } + +void MacroAssembler::or32(Imm32 imm, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(dest, scratch2); + or32(imm, scratch2); + store32(scratch2, dest); +} + +void MacroAssembler::xor64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, ImmWord(imm.value)); + as_xor(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::orPtr(Register src, Register dest) { + as_or(dest, dest, src); +} + +void MacroAssembler::orPtr(Imm32 imm, Register dest) { ma_or(dest, dest, imm); } + +void MacroAssembler::or64(Register64 src, Register64 dest) { + as_or(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::or64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + or64(scratch64, dest); + } else { + or64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::xor64(Register64 src, Register64 dest) { + as_xor(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::xor64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + xor64(scratch64, dest); + } else { + xor64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::xorPtr(Register src, Register dest) { + as_xor(dest, dest, src); +} + +void MacroAssembler::xorPtr(Imm32 imm, Register dest) { + ma_xor(dest, dest, imm); +} + +void MacroAssembler::xor32(Register src, Register dest) { + as_xor(dest, dest, src); +} + +void MacroAssembler::xor32(Imm32 imm, Register dest) { + ma_xor(dest, dest, imm); +} + +// =============================================================== +// Swap instructions +// =============================================================== +// Arithmetic functions + +void MacroAssembler::addPtr(Register src, Register dest) { + as_add_d(dest, dest, src); +} + +void MacroAssembler::addPtr(Imm32 imm, Register dest) { + ma_add_d(dest, dest, imm); +} + +void MacroAssembler::addPtr(ImmWord imm, Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(imm, scratch); + addPtr(scratch, dest); +} + +void MacroAssembler::add64(Register64 src, Register64 dest) { + addPtr(src.reg, dest.reg); +} + +void MacroAssembler::add64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + add64(scratch64, dest); + } else { + add64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::add64(Imm32 imm, Register64 dest) { + ma_add_d(dest.reg, dest.reg, imm); +} + +void MacroAssembler::add64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dest.reg != scratch); + mov(ImmWord(imm.value), scratch); + as_add_d(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::add32(Register src, Register dest) { + as_add_w(dest, dest, src); +} + +void MacroAssembler::add32(Imm32 imm, Register dest) { + ma_add_w(dest, dest, imm); +} + +void MacroAssembler::add32(Imm32 imm, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(dest, scratch2); + ma_add_w(scratch2, scratch2, imm); + store32(scratch2, dest); +} + +void MacroAssembler::addPtr(Imm32 imm, const Address& dest) { + ScratchRegisterScope scratch(asMasm()); + loadPtr(dest, scratch); + addPtr(imm, scratch); + storePtr(scratch, dest); +} + +void MacroAssembler::addPtr(const Address& src, Register dest) { + ScratchRegisterScope scratch(asMasm()); + loadPtr(src, scratch); + addPtr(scratch, dest); +} + +void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) { + as_fadd_d(dest, dest, src); +} + +void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) { + as_fadd_s(dest, dest, src); +} +void MacroAssembler::subPtr(Register src, Register dest) { + as_sub_d(dest, dest, src); +} + +void MacroAssembler::subPtr(Imm32 imm, Register dest) { + ma_sub_d(dest, dest, imm); +} + +void MacroAssembler::sub64(Register64 src, Register64 dest) { + as_sub_d(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::sub64(const Operand& src, Register64 dest) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + sub64(scratch64, dest); + } else { + sub64(Register64(src.toReg()), dest); + } +} + +void MacroAssembler::sub64(Imm64 imm, Register64 dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dest.reg != scratch); + mov(ImmWord(imm.value), scratch); + as_sub_d(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::sub32(Register src, Register dest) { + as_sub_w(dest, dest, src); +} + +void MacroAssembler::sub32(Imm32 imm, Register dest) { + ma_sub_w(dest, dest, imm); +} + +void MacroAssembler::sub32(const Address& src, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(src, scratch2); + as_sub_w(dest, dest, scratch2); +} + +void MacroAssembler::subPtr(Register src, const Address& dest) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(dest, scratch2); + subPtr(src, scratch2); + storePtr(scratch2, dest); +} + +void MacroAssembler::subPtr(const Address& addr, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(addr, scratch2); + subPtr(scratch2, dest); +} + +void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) { + as_fsub_d(dest, dest, src); +} + +void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) { + as_fsub_s(dest, dest, src); +} + +void MacroAssembler::mul64(Imm64 imm, const Register64& dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dest.reg != scratch); + mov(ImmWord(imm.value), scratch); + as_mul_d(dest.reg, dest.reg, scratch); +} + +void MacroAssembler::mul64(Imm64 imm, const Register64& dest, + const Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + mul64(imm, dest); +} + +void MacroAssembler::mul64(const Register64& src, const Register64& dest, + const Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + as_mul_d(dest.reg, dest.reg, src.reg); +} + +void MacroAssembler::mul64(const Operand& src, const Register64& dest, + const Register temp) { + if (src.getTag() == Operand::MEM) { + ScratchRegisterScope scratch(asMasm()); + Register64 scratch64(scratch); + + load64(src.toAddress(), scratch64); + mul64(scratch64, dest, temp); + } else { + mul64(Register64(src.toReg()), dest, temp); + } +} +void MacroAssembler::mulBy3(Register src, Register dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + as_add_d(scratch, src, src); + as_add_d(dest, scratch, src); +} + +void MacroAssembler::mul32(Register rhs, Register srcDest) { + as_mul_w(srcDest, srcDest, rhs); +} + +void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) { + as_fmul_s(dest, dest, src); +} + +void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) { + as_fmul_d(dest, dest, src); +} + +void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp, + FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + movePtr(imm, scratch); + loadDouble(Address(scratch, 0), fpscratch); + mulDouble(fpscratch, dest); +} + +void MacroAssembler::inc64(AbsoluteAddress dest) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch, ImmWord(uintptr_t(dest.addr))); + as_ld_d(scratch2, scratch, 0); + as_addi_d(scratch2, scratch2, 1); + as_st_d(scratch2, scratch, 0); +} + +void MacroAssembler::quotient32(Register rhs, Register srcDest, + bool isUnsigned) { + if (isUnsigned) { + as_div_wu(srcDest, srcDest, rhs); + } else { + as_div_w(srcDest, srcDest, rhs); + } +} + +void MacroAssembler::remainder32(Register rhs, Register srcDest, + bool isUnsigned) { + if (isUnsigned) { + as_mod_wu(srcDest, srcDest, rhs); + } else { + as_mod_w(srcDest, srcDest, rhs); + } +} + +void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) { + as_fdiv_s(dest, dest, src); +} + +void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) { + as_fdiv_d(dest, dest, src); +} + +void MacroAssembler::neg64(Register64 reg) { as_sub_d(reg.reg, zero, reg.reg); } +void MacroAssembler::neg32(Register reg) { as_sub_w(reg, zero, reg); } + +void MacroAssembler::negateDouble(FloatRegister reg) { as_fneg_d(reg, reg); } + +void MacroAssembler::negateFloat(FloatRegister reg) { as_fneg_s(reg, reg); } +void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) { + as_fabs_s(dest, src); +} + +void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) { + as_fabs_d(dest, src); +} + +void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) { + as_fsqrt_s(dest, src); +} + +void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) { + as_fsqrt_d(dest, src); +} + +void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + minMaxFloat32(srcDest, other, handleNaN, false); +} + +void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + minMaxDouble(srcDest, other, handleNaN, false); +} + +void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + minMaxFloat32(srcDest, other, handleNaN, true); +} + +void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + minMaxDouble(srcDest, other, handleNaN, true); +} + +// =============================================================== +// Shift functions + +void MacroAssembler::lshift32(Register src, Register dest) { + as_sll_w(dest, dest, src); +} + +void MacroAssembler::lshift32(Imm32 imm, Register dest) { + as_slli_w(dest, dest, imm.value % 32); +} +void MacroAssembler::lshift64(Register shift, Register64 dest) { + as_sll_d(dest.reg, dest.reg, shift); +} + +void MacroAssembler::lshift64(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_slli_d(dest.reg, dest.reg, imm.value); +} + +void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_slli_d(dest, dest, imm.value); +} + +void MacroAssembler::rshift32(Register src, Register dest) { + as_srl_w(dest, dest, src); +} + +void MacroAssembler::rshift32(Imm32 imm, Register dest) { + as_srli_w(dest, dest, imm.value % 32); +} +void MacroAssembler::rshift32Arithmetic(Register src, Register dest) { + as_sra_w(dest, dest, src); +} + +void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) { + as_srai_w(dest, dest, imm.value % 32); +} +void MacroAssembler::rshift64(Register shift, Register64 dest) { + as_srl_d(dest.reg, dest.reg, shift); +} + +void MacroAssembler::rshift64(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_srli_d(dest.reg, dest.reg, imm.value); +} + +void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_srai_d(dest.reg, dest.reg, imm.value); +} + +void MacroAssembler::rshift64Arithmetic(Register shift, Register64 dest) { + as_sra_d(dest.reg, dest.reg, shift); +} + +void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_srli_d(dest, dest, imm.value); +} + +void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) { + MOZ_ASSERT(0 <= imm.value && imm.value < 64); + as_srai_d(dest, dest, imm.value); +} + +// =============================================================== +// Rotation functions + +void MacroAssembler::rotateLeft(Register count, Register input, Register dest) { + ScratchRegisterScope scratch(asMasm()); + as_sub_w(scratch, zero, count); + as_rotr_w(dest, input, scratch); +} + +void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) { + as_rotri_w(dest, input, (32 - count.value) & 31); +} + +void MacroAssembler::rotateLeft64(Register count, Register64 src, + Register64 dest, Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + ScratchRegisterScope scratch(asMasm()); + as_sub_d(scratch, zero, count); + as_rotr_d(dest.reg, src.reg, scratch); +} + +void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest, + Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + as_rotri_d(dest.reg, src.reg, (64 - count.value) & 63); +} + +void MacroAssembler::rotateRight(Register count, Register input, + Register dest) { + as_rotr_w(dest, input, count); +} + +void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) { + as_rotri_w(dest, input, count.value & 31); +} + +void MacroAssembler::rotateRight64(Register count, Register64 src, + Register64 dest, Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + as_rotr_d(dest.reg, src.reg, count); +} + +void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest, + Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + as_rotri_d(dest.reg, src.reg, count.value & 63); +} + +// Bit counting functions + +void MacroAssembler::clz64(Register64 src, Register dest) { + as_clz_d(dest, src.reg); +} + +void MacroAssembler::ctz64(Register64 src, Register dest) { + as_ctz_d(dest, src.reg); +} + +void MacroAssembler::popcnt64(Register64 input, Register64 output, + Register tmp) { + ScratchRegisterScope scratch(asMasm()); + as_or(output.reg, input.reg, zero); + as_srai_d(tmp, input.reg, 1); + ma_li(scratch, Imm32(0x55555555)); + as_bstrins_d(scratch, scratch, 63, 32); + as_and(tmp, tmp, scratch); + as_sub_d(output.reg, output.reg, tmp); + as_srai_d(tmp, output.reg, 2); + ma_li(scratch, Imm32(0x33333333)); + as_bstrins_d(scratch, scratch, 63, 32); + as_and(output.reg, output.reg, scratch); + as_and(tmp, tmp, scratch); + as_add_d(output.reg, output.reg, tmp); + as_srli_d(tmp, output.reg, 4); + as_add_d(output.reg, output.reg, tmp); + ma_li(scratch, Imm32(0xF0F0F0F)); + as_bstrins_d(scratch, scratch, 63, 32); + as_and(output.reg, output.reg, scratch); + ma_li(tmp, Imm32(0x1010101)); + as_bstrins_d(tmp, tmp, 63, 32); + as_mul_d(output.reg, output.reg, tmp); + as_srai_d(output.reg, output.reg, 56); +} + +void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) { + as_clz_w(dest, src); +} + +void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) { + as_ctz_w(dest, src); +} + +void MacroAssembler::popcnt32(Register input, Register output, Register tmp) { + // Equivalent to GCC output of mozilla::CountPopulation32() + as_or(output, input, zero); + as_srai_w(tmp, input, 1); + ma_and(tmp, tmp, Imm32(0x55555555)); + as_sub_w(output, output, tmp); + as_srai_w(tmp, output, 2); + ma_and(output, output, Imm32(0x33333333)); + ma_and(tmp, tmp, Imm32(0x33333333)); + as_add_w(output, output, tmp); + as_srli_w(tmp, output, 4); + as_add_w(output, output, tmp); + ma_and(output, output, Imm32(0xF0F0F0F)); + as_slli_w(tmp, output, 8); + as_add_w(output, output, tmp); + as_slli_w(tmp, output, 16); + as_add_w(output, output, tmp); + as_srai_w(output, output, 24); +} + +// =============================================================== +// Condition functions +// Also see below for specializations of cmpPtrSet. +template +void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) { + ma_cmp_set(dest, lhs, rhs, cond); +} +template +void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) { + ma_cmp_set(dest, lhs, rhs, cond); +} + +// =============================================================== +// Branch functions +template +void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs, + L label) { + ma_b(lhs, rhs, label, cond); +} + +template +void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm, + L label) { + ma_b(lhs, imm, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + ma_b(scratch2, rhs, label, cond); +} + +void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress addr, + Imm32 imm, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(addr, scratch2); + ma_b(scratch2, imm, label, cond); +} + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val, + Label* success, Label* fail) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal || + cond == Assembler::LessThan || + cond == Assembler::LessThanOrEqual || + cond == Assembler::GreaterThan || + cond == Assembler::GreaterThanOrEqual || + cond == Assembler::Below || cond == Assembler::BelowOrEqual || + cond == Assembler::Above || cond == Assembler::AboveOrEqual, + "other condition codes not supported"); + + branchPtr(cond, lhs.reg, ImmWord(val.value), success); + if (fail) { + jump(fail); + } +} + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs, + Label* success, Label* fail) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal || + cond == Assembler::LessThan || + cond == Assembler::LessThanOrEqual || + cond == Assembler::GreaterThan || + cond == Assembler::GreaterThanOrEqual || + cond == Assembler::Below || cond == Assembler::BelowOrEqual || + cond == Assembler::Above || cond == Assembler::AboveOrEqual, + "other condition codes not supported"); + + branchPtr(cond, lhs.reg, rhs.reg, success); + if (fail) { + jump(fail); + } +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val, + Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal, + "other condition codes not supported"); + + branchPtr(cond, lhs, ImmWord(val.value), label); +} + +void MacroAssembler::branch64(Condition cond, const Address& lhs, + const Address& rhs, Register scratch, + Label* label) { + MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal, + "other condition codes not supported"); + MOZ_ASSERT(lhs.base != scratch); + MOZ_ASSERT(rhs.base != scratch); + + loadPtr(rhs, scratch); + branchPtr(cond, lhs, scratch, label); +} + +template +void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs, + L label) { + ma_b(lhs, rhs, label, cond); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + ma_b(lhs, rhs, label, cond); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs, + Label* label) { + ma_b(lhs, rhs, label, cond); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs, + Label* label) { + ma_b(lhs, rhs, label, cond); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs, + Label* label) { + ma_b(lhs, rhs, label, cond); +} + +template +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs, + L label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + ImmWord rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs, + Register rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchPtr(cond, scratch2, rhs, label); +} + +void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs, + Register rhs, Label* label) { + branchPtr(cond, lhs, rhs, label); +} + +void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs, + FloatRegister rhs, Label* label) { + ma_bc_s(lhs, rhs, label, cond); +} + +void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + as_ftintrz_l_s(fpscratch, src); + as_movfcsr2gr(scratch); + moveFromDouble(fpscratch, dest); + MOZ_ASSERT(Assembler::CauseV < 32); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, Imm32(0), fail, Assembler::NotEqual); + + as_slli_w(dest, dest, 0); +} + +void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src, + Register dest, Label* fail) { + convertFloat32ToInt32(src, dest, fail, false); +} + +void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs, + FloatRegister rhs, Label* label) { + ma_bc_d(lhs, rhs, label, cond); +} + +void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + as_ftintrz_l_d(fpscratch, src); + as_movfcsr2gr(scratch); + moveFromDouble(fpscratch, dest); + MOZ_ASSERT(Assembler::CauseV < 32); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, Imm32(0), fail, Assembler::NotEqual); + + as_slli_w(dest, dest, 0); +} + +void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src, + Register dest, Label* fail) { + convertDoubleToInt32(src, dest, fail, false); +} + +template +void MacroAssembler::branchAdd32(Condition cond, T src, Register dest, + L overflow) { + switch (cond) { + case Overflow: + ma_add32TestOverflow(dest, dest, src, overflow); + break; + case CarryClear: + case CarrySet: + ma_add32TestCarry(cond, dest, dest, src, overflow); + break; + default: + MOZ_CRASH("NYI"); + } +} + +// the type of 'T src' maybe a Register, maybe a Imm32,depends on who call it. +template +void MacroAssembler::branchSub32(Condition cond, T src, Register dest, + Label* overflow) { + switch (cond) { + case Overflow: + ma_sub32TestOverflow(dest, dest, src, overflow); + break; + case NonZero: + case Zero: + case Signed: + case NotSigned: + ma_sub_w(dest, dest, src); + ma_b(dest, dest, overflow, cond); + break; + default: + MOZ_CRASH("NYI"); + } +} +void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + subPtr(rhs, lhs); + branchPtr(cond, lhs, Imm32(0), label); +} + +template +void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs, + L label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + if (lhs == rhs) { + ma_b(lhs, rhs, label, cond); + } else { + ScratchRegisterScope scratch(asMasm()); + as_and(scratch, lhs, rhs); + ma_b(scratch, scratch, label, cond); + } +} + +template +void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs, + L label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + SecondScratchRegisterScope scratch2(asMasm()); + ma_and(scratch2, lhs, rhs); + ma_b(scratch2, scratch2, label, cond); +} + +void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs, + Label* label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + and32(rhs, scratch2); + ma_b(scratch2, scratch2, label, cond); +} + +void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + SecondScratchRegisterScope scratch2(asMasm()); + load32(lhs, scratch2); + and32(rhs, scratch2); + ma_b(scratch2, scratch2, label, cond); +} + +template +void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs, + L label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + if (lhs == rhs) { + ma_b(lhs, rhs, label, cond); + } else { + ScratchRegisterScope scratch(asMasm()); + as_and(scratch, lhs, rhs); + ma_b(scratch, scratch, label, cond); + } +} + +void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || + cond == NotSigned); + ScratchRegisterScope scratch(asMasm()); + ma_and(scratch, lhs, rhs); + ma_b(scratch, scratch, label, cond); +} + +void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs, + Imm32 rhs, Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + branchTestPtr(cond, scratch2, rhs, label); +} + +template +void MacroAssembler::branchTest64(Condition cond, Register64 lhs, + Register64 rhs, Register temp, L label) { + branchTestPtr(cond, lhs.reg, rhs.reg, label); +} + +void MacroAssembler::branchTestUndefined(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_UNDEFINED), label, cond); +} + +void MacroAssembler::branchTestUndefined(Condition cond, + const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestUndefined(cond, scratch2, label); +} + +void MacroAssembler::branchTestUndefined(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestUndefined(cond, tag, label); +} + +void MacroAssembler::branchTestUndefined(Condition cond, + const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestUndefined(cond, tag, label); +} + +void MacroAssembler::branchTestInt32(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_INT32), label, cond); +} + +void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestInt32(cond, scratch2, label); +} + +void MacroAssembler::branchTestInt32(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestInt32(cond, tag, label); +} + +void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestInt32(cond, tag, label); +} + +void MacroAssembler::branchTestInt32Truthy(bool b, const ValueOperand& value, + Label* label) { + ScratchRegisterScope scratch(*this); + as_bstrpick_d(scratch, value.valueReg(), 31, 0); + ma_b(scratch, scratch, label, b ? NonZero : Zero); +} + +void MacroAssembler::branchTestDouble(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + Condition actual = (cond == Equal) ? BelowOrEqual : Above; + ma_b(tag, ImmTag(JSVAL_TAG_MAX_DOUBLE), label, actual); +} + +void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestDouble(cond, scratch2, label); +} + +void MacroAssembler::branchTestDouble(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestDouble(cond, tag, label); +} + +void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestDouble(cond, tag, label); +} + +void MacroAssembler::branchTestDoubleTruthy(bool b, FloatRegister value, + Label* label) { + ScratchDoubleScope fpscratch(*this); + ma_lid(fpscratch, 0.0); + DoubleCondition cond = b ? DoubleNotEqual : DoubleEqualOrUnordered; + ma_bc_d(value, fpscratch, label, cond); +} + +void MacroAssembler::branchTestNumber(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + Condition actual = cond == Equal ? BelowOrEqual : Above; + ma_b(tag, ImmTag(JSVAL_UPPER_INCL_TAG_OF_NUMBER_SET), label, actual); +} + +void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestNumber(cond, scratch2, label); +} + +void MacroAssembler::branchTestBoolean(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_BOOLEAN), label, cond); +} + +void MacroAssembler::branchTestBoolean(Condition cond, + const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestBoolean(cond, scratch2, label); +} + +void MacroAssembler::branchTestBoolean(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestBoolean(cond, tag, label); +} + +void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestBoolean(cond, tag, label); +} + +void MacroAssembler::branchTestBooleanTruthy(bool b, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + unboxBoolean(value, scratch2); + ma_b(scratch2, scratch2, label, b ? NonZero : Zero); +} + +void MacroAssembler::branchTestString(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_STRING), label, cond); +} + +void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestString(cond, scratch2, label); +} + +void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestString(cond, tag, label); +} + +void MacroAssembler::branchTestStringTruthy(bool b, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + unboxString(value, scratch2); + load32(Address(scratch2, JSString::offsetOfLength()), scratch2); + ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal); +} + +void MacroAssembler::branchTestSymbol(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_SYMBOL), label, cond); +} + +void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestSymbol(cond, scratch2, label); +} + +void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestSymbol(cond, tag, label); +} + +void MacroAssembler::branchTestBigInt(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_BIGINT), label, cond); +} + +void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestBigInt(cond, scratch2, label); +} + +void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + computeEffectiveAddress(address, scratch2); + splitTag(scratch2, scratch2); + branchTestBigInt(cond, scratch2, label); +} + +void MacroAssembler::branchTestBigIntTruthy(bool b, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + unboxBigInt(value, scratch2); + loadPtr(Address(scratch2, BigInt::offsetOfLengthSignAndReservedBits()), scratch2); + ma_b(scratch2, ImmWord(0), label, b ? NotEqual : Equal); +} + +void MacroAssembler::branchTestNull(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_NULL), label, cond); +} + +void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestNull(cond, scratch2, label); +} + +void MacroAssembler::branchTestNull(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestNull(cond, tag, label); +} + +void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestNull(cond, tag, label); +} + +void MacroAssembler::branchTestObject(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_OBJECT), label, cond); +} + +void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestObject(cond, scratch2, label); +} + +void MacroAssembler::branchTestObject(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestObject(cond, tag, label); +} + +void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestObject(cond, tag, label); +} + +void MacroAssembler::branchTestPrimitive(Condition cond, + const ValueOperand& value, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + branchTestPrimitive(cond, scratch2, label); +} + +void MacroAssembler::branchTestGCThing(Condition cond, const Address& address, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + ma_b(tag, ImmTag(JSVAL_LOWER_INCL_TAG_OF_GCTHING_SET), label, + (cond == Equal) ? AboveOrEqual : Below); +} + +void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + ma_b(tag, ImmTag(JSVAL_LOWER_INCL_TAG_OF_GCTHING_SET), label, + (cond == Equal) ? AboveOrEqual : Below); +} + +void MacroAssembler::branchTestPrimitive(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_UPPER_EXCL_TAG_OF_PRIMITIVE_SET), label, + (cond == Equal) ? Below : AboveOrEqual); +} + +void MacroAssembler::branchTestMagic(Condition cond, Register tag, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ma_b(tag, ImmTag(JSVAL_TAG_MAGIC), label, cond); +} + +void MacroAssembler::branchTestMagic(Condition cond, const Address& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestMagic(cond, tag, label); +} + +void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address, + Label* label) { + SecondScratchRegisterScope scratch2(*this); + Register tag = extractTag(address, scratch2); + branchTestMagic(cond, tag, label); +} + +template +void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value, + L label) { + SecondScratchRegisterScope scratch2(*this); + splitTag(value, scratch2); + ma_b(scratch2, ImmTag(JSVAL_TAG_MAGIC), label, cond); +} + +void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, + JSWhyMagic why, Label* label) { + uint64_t magic = MagicValue(why).asRawBits(); + SecondScratchRegisterScope scratch(*this); + loadPtr(valaddr, scratch); + ma_b(scratch, ImmWord(magic), label, cond); +} + +//}}} check_macroassembler_style +// =============================================================== + +// The specializations for cmpPtrSet are outside the braces because +// check_macroassembler_style can't yet deal with specializations. + +template <> +inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs, + ImmPtr rhs, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(lhs, scratch2); + cmpPtrSet(cond, Register(scratch2), rhs, dest); +} + +template <> +inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Register lhs, + Address rhs, Register dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + loadPtr(rhs, scratch); + cmpPtrSet(cond, lhs, Register(scratch), dest); +} + +template <> +inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs, + Register rhs, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rhs != scratch2); + loadPtr(lhs, scratch2); + cmpPtrSet(cond, Register(scratch2), rhs, dest); +} + +template <> +inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Register lhs, + Address rhs, Register dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + load32(rhs, scratch); + cmp32Set(cond, lhs, Register(scratch), dest); +} + +template <> +inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Address lhs, + Register rhs, Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rhs != scratch2); + load32(lhs, scratch2); + cmp32Set(cond, Register(scratch2), rhs, dest); +} + +void MacroAssemblerLOONGARCH64Compat::incrementInt32Value(const Address& addr) { + asMasm().add32(Imm32(1), addr); +} + +void MacroAssemblerLOONGARCH64Compat::retn(Imm32 n) { + // pc <- [sp]; sp += n + loadPtr(Address(StackPointer, 0), ra); + asMasm().addPtr(n, StackPointer); + as_jirl(zero, ra, BOffImm16(0)); +} + +// If source is a double, load into dest. +// If source is int32, convert to double and store in dest. +// Else, branch to failure. +void MacroAssemblerLOONGARCH64Compat::ensureDouble(const ValueOperand& source, + FloatRegister dest, + Label* failure) { + Label isDouble, done; + { + ScratchTagScope tag(asMasm(), source); + splitTagForTest(source, tag); + asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble); + asMasm().branchTestInt32(Assembler::NotEqual, tag, failure); + } + + convertInt32ToDouble(source.valueReg(), dest); + jump(&done); + + bind(&isDouble); + unboxDouble(source, dest); + + bind(&done); +} + +} // namespace jit +} // namespace js + +#endif /* jit_loongarch64_MacroAssembler_loongarch64_inl_h */ diff --git a/js/src/jit/loongarch64/MacroAssembler-loongarch64.cpp b/js/src/jit/loongarch64/MacroAssembler-loongarch64.cpp new file mode 100644 index 0000000000..995f789305 --- /dev/null +++ b/js/src/jit/loongarch64/MacroAssembler-loongarch64.cpp @@ -0,0 +1,5285 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/loongarch64/MacroAssembler-loongarch64.h" + +#include "jsmath.h" + +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/JitFrames.h" +#include "jit/loongarch64/SharedICRegisters-loongarch64.h" +#include "jit/MacroAssembler.h" +#include "jit/MoveEmitter.h" +#include "util/Memory.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/JSContext.h" + +#include "jit/MacroAssembler-inl.h" + +namespace js { +namespace jit { + +void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) { + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + as_ftintrne_l_d(fpscratch, input); + as_movfr2gr_d(output, fpscratch); + // if (res < 0); res = 0; + as_slt(scratch, output, zero); + as_masknez(output, output, scratch); + // if res > 255; res = 255; + as_sltui(scratch, output, 255); + as_addi_d(output, output, -255); + as_maskeqz(output, output, scratch); + as_addi_d(output, output, 255); +} + +bool MacroAssemblerLOONGARCH64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr) { + uint32_t descriptor = MakeFrameDescriptor( + asMasm().framePushed(), FrameType::IonJS, ExitFrameLayout::Size()); + + asMasm().Push(Imm32(descriptor)); // descriptor_ + asMasm().Push(ImmPtr(fakeReturnAddr)); + + return true; +} + +void MacroAssemblerLOONGARCH64Compat::convertUInt32ToDouble(Register src, + FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + as_bstrpick_d(scratch, src, 31, 0); + asMasm().convertInt64ToDouble(Register64(scratch), dest); +} + +void MacroAssemblerLOONGARCH64Compat::convertUInt64ToDouble(Register src, + FloatRegister dest) { + Label positive, done; + ma_b(src, src, &positive, NotSigned, ShortJump); + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(src != scratch2); + + ma_and(scratch, src, Imm32(1)); + as_srli_d(scratch2, src, 1); + as_or(scratch, scratch, scratch2); + as_movgr2fr_d(dest, scratch); + as_ffint_d_l(dest, dest); + asMasm().addDouble(dest, dest); + ma_b(&done, ShortJump); + + bind(&positive); + as_movgr2fr_d(dest, src); + as_ffint_d_l(dest, dest); + + bind(&done); +} + +void MacroAssemblerLOONGARCH64Compat::convertUInt32ToFloat32(Register src, + FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + as_bstrpick_d(scratch, src, 31, 0); + asMasm().convertInt64ToFloat32(Register64(scratch), dest); +} + +void MacroAssemblerLOONGARCH64Compat::convertDoubleToFloat32(FloatRegister src, + FloatRegister dest) { + as_fcvt_s_d(dest, src); +} + +const int CauseBitPos = int(Assembler::CauseI); +const int CauseBitCount = 1 + int(Assembler::CauseV) - int(Assembler::CauseI); +const int CauseIOrVMask = ((1 << int(Assembler::CauseI)) | + (1 << int(Assembler::CauseV))) >> + int(Assembler::CauseI); + +// Checks whether a double is representable as a 32-bit integer. If so, the +// integer is written to the output register. Otherwise, a bailout is taken to +// the given snapshot. This function overwrites the scratch float register. +void MacroAssemblerLOONGARCH64Compat::convertDoubleToInt32(FloatRegister src, + Register dest, + Label* fail, + bool negativeZeroCheck) { + if (negativeZeroCheck) { + moveFromDouble(src, dest); + as_rotri_d(dest, dest, 63); + ma_b(dest, Imm32(1), fail, Assembler::Equal); + } + + ScratchRegisterScope scratch(asMasm()); + ScratchFloat32Scope fpscratch(asMasm()); + // Truncate double to int ; if result is inexact or invalid fail. + as_ftintrz_w_d(fpscratch, src); + as_movfcsr2gr(scratch); + moveFromFloat32(fpscratch, dest); + as_bstrpick_d(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); + as_andi(scratch, scratch, + CauseIOrVMask); // masking for Inexact and Invalid flag. + ma_b(scratch, zero, fail, Assembler::NotEqual); +} + +void MacroAssemblerLOONGARCH64Compat::convertDoubleToPtr(FloatRegister src, + Register dest, Label* fail, + bool negativeZeroCheck) { + if (negativeZeroCheck) { + moveFromDouble(src, dest); + as_rotri_d(dest, dest, 63); + ma_b(dest, Imm32(1), fail, Assembler::Equal); + } + + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + // Truncate double to int64 ; if result is inexact or invalid fail. + as_ftintrz_l_d(fpscratch, src); + as_movfcsr2gr(scratch); + moveFromDouble(fpscratch, dest); + as_bstrpick_d(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); + as_andi(scratch, scratch, + CauseIOrVMask); // masking for Inexact and Invalid flag. + ma_b(scratch, zero, fail, Assembler::NotEqual); +} + +// Checks whether a float32 is representable as a 32-bit integer. If so, the +// integer is written to the output register. Otherwise, a bailout is taken to +// the given snapshot. This function overwrites the scratch float register. +void MacroAssemblerLOONGARCH64Compat::convertFloat32ToInt32( + FloatRegister src, Register dest, Label* fail, bool negativeZeroCheck) { + if (negativeZeroCheck) { + moveFromFloat32(src, dest); + ma_b(dest, Imm32(INT32_MIN), fail, Assembler::Equal); + } + + ScratchRegisterScope scratch(asMasm()); + ScratchFloat32Scope fpscratch(asMasm()); + as_ftintrz_w_s(fpscratch, src); + as_movfcsr2gr(scratch); + moveFromFloat32(fpscratch, dest); + MOZ_ASSERT(CauseBitPos + CauseBitCount < 33); + MOZ_ASSERT(CauseBitPos < 32); + as_bstrpick_w(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); + as_andi(scratch, scratch, CauseIOrVMask); + ma_b(scratch, zero, fail, Assembler::NotEqual); +} + +void MacroAssemblerLOONGARCH64Compat::convertFloat32ToDouble(FloatRegister src, + FloatRegister dest) { + as_fcvt_d_s(dest, src); +} + +void MacroAssemblerLOONGARCH64Compat::convertInt32ToFloat32(Register src, + FloatRegister dest) { + as_movgr2fr_w(dest, src); + as_ffint_s_w(dest, dest); +} + +void MacroAssemblerLOONGARCH64Compat::convertInt32ToFloat32(const Address& src, + FloatRegister dest) { + ma_fld_s(dest, src); + as_ffint_s_w(dest, dest); +} + +void MacroAssemblerLOONGARCH64Compat::movq(Register rj, Register rd) { + as_or(rd, rj, zero); +} + +void MacroAssemblerLOONGARCH64::ma_li(Register dest, CodeLabel* label) { + BufferOffset bo = m_buffer.nextOffset(); + ma_liPatchable(dest, ImmWord(/* placeholder */ 0)); + label->patchAt()->bind(bo.getOffset()); +} + +void MacroAssemblerLOONGARCH64::ma_li(Register dest, ImmWord imm) { + int64_t value = imm.value; + + if (-1 == (value >> 11) || 0 == (value >> 11)) { + as_addi_w(dest, zero, value); + return; + } + + if (0 == (value >> 12)) { + as_ori(dest, zero, value); + return; + } + + if (-1 == (value >> 31) || 0 == (value >> 31)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + } else if (0 == (value >> 32)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + as_bstrins_d(dest, zero, 63, 32); + } else if (-1 == (value >> 51) || 0 == (value >> 51)) { + if (is_uintN((value >> 12) & 0xfffff, 20)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + } + as_lu32i_d(dest, (value >> 32) & 0xfffff); + } else if (0 == (value >> 52)) { + if (is_uintN((value >> 12) & 0xfffff, 20)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + } + as_lu32i_d(dest, (value >> 32) & 0xfffff); + as_bstrins_d(dest, zero, 63, 52); + } else { + if (is_uintN((value >> 12) & 0xfffff, 20)) { + as_lu12i_w(dest, (value >> 12) & 0xfffff); + } + if (is_uintN((value >> 32) & 0xfffff, 20)) { + as_lu32i_d(dest, (value >> 32) & 0xfffff); + } + as_lu52i_d(dest, dest, (value >> 52) & 0xfff); + } + + if (is_uintN(value & 0xfff, 12)) { + as_ori(dest, dest, value & 0xfff); + } +} + +// This method generates lu32i_d, lu12i_w and ori instruction block that can be +// modified by UpdateLoad64Value, either during compilation (eg. +// Assembler::bind), or during execution (eg. jit::PatchJump). +void MacroAssemblerLOONGARCH64::ma_liPatchable(Register dest, ImmPtr imm) { + return ma_liPatchable(dest, ImmWord(uintptr_t(imm.value))); +} + +void MacroAssemblerLOONGARCH64::ma_liPatchable(Register dest, ImmWord imm, + LiFlags flags) { + // hi12, hi20, low20, low12 + if (Li64 == flags) { // Li64: Imm data + m_buffer.ensureSpace(4 * sizeof(uint32_t)); + as_lu12i_w(dest, imm.value >> 12 & 0xfffff); // low20 + as_ori(dest, dest, imm.value & 0xfff); // low12 + as_lu32i_d(dest, imm.value >> 32 & 0xfffff); // hi20 + as_lu52i_d(dest, dest, imm.value >> 52 & 0xfff); // hi12 + } else { // Li48 address + m_buffer.ensureSpace(3 * sizeof(uint32_t)); + as_lu12i_w(dest, imm.value >> 12 & 0xfffff); // low20 + as_ori(dest, dest, imm.value & 0xfff); // low12 + as_lu32i_d(dest, imm.value >> 32 & 0xfffff); // hi20 + } +} + +// Memory access ops. + +void MacroAssemblerLOONGARCH64::ma_ld_b(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_b(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_b(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_b(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_ld_bu(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_bu(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_bu(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_bu(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_ld_h(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_h(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_h(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_h(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_ld_hu(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_hu(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_hu(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_hu(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_ld_w(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_w(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_w(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_w(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_ld_wu(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_wu(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_wu(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_wu(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_ld_d(Register dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_ld_d(dest, base, offset); + } else if (base != dest) { + ma_li(dest, Imm32(offset)); + as_ldx_d(dest, base, dest); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_ldx_d(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_st_b(Register src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_st_b(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_stx_b(src, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_st_h(Register src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_st_h(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_stx_h(src, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_st_w(Register src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_st_w(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_stx_w(src, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_st_d(Register src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_st_d(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_stx_d(src, base, scratch); + } +} + +// Arithmetic-based ops. + +// Add. +void MacroAssemblerLOONGARCH64::ma_add_d(Register rd, Register rj, Imm32 imm) { + if (is_intN(imm.value, 12)) { + as_addi_d(rd, rj, imm.value); + } else if (rd != rj) { + ma_li(rd, imm); + as_add_d(rd, rj, rd); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_add_d(rd, rj, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_add32TestOverflow(Register rd, Register rj, + Register rk, Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + as_add_d(scratch, rj, rk); + as_add_w(rd, rj, rk); + ma_b(rd, Register(scratch), overflow, Assembler::NotEqual); +} + +void MacroAssemblerLOONGARCH64::ma_add32TestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + // Check for signed range because of as_addi_d + if (is_intN(imm.value, 12)) { + ScratchRegisterScope scratch(asMasm()); + as_addi_d(scratch, rj, imm.value); + as_addi_w(rd, rj, imm.value); + ma_b(rd, scratch, overflow, Assembler::NotEqual); + } else { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_add32TestOverflow(rd, rj, scratch2, overflow); + } +} + +void MacroAssemblerLOONGARCH64::ma_addPtrTestOverflow(Register rd, Register rj, + Register rk, + Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rd != scratch); + + if (rj == rk) { + if (rj == rd) { + as_or(scratch, rj, zero); + rj = scratch; + } + + as_add_d(rd, rj, rj); + as_xor(scratch, rj, rd); + ma_b(scratch, zero, overflow, Assembler::LessThan); + } else { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rj != scratch); + MOZ_ASSERT(rd != scratch2); + + if (rj == rd) { + as_or(scratch2, rj, zero); + rj = scratch2; + } + + as_add_d(rd, rj, rk); + as_slti(scratch, rj, 0); + as_slt(scratch2, rd, rj); + ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); + } +} + +void MacroAssemblerLOONGARCH64::ma_addPtrTestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + + if (imm.value == 0) { + as_ori(rd, rj, 0); + return; + } + + if (rj == rd) { + as_ori(scratch2, rj, 0); + rj = scratch2; + } + + ma_add_d(rd, rj, imm); + + if (imm.value > 0) { + ma_b(rd, rj, overflow, Assembler::LessThan); + } else { + MOZ_ASSERT(imm.value < 0); + ma_b(rd, rj, overflow, Assembler::GreaterThan); + } +} + +void MacroAssemblerLOONGARCH64::ma_addPtrTestOverflow(Register rd, Register rj, + ImmWord imm, + Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + + if (imm.value == 0) { + as_ori(rd, rj, 0); + return; + } + + if (rj == rd) { + MOZ_ASSERT(rj != scratch2); + as_ori(scratch2, rj, 0); + rj = scratch2; + } + + ma_li(rd, imm); + as_add_d(rd, rj, rd); + + if (imm.value > 0) { + ma_b(rd, rj, overflow, Assembler::LessThan); + } else { + MOZ_ASSERT(imm.value < 0); + ma_b(rd, rj, overflow, Assembler::GreaterThan); + } +} + +void MacroAssemblerLOONGARCH64::ma_addPtrTestCarry(Condition cond, Register rd, + Register rj, Register rk, + Label* label) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rd != rk); + MOZ_ASSERT(rd != scratch); + as_add_d(rd, rj, rk); + as_sltu(scratch, rd, rk); + ma_b(scratch, Register(scratch), label, + cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); +} + +void MacroAssemblerLOONGARCH64::ma_addPtrTestCarry(Condition cond, Register rd, + Register rj, Imm32 imm, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + + // Check for signed range because of as_addi_d + if (is_intN(imm.value, 12)) { + as_addi_d(rd, rj, imm.value); + as_sltui(scratch2, rd, imm.value); + ma_b(scratch2, scratch2, label, + cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); + } else { + ma_li(scratch2, imm); + ma_addPtrTestCarry(cond, rd, rj, scratch2, label); + } +} + +void MacroAssemblerLOONGARCH64::ma_addPtrTestCarry(Condition cond, Register rd, + Register rj, ImmWord imm, + Label* label) { + SecondScratchRegisterScope scratch2(asMasm()); + + // Check for signed range because of as_addi_d + if (is_intN(imm.value, 12)) { + as_addi_d(rd, rj, imm.value); + as_sltui(scratch2, rd, imm.value); + ma_b(scratch2, scratch2, label, + cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); + } else { + ma_li(scratch2, imm); + ma_addPtrTestCarry(cond, rd, rj, scratch2, label); + } +} + +// Subtract. +void MacroAssemblerLOONGARCH64::ma_sub_d(Register rd, Register rj, Imm32 imm) { + if (is_intN(-imm.value, 12)) { + as_addi_d(rd, rj, -imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + as_sub_d(rd, rj, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_sub32TestOverflow(Register rd, Register rj, + Register rk, Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + as_sub_d(scratch, rj, rk); + as_sub_w(rd, rj, rk); + ma_b(rd, Register(scratch), overflow, Assembler::NotEqual); +} + +void MacroAssemblerLOONGARCH64::ma_subPtrTestOverflow(Register rd, Register rj, + Register rk, + Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT_IF(rj == rd, rj != rk); + MOZ_ASSERT(rj != scratch2); + MOZ_ASSERT(rk != scratch2); + MOZ_ASSERT(rd != scratch2); + + Register rj_copy = rj; + + if (rj == rd) { + as_or(scratch2, rj, zero); + rj_copy = scratch2; + } + + { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rd != scratch); + + as_sub_d(rd, rj, rk); + // If the sign of rj and rk are the same, no overflow + as_xor(scratch, rj_copy, rk); + // Check if the sign of rd and rj are the same + as_xor(scratch2, rd, rj_copy); + as_and(scratch2, scratch2, scratch); + } + + ma_b(scratch2, zero, overflow, Assembler::LessThan); +} + +void MacroAssemblerLOONGARCH64::ma_subPtrTestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + // TODO(loongarch64): Check subPtrTestOverflow + MOZ_ASSERT(imm.value != INT32_MIN); + ma_addPtrTestOverflow(rd, rj, Imm32(-imm.value), overflow); +} + +void MacroAssemblerLOONGARCH64::ma_mul_d(Register rd, Register rj, Imm32 imm) { + // li handles the relocation. + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_mul_d(rd, rj, scratch); +} + +void MacroAssemblerLOONGARCH64::ma_mulh_d(Register rd, Register rj, Imm32 imm) { + // li handles the relocation. + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_mulh_d(rd, rj, scratch); +} + +void MacroAssemblerLOONGARCH64::ma_mulPtrTestOverflow(Register rd, Register rj, + Register rk, + Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rd != scratch); + + if (rd == rj) { + as_or(scratch, rj, zero); + rj = scratch; + rk = (rd == rk) ? rj : rk; + } else if (rd == rk) { + as_or(scratch, rk, zero); + rk = scratch; + } + + as_mul_d(rd, rj, rk); + as_mulh_d(scratch, rj, rk); + as_srai_d(scratch2, rd, 63); + ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); +} + +// Memory. + +void MacroAssemblerLOONGARCH64::ma_load(Register dest, Address address, + LoadStoreSize size, + LoadStoreExtension extension) { + int32_t encodedOffset; + Register base; + + // TODO: use as_ldx_b/h/w/d, could decrease as_add_d instr. + switch (size) { + case SizeByte: + case SizeHalfWord: + if (!is_intN(address.offset, 12)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeByte) { + if (ZeroExtend == extension) { + as_ld_bu(dest, base, encodedOffset); + } else { + as_ld_b(dest, base, encodedOffset); + } + } else { + if (ZeroExtend == extension) { + as_ld_hu(dest, base, encodedOffset); + } else { + as_ld_h(dest, base, encodedOffset); + } + } + break; + case SizeWord: + case SizeDouble: + if ((address.offset & 0x3) == 0 && + (size == SizeDouble || + (size == SizeWord && SignExtend == extension))) { + if (!Imm16::IsInSignedRange(address.offset)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeWord) { + as_ldptr_w(dest, base, encodedOffset); + } else { + as_ldptr_d(dest, base, encodedOffset); + } + } else { + if (!is_intN(address.offset, 12)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeWord) { + if (ZeroExtend == extension) { + as_ld_wu(dest, base, encodedOffset); + } else { + as_ld_w(dest, base, encodedOffset); + } + } else { + as_ld_d(dest, base, encodedOffset); + } + } + break; + default: + MOZ_CRASH("Invalid argument for ma_load"); + } +} + +void MacroAssemblerLOONGARCH64::ma_store(Register data, Address address, + LoadStoreSize size, + LoadStoreExtension extension) { + int32_t encodedOffset; + Register base; + + // TODO: use as_stx_b/h/w/d, could decrease as_add_d instr. + switch (size) { + case SizeByte: + case SizeHalfWord: + if (!is_intN(address.offset, 12)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeByte) { + as_st_b(data, base, encodedOffset); + } else { + as_st_h(data, base, encodedOffset); + } + break; + case SizeWord: + case SizeDouble: + if ((address.offset & 0x3) == 0) { + if (!Imm16::IsInSignedRange(address.offset)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeWord) { + as_stptr_w(data, base, encodedOffset); + } else { + as_stptr_d(data, base, encodedOffset); + } + } else { + if (!is_intN(address.offset, 12)) { + ma_li(ScratchRegister, Imm32(address.offset)); + as_add_d(ScratchRegister, address.base, ScratchRegister); + base = ScratchRegister; + encodedOffset = 0; + } else { + encodedOffset = address.offset; + base = address.base; + } + + if (size == SizeWord) { + as_st_w(data, base, encodedOffset); + } else { + as_st_d(data, base, encodedOffset); + } + } + break; + default: + MOZ_CRASH("Invalid argument for ma_store"); + } +} + +void MacroAssemblerLOONGARCH64Compat::computeScaledAddress(const BaseIndex& address, + Register dest) { + Register base = address.base; + Register index = address.index; + int32_t shift = Imm32::ShiftOf(address.scale).value; + + if (shift) { + MOZ_ASSERT(shift <= 4); + as_alsl_d(dest, index, base, shift - 1); + } else { + as_add_d(dest, base, index); + } +} + +void MacroAssemblerLOONGARCH64::ma_pop(Register r) { + MOZ_ASSERT(r != StackPointer); + as_ld_d(r, StackPointer, 0); + as_addi_d(StackPointer, StackPointer, sizeof(intptr_t)); +} + +void MacroAssemblerLOONGARCH64::ma_push(Register r) { + if (r == StackPointer) { + ScratchRegisterScope scratch(asMasm()); + as_or(scratch, r, zero); + as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(intptr_t)); + as_st_d(scratch, StackPointer, 0); + } else { + as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(intptr_t)); + as_st_d(r, StackPointer, 0); + } +} + +// Branches when done from within loongarch-specific code. +void MacroAssemblerLOONGARCH64::ma_b(Register lhs, ImmWord imm, Label* label, + Condition c, JumpKind jumpKind) { + if (imm.value <= INT32_MAX) { + ma_b(lhs, Imm32(uint32_t(imm.value)), label, c, jumpKind); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + ma_li(scratch, imm); + ma_b(lhs, Register(scratch), label, c, jumpKind); + } +} + +void MacroAssemblerLOONGARCH64::ma_b(Register lhs, Address addr, Label* label, + Condition c, JumpKind jumpKind) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + ma_ld_d(scratch, addr); + ma_b(lhs, Register(scratch), label, c, jumpKind); +} + +void MacroAssemblerLOONGARCH64::ma_b(Address addr, Imm32 imm, Label* label, + Condition c, JumpKind jumpKind) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_ld_d(scratch2, addr); + ma_b(Register(scratch2), imm, label, c, jumpKind); +} + +void MacroAssemblerLOONGARCH64::ma_b(Address addr, ImmGCPtr imm, Label* label, + Condition c, JumpKind jumpKind) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_ld_d(scratch2, addr); + ma_b(Register(scratch2), imm, label, c, jumpKind); +} + +void MacroAssemblerLOONGARCH64::ma_bl(Label* label) { + spew("branch .Llabel %p\n", label); + if (label->bound()) { + // Generate the long jump for calls because return address has to be + // the address after the reserved block. + addLongJump(nextOffset(), BufferOffset(label->offset())); + ScratchRegisterScope scratch(asMasm()); + ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); + as_jirl(ra, scratch, BOffImm16(0)); + return; + } + + // Second word holds a pointer to the next branch in label's chain. + uint32_t nextInChain = + label->used() ? label->offset() : LabelBase::INVALID_OFFSET; + + // Make the whole branch continous in the buffer. The '5' + // instructions are writing at below. + m_buffer.ensureSpace(5 * sizeof(uint32_t)); + + spew("bal .Llabel %p\n", label); + BufferOffset bo = writeInst(getBranchCode(BranchIsCall).encode()); + writeInst(nextInChain); + if (!oom()) { + label->use(bo.getOffset()); + } + // Leave space for long jump. + as_nop(); + as_nop(); + as_nop(); +} + +void MacroAssemblerLOONGARCH64::branchWithCode(InstImm code, Label* label, + JumpKind jumpKind) { + // simply output the pointer of one label as its id, + // notice that after one label destructor, the pointer will be reused. + spew("branch .Llabel %p", label); + MOZ_ASSERT(code.encode() != + InstImm(op_jirl, BOffImm16(0), zero, ra).encode()); + InstImm inst_beq = InstImm(op_beq, BOffImm16(0), zero, zero); + + if (label->bound()) { + int32_t offset = label->offset() - m_buffer.nextOffset().getOffset(); + + if (BOffImm16::IsInRange(offset)) { + jumpKind = ShortJump; + } + + // ShortJump + if (jumpKind == ShortJump) { + MOZ_ASSERT(BOffImm16::IsInRange(offset)); + + if (code.extractBitField(31, 26) == ((uint32_t)op_bcz >> 26)) { + code.setImm21(offset); + } else { + code.setBOffImm16(BOffImm16(offset)); + } +#ifdef JS_JITSPEW + decodeBranchInstAndSpew(code); +#endif + writeInst(code.encode()); + return; + } + + // LongJump + if (code.encode() == inst_beq.encode()) { + // Handle long jump + addLongJump(nextOffset(), BufferOffset(label->offset())); + ScratchRegisterScope scratch(asMasm()); + ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); + as_jirl(zero, scratch, BOffImm16(0)); // jr scratch + as_nop(); + return; + } + + // OpenLongJump + // Handle long conditional branch, the target offset is based on self, + // point to next instruction of nop at below. + spew("invert branch .Llabel %p", label); + InstImm code_r = invertBranch(code, BOffImm16(5 * sizeof(uint32_t))); +#ifdef JS_JITSPEW + decodeBranchInstAndSpew(code_r); +#endif + writeInst(code_r.encode()); + addLongJump(nextOffset(), BufferOffset(label->offset())); + ScratchRegisterScope scratch(asMasm()); + ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); + as_jirl(zero, scratch, BOffImm16(0)); + as_nop(); + return; + } + + // Generate open jump and link it to a label. + + // Second word holds a pointer to the next branch in label's chain. + uint32_t nextInChain = + label->used() ? label->offset() : LabelBase::INVALID_OFFSET; + + if (jumpKind == ShortJump) { + // Make the whole branch continous in the buffer. + m_buffer.ensureSpace(2 * sizeof(uint32_t)); + + // Indicate that this is short jump with offset 4. + code.setBOffImm16(BOffImm16(4)); +#ifdef JS_JITSPEW + decodeBranchInstAndSpew(code); +#endif + BufferOffset bo = writeInst(code.encode()); + writeInst(nextInChain); + if (!oom()) { + label->use(bo.getOffset()); + } + return; + } + + bool conditional = code.encode() != inst_beq.encode(); + + // Make the whole branch continous in the buffer. The '5' + // instructions are writing at below (contain conditional nop). + m_buffer.ensureSpace(5 * sizeof(uint32_t)); + +#ifdef JS_JITSPEW + decodeBranchInstAndSpew(code); +#endif + BufferOffset bo = writeInst(code.encode()); // invert + writeInst(nextInChain); + if (!oom()) { + label->use(bo.getOffset()); + } + // Leave space for potential long jump. + as_nop(); + as_nop(); + if (conditional) { + as_nop(); + } +} + +void MacroAssemblerLOONGARCH64::ma_cmp_set(Register rd, Register rj, ImmWord imm, + Condition c) { + if (imm.value <= INT32_MAX) { + ma_cmp_set(rd, rj, Imm32(uint32_t(imm.value)), c); + } else { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_cmp_set(rd, rj, scratch, c); + } +} + +void MacroAssemblerLOONGARCH64::ma_cmp_set(Register rd, Register rj, ImmPtr imm, + Condition c) { + ma_cmp_set(rd, rj, ImmWord(uintptr_t(imm.value)), c); +} + +void MacroAssemblerLOONGARCH64::ma_cmp_set(Register rd, Address address, Imm32 imm, + Condition c) { + // TODO(loongarch64): 32-bit ma_cmp_set? + SecondScratchRegisterScope scratch2(asMasm()); + ma_ld_w(scratch2, address); + ma_cmp_set(rd, Register(scratch2), imm, c); +} + +void MacroAssemblerLOONGARCH64::ma_cmp_set(Register rd, Address address, + ImmWord imm, Condition c) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_ld_d(scratch2, address); + ma_cmp_set(rd, Register(scratch2), imm, c); +} + +// fp instructions +void MacroAssemblerLOONGARCH64::ma_lid(FloatRegister dest, double value) { + ImmWord imm(mozilla::BitwiseCast(value)); + + if (imm.value != 0) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + moveToDouble(scratch, dest); + } else { + moveToDouble(zero, dest); + } +} + +void MacroAssemblerLOONGARCH64::ma_mv(FloatRegister src, ValueOperand dest) { + as_movfr2gr_d(dest.valueReg(), src); +} + +void MacroAssemblerLOONGARCH64::ma_mv(ValueOperand src, FloatRegister dest) { + as_movgr2fr_d(dest, src.valueReg()); +} + +void MacroAssemblerLOONGARCH64::ma_fld_s(FloatRegister dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_fld_s(dest, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_fldx_s(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_fld_d(FloatRegister dest, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_fld_d(dest, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_fldx_d(dest, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_fst_s(FloatRegister src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_fst_s(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_fstx_s(src, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_fst_d(FloatRegister src, Address address) { + int32_t offset = address.offset; + Register base = address.base; + + if (is_intN(offset, 12)) { + as_fst_d(src, base, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(base != scratch); + ma_li(scratch, Imm32(offset)); + as_fstx_d(src, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_pop(FloatRegister f) { + as_fld_d(f, StackPointer, 0); + as_addi_d(StackPointer, StackPointer, sizeof(double)); +} + +void MacroAssemblerLOONGARCH64::ma_push(FloatRegister f) { + as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(double)); + as_fst_d(f, StackPointer, 0); +} + +void MacroAssemblerLOONGARCH64::ma_li(Register dest, ImmGCPtr ptr) { + writeDataRelocation(ptr); + asMasm().ma_liPatchable(dest, ImmPtr(ptr.value)); +} + +void MacroAssemblerLOONGARCH64::ma_li(Register dest, Imm32 imm) { + if (is_intN(imm.value, 12)) { + as_addi_w(dest, zero, imm.value); + } else if (is_uintN(imm.value, 12)) { + as_ori(dest, zero, imm.value & 0xfff); + } else { + as_lu12i_w(dest, imm.value >> 12 & 0xfffff); + if (imm.value & 0xfff) { + as_ori(dest, dest, imm.value & 0xfff); + } + } +} + +// This method generates lu12i_w and ori instruction pair that can be modified +// by UpdateLuiOriValue, either during compilation (eg. Assembler::bind), or +// during execution (eg. jit::PatchJump). +void MacroAssemblerLOONGARCH64::ma_liPatchable(Register dest, Imm32 imm) { + m_buffer.ensureSpace(2 * sizeof(uint32_t)); + as_lu12i_w(dest, imm.value >> 12 & 0xfffff); + as_ori(dest, dest, imm.value & 0xfff); +} + +void MacroAssemblerLOONGARCH64::ma_fmovz(FloatFormat fmt, FloatRegister fd, + FloatRegister fj, Register rk) { + Label done; + ma_b(rk, zero, &done, Assembler::NotEqual); + if (fmt == SingleFloat) { + as_fmov_s(fd, fj); + } else { + as_fmov_d(fd, fj); + } + bind(&done); +} + +void MacroAssemblerLOONGARCH64::ma_fmovn(FloatFormat fmt, FloatRegister fd, + FloatRegister fj, Register rk) { + Label done; + ma_b(rk, zero, &done, Assembler::Equal); + if (fmt == SingleFloat) { + as_fmov_s(fd, fj); + } else { + as_fmov_d(fd, fj); + } + bind(&done); +} + +void MacroAssemblerLOONGARCH64::ma_and(Register rd, Register rj, Imm32 imm, + bool bit32) { + if (is_uintN(imm.value, 12)) { + as_andi(rd, rj, imm.value); + } else if (rd != rj) { + ma_li(rd, imm); + as_and(rd, rj, rd); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_and(rd, rj, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_or(Register rd, Register rj, Imm32 imm, + bool bit32) { + if (is_uintN(imm.value, 12)) { + as_ori(rd, rj, imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_or(rd, rj, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_xor(Register rd, Register rj, Imm32 imm, + bool bit32) { + if (is_uintN(imm.value, 12)) { + as_xori(rd, rj, imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_xor(rd, rj, scratch); + } +} + +// Arithmetic-based ops. + +// Add. +void MacroAssemblerLOONGARCH64::ma_add_w(Register rd, Register rj, Imm32 imm) { + if (is_intN(imm.value, 12)) { + as_addi_w(rd, rj, imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_add_w(rd, rj, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_add32TestCarry(Condition cond, Register rd, + Register rj, Register rk, + Label* overflow) { + MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear); + MOZ_ASSERT_IF(rd == rj, rk != rd); + ScratchRegisterScope scratch(asMasm()); + as_add_w(rd, rj, rk); + as_sltu(scratch, rd, rd == rj ? rk : rj); + ma_b(Register(scratch), Register(scratch), overflow, + cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); +} + +void MacroAssemblerLOONGARCH64::ma_add32TestCarry(Condition cond, Register rd, + Register rj, Imm32 imm, + Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(rj != scratch2); + ma_li(scratch2, imm); + ma_add32TestCarry(cond, rd, rj, scratch2, overflow); +} + +// Subtract. +void MacroAssemblerLOONGARCH64::ma_sub_w(Register rd, Register rj, Imm32 imm) { + if (is_intN(-imm.value, 12)) { + as_addi_w(rd, rj, -imm.value); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_sub_w(rd, rj, scratch); + } +} + +void MacroAssemblerLOONGARCH64::ma_sub_w(Register rd, Register rj, Register rk) { + as_sub_w(rd, rj, rk); +} + +void MacroAssemblerLOONGARCH64::ma_sub32TestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + if (imm.value != INT32_MIN) { + asMasm().ma_add32TestOverflow(rd, rj, Imm32(-imm.value), overflow); + } else { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, Imm32(imm.value)); + asMasm().ma_sub32TestOverflow(rd, rj, scratch, overflow); + } +} + +void MacroAssemblerLOONGARCH64::ma_mul(Register rd, Register rj, Imm32 imm) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rj != scratch); + ma_li(scratch, imm); + as_mul_w(rd, rj, scratch); +} + +void MacroAssemblerLOONGARCH64::ma_mul32TestOverflow(Register rd, Register rj, + Register rk, Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + as_mulh_w(scratch, rj, rk); + as_mul_w(rd, rj, rk); + as_srai_w(scratch2, rd, 31); + ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); +} + +void MacroAssemblerLOONGARCH64::ma_mul32TestOverflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch, imm); + as_mulh_w(scratch2, rj, scratch); + as_mul_w(rd, rj, scratch); + as_srai_w(scratch, rd, 31); + ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); +} + +void MacroAssemblerLOONGARCH64::ma_div_branch_overflow(Register rd, Register rj, + Register rk, + Label* overflow) { + ScratchRegisterScope scratch(asMasm()); + as_mod_w(scratch, rj, rk); + ma_b(scratch, scratch, overflow, Assembler::NonZero); + as_div_w(rd, rj, rk); +} + +void MacroAssemblerLOONGARCH64::ma_div_branch_overflow(Register rd, Register rj, + Imm32 imm, Label* overflow) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_div_branch_overflow(rd, rj, scratch2, overflow); +} + +void MacroAssemblerLOONGARCH64::ma_mod_mask(Register src, Register dest, + Register hold, Register remain, + int32_t shift, Label* negZero) { + // MATH: + // We wish to compute x % (1< 0, store sum - C back into sum, thus performing a + // modulus. + ma_b(scratch2, Register(scratch2), &sumSigned, Signed, ShortJump); + as_or(dest, scratch2, zero); + bind(&sumSigned); + // Get rid of the bits that we extracted before. + as_srli_w(remain, remain, shift); + // If the shift produced zero, finish, otherwise, continue in the loop. + ma_b(remain, remain, &head, NonZero, ShortJump); + // Check the hold to see if we need to negate the result. + ma_b(hold, hold, &done, NotSigned, ShortJump); + + // If the hold was non-zero, negate the result to be in line with + // what JS wants + if (negZero != nullptr) { + // Jump out in case of negative zero. + ma_b(hold, hold, negZero, Zero); + as_sub_w(dest, zero, dest); + } else { + as_sub_w(dest, zero, dest); + } + + bind(&done); +} + +// Memory. + +void MacroAssemblerLOONGARCH64::ma_load(Register dest, const BaseIndex& src, + LoadStoreSize size, + LoadStoreExtension extension) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(src, scratch2); + asMasm().ma_load(dest, Address(scratch2, src.offset), size, extension); +} + +void MacroAssemblerLOONGARCH64::ma_store(Register data, const BaseIndex& dest, + LoadStoreSize size, + LoadStoreExtension extension) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(dest, scratch2); + asMasm().ma_store(data, Address(scratch2, dest.offset), size, extension); +} + +void MacroAssemblerLOONGARCH64::ma_store(Imm32 imm, const BaseIndex& dest, + LoadStoreSize size, + LoadStoreExtension extension) { + SecondScratchRegisterScope scratch2(asMasm()); + // Make sure that scratch2 contains absolute address so that offset is 0. + asMasm().computeEffectiveAddress(dest, scratch2); + + ScratchRegisterScope scratch(asMasm()); + // Scrach register is free now, use it for loading imm value + ma_li(scratch, imm); + + // with offset=0 ScratchRegister will not be used in ma_store() + // so we can use it as a parameter here + asMasm().ma_store(scratch, Address(scratch2, 0), size, extension); +} + +// Branches when done from within loongarch-specific code. +// TODO(loongarch64) Optimize ma_b +void MacroAssemblerLOONGARCH64::ma_b(Register lhs, Register rhs, Label* label, + Condition c, JumpKind jumpKind) { + switch (c) { + case Equal: + case NotEqual: + asMasm().branchWithCode(getBranchCode(lhs, rhs, c), label, jumpKind); + break; + case Always: + ma_b(label, jumpKind); + break; + case Zero: + case NonZero: + case Signed: + case NotSigned: + MOZ_ASSERT(lhs == rhs); + asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind); + break; + default: { + Condition cond = ma_cmp(ScratchRegister, lhs, rhs, c); + asMasm().branchWithCode(getBranchCode(ScratchRegister, cond), label, + jumpKind); + break; + } + } +} + +void MacroAssemblerLOONGARCH64::ma_b(Register lhs, Imm32 imm, Label* label, + Condition c, JumpKind jumpKind) { + MOZ_ASSERT(c != Overflow); + if (imm.value == 0) { + if (c == Always || c == AboveOrEqual) { + ma_b(label, jumpKind); + } else if (c == Below) { + ; // This condition is always false. No branch required. + } else { + asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind); + } + } else { + switch (c) { + case Equal: + case NotEqual: + MOZ_ASSERT(lhs != ScratchRegister); + ma_li(ScratchRegister, imm); + ma_b(lhs, ScratchRegister, label, c, jumpKind); + break; + default: + Condition cond = ma_cmp(ScratchRegister, lhs, imm, c); + asMasm().branchWithCode(getBranchCode(ScratchRegister, cond), label, + jumpKind); + } + } +} + +void MacroAssemblerLOONGARCH64::ma_b(Register lhs, ImmPtr imm, Label* l, + Condition c, JumpKind jumpKind) { + asMasm().ma_b(lhs, ImmWord(uintptr_t(imm.value)), l, c, jumpKind); +} + +void MacroAssemblerLOONGARCH64::ma_b(Label* label, JumpKind jumpKind) { + asMasm().branchWithCode(getBranchCode(BranchIsJump), label, jumpKind); +} + +Assembler::Condition MacroAssemblerLOONGARCH64::ma_cmp(Register dest, Register lhs, + Register rhs, Condition c) { + switch (c) { + case Above: + // bgtu s,t,label => + // sltu at,t,s + // bne at,$zero,offs + as_sltu(dest, rhs, lhs); + return NotEqual; + case AboveOrEqual: + // bgeu s,t,label => + // sltu at,s,t + // beq at,$zero,offs + as_sltu(dest, lhs, rhs); + return Equal; + case Below: + // bltu s,t,label => + // sltu at,s,t + // bne at,$zero,offs + as_sltu(dest, lhs, rhs); + return NotEqual; + case BelowOrEqual: + // bleu s,t,label => + // sltu at,t,s + // beq at,$zero,offs + as_sltu(dest, rhs, lhs); + return Equal; + case GreaterThan: + // bgt s,t,label => + // slt at,t,s + // bne at,$zero,offs + as_slt(dest, rhs, lhs); + return NotEqual; + case GreaterThanOrEqual: + // bge s,t,label => + // slt at,s,t + // beq at,$zero,offs + as_slt(dest, lhs, rhs); + return Equal; + case LessThan: + // blt s,t,label => + // slt at,s,t + // bne at,$zero,offs + as_slt(dest, lhs, rhs); + return NotEqual; + case LessThanOrEqual: + // ble s,t,label => + // slt at,t,s + // beq at,$zero,offs + as_slt(dest, rhs, lhs); + return Equal; + default: + MOZ_CRASH("Invalid condition."); + } + return Always; +} + +Assembler::Condition MacroAssemblerLOONGARCH64::ma_cmp(Register dest, Register lhs, + Imm32 imm, Condition c) { + ScratchRegisterScope scratch(asMasm()); + MOZ_RELEASE_ASSERT(lhs != scratch); + + switch (c) { + case Above: + case BelowOrEqual: + if (imm.value != 0x7fffffff && is_intN(imm.value + 1, 12) && + imm.value != -1) { + // lhs <= rhs via lhs < rhs + 1 if rhs + 1 does not overflow + as_sltui(dest, lhs, imm.value + 1); + + return (c == BelowOrEqual ? NotEqual : Equal); + } else { + ma_li(scratch, imm); + as_sltu(dest, scratch, lhs); + return (c == BelowOrEqual ? Equal : NotEqual); + } + case AboveOrEqual: + case Below: + if (is_intN(imm.value, 12)) { + as_sltui(dest, lhs, imm.value); + } else { + ma_li(scratch, imm); + as_sltu(dest, lhs, scratch); + } + return (c == AboveOrEqual ? Equal : NotEqual); + case GreaterThan: + case LessThanOrEqual: + if (imm.value != 0x7fffffff && is_intN(imm.value + 1, 12)) { + // lhs <= rhs via lhs < rhs + 1. + as_slti(dest, lhs, imm.value + 1); + return (c == LessThanOrEqual ? NotEqual : Equal); + } else { + ma_li(scratch, imm); + as_slt(dest, scratch, lhs); + return (c == LessThanOrEqual ? Equal : NotEqual); + } + case GreaterThanOrEqual: + case LessThan: + if (is_intN(imm.value, 12)) { + as_slti(dest, lhs, imm.value); + } else { + ma_li(scratch, imm); + as_slt(dest, lhs, scratch); + } + return (c == GreaterThanOrEqual ? Equal : NotEqual); + default: + MOZ_CRASH("Invalid condition."); + } + return Always; +} + +// fp instructions +void MacroAssemblerLOONGARCH64::ma_lis(FloatRegister dest, float value) { + Imm32 imm(mozilla::BitwiseCast(value)); + + if (imm.value != 0) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + moveToFloat32(scratch, dest); + } else { + moveToFloat32(zero, dest); + } +} + +void MacroAssemblerLOONGARCH64::ma_fst_d(FloatRegister ft, BaseIndex address) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(address, scratch2); + asMasm().ma_fst_d(ft, Address(scratch2, address.offset)); +} + +void MacroAssemblerLOONGARCH64::ma_fst_s(FloatRegister ft, BaseIndex address) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(address, scratch2); + asMasm().ma_fst_s(ft, Address(scratch2, address.offset)); +} + +void MacroAssemblerLOONGARCH64::ma_fld_d(FloatRegister ft, const BaseIndex& src) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(src, scratch2); + asMasm().ma_fld_d(ft, Address(scratch2, src.offset)); +} + +void MacroAssemblerLOONGARCH64::ma_fld_s(FloatRegister ft, const BaseIndex& src) { + SecondScratchRegisterScope scratch2(asMasm()); + asMasm().computeScaledAddress(src, scratch2); + asMasm().ma_fld_s(ft, Address(scratch2, src.offset)); +} + +void MacroAssemblerLOONGARCH64::ma_bc_s(FloatRegister lhs, FloatRegister rhs, + Label* label, DoubleCondition c, + JumpKind jumpKind, FPConditionBit fcc) { + compareFloatingPoint(SingleFloat, lhs, rhs, c, fcc); + asMasm().branchWithCode(getBranchCode(fcc), label, jumpKind); +} + +void MacroAssemblerLOONGARCH64::ma_bc_d(FloatRegister lhs, FloatRegister rhs, + Label* label, DoubleCondition c, + JumpKind jumpKind, FPConditionBit fcc) { + compareFloatingPoint(DoubleFloat, lhs, rhs, c, fcc); + asMasm().branchWithCode(getBranchCode(fcc), label, jumpKind); +} + +void MacroAssemblerLOONGARCH64::ma_call(ImmPtr dest) { + asMasm().ma_liPatchable(CallReg, dest); + as_jirl(ra, CallReg, BOffImm16(0)); +} + +void MacroAssemblerLOONGARCH64::ma_jump(ImmPtr dest) { + ScratchRegisterScope scratch(asMasm()); + asMasm().ma_liPatchable(scratch, dest); + as_jirl(zero, scratch, BOffImm16(0)); +} + +void MacroAssemblerLOONGARCH64::ma_cmp_set(Register rd, Register rj, Register rk, + Condition c) { + switch (c) { + case Equal: + // seq d,s,t => + // xor d,s,t + // sltiu d,d,1 + as_xor(rd, rj, rk); + as_sltui(rd, rd, 1); + break; + case NotEqual: + // sne d,s,t => + // xor d,s,t + // sltu d,$zero,d + as_xor(rd, rj, rk); + as_sltu(rd, zero, rd); + break; + case Above: + // sgtu d,s,t => + // sltu d,t,s + as_sltu(rd, rk, rj); + break; + case AboveOrEqual: + // sgeu d,s,t => + // sltu d,s,t + // xori d,d,1 + as_sltu(rd, rj, rk); + as_xori(rd, rd, 1); + break; + case Below: + // sltu d,s,t + as_sltu(rd, rj, rk); + break; + case BelowOrEqual: + // sleu d,s,t => + // sltu d,t,s + // xori d,d,1 + as_sltu(rd, rk, rj); + as_xori(rd, rd, 1); + break; + case GreaterThan: + // sgt d,s,t => + // slt d,t,s + as_slt(rd, rk, rj); + break; + case GreaterThanOrEqual: + // sge d,s,t => + // slt d,s,t + // xori d,d,1 + as_slt(rd, rj, rk); + as_xori(rd, rd, 1); + break; + case LessThan: + // slt d,s,t + as_slt(rd, rj, rk); + break; + case LessThanOrEqual: + // sle d,s,t => + // slt d,t,s + // xori d,d,1 + as_slt(rd, rk, rj); + as_xori(rd, rd, 1); + break; + case Zero: + MOZ_ASSERT(rj == rk); + // seq d,s,$zero => + // sltiu d,s,1 + as_sltui(rd, rj, 1); + break; + case NonZero: + MOZ_ASSERT(rj == rk); + // sne d,s,$zero => + // sltu d,$zero,s + as_sltu(rd, zero, rj); + break; + case Signed: + MOZ_ASSERT(rj == rk); + as_slt(rd, rj, zero); + break; + case NotSigned: + MOZ_ASSERT(rj == rk); + // sge d,s,$zero => + // slt d,s,$zero + // xori d,d,1 + as_slt(rd, rj, zero); + as_xori(rd, rd, 1); + break; + default: + MOZ_CRASH("Invalid condition."); + } +} + +void MacroAssemblerLOONGARCH64::ma_cmp_set_double(Register dest, FloatRegister lhs, + FloatRegister rhs, + DoubleCondition c) { + compareFloatingPoint(DoubleFloat, lhs, rhs, c); + as_movcf2gr(dest, FCC0); +} + +void MacroAssemblerLOONGARCH64::ma_cmp_set_float32(Register dest, FloatRegister lhs, + FloatRegister rhs, + DoubleCondition c) { + compareFloatingPoint(SingleFloat, lhs, rhs, c); + as_movcf2gr(dest, FCC0); +} + +void MacroAssemblerLOONGARCH64::ma_cmp_set(Register rd, Register rj, Imm32 imm, + Condition c) { + if (imm.value == 0) { + switch (c) { + case Equal: + case BelowOrEqual: + as_sltui(rd, rj, 1); + break; + case NotEqual: + case Above: + as_sltu(rd, zero, rj); + break; + case AboveOrEqual: + case Below: + as_ori(rd, zero, c == AboveOrEqual ? 1 : 0); + break; + case GreaterThan: + case LessThanOrEqual: + as_slt(rd, zero, rj); + if (c == LessThanOrEqual) { + as_xori(rd, rd, 1); + } + break; + case LessThan: + case GreaterThanOrEqual: + as_slt(rd, rj, zero); + if (c == GreaterThanOrEqual) { + as_xori(rd, rd, 1); + } + break; + case Zero: + as_sltui(rd, rj, 1); + break; + case NonZero: + as_sltu(rd, zero, rj); + break; + case Signed: + as_slt(rd, rj, zero); + break; + case NotSigned: + as_slt(rd, rj, zero); + as_xori(rd, rd, 1); + break; + default: + MOZ_CRASH("Invalid condition."); + } + return; + } + + switch (c) { + case Equal: + case NotEqual: + ma_xor(rd, rj, imm); + if (c == Equal) { + as_sltui(rd, rd, 1); + } else { + as_sltu(rd, zero, rd); + } + break; + case Zero: + case NonZero: + case Signed: + case NotSigned: + MOZ_CRASH("Invalid condition."); + default: + Condition cond = ma_cmp(rd, rj, imm, c); + MOZ_ASSERT(cond == Equal || cond == NotEqual); + + if (cond == Equal) as_xori(rd, rd, 1); + } +} + +void MacroAssemblerLOONGARCH64::compareFloatingPoint(FloatFormat fmt, + FloatRegister lhs, + FloatRegister rhs, + DoubleCondition c, + FPConditionBit fcc) { + switch (c) { + case DoubleOrdered: + as_fcmp_cor(fmt, lhs, rhs, fcc); + break; + case DoubleEqual: + as_fcmp_ceq(fmt, lhs, rhs, fcc); + break; + case DoubleNotEqual: + as_fcmp_cne(fmt, lhs, rhs, fcc); + break; + case DoubleGreaterThan: + as_fcmp_clt(fmt, rhs, lhs, fcc); + break; + case DoubleGreaterThanOrEqual: + as_fcmp_cle(fmt, rhs, lhs, fcc); + break; + case DoubleLessThan: + as_fcmp_clt(fmt, lhs, rhs, fcc); + break; + case DoubleLessThanOrEqual: + as_fcmp_cle(fmt, lhs, rhs, fcc); + break; + case DoubleUnordered: + as_fcmp_cun(fmt, lhs, rhs, fcc); + break; + case DoubleEqualOrUnordered: + as_fcmp_cueq(fmt, lhs, rhs, fcc); + break; + case DoubleNotEqualOrUnordered: + as_fcmp_cune(fmt, lhs, rhs, fcc); + break; + case DoubleGreaterThanOrUnordered: + as_fcmp_cult(fmt, rhs, lhs, fcc); + break; + case DoubleGreaterThanOrEqualOrUnordered: + as_fcmp_cule(fmt, rhs, lhs, fcc); + break; + case DoubleLessThanOrUnordered: + as_fcmp_cult(fmt, lhs, rhs, fcc); + break; + case DoubleLessThanOrEqualOrUnordered: + as_fcmp_cule(fmt, lhs, rhs, fcc); + break; + default: + MOZ_CRASH("Invalid DoubleCondition."); + } +} + +void MacroAssemblerLOONGARCH64::minMaxDouble(FloatRegister srcDest, + FloatRegister second, bool handleNaN, + bool isMax) { + if (srcDest == second) return; + + Label nan, done; + + // First or second is NaN, result is NaN. + ma_bc_d(srcDest, second, &nan, Assembler::DoubleUnordered, ShortJump); + if (isMax) { + as_fmax_d(srcDest, srcDest, second); + } else { + as_fmin_d(srcDest, srcDest, second); + } + ma_b(&done, ShortJump); + + bind(&nan); + as_fadd_d(srcDest, srcDest, second); + + bind(&done); +} + +void MacroAssemblerLOONGARCH64::minMaxFloat32(FloatRegister srcDest, + FloatRegister second, bool handleNaN, + bool isMax) { + if (srcDest == second) return; + + Label nan, done; + + // First or second is NaN, result is NaN. + ma_bc_s(srcDest, second, &nan, Assembler::DoubleUnordered, ShortJump); + if (isMax) { + as_fmax_s(srcDest, srcDest, second); + } else { + as_fmin_s(srcDest, srcDest, second); + } + ma_b(&done, ShortJump); + + bind(&nan); + as_fadd_s(srcDest, srcDest, second); + + bind(&done); +} + +void MacroAssemblerLOONGARCH64::loadDouble(const Address& address, + FloatRegister dest) { + asMasm().ma_fld_d(dest, address); +} + +void MacroAssemblerLOONGARCH64::loadDouble(const BaseIndex& src, + FloatRegister dest) { + asMasm().ma_fld_d(dest, src); +} + +void MacroAssemblerLOONGARCH64::loadFloatAsDouble(const Address& address, + FloatRegister dest) { + asMasm().ma_fld_s(dest, address); + as_fcvt_d_s(dest, dest); +} + +void MacroAssemblerLOONGARCH64::loadFloatAsDouble(const BaseIndex& src, + FloatRegister dest) { + asMasm().loadFloat32(src, dest); + as_fcvt_d_s(dest, dest); +} + +void MacroAssemblerLOONGARCH64::loadFloat32(const Address& address, + FloatRegister dest) { + asMasm().ma_fld_s(dest, address); +} + +void MacroAssemblerLOONGARCH64::loadFloat32(const BaseIndex& src, + FloatRegister dest) { + asMasm().ma_fld_s(dest, src); +} + +void MacroAssemblerLOONGARCH64::wasmLoadImpl(const wasm::MemoryAccessDesc& access, + Register memoryBase, Register ptr, + Register ptrScratch, + AnyRegister output, Register tmp) { + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit()); + MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); + + // Maybe add the offset. + if (offset) { + asMasm().addPtr(ImmWord(offset), ptrScratch); + ptr = ptrScratch; + } + + asMasm().memoryBarrierBefore(access.sync()); + + switch (access.type()) { + case Scalar::Int8: + as_ldx_b(output.gpr(), memoryBase, ptr); + break; + case Scalar::Uint8: + as_ldx_bu(output.gpr(), memoryBase, ptr); + break; + case Scalar::Int16: + as_ldx_h(output.gpr(), memoryBase, ptr); + break; + case Scalar::Uint16: + as_ldx_hu(output.gpr(), memoryBase, ptr); + break; + case Scalar::Int32: + case Scalar::Uint32: + as_ldx_w(output.gpr(), memoryBase, ptr); + break; + case Scalar::Float64: + as_fldx_d(output.fpu(), memoryBase, ptr); + break; + case Scalar::Float32: + as_fldx_s(output.fpu(), memoryBase, ptr); + break; + default: + MOZ_CRASH("unexpected array type"); + } + + asMasm().append(access, asMasm().size() - 4); + asMasm().memoryBarrierAfter(access.sync()); +} + +void MacroAssemblerLOONGARCH64::wasmStoreImpl(const wasm::MemoryAccessDesc& access, + AnyRegister value, + Register memoryBase, Register ptr, + Register ptrScratch, Register tmp) { + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit()); + MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); + + // Maybe add the offset. + if (offset) { + asMasm().addPtr(ImmWord(offset), ptrScratch); + ptr = ptrScratch; + } + + asMasm().memoryBarrierBefore(access.sync()); + + switch (access.type()) { + case Scalar::Int8: + case Scalar::Uint8: + as_stx_b(value.gpr(), memoryBase, ptr); + break; + case Scalar::Int16: + case Scalar::Uint16: + as_stx_h(value.gpr(), memoryBase, ptr); + break; + case Scalar::Int32: + case Scalar::Uint32: + as_stx_w(value.gpr(), memoryBase, ptr); + break; + case Scalar::Int64: + as_stx_d(value.gpr(), memoryBase, ptr); + break; + case Scalar::Float64: + as_fstx_d(value.fpu(), memoryBase, ptr); + break; + case Scalar::Float32: + as_fstx_s(value.fpu(), memoryBase, ptr); + break; + default: + MOZ_CRASH("unexpected array type"); + } + + // Only the last emitted instruction is a memory access. + asMasm().append(access, asMasm().size() - 4); + asMasm().memoryBarrierAfter(access.sync()); +} + +void MacroAssemblerLOONGARCH64Compat::wasmLoadI64Impl( + const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr, + Register ptrScratch, Register64 output, Register tmp) { + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit()); + MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); + + // Maybe add the offset. + if (offset) { + asMasm().addPtr(ImmWord(offset), ptrScratch); + ptr = ptrScratch; + } + + asMasm().memoryBarrierBefore(access.sync()); + + switch (access.type()) { + case Scalar::Int8: + as_ldx_b(output.reg, memoryBase, ptr); + break; + case Scalar::Uint8: + as_ldx_bu(output.reg, memoryBase, ptr); + break; + case Scalar::Int16: + as_ldx_h(output.reg, memoryBase, ptr); + break; + case Scalar::Uint16: + as_ldx_hu(output.reg, memoryBase, ptr); + break; + case Scalar::Int32: + as_ldx_w(output.reg, memoryBase, ptr); + break; + case Scalar::Uint32: + // TODO(loongarch64): Why need zero-extension here? + as_ldx_wu(output.reg, memoryBase, ptr); + break; + case Scalar::Int64: + as_ldx_d(output.reg, memoryBase, ptr); + break; + default: + MOZ_CRASH("unexpected array type"); + } + + asMasm().append(access, asMasm().size() - 4); + asMasm().memoryBarrierAfter(access.sync()); +} + +void MacroAssemblerLOONGARCH64Compat::wasmStoreI64Impl( + const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase, + Register ptr, Register ptrScratch, Register tmp) { + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit()); + MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); + + // Maybe add the offset. + if (offset) { + asMasm().addPtr(ImmWord(offset), ptrScratch); + ptr = ptrScratch; + } + + asMasm().memoryBarrierBefore(access.sync()); + + switch (access.type()) { + case Scalar::Int8: + case Scalar::Uint8: + as_stx_b(value.reg, memoryBase, ptr); + break; + case Scalar::Int16: + case Scalar::Uint16: + as_stx_h(value.reg, memoryBase, ptr); + break; + case Scalar::Int32: + case Scalar::Uint32: + as_stx_w(value.reg, memoryBase, ptr); + break; + case Scalar::Int64: + as_stx_d(value.reg, memoryBase, ptr); + break; + default: + MOZ_CRASH("unexpected array type"); + } + + asMasm().append(access, asMasm().size() - 4); + asMasm().memoryBarrierAfter(access.sync()); +} + +void MacroAssemblerLOONGARCH64Compat::profilerEnterFrame(Register framePtr, + Register scratch) { + asMasm().loadJSContext(scratch); + loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch); + storePtr(framePtr, + Address(scratch, JitActivation::offsetOfLastProfilingFrame())); + storePtr(ImmPtr(nullptr), + Address(scratch, JitActivation::offsetOfLastProfilingCallSite())); +} + +void MacroAssemblerLOONGARCH64Compat::profilerExitFrame() { + jump(GetJitContext()->runtime->jitRuntime()->getProfilerExitFrameTail()); +} + +MacroAssembler& MacroAssemblerLOONGARCH64::asMasm() { + return *static_cast(this); +} + +const MacroAssembler& MacroAssemblerLOONGARCH64::asMasm() const { + return *static_cast(this); +} + +void MacroAssembler::subFromStackPtr(Imm32 imm32) { + if (imm32.value) { + asMasm().subPtr(imm32, StackPointer); + } +} + +//{{{ check_macroassembler_style +// =============================================================== +// MacroAssembler high-level usage. + +void MacroAssembler::flush() {} + +// =============================================================== +// Stack manipulation functions. + +size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) { + return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); +} + +void MacroAssembler::PushRegsInMask(LiveRegisterSet set) { + int32_t diff = + set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); + const int32_t reserved = diff; + + reserveStack(reserved); + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diff -= sizeof(intptr_t); + storePtr(*iter, Address(StackPointer, diff)); + } + +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); + iter.more(); ++iter) { + diff -= sizeof(double); + storeDouble(*iter, Address(StackPointer, diff)); + } + MOZ_ASSERT(diff == 0); +} + +void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set, + LiveRegisterSet ignore) { + int32_t diff = + set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); + const int32_t reserved = diff; + + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diff -= sizeof(intptr_t); + if (!ignore.has(*iter)) { + loadPtr(Address(StackPointer, diff), *iter); + } + } + +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); + iter.more(); ++iter) { + diff -= sizeof(double); + if (!ignore.has(*iter)) { + loadDouble(Address(StackPointer, diff), *iter); + } + } + MOZ_ASSERT(diff == 0); + freeStack(reserved); +} + +void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest, + Register) { + FloatRegisterSet fpuSet(set.fpus().reduceSetForPush()); + mozilla::DebugOnly numFpu = fpuSet.size(); + int32_t diffF = fpuSet.getPushSizeInBytes(); + mozilla::DebugOnly diffG = set.gprs().size() * sizeof(intptr_t); + + MOZ_ASSERT(dest.offset >= diffG + diffF); + + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diffG -= sizeof(intptr_t); + dest.offset -= sizeof(intptr_t); + storePtr(*iter, dest); + } + MOZ_ASSERT(diffG == 0); + +#ifdef ENABLE_WASM_SIMD +# error "Needs more careful logic if SIMD is enabled" +#endif + + for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) { + FloatRegister reg = *iter; + diffF -= reg.size(); + numFpu -= 1; + dest.offset -= reg.size(); + if (reg.isDouble()) { + storeDouble(reg, dest); + } else if (reg.isSingle()) { + storeFloat32(reg, dest); + } else { + MOZ_CRASH("Unknown register type."); + } + } + MOZ_ASSERT(numFpu == 0); + diffF -= diffF % sizeof(uintptr_t); + MOZ_ASSERT(diffF == 0); +} + +void MacroAssembler::Push(Register reg) { + ma_push(reg); + adjustFrame(int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Push(const Imm32 imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + adjustFrame(int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Push(const ImmWord imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + adjustFrame(int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Push(const ImmPtr imm) { + Push(ImmWord(uintptr_t(imm.value))); +} + +void MacroAssembler::Push(const ImmGCPtr ptr) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, ptr); + ma_push(scratch); + adjustFrame(int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Push(FloatRegister f) { + ma_push(f); + adjustFrame(int32_t(sizeof(double))); +} + +void MacroAssembler::PushBoxed(FloatRegister reg) { + subFromStackPtr(Imm32(sizeof(double))); + boxDouble(reg, Address(getStackPointer(), 0)); + adjustFrame(sizeof(double)); +} + +void MacroAssembler::Pop(Register reg) { + ma_pop(reg); + adjustFrame(-int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::Pop(FloatRegister f) { + ma_pop(f); + adjustFrame(-int32_t(sizeof(double))); +} + +void MacroAssembler::Pop(const ValueOperand& val) { + popValue(val); + adjustFrame(-int32_t(sizeof(Value))); +} + +void MacroAssembler::PopStackPtr() { + loadPtr(Address(StackPointer, 0), StackPointer); + adjustFrame(-int32_t(sizeof(intptr_t))); +} + +// =============================================================== +// Simple call functions. + +CodeOffset MacroAssembler::call(Register reg) { + as_jirl(ra, reg, BOffImm16(0)); + return CodeOffset(currentOffset()); +} + +CodeOffset MacroAssembler::call(Label* label) { + ma_bl(label); + return CodeOffset(currentOffset()); +} + +CodeOffset MacroAssembler::callWithPatch() { + as_bl(JOffImm26(1 * sizeof(uint32_t))); + return CodeOffset(currentOffset()); +} + +void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) { + BufferOffset call(callerOffset - 1 * sizeof(uint32_t)); + + JOffImm26 offset = BufferOffset(calleeOffset).diffB(call); + if (!offset.isInvalid()) { + InstJump* bal = (InstJump*)editSrc(call); + bal->setJOffImm26(offset); + } else { + uint32_t u32Offset = callerOffset - 4 * sizeof(uint32_t); + uint32_t* u32 = + reinterpret_cast(editSrc(BufferOffset(u32Offset))); + *u32 = calleeOffset - callerOffset; + } +} + +CodeOffset MacroAssembler::farJumpWithPatch() { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + as_pcaddi(scratch, 4); + as_ld_w(scratch2, scratch, 0); + as_add_d(scratch, scratch, scratch2); + as_jirl(zero, scratch, BOffImm16(0)); + // Allocate space which will be patched by patchFarJump(). + CodeOffset farJump(currentOffset()); + spew(".space 32bit initValue 0xffff ffff"); + writeInst(UINT32_MAX); + return farJump; +} + +void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) { + uint32_t* u32 = + reinterpret_cast(editSrc(BufferOffset(farJump.offset()))); + MOZ_ASSERT(*u32 == UINT32_MAX); + *u32 = targetOffset - farJump.offset(); +} + +CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) { + movePtr(target, CallReg); + return call(CallReg); +} + +void MacroAssembler::call(const Address& addr) { + loadPtr(addr, CallReg); + call(CallReg); +} + +void MacroAssembler::call(ImmWord target) { call(ImmPtr((void*)target.value)); } + +void MacroAssembler::call(ImmPtr target) { + BufferOffset bo = m_buffer.nextOffset(); + addPendingJump(bo, target, Relocation::HARDCODED); + ma_call(target); +} + +void MacroAssembler::call(JitCode* c) { + ScratchRegisterScope scratch(asMasm()); + BufferOffset bo = m_buffer.nextOffset(); + addPendingJump(bo, ImmPtr(c->raw()), Relocation::JITCODE); + ma_liPatchable(scratch, ImmPtr(c->raw())); + callJitNoProfiler(scratch); +} + +CodeOffsetJump MacroAssembler::backedgeJump(RepatchLabel* label, + Label* documentation) { + return jumpWithPatch(label, documentation); +} + +CodeOffsetJump MacroAssembler::jumpWithPatch(RepatchLabel* label, + Label* documentation) { + MOZ_ASSERT(!label->used()); + + BufferOffset bo = nextOffset(); + label->use(bo.getOffset()); + + m_buffer.ensureSpace(4 * sizeof(uint32_t)); + writeInst(InstImm(op_beq, BOffImm16(0), zero, zero).encode()); + writeInst(LabelBase::INVALID_OFFSET); + as_nop(); + as_nop(); + return CodeOffsetJump(bo.getOffset()); +} + +CodeOffset MacroAssembler::nopPatchableToCall() { + // LOONGARCH64 + as_nop(); // lu12i_w + as_nop(); // ori + as_nop(); // lu32i_d + as_nop(); // jirl + return CodeOffset(currentOffset()); +} + +void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) { + Instruction* inst = (Instruction*)call - 4 /* four nops */; + Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)target); + inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra); +} + +void MacroAssembler::patchCallToNop(uint8_t* call) { + Instruction* inst = (Instruction*)call - 4 /* four nops */; + inst[0].makeNop(); // lu12i_w + inst[1].makeNop(); // ori + inst[2].makeNop(); // lu32i_d + inst[3].makeNop(); // jirl +} + +void MacroAssembler::pushReturnAddress() { push(ra); } + +void MacroAssembler::popReturnAddress() { pop(ra); } + +// =============================================================== +// ABI function calls. + +void MacroAssembler::setupUnalignedABICall(Register scratch) { + MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls"); + setupNativeABICall(); + dynamicAlignment_ = true; + + as_or(scratch, StackPointer, zero); + + // Force sp to be aligned + asMasm().subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1))); + storePtr(scratch, Address(StackPointer, 0)); +} + +void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { + MOZ_ASSERT(inCall_); + uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar(); + + // Reserve place for $ra. + stackForCall += sizeof(intptr_t); + + if (dynamicAlignment_) { + stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment); + } else { + uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0; + stackForCall += ComputeByteAlignment( + stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment); + } + + *stackAdjust = stackForCall; + reserveStack(stackForCall); + + // Save $ra because call is going to clobber it. Restore it in + // callWithABIPost. NOTE: This is needed for calls from SharedIC. + // Maybe we can do this differently. + storePtr(ra, Address(StackPointer, stackForCall - sizeof(intptr_t))); + + // Position all arguments. + { + enoughMemory_ &= moveResolver_.resolve(); + if (!enoughMemory_) { + return; + } + + MoveEmitter emitter(*this); + emitter.emit(moveResolver_); + emitter.finish(); + } + + assertStackAlignment(ABIStackAlignment); +} + +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result, + bool callFromWasm) { + // Restore ra value (as stored in callWithABIPre()). + loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra); + + if (dynamicAlignment_) { + // Restore sp value from stack (as stored in setupUnalignedABICall()). + loadPtr(Address(StackPointer, stackAdjust), StackPointer); + // Use adjustFrame instead of freeStack because we already restored sp. + adjustFrame(-stackAdjust); + } else { + freeStack(stackAdjust); + } + +#ifdef DEBUG + MOZ_ASSERT(inCall_); + inCall_ = false; +#endif +} + +void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) { + SecondScratchRegisterScope scratch2(asMasm()); + // Load the callee in scratch2, no instruction between the movePtr and + // call should clobber it. Note that we can't use fun because it may be + // one of the IntArg registers clobbered before the call. + movePtr(fun, scratch2); + + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + call(scratch2); + callWithABIPost(stackAdjust, result); +} + +void MacroAssembler::callWithABINoProfiler(const Address& fun, + MoveOp::Type result) { + SecondScratchRegisterScope scratch2(asMasm()); + // Load the callee in scratch2, as above. + loadPtr(fun, scratch2); + + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + call(scratch2); + callWithABIPost(stackAdjust, result); +} + +// =============================================================== +// Jit Frames. + +uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) { + CodeLabel cl; + + ma_li(scratch, &cl); + Push(scratch); + bind(&cl); + uint32_t retAddr = currentOffset(); + + addCodeLabel(cl); + return retAddr; +} + +// =============================================================== +// Move instructions + +void MacroAssembler::moveValue(const Value& src, Register dest) { + if (!src.isGCThing()) { + ma_li(dest, ImmWord(src.asRawBits())); + return; + } + + writeDataRelocation(src); + movWithPatch(ImmWord(src.asRawBits()), dest); +} + +void MacroAssembler::moveValue(const TypedOrValueRegister& src, + const ValueOperand& dest) { + if (src.hasValue()) { + moveValue(src.valueReg(), dest); + return; + } + + MIRType type = src.type(); + AnyRegister reg = src.typedReg(); + + if (!IsFloatingPointType(type)) { + boxNonDouble(ValueTypeFromMIRType(type), reg.gpr(), dest); + return; + } + + ScratchDoubleScope fpscratch(asMasm()); + FloatRegister scratch = fpscratch; + FloatRegister freg = reg.fpu(); + if (type == MIRType::Float32) { + convertFloat32ToDouble(freg, scratch); + freg = scratch; + } + boxDouble(freg, dest, scratch); +} + +void MacroAssembler::moveValue(const ValueOperand& src, + const ValueOperand& dest) { + if (src == dest) { + return; + } + movePtr(src.valueReg(), dest.valueReg()); +} + +void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) { + moveValue(src, dest.valueReg()); +} + +// =============================================================== +// Branch functions + +void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) { + if (ptr != buffer) { + movePtr(ptr, buffer); + } + orPtr(Imm32(gc::ChunkMask), buffer); + loadPtr(Address(buffer, gc::ChunkStoreBufferOffsetFromLastByte), buffer); +} + +void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, + Register temp, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + MOZ_ASSERT(ptr != temp); + MOZ_ASSERT(ptr != ScratchRegister && + ptr != SecondScratchReg); // Both may be used internally. + MOZ_ASSERT(temp != ScratchRegister && temp != SecondScratchReg); + MOZ_ASSERT(temp != InvalidReg); + + movePtr(ptr, temp); + orPtr(Imm32(gc::ChunkMask), temp); + branchPtr(InvertCondition(cond), + Address(temp, gc::ChunkStoreBufferOffsetFromLastByte), zero, label); +} + +void MacroAssembler::branchValueIsNurseryCell(Condition cond, + const Address& address, + Register temp, Label* label) { + branchValueIsNurseryCellImpl(cond, address, temp, label); +} + +void MacroAssembler::branchValueIsNurseryCell(Condition cond, + ValueOperand value, Register temp, + Label* label) { + branchValueIsNurseryCellImpl(cond, value, temp, label); +} + +template +void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, + const T& value, Register temp, + Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + MOZ_ASSERT(temp != InvalidReg); + Label done; + branchTestGCThing(Assembler::NotEqual, value, + cond == Assembler::Equal ? &done : label); + + unboxGCThingForGCBarrier(value, temp); + orPtr(Imm32(gc::ChunkMask), temp); + loadPtr(Address(temp, gc::ChunkStoreBufferOffsetFromLastByte), temp); + branchPtr(InvertCondition(cond), temp, zero, label); + + bind(&done); +} + +void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs, + const Value& rhs, Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs.valueReg() != scratch); + moveValue(rhs, ValueOperand(scratch)); + ma_b(lhs.valueReg(), scratch, label, cond); +} + +// ======================================================================== +// Memory access primitives. + +template +void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, const T& dest, + MIRType slotType) { + if (valueType == MIRType::Double) { + boxDouble(value.reg().typedReg().fpu(), dest); + return; + } + + // For known integers and booleans, we can just store the unboxed value if + // the slot has the same type. + if ((valueType == MIRType::Int32 || valueType == MIRType::Boolean) && + slotType == valueType) { + if (value.constant()) { + Value val = value.value(); + if (valueType == MIRType::Int32) { + store32(Imm32(val.toInt32()), dest); + } else { + store32(Imm32(val.toBoolean() ? 1 : 0), dest); + } + } else { + store32(value.reg().typedReg().gpr(), dest); + } + return; + } + + if (value.constant()) { + storeValue(value.value(), dest); + } else { + storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(), + dest); + } +} + +template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, + const Address& dest, + MIRType slotType); +template void MacroAssembler::storeUnboxedValue( + const ConstantOrRegister& value, MIRType valueType, + const BaseObjectElementIndex& dest, MIRType slotType); + +void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); } + +// =============================================================== +// WebAssembly + +CodeOffset MacroAssembler::wasmTrapInstruction() { + CodeOffset offset(currentOffset()); + as_break(WASM_TRAP); // TODO: as_teq(zero, zero, WASM_TRAP) + return offset; +} + +void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, + Register boundsCheckLimit, Label* ok) { + ma_b(index, boundsCheckLimit, ok, cond); +} + +void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, + Address boundsCheckLimit, Label* ok) { + SecondScratchRegisterScope scratch2(asMasm()); + load32(boundsCheckLimit, scratch2); + ma_b(index, Register(scratch2), ok, cond); +} + +void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, + Register64 boundsCheckLimit, Label* ok) { + ma_b(index.reg, boundsCheckLimit.reg, ok, cond); +} + +void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, + Address boundsCheckLimit, Label* ok) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(boundsCheckLimit, scratch2); + ma_b(index.reg, scratch2, ok, cond); +} + +// FTINTRZ behaves as follows: +// +// on NaN it produces zero +// on too large it produces INT_MAX (for appropriate type) +// on too small it produces INT_MIN (ditto) + +void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + if (!isSaturating) { + ma_bc_d(input, input, oolEntry, Assembler::DoubleUnordered); + } + as_ftintrz_l_d(fpscratch, input); + moveFromDouble(fpscratch, output); + as_srli_d(scratch, output, 32); + as_slli_w(output, output, 0); + ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); +} + +void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + if (!isSaturating) { + ma_bc_s(input, input, oolEntry, Assembler::DoubleUnordered); + } + as_ftintrz_l_s(fpscratch, input); + moveFromDouble(fpscratch, output); + as_srli_d(scratch, output, 32); + as_slli_w(output, output, 0); + ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); +} + +// Assembler::CauseV is a enum,called FCSRBit. Assembler::CauseV == 16 +void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + ScratchRegisterScope scratch(asMasm()); + ScratchFloat32Scope fpscratch(asMasm()); + as_ftintrz_w_d(fpscratch, input); + as_movfcsr2gr(scratch); + moveFromFloat32(fpscratch, output); + MOZ_ASSERT(Assembler::CauseV < 32); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); +} + +void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, + Register output, + bool isSaturating, + Label* oolEntry) { + ScratchRegisterScope scratch(asMasm()); + ScratchFloat32Scope fpscratch(asMasm()); + as_ftintrz_w_s(fpscratch, input); + as_movfcsr2gr(scratch); + moveFromFloat32(fpscratch, output); + MOZ_ASSERT(Assembler::CauseV < 32); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); +} + +void MacroAssembler::wasmTruncateDoubleToUInt64( + FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempDouble) { + MOZ_ASSERT(tempDouble.isInvalid()); + ScratchDoubleScope fpscratch(asMasm()); + Register output = output_.reg; + + Label done; + + if (!isSaturating) { + ma_bc_d(input, input, oolEntry, Assembler::DoubleUnordered); + } + as_ftintrz_l_d(fpscratch, input); + moveFromDouble(fpscratch, output); + loadConstantDouble(double(INT64_MAX + 1ULL), fpscratch); + + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, ImmWord(INT64_MAX)); + // For numbers in -1.[ : ]INT64_MAX range do nothing more + ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump); + + ma_li(scratch2, ImmWord(INT64_MIN)); + as_fsub_d(fpscratch, input, fpscratch); + as_ftintrz_l_d(fpscratch, fpscratch); + as_movfcsr2gr(scratch); + moveFromDouble(fpscratch, output); + as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + as_add_d(output, output, scratch2); + + // Guard against negative values that result in 0 due the precision loss. + as_sltui(scratch2, output, 1); + as_or(scratch, scratch, scratch2); + + ma_b(scratch, zero, oolEntry, Assembler::NotEqual); + + bind(&done); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::wasmTruncateFloat32ToUInt64( + FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempFloat) { + MOZ_ASSERT(tempFloat.isInvalid()); + ScratchDoubleScope fpscratch(asMasm()); + Register output = output_.reg; + + Label done; + + if (!isSaturating) { + ma_bc_s(input, input, oolEntry, Assembler::DoubleUnordered); + } + as_ftintrz_l_s(fpscratch, input); + moveFromDouble(fpscratch, output); + loadConstantFloat32(float(INT64_MAX + 1ULL), fpscratch); + + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, ImmWord(INT64_MAX)); + // For numbers in -1.[ : ]INT64_MAX range do nothing more + ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump); + + ma_li(scratch2, ImmWord(INT64_MIN)); + as_fsub_s(fpscratch, input, fpscratch); + as_ftintrz_l_s(fpscratch, fpscratch); + as_movfcsr2gr(scratch); + moveFromDouble(fpscratch, output); + as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + as_add_d(output, output, scratch2); + + // Guard against negative values that result in 0 due the precision loss. + as_sltui(scratch2, output, 1); + as_or(scratch, scratch, scratch2); + + ma_b(scratch, zero, oolEntry, Assembler::NotEqual); + + bind(&done); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::wasmTruncateDoubleToInt64( + FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempDouble) { + MOZ_ASSERT(tempDouble.isInvalid()); + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + + as_ftintrz_l_d(fpscratch, input); + as_movfcsr2gr(scratch); + moveFromDouble(fpscratch, output.reg); + as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, zero, oolEntry, Assembler::NotEqual); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::wasmTruncateFloat32ToInt64( + FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, + Label* oolRejoin, FloatRegister tempFloat) { + MOZ_ASSERT(tempFloat.isInvalid()); + ScratchRegisterScope scratch(asMasm()); + ScratchDoubleScope fpscratch(asMasm()); + + as_ftintrz_l_s(fpscratch, input); + as_movfcsr2gr(scratch); + moveFromDouble(fpscratch, output.reg); + as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(scratch, zero, oolEntry, Assembler::NotEqual); + + if (isSaturating) { + bind(oolRejoin); + } +} + +void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, + Register memoryBase, Register ptr, + Register ptrScratch, AnyRegister output) { + wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg); +} + +void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, + Register memoryBase, Register ptr, + Register ptrScratch, Register64 output) { + wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg); +} + +void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, + AnyRegister value, Register memoryBase, + Register ptr, Register ptrScratch) { + wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg); +} + +void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, + Register64 value, Register memoryBase, + Register ptr, Register ptrScratch) { + wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg); +} + +void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch, + ExitFrameType type) { + enterFakeExitFrame(cxreg, scratch, type); +} + +// TODO(loongarch64): widenInt32 should be nop? +void MacroAssembler::widenInt32(Register r) { + // Sign-extend the low 32 bits in-place for pointer-sized consumers. + as_slli_w(r, r, 0); +} + +// ======================================================================== +// Convert floating point. + +void MacroAssembler::convertUInt64ToFloat32(Register64 src_, FloatRegister dest, + Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + + Register src = src_.reg; + Label positive, done; + ma_b(src, src, &positive, NotSigned, ShortJump); + + MOZ_ASSERT(src != scratch); + MOZ_ASSERT(src != scratch2); + + ma_and(scratch, src, Imm32(1)); + as_srli_d(scratch2, src, 1); + as_or(scratch, scratch, scratch2); + as_movgr2fr_d(dest, scratch); + as_ffint_s_l(dest, dest); + addFloat32(dest, dest); + ma_b(&done, ShortJump); + + bind(&positive); + as_movgr2fr_d(dest, src); + as_ffint_s_l(dest, dest); + + bind(&done); +} + +void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) { + as_movgr2fr_d(dest, src.reg); + as_ffint_s_l(dest, dest); +} + +bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; } + +void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, + Register temp) { + MOZ_ASSERT(temp == Register::Invalid()); + MacroAssemblerSpecific::convertUInt64ToDouble(src.reg, dest); +} + +void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) { + as_movgr2fr_d(dest, src.reg); + as_ffint_d_l(dest, dest); +} + +void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) { + convertInt64ToDouble(Register64(src), dest); +} + +// ======================================================================== +// Primitive atomic operations. + +template +static void CompareExchange(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + Scalar::Type type, const Synchronization& sync, + const T& mem, Register oldval, Register newval, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + switch (nbytes) { + case 1: + case 2: + break; + case 4: + MOZ_ASSERT(valueTemp == InvalidReg); + MOZ_ASSERT(offsetTemp == InvalidReg); + MOZ_ASSERT(maskTemp == InvalidReg); + break; + default: + MOZ_CRASH(); + } + + Label again, end; + + masm.computeEffectiveAddress(mem, scratch); + + if (nbytes == 4) { + masm.memoryBarrierBefore(sync); + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(output, scratch, 0); + masm.ma_b(output, oldval, &end, Assembler::NotEqual, ShortJump); + masm.as_or(scratch2, newval, zero); + masm.as_sc_w(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + masm.bind(&end); + + return; + } + + masm.as_andi(offsetTemp, scratch, 3); + masm.subPtr(offsetTemp, scratch); + masm.as_slli_w(offsetTemp, offsetTemp, 3); + masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); + masm.as_sll_w(maskTemp, maskTemp, offsetTemp); + masm.as_nor(maskTemp, zero, maskTemp); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(scratch2, scratch, 0); + + masm.as_srl_w(output, scratch2, offsetTemp); + + switch (nbytes) { + case 1: + if (signExtend) { + masm.as_ext_w_b(valueTemp, oldval); + masm.as_ext_w_b(output, output); + } else { + masm.as_andi(valueTemp, oldval, 0xff); + masm.as_andi(output, output, 0xff); + } + break; + case 2: + if (signExtend) { + masm.as_ext_w_h(valueTemp, oldval); + masm.as_ext_w_h(output, output); + } else { + masm.as_bstrpick_d(valueTemp, oldval, 15, 0); + masm.as_bstrpick_d(output, output, 15, 0); + } + break; + } + + masm.ma_b(output, valueTemp, &end, Assembler::NotEqual, ShortJump); + + masm.as_sll_w(valueTemp, newval, offsetTemp); + masm.as_and(scratch2, scratch2, maskTemp); + masm.as_or(scratch2, scratch2, valueTemp); + + masm.as_sc_w(scratch2, scratch, 0); + + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + + masm.bind(&end); +} + +template +static void CompareExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + const Synchronization& sync, const T& mem, + Register64 expect, Register64 replace, + Register64 output) { + MOZ_ASSERT(expect != output && replace != output); + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + masm.computeEffectiveAddress(mem, scratch); + + Label tryAgain; + Label exit; + + masm.memoryBarrierBefore(sync); + + masm.bind(&tryAgain); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_d(output.reg, scratch, 0); + + masm.ma_b(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump); + masm.movePtr(replace.reg, scratch2); + masm.as_sc_d(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero, + ShortJump); + + masm.memoryBarrierAfter(sync); + + masm.bind(&exit); +} + +template +static void AtomicExchange(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + Scalar::Type type, const Synchronization& sync, + const T& mem, Register value, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + switch (nbytes) { + case 1: + case 2: + break; + case 4: + MOZ_ASSERT(valueTemp == InvalidReg); + MOZ_ASSERT(offsetTemp == InvalidReg); + MOZ_ASSERT(maskTemp == InvalidReg); + break; + default: + MOZ_CRASH(); + } + + Label again; + + masm.computeEffectiveAddress(mem, scratch); + + if (nbytes == 4) { + masm.memoryBarrierBefore(sync); + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(output, scratch, 0); + masm.as_or(scratch2, value, zero); + masm.as_sc_w(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + + return; + } + + masm.as_andi(offsetTemp, scratch, 3); + masm.subPtr(offsetTemp, scratch); + masm.as_slli_w(offsetTemp, offsetTemp, 3); + masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); + masm.as_sll_w(maskTemp, maskTemp, offsetTemp); + masm.as_nor(maskTemp, zero, maskTemp); + switch (nbytes) { + case 1: + masm.as_andi(valueTemp, value, 0xff); + break; + case 2: + masm.as_bstrpick_d(valueTemp, value, 15, 0); + break; + } + masm.as_sll_w(valueTemp, valueTemp, offsetTemp); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(output, scratch, 0); + masm.as_and(scratch2, output, maskTemp); + masm.as_or(scratch2, scratch2, valueTemp); + + masm.as_sc_w(scratch2, scratch, 0); + + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.as_srl_w(output, output, offsetTemp); + + switch (nbytes) { + case 1: + if (signExtend) { + masm.as_ext_w_b(output, output); + } else { + masm.as_andi(output, output, 0xff); + } + break; + case 2: + if (signExtend) { + masm.as_ext_w_h(output, output); + } else { + masm.as_bstrpick_d(output, output, 15, 0); + } + break; + } + + masm.memoryBarrierAfter(sync); +} + +template +static void AtomicExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + const Synchronization& sync, const T& mem, + Register64 value, Register64 output) { + MOZ_ASSERT(value != output); + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + masm.computeEffectiveAddress(mem, scratch); + + Label tryAgain; + + masm.memoryBarrierBefore(sync); + + masm.bind(&tryAgain); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_d(output.reg, scratch, 0); + + masm.movePtr(value.reg, scratch2); + masm.as_sc_d(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero, + ShortJump); + + masm.memoryBarrierAfter(sync); +} + +template +static void AtomicFetchOp(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + Scalar::Type type, const Synchronization& sync, + AtomicOp op, const T& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + switch (nbytes) { + case 1: + case 2: + break; + case 4: + MOZ_ASSERT(valueTemp == InvalidReg); + MOZ_ASSERT(offsetTemp == InvalidReg); + MOZ_ASSERT(maskTemp == InvalidReg); + break; + default: + MOZ_CRASH(); + } + + Label again; + + masm.computeEffectiveAddress(mem, scratch); + + if (nbytes == 4) { + masm.memoryBarrierBefore(sync); + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(output, scratch, 0); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_w(scratch2, output, value); + break; + case AtomicFetchSubOp: + masm.as_sub_w(scratch2, output, value); + break; + case AtomicFetchAndOp: + masm.as_and(scratch2, output, value); + break; + case AtomicFetchOrOp: + masm.as_or(scratch2, output, value); + break; + case AtomicFetchXorOp: + masm.as_xor(scratch2, output, value); + break; + default: + MOZ_CRASH(); + } + + masm.as_sc_w(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + + return; + } + + masm.as_andi(offsetTemp, scratch, 3); + masm.subPtr(offsetTemp, scratch); + masm.as_slli_w(offsetTemp, offsetTemp, 3); + masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); + masm.as_sll_w(maskTemp, maskTemp, offsetTemp); + masm.as_nor(maskTemp, zero, maskTemp); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(scratch2, scratch, 0); + masm.as_srl_w(output, scratch2, offsetTemp); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_w(valueTemp, output, value); + break; + case AtomicFetchSubOp: + masm.as_sub_w(valueTemp, output, value); + break; + case AtomicFetchAndOp: + masm.as_and(valueTemp, output, value); + break; + case AtomicFetchOrOp: + masm.as_or(valueTemp, output, value); + break; + case AtomicFetchXorOp: + masm.as_xor(valueTemp, output, value); + break; + default: + MOZ_CRASH(); + } + + switch (nbytes) { + case 1: + masm.as_andi(valueTemp, valueTemp, 0xff); + break; + case 2: + masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0); + break; + } + + masm.as_sll_w(valueTemp, valueTemp, offsetTemp); + + masm.as_and(scratch2, scratch2, maskTemp); + masm.as_or(scratch2, scratch2, valueTemp); + + masm.as_sc_w(scratch2, scratch, 0); + + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + switch (nbytes) { + case 1: + if (signExtend) { + masm.as_ext_w_b(output, output); + } else { + masm.as_andi(output, output, 0xff); + } + break; + case 2: + if (signExtend) { + masm.as_ext_w_h(output, output); + } else { + masm.as_bstrpick_d(output, output, 15, 0); + } + break; + } + + masm.memoryBarrierAfter(sync); +} + +template +static void AtomicFetchOp64(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + const Synchronization& sync, AtomicOp op, + Register64 value, const T& mem, Register64 temp, + Register64 output) { + MOZ_ASSERT(value != output); + MOZ_ASSERT(value != temp); + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + masm.computeEffectiveAddress(mem, scratch); + + Label tryAgain; + + masm.memoryBarrierBefore(sync); + + masm.bind(&tryAgain); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_d(output.reg, scratch, 0); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_d(temp.reg, output.reg, value.reg); + break; + case AtomicFetchSubOp: + masm.as_sub_d(temp.reg, output.reg, value.reg); + break; + case AtomicFetchAndOp: + masm.as_and(temp.reg, output.reg, value.reg); + break; + case AtomicFetchOrOp: + masm.as_or(temp.reg, output.reg, value.reg); + break; + case AtomicFetchXorOp: + masm.as_xor(temp.reg, output.reg, value.reg); + break; + default: + MOZ_CRASH(); + } + + masm.as_sc_d(temp.reg, scratch, 0); + masm.ma_b(temp.reg, temp.reg, &tryAgain, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); +} + +void MacroAssembler::compareExchange(Scalar::Type type, + const Synchronization& sync, + const Address& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, output); +} + +void MacroAssembler::compareExchange(Scalar::Type type, + const Synchronization& sync, + const BaseIndex& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, output); +} + +void MacroAssembler::compareExchange64(const Synchronization& sync, + const Address& mem, Register64 expect, + Register64 replace, Register64 output) { + CompareExchange64(*this, nullptr, sync, mem, expect, replace, output); +} + +void MacroAssembler::compareExchange64(const Synchronization& sync, + const BaseIndex& mem, Register64 expect, + Register64 replace, Register64 output) { + CompareExchange64(*this, nullptr, sync, mem, expect, replace, output); +} + +void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, + const Address& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, + newval, valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, + newval, valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, + const Address& mem, + Register64 expect, + Register64 replace, + Register64 output) { + CompareExchange64(*this, &access, access.sync(), mem, expect, replace, + output); +} + +void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, + Register64 expect, + Register64 replace, + Register64 output) { + CompareExchange64(*this, &access, access.sync(), mem, expect, replace, + output); +} + +void MacroAssembler::atomicExchange(Scalar::Type type, + const Synchronization& sync, + const Address& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, + maskTemp, output); +} + +void MacroAssembler::atomicExchange(Scalar::Type type, + const Synchronization& sync, + const BaseIndex& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, + maskTemp, output); +} + +void MacroAssembler::atomicExchange64(const Synchronization& sync, + const Address& mem, Register64 value, + Register64 output) { + AtomicExchange64(*this, nullptr, sync, mem, value, output); +} + +void MacroAssembler::atomicExchange64(const Synchronization& sync, + const BaseIndex& mem, Register64 value, + Register64 output) { + AtomicExchange64(*this, nullptr, sync, mem, value, output); +} + +void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, + const Address& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, + valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, + valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::atomicFetchOp(Scalar::Type type, + const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, + offsetTemp, maskTemp, output); +} + +void MacroAssembler::atomicFetchOp(Scalar::Type type, + const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register output) { + AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, + offsetTemp, maskTemp, output); +} + +void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, + Register64 value, const Address& mem, + Register64 temp, Register64 output) { + AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output); +} + +void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, + Register64 value, const BaseIndex& mem, + Register64 temp, Register64 output) { + AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output); +} + +void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op, + Register64 value, const Address& mem, + Register64 temp) { + AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp); +} + +void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op, + Register64 value, const BaseIndex& mem, + Register64 temp) { + AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp); +} + +void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register value, + const Address& mem, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, + valueTemp, offsetTemp, maskTemp, output); +} + +void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register value, + const BaseIndex& mem, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register output) { + AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, + valueTemp, offsetTemp, maskTemp, output); +} + +template +static void AtomicEffectOp(MacroAssembler& masm, + const wasm::MemoryAccessDesc* access, + Scalar::Type type, const Synchronization& sync, + AtomicOp op, const T& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp) { + ScratchRegisterScope scratch(masm); + SecondScratchRegisterScope scratch2(masm); + unsigned nbytes = Scalar::byteSize(type); + + switch (nbytes) { + case 1: + case 2: + break; + case 4: + MOZ_ASSERT(valueTemp == InvalidReg); + MOZ_ASSERT(offsetTemp == InvalidReg); + MOZ_ASSERT(maskTemp == InvalidReg); + break; + default: + MOZ_CRASH(); + } + + Label again; + + masm.computeEffectiveAddress(mem, scratch); + + if (nbytes == 4) { + masm.memoryBarrierBefore(sync); + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(scratch2, scratch, 0); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_w(scratch2, scratch2, value); + break; + case AtomicFetchSubOp: + masm.as_sub_w(scratch2, scratch2, value); + break; + case AtomicFetchAndOp: + masm.as_and(scratch2, scratch2, value); + break; + case AtomicFetchOrOp: + masm.as_or(scratch2, scratch2, value); + break; + case AtomicFetchXorOp: + masm.as_xor(scratch2, scratch2, value); + break; + default: + MOZ_CRASH(); + } + + masm.as_sc_w(scratch2, scratch, 0); + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); + + return; + } + + masm.as_andi(offsetTemp, scratch, 3); + masm.subPtr(offsetTemp, scratch); + masm.as_slli_w(offsetTemp, offsetTemp, 3); + masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); + masm.as_sll_w(maskTemp, maskTemp, offsetTemp); + masm.as_nor(maskTemp, zero, maskTemp); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + + if (access) { + masm.append(*access, masm.size()); + } + + masm.as_ll_w(scratch2, scratch, 0); + masm.as_srl_w(valueTemp, scratch2, offsetTemp); + + switch (op) { + case AtomicFetchAddOp: + masm.as_add_w(valueTemp, valueTemp, value); + break; + case AtomicFetchSubOp: + masm.as_sub_w(valueTemp, valueTemp, value); + break; + case AtomicFetchAndOp: + masm.as_and(valueTemp, valueTemp, value); + break; + case AtomicFetchOrOp: + masm.as_or(valueTemp, valueTemp, value); + break; + case AtomicFetchXorOp: + masm.as_xor(valueTemp, valueTemp, value); + break; + default: + MOZ_CRASH(); + } + + switch (nbytes) { + case 1: + masm.as_andi(valueTemp, valueTemp, 0xff); + break; + case 2: + masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0); + break; + } + + masm.as_sll_w(valueTemp, valueTemp, offsetTemp); + + masm.as_and(scratch2, scratch2, maskTemp); + masm.as_or(scratch2, scratch2, valueTemp); + + masm.as_sc_w(scratch2, scratch, 0); + + masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); + + masm.memoryBarrierAfter(sync); +} + +void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register value, + const Address& mem, Register valueTemp, + Register offsetTemp, + Register maskTemp) { + AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value, + valueTemp, offsetTemp, maskTemp); +} + +void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register value, + const BaseIndex& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp) { + AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value, + valueTemp, offsetTemp, maskTemp); +} + +template +static void WasmAtomicExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc& access, + const T& mem, Register64 value, + Register64 output) { + AtomicExchange64(masm, &access, access.sync(), mem, value, output); +} + +void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, + const Address& mem, Register64 src, + Register64 output) { + WasmAtomicExchange64(*this, access, mem, src, output); +} + +void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, + const BaseIndex& mem, Register64 src, + Register64 output) { + WasmAtomicExchange64(*this, access, mem, src, output); +} + +void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register64 value, + const Address& mem, Register64 temp, + Register64 output) { + AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output); +} + +void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, + AtomicOp op, Register64 value, + const BaseIndex& mem, Register64 temp, + Register64 output) { + AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output); +} + +// ======================================================================== +// JS atomic operations. + +template +static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, + const Synchronization& sync, const T& mem, + Register oldval, Register newval, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + if (arrayType == Scalar::Uint32) { + masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, temp); + masm.convertUInt32ToDouble(temp, output.fpu()); + } else { + masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, output.gpr()); + } +} + +template +static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, + const Synchronization& sync, const T& mem, + Register value, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register temp, AnyRegister output) { + if (arrayType == Scalar::Uint32) { + masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, + maskTemp, temp); + masm.convertUInt32ToDouble(temp, output.fpu()); + } else { + masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, + maskTemp, output.gpr()); + } +} + +template +static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const T& mem, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register temp, AnyRegister output) { + if (arrayType == Scalar::Uint32) { + masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, + maskTemp, temp); + masm.convertUInt32ToDouble(temp, output.fpu()); + } else { + masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, + maskTemp, output.gpr()); + } +} + +void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, + const Synchronization& sync, + const Address& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register temp, AnyRegister output) { + CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, temp, output); +} + +void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, + const Synchronization& sync, + const BaseIndex& mem, Register oldval, + Register newval, Register valueTemp, + Register offsetTemp, Register maskTemp, + Register temp, AnyRegister output) { + CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, + offsetTemp, maskTemp, temp, output); +} + +void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, + const Synchronization& sync, + const Address& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, + maskTemp, temp, output); +} + +void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, + const Synchronization& sync, + const BaseIndex& mem, Register value, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, + maskTemp, temp, output); +} + +void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, + maskTemp, temp, output); +} + +void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp, Register temp, + AnyRegister output) { + AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, + maskTemp, temp, output); +} + +void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp) { + AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp, + offsetTemp, maskTemp); +} + +void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, + const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, + Register valueTemp, Register offsetTemp, + Register maskTemp) { + AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp, + offsetTemp, maskTemp); +} + +void MacroAssembler::flexibleQuotient32(Register rhs, Register srcDest, + bool isUnsigned, + const LiveRegisterSet&) { + quotient32(rhs, srcDest, isUnsigned); +} + +void MacroAssembler::flexibleRemainder32(Register rhs, Register srcDest, + bool isUnsigned, + const LiveRegisterSet&) { + remainder32(rhs, srcDest, isUnsigned); +} + +void MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest, + Register remOutput, bool isUnsigned, + const LiveRegisterSet&) { + if (isUnsigned) { + as_mod_wu(remOutput, srcDest, rhs); + as_div_wu(srcDest, srcDest, rhs); + } else { + as_mod_w(remOutput, srcDest, rhs); + as_div_w(srcDest, srcDest, rhs); + } +} + +CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) { + return movWithPatch(ImmPtr(nullptr), dest); +} + +void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc, + CodeLocationLabel target) { + PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr)); +} + +// ======================================================================== +// Spectre Mitigations. + +void MacroAssembler::speculationBarrier() { MOZ_CRASH(); } + +void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchFloat32Scope fpscratch(asMasm()); + FloatRegister scratch = fpscratch; + Label skipCheck, done; + + // If Nan, 0 or -0 check for bailout + loadConstantFloat32(0.0f, scratch); + ma_bc_s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); + + // If high part is not zero, it is NaN or -0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromDoubleLo(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&done, ShortJump); + + bind(&skipCheck); + as_ftintrm_w_s(scratch, src); + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + bind(&done); +} + +void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchDoubleScope fpscratch(asMasm()); + FloatRegister scratch = fpscratch; + Label skipCheck, done; + + // If Nan, 0 or -0 check for bailout + loadConstantDouble(0.0, scratch); + ma_bc_d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); + + // If high part is not zero, it is NaN or -0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromDoubleHi(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&done, ShortJump); + + bind(&skipCheck); + as_ftintrm_w_d(scratch, src); + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + bind(&done); +} + +void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchFloat32Scope fpscratch(asMasm()); + FloatRegister scratch = fpscratch; + Label performCeil, done; + + // If x < -1 or x > 0 then perform ceil. + loadConstantFloat32(0.0f, scratch); + branchFloat(Assembler::DoubleGreaterThan, src, scratch, &performCeil); + loadConstantFloat32(-1.0f, scratch); + branchFloat(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil); + + // If binary value is not zero, the input was not 0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromFloat32(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&done, ShortJump); + + bind(&performCeil); + as_ftintrp_w_s(scratch, src); + moveFromFloat32(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + bind(&done); +} + +void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchDoubleScope fpscratch(asMasm()); + FloatRegister scratch = fpscratch; + Label performCeil, done; + + // If x < -1 or x > 0 then perform ceil. + loadConstantDouble(0, scratch); + branchDouble(Assembler::DoubleGreaterThan, src, scratch, &performCeil); + loadConstantDouble(-1.0, scratch); + branchDouble(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil); + + // If binary value is not zero, the input was not 0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromDoubleHi(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&done, ShortJump); + + bind(&performCeil); + as_ftintrp_w_d(scratch, src); + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + bind(&done); +} + +void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest, + FloatRegister temp, Label* fail) { + ScratchFloat32Scope scratch(*this); + + Label negative, end, skipCheck; + + // Load biggest number less than 0.5 in the temp register. + loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp); + + // Branch to a slow path for negative inputs. Doesn't catch NaN or -0. + loadConstantFloat32(0.0f, scratch); + ma_bc_s(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump); + + // If Nan, 0 or -0 check for bailout + ma_bc_s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); + + // If binary value is not zero, it is NaN or -0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromFloat32(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&end, ShortJump); + + bind(&skipCheck); + as_fadd_s(scratch, src, temp); + as_ftintrm_w_s(scratch, scratch); + + moveFromFloat32(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + jump(&end); + + // Input is negative, but isn't -0. + bind(&negative); + + // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to + // be added the biggest double less than 0.5. + Label loadJoin; + loadConstantFloat32(-0.5f, scratch); + branchFloat(Assembler::DoubleLessThan, src, scratch, &loadJoin); + loadConstantFloat32(0.5f, temp); + bind(&loadJoin); + + as_fadd_s(temp, src, temp); + + // If input + 0.5 >= 0, input is a negative number >= -0.5 and the + // result is -0. + branchFloat(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail); + + // Truncate and round toward zero. + // This is off-by-one for everything but integer-valued inputs. + as_ftintrm_w_s(scratch, temp); + moveFromFloat32(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + + bind(&end); +} + +void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest, + FloatRegister temp, Label* fail) { + ScratchDoubleScope scratch(*this); + + Label negative, end, skipCheck; + + // Load biggest number less than 0.5 in the temp register. + loadConstantDouble(GetBiggestNumberLessThan(0.5), temp); + + // Branch to a slow path for negative inputs. Doesn't catch NaN or -0. + loadConstantDouble(0.0, scratch); + ma_bc_d(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump); + + // If Nan, 0 or -0 check for bailout + ma_bc_d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump); + + // If high part is not zero, it is NaN or -0, so we bail. + { + ScratchRegisterScope scratch(asMasm()); + moveFromDoubleHi(src, scratch); + branch32(Assembler::NotEqual, scratch, zero, fail); + } + + // Input was zero, so return zero. + move32(Imm32(0), dest); + ma_b(&end, ShortJump); + + bind(&skipCheck); + as_fadd_d(scratch, src, temp); + as_ftintrm_w_d(scratch, scratch); + + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail); + + jump(&end); + + // Input is negative, but isn't -0. + bind(&negative); + + // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to + // be added the biggest double less than 0.5. + Label loadJoin; + loadConstantDouble(-0.5, scratch); + branchDouble(Assembler::DoubleLessThan, src, scratch, &loadJoin); + loadConstantDouble(0.5, temp); + bind(&loadJoin); + + addDouble(src, temp); + + // If input + 0.5 >= 0, input is a negative number >= -0.5 and the + // result is -0. + branchDouble(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail); + + // Truncate and round toward zero. + // This is off-by-one for everything but integer-valued inputs. + as_ftintrm_w_d(scratch, temp); + moveFromDoubleLo(scratch, dest); + + branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail); + + bind(&end); +} + +void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchRegisterScope scratch(asMasm()); + ScratchFloat32Scope fpscratch(asMasm()); + + Label notZero; + as_ftintrz_w_s(fpscratch, src); + as_movfcsr2gr(scratch); + moveFromFloat32(fpscratch, dest); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); + + { + // dest == zero + SecondScratchRegisterScope scratch2(asMasm()); + moveFromFloat32(src, scratch2); + // Check if input is in ]-1; -0] range by checking the sign bit. + as_slt(scratch2, scratch2, zero); + as_add_d(scratch, scratch, scratch2); + } + + bind(¬Zero); + branch32(Assembler::NotEqual, Register(scratch), zero, fail); +} + +void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchRegisterScope scratch(asMasm()); + ScratchFloat32Scope fpscratch(asMasm()); + + Label notZero; + as_ftintrz_w_d(fpscratch, src); + as_movfcsr2gr(scratch); + moveFromFloat32(fpscratch, dest); + as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); + ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); + + { + // dest == zero + SecondScratchRegisterScope scratch2(asMasm()); + moveFromDoubleHi(src, scratch2); + // Check if input is in ]-1; -0] range by checking the sign bit. + as_slt(scratch2, scratch2, zero); + as_add_d(scratch, scratch, scratch2); + } + + bind(¬Zero); + branch32(Assembler::NotEqual, Register(scratch), zero, fail); +} + +void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs, + FloatRegister output) { + MOZ_CRASH("not supported on this platform"); +} + +void MacroAssemblerLOONGARCH64Compat::move32(Imm32 imm, Register dest) { + ma_li(dest, imm); +} + +void MacroAssemblerLOONGARCH64Compat::move32(Register src, Register dest) { + as_slli_w(dest, src, 0); +} + +void MacroAssemblerLOONGARCH64Compat::movePtr(Register src, Register dest) { + as_or(dest, src, zero); +} +void MacroAssemblerLOONGARCH64Compat::movePtr(ImmWord imm, Register dest) { + ma_li(dest, imm); +} + +void MacroAssemblerLOONGARCH64Compat::movePtr(ImmGCPtr imm, Register dest) { + ma_li(dest, imm); +} + +void MacroAssemblerLOONGARCH64Compat::movePtr(ImmPtr imm, Register dest) { + movePtr(ImmWord(uintptr_t(imm.value)), dest); +} + +void MacroAssemblerLOONGARCH64Compat::movePtr(wasm::SymbolicAddress imm, + Register dest) { + append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm)); + ma_liPatchable(dest, ImmWord(-1)); +} + +void MacroAssemblerLOONGARCH64Compat::load8ZeroExtend(const Address& address, + Register dest) { + ma_load(dest, address, SizeByte, ZeroExtend); +} + +void MacroAssemblerLOONGARCH64Compat::load8ZeroExtend(const BaseIndex& src, + Register dest) { + ma_load(dest, src, SizeByte, ZeroExtend); +} + +void MacroAssemblerLOONGARCH64Compat::load8SignExtend(const Address& address, + Register dest) { + ma_load(dest, address, SizeByte, SignExtend); +} + +void MacroAssemblerLOONGARCH64Compat::load8SignExtend(const BaseIndex& src, + Register dest) { + ma_load(dest, src, SizeByte, SignExtend); +} + +void MacroAssemblerLOONGARCH64Compat::load16ZeroExtend(const Address& address, + Register dest) { + ma_load(dest, address, SizeHalfWord, ZeroExtend); +} + +void MacroAssemblerLOONGARCH64Compat::load16ZeroExtend(const BaseIndex& src, + Register dest) { + ma_load(dest, src, SizeHalfWord, ZeroExtend); +} + +void MacroAssemblerLOONGARCH64Compat::load16SignExtend(const Address& address, + Register dest) { + ma_load(dest, address, SizeHalfWord, SignExtend); +} + +void MacroAssemblerLOONGARCH64Compat::load16SignExtend(const BaseIndex& src, + Register dest) { + ma_load(dest, src, SizeHalfWord, SignExtend); +} + +void MacroAssemblerLOONGARCH64Compat::load32(const Address& address, + Register dest) { + ma_ld_w(dest, address); +} + +void MacroAssemblerLOONGARCH64Compat::load32(const BaseIndex& address, + Register dest) { + Register base = address.base; + Register index = address.index; + int32_t offset = address.offset; + uint32_t shift = Imm32::ShiftOf(address.scale).value; + + if (offset != 0) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, Imm32(offset)); + if (shift != 0) { + MOZ_ASSERT(shift <= 4); + as_alsl_d(scratch, index, scratch, shift - 1); + } else { + as_add_d(scratch, index, scratch); + } + as_ldx_w(dest, base, scratch); + } else if (shift != 0) { + ScratchRegisterScope scratch(asMasm()); + as_slli_d(scratch, index, shift); + as_ldx_w(dest, base, scratch); + } else { + as_ldx_w(dest, base, index); + } +} + +void MacroAssemblerLOONGARCH64Compat::load32(AbsoluteAddress address, + Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(ImmPtr(address.addr), scratch); + load32(Address(scratch, 0), dest); +} + +void MacroAssemblerLOONGARCH64Compat::load32(wasm::SymbolicAddress address, + Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(address, scratch); + load32(Address(scratch, 0), dest); +} + +void MacroAssemblerLOONGARCH64Compat::loadPtr(const Address& address, + Register dest) { + ma_ld_d(dest, address); +} + +void MacroAssemblerLOONGARCH64Compat::loadPtr(const BaseIndex& src, Register dest) { + Register base = src.base; + Register index = src.index; + int32_t offset = src.offset; + uint32_t shift = Imm32::ShiftOf(src.scale).value; + + if (offset != 0) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, Imm32(offset)); + if (shift != 0) { + MOZ_ASSERT(shift <= 4); + as_alsl_d(scratch, index, scratch, shift - 1); + } else { + as_add_d(scratch, index, scratch); + } + as_ldx_d(dest, base, scratch); + } else if (shift != 0) { + ScratchRegisterScope scratch(asMasm()); + as_slli_d(scratch, index, shift); + as_ldx_d(dest, base, scratch); + } else { + as_ldx_d(dest, base, index); + } +} + +void MacroAssemblerLOONGARCH64Compat::loadPtr(AbsoluteAddress address, + Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(ImmPtr(address.addr), scratch); + loadPtr(Address(scratch, 0), dest); +} + +void MacroAssemblerLOONGARCH64Compat::loadPtr(wasm::SymbolicAddress address, + Register dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(address, scratch); + loadPtr(Address(scratch, 0), dest); +} + +void MacroAssemblerLOONGARCH64Compat::loadPrivate(const Address& address, + Register dest) { + loadPtr(address, dest); +} + +void MacroAssemblerLOONGARCH64Compat::store8(Imm32 imm, const Address& address) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_store(scratch2, address, SizeByte); +} + +void MacroAssemblerLOONGARCH64Compat::store8(Register src, const Address& address) { + ma_store(src, address, SizeByte); +} + +void MacroAssemblerLOONGARCH64Compat::store8(Imm32 imm, const BaseIndex& dest) { + ma_store(imm, dest, SizeByte); +} + +void MacroAssemblerLOONGARCH64Compat::store8(Register src, const BaseIndex& dest) { + ma_store(src, dest, SizeByte); +} + +void MacroAssemblerLOONGARCH64Compat::store16(Imm32 imm, const Address& address) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_store(scratch2, address, SizeHalfWord); +} + +void MacroAssemblerLOONGARCH64Compat::store16(Register src, + const Address& address) { + ma_store(src, address, SizeHalfWord); +} + +void MacroAssemblerLOONGARCH64Compat::store16(Imm32 imm, const BaseIndex& dest) { + ma_store(imm, dest, SizeHalfWord); +} + +void MacroAssemblerLOONGARCH64Compat::store16(Register src, + const BaseIndex& address) { + ma_store(src, address, SizeHalfWord); +} + +void MacroAssemblerLOONGARCH64Compat::store32(Register src, + AbsoluteAddress address) { + ScratchRegisterScope scratch(asMasm()); + movePtr(ImmPtr(address.addr), scratch); + store32(src, Address(scratch, 0)); +} + +void MacroAssemblerLOONGARCH64Compat::store32(Register src, + const Address& address) { + ma_store(src, address, SizeWord); +} + +void MacroAssemblerLOONGARCH64Compat::store32(Imm32 src, const Address& address) { + SecondScratchRegisterScope scratch2(asMasm()); + move32(src, scratch2); + ma_store(scratch2, address, SizeWord); +} + +void MacroAssemblerLOONGARCH64Compat::store32(Imm32 imm, const BaseIndex& dest) { + ma_store(imm, dest, SizeWord); +} + +void MacroAssemblerLOONGARCH64Compat::store32(Register src, const BaseIndex& dest) { + ma_store(src, dest, SizeWord); +} + +template +void MacroAssemblerLOONGARCH64Compat::storePtr(ImmWord imm, T address) { + SecondScratchRegisterScope scratch2(asMasm()); + ma_li(scratch2, imm); + ma_store(scratch2, address, SizeDouble); +} + +template void MacroAssemblerLOONGARCH64Compat::storePtr
(ImmWord imm, + Address address); +template void MacroAssemblerLOONGARCH64Compat::storePtr( + ImmWord imm, BaseIndex address); + +template +void MacroAssemblerLOONGARCH64Compat::storePtr(ImmPtr imm, T address) { + storePtr(ImmWord(uintptr_t(imm.value)), address); +} + +template void MacroAssemblerLOONGARCH64Compat::storePtr
(ImmPtr imm, + Address address); +template void MacroAssemblerLOONGARCH64Compat::storePtr( + ImmPtr imm, BaseIndex address); + +template +void MacroAssemblerLOONGARCH64Compat::storePtr(ImmGCPtr imm, T address) { + SecondScratchRegisterScope scratch2(asMasm()); + movePtr(imm, scratch2); + storePtr(scratch2, address); +} + +template void MacroAssemblerLOONGARCH64Compat::storePtr
(ImmGCPtr imm, + Address address); +template void MacroAssemblerLOONGARCH64Compat::storePtr( + ImmGCPtr imm, BaseIndex address); + +void MacroAssemblerLOONGARCH64Compat::storePtr(Register src, + const Address& address) { + ma_st_d(src, address); +} + +void MacroAssemblerLOONGARCH64Compat::storePtr(Register src, + const BaseIndex& address) { + Register base = address.base; + Register index = address.index; + int32_t offset = address.offset; + int32_t shift = Imm32::ShiftOf(address.scale).value; + + if ((offset == 0) && (shift == 0)) { + as_stx_d(src, base, index); + } else if (is_intN(offset, 12)) { + ScratchRegisterScope scratch(asMasm()); + if (shift == 0) { + as_add_d(scratch, base, index); + } else { + as_alsl_d(scratch, index, base, shift - 1); + } + as_st_d(src, scratch, offset); + } else { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, Imm32(offset)); + if (shift == 0) { + as_add_d(scratch, scratch, index); + } else { + as_alsl_d(scratch, index, scratch, shift - 1); + } + as_stx_d(src, base, scratch); + } +} + +void MacroAssemblerLOONGARCH64Compat::storePtr(Register src, AbsoluteAddress dest) { + ScratchRegisterScope scratch(asMasm()); + movePtr(ImmPtr(dest.addr), scratch); + storePtr(src, Address(scratch, 0)); +} + +void MacroAssemblerLOONGARCH64Compat::testNullSet(Condition cond, + const ValueOperand& value, + Register dest) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(asMasm()); + splitTag(value, scratch2); + ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_NULL), cond); +} + +void MacroAssemblerLOONGARCH64Compat::testObjectSet(Condition cond, + const ValueOperand& value, + Register dest) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(asMasm()); + splitTag(value, scratch2); + ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_OBJECT), cond); +} + +void MacroAssemblerLOONGARCH64Compat::testUndefinedSet(Condition cond, + const ValueOperand& value, + Register dest) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + SecondScratchRegisterScope scratch2(asMasm()); + splitTag(value, scratch2); + ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_UNDEFINED), cond); +} + +void MacroAssemblerLOONGARCH64Compat::unboxInt32(const ValueOperand& operand, + Register dest) { + as_slli_w(dest, operand.valueReg(), 0); +} + +void MacroAssemblerLOONGARCH64Compat::unboxInt32(Register src, Register dest) { + as_slli_w(dest, src, 0); +} + +void MacroAssemblerLOONGARCH64Compat::unboxInt32(const Address& src, + Register dest) { + load32(Address(src.base, src.offset), dest); +} + +void MacroAssemblerLOONGARCH64Compat::unboxInt32(const BaseIndex& src, + Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + computeScaledAddress(src, scratch2); + load32(Address(scratch2, src.offset), dest); +} + +void MacroAssemblerLOONGARCH64Compat::unboxBoolean(const ValueOperand& operand, + Register dest) { + as_slli_w(dest, operand.valueReg(), 0); +} + +void MacroAssemblerLOONGARCH64Compat::unboxBoolean(Register src, Register dest) { + as_slli_w(dest, src, 0); +} + +void MacroAssemblerLOONGARCH64Compat::unboxBoolean(const Address& src, + Register dest) { + ma_ld_w(dest, src); +} + +void MacroAssemblerLOONGARCH64Compat::unboxBoolean(const BaseIndex& src, + Register dest) { + SecondScratchRegisterScope scratch2(asMasm()); + computeScaledAddress(src, scratch2); + ma_ld_w(dest, Address(scratch2, src.offset)); +} + +void MacroAssemblerLOONGARCH64Compat::unboxDouble(const ValueOperand& operand, + FloatRegister dest) { + as_movgr2fr_d(dest, operand.valueReg()); +} + +void MacroAssemblerLOONGARCH64Compat::unboxDouble(const Address& src, + FloatRegister dest) { + ma_fld_d(dest, Address(src.base, src.offset)); +} + +void MacroAssemblerLOONGARCH64Compat::unboxDouble(const BaseIndex& src, + FloatRegister dest) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(src, scratch2); + unboxDouble(ValueOperand(scratch2), dest); +} + +void MacroAssemblerLOONGARCH64Compat::unboxString(const ValueOperand& operand, + Register dest) { + unboxNonDouble(operand, dest, JSVAL_TYPE_STRING); +} + +void MacroAssemblerLOONGARCH64Compat::unboxString(Register src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_STRING); +} + +void MacroAssemblerLOONGARCH64Compat::unboxString(const Address& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_STRING); +} + +void MacroAssemblerLOONGARCH64Compat::unboxSymbol(const ValueOperand& operand, + Register dest) { + unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL); +} + +void MacroAssemblerLOONGARCH64Compat::unboxSymbol(Register src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); +} + +void MacroAssemblerLOONGARCH64Compat::unboxSymbol(const Address& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); +} + +void MacroAssemblerLOONGARCH64Compat::unboxBigInt(const ValueOperand& operand, + Register dest) { + unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT); +} + +void MacroAssemblerLOONGARCH64Compat::unboxBigInt(Register src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); +} + +void MacroAssemblerLOONGARCH64Compat::unboxBigInt(const Address& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); +} + +void MacroAssemblerLOONGARCH64Compat::unboxObject(const ValueOperand& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); +} + +void MacroAssemblerLOONGARCH64Compat::unboxObject(Register src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); +} + +void MacroAssemblerLOONGARCH64Compat::unboxObject(const Address& src, + Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); +} + +void MacroAssemblerLOONGARCH64Compat::unboxNonDouble(const ValueOperand& operand, + Register dest) { + Label isInt32, done; + SecondScratchRegisterScope scratch2(asMasm()); + splitTag(operand, scratch2); + asMasm().branchTestInt32(Assembler::Equal, scratch2, &isInt32); + + as_bstrpick_d(dest, operand.valueReg(), JSVAL_TAG_SHIFT - 1, 0); + jump(&done); + + bind(&isInt32); + as_slli_w(dest, operand.valueReg(), 0); + + bind(&done); +} + +void MacroAssemblerLOONGARCH64Compat::unboxNonDouble(const Address& src, + Register dest) { + Label isInt32, done; + loadPtr(src, dest); + SecondScratchRegisterScope scratch2(asMasm()); + splitTag(dest, scratch2); + asMasm().branchTestInt32(Assembler::Equal, scratch2, &isInt32); + + as_bstrpick_d(dest, dest, JSVAL_TAG_SHIFT - 1, 0); + jump(&done); + + bind(&isInt32); + as_slli_w(dest, dest, 0); + + bind(&done); +} + +void MacroAssemblerLOONGARCH64Compat::unboxNonDouble(const BaseIndex& src, + Register dest) { + Label isInt32, done; + computeScaledAddress(src, SecondScratchReg); + loadPtr(Address(SecondScratchReg, src.offset), dest); + splitTag(dest, SecondScratchReg); + asMasm().branchTestInt32(Assembler::Equal, SecondScratchReg, &isInt32); + + as_bstrpick_d(dest, dest, JSVAL_TAG_SHIFT - 1, 0); + jump(&done); + + bind(&isInt32); + as_slli_w(dest, dest, 0); + + bind(&done); +} + +void MacroAssemblerLOONGARCH64Compat::unboxValue(const ValueOperand& src, + AnyRegister dest) { + if (dest.isFloat()) { + Label notInt32, end; + asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); + convertInt32ToDouble(src.valueReg(), dest.fpu()); + ma_b(&end, ShortJump); + bind(¬Int32); + unboxDouble(src, dest.fpu()); + bind(&end); + } else { + unboxNonDouble(src, dest.gpr()); + } +} + +void MacroAssemblerLOONGARCH64Compat::boxDouble(FloatRegister src, + const ValueOperand& dest, + FloatRegister) { + as_movfr2gr_d(dest.valueReg(), src); +} + +void MacroAssemblerLOONGARCH64Compat::boxNonDouble(JSValueType type, Register src, + const ValueOperand& dest) { + boxValue(type, src, dest.valueReg()); +} + +void MacroAssemblerLOONGARCH64Compat::boolValueToDouble(const ValueOperand& operand, + FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + convertBoolToInt32(operand.valueReg(), scratch); + convertInt32ToDouble(scratch, dest); +} + +void MacroAssemblerLOONGARCH64Compat::int32ValueToDouble( + const ValueOperand& operand, FloatRegister dest) { + convertInt32ToDouble(operand.valueReg(), dest); +} + +void MacroAssemblerLOONGARCH64Compat::boolValueToFloat32( + const ValueOperand& operand, FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + convertBoolToInt32(operand.valueReg(), scratch); + convertInt32ToFloat32(scratch, dest); +} + +void MacroAssemblerLOONGARCH64Compat::int32ValueToFloat32( + const ValueOperand& operand, FloatRegister dest) { + convertInt32ToFloat32(operand.valueReg(), dest); +} + +void MacroAssemblerLOONGARCH64Compat::loadConstantFloat32(float f, + FloatRegister dest) { + ma_lis(dest, f); +} + +void MacroAssemblerLOONGARCH64Compat::loadInt32OrDouble(const Address& src, + FloatRegister dest) { + SecondScratchRegisterScope scratch2(asMasm()); + Label end; + + // If it's an int, convert it to double. + loadPtr(Address(src.base, src.offset), scratch2); + as_movgr2fr_d(dest, scratch2); + as_srli_d(scratch2, scratch2, JSVAL_TAG_SHIFT); + asMasm().branchTestInt32(Assembler::NotEqual, scratch2, &end); + as_ffint_d_w(dest, dest); + + bind(&end); +} + +void MacroAssemblerLOONGARCH64Compat::loadInt32OrDouble(const BaseIndex& addr, + FloatRegister dest) { + SecondScratchRegisterScope scratch2(asMasm()); + Label end; + + // If it's an int, convert it to double. + computeScaledAddress(addr, scratch2); + // Since we only have one scratch, we need to stomp over it with the tag. + loadPtr(Address(scratch2, 0), scratch2); + as_movgr2fr_d(dest, scratch2); + as_srli_d(scratch2, scratch2, JSVAL_TAG_SHIFT); + asMasm().branchTestInt32(Assembler::NotEqual, scratch2, &end); + as_ffint_d_w(dest, dest); + + bind(&end); +} + +void MacroAssemblerLOONGARCH64Compat::loadConstantDouble(double dp, + FloatRegister dest) { + ma_lid(dest, dp); +} + +Register MacroAssemblerLOONGARCH64Compat::extractObject(const Address& address, + Register scratch) { + loadPtr(Address(address.base, address.offset), scratch); + as_bstrpick_d(scratch, scratch, JSVAL_TAG_SHIFT - 1, 0); + return scratch; +} + +Register MacroAssemblerLOONGARCH64Compat::extractTag(const Address& address, + Register scratch) { + loadPtr(Address(address.base, address.offset), scratch); + as_bstrpick_d(scratch, scratch, 63, JSVAL_TAG_SHIFT); + return scratch; +} + +Register MacroAssemblerLOONGARCH64Compat::extractTag(const BaseIndex& address, + Register scratch) { + computeScaledAddress(address, scratch); + return extractTag(Address(scratch, address.offset), scratch); +} + +///////////////////////////////////////////////////////////////// +// X86/X64-common/ARM/LoongArch interface. +///////////////////////////////////////////////////////////////// +void MacroAssemblerLOONGARCH64Compat::storeValue(ValueOperand val, + const Address& dest) { + storePtr(val.valueReg(), Address(dest.base, dest.offset)); +} + +void MacroAssemblerLOONGARCH64Compat::storeValue(ValueOperand val, + const BaseIndex& dest) { + storePtr(val.valueReg(), dest); +} + +void MacroAssemblerLOONGARCH64Compat::storeValue(JSValueType type, Register reg, + Address dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(dest.base != scratch2); + + tagValue(type, reg, ValueOperand(scratch2)); + storePtr(scratch2, dest); +} + +void MacroAssemblerLOONGARCH64Compat::storeValue(JSValueType type, Register reg, + BaseIndex dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(dest.base != scratch2); + + tagValue(type, reg, ValueOperand(scratch2)); + storePtr(scratch2, dest); +} + +void MacroAssemblerLOONGARCH64Compat::storeValue(const Value& val, Address dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(dest.base != scratch2); + + if (val.isGCThing()) { + writeDataRelocation(val); + movWithPatch(ImmWord(val.asRawBits()), scratch2); + } else { + ma_li(scratch2, ImmWord(val.asRawBits())); + } + storePtr(scratch2, dest); +} + +void MacroAssemblerLOONGARCH64Compat::storeValue(const Value& val, BaseIndex dest) { + SecondScratchRegisterScope scratch2(asMasm()); + MOZ_ASSERT(dest.base != scratch2); + + if (val.isGCThing()) { + writeDataRelocation(val); + movWithPatch(ImmWord(val.asRawBits()), scratch2); + } else { + ma_li(scratch2, ImmWord(val.asRawBits())); + } + storePtr(scratch2, dest); +} + +void MacroAssemblerLOONGARCH64Compat::loadValue(Address src, ValueOperand val) { + loadPtr(src, val.valueReg()); +} + +void MacroAssemblerLOONGARCH64Compat::loadValue(const BaseIndex& src, + ValueOperand val) { + loadPtr(src, val.valueReg()); +} + +void MacroAssemblerLOONGARCH64Compat::tagValue(JSValueType type, Register payload, + ValueOperand dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dest.valueReg() != scratch); + + if (payload == dest.valueReg()) { + as_or(scratch, payload, zero); + payload = scratch; + } + ma_li(dest.valueReg(), ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type))); + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + as_bstrins_d(dest.valueReg(), payload, 31, 0); + } else { + as_bstrins_d(dest.valueReg(), payload, JSVAL_TAG_SHIFT - 1, 0); + } +} + +void MacroAssemblerLOONGARCH64Compat::pushValue(ValueOperand val) { + push(val.valueReg()); +} + +void MacroAssemblerLOONGARCH64Compat::pushValue(const Address& addr) { push(addr); } + +void MacroAssemblerLOONGARCH64Compat::popValue(ValueOperand val) { + pop(val.valueReg()); +} + +void MacroAssemblerLOONGARCH64Compat::breakpoint(uint32_t value) { + as_break(value); +} + +void MacroAssemblerLOONGARCH64Compat::handleFailureWithHandlerTail( + Label* profilerExitTail) { + // Reserve space for exception information. + int size = (sizeof(ResumeFromException) + ABIStackAlignment) & + ~(ABIStackAlignment - 1); + asMasm().subPtr(Imm32(size), StackPointer); + mov(StackPointer, a0); // Use a0 since it is a first function argument + + // Call the handler. + using Fn = void (*)(ResumeFromException * rfe); + asMasm().setupUnalignedABICall(a1); + asMasm().passABIArg(a0); + asMasm().callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + Label entryFrame; + Label catch_; + Label finally; + Label return_; + Label bailout; + Label wasm; + Label wasmCatch; + + // Already clobbered a0, so use it... + load32(Address(StackPointer, offsetof(ResumeFromException, kind)), a0); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_ENTRY_FRAME), + &entryFrame); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_CATCH), &catch_); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_FINALLY), &finally); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_FORCED_RETURN), &return_); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_BAILOUT), &bailout); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_WASM), &wasm); + asMasm().branch32(Assembler::Equal, a0, + Imm32(ResumeFromException::RESUME_WASM_CATCH), &wasmCatch); + + breakpoint(); // Invalid kind. + + // No exception handler. Load the error value, load the new stack pointer + // and return from the entry frame. + bind(&entryFrame); + asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + + // We're going to be returning by the ion calling convention + ma_pop(ra); + as_jirl(zero, ra, BOffImm16(0)); + + // If we found a catch handler, this must be a baseline frame. Restore + // state and jump to the catch block. + bind(&catch_); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, target)), a0); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), + BaselineFrameReg); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + jump(a0); + + // If we found a finally block, this must be a baseline frame. Push + // two values expected by JSOp::Retsub: the exception and BooleanValue(true). + bind(&finally); + ValueOperand exception = ValueOperand(a1); + loadValue(Address(sp, offsetof(ResumeFromException, exception)), exception); + + loadPtr(Address(sp, offsetof(ResumeFromException, target)), a0); + loadPtr(Address(sp, offsetof(ResumeFromException, framePointer)), + BaselineFrameReg); + loadPtr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp); + + pushValue(exception); + pushValue(BooleanValue(true)); + jump(a0); + + // Only used in debug mode. Return BaselineFrame->returnValue() to the + // caller. + bind(&return_); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), + BaselineFrameReg); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + loadValue( + Address(BaselineFrameReg, BaselineFrame::reverseOffsetOfReturnValue()), + JSReturnOperand); + as_or(StackPointer, BaselineFrameReg, zero); + pop(BaselineFrameReg); + + // If profiling is enabled, then update the lastProfilingFrame to refer to + // caller frame before returning. + { + Label skipProfilingInstrumentation; + // Test if profiler enabled. + AbsoluteAddress addressOfEnabled( + GetJitContext()->runtime->geckoProfiler().addressOfEnabled()); + asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + jump(profilerExitTail); + bind(&skipProfilingInstrumentation); + } + + ret(); + + // If we are bailing out to baseline to handle an exception, jump to + // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success. + bind(&bailout); + loadPtr(Address(sp, offsetof(ResumeFromException, bailoutInfo)), a2); + ma_li(ReturnReg, Imm32(1)); + loadPtr(Address(sp, offsetof(ResumeFromException, target)), a1); + jump(a1); + + // If we are throwing and the innermost frame was a wasm frame, reset SP and + // FP; SP is pointing to the unwound return address to the wasm entry, so + // we can just ret(). + bind(&wasm); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), + FramePointer); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + ret(); + + // Found a wasm catch handler, restore state and jump to it. + bind(&wasmCatch); + loadPtr(Address(sp, offsetof(ResumeFromException, target)), a1); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, framePointer)), + FramePointer); + loadPtr(Address(StackPointer, offsetof(ResumeFromException, stackPointer)), + StackPointer); + jump(a1); +} + +CodeOffset MacroAssemblerLOONGARCH64Compat::toggledJump(Label* label) { + CodeOffset ret(nextOffset().getOffset()); + ma_b(label); + return ret; +} + +CodeOffset MacroAssemblerLOONGARCH64Compat::toggledCall(JitCode* target, + bool enabled) { + ScratchRegisterScope scratch(asMasm()); + BufferOffset bo = nextOffset(); + CodeOffset offset(bo.getOffset()); // first instruction location,not changed. + addPendingJump(bo, ImmPtr(target->raw()), Relocation::JITCODE); + ma_liPatchable(scratch, ImmPtr(target->raw())); + if (enabled) { + as_jirl(ra, scratch, BOffImm16(0)); + } else { + as_nop(); + } + MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() == + ToggledCallSize(nullptr)); + return offset; // location of first instruction of call instr sequence. +} + +void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift, + Register pointer) { + if (IsShiftInScaleRange(shift)) { + computeEffectiveAddress( + BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer); + return; + } + lshift32(Imm32(shift), indexTemp32); + addPtr(indexTemp32, pointer); +} + +//}}} check_macroassembler_style + +} // namespace jit +} // namespace js diff --git a/js/src/jit/loongarch64/MacroAssembler-loongarch64.h b/js/src/jit/loongarch64/MacroAssembler-loongarch64.h new file mode 100644 index 0000000000..4e8a25ab07 --- /dev/null +++ b/js/src/jit/loongarch64/MacroAssembler-loongarch64.h @@ -0,0 +1,1088 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loongarch64_MacroAssembler_loongarch64_h +#define jit_loongarch64_MacroAssembler_loongarch64_h + +#include "jit/loongarch64/Assembler-loongarch64.h" +#include "jit/MoveResolver.h" +#include "wasm/WasmTypes.h" + +namespace js { +namespace jit { + +enum LoadStoreSize { + SizeByte = 8, + SizeHalfWord = 16, + SizeWord = 32, + SizeDouble = 64 +}; + +enum LoadStoreExtension { ZeroExtend = 0, SignExtend = 1 }; + +enum JumpKind { LongJump = 0, ShortJump = 1 }; + +static Register CallReg = t8; + +enum LiFlags { + Li64 = 0, + Li48 = 1, +}; + +struct ImmShiftedTag : public ImmWord { + explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {} + + explicit ImmShiftedTag(JSValueType type) + : ImmWord(uintptr_t(JSValueShiftedTag(JSVAL_TYPE_TO_SHIFTED_TAG(type)))) { + } +}; + +struct ImmTag : public Imm32 { + ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {} +}; + +static const int defaultShift = 3; +static_assert(1 << defaultShift == sizeof(JS::Value), + "The defaultShift is wrong"); + +// See documentation for ScratchTagScope and ScratchTagScopeRelease in +// MacroAssembler-x64.h. + +class ScratchTagScope : public SecondScratchRegisterScope { + public: + ScratchTagScope(MacroAssembler& masm, const ValueOperand&) + : SecondScratchRegisterScope(masm) {} + + void release() {} + void reacquire() {} +}; + +class ScratchTagScopeRelease { + ScratchTagScope* ts_; + + public: + explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) { + ts_->release(); + } + ~ScratchTagScopeRelease() { ts_->reacquire(); } +}; + +class MacroAssemblerLOONGARCH64 : public Assembler { + protected: + // Perform a downcast. Should be removed by Bug 996602. + MacroAssembler& asMasm(); + const MacroAssembler& asMasm() const; + + Condition ma_cmp(Register rd, Register lhs, Register rhs, Condition c); + Condition ma_cmp(Register rd, Register lhs, Imm32 imm, Condition c); + + void compareFloatingPoint(FloatFormat fmt, FloatRegister lhs, + FloatRegister rhs, DoubleCondition c, + FPConditionBit fcc = FCC0); + + public: + void ma_li(Register dest, CodeLabel* label); + void ma_li(Register dest, ImmWord imm); + void ma_liPatchable(Register dest, ImmPtr imm); + void ma_liPatchable(Register dest, ImmWord imm, LiFlags flags = Li48); + + // load + void ma_ld_b(Register dest, Address address); + void ma_ld_h(Register dest, Address address); + void ma_ld_w(Register dest, Address address); + void ma_ld_d(Register dest, Address address); + void ma_ld_bu(Register dest, Address address); + void ma_ld_hu(Register dest, Address address); + void ma_ld_wu(Register dest, Address address); + void ma_load(Register dest, Address address, LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + + // store + void ma_st_b(Register src, Address address); + void ma_st_h(Register src, Address address); + void ma_st_w(Register src, Address address); + void ma_st_d(Register src, Address address); + void ma_store(Register data, Address address, LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + + // arithmetic based ops + // add + void ma_add_d(Register rd, Register rj, Imm32 imm); + void ma_add32TestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_add32TestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + void ma_addPtrTestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_addPtrTestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + void ma_addPtrTestOverflow(Register rd, Register rj, ImmWord imm, + Label* overflow); + void ma_addPtrTestCarry(Condition cond, Register rd, Register rj, Register rk, + Label* overflow); + void ma_addPtrTestCarry(Condition cond, Register rd, Register rj, Imm32 imm, + Label* overflow); + void ma_addPtrTestCarry(Condition cond, Register rd, Register rj, ImmWord imm, + Label* overflow); + + // subtract + void ma_sub_d(Register rd, Register rj, Imm32 imm); + void ma_sub32TestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_subPtrTestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_subPtrTestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + + // multiplies. For now, there are only few that we care about. + void ma_mul_d(Register rd, Register rj, Imm32 imm); + void ma_mulh_d(Register rd, Register rj, Imm32 imm); + void ma_mulPtrTestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + + // stack + void ma_pop(Register r); + void ma_push(Register r); + + void branchWithCode(InstImm code, Label* label, JumpKind jumpKind); + // branches when done from within la-specific code + void ma_b(Register lhs, ImmWord imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Register lhs, Address addr, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Address addr, Imm32 imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Address addr, ImmGCPtr imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Address addr, Register rhs, Label* l, Condition c, + JumpKind jumpKind = LongJump) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(rhs != scratch); + ma_ld_d(scratch, addr); + ma_b(scratch, rhs, l, c, jumpKind); + } + + void ma_bl(Label* l); + + // fp instructions + void ma_lid(FloatRegister dest, double value); + + void ma_mv(FloatRegister src, ValueOperand dest); + void ma_mv(ValueOperand src, FloatRegister dest); + + void ma_fld_s(FloatRegister ft, Address address); + void ma_fld_d(FloatRegister ft, Address address); + void ma_fst_d(FloatRegister ft, Address address); + void ma_fst_s(FloatRegister ft, Address address); + + void ma_pop(FloatRegister f); + void ma_push(FloatRegister f); + + void ma_cmp_set(Register dst, Register lhs, ImmWord imm, Condition c); + void ma_cmp_set(Register dst, Register lhs, ImmPtr imm, Condition c); + void ma_cmp_set(Register dst, Address address, Imm32 imm, Condition c); + void ma_cmp_set(Register dst, Address address, ImmWord imm, Condition c); + + void moveIfZero(Register dst, Register src, Register cond) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dst != scratch && cond != scratch); + as_masknez(scratch, src, cond); + as_maskeqz(dst, dst, cond); + as_or(dst, dst, scratch); + } + void moveIfNotZero(Register dst, Register src, Register cond) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(dst != scratch && cond != scratch); + as_maskeqz(scratch, src, cond); + as_masknez(dst, dst, cond); + as_or(dst, dst, scratch); + } + + // These functions abstract the access to high part of the double precision + // float register. They are intended to work on both 32 bit and 64 bit + // floating point coprocessor. + void moveToDoubleHi(Register src, FloatRegister dest) { + as_movgr2frh_w(dest, src); + } + void moveFromDoubleHi(FloatRegister src, Register dest) { + as_movfrh2gr_s(dest, src); + } + + void moveToDouble(Register src, FloatRegister dest) { + as_movgr2fr_d(dest, src); + } + void moveFromDouble(FloatRegister src, Register dest) { + as_movfr2gr_d(dest, src); + } + + public: + void ma_li(Register dest, ImmGCPtr ptr); + + void ma_li(Register dest, Imm32 imm); + void ma_liPatchable(Register dest, Imm32 imm); + + void ma_rotr_w(Register rd, Register rj, Imm32 shift); + + void ma_fmovz(FloatFormat fmt, FloatRegister fd, FloatRegister fj, + Register rk); + void ma_fmovn(FloatFormat fmt, FloatRegister fd, FloatRegister fj, + Register rk); + + void ma_and(Register rd, Register rj, Imm32 imm, bool bit32 = false); + + void ma_or(Register rd, Register rj, Imm32 imm, bool bit32 = false); + + void ma_xor(Register rd, Register rj, Imm32 imm, bool bit32 = false); + + // load + void ma_load(Register dest, const BaseIndex& src, + LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + + // store + void ma_store(Register data, const BaseIndex& dest, + LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + void ma_store(Imm32 imm, const BaseIndex& dest, LoadStoreSize size = SizeWord, + LoadStoreExtension extension = SignExtend); + + // arithmetic based ops + // add + void ma_add_w(Register rd, Register rj, Imm32 imm); + void ma_add32TestCarry(Condition cond, Register rd, Register rj, Register rk, + Label* overflow); + void ma_add32TestCarry(Condition cond, Register rd, Register rj, Imm32 imm, + Label* overflow); + + // subtract + void ma_sub_w(Register rd, Register rj, Imm32 imm); + void ma_sub_w(Register rd, Register rj, Register rk); + void ma_sub32TestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + + // multiplies. For now, there are only few that we care about. + void ma_mul(Register rd, Register rj, Imm32 imm); + void ma_mul32TestOverflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_mul32TestOverflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + + // divisions + void ma_div_branch_overflow(Register rd, Register rj, Register rk, + Label* overflow); + void ma_div_branch_overflow(Register rd, Register rj, Imm32 imm, + Label* overflow); + + // fast mod, uses scratch registers, and thus needs to be in the assembler + // implicitly assumes that we can overwrite dest at the beginning of the + // sequence + void ma_mod_mask(Register src, Register dest, Register hold, Register remain, + int32_t shift, Label* negZero = nullptr); + + // branches when done from within la-specific code + void ma_b(Register lhs, Register rhs, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Register lhs, Imm32 imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Register lhs, ImmPtr imm, Label* l, Condition c, + JumpKind jumpKind = LongJump); + void ma_b(Register lhs, ImmGCPtr imm, Label* l, Condition c, + JumpKind jumpKind = LongJump) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(lhs != scratch); + ma_li(scratch, imm); + ma_b(lhs, scratch, l, c, jumpKind); + } + + template + void ma_b(Register lhs, const T& rhs, wasm::TrapDesc target, + Condition c, JumpKind jumpKind = LongJump) { + (void)lhs; + (void)rhs; + (void)target; + (void)c; + (void)jumpKind; + MOZ_CRASH("wasm trap branches are not supported on loongarch64 yet"); + } + + void ma_b(wasm::TrapDesc target, JumpKind jumpKind = LongJump) { + (void)target; + (void)jumpKind; + MOZ_CRASH("wasm trap jumps are not supported on loongarch64 yet"); + } + + void ma_b(Label* l, JumpKind jumpKind = LongJump); + + // fp instructions + void ma_lis(FloatRegister dest, float value); + + void ma_fst_d(FloatRegister src, BaseIndex address); + void ma_fst_s(FloatRegister src, BaseIndex address); + + void ma_fld_d(FloatRegister dest, const BaseIndex& src); + void ma_fld_s(FloatRegister dest, const BaseIndex& src); + + // FP branches + void ma_bc_s(FloatRegister lhs, FloatRegister rhs, Label* label, + DoubleCondition c, JumpKind jumpKind = LongJump, + FPConditionBit fcc = FCC0); + void ma_bc_d(FloatRegister lhs, FloatRegister rhs, Label* label, + DoubleCondition c, JumpKind jumpKind = LongJump, + FPConditionBit fcc = FCC0); + + void ma_call(ImmPtr dest); + + void ma_jump(ImmPtr dest); + + void ma_cmp_set(Register dst, Register lhs, Register rhs, Condition c); + void ma_cmp_set(Register dst, Register lhs, Imm32 imm, Condition c); + void ma_cmp_set_double(Register dst, FloatRegister lhs, FloatRegister rhs, + DoubleCondition c); + void ma_cmp_set_float32(Register dst, FloatRegister lhs, FloatRegister rhs, + DoubleCondition c); + + void moveToDoubleLo(Register src, FloatRegister dest) { + as_movgr2fr_w(dest, src); + } + void moveFromDoubleLo(FloatRegister src, Register dest) { + as_movfr2gr_s(dest, src); + } + + void moveToFloat32(Register src, FloatRegister dest) { + as_movgr2fr_w(dest, src); + } + void moveFromFloat32(FloatRegister src, Register dest) { + as_movfr2gr_s(dest, src); + } + + // Evaluate srcDest = minmax{Float32,Double}(srcDest, other). + // Handle NaN specially if handleNaN is true. + void minMaxDouble(FloatRegister srcDest, FloatRegister other, bool handleNaN, + bool isMax); + void minMaxFloat32(FloatRegister srcDest, FloatRegister other, bool handleNaN, + bool isMax); + + void loadDouble(const Address& addr, FloatRegister dest); + void loadDouble(const BaseIndex& src, FloatRegister dest); + + // Load a float value into a register, then expand it to a double. + void loadFloatAsDouble(const Address& addr, FloatRegister dest); + void loadFloatAsDouble(const BaseIndex& src, FloatRegister dest); + + void loadFloat32(const Address& addr, FloatRegister dest); + void loadFloat32(const BaseIndex& src, FloatRegister dest); + + + protected: + void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register memoryBase, + Register ptr, Register ptrScratch, AnyRegister output, + Register tmp); + void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister value, + Register memoryBase, Register ptr, Register ptrScratch, + Register tmp); +}; + +class MacroAssembler; + +class MacroAssemblerLOONGARCH64Compat : public MacroAssemblerLOONGARCH64 { + public: + using MacroAssemblerLOONGARCH64::call; + + MacroAssemblerLOONGARCH64Compat() {} + + void convertBoolToInt32(Register src, Register dest) { + ma_and(dest, src, Imm32(0xff)); + }; + void convertInt32ToDouble(Register src, FloatRegister dest) { + as_movgr2fr_w(dest, src); + as_ffint_d_w(dest, dest); + }; + void convertInt32ToDouble(const Address& src, FloatRegister dest) { + ma_fld_s(dest, src); + as_ffint_d_w(dest, dest); + }; + void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest) { + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(scratch != src.base); + MOZ_ASSERT(scratch != src.index); + computeScaledAddress(src, scratch); + convertInt32ToDouble(Address(scratch, src.offset), dest); + }; + void convertUInt32ToDouble(Register src, FloatRegister dest); + void convertUInt32ToFloat32(Register src, FloatRegister dest); + void convertDoubleToFloat32(FloatRegister src, FloatRegister dest); + void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true); + void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true); + void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true); + + void convertFloat32ToDouble(FloatRegister src, FloatRegister dest); + void convertInt32ToFloat32(Register src, FloatRegister dest); + void convertInt32ToFloat32(const Address& src, FloatRegister dest); + + void movq(Register rj, Register rd); + + void computeScaledAddress(const BaseIndex& address, Register dest); + + void computeEffectiveAddress(const Address& address, Register dest) { + ma_add_d(dest, address.base, Imm32(address.offset)); + } + + void computeEffectiveAddress(const BaseIndex& address, Register dest) { + computeScaledAddress(address, dest); + if (address.offset) { + ma_add_d(dest, dest, Imm32(address.offset)); + } + } + + void j(Label* dest) { ma_b(dest); } + + void mov(Register src, Register dest) { as_ori(dest, src, 0); } + void mov(ImmWord imm, Register dest) { ma_li(dest, imm); } + void mov(ImmPtr imm, Register dest) { + mov(ImmWord(uintptr_t(imm.value)), dest); + } + void mov(CodeLabel* label, Register dest) { ma_li(dest, label); } + void mov(Register src, Address dest) { MOZ_CRASH("NYI-IC"); } + void mov(Address src, Register dest) { MOZ_CRASH("NYI-IC"); } + + void writeDataRelocation(const Value& val) { + // Raw GC pointer relocations and Value relocations both end up in + // TraceOneDataRelocation. + if (val.isGCThing()) { + gc::Cell* cell = val.toGCThing(); + if (cell && gc::IsInsideNursery(cell)) { + embedsNurseryPointers_ = true; + } + dataRelocations_.writeUnsigned(currentOffset()); + } + } + + void branch(JitCode* c) { + ScratchRegisterScope scratch(asMasm()); + BufferOffset bo = m_buffer.nextOffset(); + addPendingJump(bo, ImmPtr(c->raw()), Relocation::JITCODE); + ma_liPatchable(scratch, ImmPtr(c->raw())); + as_jirl(zero, scratch, BOffImm16(0)); + } + void branch(const Register reg) { as_jirl(zero, reg, BOffImm16(0)); } + void nop() { as_nop(); } + void ret() { + ma_pop(ra); + as_jirl(zero, ra, BOffImm16(0)); + } + inline void retn(Imm32 n); + void push(Imm32 imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + } + void push(ImmWord imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + } + void push(ImmGCPtr imm) { + ScratchRegisterScope scratch(asMasm()); + ma_li(scratch, imm); + ma_push(scratch); + } + void push(const Address& address) { + SecondScratchRegisterScope scratch2(asMasm()); + loadPtr(address, scratch2); + ma_push(scratch2); + } + void push(Register reg) { ma_push(reg); } + void push(FloatRegister reg) { ma_push(reg); } + void pop(Register reg) { ma_pop(reg); } + void pop(FloatRegister reg) { ma_pop(reg); } + + // Emit a branch that can be toggled to a non-operation. On LOONGARCH64 we use + // "andi" instruction to toggle the branch. + // See ToggleToJmp(), ToggleToCmp(). + CodeOffset toggledJump(Label* label); + + // Emit a "jalr" or "nop" instruction. ToggleCall can be used to patch + // this instruction. + CodeOffset toggledCall(JitCode* target, bool enabled); + + static size_t ToggledCallSize(uint8_t* code) { + // Four instructions used in: MacroAssemblerLOONGARCH64Compat::toggledCall + return 4 * sizeof(uint32_t); + } + + CodeOffset pushWithPatch(ImmWord imm) { + ScratchRegisterScope scratch(asMasm()); + CodeOffset offset = movWithPatch(imm, scratch); + ma_push(scratch); + return offset; + } + + CodeOffset movWithPatch(ImmWord imm, Register dest) { + CodeOffset offset = CodeOffset(currentOffset()); + ma_liPatchable(dest, imm, Li64); + return offset; + } + CodeOffset movWithPatch(ImmPtr imm, Register dest) { + CodeOffset offset = CodeOffset(currentOffset()); + ma_liPatchable(dest, imm); + return offset; + } + + void writeCodePointer(CodeLabel* label) { + label->patchAt()->bind(currentOffset()); + m_buffer.ensureSpace(sizeof(void*)); + writeInst(-1); + writeInst(-1); + } + + void jump(Label* label) { ma_b(label); } + void jump(Register reg) { as_jirl(zero, reg, BOffImm16(0)); } + void jump(const Address& address) { + ScratchRegisterScope scratch(asMasm()); + loadPtr(address, scratch); + as_jirl(zero, scratch, BOffImm16(0)); + } + + void jump(JitCode* code) { branch(code); } + + void jump(ImmPtr ptr) { + BufferOffset bo = m_buffer.nextOffset(); + addPendingJump(bo, ptr, Relocation::HARDCODED); + ma_jump(ptr); + } + + CodeOffsetJump backedgeJump(RepatchLabel* label, Label* documentation = nullptr); + CodeOffsetJump jumpWithPatch(RepatchLabel* label, Label* documentation = nullptr); + + void splitTag(Register src, Register dest) { + as_srli_d(dest, src, JSVAL_TAG_SHIFT); + } + + void splitTag(const ValueOperand& operand, Register dest) { + splitTag(operand.valueReg(), dest); + } + + Register splitTagForTest(const ValueOperand& value) { + splitTag(value, SecondScratchReg); + return SecondScratchReg; + } + + void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) { + splitTag(value, tag); + } + + // unboxing code + void unboxNonDouble(const ValueOperand& operand, Register dest); + void unboxNonDouble(const Address& src, Register dest); + void unboxNonDouble(const BaseIndex& src, Register dest); + + void unboxNonDouble(const ValueOperand& operand, Register dest, + JSValueType type) { + unboxNonDouble(operand.valueReg(), dest, type); + } + + template + void unboxNonDouble(T src, Register dest, JSValueType type) { + MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + load32(src, dest); + return; + } + loadPtr(src, dest); + unboxNonDouble(dest, dest, type); + } + + void unboxNonDouble(Register src, Register dest, JSValueType type) { + MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + as_slli_w(dest, src, 0); + return; + } + ScratchRegisterScope scratch(asMasm()); + MOZ_ASSERT(scratch != src); + mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch); + as_xor(dest, src, scratch); + } + + template + void unboxObjectOrNull(const T& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); + // Object and null differ by a single tag bit in the boxed value. + as_bstrins_d(dest, zero, JSVAL_TAG_SHIFT + 3, JSVAL_TAG_SHIFT + 3); + } + + void unboxGCThingForGCBarrier(const Address& src, Register dest) { + loadPtr(src, dest); + as_bstrpick_d(dest, dest, JSVAL_TAG_SHIFT - 1, 0); + } + void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) { + as_bstrpick_d(dest, src.valueReg(), JSVAL_TAG_SHIFT - 1, 0); + } + + void unboxInt32(const ValueOperand& operand, Register dest); + void unboxInt32(Register src, Register dest); + void unboxInt32(const Address& src, Register dest); + void unboxInt32(const BaseIndex& src, Register dest); + void unboxBoolean(const ValueOperand& operand, Register dest); + void unboxBoolean(Register src, Register dest); + void unboxBoolean(const Address& src, Register dest); + void unboxBoolean(const BaseIndex& src, Register dest); + void unboxDouble(const ValueOperand& operand, FloatRegister dest); + void unboxDouble(Register src, Register dest); + void unboxDouble(const Address& src, FloatRegister dest); + void unboxDouble(const BaseIndex& src, FloatRegister dest); + void unboxString(const ValueOperand& operand, Register dest); + void unboxString(Register src, Register dest); + void unboxString(const Address& src, Register dest); + void unboxSymbol(const ValueOperand& src, Register dest); + void unboxSymbol(Register src, Register dest); + void unboxSymbol(const Address& src, Register dest); + void unboxBigInt(const ValueOperand& operand, Register dest); + void unboxBigInt(Register src, Register dest); + void unboxBigInt(const Address& src, Register dest); + void unboxObject(const ValueOperand& src, Register dest); + void unboxObject(Register src, Register dest); + void unboxObject(const Address& src, Register dest); + void unboxObject(const BaseIndex& src, Register dest) { + unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); + } + void unboxValue(const ValueOperand& src, AnyRegister dest); + + void notBoolean(const ValueOperand& val) { + as_xori(val.valueReg(), val.valueReg(), 1); + } + + // boxing code + void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister); + void boxDouble(FloatRegister src, const ValueOperand& dest) { + boxDouble(src, dest, ScratchDoubleReg); + } + void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest); + + // Extended unboxing API. If the payload is already in a register, returns + // that register. Otherwise, provides a move to the given scratch register, + // and returns that. + [[nodiscard]] Register extractObject(const Address& address, + Register scratch); + [[nodiscard]] Register extractObject(const ValueOperand& value, + Register scratch) { + unboxObject(value, scratch); + return scratch; + } + [[nodiscard]] Register extractString(const ValueOperand& value, + Register scratch) { + unboxString(value, scratch); + return scratch; + } + [[nodiscard]] Register extractSymbol(const ValueOperand& value, + Register scratch) { + unboxSymbol(value, scratch); + return scratch; + } + [[nodiscard]] Register extractInt32(const ValueOperand& value, + Register scratch) { + unboxInt32(value, scratch); + return scratch; + } + [[nodiscard]] Register extractBoolean(const ValueOperand& value, + Register scratch) { + unboxBoolean(value, scratch); + return scratch; + } + [[nodiscard]] Register extractTag(const Address& address, Register scratch); + [[nodiscard]] Register extractTag(const BaseIndex& address, Register scratch); + [[nodiscard]] Register extractTag(const ValueOperand& value, + Register scratch) { + splitTag(value, scratch); + return scratch; + } + + inline void ensureDouble(const ValueOperand& source, FloatRegister dest, + Label* failure); + + void boolValueToDouble(const ValueOperand& operand, FloatRegister dest); + void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest); + void loadInt32OrDouble(const Address& src, FloatRegister dest); + void loadInt32OrDouble(const BaseIndex& addr, FloatRegister dest); + void loadConstantDouble(double dp, FloatRegister dest); + + void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest); + void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest); + void loadConstantFloat32(float f, FloatRegister dest); + + void testNullSet(Condition cond, const ValueOperand& value, Register dest); + + void testObjectSet(Condition cond, const ValueOperand& value, Register dest); + + void testUndefinedSet(Condition cond, const ValueOperand& value, + Register dest); + + // higher level tag testing code + Address ToPayload(Address value) { return value; } + + void moveValue(const Value& val, Register dest); + void moveValue(const TypedOrValueRegister& src, const ValueOperand& dest); + void moveValue(const Value& src, const ValueOperand& dest); + void moveValue(const ValueOperand& src, const ValueOperand& dest); + + template + void loadUnboxedValue(const T& address, MIRType type, AnyRegister dest) { + if (dest.isFloat()) { + loadInt32OrDouble(address, dest.fpu()); + } else { + unboxNonDouble(address, dest.gpr(), ValueTypeFromMIRType(type)); + } + } + + template + void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes) { + switch (nbytes) { + case 8: + unboxNonDouble(value, ScratchRegister); + storePtr(ScratchRegister, address); + return; + case 4: + store32(value.valueReg(), address); + return; + case 1: + store8(value.valueReg(), address); + return; + default: + MOZ_CRASH("Bad payload width"); + } + } + + void storeUnboxedPayload(ValueOperand value, BaseIndex address, size_t nbytes, + JSValueType type) { + switch (nbytes) { + case 8: { + ScratchRegisterScope scratch(asMasm()); + SecondScratchRegisterScope scratch2(asMasm()); + if (type == JSVAL_TYPE_OBJECT) { + unboxObjectOrNull(value, scratch2); + } else { + unboxNonDouble(value, scratch2, type); + } + computeEffectiveAddress(address, scratch); + as_st_d(scratch2, scratch, 0); + return; + } + case 4: + store32(value.valueReg(), address); + return; + case 1: + store8(value.valueReg(), address); + return; + default: + MOZ_CRASH("Bad payload width"); + } + } + + void storeUnboxedPayload(ValueOperand value, Address address, size_t nbytes, + JSValueType type) { + switch (nbytes) { + case 8: { + SecondScratchRegisterScope scratch2(asMasm()); + if (type == JSVAL_TYPE_OBJECT) { + unboxObjectOrNull(value, scratch2); + } else { + unboxNonDouble(value, scratch2, type); + } + storePtr(scratch2, address); + return; + } + case 4: + store32(value.valueReg(), address); + return; + case 1: + store8(value.valueReg(), address); + return; + default: + MOZ_CRASH("Bad payload width"); + } + } + + void boxValue(JSValueType type, Register src, Register dest) { + ScratchRegisterScope scratch(asMasm()); + if (src == dest) { + as_ori(scratch, src, 0); + src = scratch; + } +#ifdef DEBUG + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + Label upper32BitsSignExtended; + as_slli_w(dest, src, 0); + ma_b(src, dest, &upper32BitsSignExtended, Equal, ShortJump); + breakpoint(); + bind(&upper32BitsSignExtended); + } +#endif + ma_li(dest, ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type))); + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + as_bstrins_d(dest, src, 31, 0); + } else { + as_bstrins_d(dest, src, JSVAL_TAG_SHIFT - 1, 0); + } + } + + void storeValue(ValueOperand val, const Address& dest); + void storeValue(ValueOperand val, const BaseIndex& dest); + void storeValue(JSValueType type, Register reg, Address dest); + void storeValue(JSValueType type, Register reg, BaseIndex dest); + void storeValue(const Value& val, Address dest); + void storeValue(const Value& val, BaseIndex dest); + void storeValue(const Address& src, const Address& dest, Register temp) { + loadPtr(src, temp); + storePtr(temp, dest); + } + + void storePrivateValue(Register src, const Address& dest) { + storePtr(src, dest); + } + void storePrivateValue(ImmGCPtr imm, const Address& dest) { + storePtr(imm, dest); + } + + void loadValue(Address src, ValueOperand val); + void loadValue(const BaseIndex& src, ValueOperand val); + + void loadUnalignedValue(const Address& src, ValueOperand dest) { + loadValue(src, dest); + } + + void tagValue(JSValueType type, Register payload, ValueOperand dest); + + void pushValue(ValueOperand val); + void popValue(ValueOperand val); + void pushValue(const Value& val) { + if (val.isGCThing()) { + ScratchRegisterScope scratch(asMasm()); + writeDataRelocation(val); + movWithPatch(ImmWord(val.asRawBits()), scratch); + push(scratch); + } else { + push(ImmWord(val.asRawBits())); + } + } + void pushValue(JSValueType type, Register reg) { + SecondScratchRegisterScope scratch2(asMasm()); + boxValue(type, reg, scratch2); + push(scratch2); + } + void pushValue(const Address& addr); + + void handleFailureWithHandlerTail(Label* profilerExitTail); + + ///////////////////////////////////////////////////////////////// + // Common interface. + ///////////////////////////////////////////////////////////////// + public: + // The following functions are exposed for use in platform-shared code. + + inline void incrementInt32Value(const Address& addr); + + void move32(Imm32 imm, Register dest); + void move32(Register src, Register dest); + + void movePtr(Register src, Register dest); + void movePtr(ImmWord imm, Register dest); + void movePtr(ImmPtr imm, Register dest); + void movePtr(wasm::SymbolicAddress imm, Register dest); + void movePtr(ImmGCPtr imm, Register dest); + + void load8SignExtend(const Address& address, Register dest); + void load8SignExtend(const BaseIndex& src, Register dest); + + void load8ZeroExtend(const Address& address, Register dest); + void load8ZeroExtend(const BaseIndex& src, Register dest); + + void load16SignExtend(const Address& address, Register dest); + void load16SignExtend(const BaseIndex& src, Register dest); + + template + void load16UnalignedSignExtend(const S& src, Register dest) { + load16SignExtend(src, dest); + } + + void load16ZeroExtend(const Address& address, Register dest); + void load16ZeroExtend(const BaseIndex& src, Register dest); + + template + void load16UnalignedZeroExtend(const S& src, Register dest) { + load16ZeroExtend(src, dest); + } + + void load32(const Address& address, Register dest); + void load32(const BaseIndex& address, Register dest); + void load32(AbsoluteAddress address, Register dest); + void load32(wasm::SymbolicAddress address, Register dest); + + template + void load32Unaligned(const S& src, Register dest) { + load32(src, dest); + } + + void load64(const Address& address, Register64 dest) { + loadPtr(address, dest.reg); + } + void load64(const BaseIndex& address, Register64 dest) { + loadPtr(address, dest.reg); + } + + template + void load64Unaligned(const S& src, Register64 dest) { + load64(src, dest); + } + + void loadPtr(const Address& address, Register dest); + void loadPtr(const BaseIndex& src, Register dest); + void loadPtr(AbsoluteAddress address, Register dest); + void loadPtr(wasm::SymbolicAddress address, Register dest); + + void loadPrivate(const Address& address, Register dest); + + void store8(Register src, const Address& address); + void store8(Imm32 imm, const Address& address); + void store8(Register src, const BaseIndex& address); + void store8(Imm32 imm, const BaseIndex& address); + + void store16(Register src, const Address& address); + void store16(Imm32 imm, const Address& address); + void store16(Register src, const BaseIndex& address); + void store16(Imm32 imm, const BaseIndex& address); + + template + void store16Unaligned(Register src, const T& dest) { + store16(src, dest); + } + + void store32(Register src, AbsoluteAddress address); + void store32(Register src, const Address& address); + void store32(Register src, const BaseIndex& address); + void store32(Imm32 src, const Address& address); + void store32(Imm32 src, const BaseIndex& address); + + // NOTE: This will use second scratch on LOONGARCH64. Only ARM needs the + // implementation without second scratch. + void store32_NoSecondScratch(Imm32 src, const Address& address) { + store32(src, address); + } + + template + void store32Unaligned(Register src, const T& dest) { + store32(src, dest); + } + + void store64(Imm64 imm, Address address) { + storePtr(ImmWord(imm.value), address); + } + void store64(Imm64 imm, const BaseIndex& address) { + storePtr(ImmWord(imm.value), address); + } + + void store64(Register64 src, Address address) { storePtr(src.reg, address); } + void store64(Register64 src, const BaseIndex& address) { + storePtr(src.reg, address); + } + + template + void store64Unaligned(Register64 src, const T& dest) { + store64(src, dest); + } + + template + void storePtr(ImmWord imm, T address); + template + void storePtr(ImmPtr imm, T address); + template + void storePtr(ImmGCPtr imm, T address); + void storePtr(Register src, const Address& address); + void storePtr(Register src, const BaseIndex& address); + void storePtr(Register src, AbsoluteAddress dest); + + void moveDouble(FloatRegister src, FloatRegister dest) { + as_fmov_d(dest, src); + } + + void zeroDouble(FloatRegister reg) { moveToDouble(zero, reg); } + + void convertUInt64ToDouble(Register src, FloatRegister dest); + static bool convertUInt64ToDoubleNeedsTemp(); + void convertUInt64ToDouble(Register64 src, FloatRegister dest, Register temp); + + void breakpoint(uint32_t value = 0); + + void checkStackAlignment() { +#ifdef DEBUG + Label aligned; + ScratchRegisterScope scratch(asMasm()); + as_andi(scratch, sp, ABIStackAlignment - 1); + ma_b(scratch, zero, &aligned, Equal, ShortJump); + breakpoint(); + bind(&aligned); +#endif + }; + + static void calculateAlignedStackPointer(void** stackPointer); + + void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest) { + (void)globalDataOffset; + (void)dest; + MOZ_CRASH("wasm globals are not supported on loongarch64 yet"); + } + void loadWasmPinnedRegsFromTls() { + MOZ_CRASH("wasm pinned registers are not supported on loongarch64 yet"); + } + + void cmpPtrSet(Assembler::Condition cond, Address lhs, ImmPtr rhs, + Register dest); + void cmpPtrSet(Assembler::Condition cond, Register lhs, Address rhs, + Register dest); + void cmpPtrSet(Assembler::Condition cond, Address lhs, Register rhs, + Register dest); + + void cmp32Set(Assembler::Condition cond, Register lhs, Address rhs, + Register dest); + + protected: + bool buildOOLFakeExitFrame(void* fakeReturnAddr); + + void wasmLoadI64Impl(const wasm::MemoryAccessDesc& access, + Register memoryBase, Register ptr, Register ptrScratch, + Register64 output, Register tmp); + void wasmStoreI64Impl(const wasm::MemoryAccessDesc& access, Register64 value, + Register memoryBase, Register ptr, Register ptrScratch, + Register tmp); + + public: + CodeOffset labelForPatch() { + return CodeOffset(nextOffset().getOffset()); + } + + void lea(Operand addr, Register dest) { + ma_add_d(dest, addr.baseReg(), Imm32(addr.disp())); + } + + void abiret() { as_jirl(zero, ra, BOffImm16(0)); } + + void moveFloat32(FloatRegister src, FloatRegister dest) { + as_fmov_s(dest, src); + } + + // Instrumentation for entering and leaving the profiler. + void profilerEnterFrame(Register framePtr, Register scratch); + void profilerExitFrame(); +}; + +typedef MacroAssemblerLOONGARCH64Compat MacroAssemblerSpecific; + +} // namespace jit +} // namespace js + +#endif /* jit_loongarch64_MacroAssembler_loongarch64_h */ diff --git a/js/src/jit/loongarch64/SharedIC-loongarch64.cpp b/js/src/jit/loongarch64/SharedIC-loongarch64.cpp new file mode 100644 index 0000000000..6903f1958b --- /dev/null +++ b/js/src/jit/loongarch64/SharedIC-loongarch64.cpp @@ -0,0 +1,190 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jsiter.h" + +#include "jit/BaselineCompiler.h" +#include "jit/BaselineIC.h" +#include "jit/BaselineJIT.h" +#include "jit/Linker.h" +#include "jit/SharedICHelpers.h" + +#include "jsboolinlines.h" + +using namespace js; +using namespace js::jit; + +namespace js { +namespace jit { + +// ICBinaryArith_Int32 + +bool +ICBinaryArith_Int32::Compiler::generateStubCode(MacroAssembler& masm) +{ + // Guard that R0 is an integer and R1 is an integer. + Label failure; + masm.branchTestInt32(Assembler::NotEqual, R0, &failure); + masm.branchTestInt32(Assembler::NotEqual, R1, &failure); + + // Add R0 and R1. Don't need to explicitly unbox, just use R2's valueReg. + Register scratchReg = R2.valueReg(); + + Label goodMul, divTest1, divTest2; + switch(op_) { + case JSOP_ADD: + masm.unboxInt32(R0, ExtractTemp0); + masm.unboxInt32(R1, ExtractTemp1); + masm.ma_addTestOverflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); + masm.boxValue(JSVAL_TYPE_INT32, scratchReg, R0.valueReg()); + break; + case JSOP_SUB: + masm.unboxInt32(R0, ExtractTemp0); + masm.unboxInt32(R1, ExtractTemp1); + masm.ma_subTestOverflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); + masm.boxValue(JSVAL_TYPE_INT32, scratchReg, R0.valueReg()); + break; + case JSOP_MUL: { + masm.unboxInt32(R0, ExtractTemp0); + masm.unboxInt32(R1, ExtractTemp1); + masm.ma_mul_branch_overflow(scratchReg, ExtractTemp0, ExtractTemp1, &failure); + + masm.ma_b(scratchReg, Imm32(0), &goodMul, Assembler::NotEqual, ShortJump); + + // Result is -0 if operands have different signs. + masm.as_xor(t8, ExtractTemp0, ExtractTemp1); + masm.ma_b(t8, Imm32(0), &failure, Assembler::LessThan, ShortJump); + + masm.bind(&goodMul); + masm.boxValue(JSVAL_TYPE_INT32, scratchReg, R0.valueReg()); + break; + } + case JSOP_DIV: + case JSOP_MOD: { + masm.unboxInt32(R0, ExtractTemp0); + masm.unboxInt32(R1, ExtractTemp1); + // Check for INT_MIN / -1, it results in a double. + masm.ma_b(ExtractTemp0, Imm32(INT_MIN), &divTest1, Assembler::NotEqual, ShortJump); + masm.ma_b(ExtractTemp1, Imm32(-1), &failure, Assembler::Equal, ShortJump); + masm.bind(&divTest1); + + // Check for division by zero + masm.ma_b(ExtractTemp1, Imm32(0), &failure, Assembler::Equal, ShortJump); + + // Check for 0 / X with X < 0 (results in -0). + masm.ma_b(ExtractTemp0, Imm32(0), &divTest2, Assembler::NotEqual, ShortJump); + masm.ma_b(ExtractTemp1, Imm32(0), &failure, Assembler::LessThan, ShortJump); + masm.bind(&divTest2); + + masm.as_div(ExtractTemp0, ExtractTemp1); + + if (op_ == JSOP_DIV) { + // Result is a double if the remainder != 0. + masm.as_mfhi(scratchReg); + masm.ma_b(scratchReg, Imm32(0), &failure, Assembler::NotEqual, ShortJump); + masm.as_mflo(scratchReg); + masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); + } else { + Label done; + // If X % Y == 0 and X < 0, the result is -0. + masm.as_mfhi(scratchReg); + masm.ma_b(scratchReg, Imm32(0), &done, Assembler::NotEqual, ShortJump); + masm.ma_b(ExtractTemp0, Imm32(0), &failure, Assembler::LessThan, ShortJump); + masm.bind(&done); + masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); + } + break; + } + case JSOP_BITOR: + masm.as_or(R0.valueReg() , R0.valueReg(), R1.valueReg()); + break; + case JSOP_BITXOR: + masm.as_xor(scratchReg, R0.valueReg(), R1.valueReg()); + masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); + break; + case JSOP_BITAND: + masm.as_and(R0.valueReg() , R0.valueReg(), R1.valueReg()); + break; + case JSOP_LSH: + masm.unboxInt32(R0, ExtractTemp0); + masm.unboxInt32(R1, ExtractTemp1); + // MIPS will only use 5 lowest bits in R1 as shift offset. + masm.ma_sll(scratchReg, ExtractTemp0, ExtractTemp1); + masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); + break; + case JSOP_RSH: + masm.unboxInt32(R0, ExtractTemp0); + masm.unboxInt32(R1, ExtractTemp1); + masm.ma_sra(scratchReg, ExtractTemp0, ExtractTemp1); + masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); + break; + case JSOP_URSH: + masm.unboxInt32(R0, ExtractTemp0); + masm.unboxInt32(R1, ExtractTemp1); + masm.ma_srl(scratchReg, ExtractTemp0, ExtractTemp1); + if (allowDouble_) { + Label toUint; + masm.ma_b(scratchReg, Imm32(0), &toUint, Assembler::LessThan, ShortJump); + + // Move result and box for return. + masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); + EmitReturnFromIC(masm); + + masm.bind(&toUint); + masm.convertUInt32ToDouble(scratchReg, FloatReg1); + masm.boxDouble(FloatReg1, R0); + } else { + masm.ma_b(scratchReg, Imm32(0), &failure, Assembler::LessThan, ShortJump); + // Move result for return. + masm.tagValue(JSVAL_TYPE_INT32, scratchReg, R0); + } + break; + default: + MOZ_CRASH("Unhandled op for BinaryArith_Int32."); + } + + EmitReturnFromIC(masm); + + // Failure case - jump to next stub + masm.bind(&failure); + EmitStubGuardFailure(masm); + + return true; +} + +bool +ICUnaryArith_Int32::Compiler::generateStubCode(MacroAssembler& masm) +{ + Label failure; + masm.branchTestInt32(Assembler::NotEqual, R0, &failure); + + switch (op) { + case JSOP_BITNOT: + masm.not32(R0.valueReg()); + masm.tagValue(JSVAL_TYPE_INT32, R0.valueReg(), R0); + break; + case JSOP_NEG: + masm.unboxInt32(R0, ExtractTemp0); + // Guard against 0 and MIN_INT, both result in a double. + masm.branchTest32(Assembler::Zero, ExtractTemp0, Imm32(INT32_MAX), &failure); + + masm.neg32(ExtractTemp0); + masm.tagValue(JSVAL_TYPE_INT32, ExtractTemp0, R0); + break; + default: + MOZ_CRASH("Unexpected op"); + return false; + } + + EmitReturnFromIC(masm); + + masm.bind(&failure); + EmitStubGuardFailure(masm); + return true; +} + + +} // namespace jit +} // namespace js diff --git a/js/src/jit/loongarch64/SharedICHelpers-loongarch64.h b/js/src/jit/loongarch64/SharedICHelpers-loongarch64.h new file mode 100644 index 0000000000..ab13b04752 --- /dev/null +++ b/js/src/jit/loongarch64/SharedICHelpers-loongarch64.h @@ -0,0 +1,381 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loongarch64_SharedICHelpers_loongarch64_h +#define jit_loongarch64_SharedICHelpers_loongarch64_h + +#include "jit/BaselineFrame.h" +#include "jit/BaselineIC.h" +#include "jit/MacroAssembler.h" +#include "jit/SharedICRegisters.h" + +namespace js { +namespace jit { + +// Distance from sp to the top Value inside an IC stub (no return address on +// the stack on LoongArch). +static const size_t ICStackValueOffset = 0; + +inline void +EmitRestoreTailCallReg(MacroAssembler& masm) +{ + // No-op on LoongArch because ra register is always holding the return address. +} + +inline void +EmitRepushTailCallReg(MacroAssembler& masm) +{ + // No-op on LoongArch because ra register is always holding the return address. +} + +inline void +EmitCallIC(CodeOffset* patchOffset, MacroAssembler& masm) +{ + // Move ICEntry offset into ICStubReg. + CodeOffset offset = masm.movWithPatch(ImmWord(-1), ICStubReg); + *patchOffset = offset; + + // Load stub pointer into ICStubReg. + masm.loadPtr(Address(ICStubReg, ICEntry::offsetOfFirstStub()), ICStubReg); + + // Load stubcode pointer from BaselineStubEntry. + // R2 won't be active when we call ICs, so we can use it as scratch. + masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg()); + + // Call the stubcode via a direct jump-and-link + masm.call(R2.scratchReg()); +} + +inline void +EmitEnterTypeMonitorIC(MacroAssembler& masm, + size_t monitorStubOffset = ICMonitoredStub::offsetOfFirstMonitorStub()) +{ + // This is expected to be called from within an IC, when ICStubReg + // is properly initialized to point to the stub. + masm.loadPtr(Address(ICStubReg, (uint32_t) monitorStubOffset), ICStubReg); + + // Load stubcode pointer from BaselineStubEntry. + // R2 won't be active when we call ICs, so we can use it. + masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg()); + + // Jump to the stubcode. + masm.branch(R2.scratchReg()); +} + +inline void +EmitReturnFromIC(MacroAssembler& masm) +{ + masm.branch(ra); +} + +inline void +EmitChangeICReturnAddress(MacroAssembler& masm, Register reg) +{ + masm.movePtr(reg, ra); +} + +inline void +EmitBaselineTailCallVM(JitCode* target, MacroAssembler& masm, uint32_t argSize) +{ + Register scratch = R2.scratchReg(); + + // Compute frame size. + masm.movePtr(BaselineFrameReg, scratch); + masm.addPtr(Imm32(BaselineFrame::FramePointerOffset), scratch); + masm.subPtr(BaselineStackReg, scratch); + + // Store frame size without VMFunction arguments for GC marking. + masm.subPtr(Imm32(argSize), scratch); + masm.store32(scratch, Address(BaselineFrameReg, BaselineFrame::reverseOffsetOfFrameSize())); + masm.addPtr(Imm32(argSize), scratch); + + // Push frame descriptor and perform the tail call. + // ICTailCallReg (ra) already contains the return address (as we + // keep it there through the stub calls), but the VMWrapper code being + // called expects the return address to also be pushed on the stack. + MOZ_ASSERT(ICTailCallReg == ra); + masm.makeFrameDescriptor(scratch, JitFrame_BaselineJS, ExitFrameLayout::Size()); + masm.subPtr(Imm32(sizeof(CommonFrameLayout)), StackPointer); + masm.storePtr(scratch, Address(StackPointer, CommonFrameLayout::offsetOfDescriptor())); + masm.storePtr(ra, Address(StackPointer, CommonFrameLayout::offsetOfReturnAddress())); + + masm.branch(target); +} + +inline void +EmitIonTailCallVM(JitCode* target, MacroAssembler& masm, uint32_t stackSize) +{ + Register scratch = R2.scratchReg(); + + masm.loadPtr(Address(sp, stackSize), scratch); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch); + masm.addPtr(Imm32(stackSize + JitStubFrameLayout::Size() - sizeof(intptr_t)), scratch); + + // Push frame descriptor and perform the tail call. + MOZ_ASSERT(ICTailCallReg == ra); + masm.makeFrameDescriptor(scratch, JitFrame_IonJS, ExitFrameLayout::Size()); + masm.push(scratch); + masm.push(ICTailCallReg); + masm.branch(target); +} + +inline void +EmitBaselineCreateStubFrameDescriptor(MacroAssembler& masm, Register reg, uint32_t headerSize) +{ + // Compute stub frame size. We have to add two pointers: the stub reg and + // previous frame pointer pushed by EmitEnterStubFrame. + masm.movePtr(BaselineFrameReg, reg); + masm.addPtr(Imm32(sizeof(intptr_t) * 2), reg); + masm.subPtr(BaselineStackReg, reg); + + masm.makeFrameDescriptor(reg, JitFrame_BaselineStub, headerSize); +} + +inline void +EmitBaselineCallVM(JitCode* target, MacroAssembler& masm) +{ + Register scratch = R2.scratchReg(); + EmitBaselineCreateStubFrameDescriptor(masm, scratch, ExitFrameLayout::Size()); + masm.push(scratch); + masm.call(target); +} + +inline void +EmitIonCallVM(JitCode* target, size_t stackSlots, MacroAssembler& masm) +{ + uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonStub, + ExitFrameLayout::Size()); + masm.Push(Imm32(descriptor)); + masm.callJit(target); + + // Remove rest of the frame left on the stack. We remove the return address + // which is implicitly popped when returning. + size_t framePop = sizeof(ExitFrameLayout) - sizeof(void*); + + // Pop arguments from framePushed. + masm.implicitPop(stackSlots * sizeof(void*) + framePop); +} + +struct BaselineStubFrame { + uintptr_t savedFrame; + uintptr_t savedStub; + uintptr_t returnAddress; + uintptr_t descriptor; +}; + +static const uint32_t STUB_FRAME_SIZE = sizeof(BaselineStubFrame); +static const uint32_t STUB_FRAME_SAVED_STUB_OFFSET = offsetof(BaselineStubFrame, savedStub); + +inline void +EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) +{ + MOZ_ASSERT(scratch != ICTailCallReg); + + // Compute frame size. + masm.movePtr(BaselineFrameReg, scratch); + masm.addPtr(Imm32(BaselineFrame::FramePointerOffset), scratch); + masm.subPtr(BaselineStackReg, scratch); + + masm.store32(scratch, Address(BaselineFrameReg, BaselineFrame::reverseOffsetOfFrameSize())); + + // Note: when making changes here, don't forget to update + // BaselineStubFrame if needed. + + // Push frame descriptor and return address. + masm.makeFrameDescriptor(scratch, JitFrame_BaselineJS, BaselineStubFrameLayout::Size()); + masm.subPtr(Imm32(STUB_FRAME_SIZE), StackPointer); + masm.storePtr(scratch, Address(StackPointer, offsetof(BaselineStubFrame, descriptor))); + masm.storePtr(ICTailCallReg, Address(StackPointer, + offsetof(BaselineStubFrame, returnAddress))); + + // Save old frame pointer, stack pointer and stub reg. + masm.storePtr(ICStubReg, Address(StackPointer, + offsetof(BaselineStubFrame, savedStub))); + masm.storePtr(BaselineFrameReg, Address(StackPointer, + offsetof(BaselineStubFrame, savedFrame))); + masm.movePtr(BaselineStackReg, BaselineFrameReg); + + // Stack should remain aligned. + masm.assertStackAlignment(sizeof(Value), 0); +} + +inline void +EmitIonEnterStubFrame(MacroAssembler& masm, Register scratch) +{ + MOZ_ASSERT(ICTailCallReg == ra); + + // In LoongArch the ra register contains the return address, + // but in jit frames we expect it to be on the stack. As a result + // push the link register (which is actually part of the previous frame. + // Therefore using push instead of Push). + masm.push(ICTailCallReg); + + masm.Push(ICStubReg); +} + +inline void +EmitBaselineLeaveStubFrame(MacroAssembler& masm, bool calledIntoIon = false) +{ + // Ion frames do not save and restore the frame pointer. If we called + // into Ion, we have to restore the stack pointer from the frame descriptor. + // If we performed a VM call, the descriptor has been popped already so + // in that case we use the frame pointer. + if (calledIntoIon) { + masm.pop(ScratchRegister); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), ScratchRegister); + masm.addPtr(ScratchRegister, BaselineStackReg); + } else { + masm.movePtr(BaselineFrameReg, BaselineStackReg); + } + + masm.loadPtr(Address(StackPointer, offsetof(BaselineStubFrame, savedFrame)), + BaselineFrameReg); + masm.loadPtr(Address(StackPointer, offsetof(BaselineStubFrame, savedStub)), + ICStubReg); + + // Load the return address. + masm.loadPtr(Address(StackPointer, offsetof(BaselineStubFrame, returnAddress)), + ICTailCallReg); + + // Discard the frame descriptor. + masm.loadPtr(Address(StackPointer, offsetof(BaselineStubFrame, descriptor)), ScratchRegister); + masm.addPtr(Imm32(STUB_FRAME_SIZE), StackPointer); +} + +inline void +EmitIonLeaveStubFrame(MacroAssembler& masm) +{ + masm.Pop(ICStubReg); + masm.pop(ICTailCallReg); // See EmitIonEnterStubFrame for explanation on pop/Pop. +} + +inline void +EmitStowICValues(MacroAssembler& masm, int values) +{ + MOZ_ASSERT(values >= 0 && values <= 2); + switch(values) { + case 1: + // Stow R0 + masm.Push(R0); + break; + case 2: + // Stow R0 and R1 + masm.Push(R0); + masm.Push(R1); + } +} + +inline void +EmitUnstowICValues(MacroAssembler& masm, int values, bool discard = false) +{ + MOZ_ASSERT(values >= 0 && values <= 2); + switch(values) { + case 1: + // Unstow R0. + if (discard) + masm.addPtr(Imm32(sizeof(Value)), BaselineStackReg); + else + masm.popValue(R0); + break; + case 2: + // Unstow R0 and R1. + if (discard) { + masm.addPtr(Imm32(sizeof(Value) * 2), BaselineStackReg); + } else { + masm.popValue(R1); + masm.popValue(R0); + } + break; + } + masm.adjustFrame(-values * sizeof(Value)); +} + +inline void +EmitCallTypeUpdateIC(MacroAssembler& masm, JitCode* code, uint32_t objectOffset) +{ + // R0 contains the value that needs to be typechecked. + // The object we're updating is a boxed Value on the stack, at offset + // objectOffset from $sp, excluding the return address. + + // Save the current ICStubReg to stack, as well as the TailCallReg, + // since on LoongArch, the $ra is live. + masm.subPtr(Imm32(2 * sizeof(intptr_t)), StackPointer); + masm.storePtr(ICStubReg, Address(StackPointer, sizeof(intptr_t))); + masm.storePtr(ICTailCallReg, Address(StackPointer, 0)); + + // This is expected to be called from within an IC, when ICStubReg + // is properly initialized to point to the stub. + masm.loadPtr(Address(ICStubReg, ICUpdatedStub::offsetOfFirstUpdateStub()), + ICStubReg); + + // Load stubcode pointer from ICStubReg into ICTailCallReg. + masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg()); + + // Call the stubcode. + masm.call(R2.scratchReg()); + + // Restore the old stub reg and tailcall reg. + masm.loadPtr(Address(StackPointer, 0), ICTailCallReg); + masm.loadPtr(Address(StackPointer, sizeof(intptr_t)), ICStubReg); + masm.addPtr(Imm32(2 * sizeof(intptr_t)), StackPointer); + + // The update IC will store 0 or 1 in R1.scratchReg() reflecting if the + // value in R0 type-checked properly or not. + Label success; + masm.ma_b(R1.scratchReg(), Imm32(1), &success, Assembler::Equal, ShortJump); + + // If the IC failed, then call the update fallback function. + EmitBaselineEnterStubFrame(masm, R1.scratchReg()); + + masm.loadValue(Address(BaselineStackReg, STUB_FRAME_SIZE + objectOffset), R1); + + masm.Push(R0); + masm.Push(R1); + masm.Push(ICStubReg); + + // Load previous frame pointer, push BaselineFrame*. + masm.loadPtr(Address(BaselineFrameReg, 0), R0.scratchReg()); + masm.pushBaselineFramePtr(R0.scratchReg(), R0.scratchReg()); + + EmitBaselineCallVM(code, masm); + EmitBaselineLeaveStubFrame(masm); + + // Success at end. + masm.bind(&success); +} + +template +inline void +EmitPreBarrier(MacroAssembler& masm, const AddrType& addr, MIRType type) +{ + // On LoongArch, $ra is clobbered by patchableCallPreBarrier. Save it first. + masm.push(ra); + masm.patchableCallPreBarrier(addr, type); + masm.pop(ra); +} + +inline void +EmitStubGuardFailure(MacroAssembler& masm) +{ + // NOTE: This routine assumes that the stub guard code left the stack in + // the same state it was in when it was entered. + + // BaselineStubEntry points to the current stub. + + // Load next stub into ICStubReg + masm.loadPtr(Address(ICStubReg, ICStub::offsetOfNext()), ICStubReg); + + // Load stubcode pointer from BaselineStubEntry into scratch register. + masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg()); + + // Return address is already loaded, just jump to the next stubcode. + MOZ_ASSERT(ICTailCallReg == ra); + masm.branch(R2.scratchReg()); +} + +} // namespace jit +} // namespace js + +#endif /* jit_loongarch64_SharedICHelpers_loongarch64_h */ diff --git a/js/src/jit/loongarch64/SharedICRegisters-loongarch64.h b/js/src/jit/loongarch64/SharedICRegisters-loongarch64.h new file mode 100644 index 0000000000..1de49dc39a --- /dev/null +++ b/js/src/jit/loongarch64/SharedICRegisters-loongarch64.h @@ -0,0 +1,53 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_loongarch64_SharedICRegisters_loongarch64_h +#define jit_loongarch64_SharedICRegisters_loongarch64_h + +#include "jit/loongarch64/Assembler-loongarch64.h" +#include "jit/Registers.h" +#include "jit/RegisterSets.h" + +namespace js { +namespace jit { + +// Must be a callee-saved register for preservation around generateEnterJIT(). +static constexpr Register BaselineFrameReg = s0; +static constexpr Register BaselineStackReg = sp; + +// ValueOperands R0, R1, and R2. +// R0 == JSReturnReg, and R2 uses registers not preserved across calls. R1 value +// should be preserved across calls. +static constexpr ValueOperand R0(a2); +static constexpr ValueOperand R1(s1); +static constexpr ValueOperand R2(a0); + +// ICTailCallReg and ICStubReg +// These use registers that are not preserved across calls. +static constexpr Register ICTailCallReg = ra; +static constexpr Register ICStubReg = t0; + +// ExtractTemps must be callee-save registers: +// ICSetProp_Native::Compiler::generateStubCode() stores the object +// in ExtractTemp0, but then calls callTypeUpdateIC(), which clobbers +// caller-save registers. +static constexpr Register ExtractTemp0 = s7; +static constexpr Register ExtractTemp1 = s8; + +// Note that ICTailCallReg is actually just the link register. +// In LoongArch code emission, we do not clobber ICTailCallReg since we keep +// the return address for calls there. + +// FloatReg0 must be equal to ReturnFloatReg. +static constexpr FloatRegister FloatReg0 = f0; +static constexpr FloatRegister FloatReg1 = f1; +static constexpr FloatRegister FloatReg2 = f2; +static constexpr FloatRegister FloatReg3 = f3; + +} // namespace jit +} // namespace js + +#endif /* jit_loongarch64_SharedICRegisters_loongarch64_h */ diff --git a/js/src/jit/loongarch64/Trampoline-loongarch64.cpp b/js/src/jit/loongarch64/Trampoline-loongarch64.cpp new file mode 100644 index 0000000000..246052a4ef --- /dev/null +++ b/js/src/jit/loongarch64/Trampoline-loongarch64.cpp @@ -0,0 +1,1362 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/DebugOnly.h" + +#include "jscompartment.h" + +#include "jit/Bailouts.h" +#include "jit/JitCompartment.h" +#include "jit/JitFrames.h" +#include "jit/JitSpewer.h" +#include "jit/Linker.h" +#include "jit/loongarch64/SharedICHelpers-loongarch64.h" +#include "jit/loongarch64/Bailouts-loongarch64.h" +#ifdef JS_ION_PERF +# include "jit/PerfSpewer.h" +#endif +#include "jit/VMFunctions.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +// All registers to save and restore. This includes the stack pointer, since we +// use the ability to reference register values on the stack by index. +static const LiveRegisterSet AllRegs = + LiveRegisterSet(GeneralRegisterSet(Registers::AllMask), + FloatRegisterSet(FloatRegisters::AllMask)); + +static_assert(sizeof(uintptr_t) == sizeof(uint64_t), "Not 32-bit clean."); + +struct EnterJITRegs +{ + double f31; + double f30; + double f29; + double f28; + double f27; + double f26; + double f25; + double f24; + + // non-volatile registers. + uint64_t ra; + uint64_t s7; + uint64_t s6; + uint64_t s5; + uint64_t s4; + uint64_t s3; + uint64_t s2; + uint64_t s1; + uint64_t s0; + // Save reg_vp(a7) on stack, use it after call jit code. + uint64_t a7; +}; + +static void +GenerateReturn(MacroAssembler& masm, int returnCode) +{ + MOZ_ASSERT(masm.framePushed() == sizeof(EnterJITRegs)); + + if (isLoongson()) { + // Restore non-volatile registers + masm.as_ld(s0, StackPointer, offsetof(EnterJITRegs, s0)); + masm.as_gslq(s1, s2, StackPointer, offsetof(EnterJITRegs, s2)); + masm.as_gslq(s3, s4, StackPointer, offsetof(EnterJITRegs, s4)); + masm.as_gslq(s5, s6, StackPointer, offsetof(EnterJITRegs, s6)); + masm.as_gslq(s7, ra, StackPointer, offsetof(EnterJITRegs, ra)); + + // Restore non-volatile floating point registers + masm.as_gslq(f24, f25, StackPointer, offsetof(EnterJITRegs, f25)); + masm.as_gslq(f26, f27, StackPointer, offsetof(EnterJITRegs, f27)); + masm.as_gslq(f28, f29, StackPointer, offsetof(EnterJITRegs, f29)); + masm.as_gslq(f30, f31, StackPointer, offsetof(EnterJITRegs, f31)); + } else { + // Restore non-volatile registers + masm.as_ld(s0, StackPointer, offsetof(EnterJITRegs, s0)); + masm.as_ld(s1, StackPointer, offsetof(EnterJITRegs, s1)); + masm.as_ld(s2, StackPointer, offsetof(EnterJITRegs, s2)); + masm.as_ld(s3, StackPointer, offsetof(EnterJITRegs, s3)); + masm.as_ld(s4, StackPointer, offsetof(EnterJITRegs, s4)); + masm.as_ld(s5, StackPointer, offsetof(EnterJITRegs, s5)); + masm.as_ld(s6, StackPointer, offsetof(EnterJITRegs, s6)); + masm.as_ld(s7, StackPointer, offsetof(EnterJITRegs, s7)); + masm.as_ld(ra, StackPointer, offsetof(EnterJITRegs, ra)); + + // Restore non-volatile floating point registers + masm.as_ld(f24, StackPointer, offsetof(EnterJITRegs, f24)); + masm.as_ld(f25, StackPointer, offsetof(EnterJITRegs, f25)); + masm.as_ld(f26, StackPointer, offsetof(EnterJITRegs, f26)); + masm.as_ld(f27, StackPointer, offsetof(EnterJITRegs, f27)); + masm.as_ld(f28, StackPointer, offsetof(EnterJITRegs, f28)); + masm.as_ld(f29, StackPointer, offsetof(EnterJITRegs, f29)); + masm.as_ld(f30, StackPointer, offsetof(EnterJITRegs, f30)); + masm.as_ld(f31, StackPointer, offsetof(EnterJITRegs, f31)); + } + + masm.freeStack(sizeof(EnterJITRegs)); + + masm.branch(ra); +} + +static void +GeneratePrologue(MacroAssembler& masm) +{ + masm.reserveStack(sizeof(EnterJITRegs)); + + if (isLoongson()) { + masm.as_gssq(a7, s0, StackPointer, offsetof(EnterJITRegs, s0)); + masm.as_gssq(s1, s2, StackPointer, offsetof(EnterJITRegs, s2)); + masm.as_gssq(s3, s4, StackPointer, offsetof(EnterJITRegs, s4)); + masm.as_gssq(s5, s6, StackPointer, offsetof(EnterJITRegs, s6)); + masm.as_gssq(s7, ra, StackPointer, offsetof(EnterJITRegs, ra)); + + masm.as_gssq(f24, f25, StackPointer, offsetof(EnterJITRegs, f25)); + masm.as_gssq(f26, f27, StackPointer, offsetof(EnterJITRegs, f27)); + masm.as_gssq(f28, f29, StackPointer, offsetof(EnterJITRegs, f29)); + masm.as_gssq(f30, f31, StackPointer, offsetof(EnterJITRegs, f31)); + return; + } + + masm.as_sd(s0, StackPointer, offsetof(EnterJITRegs, s0)); + masm.as_sd(s1, StackPointer, offsetof(EnterJITRegs, s1)); + masm.as_sd(s2, StackPointer, offsetof(EnterJITRegs, s2)); + masm.as_sd(s3, StackPointer, offsetof(EnterJITRegs, s3)); + masm.as_sd(s4, StackPointer, offsetof(EnterJITRegs, s4)); + masm.as_sd(s5, StackPointer, offsetof(EnterJITRegs, s5)); + masm.as_sd(s6, StackPointer, offsetof(EnterJITRegs, s6)); + masm.as_sd(s7, StackPointer, offsetof(EnterJITRegs, s7)); + masm.as_sd(ra, StackPointer, offsetof(EnterJITRegs, ra)); + masm.as_sd(a7, StackPointer, offsetof(EnterJITRegs, a7)); + + masm.as_sd(f24, StackPointer, offsetof(EnterJITRegs, f24)); + masm.as_sd(f25, StackPointer, offsetof(EnterJITRegs, f25)); + masm.as_sd(f26, StackPointer, offsetof(EnterJITRegs, f26)); + masm.as_sd(f27, StackPointer, offsetof(EnterJITRegs, f27)); + masm.as_sd(f28, StackPointer, offsetof(EnterJITRegs, f28)); + masm.as_sd(f29, StackPointer, offsetof(EnterJITRegs, f29)); + masm.as_sd(f30, StackPointer, offsetof(EnterJITRegs, f30)); + masm.as_sd(f31, StackPointer, offsetof(EnterJITRegs, f31)); +} + + +// Generates a trampoline for calling Jit compiled code from a C++ function. +// The trampoline use the EnterJitCode signature, with the standard x64 fastcall +// calling convention. +JitCode * +JitRuntime::generateEnterJIT(JSContext* cx, EnterJitType type) +{ + const Register reg_code = IntArgReg0; + const Register reg_argc = IntArgReg1; + const Register reg_argv = IntArgReg2; + const mozilla::DebugOnly reg_frame = IntArgReg3; + const Register reg_token = IntArgReg4; + const Register reg_chain = IntArgReg5; + const Register reg_values = IntArgReg6; + const Register reg_vp = IntArgReg7; + MacroAssembler masm(cx); + + MOZ_ASSERT(OsrFrameReg == reg_frame); + + GeneratePrologue(masm); + + // Save stack pointer into s4 + masm.movePtr(StackPointer, s4); + + // Save stack pointer as baseline frame. + if (type == EnterJitBaseline) + masm.movePtr(StackPointer, BaselineFrameReg); + + // Load the number of actual arguments into s3. + masm.unboxInt32(Address(reg_vp, 0), s3); + + /*************************************************************** + Loop over argv vector, push arguments onto stack in reverse order + ***************************************************************/ + + // if we are constructing, that also needs to include newTarget + { + Label noNewTarget; + masm.branchTest32(Assembler::Zero, reg_token, Imm32(CalleeToken_FunctionConstructing), + &noNewTarget); + + masm.add32(Imm32(1), reg_argc); + + masm.bind(&noNewTarget); + } + + // Make stack algined + masm.ma_and(s0, reg_argc, Imm32(1)); + masm.ma_dsubu(s1, StackPointer, Imm32(sizeof(Value))); + masm.as_movn(StackPointer, s1, s0); + + masm.as_dsll(s0, reg_argc, 3); // Value* argv + masm.addPtr(reg_argv, s0); // s0 = &argv[argc] + + // Loop over arguments, copying them from an unknown buffer onto the Ion + // stack so they can be accessed from JIT'ed code. + Label header, footer; + // If there aren't any arguments, don't do anything + masm.ma_b(s0, reg_argv, &footer, Assembler::BelowOrEqual, ShortJump); + { + masm.bind(&header); + + masm.subPtr(Imm32(sizeof(Value)), s0); + masm.subPtr(Imm32(sizeof(Value)), StackPointer); + + ValueOperand value = ValueOperand(s6); + masm.loadValue(Address(s0, 0), value); + masm.storeValue(value, Address(StackPointer, 0)); + + masm.ma_b(s0, reg_argv, &header, Assembler::Above, ShortJump); + } + masm.bind(&footer); + + masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + masm.storePtr(s3, Address(StackPointer, sizeof(uintptr_t))); // actual arguments + masm.storePtr(reg_token, Address(StackPointer, 0)); // callee token + + masm.subPtr(StackPointer, s4); + masm.makeFrameDescriptor(s4, JitFrame_Entry, JitFrameLayout::Size()); + masm.push(s4); // descriptor + + CodeLabel returnLabel; + CodeLabel oomReturnLabel; + if (type == EnterJitBaseline) { + // Handle OSR. + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + regs.take(OsrFrameReg); + regs.take(BaselineFrameReg); + regs.take(reg_code); + regs.take(ReturnReg); + regs.take(JSReturnOperand); + + Label notOsr; + masm.ma_b(OsrFrameReg, OsrFrameReg, ¬Osr, Assembler::Zero, ShortJump); + + Register numStackValues = reg_values; + regs.take(numStackValues); + Register scratch = regs.takeAny(); + + // Push return address. + masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + masm.ma_li(scratch, returnLabel.patchAt()); + masm.storePtr(scratch, Address(StackPointer, 0)); + + // Push previous frame pointer. + masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + masm.storePtr(BaselineFrameReg, Address(StackPointer, 0)); + + // Reserve frame. + Register framePtr = BaselineFrameReg; + masm.subPtr(Imm32(BaselineFrame::Size()), StackPointer); + masm.movePtr(StackPointer, framePtr); + + // Reserve space for locals and stack values. + masm.ma_dsll(scratch, numStackValues, Imm32(3)); + masm.subPtr(scratch, StackPointer); + + // Enter exit frame. + masm.addPtr(Imm32(BaselineFrame::Size() + BaselineFrame::FramePointerOffset), scratch); + masm.makeFrameDescriptor(scratch, JitFrame_BaselineJS, ExitFrameLayout::Size()); + + // Push frame descriptor and fake return address. + masm.reserveStack(2 * sizeof(uintptr_t)); + masm.storePtr(scratch, Address(StackPointer, sizeof(uintptr_t))); // Frame descriptor + masm.storePtr(zero, Address(StackPointer, 0)); // fake return address + + // No GC things to mark, push a bare token. + masm.enterFakeExitFrame(ExitFrameLayoutBareToken); + + masm.reserveStack(2 * sizeof(uintptr_t)); + masm.storePtr(framePtr, Address(StackPointer, sizeof(uintptr_t))); // BaselineFrame + masm.storePtr(reg_code, Address(StackPointer, 0)); // jitcode + + masm.setupUnalignedABICall(scratch); + masm.passABIArg(BaselineFrameReg); // BaselineFrame + masm.passABIArg(OsrFrameReg); // InterpreterFrame + masm.passABIArg(numStackValues); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, jit::InitBaselineFrameForOsr)); + + regs.add(OsrFrameReg); + Register jitcode = regs.takeAny(); + masm.loadPtr(Address(StackPointer, 0), jitcode); + masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), framePtr); + masm.freeStack(2 * sizeof(uintptr_t)); + + Label error; + masm.freeStack(ExitFrameLayout::SizeWithFooter()); + masm.addPtr(Imm32(BaselineFrame::Size()), framePtr); + masm.branchIfFalseBool(ReturnReg, &error); + + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + Register realFramePtr = numStackValues; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.ma_daddu(realFramePtr, framePtr, Imm32(sizeof(void*))); + masm.profilerEnterFrame(realFramePtr, scratch); + masm.bind(&skipProfilingInstrumentation); + } + + masm.jump(jitcode); + + // OOM: load error value, discard return address and previous frame + // pointer and return. + masm.bind(&error); + masm.movePtr(framePtr, StackPointer); + masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + masm.ma_li(scratch, oomReturnLabel.patchAt()); + masm.jump(scratch); + + masm.bind(¬Osr); + // Load the scope chain in R1. + MOZ_ASSERT(R1.scratchReg() != reg_code); + masm.ma_move(R1.scratchReg(), reg_chain); + } + + // The call will push the return address on the stack, thus we check that + // the stack would be aligned once the call is complete. + masm.assertStackAlignment(JitStackAlignment, sizeof(uintptr_t)); + + // Call the function with pushing return address to stack. + masm.callJitNoProfiler(reg_code); + + if (type == EnterJitBaseline) { + // Baseline OSR will return here. + masm.bind(returnLabel.target()); + masm.addCodeLabel(returnLabel); + masm.bind(oomReturnLabel.target()); + masm.addCodeLabel(oomReturnLabel); + } + + // Pop arguments off the stack. + // s0 <- 8*argc (size of all arguments we pushed on the stack) + masm.pop(s0); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), s0); + masm.addPtr(s0, StackPointer); + + // Store the returned value into the vp + masm.as_ld(reg_vp, StackPointer, offsetof(EnterJITRegs, a7)); + masm.storeValue(JSReturnOperand, Address(reg_vp, 0)); + + // Restore non-volatile registers and return. + GenerateReturn(masm, ShortJump); + + Linker linker(masm); + AutoFlushICache afc("GenerateEnterJIT"); + JitCode* code = linker.newCode(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "EnterJIT"); +#endif + + return code; +} + +JitCode* +JitRuntime::generateInvalidator(JSContext* cx) +{ + MacroAssembler masm(cx); + + // Stack has to be alligned here. If not, we will have to fix it. + masm.checkStackAlignment(); + + // Push registers such that we can access them from [base + code]. + masm.PushRegsInMask(AllRegs); + + // Pass pointer to InvalidationBailoutStack structure. + masm.movePtr(StackPointer, a0); + + // Reserve place for return value and BailoutInfo pointer + masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + // Pass pointer to return value. + masm.ma_daddu(a1, StackPointer, Imm32(sizeof(uintptr_t))); + // Pass pointer to BailoutInfo + masm.movePtr(StackPointer, a2); + + masm.setupAlignedABICall(); + masm.passABIArg(a0); + masm.passABIArg(a1); + masm.passABIArg(a2); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, InvalidationBailout)); + + masm.loadPtr(Address(StackPointer, 0), a2); + masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), a1); + // Remove the return address, the IonScript, the register state + // (InvaliationBailoutStack) and the space that was allocated for the + // return value. + masm.addPtr(Imm32(sizeof(InvalidationBailoutStack) + 2 * sizeof(uintptr_t)), StackPointer); + // remove the space that this frame was using before the bailout + // (computed by InvalidationBailout) + masm.addPtr(a1, StackPointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2. + JitCode* bailoutTail = cx->runtime()->jitRuntime()->getBailoutTail(); + masm.branch(bailoutTail); + + Linker linker(masm); + AutoFlushICache afc("Invalidator"); + JitCode* code = linker.newCode(cx, OTHER_CODE); + JitSpew(JitSpew_IonInvalidate, " invalidation thunk created at %p", (void*) code->raw()); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "Invalidator"); +#endif + + return code; +} + +JitCode* +JitRuntime::generateArgumentsRectifier(JSContext* cx, void** returnAddrOut) +{ + // Do not erase the frame pointer in this function. + + MacroAssembler masm(cx); + masm.pushReturnAddress(); + // Caller: + // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- sp + // '--- s3 ---' + + // ArgumentsRectifierReg contains the |nargs| pushed onto the current + // frame. Including |this|, there are (|nargs| + 1) arguments to copy. + MOZ_ASSERT(ArgumentsRectifierReg == s3); + + // Add |this|, in the counter of known arguments. + masm.addPtr(Imm32(1), ArgumentsRectifierReg); + + Register numActArgsReg = a6; + Register calleeTokenReg = a7; + Register numArgsReg = a5; + + // Load |nformals| into numArgsReg. + masm.loadPtr(Address(StackPointer, RectifierFrameLayout::offsetOfCalleeToken()), + calleeTokenReg); + masm.mov(calleeTokenReg, numArgsReg); + masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), numArgsReg); + masm.load16ZeroExtend(Address(numArgsReg, JSFunction::offsetOfNargs()), numArgsReg); + + // Stash another copy in t3, since we are going to do destructive operations + // on numArgsReg + masm.mov(numArgsReg, t3); + + static_assert(CalleeToken_FunctionConstructing == 1, + "Ensure that we can use the constructing bit to count the value"); + masm.mov(calleeTokenReg, t2); + masm.ma_and(t2, Imm32(uint32_t(CalleeToken_FunctionConstructing))); + + // Including |this|, and |new.target|, there are (|nformals| + 1 + isConstructing) + // arguments to push to the stack. Then we push a JitFrameLayout. We + // compute the padding expressed in the number of extra |undefined| values + // to push on the stack. + static_assert(sizeof(JitFrameLayout) % JitStackAlignment == 0, + "No need to consider the JitFrameLayout for aligning the stack"); + static_assert(JitStackAlignment % sizeof(Value) == 0, + "Ensure that we can pad the stack by pushing extra UndefinedValue"); + + MOZ_ASSERT(mozilla::IsPowerOfTwo(JitStackValueAlignment)); + masm.add32(Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */), numArgsReg); + masm.add32(t2, numArgsReg); + masm.and32(Imm32(~(JitStackValueAlignment - 1)), numArgsReg); + + // Load the number of |undefined|s to push into t1. + masm.as_dsubu(t1, numArgsReg, s3); + + // Caller: + // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- sp <- t2 + // '------ s3 -------' + // + // Rectifier frame: + // [undef] [undef] [undef] [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] + // '-------- t1 ---------' '------- s3 -------' + + // Copy number of actual arguments into numActArgsReg + masm.loadPtr(Address(StackPointer, RectifierFrameLayout::offsetOfNumActualArgs()), + numActArgsReg); + + + masm.moveValue(UndefinedValue(), ValueOperand(t0)); + + masm.movePtr(StackPointer, t2); // Save %sp. + + // Push undefined. (including the padding) + { + Label undefLoopTop; + + masm.bind(&undefLoopTop); + masm.sub32(Imm32(1), t1); + masm.subPtr(Imm32(sizeof(Value)), StackPointer); + masm.storeValue(ValueOperand(t0), Address(StackPointer, 0)); + + masm.ma_b(t1, t1, &undefLoopTop, Assembler::NonZero, ShortJump); + } + + // Get the topmost argument. + static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments"); + + // | - sizeof(Value)| is used to put rcx such that we can read the last + // argument, and not the value which is after. + masm.ma_dsll(t0, s3, Imm32(3)); // t0 <- nargs * 8 + masm.as_daddu(t1, t2, t0); // t1 <- t2(saved sp) + nargs * 8 + masm.addPtr(Imm32(sizeof(RectifierFrameLayout) - sizeof(Value)), t1); + + // Copy & Push arguments, |nargs| + 1 times (to include |this|). + { + Label copyLoopTop; + + masm.bind(©LoopTop); + masm.sub32(Imm32(1), s3); + masm.subPtr(Imm32(sizeof(Value)), StackPointer); + masm.loadValue(Address(t1, 0), ValueOperand(t0)); + masm.storeValue(ValueOperand(t0), Address(StackPointer, 0)); + masm.subPtr(Imm32(sizeof(Value)), t1); + + masm.ma_b(s3, s3, ©LoopTop, Assembler::NonZero, ShortJump); + } + + // if constructing, copy newTarget + { + Label notConstructing; + + masm.branchTest32(Assembler::Zero, calleeTokenReg, Imm32(CalleeToken_FunctionConstructing), + ¬Constructing); + + // thisFrame[numFormals] = prevFrame[argc] + ValueOperand newTarget(t0); + + // +1 for |this|. We want vp[argc], so don't subtract 1 + BaseIndex newTargetSrc(t2, numActArgsReg, TimesEight, sizeof(RectifierFrameLayout) + sizeof(Value)); + masm.loadValue(newTargetSrc, newTarget); + + // Again, 1 for |this| + BaseIndex newTargetDest(StackPointer, t3, TimesEight, sizeof(Value)); + masm.storeValue(newTarget, newTargetDest); + + masm.bind(¬Constructing); + } + + // Caller: + // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- t2 + // + // + // Rectifier frame: + // [undef] [undef] [undef] [arg2] [arg1] [this] <- sp [[argc] [callee] [descr] [raddr]] + + // Construct sizeDescriptor. + masm.subPtr(StackPointer, t2); + masm.makeFrameDescriptor(t2, JitFrame_Rectifier, JitFrameLayout::Size()); + + // Construct JitFrameLayout. + masm.subPtr(Imm32(3 * sizeof(uintptr_t)), StackPointer); + // Push actual arguments. + masm.storePtr(numActArgsReg, Address(StackPointer, 2 * sizeof(uintptr_t))); + // Push callee token. + masm.storePtr(calleeTokenReg, Address(StackPointer, sizeof(uintptr_t))); + // Push frame descriptor. + masm.storePtr(t2, Address(StackPointer, 0)); + + // Call the target function. + // Note that this code assumes the function is JITted. + masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), calleeTokenReg); + masm.loadPtr(Address(calleeTokenReg, JSFunction::offsetOfNativeOrScript()), t1); + masm.loadBaselineOrIonRaw(t1, t1, nullptr); + uint32_t returnOffset = masm.callJitNoProfiler(t1); + + // Remove the rectifier frame. + // t2 <- descriptor with FrameType. + masm.loadPtr(Address(StackPointer, 0), t2); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), t2); // t2 <- descriptor. + + // Discard descriptor, calleeToken and number of actual arguments. + masm.addPtr(Imm32(3 * sizeof(uintptr_t)), StackPointer); + + // Discard pushed arguments. + masm.addPtr(t2, StackPointer); + + masm.ret(); + Linker linker(masm); + AutoFlushICache afc("ArgumentsRectifier"); + JitCode* code = linker.newCode(cx, OTHER_CODE); + + if (returnAddrOut) + *returnAddrOut = (void*) (code->raw() + returnOffset); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "ArgumentsRectifier"); +#endif + + return code; +} + +/* - When bailout is done via out of line code (lazy bailout). + * Frame size is stored in $ra (look at + * CodeGeneratorLOONGARCH64::generateOutOfLineCode()) and thunk code should save it + * on stack. Other difference is that members snapshotOffset_ and padding_ are + * pushed to the stack by CodeGeneratorLOONGARCH64::visitOutOfLineBailout(). Field + * frameClassId_ is forced to be NO_FRAME_SIZE_CLASS_ID + * (See: JitRuntime::generateBailoutHandler). + */ +static void +PushBailoutFrame(MacroAssembler& masm, Register spArg) +{ + // Push the frameSize_ stored in ra + // See: CodeGeneratorLOONGARCH64::generateOutOfLineCode() + masm.push(ra); + + // Push registers such that we can access them from [base + code]. + masm.PushRegsInMask(AllRegs); + + // Put pointer to BailoutStack as first argument to the Bailout() + masm.movePtr(StackPointer, spArg); +} + +static void +GenerateBailoutThunk(JSContext* cx, MacroAssembler& masm, uint32_t frameClass) +{ + PushBailoutFrame(masm, a0); + + // Put pointer to BailoutInfo + static const uint32_t sizeOfBailoutInfo = sizeof(uintptr_t) * 2; + masm.subPtr(Imm32(sizeOfBailoutInfo), StackPointer); + masm.movePtr(StackPointer, a1); + + masm.setupAlignedABICall(); + masm.passABIArg(a0); + masm.passABIArg(a1); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, Bailout)); + + // Get BailoutInfo pointer + masm.loadPtr(Address(StackPointer, 0), a2); + + // Stack is: + // [frame] + // snapshotOffset + // frameSize + // [bailoutFrame] + // [bailoutInfo] + // + // Remove both the bailout frame and the topmost Ion frame's stack. + // Load frameSize from stack + masm.loadPtr(Address(StackPointer, + sizeOfBailoutInfo + BailoutStack::offsetOfFrameSize()), a1); + // Remove complete BailoutStack class and data after it + masm.addPtr(Imm32(sizeof(BailoutStack) + sizeOfBailoutInfo), StackPointer); + // Remove frame size srom stack + masm.addPtr(a1, StackPointer); + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in a2. + JitCode* bailoutTail = cx->runtime()->jitRuntime()->getBailoutTail(); + masm.branch(bailoutTail); +} + +JitCode* +JitRuntime::generateBailoutTable(JSContext* cx, uint32_t frameClass) +{ + MOZ_CRASH("LOONGARCH64 does not use bailout tables"); +} + +JitCode* +JitRuntime::generateBailoutHandler(JSContext* cx) +{ + MacroAssembler masm(cx); + GenerateBailoutThunk(cx, masm, NO_FRAME_SIZE_CLASS_ID); + + Linker linker(masm); + AutoFlushICache afc("BailoutHandler"); + JitCode* code = linker.newCode(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "BailoutHandler"); +#endif + + return code; +} + +JitCode* +JitRuntime::generateVMWrapper(JSContext* cx, const VMFunction& f) +{ + MOZ_ASSERT(functionWrappers_); + MOZ_ASSERT(functionWrappers_->initialized()); + VMWrapperMap::AddPtr p = functionWrappers_->lookupForAdd(&f); + if (p) + return p->value(); + + MacroAssembler masm(cx); + + AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask); + + static_assert((Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0, + "Wrapper register set should be a superset of Volatile register set."); + + // The context is the first argument; a0 is the first argument register. + Register cxreg = a0; + regs.take(cxreg); + + // If it isn't a tail call, then the return address needs to be saved + if (f.expectTailCall == NonTailCall) + masm.pushReturnAddress(); + + // We're aligned to an exit frame, so link it up. + masm.enterExitFrame(&f); + masm.loadJSContext(cxreg); + + // Save the base of the argument set stored on the stack. + Register argsBase = InvalidReg; + if (f.explicitArgs) { + argsBase = t1; // Use temporary register. + regs.take(argsBase); + masm.ma_daddu(argsBase, StackPointer, Imm32(ExitFrameLayout::SizeWithFooter())); + } + + // Reserve space for the outparameter. + Register outReg = InvalidReg; + switch (f.outParam) { + case Type_Value: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(Value)); + masm.movePtr(StackPointer, outReg); + break; + + case Type_Handle: + outReg = regs.takeAny(); + masm.PushEmptyRooted(f.outParamRootType); + masm.movePtr(StackPointer, outReg); + break; + + case Type_Bool: + case Type_Int32: + outReg = regs.takeAny(); + // Reserve 4-byte space to make stack aligned to 8-byte. + masm.reserveStack(2 * sizeof(int32_t)); + masm.movePtr(StackPointer, outReg); + break; + + case Type_Pointer: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(uintptr_t)); + masm.movePtr(StackPointer, outReg); + break; + + case Type_Double: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(double)); + masm.movePtr(StackPointer, outReg); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + if (!generateTLEnterVM(cx, masm, f)) + return nullptr; + + masm.setupUnalignedABICall(regs.getAny()); + masm.passABIArg(cxreg); + + size_t argDisp = 0; + + // Copy any arguments. + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) { + MoveOperand from; + switch (f.argProperties(explicitArg)) { + case VMFunction::WordByValue: + if (f.argPassedInFloatReg(explicitArg)) + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE); + else + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunction::WordByRef: + masm.passABIArg(MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS), + MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + case VMFunction::DoubleByValue: + case VMFunction::DoubleByRef: + MOZ_CRASH("NYI: LOONGARCH64 callVM should not be used with 128bits values."); + break; + } + } + + // Copy the implicit outparam, if any. + if (InvalidReg != outReg) + masm.passABIArg(outReg); + + masm.callWithABI(f.wrapped); + + if (!generateTLExitVM(cx, masm, f)) + return nullptr; + + // Test for failure. + switch (f.failType()) { + case Type_Object: + masm.branchTestPtr(Assembler::Zero, v0, v0, masm.failureLabel()); + break; + case Type_Bool: + // Called functions return bools, which are 0/false and non-zero/true + masm.branchIfFalseBool(v0, masm.failureLabel()); + break; + default: + MOZ_CRASH("unknown failure kind"); + } + + // Load the outparam and free any allocated stack. + switch (f.outParam) { + case Type_Handle: + masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand); + break; + + case Type_Value: + masm.loadValue(Address(StackPointer, 0), JSReturnOperand); + masm.freeStack(sizeof(Value)); + break; + + case Type_Int32: + masm.load32(Address(StackPointer, 0), ReturnReg); + masm.freeStack(2 * sizeof(int32_t)); + break; + + case Type_Pointer: + masm.loadPtr(Address(StackPointer, 0), ReturnReg); + masm.freeStack(sizeof(uintptr_t)); + break; + + case Type_Bool: + masm.load8ZeroExtend(Address(StackPointer, 0), ReturnReg); + masm.freeStack(2 * sizeof(int32_t)); + break; + + case Type_Double: + if (cx->runtime()->jitSupportsFloatingPoint) { + masm.as_ld(ReturnDoubleReg, StackPointer, 0); + } else { + masm.assumeUnreachable("Unable to load into float reg, with no FP support."); + } + masm.freeStack(sizeof(double)); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + masm.leaveExitFrame(); + masm.retn(Imm32(sizeof(ExitFrameLayout) + + f.explicitStackSlots() * sizeof(void*) + + f.extraValuesToPop * sizeof(Value))); + + Linker linker(masm); + AutoFlushICache afc("VMWrapper"); + JitCode* wrapper = linker.newCode(cx, OTHER_CODE); + if (!wrapper) + return nullptr; + + // linker.newCode may trigger a GC and sweep functionWrappers_ so we have + // to use relookupOrAdd instead of add. + if (!functionWrappers_->relookupOrAdd(p, &f, wrapper)) + return nullptr; + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(wrapper, "VMWrapper"); +#endif + + return wrapper; +} + +JitCode* +JitRuntime::generatePreBarrier(JSContext* cx, MIRType type) +{ + MacroAssembler masm(cx); + + LiveRegisterSet save; + if (cx->runtime()->jitSupportsFloatingPoint) { + save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask), + FloatRegisterSet(FloatRegisters::VolatileMask)); + } else { + save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask), + FloatRegisterSet()); + } + save.add(ra); + masm.PushRegsInMask(save); + + MOZ_ASSERT(PreBarrierReg == a1); + masm.movePtr(ImmPtr(cx->runtime()), a0); + + masm.setupUnalignedABICall(a2); + masm.passABIArg(a0); + masm.passABIArg(a1); + masm.callWithABI(IonMarkFunction(type)); + + save.take(AnyRegister(ra)); + masm.PopRegsInMask(save); + masm.ret(); + + Linker linker(masm); + AutoFlushICache afc("PreBarrier"); + JitCode* code = linker.newCode(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "PreBarrier"); +#endif + + return code; +} + +typedef bool (*HandleDebugTrapFn)(JSContext*, BaselineFrame*, uint8_t*, bool*); +static const VMFunction HandleDebugTrapInfo = + FunctionInfo(HandleDebugTrap, "HandleDebugTrap"); + +JitCode* +JitRuntime::generateDebugTrapHandler(JSContext* cx) +{ + MacroAssembler masm(cx); + + Register scratch1 = t0; + Register scratch2 = t1; + + // Load BaselineFrame pointer in scratch1. + masm.movePtr(s5, scratch1); + masm.subPtr(Imm32(BaselineFrame::Size()), scratch1); + + // Enter a stub frame and call the HandleDebugTrap VM function. Ensure + // the stub frame has a nullptr ICStub pointer, since this pointer is + // marked during GC. + masm.movePtr(ImmPtr(nullptr), ICStubReg); + EmitBaselineEnterStubFrame(masm, scratch2); + + JitCode* code = cx->runtime()->jitRuntime()->getVMWrapper(HandleDebugTrapInfo); + if (!code) + return nullptr; + + masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + masm.storePtr(ra, Address(StackPointer, sizeof(uintptr_t))); + masm.storePtr(scratch1, Address(StackPointer, 0)); + + EmitBaselineCallVM(code, masm); + + EmitBaselineLeaveStubFrame(masm); + + // If the stub returns |true|, we have to perform a forced return + // (return from the JS frame). If the stub returns |false|, just return + // from the trap stub so that execution continues at the current pc. + Label forcedReturn; + masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg, &forcedReturn); + + // ra was restored by EmitLeaveStubFrame + masm.branch(ra); + + masm.bind(&forcedReturn); + masm.loadValue(Address(s5, BaselineFrame::reverseOffsetOfReturnValue()), + JSReturnOperand); + masm.movePtr(s5, StackPointer); + masm.pop(s5); + + // Before returning, if profiling is turned on, make sure that lastProfilingFrame + // is set to the correct caller frame. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + masm.profilerExitFrame(); + masm.bind(&skipProfilingInstrumentation); + } + + masm.ret(); + + Linker linker(masm); + AutoFlushICache afc("DebugTrapHandler"); + JitCode* codeDbg = linker.newCode(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(codeDbg, "DebugTrapHandler"); +#endif + + return codeDbg; +} + + +JitCode* +JitRuntime::generateExceptionTailStub(JSContext* cx, void* handler) +{ + MacroAssembler masm; + + masm.handleFailureWithHandlerTail(handler); + + Linker linker(masm); + AutoFlushICache afc("ExceptionTailStub"); + JitCode* code = linker.newCode(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "ExceptionTailStub"); +#endif + + return code; +} + +JitCode* +JitRuntime::generateBailoutTailStub(JSContext* cx) +{ + MacroAssembler masm; + + masm.generateBailoutTail(a1, a2); + + Linker linker(masm); + AutoFlushICache afc("BailoutTailStub"); + JitCode* code = linker.newCode(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "BailoutTailStub"); +#endif + + return code; +} + +JitCode* +JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) +{ + MacroAssembler masm; + + Register scratch1 = t0; + Register scratch2 = t1; + Register scratch3 = t2; + Register scratch4 = t3; + + // + // The code generated below expects that the current stack pointer points + // to an Ion or Baseline frame, at the state it would be immediately + // before a ret(). Thus, after this stub's business is done, it executes + // a ret() and returns directly to the caller script, on behalf of the + // callee script that jumped to this code. + // + // Thus the expected stack is: + // + // StackPointer ----+ + // v + // ..., ActualArgc, CalleeToken, Descriptor, ReturnAddr + // MEM-HI MEM-LOW + // + // + // The generated jitcode is responsible for overwriting the + // jitActivation->lastProfilingFrame field with a pointer to the previous + // Ion or Baseline jit-frame that was pushed before this one. It is also + // responsible for overwriting jitActivation->lastProfilingCallSite with + // the return address into that frame. The frame could either be an + // immediate "caller" frame, or it could be a frame in a previous + // JitActivation (if the current frame was entered from C++, and the C++ + // was entered by some caller jit-frame further down the stack). + // + // So this jitcode is responsible for "walking up" the jit stack, finding + // the previous Ion or Baseline JS frame, and storing its address and the + // return address into the appropriate fields on the current jitActivation. + // + // There are a fixed number of different path types that can lead to the + // current frame, which is either a baseline or ion frame: + // + // + // ^ + // | + // ^--- Ion + // | + // ^--- Baseline Stub <---- Baseline + // | + // ^--- Argument Rectifier + // | ^ + // | | + // | ^--- Ion + // | | + // | ^--- Baseline Stub <---- Baseline + // | + // ^--- Entry Frame (From C++) + // + Register actReg = scratch4; + AbsoluteAddress activationAddr(GetJitContext()->runtime->addressOfProfilingActivation()); + masm.loadPtr(activationAddr, actReg); + + Address lastProfilingFrame(actReg, JitActivation::offsetOfLastProfilingFrame()); + Address lastProfilingCallSite(actReg, JitActivation::offsetOfLastProfilingCallSite()); + +#ifdef DEBUG + // Ensure that frame we are exiting is current lastProfilingFrame + { + masm.loadPtr(lastProfilingFrame, scratch1); + Label checkOk; + masm.branchPtr(Assembler::Equal, scratch1, ImmWord(0), &checkOk); + masm.branchPtr(Assembler::Equal, StackPointer, scratch1, &checkOk); + masm.assumeUnreachable( + "Mismatch between stored lastProfilingFrame and current stack pointer."); + masm.bind(&checkOk); + } +#endif + + // Load the frame descriptor into |scratch1|, figure out what to do depending on its type. + masm.loadPtr(Address(StackPointer, JitFrameLayout::offsetOfDescriptor()), scratch1); + + // Going into the conditionals, we will have: + // FrameDescriptor.size in scratch1 + // FrameDescriptor.type in scratch2 + masm.ma_and(scratch2, scratch1, Imm32((1 << FRAMETYPE_BITS) - 1)); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch1); + + // Handling of each case is dependent on FrameDescriptor.type + Label handle_IonJS; + Label handle_BaselineStub; + Label handle_Rectifier; + Label handle_IonAccessorIC; + Label handle_Entry; + Label end; + + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_IonJS), &handle_IonJS); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_BaselineJS), &handle_IonJS); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_BaselineStub), &handle_BaselineStub); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_Rectifier), &handle_Rectifier); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_IonAccessorIC), &handle_IonAccessorIC); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_Entry), &handle_Entry); + + masm.assumeUnreachable("Invalid caller frame type when exiting from Ion frame."); + + // + // JitFrame_IonJS + // + // Stack layout: + // ... + // Ion-Descriptor + // Prev-FP ---> Ion-ReturnAddr + // ... previous frame data ... |- Descriptor.Size + // ... arguments ... | + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + masm.bind(&handle_IonJS); + { + // |scratch1| contains Descriptor.size + + // returning directly to an IonJS frame. Store return addr to frame + // in lastProfilingCallSite. + masm.loadPtr(Address(StackPointer, JitFrameLayout::offsetOfReturnAddress()), scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + // Store return frame in lastProfilingFrame. + // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size(); + masm.as_daddu(scratch2, StackPointer, scratch1); + masm.ma_daddu(scratch2, scratch2, Imm32(JitFrameLayout::Size())); + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + // + // JitFrame_BaselineStub + // + // Look past the stub and store the frame pointer to + // the baselineJS frame prior to it. + // + // Stack layout: + // ... + // BL-Descriptor + // Prev-FP ---> BL-ReturnAddr + // +-----> BL-PrevFramePointer + // | ... BL-FrameData ... + // | BLStub-Descriptor + // | BLStub-ReturnAddr + // | BLStub-StubPointer | + // +------ BLStub-SavedFramePointer |- Descriptor.Size + // ... arguments ... | + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + // We take advantage of the fact that the stub frame saves the frame + // pointer pointing to the baseline frame, so a bunch of calculation can + // be avoided. + // + masm.bind(&handle_BaselineStub); + { + masm.as_daddu(scratch3, StackPointer, scratch1); + Address stubFrameReturnAddr(scratch3, + JitFrameLayout::Size() + + BaselineStubFrameLayout::offsetOfReturnAddress()); + masm.loadPtr(stubFrameReturnAddr, scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + Address stubFrameSavedFramePtr(scratch3, + JitFrameLayout::Size() - (2 * sizeof(void*))); + masm.loadPtr(stubFrameSavedFramePtr, scratch2); + masm.addPtr(Imm32(sizeof(void*)), scratch2); // Skip past BL-PrevFramePtr + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + + // + // JitFrame_Rectifier + // + // The rectifier frame can be preceded by either an IonJS or a + // BaselineStub frame. + // + // Stack layout if caller of rectifier was Ion: + // + // Ion-Descriptor + // Ion-ReturnAddr + // ... ion frame data ... |- Rect-Descriptor.Size + // < COMMON LAYOUT > + // + // Stack layout if caller of rectifier was Baseline: + // + // BL-Descriptor + // Prev-FP ---> BL-ReturnAddr + // +-----> BL-SavedFramePointer + // | ... baseline frame data ... + // | BLStub-Descriptor + // | BLStub-ReturnAddr + // | BLStub-StubPointer | + // +------ BLStub-SavedFramePointer |- Rect-Descriptor.Size + // ... args to rectifier ... | + // < COMMON LAYOUT > + // + // Common stack layout: + // + // ActualArgc | + // CalleeToken |- IonRectitiferFrameLayout::Size() + // Rect-Descriptor | + // Rect-ReturnAddr | + // ... rectifier data & args ... |- Descriptor.Size + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + masm.bind(&handle_Rectifier); + { + // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size(); + masm.as_daddu(scratch2, StackPointer, scratch1); + masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2); + masm.loadPtr(Address(scratch2, RectifierFrameLayout::offsetOfDescriptor()), scratch3); + masm.ma_dsrl(scratch1, scratch3, Imm32(FRAMESIZE_SHIFT)); + masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch3); + + // Now |scratch1| contains Rect-Descriptor.Size + // and |scratch2| points to Rectifier frame + // and |scratch3| contains Rect-Descriptor.Type + + // Check for either Ion or BaselineStub frame. + Label handle_Rectifier_BaselineStub; + masm.branch32(Assembler::NotEqual, scratch3, Imm32(JitFrame_IonJS), + &handle_Rectifier_BaselineStub); + + // Handle Rectifier <- IonJS + // scratch3 := RectFrame[ReturnAddr] + masm.loadPtr(Address(scratch2, RectifierFrameLayout::offsetOfReturnAddress()), scratch3); + masm.storePtr(scratch3, lastProfilingCallSite); + + // scratch3 := RectFrame + Rect-Descriptor.Size + RectifierFrameLayout::Size() + masm.as_daddu(scratch3, scratch2, scratch1); + masm.addPtr(Imm32(RectifierFrameLayout::Size()), scratch3); + masm.storePtr(scratch3, lastProfilingFrame); + masm.ret(); + + // Handle Rectifier <- BaselineStub <- BaselineJS + masm.bind(&handle_Rectifier_BaselineStub); +#ifdef DEBUG + { + Label checkOk; + masm.branch32(Assembler::Equal, scratch3, Imm32(JitFrame_BaselineStub), &checkOk); + masm.assumeUnreachable("Unrecognized frame preceding baselineStub."); + masm.bind(&checkOk); + } +#endif + masm.as_daddu(scratch3, scratch2, scratch1); + Address stubFrameReturnAddr(scratch3, RectifierFrameLayout::Size() + + BaselineStubFrameLayout::offsetOfReturnAddress()); + masm.loadPtr(stubFrameReturnAddr, scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + Address stubFrameSavedFramePtr(scratch3, + RectifierFrameLayout::Size() - (2 * sizeof(void*))); + masm.loadPtr(stubFrameSavedFramePtr, scratch2); + masm.addPtr(Imm32(sizeof(void*)), scratch2); + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + // JitFrame_IonAccessorIC + // + // The caller is always an IonJS frame. + // + // Ion-Descriptor + // Ion-ReturnAddr + // ... ion frame data ... |- AccFrame-Descriptor.Size + // StubCode | + // AccFrame-Descriptor |- IonAccessorICFrameLayout::Size() + // AccFrame-ReturnAddr | + // ... accessor frame data & args ... |- Descriptor.Size + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + masm.bind(&handle_IonAccessorIC); + { + // scratch2 := StackPointer + Descriptor.size + JitFrameLayout::Size() + masm.as_daddu(scratch2, StackPointer, scratch1); + masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2); + + // scratch3 := AccFrame-Descriptor.Size + masm.loadPtr(Address(scratch2, IonAccessorICFrameLayout::offsetOfDescriptor()), scratch3); +#ifdef DEBUG + // Assert previous frame is an IonJS frame. + masm.movePtr(scratch3, scratch1); + masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch1); + { + Label checkOk; + masm.branch32(Assembler::Equal, scratch1, Imm32(JitFrame_IonJS), &checkOk); + masm.assumeUnreachable("IonAccessorIC frame must be preceded by IonJS frame"); + masm.bind(&checkOk); + } +#endif + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch3); + + // lastProfilingCallSite := AccFrame-ReturnAddr + masm.loadPtr(Address(scratch2, IonAccessorICFrameLayout::offsetOfReturnAddress()), scratch1); + masm.storePtr(scratch1, lastProfilingCallSite); + + // lastProfilingFrame := AccessorFrame + AccFrame-Descriptor.Size + + // IonAccessorICFrameLayout::Size() + masm.as_daddu(scratch1, scratch2, scratch3); + masm.addPtr(Imm32(IonAccessorICFrameLayout::Size()), scratch1); + masm.storePtr(scratch1, lastProfilingFrame); + masm.ret(); + } + + // + // JitFrame_Entry + // + // If at an entry frame, store null into both fields. + // + masm.bind(&handle_Entry); + { + masm.movePtr(ImmPtr(nullptr), scratch1); + masm.storePtr(scratch1, lastProfilingCallSite); + masm.storePtr(scratch1, lastProfilingFrame); + masm.ret(); + } + + Linker linker(masm); + AutoFlushICache afc("ProfilerExitFrameTailStub"); + JitCode* code = linker.newCode(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "ProfilerExitFrameStub"); +#endif + + return code; +} diff --git a/js/src/jit/shared/Assembler-shared.h b/js/src/jit/shared/Assembler-shared.h index 308d9f1a12..61185b3835 100644 --- a/js/src/jit/shared/Assembler-shared.h +++ b/js/src/jit/shared/Assembler-shared.h @@ -17,7 +17,8 @@ #include "wasm/WasmTypes.h" #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONGARCH64) // Push return addresses callee-side. # define JS_USE_LINK_REGISTER #endif diff --git a/js/src/jit/shared/Lowering-shared-inl.h b/js/src/jit/shared/Lowering-shared-inl.h index 75869595b5..95e315614e 100644 --- a/js/src/jit/shared/Lowering-shared-inl.h +++ b/js/src/jit/shared/Lowering-shared-inl.h @@ -301,7 +301,7 @@ LIRGeneratorShared::defineSinCos(LInstructionHelper<2, Ops, Temps> *lir, MDefini #elif defined(JS_CODEGEN_ARM64) lir->setDef(1, LDefinition(vreg + VREG_INCREMENT, LDefinition::DOUBLE, LFloatReg(FloatRegister(FloatRegisters::d1, FloatRegisters::Double)))); -#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONGARCH64) lir->setDef(1, LDefinition(vreg + VREG_INCREMENT, LDefinition::DOUBLE, LFloatReg(f2))); #elif defined(JS_CODEGEN_NONE) MOZ_CRASH(); diff --git a/js/src/jsutil.h b/js/src/jsutil.h index 00c2806c1d..1969a2e823 100644 --- a/js/src/jsutil.h +++ b/js/src/jsutil.h @@ -443,7 +443,7 @@ PodSet(T* aDst, const T& aSrc, size_t aNElem) # define JS_SWEPT_CODE_PATTERN 0xED // IN instruction, crashes in user mode. #elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) # define JS_SWEPT_CODE_PATTERN 0xA3 // undefined instruction -#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONGARCH64) # define JS_SWEPT_CODE_PATTERN 0x01 // undefined instruction #else # error "JS_SWEPT_CODE_PATTERN not defined for this platform" diff --git a/js/src/moz.build b/js/src/moz.build index 4cfec1b74f..23db100e7b 100644 --- a/js/src/moz.build +++ b/js/src/moz.build @@ -524,6 +524,24 @@ elif CONFIG['JS_CODEGEN_ARM64']: 'jit/arm64/vixl/MozSimulator-vixl.cpp', 'jit/arm64/vixl/Simulator-vixl.cpp' ] +elif CONFIG['JS_CODEGEN_LOONGARCH64']: + jit_ioncheck1_deunified_sources += [ + 'jit/mips-shared/Bailouts-mips-shared.cpp', + 'jit/mips-shared/BaselineCompiler-mips-shared.cpp', + 'jit/mips-shared/BaselineIC-mips-shared.cpp', + 'jit/mips-shared/MoveEmitter-mips-shared.cpp', + ] + jit_ioncheck2_deunified_sources += [ + 'jit/loongarch64/Architecture-loongarch64.cpp', + 'jit/loongarch64/Assembler-loongarch64.cpp', + 'jit/loongarch64/Bailouts-loongarch64.cpp', + 'jit/loongarch64/BaselineCompiler-loongarch64.cpp', + 'jit/loongarch64/BaselineIC-loongarch64.cpp', + 'jit/loongarch64/MacroAssembler-loongarch64.cpp', + 'jit/mips64/MoveEmitter-mips64.cpp', + 'jit/loongarch64/SharedIC-loongarch64.cpp', + 'jit/loongarch64/Trampoline-loongarch64.cpp', + ] elif CONFIG['JS_CODEGEN_MIPS32'] or CONFIG['JS_CODEGEN_MIPS64']: jit_ioncheck1_deunified_sources += [ 'jit/mips-shared/Architecture-mips-shared.cpp', diff --git a/js/src/old-configure.in b/js/src/old-configure.in index 7329fc470d..8838524219 100644 --- a/js/src/old-configure.in +++ b/js/src/old-configure.in @@ -976,6 +976,10 @@ arm*-*) ENABLE_ION=1 AC_DEFINE(JS_CPU_ARM) ;; +loongarch64*-*) + ENABLE_ION=1 + AC_DEFINE(JS_CPU_LOONGARCH64) + ;; sparc-*) dnl ENABLE_ION=0 AC_DEFINE(JS_CPU_SPARC) @@ -1781,6 +1785,9 @@ elif test "$CPU_ARCH" = "arm"; then dnl ARM platforms may trap on unaligned accesses; catch the signal and dnl recover. +elif test "$CPU_ARCH" = "loongarch64"; then + AC_DEFINE(JS_CODEGEN_LOONGARCH64) + JS_CODEGEN_LOONGARCH64=1 elif test "$CPU_ARCH" = "mips" || test "$CPU_ARCH" = "mips32" || test "$CPU_ARCH" = "mips64"; then if test ! "$HAVE_64BIT_BUILD"; then AC_DEFINE(JS_CODEGEN_MIPS32) @@ -1798,6 +1805,7 @@ AC_SUBST(JS_SIMULATOR_MIPS32) AC_SUBST(JS_SIMULATOR_MIPS64) AC_SUBST(JS_CODEGEN_ARM) AC_SUBST(JS_CODEGEN_ARM64) +AC_SUBST(JS_CODEGEN_LOONGARCH64) AC_SUBST(JS_CODEGEN_MIPS32) AC_SUBST(JS_CODEGEN_MIPS64) AC_SUBST(JS_CODEGEN_X86) diff --git a/js/src/wasm/WasmJS.cpp b/js/src/wasm/WasmJS.cpp index dea2bddc46..be953f0aa5 100644 --- a/js/src/wasm/WasmJS.cpp +++ b/js/src/wasm/WasmJS.cpp @@ -66,7 +66,7 @@ wasm::HasCompilerSupport(ExclusiveContext* cx) return false; #endif -#if defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_ARM64) +#if defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_LOONGARCH64) return false; #else return true; diff --git a/js/src/wasm/WasmSignalHandlers.cpp b/js/src/wasm/WasmSignalHandlers.cpp index f68285e401..733ad48f99 100644 --- a/js/src/wasm/WasmSignalHandlers.cpp +++ b/js/src/wasm/WasmSignalHandlers.cpp @@ -392,7 +392,7 @@ struct macos_aarch64_context { # define PC_sig(p) R15_sig(p) #elif defined(__aarch64__) # define PC_sig(p) EPC_sig(p) -#elif defined(JS_CPU_MIPS) +#elif defined(JS_CPU_MIPS) || defined(JS_CPU_LOONGARCH64) # define PC_sig(p) EPC_sig(p) #endif