From 0b36c23dc56c5ef9839b9562fdc55eb2f10d732a Mon Sep 17 00:00:00 2001 From: win7-7 Date: Tue, 30 Dec 2025 05:05:26 +0200 Subject: [PATCH] Bug 1312751 - Merge ARM load and store code for Wasm 1312751 - Refactor wasm{Load,Store}{,I64} to MacroAssembler-arm. 1312751 - Wasm baseline, use refactored ARM load/store methods. --- js/src/jit/MacroAssembler.h | 37 +++ js/src/jit/arm/CodeGenerator-arm.cpp | 249 +++------------ js/src/jit/arm/MacroAssembler-arm.cpp | 316 +++++++++++++++++++ js/src/jit/arm/MacroAssembler-arm.h | 23 +- js/src/wasm/WasmBaselineCompile.cpp | 418 ++++++-------------------- 5 files changed, 503 insertions(+), 540 deletions(-) diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h index b14d91e64d..c51210dd90 100644 --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -1376,6 +1376,43 @@ class MacroAssembler : public MacroAssemblerSpecific void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Operand dstAddr) DEFINED_ON(x86, x64); void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Operand dstAddr) DEFINED_ON(x86); + // For all the ARM wasmLoad and wasmStore functions, `ptr` MUST equal + // `ptrScratch`, and that register will be updated based on conditions + // listed below (where it is only mentioned as `ptr`). + + // `ptr` will be updated if access.offset() != 0 or access.type() == Scalar::Int64. + void wasmLoad(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, AnyRegister output) DEFINED_ON(arm); + void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, Register64 output) DEFINED_ON(arm); + void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Register ptr, Register ptrScratch) DEFINED_ON(arm); + void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Register ptr, Register ptrScratch) DEFINED_ON(arm); + + // `ptr` will always be updated. + void wasmUnalignedLoad(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, + Register output, Register tmp) DEFINED_ON(arm); + + // `ptr` will always be updated and `tmp1` is always needed. `tmp2` is + // needed for Float32; `tmp2` and `tmp3` are needed for Float64. Temps must + // be Invalid when they are not needed. + void wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, + FloatRegister output, Register tmp1, Register tmp2, Register tmp3) DEFINED_ON(arm); + + // `ptr` will always be updated. + void wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, + Register64 output, Register tmp) DEFINED_ON(arm); + + // `ptr` and `value` will always be updated. + void wasmUnalignedStore(const wasm::MemoryAccessDesc& access, Register value, Register ptr, Register ptrScratch) + DEFINED_ON(arm); + + // `ptr` will always be updated. + void wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access, FloatRegister floatValue, Register ptr, + Register ptrScratch, Register tmp) DEFINED_ON(arm); + + // `ptr` will always be updated. + void wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Register ptr, Register ptrScratch, + Register tmp) DEFINED_ON(arm); + + // wasm specific methods, used in both the wasm baseline compiler and ion. void wasmTruncateDoubleToUInt32(FloatRegister input, Register output, Label* oolEntry) DEFINED_ON(x86, x64, arm); void wasmTruncateDoubleToInt32(FloatRegister input, Register output, Label* oolEntry) DEFINED_ON(x86_shared, arm); diff --git a/js/src/jit/arm/CodeGenerator-arm.cpp b/js/src/jit/arm/CodeGenerator-arm.cpp index 25ed23a2c8..4325aac9a6 100644 --- a/js/src/jit/arm/CodeGenerator-arm.cpp +++ b/js/src/jit/arm/CodeGenerator-arm.cpp @@ -2366,68 +2366,20 @@ CodeGeneratorARM::emitWasmLoad(T* lir) { const MWasmLoad* mir = lir->mir(); - uint32_t offset = mir->access().offset(); - MOZ_ASSERT(offset < wasm::OffsetGuardLimit); + MIRType resultType = mir->type(); + Register ptr; - Register ptr = ToRegister(lir->ptr()); - Scalar::Type type = mir->access().type(); - - // Maybe add the offset. - if (offset || type == Scalar::Int64) { - ScratchRegisterScope scratch(masm); - Register ptrPlusOffset = ToRegister(lir->ptrCopy()); - if (offset) - masm.ma_add(Imm32(offset), ptrPlusOffset, scratch); - ptr = ptrPlusOffset; + if (mir->access().offset() || mir->access().type() == Scalar::Int64) { + ptr = ToRegister(lir->ptrCopy()); } else { MOZ_ASSERT(lir->ptrCopy()->isBogusTemp()); + ptr = ToRegister(lir->ptr()); } - bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 || type == Scalar::Int32 || - type == Scalar::Int64; - unsigned byteSize = mir->access().byteSize(); - - masm.memoryBarrier(mir->access().barrierBefore()); - - BufferOffset load; - if (mir->type() == MIRType::Int64) { - Register64 output = ToOutRegister64(lir); - if (type == Scalar::Int64) { - MOZ_ASSERT(INT64LOW_OFFSET == 0); - - load = masm.ma_dataTransferN(IsLoad, 32, /* signed = */ false, HeapReg, ptr, output.low); - masm.append(mir->access(), load.getOffset(), masm.framePushed()); - - masm.as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET)); - - load = masm.ma_dataTransferN(IsLoad, 32, isSigned, HeapReg, ptr, output.high); - masm.append(mir->access(), load.getOffset(), masm.framePushed()); - } else { - load = masm.ma_dataTransferN(IsLoad, byteSize * 8, isSigned, HeapReg, ptr, output.low); - masm.append(mir->access(), load.getOffset(), masm.framePushed()); - - if (isSigned) - masm.ma_asr(Imm32(31), output.low, output.high); - else - masm.ma_mov(Imm32(0), output.high); - } - } else { - AnyRegister output = ToAnyRegister(lir->output()); - bool isFloat = output.isFloat(); - if (isFloat) { - MOZ_ASSERT((byteSize == 4) == output.fpu().isSingle()); - ScratchRegisterScope scratch(masm); - masm.ma_add(HeapReg, ptr, scratch); - - load = masm.ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), output.fpu()); - masm.append(mir->access(), load.getOffset(), masm.framePushed()); - } else { - load = masm.ma_dataTransferN(IsLoad, byteSize * 8, isSigned, HeapReg, ptr, output.gpr()); - masm.append(mir->access(), load.getOffset(), masm.framePushed()); - } - } - - masm.memoryBarrier(mir->access().barrierAfter()); + if (resultType == MIRType::Int64) + masm.wasmLoadI64(mir->access(), ptr, ptr, ToOutRegister64(lir)); + else + masm.wasmLoad(mir->access(), ptr, ptr, ToAnyRegister(lir->output())); } void @@ -2448,70 +2400,22 @@ CodeGeneratorARM::emitWasmUnalignedLoad(T* lir) { const MWasmLoad* mir = lir->mir(); - uint32_t offset = mir->access().offset(); - MOZ_ASSERT(offset < wasm::OffsetGuardLimit); + MIRType resultType = mir->type(); Register ptr = ToRegister(lir->ptrCopy()); - if (offset) { - ScratchRegisterScope scratch(masm); - masm.ma_add(Imm32(offset), ptr, scratch); + Register tmp1 = ToRegister(lir->getTemp(1)); + + if (resultType == MIRType::Int64) { + masm.wasmUnalignedLoadI64(mir->access(), ptr, ptr, ToOutRegister64(lir), tmp1); + } else if (IsFloatingPointType(resultType)) { + Register tmp2(ToRegister(lir->getTemp(2))); + Register tmp3(Register::Invalid()); + if (mir->access().byteSize() == 8) + tmp3 = ToRegister(lir->getTemp(3)); + masm.wasmUnalignedLoadFP(mir->access(), ptr, ptr, ToFloatRegister(lir->output()), tmp1, tmp2, tmp3); + } else { + masm.wasmUnalignedLoad(mir->access(), ptr, ptr, ToRegister(lir->output()), tmp1); } - - // Add HeapReg to ptr, so we can use base+index addressing in the byte loads. - masm.ma_add(HeapReg, ptr); - - unsigned byteSize = mir->access().byteSize(); - Scalar::Type type = mir->access().type(); - bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 || type == Scalar::Int32 || - type == Scalar::Int64; - - MIRType mirType = mir->type(); - - Register tmp = ToRegister(lir->getTemp(1)); - - Register low; - if (IsFloatingPointType(mirType)) - low = ToRegister(lir->getTemp(2)); - else if (mirType == MIRType::Int64) - low = ToOutRegister64(lir).low; - else - low = ToRegister(lir->output()); - - MOZ_ASSERT(low != tmp); - MOZ_ASSERT(low != ptr); - - masm.memoryBarrier(mir->access().barrierBefore()); - - masm.emitUnalignedLoad(isSigned, Min(byteSize, 4u), ptr, tmp, low); - - if (IsFloatingPointType(mirType)) { - FloatRegister output = ToFloatRegister(lir->output()); - if (byteSize == 4) { - MOZ_ASSERT(output.isSingle()); - masm.ma_vxfer(low, output); - } else { - MOZ_ASSERT(byteSize == 8); - MOZ_ASSERT(output.isDouble()); - Register high = ToRegister(lir->getTemp(3)); - masm.emitUnalignedLoad(/* signed */ false, 4, ptr, tmp, high, /* offset */ 4); - masm.ma_vxfer(low, high, output); - } - } else if (mirType == MIRType::Int64) { - Register64 output = ToOutRegister64(lir); - if (type == Scalar::Int64) { - MOZ_ASSERT(byteSize == 8); - masm.emitUnalignedLoad(isSigned, 4, ptr, tmp, output.high, /* offset */ 4); - } else { - MOZ_ASSERT(byteSize <= 4); - // Propagate sign. - if (isSigned) - masm.ma_asr(Imm32(31), output.low, output.high); - else - masm.ma_mov(Imm32(0), output.high); - } - } - - masm.memoryBarrier(mir->access().barrierAfter()); } void @@ -2545,59 +2449,22 @@ CodeGeneratorARM::emitWasmStore(T* lir) { const MWasmStore* mir = lir->mir(); - uint32_t offset = mir->access().offset(); - MOZ_ASSERT(offset < wasm::OffsetGuardLimit); - - Register ptr = ToRegister(lir->ptr()); - unsigned byteSize = mir->access().byteSize(); - Scalar::Type type = mir->access().type(); + Scalar::Type accessType = mir->access().type(); + Register ptr; // Maybe add the offset. - if (offset || type == Scalar::Int64) { - ScratchRegisterScope scratch(masm); - Register ptrPlusOffset = ToRegister(lir->ptrCopy()); - if (offset) - masm.ma_add(Imm32(offset), ptrPlusOffset, scratch); - ptr = ptrPlusOffset; + if (mir->access().offset() || accessType == Scalar::Int64) { + ptr = ToRegister(lir->ptrCopy()); } else { MOZ_ASSERT(lir->ptrCopy()->isBogusTemp()); + ptr = ToRegister(lir->ptr()); } - masm.memoryBarrier(mir->access().barrierBefore()); - - BufferOffset store; - if (type == Scalar::Int64) { - MOZ_ASSERT(INT64LOW_OFFSET == 0); - - Register64 value = ToRegister64(lir->getInt64Operand(lir->ValueIndex)); - - store = masm.ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ false, HeapReg, ptr, value.low); - masm.append(mir->access(), store.getOffset(), masm.framePushed()); - - masm.as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET)); - - store = masm.ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ true, HeapReg, ptr, value.high); - masm.append(mir->access(), store.getOffset(), masm.framePushed()); - } else { - AnyRegister value = ToAnyRegister(lir->getOperand(lir->ValueIndex)); - if (value.isFloat()) { - ScratchRegisterScope scratch(masm); - FloatRegister val = value.fpu(); - MOZ_ASSERT((byteSize == 4) == val.isSingle()); - masm.ma_add(HeapReg, ptr, scratch); - - store = masm.ma_vstr(val, Operand(Address(scratch, 0)).toVFPAddr()); - masm.append(mir->access(), store.getOffset(), masm.framePushed()); - } else { - bool isSigned = type == Scalar::Uint32 || type == Scalar::Int32; // see AsmJSStoreHeap; - Register val = value.gpr(); - - store = masm.ma_dataTransferN(IsStore, 8 * byteSize /* bits */, isSigned, HeapReg, ptr, val); - masm.append(mir->access(), store.getOffset(), masm.framePushed()); - } - } - - masm.memoryBarrier(mir->access().barrierAfter()); + if (accessType == Scalar::Int64) + masm.wasmStoreI64(mir->access(), ToRegister64(lir->getInt64Operand(lir->ValueIndex)), + ptr, ptr); + else + masm.wasmStore(mir->access(), ToAnyRegister(lir->getOperand(lir->ValueIndex)), ptr, ptr); } void @@ -2618,50 +2485,20 @@ CodeGeneratorARM::emitWasmUnalignedStore(T* lir) { const MWasmStore* mir = lir->mir(); - uint32_t offset = mir->access().offset(); - MOZ_ASSERT(offset < wasm::OffsetGuardLimit); + Scalar::Type accessType = mir->access().type(); Register ptr = ToRegister(lir->ptrCopy()); - if (offset) { - ScratchRegisterScope scratch(masm); - masm.ma_add(Imm32(offset), ptr, scratch); + Register valOrTmp = ToRegister(lir->valueHelper()); + if (accessType == Scalar::Int64) { + masm.wasmUnalignedStoreI64(mir->access(), + ToRegister64(lir->getInt64Operand(LWasmUnalignedStoreI64::ValueIndex)), + ptr, ptr, valOrTmp); + } else if (accessType == Scalar::Float32 || accessType == Scalar::Float64) { + FloatRegister value = ToFloatRegister(lir->getOperand(LWasmUnalignedStore::ValueIndex)); + masm.wasmUnalignedStoreFP(mir->access(), value, ptr, ptr, valOrTmp); + } else { + masm.wasmUnalignedStore(mir->access(), valOrTmp, ptr, ptr); } - - // Add HeapReg to ptr, so we can use base+index addressing in the byte loads. - masm.ma_add(HeapReg, ptr); - - MIRType mirType = mir->value()->type(); - - masm.memoryBarrier(mir->access().barrierAfter()); - - Register val = ToRegister(lir->valueHelper()); - if (IsFloatingPointType(mirType)) { - masm.ma_vxfer(ToFloatRegister(lir->getOperand(LWasmUnalignedStore::ValueIndex)), val); - } else if (mirType == MIRType::Int64) { - Register64 input = ToRegister64(lir->getInt64Operand(LWasmUnalignedStoreI64::ValueIndex)); - if (input.low != val) - masm.ma_mov(input.low, val); - } - - unsigned byteSize = mir->access().byteSize(); - masm.emitUnalignedStore(Min(byteSize, 4u), ptr, val); - - if (byteSize > 4) { - // It's a double or an int64 load. - // Load the high 32 bits when counter == 4. - if (IsFloatingPointType(mirType)) { - FloatRegister fp = ToFloatRegister(lir->getOperand(LWasmUnalignedStore::ValueIndex)); - MOZ_ASSERT(fp.isDouble()); - ScratchRegisterScope scratch(masm); - masm.ma_vxfer(fp, scratch, val); - } else { - MOZ_ASSERT(mirType == MIRType::Int64); - masm.ma_mov(ToRegister64(lir->getInt64Operand(LWasmUnalignedStoreI64::ValueIndex)).high, val); - } - masm.emitUnalignedStore(4, ptr, val, /* offset */ 4); - } - - masm.memoryBarrier(mir->access().barrierBefore()); } void diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp index a8a489af20..7b91f6527f 100644 --- a/js/src/jit/arm/MacroAssembler-arm.cpp +++ b/js/src/jit/arm/MacroAssembler-arm.cpp @@ -5438,6 +5438,80 @@ MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, Register output, wasmTruncateToInt32(input, output, MIRType::Float32, /* isUnsigned= */ false, oolEntry); } +void +MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, + AnyRegister output) +{ + wasmLoadImpl(access, ptr, ptrScratch, output, Register64::Invalid()); +} + +void +MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, + Register64 output) +{ + wasmLoadImpl(access, ptr, ptrScratch, AnyRegister(), output); +} + +void +MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Register ptr, + Register ptrScratch) +{ + wasmStoreImpl(access, value, Register64::Invalid(), ptr, ptrScratch); +} + +void +MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Register ptr, + Register ptrScratch) +{ + wasmStoreImpl(access, AnyRegister(), value, ptr, ptrScratch); +} + +void +MacroAssembler::wasmUnalignedLoad(const wasm::MemoryAccessDesc& access, Register ptr, + Register ptrScratch, Register output, Register tmp) +{ + wasmUnalignedLoadImpl(access, ptr, ptrScratch, AnyRegister(output), Register64::Invalid(), tmp, + Register::Invalid(), Register::Invalid()); +} + +void +MacroAssembler::wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access, Register ptr, + Register ptrScratch, FloatRegister outFP, Register tmp1, + Register tmp2, Register tmp3) +{ + wasmUnalignedLoadImpl(access, ptr, ptrScratch, AnyRegister(outFP), Register64::Invalid(), + tmp1, tmp2, tmp3); +} + +void +MacroAssembler::wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access, Register ptr, + Register ptrScratch, Register64 out64, Register tmp) +{ + wasmUnalignedLoadImpl(access, ptr, ptrScratch, AnyRegister(), out64, tmp, Register::Invalid(), + Register::Invalid()); +} + +void +MacroAssembler::wasmUnalignedStore(const wasm::MemoryAccessDesc& access, Register value, + Register ptr, Register ptrScratch) +{ + wasmUnalignedStoreImpl(access, FloatRegister(), Register64::Invalid(), ptr, ptrScratch, value); +} + +void +MacroAssembler::wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access, FloatRegister floatVal, + Register ptr, Register ptrScratch, Register tmp) +{ + wasmUnalignedStoreImpl(access, floatVal, Register64::Invalid(), ptr, ptrScratch, tmp); +} + +void +MacroAssembler::wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access, Register64 val64, + Register ptr, Register ptrScratch, Register tmp) +{ + wasmUnalignedStoreImpl(access, FloatRegister(), val64, ptr, ptrScratch, tmp); +} + //}}} check_macroassembler_style void @@ -5582,6 +5656,248 @@ MacroAssemblerARM::outOfLineWasmTruncateToIntCheck(FloatRegister input, MIRType asMasm().framePushed())); } +void +MacroAssemblerARM::wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register ptr, + Register ptrScratch, AnyRegister output, Register64 out64) +{ + MOZ_ASSERT(ptr == ptrScratch); + + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < wasm::OffsetGuardLimit); + + Scalar::Type type = access.type(); + + // Maybe add the offset. + if (offset || type == Scalar::Int64) { + ScratchRegisterScope scratch(asMasm()); + if (offset) + ma_add(Imm32(offset), ptr, scratch); + } + + bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 || type == Scalar::Int32 || + type == Scalar::Int64; + unsigned byteSize = access.byteSize(); + + asMasm().memoryBarrier(access.barrierBefore()); + + uint32_t framePushed = asMasm().framePushed(); + BufferOffset load; + if (out64 != Register64::Invalid()) { + if (type == Scalar::Int64) { + MOZ_ASSERT(INT64LOW_OFFSET == 0); + + load = ma_dataTransferN(IsLoad, 32, /* signed = */ false, HeapReg, ptr, out64.low); + append(access, load.getOffset(), framePushed); + + as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET)); + + load = ma_dataTransferN(IsLoad, 32, isSigned, HeapReg, ptr, out64.high); + append(access, load.getOffset(), framePushed); + } else { + load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, HeapReg, ptr, out64.low); + append(access, load.getOffset(), framePushed); + + if (isSigned) + ma_asr(Imm32(31), out64.low, out64.high); + else + ma_mov(Imm32(0), out64.high); + } + } else { + bool isFloat = output.isFloat(); + if (isFloat) { + MOZ_ASSERT((byteSize == 4) == output.fpu().isSingle()); + ScratchRegisterScope scratch(asMasm()); + ma_add(HeapReg, ptr, scratch); + + load = ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), output.fpu()); + append(access, load.getOffset(), framePushed); + } else { + load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, HeapReg, ptr, output.gpr()); + append(access, load.getOffset(), framePushed); + } + } + + asMasm().memoryBarrier(access.barrierAfter()); +} + +void +MacroAssemblerARM::wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister value, + Register64 val64, Register ptr, Register ptrScratch) +{ + MOZ_ASSERT(ptr == ptrScratch); + + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < wasm::OffsetGuardLimit); + + unsigned byteSize = access.byteSize(); + Scalar::Type type = access.type(); + + // Maybe add the offset. + if (offset || type == Scalar::Int64) { + ScratchRegisterScope scratch(asMasm()); + if (offset) + ma_add(Imm32(offset), ptr, scratch); + } + + asMasm().memoryBarrier(access.barrierBefore()); + + uint32_t framePushed = asMasm().framePushed(); + + BufferOffset store; + if (type == Scalar::Int64) { + MOZ_ASSERT(INT64LOW_OFFSET == 0); + + store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ false, HeapReg, ptr, val64.low); + append(access, store.getOffset(), framePushed); + + as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET)); + + store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ true, HeapReg, ptr, val64.high); + append(access, store.getOffset(), framePushed); + } else { + if (value.isFloat()) { + ScratchRegisterScope scratch(asMasm()); + FloatRegister val = value.fpu(); + MOZ_ASSERT((byteSize == 4) == val.isSingle()); + ma_add(HeapReg, ptr, scratch); + + store = ma_vstr(val, Operand(Address(scratch, 0)).toVFPAddr()); + append(access, store.getOffset(), framePushed); + } else { + bool isSigned = type == Scalar::Uint32 || type == Scalar::Int32; // see AsmJSStoreHeap; + Register val = value.gpr(); + + store = ma_dataTransferN(IsStore, 8 * byteSize /* bits */, isSigned, HeapReg, ptr, val); + append(access, store.getOffset(), framePushed); + } + } + + asMasm().memoryBarrier(access.barrierAfter()); +} + +void +MacroAssemblerARM::wasmUnalignedLoadImpl(const wasm::MemoryAccessDesc& access, Register ptr, + Register ptrScratch, AnyRegister outAny, Register64 out64, + Register tmp, Register tmp2, Register tmp3) +{ + MOZ_ASSERT(ptr == ptrScratch); + MOZ_ASSERT_IF(access.type() != Scalar::Float32 && access.type() != Scalar::Float64, + tmp2 == Register::Invalid() && tmp3 == Register::Invalid()); + MOZ_ASSERT_IF(access.type() == Scalar::Float32, + tmp2 != Register::Invalid() && tmp3 == Register::Invalid()); + MOZ_ASSERT_IF(access.type() == Scalar::Float64, + tmp2 != Register::Invalid() && tmp3 != Register::Invalid()); + + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < wasm::OffsetGuardLimit); + + if (offset) { + ScratchRegisterScope scratch(asMasm()); + ma_add(Imm32(offset), ptr, scratch); + } + + // Add HeapReg to ptr, so we can use baseindex addressing in the byte loads. + ma_add(HeapReg, ptr); + + unsigned byteSize = access.byteSize(); + Scalar::Type type = access.type(); + bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 || type == Scalar::Int32 || + type == Scalar::Int64; + + Register low; + if (out64 != Register64::Invalid()) + low = out64.low; + else if (outAny.isFloat()) + low = tmp2; + else + low = outAny.gpr(); + + MOZ_ASSERT(low != tmp); + MOZ_ASSERT(low != ptr); + + asMasm().memoryBarrier(access.barrierBefore()); + + emitUnalignedLoad(isSigned, Min(byteSize, 4u), ptr, tmp, low); + + if (out64 != Register64::Invalid()) { + if (type == Scalar::Int64) { + MOZ_ASSERT(byteSize == 8); + emitUnalignedLoad(isSigned, 4, ptr, tmp, out64.high, /* offset */ 4); + } else { + MOZ_ASSERT(byteSize <= 4); + // Propagate sign. + if (isSigned) + ma_asr(Imm32(31), out64.low, out64.high); + else + ma_mov(Imm32(0), out64.high); + } + } else if (outAny.isFloat()) { + FloatRegister output = outAny.fpu(); + if (byteSize == 4) { + MOZ_ASSERT(output.isSingle()); + ma_vxfer(low, output); + } else { + MOZ_ASSERT(byteSize == 8); + MOZ_ASSERT(output.isDouble()); + Register high = tmp3; + emitUnalignedLoad(/* signed */ false, 4, ptr, tmp, high, /* offset */ 4); + ma_vxfer(low, high, output); + } + } + + asMasm().memoryBarrier(access.barrierAfter()); +} + +void +MacroAssemblerARM::wasmUnalignedStoreImpl(const wasm::MemoryAccessDesc& access, FloatRegister floatValue, + Register64 val64, Register ptr, Register ptrScratch, Register tmp) +{ + MOZ_ASSERT(ptr == ptrScratch); + // They can't both be valid, but they can both be invalid. + MOZ_ASSERT_IF(!floatValue.isInvalid(), val64 == Register64::Invalid()); + MOZ_ASSERT_IF(val64 != Register64::Invalid(), floatValue.isInvalid()); + + uint32_t offset = access.offset(); + MOZ_ASSERT(offset < wasm::OffsetGuardLimit); + + unsigned byteSize = access.byteSize(); + + if (offset) { + ScratchRegisterScope scratch(asMasm()); + ma_add(Imm32(offset), ptr, scratch); + } + + // Add HeapReg to ptr, so we can use baseindex addressing in the byte loads. + ma_add(HeapReg, ptr); + + asMasm().memoryBarrier(access.barrierBefore()); + + if (val64 != Register64::Invalid()) { + if (val64.low != tmp) + ma_mov(val64.low, tmp); + } else if (!floatValue.isInvalid()) { + ma_vxfer(floatValue, tmp); + } + // Otherwise, tmp has the integer value to store. + + emitUnalignedStore(Min(byteSize, 4u), ptr, tmp); + + if (byteSize > 4) { + if (val64 != Register64::Invalid()) { + if (val64.high != tmp) + ma_mov(val64.high, tmp); + } else { + MOZ_ASSERT(!floatValue.isInvalid()); + MOZ_ASSERT(floatValue.isDouble()); + ScratchRegisterScope scratch(asMasm()); + ma_vxfer(floatValue, scratch, tmp); + } + emitUnalignedStore(4, ptr, tmp, /* offset */ 4); + } + + asMasm().memoryBarrier(access.barrierAfter()); +} + void MacroAssemblerARM::emitUnalignedLoad(bool isSigned, unsigned byteSize, Register ptr, Register tmp, Register dest, unsigned offset) diff --git a/js/src/jit/arm/MacroAssembler-arm.h b/js/src/jit/arm/MacroAssembler-arm.h index 745b8b1e53..4d8f1aff97 100644 --- a/js/src/jit/arm/MacroAssembler-arm.h +++ b/js/src/jit/arm/MacroAssembler-arm.h @@ -452,6 +452,28 @@ class MacroAssemblerARM : public Assembler MOZ_CRASH("Invalid data transfer addressing mode"); } + // `outAny` is valid if and only if `out64` == Register64::Invalid(). + void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, + AnyRegister outAny, Register64 out64); + + // `valAny` is valid if and only if `val64` == Register64::Invalid(). + void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister valAny, Register64 val64, + Register ptr, Register ptrScratch); + + protected: + // `outAny` is valid if and only if `out64` == Register64::Invalid(). + void wasmUnalignedLoadImpl(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, + AnyRegister outAny, Register64 out64, Register tmp1, Register tmp2, + Register tmp3); + + // The value to be stored is in `floatValue` (if not invalid), `val64` (if not invalid), + // or in `valOrTmp` (if `floatValue` and `val64` are both invalid). Note `valOrTmp` must + // always be valid. + void wasmUnalignedStoreImpl(const wasm::MemoryAccessDesc& access, FloatRegister floatValue, + Register64 val64, Register ptr, Register ptrScratch, Register valOrTmp); + + private: + // Loads `byteSize` bytes, byte by byte, by reading from ptr[offset], // applying the indicated signedness (defined by isSigned). // - all three registers must be different. @@ -466,7 +488,6 @@ class MacroAssemblerARM : public Assembler // - byteSize can be up to 4 bytes and no more (GPR are 32 bits on ARM). void emitUnalignedStore(unsigned byteSize, Register ptr, Register val, unsigned offset = 0); -private: // Implementation for transferMultipleByRuns so we can use different // iterators for forward/backward traversals. The sign argument should be 1 // if we traverse forwards, -1 if we traverse backwards. diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp index 09cb43306b..f1476ab08d 100644 --- a/js/src/wasm/WasmBaselineCompile.cpp +++ b/js/src/wasm/WasmBaselineCompile.cpp @@ -3588,16 +3588,16 @@ class BaseCompiler } // This is the temp register passed as the last argument to load() - MOZ_MUST_USE size_t loadStoreTemps(MemoryAccessDesc& access) { + [[nodiscard]] size_t loadTemps(MemoryAccessDesc& access) { #if defined(JS_CODEGEN_ARM) if (IsUnaligned(access)) { switch (access.type()) { case Scalar::Float32: - return 1; - case Scalar::Float64: return 2; + case Scalar::Float64: + return 3; default: - break; + return 1; } } return 0; @@ -3610,8 +3610,8 @@ class BaseCompiler // ptr and dest may be the same iff dest is I32. // This may destroy ptr even if ptr and dest are not the same. - MOZ_MUST_USE bool load(MemoryAccessDesc& access, RegI32 ptr, AnyReg dest, RegI32 tmp1, - RegI32 tmp2) + [[nodiscard]] bool load(MemoryAccessDesc& access, RegI32 ptr, AnyReg dest, + RegI32 tmp1, RegI32 tmp2, RegI32 tmp3) { checkOffset(&access, ptr); @@ -3650,40 +3650,26 @@ class BaseCompiler masm.mov(ScratchRegX86, dest.i32().reg); } #elif defined(JS_CODEGEN_ARM) - if (access.offset() != 0) - masm.add32(Imm32(access.offset()), ptr.reg); - - bool isSigned = true; - switch (access.type()) { - case Scalar::Uint8: - case Scalar::Uint16: - case Scalar::Uint32: { - isSigned = false; - MOZ_FALLTHROUGH; - case Scalar::Int8: - case Scalar::Int16: - case Scalar::Int32: - Register rt = dest.tag == AnyReg::I64 ? dest.i64().reg.low : dest.i32().reg; - loadI32(access, isSigned, ptr, rt); - if (dest.tag == AnyReg::I64) { - if (isSigned) - masm.ma_asr(Imm32(31), rt, dest.i64().reg.high); - else - masm.move32(Imm32(0), dest.i64().reg.high); + if (access.isUnaligned()) { + switch (dest.tag) { + case AnyReg::I64: + masm.wasmUnalignedLoadI64(access, ptr, ptr, dest.i64(), tmp1); + break; + case AnyReg::F32: + masm.wasmUnalignedLoadFP(access, ptr, ptr, dest.f32(), tmp1, tmp2, Register::Invalid()); + break; + case AnyReg::F64: + masm.wasmUnalignedLoadFP(access, ptr, ptr, dest.f64(), tmp1, tmp2, tmp3); + break; + default: + masm.wasmUnalignedLoad(access, ptr, ptr, dest.i32(), tmp1); + break; } - break; - } - case Scalar::Int64: - loadI64(access, ptr, dest.i64()); - break; - case Scalar::Float32: - loadF32(access, ptr, dest.f32(), tmp1); - break; - case Scalar::Float64: - loadF64(access, ptr, dest.f64(), tmp1, tmp2); - break; - default: - MOZ_CRASH("Compiler bug: unexpected array type"); + } else { + if (dest.tag == AnyReg::I64) + masm.wasmLoadI64(access, ptr, ptr, dest.i64()); + else + masm.wasmLoad(access, ptr, ptr, dest.any()); } #elif defined(JS_CODEGEN_LOONGARCH64) switch (access.type()) { @@ -3752,10 +3738,21 @@ class BaseCompiler return true; } + [[nodiscard]] size_t storeTemps(MemoryAccessDesc& access) { +#if defined(JS_CODEGEN_ARM) + if (access.isUnaligned()) { + // See comment in store() about how this temp could be avoided for + // unaligned i8/i16/i32 stores with some restructuring elsewhere. + return 1; + } +#endif + return 0; + } + // ptr and src must not be the same register. - // This may destroy ptr. - MOZ_MUST_USE bool store(MemoryAccessDesc access, RegI32 ptr, AnyReg src, RegI32 tmp1, - RegI32 tmp2) + // This may destroy ptr but will not destroy src. + [[nodiscard]] bool store(MemoryAccessDesc access, RegI32 ptr, AnyReg src, + RegI32 tmp) { checkOffset(&access, ptr); @@ -3791,36 +3788,36 @@ class BaseCompiler masm.wasmStore(access, value, dstAddr); } #elif defined(JS_CODEGEN_ARM) - if (access.offset() != 0) - masm.add32(Imm32(access.offset()), ptr.reg); - - switch (access.type()) { - case Scalar::Uint8: - MOZ_FALLTHROUGH; - case Scalar::Uint16: - MOZ_FALLTHROUGH; - case Scalar::Int8: - MOZ_FALLTHROUGH; - case Scalar::Int16: - MOZ_FALLTHROUGH; - case Scalar::Int32: - MOZ_FALLTHROUGH; - case Scalar::Uint32: { - Register rt = src.tag == AnyReg::I64 ? src.i64().reg.low : src.i32().reg; - storeI32(access, ptr, rt); - break; - } - case Scalar::Int64: - storeI64(access, ptr, src.i64()); - break; - case Scalar::Float32: - storeF32(access, ptr, src.f32(), tmp1); - break; - case Scalar::Float64: - storeF64(access, ptr, src.f64(), tmp1, tmp2); - break; - default: - MOZ_CRASH("Compiler bug: unexpected array type"); + if (access.isUnaligned()) { + // TODO / OPTIMIZE (bug 1331264): We perform the copy on the i32 + // path (and allocate the temp for the copy) because we will destroy + // the value in the temp. We could avoid the copy and the temp if + // the caller would instead preserve src when it needs to return its + // value as a result (for teeStore). If unaligned accesses are + // common it will be worthwhile to make that change, but there's no + // evidence yet that they will be common. + switch (src.tag) { + case AnyReg::I64: + masm.wasmUnalignedStoreI64(access, src.i64(), ptr, ptr, tmp); + break; + case AnyReg::F32: + masm.wasmUnalignedStoreFP(access, src.f32(), ptr, ptr, tmp); + break; + case AnyReg::F64: + masm.wasmUnalignedStoreFP(access, src.f64(), ptr, ptr, tmp); + break; + default: + moveI32(src.i32(), tmp); + masm.wasmUnalignedStore(access, tmp, ptr, ptr); + break; + } + } else { + if (access.type() == Scalar::Int64) + masm.wasmStoreI64(access, src.i64(), ptr, ptr); + else if (src.tag == AnyReg::I64) + masm.wasmStore(access, AnyRegister(src.i64().low), ptr, ptr); + else + masm.wasmStore(access, src.any(), ptr, ptr); } #elif defined(JS_CODEGEN_LOONGARCH64) switch (access.type()) { @@ -3885,248 +3882,6 @@ class BaseCompiler return true; } -#if defined(JS_CODEGEN_LOONGARCH64) - void coerceAtomicStoreResult(Scalar::Type viewType, RegI32 value) { - switch (viewType) { - case Scalar::Int8: - masm.as_ext_w_b(value.reg, value.reg); - break; - case Scalar::Uint8: - masm.as_bstrpick_d(value.reg, value.reg, 7, 0); - break; - case Scalar::Int16: - masm.as_ext_w_h(value.reg, value.reg); - break; - case Scalar::Uint16: - masm.as_bstrpick_d(value.reg, value.reg, 15, 0); - break; - case Scalar::Int32: - case Scalar::Uint32: - break; - default: - MOZ_CRASH("Unexpected atomic array type"); - } - } - - void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type viewType, Register value, - const BaseIndex& addr, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - switch (viewType) { - case Scalar::Int8: - masm.atomicFetchOp(1, true, op, value, addr, valueTemp, offsetTemp, - maskTemp, output); - break; - case Scalar::Uint8: - masm.atomicFetchOp(1, false, op, value, addr, valueTemp, offsetTemp, - maskTemp, output); - break; - case Scalar::Int16: - masm.atomicFetchOp(2, true, op, value, addr, valueTemp, offsetTemp, - maskTemp, output); - break; - case Scalar::Uint16: - masm.atomicFetchOp(2, false, op, value, addr, valueTemp, offsetTemp, - maskTemp, output); - break; - case Scalar::Int32: - case Scalar::Uint32: - masm.atomicFetchOp(4, false, op, value, addr, valueTemp, offsetTemp, - maskTemp, output); - break; - default: - MOZ_CRASH("Unexpected atomic array type"); - } - } - - void atomicCompareExchangeToTypedIntArray(Scalar::Type viewType, const BaseIndex& addr, - Register oldval, Register newval, - Register valueTemp, Register offsetTemp, - Register maskTemp, Register output) { - switch (viewType) { - case Scalar::Int8: - masm.compareExchange(1, true, addr, oldval, newval, valueTemp, - offsetTemp, maskTemp, output); - break; - case Scalar::Uint8: - masm.compareExchange(1, false, addr, oldval, newval, valueTemp, - offsetTemp, maskTemp, output); - break; - case Scalar::Int16: - masm.compareExchange(2, true, addr, oldval, newval, valueTemp, - offsetTemp, maskTemp, output); - break; - case Scalar::Uint16: - masm.compareExchange(2, false, addr, oldval, newval, valueTemp, - offsetTemp, maskTemp, output); - break; - case Scalar::Int32: - case Scalar::Uint32: - masm.compareExchange(4, false, addr, oldval, newval, valueTemp, - offsetTemp, maskTemp, output); - break; - default: - MOZ_CRASH("Unexpected atomic array type"); - } - } - - void atomicExchangeToTypedIntArray(Scalar::Type viewType, const BaseIndex& addr, - Register value, Register valueTemp, - Register offsetTemp, Register maskTemp, - Register output) { - switch (viewType) { - case Scalar::Int8: - masm.atomicExchange(1, true, addr, value, valueTemp, offsetTemp, - maskTemp, output); - break; - case Scalar::Uint8: - masm.atomicExchange(1, false, addr, value, valueTemp, offsetTemp, - maskTemp, output); - break; - case Scalar::Int16: - masm.atomicExchange(2, true, addr, value, valueTemp, offsetTemp, - maskTemp, output); - break; - case Scalar::Uint16: - masm.atomicExchange(2, false, addr, value, valueTemp, offsetTemp, - maskTemp, output); - break; - case Scalar::Int32: - case Scalar::Uint32: - masm.atomicExchange(4, false, addr, value, valueTemp, offsetTemp, - maskTemp, output); - break; - default: - MOZ_CRASH("Unexpected atomic array type"); - } - } -#endif - -#ifdef JS_CODEGEN_ARM - void - loadI32(MemoryAccessDesc access, bool isSigned, RegI32 ptr, Register rt) { - if (access.byteSize() > 1 && IsUnaligned(access)) { - masm.add32(HeapReg, ptr.reg); - SecondScratchRegisterScope scratch(*this); - masm.emitUnalignedLoad(isSigned, access.byteSize(), ptr.reg, scratch, rt, 0); - } else { - BufferOffset ld = - masm.ma_dataTransferN(js::jit::IsLoad, BitSize(access.byteSize()*8), - isSigned, HeapReg, ptr.reg, rt, Offset, Assembler::Always); - masm.append(access, ld.getOffset(), masm.framePushed()); - } - } - - void - storeI32(MemoryAccessDesc access, RegI32 ptr, Register rt) { - if (access.byteSize() > 1 && IsUnaligned(access)) { - masm.add32(HeapReg, ptr.reg); - masm.emitUnalignedStore(access.byteSize(), ptr.reg, rt, 0); - } else { - BufferOffset st = - masm.ma_dataTransferN(js::jit::IsStore, BitSize(access.byteSize()*8), - IsSigned(false), ptr.reg, HeapReg, rt, Offset, - Assembler::Always); - masm.append(access, st.getOffset(), masm.framePushed()); - } - } - - void - loadI64(MemoryAccessDesc access, RegI32 ptr, RegI64 dest) { - if (IsUnaligned(access)) { - masm.add32(HeapReg, ptr.reg); - SecondScratchRegisterScope scratch(*this); - masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, dest.reg.low, - 0); - masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, dest.reg.high, - 4); - } else { - BufferOffset ld; - ld = masm.ma_dataTransferN(js::jit::IsLoad, BitSize(32), IsSigned(false), HeapReg, - ptr.reg, dest.reg.low, Offset, Assembler::Always); - masm.append(access, ld.getOffset(), masm.framePushed()); - masm.add32(Imm32(4), ptr.reg); - ld = masm.ma_dataTransferN(js::jit::IsLoad, BitSize(32), IsSigned(false), HeapReg, - ptr.reg, dest.reg.high, Offset, Assembler::Always); - masm.append(access, ld.getOffset(), masm.framePushed()); - } - } - - void - storeI64(MemoryAccessDesc access, RegI32 ptr, RegI64 src) { - if (IsUnaligned(access)) { - masm.add32(HeapReg, ptr.reg); - masm.emitUnalignedStore(ByteSize(4), ptr.reg, src.reg.low, 0); - masm.emitUnalignedStore(ByteSize(4), ptr.reg, src.reg.high, 4); - } else { - BufferOffset st; - st = masm.ma_dataTransferN(js::jit::IsStore, BitSize(32), IsSigned(false), HeapReg, - ptr.reg, src.reg.low, Offset, Assembler::Always); - masm.append(access, st.getOffset(), masm.framePushed()); - masm.add32(Imm32(4), ptr.reg); - st = masm.ma_dataTransferN(js::jit::IsStore, BitSize(32), IsSigned(false), HeapReg, - ptr.reg, src.reg.high, Offset, Assembler::Always); - masm.append(access, st.getOffset(), masm.framePushed()); - } - } - - void - loadF32(MemoryAccessDesc access, RegI32 ptr, RegF32 dest, RegI32 tmp1) { - masm.add32(HeapReg, ptr.reg); - if (IsUnaligned(access)) { - SecondScratchRegisterScope scratch(*this); - masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp1.reg, 0); - masm.ma_vxfer(tmp1.reg, dest.reg); - } else { - BufferOffset ld = masm.ma_vldr(VFPAddr(ptr.reg, VFPOffImm(0)), dest.reg, - Assembler::Always); - masm.append(access, ld.getOffset(), masm.framePushed()); - } - } - - void - storeF32(MemoryAccessDesc access, RegI32 ptr, RegF32 src, RegI32 tmp1) { - masm.add32(HeapReg, ptr.reg); - if (IsUnaligned(access)) { - masm.ma_vxfer(src.reg, tmp1.reg); - masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp1.reg, 0); - } else { - BufferOffset st = - masm.ma_vstr(src.reg, VFPAddr(ptr.reg, VFPOffImm(0)), Assembler::Always); - masm.append(access, st.getOffset(), masm.framePushed()); - } - } - - void - loadF64(MemoryAccessDesc access, RegI32 ptr, RegF64 dest, RegI32 tmp1, RegI32 tmp2) { - masm.add32(HeapReg, ptr.reg); - if (IsUnaligned(access)) { - SecondScratchRegisterScope scratch(*this); - masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp1.reg, 0); - masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp2.reg, 4); - masm.ma_vxfer(tmp1.reg, tmp2.reg, dest.reg); - } else { - BufferOffset ld = masm.ma_vldr(VFPAddr(ptr.reg, VFPOffImm(0)), dest.reg, - Assembler::Always); - masm.append(access, ld.getOffset(), masm.framePushed()); - } - } - - void - storeF64(MemoryAccessDesc access, RegI32 ptr, RegF64 src, RegI32 tmp1, RegI32 tmp2) { - masm.add32(HeapReg, ptr.reg); - if (IsUnaligned(access)) { - masm.ma_vxfer(src.reg, tmp1.reg, tmp2.reg); - masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp1.reg, 0); - masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp2.reg, 4); - } else { - BufferOffset st = - masm.ma_vstr(src.reg, VFPAddr(ptr.reg, VFPOffImm(0)), Assembler::Always); - masm.append(access, st.getOffset(), masm.framePushed()); - } - } -#endif // JS_CODEGEN_ARM - //////////////////////////////////////////////////////////// // Generally speaking, ABOVE this point there should be no value @@ -6868,9 +6623,10 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS()); - size_t temps = loadStoreTemps(access); + size_t temps = loadTemps(access); RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32(); RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32(); + RegI32 tmp3 = temps >= 3 ? needI32() : invalidI32(); switch (type) { case ValType::I32: { @@ -6880,7 +6636,7 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) #else RegI32 rv = rp; #endif - if (!load(access, rp, AnyReg(rv), tmp1, tmp2)) + if (!load(access, rp, AnyReg(rv), tmp1, tmp2, tmp3)) return false; pushI32(rv); if (rp != rv) @@ -6898,7 +6654,7 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) rp = popI32(); rv = needI64(); #endif - if (!load(access, rp, AnyReg(rv), tmp1, tmp2)) + if (!load(access, rp, AnyReg(rv), tmp1, tmp2, tmp3)) return false; pushI64(rv); freeI32(rp); @@ -6907,7 +6663,7 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) case ValType::F32: { RegI32 rp = popI32(); RegF32 rv = needF32(); - if (!load(access, rp, AnyReg(rv), tmp1, tmp2)) + if (!load(access, rp, AnyReg(rv), tmp1, tmp2, tmp3)) return false; pushF32(rv); freeI32(rp); @@ -6916,7 +6672,7 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) case ValType::F64: { RegI32 rp = popI32(); RegF64 rv = needF64(); - if (!load(access, rp, AnyReg(rv), tmp1, tmp2)) + if (!load(access, rp, AnyReg(rv), tmp1, tmp2, tmp3)) return false; pushF64(rv); freeI32(rp); @@ -6931,6 +6687,8 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) freeI32(tmp1); if (temps >= 2) freeI32(tmp2); + if (temps >= 3) + freeI32(tmp3); return true; } @@ -6951,15 +6709,14 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType) MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS()); - size_t temps = loadStoreTemps(access); + size_t temps = storeTemps(access); RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32(); - RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32(); switch (resultType) { case ValType::I32: { RegI32 rp, rv; pop2xI32(&rp, &rv); - if (!store(access, rp, AnyReg(rv), tmp1, tmp2)) + if (!store(access, rp, AnyReg(rv), tmp1)) return false; freeI32(rp); freeI32(rv); @@ -6968,7 +6725,7 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType) case ValType::I64: { RegI64 rv = popI64(); RegI32 rp = popI32(); - if (!store(access, rp, AnyReg(rv), tmp1, tmp2)) + if (!store(access, rp, AnyReg(rv), tmp1)) return false; freeI32(rp); freeI64(rv); @@ -6977,7 +6734,7 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType) case ValType::F32: { RegF32 rv = popF32(); RegI32 rp = popI32(); - if (!store(access, rp, AnyReg(rv), tmp1, tmp2)) + if (!store(access, rp, AnyReg(rv), tmp1)) return false; freeI32(rp); freeF32(rv); @@ -6986,7 +6743,7 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType) case ValType::F64: { RegF64 rv = popF64(); RegI32 rp = popI32(); - if (!store(access, rp, AnyReg(rv), tmp1, tmp2)) + if (!store(access, rp, AnyReg(rv), tmp1)) return false; freeI32(rp); freeF64(rv); @@ -6999,8 +6756,6 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType) if (temps >= 1) freeI32(tmp1); - if (temps >= 2) - freeI32(tmp2); return true; } @@ -7314,16 +7069,15 @@ BaseCompiler::emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS()); - size_t temps = loadStoreTemps(access); + size_t temps = storeTemps(access); RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32(); - RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32(); if (resultType == ValType::F32 && viewType == Scalar::Float64) { RegF32 rv = popF32(); RegF64 rw = needF64(); masm.convertFloat32ToDouble(rv.reg, rw.reg); RegI32 rp = popI32(); - if (!store(access, rp, AnyReg(rw), tmp1, tmp2)) + if (!store(access, rp, AnyReg(rw), tmp1)) return false; pushF32(rv); freeI32(rp); @@ -7334,7 +7088,7 @@ BaseCompiler::emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType RegF32 rw = needF32(); masm.convertDoubleToFloat32(rv.reg, rw.reg); RegI32 rp = popI32(); - if (!store(access, rp, AnyReg(rw), tmp1, tmp2)) + if (!store(access, rp, AnyReg(rw), tmp1)) return false; pushF64(rv); freeI32(rp); @@ -7345,8 +7099,6 @@ BaseCompiler::emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType if (temps >= 1) freeI32(tmp1); - if (temps >= 2) - freeI32(tmp2); return true; }