mirror of
https://repo.dactyloidae.xyz/Dactyloidae/UXP.git
synced 2026-09-05 23:38:38 +09:00
Bug 1312751 - Merge ARM load and store code for Wasm
1312751 - Refactor wasm{Load,Store}{,I64} to MacroAssembler-arm.
1312751 - Wasm baseline, use refactored ARM load/store methods.
This commit is contained in:
win7-7
wuggy
parent
5d9bc9e53d
commit
0b36c23dc5
5 changed files with 503 additions and 540 deletions
|
|
@ -1376,6 +1376,43 @@ class MacroAssembler : public MacroAssemblerSpecific
|
|||
void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Operand dstAddr) DEFINED_ON(x86, x64);
|
||||
void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Operand dstAddr) DEFINED_ON(x86);
|
||||
|
||||
// For all the ARM wasmLoad and wasmStore functions, `ptr` MUST equal
|
||||
// `ptrScratch`, and that register will be updated based on conditions
|
||||
// listed below (where it is only mentioned as `ptr`).
|
||||
|
||||
// `ptr` will be updated if access.offset() != 0 or access.type() == Scalar::Int64.
|
||||
void wasmLoad(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, AnyRegister output) DEFINED_ON(arm);
|
||||
void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch, Register64 output) DEFINED_ON(arm);
|
||||
void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Register ptr, Register ptrScratch) DEFINED_ON(arm);
|
||||
void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Register ptr, Register ptrScratch) DEFINED_ON(arm);
|
||||
|
||||
// `ptr` will always be updated.
|
||||
void wasmUnalignedLoad(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch,
|
||||
Register output, Register tmp) DEFINED_ON(arm);
|
||||
|
||||
// `ptr` will always be updated and `tmp1` is always needed. `tmp2` is
|
||||
// needed for Float32; `tmp2` and `tmp3` are needed for Float64. Temps must
|
||||
// be Invalid when they are not needed.
|
||||
void wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch,
|
||||
FloatRegister output, Register tmp1, Register tmp2, Register tmp3) DEFINED_ON(arm);
|
||||
|
||||
// `ptr` will always be updated.
|
||||
void wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch,
|
||||
Register64 output, Register tmp) DEFINED_ON(arm);
|
||||
|
||||
// `ptr` and `value` will always be updated.
|
||||
void wasmUnalignedStore(const wasm::MemoryAccessDesc& access, Register value, Register ptr, Register ptrScratch)
|
||||
DEFINED_ON(arm);
|
||||
|
||||
// `ptr` will always be updated.
|
||||
void wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access, FloatRegister floatValue, Register ptr,
|
||||
Register ptrScratch, Register tmp) DEFINED_ON(arm);
|
||||
|
||||
// `ptr` will always be updated.
|
||||
void wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Register ptr, Register ptrScratch,
|
||||
Register tmp) DEFINED_ON(arm);
|
||||
|
||||
|
||||
// wasm specific methods, used in both the wasm baseline compiler and ion.
|
||||
void wasmTruncateDoubleToUInt32(FloatRegister input, Register output, Label* oolEntry) DEFINED_ON(x86, x64, arm);
|
||||
void wasmTruncateDoubleToInt32(FloatRegister input, Register output, Label* oolEntry) DEFINED_ON(x86_shared, arm);
|
||||
|
|
|
|||
|
|
@ -2366,68 +2366,20 @@ CodeGeneratorARM::emitWasmLoad(T* lir)
|
|||
{
|
||||
const MWasmLoad* mir = lir->mir();
|
||||
|
||||
uint32_t offset = mir->access().offset();
|
||||
MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
|
||||
MIRType resultType = mir->type();
|
||||
Register ptr;
|
||||
|
||||
Register ptr = ToRegister(lir->ptr());
|
||||
Scalar::Type type = mir->access().type();
|
||||
|
||||
// Maybe add the offset.
|
||||
if (offset || type == Scalar::Int64) {
|
||||
ScratchRegisterScope scratch(masm);
|
||||
Register ptrPlusOffset = ToRegister(lir->ptrCopy());
|
||||
if (offset)
|
||||
masm.ma_add(Imm32(offset), ptrPlusOffset, scratch);
|
||||
ptr = ptrPlusOffset;
|
||||
if (mir->access().offset() || mir->access().type() == Scalar::Int64) {
|
||||
ptr = ToRegister(lir->ptrCopy());
|
||||
} else {
|
||||
MOZ_ASSERT(lir->ptrCopy()->isBogusTemp());
|
||||
ptr = ToRegister(lir->ptr());
|
||||
}
|
||||
|
||||
bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 || type == Scalar::Int32 ||
|
||||
type == Scalar::Int64;
|
||||
unsigned byteSize = mir->access().byteSize();
|
||||
|
||||
masm.memoryBarrier(mir->access().barrierBefore());
|
||||
|
||||
BufferOffset load;
|
||||
if (mir->type() == MIRType::Int64) {
|
||||
Register64 output = ToOutRegister64(lir);
|
||||
if (type == Scalar::Int64) {
|
||||
MOZ_ASSERT(INT64LOW_OFFSET == 0);
|
||||
|
||||
load = masm.ma_dataTransferN(IsLoad, 32, /* signed = */ false, HeapReg, ptr, output.low);
|
||||
masm.append(mir->access(), load.getOffset(), masm.framePushed());
|
||||
|
||||
masm.as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
|
||||
|
||||
load = masm.ma_dataTransferN(IsLoad, 32, isSigned, HeapReg, ptr, output.high);
|
||||
masm.append(mir->access(), load.getOffset(), masm.framePushed());
|
||||
} else {
|
||||
load = masm.ma_dataTransferN(IsLoad, byteSize * 8, isSigned, HeapReg, ptr, output.low);
|
||||
masm.append(mir->access(), load.getOffset(), masm.framePushed());
|
||||
|
||||
if (isSigned)
|
||||
masm.ma_asr(Imm32(31), output.low, output.high);
|
||||
else
|
||||
masm.ma_mov(Imm32(0), output.high);
|
||||
}
|
||||
} else {
|
||||
AnyRegister output = ToAnyRegister(lir->output());
|
||||
bool isFloat = output.isFloat();
|
||||
if (isFloat) {
|
||||
MOZ_ASSERT((byteSize == 4) == output.fpu().isSingle());
|
||||
ScratchRegisterScope scratch(masm);
|
||||
masm.ma_add(HeapReg, ptr, scratch);
|
||||
|
||||
load = masm.ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), output.fpu());
|
||||
masm.append(mir->access(), load.getOffset(), masm.framePushed());
|
||||
} else {
|
||||
load = masm.ma_dataTransferN(IsLoad, byteSize * 8, isSigned, HeapReg, ptr, output.gpr());
|
||||
masm.append(mir->access(), load.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
masm.memoryBarrier(mir->access().barrierAfter());
|
||||
if (resultType == MIRType::Int64)
|
||||
masm.wasmLoadI64(mir->access(), ptr, ptr, ToOutRegister64(lir));
|
||||
else
|
||||
masm.wasmLoad(mir->access(), ptr, ptr, ToAnyRegister(lir->output()));
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -2448,70 +2400,22 @@ CodeGeneratorARM::emitWasmUnalignedLoad(T* lir)
|
|||
{
|
||||
const MWasmLoad* mir = lir->mir();
|
||||
|
||||
uint32_t offset = mir->access().offset();
|
||||
MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
|
||||
MIRType resultType = mir->type();
|
||||
|
||||
Register ptr = ToRegister(lir->ptrCopy());
|
||||
if (offset) {
|
||||
ScratchRegisterScope scratch(masm);
|
||||
masm.ma_add(Imm32(offset), ptr, scratch);
|
||||
Register tmp1 = ToRegister(lir->getTemp(1));
|
||||
|
||||
if (resultType == MIRType::Int64) {
|
||||
masm.wasmUnalignedLoadI64(mir->access(), ptr, ptr, ToOutRegister64(lir), tmp1);
|
||||
} else if (IsFloatingPointType(resultType)) {
|
||||
Register tmp2(ToRegister(lir->getTemp(2)));
|
||||
Register tmp3(Register::Invalid());
|
||||
if (mir->access().byteSize() == 8)
|
||||
tmp3 = ToRegister(lir->getTemp(3));
|
||||
masm.wasmUnalignedLoadFP(mir->access(), ptr, ptr, ToFloatRegister(lir->output()), tmp1, tmp2, tmp3);
|
||||
} else {
|
||||
masm.wasmUnalignedLoad(mir->access(), ptr, ptr, ToRegister(lir->output()), tmp1);
|
||||
}
|
||||
|
||||
// Add HeapReg to ptr, so we can use base+index addressing in the byte loads.
|
||||
masm.ma_add(HeapReg, ptr);
|
||||
|
||||
unsigned byteSize = mir->access().byteSize();
|
||||
Scalar::Type type = mir->access().type();
|
||||
bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 || type == Scalar::Int32 ||
|
||||
type == Scalar::Int64;
|
||||
|
||||
MIRType mirType = mir->type();
|
||||
|
||||
Register tmp = ToRegister(lir->getTemp(1));
|
||||
|
||||
Register low;
|
||||
if (IsFloatingPointType(mirType))
|
||||
low = ToRegister(lir->getTemp(2));
|
||||
else if (mirType == MIRType::Int64)
|
||||
low = ToOutRegister64(lir).low;
|
||||
else
|
||||
low = ToRegister(lir->output());
|
||||
|
||||
MOZ_ASSERT(low != tmp);
|
||||
MOZ_ASSERT(low != ptr);
|
||||
|
||||
masm.memoryBarrier(mir->access().barrierBefore());
|
||||
|
||||
masm.emitUnalignedLoad(isSigned, Min(byteSize, 4u), ptr, tmp, low);
|
||||
|
||||
if (IsFloatingPointType(mirType)) {
|
||||
FloatRegister output = ToFloatRegister(lir->output());
|
||||
if (byteSize == 4) {
|
||||
MOZ_ASSERT(output.isSingle());
|
||||
masm.ma_vxfer(low, output);
|
||||
} else {
|
||||
MOZ_ASSERT(byteSize == 8);
|
||||
MOZ_ASSERT(output.isDouble());
|
||||
Register high = ToRegister(lir->getTemp(3));
|
||||
masm.emitUnalignedLoad(/* signed */ false, 4, ptr, tmp, high, /* offset */ 4);
|
||||
masm.ma_vxfer(low, high, output);
|
||||
}
|
||||
} else if (mirType == MIRType::Int64) {
|
||||
Register64 output = ToOutRegister64(lir);
|
||||
if (type == Scalar::Int64) {
|
||||
MOZ_ASSERT(byteSize == 8);
|
||||
masm.emitUnalignedLoad(isSigned, 4, ptr, tmp, output.high, /* offset */ 4);
|
||||
} else {
|
||||
MOZ_ASSERT(byteSize <= 4);
|
||||
// Propagate sign.
|
||||
if (isSigned)
|
||||
masm.ma_asr(Imm32(31), output.low, output.high);
|
||||
else
|
||||
masm.ma_mov(Imm32(0), output.high);
|
||||
}
|
||||
}
|
||||
|
||||
masm.memoryBarrier(mir->access().barrierAfter());
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -2545,59 +2449,22 @@ CodeGeneratorARM::emitWasmStore(T* lir)
|
|||
{
|
||||
const MWasmStore* mir = lir->mir();
|
||||
|
||||
uint32_t offset = mir->access().offset();
|
||||
MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
|
||||
|
||||
Register ptr = ToRegister(lir->ptr());
|
||||
unsigned byteSize = mir->access().byteSize();
|
||||
Scalar::Type type = mir->access().type();
|
||||
Scalar::Type accessType = mir->access().type();
|
||||
Register ptr;
|
||||
|
||||
// Maybe add the offset.
|
||||
if (offset || type == Scalar::Int64) {
|
||||
ScratchRegisterScope scratch(masm);
|
||||
Register ptrPlusOffset = ToRegister(lir->ptrCopy());
|
||||
if (offset)
|
||||
masm.ma_add(Imm32(offset), ptrPlusOffset, scratch);
|
||||
ptr = ptrPlusOffset;
|
||||
if (mir->access().offset() || accessType == Scalar::Int64) {
|
||||
ptr = ToRegister(lir->ptrCopy());
|
||||
} else {
|
||||
MOZ_ASSERT(lir->ptrCopy()->isBogusTemp());
|
||||
ptr = ToRegister(lir->ptr());
|
||||
}
|
||||
|
||||
masm.memoryBarrier(mir->access().barrierBefore());
|
||||
|
||||
BufferOffset store;
|
||||
if (type == Scalar::Int64) {
|
||||
MOZ_ASSERT(INT64LOW_OFFSET == 0);
|
||||
|
||||
Register64 value = ToRegister64(lir->getInt64Operand(lir->ValueIndex));
|
||||
|
||||
store = masm.ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ false, HeapReg, ptr, value.low);
|
||||
masm.append(mir->access(), store.getOffset(), masm.framePushed());
|
||||
|
||||
masm.as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
|
||||
|
||||
store = masm.ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ true, HeapReg, ptr, value.high);
|
||||
masm.append(mir->access(), store.getOffset(), masm.framePushed());
|
||||
} else {
|
||||
AnyRegister value = ToAnyRegister(lir->getOperand(lir->ValueIndex));
|
||||
if (value.isFloat()) {
|
||||
ScratchRegisterScope scratch(masm);
|
||||
FloatRegister val = value.fpu();
|
||||
MOZ_ASSERT((byteSize == 4) == val.isSingle());
|
||||
masm.ma_add(HeapReg, ptr, scratch);
|
||||
|
||||
store = masm.ma_vstr(val, Operand(Address(scratch, 0)).toVFPAddr());
|
||||
masm.append(mir->access(), store.getOffset(), masm.framePushed());
|
||||
} else {
|
||||
bool isSigned = type == Scalar::Uint32 || type == Scalar::Int32; // see AsmJSStoreHeap;
|
||||
Register val = value.gpr();
|
||||
|
||||
store = masm.ma_dataTransferN(IsStore, 8 * byteSize /* bits */, isSigned, HeapReg, ptr, val);
|
||||
masm.append(mir->access(), store.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
masm.memoryBarrier(mir->access().barrierAfter());
|
||||
if (accessType == Scalar::Int64)
|
||||
masm.wasmStoreI64(mir->access(), ToRegister64(lir->getInt64Operand(lir->ValueIndex)),
|
||||
ptr, ptr);
|
||||
else
|
||||
masm.wasmStore(mir->access(), ToAnyRegister(lir->getOperand(lir->ValueIndex)), ptr, ptr);
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -2618,50 +2485,20 @@ CodeGeneratorARM::emitWasmUnalignedStore(T* lir)
|
|||
{
|
||||
const MWasmStore* mir = lir->mir();
|
||||
|
||||
uint32_t offset = mir->access().offset();
|
||||
MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
|
||||
Scalar::Type accessType = mir->access().type();
|
||||
|
||||
Register ptr = ToRegister(lir->ptrCopy());
|
||||
if (offset) {
|
||||
ScratchRegisterScope scratch(masm);
|
||||
masm.ma_add(Imm32(offset), ptr, scratch);
|
||||
Register valOrTmp = ToRegister(lir->valueHelper());
|
||||
if (accessType == Scalar::Int64) {
|
||||
masm.wasmUnalignedStoreI64(mir->access(),
|
||||
ToRegister64(lir->getInt64Operand(LWasmUnalignedStoreI64::ValueIndex)),
|
||||
ptr, ptr, valOrTmp);
|
||||
} else if (accessType == Scalar::Float32 || accessType == Scalar::Float64) {
|
||||
FloatRegister value = ToFloatRegister(lir->getOperand(LWasmUnalignedStore::ValueIndex));
|
||||
masm.wasmUnalignedStoreFP(mir->access(), value, ptr, ptr, valOrTmp);
|
||||
} else {
|
||||
masm.wasmUnalignedStore(mir->access(), valOrTmp, ptr, ptr);
|
||||
}
|
||||
|
||||
// Add HeapReg to ptr, so we can use base+index addressing in the byte loads.
|
||||
masm.ma_add(HeapReg, ptr);
|
||||
|
||||
MIRType mirType = mir->value()->type();
|
||||
|
||||
masm.memoryBarrier(mir->access().barrierAfter());
|
||||
|
||||
Register val = ToRegister(lir->valueHelper());
|
||||
if (IsFloatingPointType(mirType)) {
|
||||
masm.ma_vxfer(ToFloatRegister(lir->getOperand(LWasmUnalignedStore::ValueIndex)), val);
|
||||
} else if (mirType == MIRType::Int64) {
|
||||
Register64 input = ToRegister64(lir->getInt64Operand(LWasmUnalignedStoreI64::ValueIndex));
|
||||
if (input.low != val)
|
||||
masm.ma_mov(input.low, val);
|
||||
}
|
||||
|
||||
unsigned byteSize = mir->access().byteSize();
|
||||
masm.emitUnalignedStore(Min(byteSize, 4u), ptr, val);
|
||||
|
||||
if (byteSize > 4) {
|
||||
// It's a double or an int64 load.
|
||||
// Load the high 32 bits when counter == 4.
|
||||
if (IsFloatingPointType(mirType)) {
|
||||
FloatRegister fp = ToFloatRegister(lir->getOperand(LWasmUnalignedStore::ValueIndex));
|
||||
MOZ_ASSERT(fp.isDouble());
|
||||
ScratchRegisterScope scratch(masm);
|
||||
masm.ma_vxfer(fp, scratch, val);
|
||||
} else {
|
||||
MOZ_ASSERT(mirType == MIRType::Int64);
|
||||
masm.ma_mov(ToRegister64(lir->getInt64Operand(LWasmUnalignedStoreI64::ValueIndex)).high, val);
|
||||
}
|
||||
masm.emitUnalignedStore(4, ptr, val, /* offset */ 4);
|
||||
}
|
||||
|
||||
masm.memoryBarrier(mir->access().barrierBefore());
|
||||
}
|
||||
|
||||
void
|
||||
|
|
|
|||
|
|
@ -5438,6 +5438,80 @@ MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, Register output,
|
|||
wasmTruncateToInt32(input, output, MIRType::Float32, /* isUnsigned= */ false, oolEntry);
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch,
|
||||
AnyRegister output)
|
||||
{
|
||||
wasmLoadImpl(access, ptr, ptrScratch, output, Register64::Invalid());
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch,
|
||||
Register64 output)
|
||||
{
|
||||
wasmLoadImpl(access, ptr, ptrScratch, AnyRegister(), output);
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Register ptr,
|
||||
Register ptrScratch)
|
||||
{
|
||||
wasmStoreImpl(access, value, Register64::Invalid(), ptr, ptrScratch);
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Register ptr,
|
||||
Register ptrScratch)
|
||||
{
|
||||
wasmStoreImpl(access, AnyRegister(), value, ptr, ptrScratch);
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmUnalignedLoad(const wasm::MemoryAccessDesc& access, Register ptr,
|
||||
Register ptrScratch, Register output, Register tmp)
|
||||
{
|
||||
wasmUnalignedLoadImpl(access, ptr, ptrScratch, AnyRegister(output), Register64::Invalid(), tmp,
|
||||
Register::Invalid(), Register::Invalid());
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access, Register ptr,
|
||||
Register ptrScratch, FloatRegister outFP, Register tmp1,
|
||||
Register tmp2, Register tmp3)
|
||||
{
|
||||
wasmUnalignedLoadImpl(access, ptr, ptrScratch, AnyRegister(outFP), Register64::Invalid(),
|
||||
tmp1, tmp2, tmp3);
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access, Register ptr,
|
||||
Register ptrScratch, Register64 out64, Register tmp)
|
||||
{
|
||||
wasmUnalignedLoadImpl(access, ptr, ptrScratch, AnyRegister(), out64, tmp, Register::Invalid(),
|
||||
Register::Invalid());
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmUnalignedStore(const wasm::MemoryAccessDesc& access, Register value,
|
||||
Register ptr, Register ptrScratch)
|
||||
{
|
||||
wasmUnalignedStoreImpl(access, FloatRegister(), Register64::Invalid(), ptr, ptrScratch, value);
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access, FloatRegister floatVal,
|
||||
Register ptr, Register ptrScratch, Register tmp)
|
||||
{
|
||||
wasmUnalignedStoreImpl(access, floatVal, Register64::Invalid(), ptr, ptrScratch, tmp);
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssembler::wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access, Register64 val64,
|
||||
Register ptr, Register ptrScratch, Register tmp)
|
||||
{
|
||||
wasmUnalignedStoreImpl(access, FloatRegister(), val64, ptr, ptrScratch, tmp);
|
||||
}
|
||||
|
||||
//}}} check_macroassembler_style
|
||||
|
||||
void
|
||||
|
|
@ -5582,6 +5656,248 @@ MacroAssemblerARM::outOfLineWasmTruncateToIntCheck(FloatRegister input, MIRType
|
|||
asMasm().framePushed()));
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssemblerARM::wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register ptr,
|
||||
Register ptrScratch, AnyRegister output, Register64 out64)
|
||||
{
|
||||
MOZ_ASSERT(ptr == ptrScratch);
|
||||
|
||||
uint32_t offset = access.offset();
|
||||
MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
|
||||
|
||||
Scalar::Type type = access.type();
|
||||
|
||||
// Maybe add the offset.
|
||||
if (offset || type == Scalar::Int64) {
|
||||
ScratchRegisterScope scratch(asMasm());
|
||||
if (offset)
|
||||
ma_add(Imm32(offset), ptr, scratch);
|
||||
}
|
||||
|
||||
bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 || type == Scalar::Int32 ||
|
||||
type == Scalar::Int64;
|
||||
unsigned byteSize = access.byteSize();
|
||||
|
||||
asMasm().memoryBarrier(access.barrierBefore());
|
||||
|
||||
uint32_t framePushed = asMasm().framePushed();
|
||||
BufferOffset load;
|
||||
if (out64 != Register64::Invalid()) {
|
||||
if (type == Scalar::Int64) {
|
||||
MOZ_ASSERT(INT64LOW_OFFSET == 0);
|
||||
|
||||
load = ma_dataTransferN(IsLoad, 32, /* signed = */ false, HeapReg, ptr, out64.low);
|
||||
append(access, load.getOffset(), framePushed);
|
||||
|
||||
as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
|
||||
|
||||
load = ma_dataTransferN(IsLoad, 32, isSigned, HeapReg, ptr, out64.high);
|
||||
append(access, load.getOffset(), framePushed);
|
||||
} else {
|
||||
load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, HeapReg, ptr, out64.low);
|
||||
append(access, load.getOffset(), framePushed);
|
||||
|
||||
if (isSigned)
|
||||
ma_asr(Imm32(31), out64.low, out64.high);
|
||||
else
|
||||
ma_mov(Imm32(0), out64.high);
|
||||
}
|
||||
} else {
|
||||
bool isFloat = output.isFloat();
|
||||
if (isFloat) {
|
||||
MOZ_ASSERT((byteSize == 4) == output.fpu().isSingle());
|
||||
ScratchRegisterScope scratch(asMasm());
|
||||
ma_add(HeapReg, ptr, scratch);
|
||||
|
||||
load = ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), output.fpu());
|
||||
append(access, load.getOffset(), framePushed);
|
||||
} else {
|
||||
load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, HeapReg, ptr, output.gpr());
|
||||
append(access, load.getOffset(), framePushed);
|
||||
}
|
||||
}
|
||||
|
||||
asMasm().memoryBarrier(access.barrierAfter());
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssemblerARM::wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister value,
|
||||
Register64 val64, Register ptr, Register ptrScratch)
|
||||
{
|
||||
MOZ_ASSERT(ptr == ptrScratch);
|
||||
|
||||
uint32_t offset = access.offset();
|
||||
MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
|
||||
|
||||
unsigned byteSize = access.byteSize();
|
||||
Scalar::Type type = access.type();
|
||||
|
||||
// Maybe add the offset.
|
||||
if (offset || type == Scalar::Int64) {
|
||||
ScratchRegisterScope scratch(asMasm());
|
||||
if (offset)
|
||||
ma_add(Imm32(offset), ptr, scratch);
|
||||
}
|
||||
|
||||
asMasm().memoryBarrier(access.barrierBefore());
|
||||
|
||||
uint32_t framePushed = asMasm().framePushed();
|
||||
|
||||
BufferOffset store;
|
||||
if (type == Scalar::Int64) {
|
||||
MOZ_ASSERT(INT64LOW_OFFSET == 0);
|
||||
|
||||
store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ false, HeapReg, ptr, val64.low);
|
||||
append(access, store.getOffset(), framePushed);
|
||||
|
||||
as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
|
||||
|
||||
store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ true, HeapReg, ptr, val64.high);
|
||||
append(access, store.getOffset(), framePushed);
|
||||
} else {
|
||||
if (value.isFloat()) {
|
||||
ScratchRegisterScope scratch(asMasm());
|
||||
FloatRegister val = value.fpu();
|
||||
MOZ_ASSERT((byteSize == 4) == val.isSingle());
|
||||
ma_add(HeapReg, ptr, scratch);
|
||||
|
||||
store = ma_vstr(val, Operand(Address(scratch, 0)).toVFPAddr());
|
||||
append(access, store.getOffset(), framePushed);
|
||||
} else {
|
||||
bool isSigned = type == Scalar::Uint32 || type == Scalar::Int32; // see AsmJSStoreHeap;
|
||||
Register val = value.gpr();
|
||||
|
||||
store = ma_dataTransferN(IsStore, 8 * byteSize /* bits */, isSigned, HeapReg, ptr, val);
|
||||
append(access, store.getOffset(), framePushed);
|
||||
}
|
||||
}
|
||||
|
||||
asMasm().memoryBarrier(access.barrierAfter());
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssemblerARM::wasmUnalignedLoadImpl(const wasm::MemoryAccessDesc& access, Register ptr,
|
||||
Register ptrScratch, AnyRegister outAny, Register64 out64,
|
||||
Register tmp, Register tmp2, Register tmp3)
|
||||
{
|
||||
MOZ_ASSERT(ptr == ptrScratch);
|
||||
MOZ_ASSERT_IF(access.type() != Scalar::Float32 && access.type() != Scalar::Float64,
|
||||
tmp2 == Register::Invalid() && tmp3 == Register::Invalid());
|
||||
MOZ_ASSERT_IF(access.type() == Scalar::Float32,
|
||||
tmp2 != Register::Invalid() && tmp3 == Register::Invalid());
|
||||
MOZ_ASSERT_IF(access.type() == Scalar::Float64,
|
||||
tmp2 != Register::Invalid() && tmp3 != Register::Invalid());
|
||||
|
||||
uint32_t offset = access.offset();
|
||||
MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
|
||||
|
||||
if (offset) {
|
||||
ScratchRegisterScope scratch(asMasm());
|
||||
ma_add(Imm32(offset), ptr, scratch);
|
||||
}
|
||||
|
||||
// Add HeapReg to ptr, so we can use baseindex addressing in the byte loads.
|
||||
ma_add(HeapReg, ptr);
|
||||
|
||||
unsigned byteSize = access.byteSize();
|
||||
Scalar::Type type = access.type();
|
||||
bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 || type == Scalar::Int32 ||
|
||||
type == Scalar::Int64;
|
||||
|
||||
Register low;
|
||||
if (out64 != Register64::Invalid())
|
||||
low = out64.low;
|
||||
else if (outAny.isFloat())
|
||||
low = tmp2;
|
||||
else
|
||||
low = outAny.gpr();
|
||||
|
||||
MOZ_ASSERT(low != tmp);
|
||||
MOZ_ASSERT(low != ptr);
|
||||
|
||||
asMasm().memoryBarrier(access.barrierBefore());
|
||||
|
||||
emitUnalignedLoad(isSigned, Min(byteSize, 4u), ptr, tmp, low);
|
||||
|
||||
if (out64 != Register64::Invalid()) {
|
||||
if (type == Scalar::Int64) {
|
||||
MOZ_ASSERT(byteSize == 8);
|
||||
emitUnalignedLoad(isSigned, 4, ptr, tmp, out64.high, /* offset */ 4);
|
||||
} else {
|
||||
MOZ_ASSERT(byteSize <= 4);
|
||||
// Propagate sign.
|
||||
if (isSigned)
|
||||
ma_asr(Imm32(31), out64.low, out64.high);
|
||||
else
|
||||
ma_mov(Imm32(0), out64.high);
|
||||
}
|
||||
} else if (outAny.isFloat()) {
|
||||
FloatRegister output = outAny.fpu();
|
||||
if (byteSize == 4) {
|
||||
MOZ_ASSERT(output.isSingle());
|
||||
ma_vxfer(low, output);
|
||||
} else {
|
||||
MOZ_ASSERT(byteSize == 8);
|
||||
MOZ_ASSERT(output.isDouble());
|
||||
Register high = tmp3;
|
||||
emitUnalignedLoad(/* signed */ false, 4, ptr, tmp, high, /* offset */ 4);
|
||||
ma_vxfer(low, high, output);
|
||||
}
|
||||
}
|
||||
|
||||
asMasm().memoryBarrier(access.barrierAfter());
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssemblerARM::wasmUnalignedStoreImpl(const wasm::MemoryAccessDesc& access, FloatRegister floatValue,
|
||||
Register64 val64, Register ptr, Register ptrScratch, Register tmp)
|
||||
{
|
||||
MOZ_ASSERT(ptr == ptrScratch);
|
||||
// They can't both be valid, but they can both be invalid.
|
||||
MOZ_ASSERT_IF(!floatValue.isInvalid(), val64 == Register64::Invalid());
|
||||
MOZ_ASSERT_IF(val64 != Register64::Invalid(), floatValue.isInvalid());
|
||||
|
||||
uint32_t offset = access.offset();
|
||||
MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
|
||||
|
||||
unsigned byteSize = access.byteSize();
|
||||
|
||||
if (offset) {
|
||||
ScratchRegisterScope scratch(asMasm());
|
||||
ma_add(Imm32(offset), ptr, scratch);
|
||||
}
|
||||
|
||||
// Add HeapReg to ptr, so we can use baseindex addressing in the byte loads.
|
||||
ma_add(HeapReg, ptr);
|
||||
|
||||
asMasm().memoryBarrier(access.barrierBefore());
|
||||
|
||||
if (val64 != Register64::Invalid()) {
|
||||
if (val64.low != tmp)
|
||||
ma_mov(val64.low, tmp);
|
||||
} else if (!floatValue.isInvalid()) {
|
||||
ma_vxfer(floatValue, tmp);
|
||||
}
|
||||
// Otherwise, tmp has the integer value to store.
|
||||
|
||||
emitUnalignedStore(Min(byteSize, 4u), ptr, tmp);
|
||||
|
||||
if (byteSize > 4) {
|
||||
if (val64 != Register64::Invalid()) {
|
||||
if (val64.high != tmp)
|
||||
ma_mov(val64.high, tmp);
|
||||
} else {
|
||||
MOZ_ASSERT(!floatValue.isInvalid());
|
||||
MOZ_ASSERT(floatValue.isDouble());
|
||||
ScratchRegisterScope scratch(asMasm());
|
||||
ma_vxfer(floatValue, scratch, tmp);
|
||||
}
|
||||
emitUnalignedStore(4, ptr, tmp, /* offset */ 4);
|
||||
}
|
||||
|
||||
asMasm().memoryBarrier(access.barrierAfter());
|
||||
}
|
||||
|
||||
void
|
||||
MacroAssemblerARM::emitUnalignedLoad(bool isSigned, unsigned byteSize, Register ptr, Register tmp,
|
||||
Register dest, unsigned offset)
|
||||
|
|
|
|||
|
|
@ -452,6 +452,28 @@ class MacroAssemblerARM : public Assembler
|
|||
MOZ_CRASH("Invalid data transfer addressing mode");
|
||||
}
|
||||
|
||||
// `outAny` is valid if and only if `out64` == Register64::Invalid().
|
||||
void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch,
|
||||
AnyRegister outAny, Register64 out64);
|
||||
|
||||
// `valAny` is valid if and only if `val64` == Register64::Invalid().
|
||||
void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister valAny, Register64 val64,
|
||||
Register ptr, Register ptrScratch);
|
||||
|
||||
protected:
|
||||
// `outAny` is valid if and only if `out64` == Register64::Invalid().
|
||||
void wasmUnalignedLoadImpl(const wasm::MemoryAccessDesc& access, Register ptr, Register ptrScratch,
|
||||
AnyRegister outAny, Register64 out64, Register tmp1, Register tmp2,
|
||||
Register tmp3);
|
||||
|
||||
// The value to be stored is in `floatValue` (if not invalid), `val64` (if not invalid),
|
||||
// or in `valOrTmp` (if `floatValue` and `val64` are both invalid). Note `valOrTmp` must
|
||||
// always be valid.
|
||||
void wasmUnalignedStoreImpl(const wasm::MemoryAccessDesc& access, FloatRegister floatValue,
|
||||
Register64 val64, Register ptr, Register ptrScratch, Register valOrTmp);
|
||||
|
||||
private:
|
||||
|
||||
// Loads `byteSize` bytes, byte by byte, by reading from ptr[offset],
|
||||
// applying the indicated signedness (defined by isSigned).
|
||||
// - all three registers must be different.
|
||||
|
|
@ -466,7 +488,6 @@ class MacroAssemblerARM : public Assembler
|
|||
// - byteSize can be up to 4 bytes and no more (GPR are 32 bits on ARM).
|
||||
void emitUnalignedStore(unsigned byteSize, Register ptr, Register val, unsigned offset = 0);
|
||||
|
||||
private:
|
||||
// Implementation for transferMultipleByRuns so we can use different
|
||||
// iterators for forward/backward traversals. The sign argument should be 1
|
||||
// if we traverse forwards, -1 if we traverse backwards.
|
||||
|
|
|
|||
|
|
@ -3588,16 +3588,16 @@ class BaseCompiler
|
|||
}
|
||||
|
||||
// This is the temp register passed as the last argument to load()
|
||||
MOZ_MUST_USE size_t loadStoreTemps(MemoryAccessDesc& access) {
|
||||
[[nodiscard]] size_t loadTemps(MemoryAccessDesc& access) {
|
||||
#if defined(JS_CODEGEN_ARM)
|
||||
if (IsUnaligned(access)) {
|
||||
switch (access.type()) {
|
||||
case Scalar::Float32:
|
||||
return 1;
|
||||
case Scalar::Float64:
|
||||
return 2;
|
||||
case Scalar::Float64:
|
||||
return 3;
|
||||
default:
|
||||
break;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
|
|
@ -3610,8 +3610,8 @@ class BaseCompiler
|
|||
|
||||
// ptr and dest may be the same iff dest is I32.
|
||||
// This may destroy ptr even if ptr and dest are not the same.
|
||||
MOZ_MUST_USE bool load(MemoryAccessDesc& access, RegI32 ptr, AnyReg dest, RegI32 tmp1,
|
||||
RegI32 tmp2)
|
||||
[[nodiscard]] bool load(MemoryAccessDesc& access, RegI32 ptr, AnyReg dest,
|
||||
RegI32 tmp1, RegI32 tmp2, RegI32 tmp3)
|
||||
{
|
||||
checkOffset(&access, ptr);
|
||||
|
||||
|
|
@ -3650,40 +3650,26 @@ class BaseCompiler
|
|||
masm.mov(ScratchRegX86, dest.i32().reg);
|
||||
}
|
||||
#elif defined(JS_CODEGEN_ARM)
|
||||
if (access.offset() != 0)
|
||||
masm.add32(Imm32(access.offset()), ptr.reg);
|
||||
|
||||
bool isSigned = true;
|
||||
switch (access.type()) {
|
||||
case Scalar::Uint8:
|
||||
case Scalar::Uint16:
|
||||
case Scalar::Uint32: {
|
||||
isSigned = false;
|
||||
MOZ_FALLTHROUGH;
|
||||
case Scalar::Int8:
|
||||
case Scalar::Int16:
|
||||
case Scalar::Int32:
|
||||
Register rt = dest.tag == AnyReg::I64 ? dest.i64().reg.low : dest.i32().reg;
|
||||
loadI32(access, isSigned, ptr, rt);
|
||||
if (dest.tag == AnyReg::I64) {
|
||||
if (isSigned)
|
||||
masm.ma_asr(Imm32(31), rt, dest.i64().reg.high);
|
||||
else
|
||||
masm.move32(Imm32(0), dest.i64().reg.high);
|
||||
if (access.isUnaligned()) {
|
||||
switch (dest.tag) {
|
||||
case AnyReg::I64:
|
||||
masm.wasmUnalignedLoadI64(access, ptr, ptr, dest.i64(), tmp1);
|
||||
break;
|
||||
case AnyReg::F32:
|
||||
masm.wasmUnalignedLoadFP(access, ptr, ptr, dest.f32(), tmp1, tmp2, Register::Invalid());
|
||||
break;
|
||||
case AnyReg::F64:
|
||||
masm.wasmUnalignedLoadFP(access, ptr, ptr, dest.f64(), tmp1, tmp2, tmp3);
|
||||
break;
|
||||
default:
|
||||
masm.wasmUnalignedLoad(access, ptr, ptr, dest.i32(), tmp1);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Scalar::Int64:
|
||||
loadI64(access, ptr, dest.i64());
|
||||
break;
|
||||
case Scalar::Float32:
|
||||
loadF32(access, ptr, dest.f32(), tmp1);
|
||||
break;
|
||||
case Scalar::Float64:
|
||||
loadF64(access, ptr, dest.f64(), tmp1, tmp2);
|
||||
break;
|
||||
default:
|
||||
MOZ_CRASH("Compiler bug: unexpected array type");
|
||||
} else {
|
||||
if (dest.tag == AnyReg::I64)
|
||||
masm.wasmLoadI64(access, ptr, ptr, dest.i64());
|
||||
else
|
||||
masm.wasmLoad(access, ptr, ptr, dest.any());
|
||||
}
|
||||
#elif defined(JS_CODEGEN_LOONGARCH64)
|
||||
switch (access.type()) {
|
||||
|
|
@ -3752,10 +3738,21 @@ class BaseCompiler
|
|||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] size_t storeTemps(MemoryAccessDesc& access) {
|
||||
#if defined(JS_CODEGEN_ARM)
|
||||
if (access.isUnaligned()) {
|
||||
// See comment in store() about how this temp could be avoided for
|
||||
// unaligned i8/i16/i32 stores with some restructuring elsewhere.
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ptr and src must not be the same register.
|
||||
// This may destroy ptr.
|
||||
MOZ_MUST_USE bool store(MemoryAccessDesc access, RegI32 ptr, AnyReg src, RegI32 tmp1,
|
||||
RegI32 tmp2)
|
||||
// This may destroy ptr but will not destroy src.
|
||||
[[nodiscard]] bool store(MemoryAccessDesc access, RegI32 ptr, AnyReg src,
|
||||
RegI32 tmp)
|
||||
{
|
||||
checkOffset(&access, ptr);
|
||||
|
||||
|
|
@ -3791,36 +3788,36 @@ class BaseCompiler
|
|||
masm.wasmStore(access, value, dstAddr);
|
||||
}
|
||||
#elif defined(JS_CODEGEN_ARM)
|
||||
if (access.offset() != 0)
|
||||
masm.add32(Imm32(access.offset()), ptr.reg);
|
||||
|
||||
switch (access.type()) {
|
||||
case Scalar::Uint8:
|
||||
MOZ_FALLTHROUGH;
|
||||
case Scalar::Uint16:
|
||||
MOZ_FALLTHROUGH;
|
||||
case Scalar::Int8:
|
||||
MOZ_FALLTHROUGH;
|
||||
case Scalar::Int16:
|
||||
MOZ_FALLTHROUGH;
|
||||
case Scalar::Int32:
|
||||
MOZ_FALLTHROUGH;
|
||||
case Scalar::Uint32: {
|
||||
Register rt = src.tag == AnyReg::I64 ? src.i64().reg.low : src.i32().reg;
|
||||
storeI32(access, ptr, rt);
|
||||
break;
|
||||
}
|
||||
case Scalar::Int64:
|
||||
storeI64(access, ptr, src.i64());
|
||||
break;
|
||||
case Scalar::Float32:
|
||||
storeF32(access, ptr, src.f32(), tmp1);
|
||||
break;
|
||||
case Scalar::Float64:
|
||||
storeF64(access, ptr, src.f64(), tmp1, tmp2);
|
||||
break;
|
||||
default:
|
||||
MOZ_CRASH("Compiler bug: unexpected array type");
|
||||
if (access.isUnaligned()) {
|
||||
// TODO / OPTIMIZE (bug 1331264): We perform the copy on the i32
|
||||
// path (and allocate the temp for the copy) because we will destroy
|
||||
// the value in the temp. We could avoid the copy and the temp if
|
||||
// the caller would instead preserve src when it needs to return its
|
||||
// value as a result (for teeStore). If unaligned accesses are
|
||||
// common it will be worthwhile to make that change, but there's no
|
||||
// evidence yet that they will be common.
|
||||
switch (src.tag) {
|
||||
case AnyReg::I64:
|
||||
masm.wasmUnalignedStoreI64(access, src.i64(), ptr, ptr, tmp);
|
||||
break;
|
||||
case AnyReg::F32:
|
||||
masm.wasmUnalignedStoreFP(access, src.f32(), ptr, ptr, tmp);
|
||||
break;
|
||||
case AnyReg::F64:
|
||||
masm.wasmUnalignedStoreFP(access, src.f64(), ptr, ptr, tmp);
|
||||
break;
|
||||
default:
|
||||
moveI32(src.i32(), tmp);
|
||||
masm.wasmUnalignedStore(access, tmp, ptr, ptr);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (access.type() == Scalar::Int64)
|
||||
masm.wasmStoreI64(access, src.i64(), ptr, ptr);
|
||||
else if (src.tag == AnyReg::I64)
|
||||
masm.wasmStore(access, AnyRegister(src.i64().low), ptr, ptr);
|
||||
else
|
||||
masm.wasmStore(access, src.any(), ptr, ptr);
|
||||
}
|
||||
#elif defined(JS_CODEGEN_LOONGARCH64)
|
||||
switch (access.type()) {
|
||||
|
|
@ -3885,248 +3882,6 @@ class BaseCompiler
|
|||
return true;
|
||||
}
|
||||
|
||||
#if defined(JS_CODEGEN_LOONGARCH64)
|
||||
void coerceAtomicStoreResult(Scalar::Type viewType, RegI32 value) {
|
||||
switch (viewType) {
|
||||
case Scalar::Int8:
|
||||
masm.as_ext_w_b(value.reg, value.reg);
|
||||
break;
|
||||
case Scalar::Uint8:
|
||||
masm.as_bstrpick_d(value.reg, value.reg, 7, 0);
|
||||
break;
|
||||
case Scalar::Int16:
|
||||
masm.as_ext_w_h(value.reg, value.reg);
|
||||
break;
|
||||
case Scalar::Uint16:
|
||||
masm.as_bstrpick_d(value.reg, value.reg, 15, 0);
|
||||
break;
|
||||
case Scalar::Int32:
|
||||
case Scalar::Uint32:
|
||||
break;
|
||||
default:
|
||||
MOZ_CRASH("Unexpected atomic array type");
|
||||
}
|
||||
}
|
||||
|
||||
void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type viewType, Register value,
|
||||
const BaseIndex& addr, Register valueTemp,
|
||||
Register offsetTemp, Register maskTemp,
|
||||
Register output) {
|
||||
switch (viewType) {
|
||||
case Scalar::Int8:
|
||||
masm.atomicFetchOp(1, true, op, value, addr, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
case Scalar::Uint8:
|
||||
masm.atomicFetchOp(1, false, op, value, addr, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
case Scalar::Int16:
|
||||
masm.atomicFetchOp(2, true, op, value, addr, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
case Scalar::Uint16:
|
||||
masm.atomicFetchOp(2, false, op, value, addr, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
case Scalar::Int32:
|
||||
case Scalar::Uint32:
|
||||
masm.atomicFetchOp(4, false, op, value, addr, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
default:
|
||||
MOZ_CRASH("Unexpected atomic array type");
|
||||
}
|
||||
}
|
||||
|
||||
void atomicCompareExchangeToTypedIntArray(Scalar::Type viewType, const BaseIndex& addr,
|
||||
Register oldval, Register newval,
|
||||
Register valueTemp, Register offsetTemp,
|
||||
Register maskTemp, Register output) {
|
||||
switch (viewType) {
|
||||
case Scalar::Int8:
|
||||
masm.compareExchange(1, true, addr, oldval, newval, valueTemp,
|
||||
offsetTemp, maskTemp, output);
|
||||
break;
|
||||
case Scalar::Uint8:
|
||||
masm.compareExchange(1, false, addr, oldval, newval, valueTemp,
|
||||
offsetTemp, maskTemp, output);
|
||||
break;
|
||||
case Scalar::Int16:
|
||||
masm.compareExchange(2, true, addr, oldval, newval, valueTemp,
|
||||
offsetTemp, maskTemp, output);
|
||||
break;
|
||||
case Scalar::Uint16:
|
||||
masm.compareExchange(2, false, addr, oldval, newval, valueTemp,
|
||||
offsetTemp, maskTemp, output);
|
||||
break;
|
||||
case Scalar::Int32:
|
||||
case Scalar::Uint32:
|
||||
masm.compareExchange(4, false, addr, oldval, newval, valueTemp,
|
||||
offsetTemp, maskTemp, output);
|
||||
break;
|
||||
default:
|
||||
MOZ_CRASH("Unexpected atomic array type");
|
||||
}
|
||||
}
|
||||
|
||||
void atomicExchangeToTypedIntArray(Scalar::Type viewType, const BaseIndex& addr,
|
||||
Register value, Register valueTemp,
|
||||
Register offsetTemp, Register maskTemp,
|
||||
Register output) {
|
||||
switch (viewType) {
|
||||
case Scalar::Int8:
|
||||
masm.atomicExchange(1, true, addr, value, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
case Scalar::Uint8:
|
||||
masm.atomicExchange(1, false, addr, value, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
case Scalar::Int16:
|
||||
masm.atomicExchange(2, true, addr, value, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
case Scalar::Uint16:
|
||||
masm.atomicExchange(2, false, addr, value, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
case Scalar::Int32:
|
||||
case Scalar::Uint32:
|
||||
masm.atomicExchange(4, false, addr, value, valueTemp, offsetTemp,
|
||||
maskTemp, output);
|
||||
break;
|
||||
default:
|
||||
MOZ_CRASH("Unexpected atomic array type");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef JS_CODEGEN_ARM
|
||||
void
|
||||
loadI32(MemoryAccessDesc access, bool isSigned, RegI32 ptr, Register rt) {
|
||||
if (access.byteSize() > 1 && IsUnaligned(access)) {
|
||||
masm.add32(HeapReg, ptr.reg);
|
||||
SecondScratchRegisterScope scratch(*this);
|
||||
masm.emitUnalignedLoad(isSigned, access.byteSize(), ptr.reg, scratch, rt, 0);
|
||||
} else {
|
||||
BufferOffset ld =
|
||||
masm.ma_dataTransferN(js::jit::IsLoad, BitSize(access.byteSize()*8),
|
||||
isSigned, HeapReg, ptr.reg, rt, Offset, Assembler::Always);
|
||||
masm.append(access, ld.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
storeI32(MemoryAccessDesc access, RegI32 ptr, Register rt) {
|
||||
if (access.byteSize() > 1 && IsUnaligned(access)) {
|
||||
masm.add32(HeapReg, ptr.reg);
|
||||
masm.emitUnalignedStore(access.byteSize(), ptr.reg, rt, 0);
|
||||
} else {
|
||||
BufferOffset st =
|
||||
masm.ma_dataTransferN(js::jit::IsStore, BitSize(access.byteSize()*8),
|
||||
IsSigned(false), ptr.reg, HeapReg, rt, Offset,
|
||||
Assembler::Always);
|
||||
masm.append(access, st.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
loadI64(MemoryAccessDesc access, RegI32 ptr, RegI64 dest) {
|
||||
if (IsUnaligned(access)) {
|
||||
masm.add32(HeapReg, ptr.reg);
|
||||
SecondScratchRegisterScope scratch(*this);
|
||||
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, dest.reg.low,
|
||||
0);
|
||||
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, dest.reg.high,
|
||||
4);
|
||||
} else {
|
||||
BufferOffset ld;
|
||||
ld = masm.ma_dataTransferN(js::jit::IsLoad, BitSize(32), IsSigned(false), HeapReg,
|
||||
ptr.reg, dest.reg.low, Offset, Assembler::Always);
|
||||
masm.append(access, ld.getOffset(), masm.framePushed());
|
||||
masm.add32(Imm32(4), ptr.reg);
|
||||
ld = masm.ma_dataTransferN(js::jit::IsLoad, BitSize(32), IsSigned(false), HeapReg,
|
||||
ptr.reg, dest.reg.high, Offset, Assembler::Always);
|
||||
masm.append(access, ld.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
storeI64(MemoryAccessDesc access, RegI32 ptr, RegI64 src) {
|
||||
if (IsUnaligned(access)) {
|
||||
masm.add32(HeapReg, ptr.reg);
|
||||
masm.emitUnalignedStore(ByteSize(4), ptr.reg, src.reg.low, 0);
|
||||
masm.emitUnalignedStore(ByteSize(4), ptr.reg, src.reg.high, 4);
|
||||
} else {
|
||||
BufferOffset st;
|
||||
st = masm.ma_dataTransferN(js::jit::IsStore, BitSize(32), IsSigned(false), HeapReg,
|
||||
ptr.reg, src.reg.low, Offset, Assembler::Always);
|
||||
masm.append(access, st.getOffset(), masm.framePushed());
|
||||
masm.add32(Imm32(4), ptr.reg);
|
||||
st = masm.ma_dataTransferN(js::jit::IsStore, BitSize(32), IsSigned(false), HeapReg,
|
||||
ptr.reg, src.reg.high, Offset, Assembler::Always);
|
||||
masm.append(access, st.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
loadF32(MemoryAccessDesc access, RegI32 ptr, RegF32 dest, RegI32 tmp1) {
|
||||
masm.add32(HeapReg, ptr.reg);
|
||||
if (IsUnaligned(access)) {
|
||||
SecondScratchRegisterScope scratch(*this);
|
||||
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp1.reg, 0);
|
||||
masm.ma_vxfer(tmp1.reg, dest.reg);
|
||||
} else {
|
||||
BufferOffset ld = masm.ma_vldr(VFPAddr(ptr.reg, VFPOffImm(0)), dest.reg,
|
||||
Assembler::Always);
|
||||
masm.append(access, ld.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
storeF32(MemoryAccessDesc access, RegI32 ptr, RegF32 src, RegI32 tmp1) {
|
||||
masm.add32(HeapReg, ptr.reg);
|
||||
if (IsUnaligned(access)) {
|
||||
masm.ma_vxfer(src.reg, tmp1.reg);
|
||||
masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp1.reg, 0);
|
||||
} else {
|
||||
BufferOffset st =
|
||||
masm.ma_vstr(src.reg, VFPAddr(ptr.reg, VFPOffImm(0)), Assembler::Always);
|
||||
masm.append(access, st.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
loadF64(MemoryAccessDesc access, RegI32 ptr, RegF64 dest, RegI32 tmp1, RegI32 tmp2) {
|
||||
masm.add32(HeapReg, ptr.reg);
|
||||
if (IsUnaligned(access)) {
|
||||
SecondScratchRegisterScope scratch(*this);
|
||||
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp1.reg, 0);
|
||||
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp2.reg, 4);
|
||||
masm.ma_vxfer(tmp1.reg, tmp2.reg, dest.reg);
|
||||
} else {
|
||||
BufferOffset ld = masm.ma_vldr(VFPAddr(ptr.reg, VFPOffImm(0)), dest.reg,
|
||||
Assembler::Always);
|
||||
masm.append(access, ld.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
storeF64(MemoryAccessDesc access, RegI32 ptr, RegF64 src, RegI32 tmp1, RegI32 tmp2) {
|
||||
masm.add32(HeapReg, ptr.reg);
|
||||
if (IsUnaligned(access)) {
|
||||
masm.ma_vxfer(src.reg, tmp1.reg, tmp2.reg);
|
||||
masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp1.reg, 0);
|
||||
masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp2.reg, 4);
|
||||
} else {
|
||||
BufferOffset st =
|
||||
masm.ma_vstr(src.reg, VFPAddr(ptr.reg, VFPOffImm(0)), Assembler::Always);
|
||||
masm.append(access, st.getOffset(), masm.framePushed());
|
||||
}
|
||||
}
|
||||
#endif // JS_CODEGEN_ARM
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
|
||||
// Generally speaking, ABOVE this point there should be no value
|
||||
|
|
@ -6868,9 +6623,10 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType)
|
|||
|
||||
MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
|
||||
|
||||
size_t temps = loadStoreTemps(access);
|
||||
size_t temps = loadTemps(access);
|
||||
RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
|
||||
RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
|
||||
RegI32 tmp3 = temps >= 3 ? needI32() : invalidI32();
|
||||
|
||||
switch (type) {
|
||||
case ValType::I32: {
|
||||
|
|
@ -6880,7 +6636,7 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType)
|
|||
#else
|
||||
RegI32 rv = rp;
|
||||
#endif
|
||||
if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
|
||||
if (!load(access, rp, AnyReg(rv), tmp1, tmp2, tmp3))
|
||||
return false;
|
||||
pushI32(rv);
|
||||
if (rp != rv)
|
||||
|
|
@ -6898,7 +6654,7 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType)
|
|||
rp = popI32();
|
||||
rv = needI64();
|
||||
#endif
|
||||
if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
|
||||
if (!load(access, rp, AnyReg(rv), tmp1, tmp2, tmp3))
|
||||
return false;
|
||||
pushI64(rv);
|
||||
freeI32(rp);
|
||||
|
|
@ -6907,7 +6663,7 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType)
|
|||
case ValType::F32: {
|
||||
RegI32 rp = popI32();
|
||||
RegF32 rv = needF32();
|
||||
if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
|
||||
if (!load(access, rp, AnyReg(rv), tmp1, tmp2, tmp3))
|
||||
return false;
|
||||
pushF32(rv);
|
||||
freeI32(rp);
|
||||
|
|
@ -6916,7 +6672,7 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType)
|
|||
case ValType::F64: {
|
||||
RegI32 rp = popI32();
|
||||
RegF64 rv = needF64();
|
||||
if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
|
||||
if (!load(access, rp, AnyReg(rv), tmp1, tmp2, tmp3))
|
||||
return false;
|
||||
pushF64(rv);
|
||||
freeI32(rp);
|
||||
|
|
@ -6931,6 +6687,8 @@ BaseCompiler::emitLoad(ValType type, Scalar::Type viewType)
|
|||
freeI32(tmp1);
|
||||
if (temps >= 2)
|
||||
freeI32(tmp2);
|
||||
if (temps >= 3)
|
||||
freeI32(tmp3);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -6951,15 +6709,14 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType)
|
|||
|
||||
MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
|
||||
|
||||
size_t temps = loadStoreTemps(access);
|
||||
size_t temps = storeTemps(access);
|
||||
RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
|
||||
RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
|
||||
|
||||
switch (resultType) {
|
||||
case ValType::I32: {
|
||||
RegI32 rp, rv;
|
||||
pop2xI32(&rp, &rv);
|
||||
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
|
||||
if (!store(access, rp, AnyReg(rv), tmp1))
|
||||
return false;
|
||||
freeI32(rp);
|
||||
freeI32(rv);
|
||||
|
|
@ -6968,7 +6725,7 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType)
|
|||
case ValType::I64: {
|
||||
RegI64 rv = popI64();
|
||||
RegI32 rp = popI32();
|
||||
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
|
||||
if (!store(access, rp, AnyReg(rv), tmp1))
|
||||
return false;
|
||||
freeI32(rp);
|
||||
freeI64(rv);
|
||||
|
|
@ -6977,7 +6734,7 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType)
|
|||
case ValType::F32: {
|
||||
RegF32 rv = popF32();
|
||||
RegI32 rp = popI32();
|
||||
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
|
||||
if (!store(access, rp, AnyReg(rv), tmp1))
|
||||
return false;
|
||||
freeI32(rp);
|
||||
freeF32(rv);
|
||||
|
|
@ -6986,7 +6743,7 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType)
|
|||
case ValType::F64: {
|
||||
RegF64 rv = popF64();
|
||||
RegI32 rp = popI32();
|
||||
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
|
||||
if (!store(access, rp, AnyReg(rv), tmp1))
|
||||
return false;
|
||||
freeI32(rp);
|
||||
freeF64(rv);
|
||||
|
|
@ -6999,8 +6756,6 @@ BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType)
|
|||
|
||||
if (temps >= 1)
|
||||
freeI32(tmp1);
|
||||
if (temps >= 2)
|
||||
freeI32(tmp2);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -7314,16 +7069,15 @@ BaseCompiler::emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType
|
|||
|
||||
MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
|
||||
|
||||
size_t temps = loadStoreTemps(access);
|
||||
size_t temps = storeTemps(access);
|
||||
RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
|
||||
RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
|
||||
|
||||
if (resultType == ValType::F32 && viewType == Scalar::Float64) {
|
||||
RegF32 rv = popF32();
|
||||
RegF64 rw = needF64();
|
||||
masm.convertFloat32ToDouble(rv.reg, rw.reg);
|
||||
RegI32 rp = popI32();
|
||||
if (!store(access, rp, AnyReg(rw), tmp1, tmp2))
|
||||
if (!store(access, rp, AnyReg(rw), tmp1))
|
||||
return false;
|
||||
pushF32(rv);
|
||||
freeI32(rp);
|
||||
|
|
@ -7334,7 +7088,7 @@ BaseCompiler::emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType
|
|||
RegF32 rw = needF32();
|
||||
masm.convertDoubleToFloat32(rv.reg, rw.reg);
|
||||
RegI32 rp = popI32();
|
||||
if (!store(access, rp, AnyReg(rw), tmp1, tmp2))
|
||||
if (!store(access, rp, AnyReg(rw), tmp1))
|
||||
return false;
|
||||
pushF64(rv);
|
||||
freeI32(rp);
|
||||
|
|
@ -7345,8 +7099,6 @@ BaseCompiler::emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType
|
|||
|
||||
if (temps >= 1)
|
||||
freeI32(tmp1);
|
||||
if (temps >= 2)
|
||||
freeI32(tmp2);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue