From 76d0a10fee50190aa79de1af568b7db3cfe4aef3 Mon Sep 17 00:00:00 2001 From: win7-7 Date: Tue, 30 Dec 2025 06:22:08 +0200 Subject: [PATCH] 1335068 - Wasm: Split the global allocation out of the CodeSegment 1335068 - Break wasm globals out of the code segment. --- js/src/jit/Lowering.cpp | 10 +- js/src/jit/MIR.h | 21 ++- js/src/jit/RegisterAllocator.h | 8 +- js/src/jit/arm/Assembler-arm.h | 6 - js/src/jit/arm/CodeGenerator-arm.cpp | 72 +------- js/src/jit/arm/MacroAssembler-arm.h | 4 +- js/src/jit/arm64/Assembler-arm64.h | 2 - js/src/jit/arm64/CodeGenerator-arm64.cpp | 13 -- js/src/jit/arm64/CodeGenerator-arm64.h | 3 - js/src/jit/arm64/MacroAssembler-arm64.h | 4 +- .../mips-shared/CodeGenerator-mips-shared.cpp | 34 +--- .../mips-shared/CodeGenerator-mips-shared.h | 2 - js/src/jit/mips32/CodeGenerator-mips32.cpp | 24 --- js/src/jit/mips32/CodeGenerator-mips32.h | 2 - js/src/jit/mips64/CodeGenerator-mips64.cpp | 18 -- js/src/jit/mips64/CodeGenerator-mips64.h | 2 - js/src/jit/none/MacroAssembler-none.h | 1 - js/src/jit/shared/Assembler-shared.h | 9 - js/src/jit/shared/CodeGenerator-shared.cpp | 98 ++++++++++ js/src/jit/shared/CodeGenerator-shared.h | 4 + js/src/jit/shared/LIR-shared.h | 35 +++- js/src/jit/x64/CodeGenerator-x64.cpp | 93 ---------- js/src/jit/x64/CodeGenerator-x64.h | 4 - js/src/jit/x64/MacroAssembler-x64.h | 9 +- .../x86-shared/MacroAssembler-x86-shared.h | 13 ++ js/src/jit/x86/CodeGenerator-x86.cpp | 62 ------- js/src/jit/x86/CodeGenerator-x86.h | 4 - js/src/jit/x86/MacroAssembler-x86.h | 3 +- js/src/wasm/WasmBaselineCompile.cpp | 168 +++--------------- js/src/wasm/WasmBinaryConstants.h | 7 - js/src/wasm/WasmCode.cpp | 53 +++--- js/src/wasm/WasmCode.h | 25 +-- js/src/wasm/WasmGenerator.cpp | 27 +-- js/src/wasm/WasmInstance.cpp | 77 +++++--- js/src/wasm/WasmInstance.h | 37 +++- js/src/wasm/WasmIonCompile.cpp | 4 +- js/src/wasm/WasmJS.cpp | 2 + js/src/wasm/WasmJS.h | 1 + js/src/wasm/WasmModule.cpp | 7 + js/src/wasm/WasmStubs.cpp | 5 - js/src/wasm/WasmTable.cpp | 2 +- js/src/wasm/WasmTypes.h | 9 + 42 files changed, 352 insertions(+), 632 deletions(-) diff --git a/js/src/jit/Lowering.cpp b/js/src/jit/Lowering.cpp index 7f29d537a7..2ac1cd2602 100644 --- a/js/src/jit/Lowering.cpp +++ b/js/src/jit/Lowering.cpp @@ -4244,20 +4244,22 @@ LIRGenerator::visitWasmBoundsCheck(MWasmBoundsCheck* ins) void LIRGenerator::visitWasmLoadGlobalVar(MWasmLoadGlobalVar* ins) { + LAllocation tlsPtr = useRegisterAtStart(ins->tlsPtr()); if (ins->type() == MIRType::Int64) - defineInt64(new(alloc()) LWasmLoadGlobalVarI64, ins); + defineInt64(new(alloc()) LWasmLoadGlobalVarI64(tlsPtr), ins); else - define(new(alloc()) LWasmLoadGlobalVar, ins); + define(new(alloc()) LWasmLoadGlobalVar(tlsPtr), ins); } void LIRGenerator::visitWasmStoreGlobalVar(MWasmStoreGlobalVar* ins) { MDefinition* value = ins->value(); + LAllocation tlsPtr = useRegisterAtStart(ins->tlsPtr()); if (value->type() == MIRType::Int64) - add(new(alloc()) LWasmStoreGlobalVarI64(useInt64RegisterAtStart(value)), ins); + add(new(alloc()) LWasmStoreGlobalVarI64(useInt64RegisterAtStart(value), tlsPtr), ins); else - add(new(alloc()) LWasmStoreGlobalVar(useRegisterAtStart(value)), ins); + add(new(alloc()) LWasmStoreGlobalVar(useRegisterAtStart(value), tlsPtr), ins); } void diff --git a/js/src/jit/MIR.h b/js/src/jit/MIR.h index 9dceb5ea7c..d2247b56eb 100644 --- a/js/src/jit/MIR.h +++ b/js/src/jit/MIR.h @@ -12860,10 +12860,13 @@ class MAsmJSAtomicBinopHeap } }; -class MWasmLoadGlobalVar : public MNullaryInstruction +class MWasmLoadGlobalVar + : public MUnaryInstruction, + public NoTypePolicy::Data { - MWasmLoadGlobalVar(MIRType type, unsigned globalDataOffset, bool isConstant) - : globalDataOffset_(globalDataOffset), isConstant_(isConstant) + MWasmLoadGlobalVar(MIRType type, unsigned globalDataOffset, bool isConstant, MDefinition* tlsPtr) + : MUnaryInstruction(classOpcode, tlsPtr), + globalDataOffset_(globalDataOffset), isConstant_(isConstant) { MOZ_ASSERT(IsNumberType(type)); setResultType(type); @@ -12876,6 +12879,7 @@ class MWasmLoadGlobalVar : public MNullaryInstruction public: INSTRUCTION_HEADER(WasmLoadGlobalVar) TRIVIAL_NEW_WRAPPERS + NAMED_OPERANDS((0, tlsPtr)) unsigned globalDataOffset() const { return globalDataOffset_; } @@ -12891,21 +12895,22 @@ class MWasmLoadGlobalVar : public MNullaryInstruction }; class MWasmStoreGlobalVar - : public MUnaryInstruction, + : public MBinaryInstruction, public NoTypePolicy::Data { - MWasmStoreGlobalVar(unsigned globalDataOffset, MDefinition* v) - : MUnaryInstruction(v), globalDataOffset_(globalDataOffset) - {} + MWasmStoreGlobalVar(unsigned globalDataOffset, MDefinition* value, MDefinition* tlsPtr) + : MBinaryInstruction(classOpcode, value, tlsPtr), + globalDataOffset_(globalDataOffset) + { } unsigned globalDataOffset_; public: INSTRUCTION_HEADER(WasmStoreGlobalVar) TRIVIAL_NEW_WRAPPERS + NAMED_OPERANDS((0, value), (1, tlsPtr)) unsigned globalDataOffset() const { return globalDataOffset_; } - MDefinition* value() const { return getOperand(0); } AliasSet getAliasSet() const override { return AliasSet::Store(AliasSet::WasmGlobalVar); diff --git a/js/src/jit/RegisterAllocator.h b/js/src/jit/RegisterAllocator.h index 8b5d0736a6..ad7dcf1d98 100644 --- a/js/src/jit/RegisterAllocator.h +++ b/js/src/jit/RegisterAllocator.h @@ -280,16 +280,12 @@ class RegisterAllocator allRegisters_(RegisterSet::All()) { if (mir->compilingWasm()) { -#if defined(JS_CODEGEN_X64) +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) allRegisters_.take(AnyRegister(HeapReg)); -#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || \ - defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONGARCH64) - allRegisters_.take(AnyRegister(HeapReg)); - allRegisters_.take(AnyRegister(GlobalReg)); #elif defined(JS_CODEGEN_ARM64) allRegisters_.take(AnyRegister(HeapReg)); allRegisters_.take(AnyRegister(HeapLenReg)); - allRegisters_.take(AnyRegister(GlobalReg)); #endif } else { if (FramePointer != InvalidReg && mir->instrumentedProfiling()) diff --git a/js/src/jit/arm/Assembler-arm.h b/js/src/jit/arm/Assembler-arm.h index 7f71e52f11..7c314a0c32 100644 --- a/js/src/jit/arm/Assembler-arm.h +++ b/js/src/jit/arm/Assembler-arm.h @@ -72,7 +72,6 @@ static constexpr Register IntArgReg0 = r0; static constexpr Register IntArgReg1 = r1; static constexpr Register IntArgReg2 = r2; static constexpr Register IntArgReg3 = r3; -static constexpr Register GlobalReg = r10; static constexpr Register HeapReg = r11; static constexpr Register CallTempNonArgRegs[] = { r5, r6, r7, r8 }; static const uint32_t NumCallTempNonArgRegs = @@ -159,11 +158,6 @@ struct ScratchDoubleScope : public AutoFloatRegisterScope { } }; -// A bias applied to the GlobalReg to allow the use of instructions with small -// negative immediate offsets which doubles the range of global data that can be -// accessed with a single instruction. -static const int32_t WasmGlobalRegBias = 1024; - // Registers used in the GenerateFFIIonExit Enable Activation block. static constexpr Register WasmIonExitRegCallee = r4; static constexpr Register WasmIonExitRegE0 = r0; diff --git a/js/src/jit/arm/CodeGenerator-arm.cpp b/js/src/jit/arm/CodeGenerator-arm.cpp index 4325aac9a6..1d62cc26ae 100644 --- a/js/src/jit/arm/CodeGenerator-arm.cpp +++ b/js/src/jit/arm/CodeGenerator-arm.cpp @@ -2336,9 +2336,10 @@ CodeGeneratorARM::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) BufferOffset cmp = masm.as_cmp(ptrReg, Imm8(0)); masm.append(wasm::BoundsCheck(cmp.getOffset())); - size_t nanOffset = size == 32 ? wasm::NaN32GlobalDataOffset : wasm::NaN64GlobalDataOffset; - masm.ma_vldr(Address(GlobalReg, nanOffset - WasmGlobalRegBias), output, scratch, - Assembler::AboveOrEqual); + if (size == 32) + masm.ma_vimm_f32(GenericNaN(), output, Assembler::AboveOrEqual); + else + masm.ma_vimm(GenericNaN(), output, Assembler::AboveOrEqual); cond = Assembler::Below; } @@ -2924,71 +2925,6 @@ CodeGeneratorARM::visitEffectiveAddress(LEffectiveAddress* ins) masm.ma_add(Imm32(mir->displacement()), output, scratch); } -void -CodeGeneratorARM::visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins) -{ - const MWasmLoadGlobalVar* mir = ins->mir(); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - - ScratchRegisterScope scratch(masm); - - if (mir->type() == MIRType::Int32) { - masm.ma_dtr(IsLoad, GlobalReg, Imm32(addr), ToRegister(ins->output()), scratch); - } else if (mir->type() == MIRType::Float32) { - VFPRegister vd(ToFloatRegister(ins->output())); - masm.ma_vldr(Address(GlobalReg, addr), vd.singleOverlay(), scratch); - } else { - MOZ_ASSERT(mir->type() == MIRType::Double); - masm.ma_vldr(Address(GlobalReg, addr), ToFloatRegister(ins->output()), scratch); - } -} - -void -CodeGeneratorARM::visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins) -{ - const MWasmLoadGlobalVar* mir = ins->mir(); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - MOZ_ASSERT(mir->type() == MIRType::Int64); - Register64 output = ToOutRegister64(ins); - - ScratchRegisterScope scratch(masm); - masm.ma_dtr(IsLoad, GlobalReg, Imm32(addr + INT64LOW_OFFSET), output.low, scratch); - masm.ma_dtr(IsLoad, GlobalReg, Imm32(addr + INT64HIGH_OFFSET), output.high, scratch); -} - -void -CodeGeneratorARM::visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins) -{ - const MWasmStoreGlobalVar* mir = ins->mir(); - MIRType type = mir->value()->type(); - - ScratchRegisterScope scratch(masm); - - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - if (type == MIRType::Int32) { - masm.ma_dtr(IsStore, GlobalReg, Imm32(addr), ToRegister(ins->value()), scratch); - } else if (type == MIRType::Float32) { - VFPRegister vd(ToFloatRegister(ins->value())); - masm.ma_vstr(vd.singleOverlay(), Address(GlobalReg, addr), scratch); - } else { - MOZ_ASSERT(type == MIRType::Double); - masm.ma_vstr(ToFloatRegister(ins->value()), Address(GlobalReg, addr), scratch); - } -} - -void -CodeGeneratorARM::visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins) -{ - const MWasmStoreGlobalVar* mir = ins->mir(); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - MOZ_ASSERT (mir->value()->type() == MIRType::Int64); - Register64 input = ToRegister64(ins->value()); - - ScratchRegisterScope scratch(masm); - masm.ma_dtr(IsStore, GlobalReg, Imm32(addr + INT64LOW_OFFSET), input.low, scratch); - masm.ma_dtr(IsStore, GlobalReg, Imm32(addr + INT64HIGH_OFFSET), input.high, scratch); -} - void CodeGeneratorARM::visitNegI(LNegI* ins) { diff --git a/js/src/jit/arm/MacroAssembler-arm.h b/js/src/jit/arm/MacroAssembler-arm.h index 4d8f1aff97..218939c2a4 100644 --- a/js/src/jit/arm/MacroAssembler-arm.h +++ b/js/src/jit/arm/MacroAssembler-arm.h @@ -1525,13 +1525,11 @@ class MacroAssemblerARMCompat : public MacroAssemblerARM } void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest) { - loadPtr(Address(GlobalReg, globalDataOffset - WasmGlobalRegBias), dest); + loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, globalArea) + globalDataOffset), dest); } void loadWasmPinnedRegsFromTls() { ScratchRegisterScope scratch(asMasm()); ma_ldr(Address(WasmTlsReg, offsetof(wasm::TlsData, memoryBase)), HeapReg, scratch); - ma_ldr(Address(WasmTlsReg, offsetof(wasm::TlsData, globalData)), GlobalReg, scratch); - ma_add(Imm32(WasmGlobalRegBias), GlobalReg, scratch); } // Instrumentation for entering and leaving the profiler. diff --git a/js/src/jit/arm64/Assembler-arm64.h b/js/src/jit/arm64/Assembler-arm64.h index 47f08012ce..5a07900002 100644 --- a/js/src/jit/arm64/Assembler-arm64.h +++ b/js/src/jit/arm64/Assembler-arm64.h @@ -82,7 +82,6 @@ static constexpr Register IntArgReg4 = { Registers::x4 }; static constexpr Register IntArgReg5 = { Registers::x5 }; static constexpr Register IntArgReg6 = { Registers::x6 }; static constexpr Register IntArgReg7 = { Registers::x7 }; -static constexpr Register GlobalReg = { Registers::x20 }; static constexpr Register HeapReg = { Registers::x21 }; static constexpr Register HeapLenReg = { Registers::x22 }; @@ -169,7 +168,6 @@ static_assert(CodeAlignment % SimdMemoryAlignment == 0, "alignment for SIMD constants."); static const uint32_t WasmStackAlignment = SimdMemoryAlignment; -static const int32_t WasmGlobalRegBias = 1024; // Does this architecture support SIMD conversions between Uint32x4 and Float32x4? static constexpr bool SupportsUint32x4FloatConversions = false; diff --git a/js/src/jit/arm64/CodeGenerator-arm64.cpp b/js/src/jit/arm64/CodeGenerator-arm64.cpp index 9668fa2162..e746c76b50 100644 --- a/js/src/jit/arm64/CodeGenerator-arm64.cpp +++ b/js/src/jit/arm64/CodeGenerator-arm64.cpp @@ -619,7 +619,6 @@ getBase(U* mir) { switch (mir->base()) { case U::Heap: return HeapReg; - case U::Global: return GlobalReg; } return InvalidReg; } @@ -696,18 +695,6 @@ CodeGeneratorARM64::visitEffectiveAddress(LEffectiveAddress* ins) MOZ_CRASH("visitEffectiveAddress"); } -void -CodeGeneratorARM64::visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins) -{ - MOZ_CRASH("visitWasmLoadGlobalVar"); -} - -void -CodeGeneratorARM64::visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins) -{ - MOZ_CRASH("visitWasmStoreGlobalVar"); -} - void CodeGeneratorARM64::visitNegI(LNegI* ins) { diff --git a/js/src/jit/arm64/CodeGenerator-arm64.h b/js/src/jit/arm64/CodeGenerator-arm64.h index e415dc0fc1..505b61ff3e 100644 --- a/js/src/jit/arm64/CodeGenerator-arm64.h +++ b/js/src/jit/arm64/CodeGenerator-arm64.h @@ -205,9 +205,6 @@ class CodeGeneratorARM64 : public CodeGeneratorShared void visitAsmJSAtomicBinopHeap(LAsmJSAtomicBinopHeap* ins); void visitWasmStackArg(LWasmStackArg* ins); - void visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins); - void visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins); - void generateInvalidateEpilogue(); void setReturnDoubleRegs(LiveRegisterSet* regs); diff --git a/js/src/jit/arm64/MacroAssembler-arm64.h b/js/src/jit/arm64/MacroAssembler-arm64.h index 945f65a01e..4b2cda2188 100644 --- a/js/src/jit/arm64/MacroAssembler-arm64.h +++ b/js/src/jit/arm64/MacroAssembler-arm64.h @@ -2252,12 +2252,10 @@ class MacroAssemblerCompat : public vixl::MacroAssembler } void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest) { - loadPtr(Address(GlobalReg, globalDataOffset - WasmGlobalRegBias), dest); + loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, globalArea) + globalDataOffset), dest); } void loadWasmPinnedRegsFromTls() { loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, memoryBase)), HeapReg); - loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, globalData)), GlobalReg); - adds32(Imm32(WasmGlobalRegBias), GlobalReg); } // Overwrites the payload bits of a dest register containing a Value. diff --git a/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp b/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp index 9e250c5b42..91b9878469 100644 --- a/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp +++ b/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp @@ -2132,11 +2132,9 @@ CodeGeneratorMIPSShared::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) // Offset is out of range. Load default values. if (isFloat) { if (size == 32) - masm.loadFloat32(Address(GlobalReg, wasm::NaN32GlobalDataOffset - WasmGlobalRegBias), - ToFloatRegister(out)); + masm.loadConstantFloat32(float(GenericNaN()), ToFloatRegister(out)); else - masm.loadDouble(Address(GlobalReg, wasm::NaN64GlobalDataOffset - WasmGlobalRegBias), - ToFloatRegister(out)); + masm.loadConstantDouble(GenericNaN(), ToFloatRegister(out)); } else { masm.move32(Imm32(0), ToRegister(out)); } @@ -2469,34 +2467,6 @@ CodeGeneratorMIPSShared::visitEffectiveAddress(LEffectiveAddress* ins) masm.computeEffectiveAddress(address, output); } -void -CodeGeneratorMIPSShared::visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins) -{ - const MWasmLoadGlobalVar* mir = ins->mir(); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - if (mir->type() == MIRType::Int32) - masm.load32(Address(GlobalReg, addr), ToRegister(ins->output())); - else if (mir->type() == MIRType::Float32) - masm.loadFloat32(Address(GlobalReg, addr), ToFloatRegister(ins->output())); - else - masm.loadDouble(Address(GlobalReg, addr), ToFloatRegister(ins->output())); -} - -void -CodeGeneratorMIPSShared::visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins) -{ - const MWasmStoreGlobalVar* mir = ins->mir(); - - MOZ_ASSERT(IsNumberType(mir->value()->type())); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - if (mir->value()->type() == MIRType::Int32) - masm.store32(ToRegister(ins->value()), Address(GlobalReg, addr)); - else if (mir->value()->type() == MIRType::Float32) - masm.storeFloat32(ToFloatRegister(ins->value()), Address(GlobalReg, addr)); - else - masm.storeDouble(ToFloatRegister(ins->value()), Address(GlobalReg, addr)); -} - void CodeGeneratorMIPSShared::visitNegI(LNegI* ins) { diff --git a/js/src/jit/mips-shared/CodeGenerator-mips-shared.h b/js/src/jit/mips-shared/CodeGenerator-mips-shared.h index b8aa95e912..6603d0862b 100644 --- a/js/src/jit/mips-shared/CodeGenerator-mips-shared.h +++ b/js/src/jit/mips-shared/CodeGenerator-mips-shared.h @@ -186,8 +186,6 @@ class CodeGeneratorMIPSShared : public CodeGeneratorShared virtual void visitTruncateFToInt32(LTruncateFToInt32* ins); void visitWasmTruncateToInt32(LWasmTruncateToInt32* lir); - void visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins); - void visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins); // Out of line visitors. virtual void visitOutOfLineBailout(OutOfLineBailout* ool) = 0; diff --git a/js/src/jit/mips32/CodeGenerator-mips32.cpp b/js/src/jit/mips32/CodeGenerator-mips32.cpp index 55345334de..7e96cbbb28 100644 --- a/js/src/jit/mips32/CodeGenerator-mips32.cpp +++ b/js/src/jit/mips32/CodeGenerator-mips32.cpp @@ -618,30 +618,6 @@ CodeGeneratorMIPS::visitWasmUnalignedStoreI64(LWasmUnalignedStoreI64* lir) emitWasmStoreI64(lir); } -void -CodeGeneratorMIPS::visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins) -{ - const MWasmLoadGlobalVar* mir = ins->mir(); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - MOZ_ASSERT(mir->type() == MIRType::Int64); - Register64 output = ToOutRegister64(ins); - - masm.load32(Address(GlobalReg, addr + INT64LOW_OFFSET), output.low); - masm.load32(Address(GlobalReg, addr + INT64HIGH_OFFSET), output.high); -} - -void -CodeGeneratorMIPS::visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins) -{ - const MWasmStoreGlobalVar* mir = ins->mir(); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - MOZ_ASSERT (mir->value()->type() == MIRType::Int64); - Register64 input = ToRegister64(ins->value()); - - masm.store32(input.low, Address(GlobalReg, addr + INT64LOW_OFFSET)); - masm.store32(input.high, Address(GlobalReg, addr + INT64HIGH_OFFSET)); -} - void CodeGeneratorMIPS::visitWasmSelectI64(LWasmSelectI64* lir) { diff --git a/js/src/jit/mips32/CodeGenerator-mips32.h b/js/src/jit/mips32/CodeGenerator-mips32.h index 3a892c9dd9..19dc790c3a 100644 --- a/js/src/jit/mips32/CodeGenerator-mips32.h +++ b/js/src/jit/mips32/CodeGenerator-mips32.h @@ -50,8 +50,6 @@ class CodeGeneratorMIPS : public CodeGeneratorMIPSShared void visitWasmUnalignedLoadI64(LWasmUnalignedLoadI64* lir); void visitWasmStoreI64(LWasmStoreI64* ins); void visitWasmUnalignedStoreI64(LWasmUnalignedStoreI64* ins); - void visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins); - void visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins); void visitWasmSelectI64(LWasmSelectI64* lir); void visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir); void visitWasmReinterpretToI64(LWasmReinterpretToI64* lir); diff --git a/js/src/jit/mips64/CodeGenerator-mips64.cpp b/js/src/jit/mips64/CodeGenerator-mips64.cpp index 8e96171615..58a68437a4 100644 --- a/js/src/jit/mips64/CodeGenerator-mips64.cpp +++ b/js/src/jit/mips64/CodeGenerator-mips64.cpp @@ -547,24 +547,6 @@ CodeGeneratorMIPS64::visitWasmUnalignedStoreI64(LWasmUnalignedStoreI64* lir) emitWasmStoreI64(lir); } -void -CodeGeneratorMIPS64::visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins) -{ - const MWasmLoadGlobalVar* mir = ins->mir(); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - MOZ_ASSERT(mir->type() == MIRType::Int64); - masm.load64(Address(GlobalReg, addr), ToOutRegister64(ins)); -} - -void -CodeGeneratorMIPS64::visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins) -{ - const MWasmStoreGlobalVar* mir = ins->mir(); - unsigned addr = mir->globalDataOffset() - WasmGlobalRegBias; - MOZ_ASSERT(mir->value()->type() == MIRType::Int64); - masm.store64(ToRegister64(ins->value()), Address(GlobalReg, addr)); -} - void CodeGeneratorMIPS64::visitWasmSelectI64(LWasmSelectI64* lir) { diff --git a/js/src/jit/mips64/CodeGenerator-mips64.h b/js/src/jit/mips64/CodeGenerator-mips64.h index 7a30c795a6..738b8d2c0b 100644 --- a/js/src/jit/mips64/CodeGenerator-mips64.h +++ b/js/src/jit/mips64/CodeGenerator-mips64.h @@ -56,8 +56,6 @@ class CodeGeneratorMIPS64 : public CodeGeneratorMIPSShared void visitWasmUnalignedLoadI64(LWasmUnalignedLoadI64* lir); void visitWasmStoreI64(LWasmStoreI64* ins); void visitWasmUnalignedStoreI64(LWasmUnalignedStoreI64* ins); - void visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins); - void visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins); void visitWasmSelectI64(LWasmSelectI64* ins); void visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir); void visitWasmReinterpretToI64(LWasmReinterpretToI64* lir); diff --git a/js/src/jit/none/MacroAssembler-none.h b/js/src/jit/none/MacroAssembler-none.h index 4e9afc524a..b501f9a7d8 100644 --- a/js/src/jit/none/MacroAssembler-none.h +++ b/js/src/jit/none/MacroAssembler-none.h @@ -39,7 +39,6 @@ static constexpr Register IntArgReg0 = { Registers::invalid_reg }; static constexpr Register IntArgReg1 = { Registers::invalid_reg }; static constexpr Register IntArgReg2 = { Registers::invalid_reg }; static constexpr Register IntArgReg3 = { Registers::invalid_reg }; -static constexpr Register GlobalReg = { Registers::invalid_reg }; static constexpr Register HeapReg = { Registers::invalid_reg }; static constexpr Register WasmIonExitRegCallee = { Registers::invalid_reg }; diff --git a/js/src/jit/shared/Assembler-shared.h b/js/src/jit/shared/Assembler-shared.h index 9d550ee00d..f5a3623c95 100644 --- a/js/src/jit/shared/Assembler-shared.h +++ b/js/src/jit/shared/Assembler-shared.h @@ -798,7 +798,6 @@ class AssemblerShared wasm::MemoryAccessVector memoryAccesses_; wasm::MemoryPatchVector memoryPatches_; wasm::BoundsCheckVector boundsChecks_; - wasm::GlobalAccessVector globalAccesses_; wasm::SymbolicAccessVector symbolicAccesses_; protected: @@ -879,9 +878,6 @@ class AssemblerShared void append(wasm::BoundsCheck check) { enoughMemory_ &= boundsChecks_.append(check); } wasm::BoundsCheckVector&& extractBoundsChecks() { return Move(boundsChecks_); } - void append(wasm::GlobalAccess access) { enoughMemory_ &= globalAccesses_.append(access); } - const wasm::GlobalAccessVector& globalAccesses() const { return globalAccesses_; } - void append(wasm::SymbolicAccess access) { enoughMemory_ &= symbolicAccesses_.append(access); } size_t numSymbolicAccesses() const { return symbolicAccesses_.length(); } wasm::SymbolicAccess symbolicAccess(size_t i) const { return symbolicAccesses_[i]; } @@ -928,11 +924,6 @@ class AssemblerShared for (; i < boundsChecks_.length(); i++) boundsChecks_[i].offsetBy(delta); - i = globalAccesses_.length(); - enoughMemory_ &= globalAccesses_.appendAll(other.globalAccesses_); - for (; i < globalAccesses_.length(); i++) - globalAccesses_[i].patchAt.offsetBy(delta); - i = symbolicAccesses_.length(); enoughMemory_ &= symbolicAccesses_.appendAll(other.symbolicAccesses_); for (; i < symbolicAccesses_.length(); i++) diff --git a/js/src/jit/shared/CodeGenerator-shared.cpp b/js/src/jit/shared/CodeGenerator-shared.cpp index 78f66bb9da..3492086f6a 100644 --- a/js/src/jit/shared/CodeGenerator-shared.cpp +++ b/js/src/jit/shared/CodeGenerator-shared.cpp @@ -1529,6 +1529,104 @@ CodeGeneratorShared::emitWasmCallBase(LWasmCallBase* ins) masm.reserveStack(mir->spIncrement()); } +void +CodeGeneratorShared::visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins) +{ + MWasmLoadGlobalVar* mir = ins->mir(); + + MIRType type = mir->type(); + MOZ_ASSERT(IsNumberType(type) || IsSimdType(type)); + + Register tls = ToRegister(ins->tlsPtr()); + Address addr(tls, offsetof(wasm::TlsData, globalArea) + mir->globalDataOffset()); + switch (type) { + case MIRType::Int32: + masm.load32(addr, ToRegister(ins->output())); + break; + case MIRType::Float32: + masm.loadFloat32(addr, ToFloatRegister(ins->output())); + break; + case MIRType::Double: + masm.loadDouble(addr, ToFloatRegister(ins->output())); + break; + // Aligned access: code is aligned on PageSize + there is padding + // before the global data section. + case MIRType::Int8x16: + case MIRType::Int16x8: + case MIRType::Int32x4: + case MIRType::Bool8x16: + case MIRType::Bool16x8: + case MIRType::Bool32x4: + masm.loadInt32x4(addr, ToFloatRegister(ins->output())); + break; + case MIRType::Float32x4: + masm.loadFloat32x4(addr, ToFloatRegister(ins->output())); + break; + default: + MOZ_CRASH("unexpected type in visitWasmLoadGlobalVar"); + } +} + +void +CodeGeneratorShared::visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins) +{ + MWasmStoreGlobalVar* mir = ins->mir(); + + MIRType type = mir->value()->type(); + MOZ_ASSERT(IsNumberType(type) || IsSimdType(type)); + + Register tls = ToRegister(ins->tlsPtr()); + Address addr(tls, offsetof(wasm::TlsData, globalArea) + mir->globalDataOffset()); + switch (type) { + case MIRType::Int32: + masm.store32(ToRegister(ins->value()), addr); + break; + case MIRType::Float32: + masm.storeFloat32(ToFloatRegister(ins->value()), addr); + break; + case MIRType::Double: + masm.storeDouble(ToFloatRegister(ins->value()), addr); + break; + // Aligned access: code is aligned on PageSize + there is padding + // before the global data section. + case MIRType::Int32x4: + case MIRType::Bool32x4: + masm.storeInt32x4(ToFloatRegister(ins->value()), addr); + break; + case MIRType::Float32x4: + masm.storeFloat32x4(ToFloatRegister(ins->value()), addr); + break; + default: + MOZ_CRASH("unexpected type in visitWasmStoreGlobalVar"); + } +} + +void +CodeGeneratorShared::visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins) +{ + MWasmLoadGlobalVar* mir = ins->mir(); + MOZ_ASSERT(mir->type() == MIRType::Int64); + + Register tls = ToRegister(ins->tlsPtr()); + Address addr(tls, offsetof(wasm::TlsData, globalArea) + mir->globalDataOffset()); + + Register64 output = ToOutRegister64(ins); + masm.load64(addr, output); +} + +void +CodeGeneratorShared::visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins) +{ + MWasmStoreGlobalVar* mir = ins->mir(); + MOZ_ASSERT(mir->value()->type() == MIRType::Int64); + + Register tls = ToRegister(ins->tlsPtr()); + Address addr(tls, offsetof(wasm::TlsData, globalArea) + mir->globalDataOffset()); + + Register64 value = ToRegister64(ins->value()); + masm.store64(value, addr); +} + void CodeGeneratorShared::emitPreBarrier(Register base, const LAllocation* index, int32_t offsetAdjustment) { diff --git a/js/src/jit/shared/CodeGenerator-shared.h b/js/src/jit/shared/CodeGenerator-shared.h index c4245614cf..202b366dae 100644 --- a/js/src/jit/shared/CodeGenerator-shared.h +++ b/js/src/jit/shared/CodeGenerator-shared.h @@ -356,6 +356,10 @@ class CodeGeneratorShared : public LElementVisitor void emitTruncateFloat32(FloatRegister src, Register dest, MInstruction* mir); void emitWasmCallBase(LWasmCallBase* ins); + void visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins); + void visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins); + void visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins); + void visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins); void emitPreBarrier(Register base, const LAllocation* index, int32_t offsetAdjustment); void emitPreBarrier(Address address); diff --git a/js/src/jit/shared/LIR-shared.h b/js/src/jit/shared/LIR-shared.h index 6c875b951a..3e7b7bdb02 100644 --- a/js/src/jit/shared/LIR-shared.h +++ b/js/src/jit/shared/LIR-shared.h @@ -7751,30 +7751,43 @@ class LAsmJSAtomicBinopHeapForEffect : public LInstructionHelper<0, 2, 5> } }; -class LWasmLoadGlobalVar : public LInstructionHelper<1, 0, 0> +class LWasmLoadGlobalVar : public LInstructionHelper<1, 1, 0> { public: LIR_HEADER(WasmLoadGlobalVar); + explicit LWasmLoadGlobalVar(const LAllocation& tlsPtr) { + setOperand(0, tlsPtr); + } MWasmLoadGlobalVar* mir() const { return mir_->toWasmLoadGlobalVar(); } + const LAllocation* tlsPtr() { + return getOperand(0); + } }; -class LWasmLoadGlobalVarI64 : public LInstructionHelper +class LWasmLoadGlobalVarI64 : public LInstructionHelper { public: LIR_HEADER(WasmLoadGlobalVarI64); + explicit LWasmLoadGlobalVarI64(const LAllocation& tlsPtr) { + setOperand(0, tlsPtr); + } MWasmLoadGlobalVar* mir() const { return mir_->toWasmLoadGlobalVar(); } + const LAllocation* tlsPtr() { + return getOperand(0); + } }; -class LWasmStoreGlobalVar : public LInstructionHelper<0, 1, 0> +class LWasmStoreGlobalVar : public LInstructionHelper<0, 2, 0> { public: LIR_HEADER(WasmStoreGlobalVar); - explicit LWasmStoreGlobalVar(const LAllocation& value) { + explicit LWasmStoreGlobalVar(const LAllocation& value, const LAllocation& tlsPtr) { setOperand(0, value); + setOperand(1, tlsPtr); } MWasmStoreGlobalVar* mir() const { return mir_->toWasmStoreGlobalVar(); @@ -7782,22 +7795,28 @@ class LWasmStoreGlobalVar : public LInstructionHelper<0, 1, 0> const LAllocation* value() { return getOperand(0); } + const LAllocation* tlsPtr() { + return getOperand(1); + } }; -class LWasmStoreGlobalVarI64 : public LInstructionHelper<0, INT64_PIECES, 0> +class LWasmStoreGlobalVarI64 : public LInstructionHelper<0, INT64_PIECES + 1, 0> { public: LIR_HEADER(WasmStoreGlobalVarI64); - explicit LWasmStoreGlobalVarI64(const LInt64Allocation& value) { + explicit LWasmStoreGlobalVarI64(const LInt64Allocation& value, const LAllocation& tlsPtr) { setInt64Operand(0, value); + setOperand(INT64_PIECES, tlsPtr); } MWasmStoreGlobalVar* mir() const { return mir_->toWasmStoreGlobalVar(); } - static const uint32_t InputIndex = 0; const LInt64Allocation value() { - return getInt64Operand(InputIndex); + return getInt64Operand(0); + } + const LAllocation* tlsPtr() { + return getOperand(INT64_PIECES); } }; diff --git a/js/src/jit/x64/CodeGenerator-x64.cpp b/js/src/jit/x64/CodeGenerator-x64.cpp index 47b6e9f278..0fff027b2d 100644 --- a/js/src/jit/x64/CodeGenerator-x64.cpp +++ b/js/src/jit/x64/CodeGenerator-x64.cpp @@ -653,99 +653,6 @@ CodeGeneratorX64::visitAsmJSAtomicBinopHeapForEffect(LAsmJSAtomicBinopHeapForEff atomicBinopToTypedIntArray(op, accessType, ToRegister(value), srcAddr); } -void -CodeGeneratorX64::visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins) -{ - MWasmLoadGlobalVar* mir = ins->mir(); - - MIRType type = mir->type(); - MOZ_ASSERT(IsNumberType(type) || IsSimdType(type)); - - CodeOffset label; - switch (type) { - case MIRType::Int32: - label = masm.loadRipRelativeInt32(ToRegister(ins->output())); - break; - case MIRType::Float32: - label = masm.loadRipRelativeFloat32(ToFloatRegister(ins->output())); - break; - case MIRType::Double: - label = masm.loadRipRelativeDouble(ToFloatRegister(ins->output())); - break; - // Aligned access: code is aligned on PageSize + there is padding - // before the global data section. - case MIRType::Int8x16: - case MIRType::Int16x8: - case MIRType::Int32x4: - case MIRType::Bool8x16: - case MIRType::Bool16x8: - case MIRType::Bool32x4: - label = masm.loadRipRelativeInt32x4(ToFloatRegister(ins->output())); - break; - case MIRType::Float32x4: - label = masm.loadRipRelativeFloat32x4(ToFloatRegister(ins->output())); - break; - default: - MOZ_CRASH("unexpected type in visitWasmLoadGlobalVar"); - } - - masm.append(wasm::GlobalAccess(label, mir->globalDataOffset())); -} - -void -CodeGeneratorX64::visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins) -{ - MWasmLoadGlobalVar* mir = ins->mir(); - MOZ_ASSERT(mir->type() == MIRType::Int64); - CodeOffset label = masm.loadRipRelativeInt64(ToRegister(ins->output())); - masm.append(wasm::GlobalAccess(label, mir->globalDataOffset())); -} - -void -CodeGeneratorX64::visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins) -{ - MWasmStoreGlobalVar* mir = ins->mir(); - - MIRType type = mir->value()->type(); - MOZ_ASSERT(IsNumberType(type) || IsSimdType(type)); - - CodeOffset label; - switch (type) { - case MIRType::Int32: - label = masm.storeRipRelativeInt32(ToRegister(ins->value())); - break; - case MIRType::Float32: - label = masm.storeRipRelativeFloat32(ToFloatRegister(ins->value())); - break; - case MIRType::Double: - label = masm.storeRipRelativeDouble(ToFloatRegister(ins->value())); - break; - // Aligned access: code is aligned on PageSize + there is padding - // before the global data section. - case MIRType::Int32x4: - case MIRType::Bool32x4: - label = masm.storeRipRelativeInt32x4(ToFloatRegister(ins->value())); - break; - case MIRType::Float32x4: - label = masm.storeRipRelativeFloat32x4(ToFloatRegister(ins->value())); - break; - default: - MOZ_CRASH("unexpected type in visitWasmStoreGlobalVar"); - } - - masm.append(wasm::GlobalAccess(label, mir->globalDataOffset())); -} - -void -CodeGeneratorX64::visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins) -{ - MWasmStoreGlobalVar* mir = ins->mir(); - MOZ_ASSERT(mir->value()->type() == MIRType::Int64); - Register value = ToRegister(ins->getOperand(LWasmStoreGlobalVarI64::InputIndex)); - CodeOffset label = masm.storeRipRelativeInt64(value); - masm.append(wasm::GlobalAccess(label, mir->globalDataOffset())); -} - void CodeGeneratorX64::visitTruncateDToInt32(LTruncateDToInt32* ins) { diff --git a/js/src/jit/x64/CodeGenerator-x64.h b/js/src/jit/x64/CodeGenerator-x64.h index 2bb3b389b4..0e2f8cbcbc 100644 --- a/js/src/jit/x64/CodeGenerator-x64.h +++ b/js/src/jit/x64/CodeGenerator-x64.h @@ -61,10 +61,6 @@ class CodeGeneratorX64 : public CodeGeneratorX86Shared void visitWasmLoadI64(LWasmLoadI64* ins); void visitWasmStore(LWasmStore* ins); void visitWasmStoreI64(LWasmStoreI64* ins); - void visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins); - void visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins); - void visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins); - void visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins); void visitWasmSelectI64(LWasmSelectI64* ins); void visitWasmCall(LWasmCall* ins); void visitWasmCallI64(LWasmCallI64* ins); diff --git a/js/src/jit/x64/MacroAssembler-x64.h b/js/src/jit/x64/MacroAssembler-x64.h index 85fc4596a9..e0f8536a99 100644 --- a/js/src/jit/x64/MacroAssembler-x64.h +++ b/js/src/jit/x64/MacroAssembler-x64.h @@ -582,6 +582,9 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared void loadPtr(const Address& address, Register dest) { movq(Operand(address), dest); } + void load64(const Address& address, Register dest) { + movq(Operand(address), dest); + } void loadPtr(const Operand& src, Register dest) { movq(src, dest); } @@ -627,6 +630,9 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared void storePtr(Register src, const Address& address) { movq(src, Operand(address)); } + void store64(Register src, const Address& address) { + movq(src, Operand(address)); + } void storePtr(Register src, const BaseIndex& address) { movq(src, Operand(address)); } @@ -897,8 +903,7 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared Label* oolRejoin, FloatRegister tempDouble); void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest) { - CodeOffset label = loadRipRelativeInt64(dest); - append(wasm::GlobalAccess(label, globalDataOffset)); + loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, globalArea) + globalDataOffset), dest); } void loadWasmPinnedRegsFromTls() { loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, memoryBase)), HeapReg); diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared.h b/js/src/jit/x86-shared/MacroAssembler-x86-shared.h index 76d6ac86db..c88ae1bb3c 100644 --- a/js/src/jit/x86-shared/MacroAssembler-x86-shared.h +++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared.h @@ -809,6 +809,19 @@ class MacroAssemblerX86Shared : public Assembler vcvtsd2ss(src, dest, dest); } + void loadInt32x4(const Address& addr, FloatRegister dest) { + vmovdqa(Operand(addr), dest); + } + void loadFloat32x4(const Address& addr, FloatRegister dest) { + vmovaps(Operand(addr), dest); + } + void storeInt32x4(FloatRegister src, const Address& addr) { + vmovdqa(src, Operand(addr)); + } + void storeFloat32x4(FloatRegister src, const Address& addr) { + vmovaps(src, Operand(addr)); + } + void convertFloat32x4ToInt32x4(FloatRegister src, FloatRegister dest) { // Note that if the conversion failed (because the converted // result is larger than the maximum signed int32, or less than the diff --git a/js/src/jit/x86/CodeGenerator-x86.cpp b/js/src/jit/x86/CodeGenerator-x86.cpp index 65f0e40bcc..dd6e087eaa 100644 --- a/js/src/jit/x86/CodeGenerator-x86.cpp +++ b/js/src/jit/x86/CodeGenerator-x86.cpp @@ -663,68 +663,6 @@ CodeGeneratorX86::visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins) masm.append(wasm::GlobalAccess(label, mir->globalDataOffset())); } -void -CodeGeneratorX86::visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins) -{ - MWasmLoadGlobalVar* mir = ins->mir(); - - MOZ_ASSERT(mir->type() == MIRType::Int64); - Register64 output = ToOutRegister64(ins); - - CodeOffset labelLow = masm.movlWithPatch(PatchedAbsoluteAddress(), output.low); - masm.append(wasm::GlobalAccess(labelLow, mir->globalDataOffset() + INT64LOW_OFFSET)); - CodeOffset labelHigh = masm.movlWithPatch(PatchedAbsoluteAddress(), output.high); - masm.append(wasm::GlobalAccess(labelHigh, mir->globalDataOffset() + INT64HIGH_OFFSET)); -} - -void -CodeGeneratorX86::visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins) -{ - MWasmStoreGlobalVar* mir = ins->mir(); - - MIRType type = mir->value()->type(); - MOZ_ASSERT(IsNumberType(type) || IsSimdType(type)); - - CodeOffset label; - switch (type) { - case MIRType::Int32: - label = masm.movlWithPatch(ToRegister(ins->value()), PatchedAbsoluteAddress()); - break; - case MIRType::Float32: - label = masm.vmovssWithPatch(ToFloatRegister(ins->value()), PatchedAbsoluteAddress()); - break; - case MIRType::Double: - label = masm.vmovsdWithPatch(ToFloatRegister(ins->value()), PatchedAbsoluteAddress()); - break; - // Aligned access: code is aligned on PageSize + there is padding - // before the global data section. - case MIRType::Int32x4: - case MIRType::Bool32x4: - label = masm.vmovdqaWithPatch(ToFloatRegister(ins->value()), PatchedAbsoluteAddress()); - break; - case MIRType::Float32x4: - label = masm.vmovapsWithPatch(ToFloatRegister(ins->value()), PatchedAbsoluteAddress()); - break; - default: - MOZ_CRASH("unexpected type in visitWasmStoreGlobalVar"); - } - masm.append(wasm::GlobalAccess(label, mir->globalDataOffset())); -} - -void -CodeGeneratorX86::visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins) -{ - MWasmStoreGlobalVar* mir = ins->mir(); - - MOZ_ASSERT(mir->value()->type() == MIRType::Int64); - Register64 input = ToRegister64(ins->value()); - - CodeOffset labelLow = masm.movlWithPatch(input.low, PatchedAbsoluteAddress()); - masm.append(wasm::GlobalAccess(labelLow, mir->globalDataOffset() + INT64LOW_OFFSET)); - CodeOffset labelHigh = masm.movlWithPatch(input.high, PatchedAbsoluteAddress()); - masm.append(wasm::GlobalAccess(labelHigh, mir->globalDataOffset() + INT64HIGH_OFFSET)); -} - namespace js { namespace jit { diff --git a/js/src/jit/x86/CodeGenerator-x86.h b/js/src/jit/x86/CodeGenerator-x86.h index 7bc48d9f3f..e1e7c6dfdd 100644 --- a/js/src/jit/x86/CodeGenerator-x86.h +++ b/js/src/jit/x86/CodeGenerator-x86.h @@ -55,10 +55,6 @@ class CodeGeneratorX86 : public CodeGeneratorX86Shared void visitWasmLoadI64(LWasmLoadI64* ins); void visitWasmStore(LWasmStore* ins); void visitWasmStoreI64(LWasmStoreI64* ins); - void visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins); - void visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins); - void visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins); - void visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins); void visitAsmJSLoadHeap(LAsmJSLoadHeap* ins); void visitAsmJSStoreHeap(LAsmJSStoreHeap* ins); void visitAsmJSCompareExchangeHeap(LAsmJSCompareExchangeHeap* ins); diff --git a/js/src/jit/x86/MacroAssembler-x86.h b/js/src/jit/x86/MacroAssembler-x86.h index cf4edccaa7..190ef34714 100644 --- a/js/src/jit/x86/MacroAssembler-x86.h +++ b/js/src/jit/x86/MacroAssembler-x86.h @@ -864,8 +864,7 @@ class MacroAssemblerX86 : public MacroAssemblerX86Shared inline void ensureDouble(const ValueOperand& source, FloatRegister dest, Label* failure); void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest) { - CodeOffset label = movlWithPatch(PatchedAbsoluteAddress(), dest); - append(wasm::GlobalAccess(label, globalDataOffset)); + loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, globalArea) + globalDataOffset), dest); } void loadWasmPinnedRegsFromTls() { // x86 doesn't have any pinned registers. diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp index f1476ab08d..4fb6be1d5f 100644 --- a/js/src/wasm/WasmBaselineCompile.cpp +++ b/js/src/wasm/WasmBaselineCompile.cpp @@ -3360,174 +3360,65 @@ class BaseCompiler // // Global variable access. - // CodeGenerator{X86,X64}::visitWasmLoadGlobal() + uint32_t globalToTlsOffset(uint32_t globalOffset) { + return offsetof(TlsData, globalArea) + globalOffset; + } + void loadGlobalVarI32(unsigned globalDataOffset, RegI32 r) { -#if defined(JS_CODEGEN_X64) - CodeOffset label = masm.loadRipRelativeInt32(r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_X86) - CodeOffset label = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_ARM) - ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr), r.reg, scratch); -#elif defined(JS_CODEGEN_LOONGARCH64) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.load32(Address(GlobalReg, addr), r.reg); -#else - MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarI32"); -#endif +ScratchI32 tmp(*this); + loadFromFramePtr(tmp, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); + masm.load32(Address(tmp, globalToTlsOffset(globalDataOffset)), r); } void loadGlobalVarI64(unsigned globalDataOffset, RegI64 r) { -#if defined(JS_CODEGEN_X64) - CodeOffset label = masm.loadRipRelativeInt64(r.reg.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_X86) - CodeOffset labelLow = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg.low); - masm.append(GlobalAccess(labelLow, globalDataOffset + INT64LOW_OFFSET)); - CodeOffset labelHigh = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg.high); - masm.append(GlobalAccess(labelHigh, globalDataOffset + INT64HIGH_OFFSET)); -#elif defined(JS_CODEGEN_ARM) - ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr + INT64LOW_OFFSET), r.reg.low, scratch); - masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr + INT64HIGH_OFFSET), r.reg.high, - scratch); -#elif defined(JS_CODEGEN_LOONGARCH64) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.load64(Address(GlobalReg, addr), r.reg); -#else - MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarI64"); -#endif +ScratchI32 tmp(*this); + loadFromFramePtr(tmp, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); + masm.load64(Address(tmp, globalToTlsOffset(globalDataOffset)), r); } void loadGlobalVarF32(unsigned globalDataOffset, RegF32 r) { -#if defined(JS_CODEGEN_X64) - CodeOffset label = masm.loadRipRelativeFloat32(r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_X86) - CodeOffset label = masm.vmovssWithPatch(PatchedAbsoluteAddress(), r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_ARM) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - VFPRegister vd(r.reg); - masm.ma_vldr(VFPAddr(GlobalReg, VFPOffImm(addr)), vd.singleOverlay()); -#elif defined(JS_CODEGEN_LOONGARCH64) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.loadFloat32(Address(GlobalReg, addr), r.reg); -#else - MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarF32"); -#endif +ScratchI32 tmp(*this); + loadFromFramePtr(tmp, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); + masm.loadFloat32(Address(tmp, globalToTlsOffset(globalDataOffset)), r); } void loadGlobalVarF64(unsigned globalDataOffset, RegF64 r) { -#if defined(JS_CODEGEN_X64) - CodeOffset label = masm.loadRipRelativeDouble(r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_X86) - CodeOffset label = masm.vmovsdWithPatch(PatchedAbsoluteAddress(), r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_ARM) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.ma_vldr(VFPAddr(GlobalReg, VFPOffImm(addr)), r.reg); -#elif defined(JS_CODEGEN_LOONGARCH64) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.loadDouble(Address(GlobalReg, addr), r.reg); -#else - MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarF64"); -#endif +ScratchI32 tmp(*this); + loadFromFramePtr(tmp, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); + masm.loadDouble(Address(tmp, globalToTlsOffset(globalDataOffset)), r); } - // CodeGeneratorX64::visitWasmStoreGlobal() - void storeGlobalVarI32(unsigned globalDataOffset, RegI32 r) { -#if defined(JS_CODEGEN_X64) - CodeOffset label = masm.storeRipRelativeInt32(r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_X86) - CodeOffset label = masm.movlWithPatch(r.reg, PatchedAbsoluteAddress()); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_ARM) - ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr), r.reg, scratch); -#elif defined(JS_CODEGEN_LOONGARCH64) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.store32(r.reg, Address(GlobalReg, addr)); -#else - MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarI32"); -#endif +ScratchI32 tmp(*this); + loadFromFramePtr(tmp, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); + masm.store32(r, Address(tmp, globalToTlsOffset(globalDataOffset))); } void storeGlobalVarI64(unsigned globalDataOffset, RegI64 r) { -#if defined(JS_CODEGEN_X64) - CodeOffset label = masm.storeRipRelativeInt64(r.reg.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_X86) - CodeOffset labelLow = masm.movlWithPatch(r.reg.low, PatchedAbsoluteAddress()); - masm.append(GlobalAccess(labelLow, globalDataOffset + INT64LOW_OFFSET)); - CodeOffset labelHigh = masm.movlWithPatch(r.reg.high, PatchedAbsoluteAddress()); - masm.append(GlobalAccess(labelHigh, globalDataOffset + INT64HIGH_OFFSET)); -#elif defined(JS_CODEGEN_ARM) - ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr + INT64LOW_OFFSET), r.reg.low, scratch); - masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr + INT64HIGH_OFFSET), r.reg.high, - scratch); -#elif defined(JS_CODEGEN_LOONGARCH64) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.store64(r.reg, Address(GlobalReg, addr)); -#else - MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarI64"); -#endif +ScratchI32 tmp(*this); + loadFromFramePtr(tmp, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); + masm.store64(r, Address(tmp, globalToTlsOffset(globalDataOffset))); } void storeGlobalVarF32(unsigned globalDataOffset, RegF32 r) { -#if defined(JS_CODEGEN_X64) - CodeOffset label = masm.storeRipRelativeFloat32(r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_X86) - CodeOffset label = masm.vmovssWithPatch(r.reg, PatchedAbsoluteAddress()); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_ARM) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - VFPRegister vd(r.reg); - masm.ma_vstr(vd.singleOverlay(), VFPAddr(GlobalReg, VFPOffImm(addr))); -#elif defined(JS_CODEGEN_LOONGARCH64) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.storeFloat32(r.reg, Address(GlobalReg, addr)); -#else - MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarF32"); -#endif +ScratchI32 tmp(*this); + loadFromFramePtr(tmp, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); + masm.storeFloat32(r, Address(tmp, globalToTlsOffset(globalDataOffset))); } void storeGlobalVarF64(unsigned globalDataOffset, RegF64 r) { -#if defined(JS_CODEGEN_X64) - CodeOffset label = masm.storeRipRelativeDouble(r.reg); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_X86) - CodeOffset label = masm.vmovsdWithPatch(r.reg, PatchedAbsoluteAddress()); - masm.append(GlobalAccess(label, globalDataOffset)); -#elif defined(JS_CODEGEN_ARM) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.ma_vstr(r.reg, VFPAddr(GlobalReg, VFPOffImm(addr))); -#elif defined(JS_CODEGEN_LOONGARCH64) - unsigned addr = globalDataOffset - WasmGlobalRegBias; - masm.storeDouble(r.reg, Address(GlobalReg, addr)); -#else - MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarF64"); -#endif +ScratchI32 tmp(*this); + loadFromFramePtr(tmp, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer)); + masm.storeDouble(r, Address(tmp, globalToTlsOffset(globalDataOffset))); } ////////////////////////////////////////////////////////////////////// @@ -8072,18 +7963,15 @@ BaseCompiler::BaseCompiler(const ModuleEnvironment& env, availGPR_.take(HeapReg); #elif defined(JS_CODEGEN_ARM) availGPR_.take(HeapReg); - availGPR_.take(GlobalReg); availGPR_.take(ScratchRegARM); #elif defined(JS_CODEGEN_ARM64) availGPR_.take(HeapReg); availGPR_.take(HeapLenReg); - availGPR_.take(GlobalReg); #elif defined(JS_CODEGEN_X86) availGPR_.take(ScratchRegX86); #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ defined(JS_CODEGEN_LOONGARCH64) availGPR_.take(HeapReg); - availGPR_.take(GlobalReg); #endif } diff --git a/js/src/wasm/WasmBinaryConstants.h b/js/src/wasm/WasmBinaryConstants.h index 616a9b05e8..701adb0d8f 100644 --- a/js/src/wasm/WasmBinaryConstants.h +++ b/js/src/wasm/WasmBinaryConstants.h @@ -342,13 +342,6 @@ enum class Op Limit }; - -// Static offsets into the global data of every module that is compiled. - -static const unsigned NaN64GlobalDataOffset = 0; -static const unsigned NaN32GlobalDataOffset = NaN64GlobalDataOffset + sizeof(double); -static const unsigned InitialGlobalDataBytes = NaN32GlobalDataOffset + sizeof(float); - // These limits are agreed upon with other engines for consistency. static const unsigned MaxTypes = 1000000; diff --git a/js/src/wasm/WasmCode.cpp b/js/src/wasm/WasmCode.cpp index 39effecea7..5b3bb73345 100644 --- a/js/src/wasm/WasmCode.cpp +++ b/js/src/wasm/WasmCode.cpp @@ -57,25 +57,24 @@ static Atomic wasmCodeAllocations(0); static const uint32_t MaxWasmCodeAllocations = 16384; static uint8_t* -AllocateCodeSegment(JSContext* cx, uint32_t totalLength) +AllocateCodeSegment(JSContext* cx, uint32_t codeLength) { if (wasmCodeAllocations >= MaxWasmCodeAllocations) return nullptr; - // codeLength is a multiple of the system's page size, but not necessarily + // length is a multiple of the system's page size, but not necessarily // a multiple of ExecutableCodePageSize. - totalLength = JS_ROUNDUP(totalLength, ExecutableCodePageSize); + codeLength = JS_ROUNDUP(codeLength, ExecutableCodePageSize); - void* p = AllocateExecutableMemory(totalLength, ProtectionSetting::Writable); + void* p = AllocateExecutableMemory(codeLength, ProtectionSetting::Writable); // If the allocation failed and the embedding gives us a last-ditch attempt // to purge all memory (which, in gecko, does a purging GC/CC/GC), do that // then retry the allocation. if (!p) { - JSRuntime* rt = cx->runtime(); - if (rt->largeAllocationFailureCallback) { - rt->largeAllocationFailureCallback(rt->largeAllocationFailureCallbackData); - p = AllocateExecutableMemory(totalLength, ProtectionSetting::Writable); + if (OnLargeAllocationFailure) { + OnLargeAllocationFailure(); + p = AllocateExecutableMemory(codeLength, ProtectionSetting::Writable); } } @@ -84,6 +83,8 @@ AllocateCodeSegment(JSContext* cx, uint32_t totalLength) return nullptr; } + cx->zone()->updateJitCodeMallocBytes(codeLength); + wasmCodeAllocations++; return (uint8_t*)p; } @@ -110,11 +111,6 @@ StaticallyLink(CodeSegment& cs, const LinkData& linkData, ExclusiveContext* cx) PatchedImmPtr((void*)-1)); } } - - // These constants are logically part of the code: - - *(double*)(cs.globalData() + NaN64GlobalDataOffset) = GenericNaN(); - *(float*)(cs.globalData() + NaN32GlobalDataOffset) = GenericNaN(); } static void @@ -225,22 +221,20 @@ CodeSegment::create(JSContext* cx, HandleWasmMemoryObject memory) { MOZ_ASSERT(bytecode.length() % gc::SystemPageSize() == 0); - MOZ_ASSERT(linkData.globalDataLength % gc::SystemPageSize() == 0); MOZ_ASSERT(linkData.functionCodeLength < bytecode.length()); auto cs = cx->make_unique(); if (!cs) return nullptr; - cs->bytes_ = AllocateCodeSegment(cx, bytecode.length() + linkData.globalDataLength); + cs->bytes_ = AllocateCodeSegment(cx, bytecode.length()); if (!cs->bytes_) return nullptr; uint8_t* codeBase = cs->base(); - cs->functionCodeLength_ = linkData.functionCodeLength; - cs->codeLength_ = bytecode.length(); - cs->globalDataLength_ = linkData.globalDataLength; + cs->functionLength_ = linkData.functionCodeLength; + cs->length_ = bytecode.length(); cs->interruptCode_ = codeBase + linkData.interruptOffset; cs->outOfBoundsCode_ = codeBase + linkData.outOfBoundsOffset; cs->unalignedAccessCode_ = codeBase + linkData.unalignedAccessOffset; @@ -248,7 +242,7 @@ CodeSegment::create(JSContext* cx, { JitContext jcx(CompileRuntime::get(cx->compartment()->runtimeFromAnyThread())); AutoFlushICache afc("CodeSegment::create"); - AutoFlushICache::setRange(uintptr_t(codeBase), cs->codeLength()); + AutoFlushICache::setRange(uintptr_t(codeBase), cs->length()); memcpy(codeBase, bytecode.begin(), bytecode.length()); StaticallyLink(*cs, linkData, cx); @@ -256,7 +250,7 @@ CodeSegment::create(JSContext* cx, SpecializeToMemory(nullptr, *cs, metadata, memory->buffer()); } - if (!ExecutableAllocator::makeExecutable(codeBase, cs->codeLength())) { + if (!ExecutableAllocator::makeExecutable(codeBase, cs->length())) { ReportOutOfMemory(cx); return nullptr; } @@ -275,19 +269,19 @@ CodeSegment::~CodeSegment() MOZ_ASSERT(wasmCodeAllocations > 0); wasmCodeAllocations--; - MOZ_ASSERT(totalLength() > 0); + MOZ_ASSERT(length() > 0); // Match AllocateCodeSegment. - uint32_t size = JS_ROUNDUP(totalLength(), ExecutableCodePageSize); + uint32_t size = JS_ROUNDUP(length(), ExecutableCodePageSize); DeallocateExecutableMemory(bytes_, size); } void CodeSegment::onMovingGrow(uint8_t* prevMemoryBase, const Metadata& metadata, ArrayBufferObject& buffer) { - AutoWritableJitCode awjc(base(), codeLength()); + AutoWritableJitCode awjc(base(), length()); AutoFlushICache afc("CodeSegment::onMovingGrow"); - AutoFlushICache::setRange(uintptr_t(base()), codeLength()); + AutoFlushICache::setRange(uintptr_t(base()), length()); SpecializeToMemory(prevMemoryBase, *this, metadata, buffer); } @@ -813,9 +807,9 @@ Code::ensureProfilingState(JSContext* cx, bool newProfilingEnabled) profilingEnabled_ = newProfilingEnabled; { - AutoWritableJitCode awjc(cx->runtime(), segment_->base(), segment_->codeLength()); + AutoWritableJitCode awjc(segment_->base(), segment_->length()); AutoFlushICache afc("Code::ensureProfilingState"); - AutoFlushICache::setRange(uintptr_t(segment_->base()), segment_->codeLength()); + AutoFlushICache::setRange(uintptr_t(segment_->base()), segment_->length()); for (const CallSite& callSite : metadata_->callSites) ToggleProfiling(*this, callSite, newProfilingEnabled); @@ -864,9 +858,9 @@ Code::adjustEnterAndLeaveFrameTrapsState(JSContext* cx, bool enabled) if (wasEnabled == stillEnabled) return; - AutoWritableJitCode awjc(cx->runtime(), segment_->base(), segment_->codeLength()); + AutoWritableJitCode awjc(cx->runtime(), segment_->base(), segment_->length()); AutoFlushICache afc("Code::adjustEnterAndLeaveFrameTrapsState"); - AutoFlushICache::setRange(uintptr_t(segment_->base()), segment_->codeLength()); + AutoFlushICache::setRange(uintptr_t(segment_->base()), segment_->length()); for (const CallSite& callSite : metadata_->callSites) { if (callSite.kind() != CallSite::EnterFrame && callSite.kind() != CallSite::LeaveFrame) continue; @@ -881,9 +875,8 @@ Code::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code, size_t* data) const { - *code += segment_->codeLength(); + *code += segment_->length(); *data += mallocSizeOf(this) + - segment_->globalDataLength() + metadata_->sizeOfIncludingThisIfNotSeen(mallocSizeOf, seenMetadata); if (maybeBytecode_) diff --git a/js/src/wasm/WasmCode.h b/js/src/wasm/WasmCode.h index 7e5ad65993..abd0c661c1 100644 --- a/js/src/wasm/WasmCode.h +++ b/js/src/wasm/WasmCode.h @@ -35,24 +35,18 @@ struct Metadata; class FrameIterator; // A wasm CodeSegment owns the allocated executable code for a wasm module. -// This allocation also currently includes the global data segment, which allows -// RIP-relative access to global data on some architectures, but this will -// change in the future to give global data its own allocation. class CodeSegment; typedef UniquePtr UniqueCodeSegment; class CodeSegment { - // bytes_ points to a single allocation with two contiguous ranges: - // executable machine code in the range [0, codeLength) and global data in - // the range [codeLength, codeLength + globalDataLength). The range - // [0, functionCodeLength) is the subrange of [0, codeLength) which contains - // function code. + // bytes_ points to a single allocation of executable machine code in + // the range [0, length_). The range [0, functionLength_) is + // the subrange of [0, length_) which contains function code. uint8_t* bytes_; - uint32_t functionCodeLength_; - uint32_t codeLength_; - uint32_t globalDataLength_; + uint32_t functionLength_; + uint32_t length_; // These are pointers into code for stubs used for asynchronous // signal-handler control-flow transfer. @@ -80,10 +74,7 @@ class CodeSegment ~CodeSegment(); uint8_t* base() const { return bytes_; } - uint8_t* globalData() const { return bytes_ + codeLength_; } - uint32_t codeLength() const { return codeLength_; } - uint32_t globalDataLength() const { return globalDataLength_; } - uint32_t totalLength() const { return codeLength_ + globalDataLength_; } + uint32_t length() const { return length_; } uint8_t* interruptCode() const { return interruptCode_; } uint8_t* outOfBoundsCode() const { return outOfBoundsCode_; } @@ -96,10 +87,10 @@ class CodeSegment // enter/exit. bool containsFunctionPC(const void* pc) const { - return pc >= base() && pc < (base() + functionCodeLength_); + return pc >= base() && pc < (base() + functionLength_); } bool containsCodePC(const void* pc) const { - return pc >= base() && pc < (base() + codeLength_); + return pc >= base() && pc < (base() + length_); } // onMovingGrow must be called if the memory passed to 'create' performs a diff --git a/js/src/wasm/WasmGenerator.cpp b/js/src/wasm/WasmGenerator.cpp index e9a30e171a..a30933028b 100644 --- a/js/src/wasm/WasmGenerator.cpp +++ b/js/src/wasm/WasmGenerator.cpp @@ -196,7 +196,7 @@ ModuleGenerator::init(UniqueModuleEnvironment env, const CompileArgs& args, { env_ = Move(env); - linkData_.globalDataLength = AlignBytes(InitialGlobalDataBytes, sizeof(void*)); + linkData_.globalDataLength = 0; alwaysBaseline_ = args.alwaysBaseline; debugEnabled_ = args.debugEnabled; @@ -661,31 +661,6 @@ ModuleGenerator::finishLinkData(Bytes& code) if (!linkData_.internalLinks.append(inLink)) return false; } - -#if defined(JS_CODEGEN_X86) - // Global data accesses in x86 need to be patched with the absolute - // address of the global. Globals are allocated sequentially after the - // code section so we can just use an InternalLink. - for (GlobalAccess a : masm_.globalAccesses()) { - LinkData::InternalLink inLink(LinkData::InternalLink::RawPointer); - inLink.patchAtOffset = masm_.labelToPatchOffset(a.patchAt); - inLink.targetOffset = code.length() + a.globalDataOffset; - if (!linkData_.internalLinks.append(inLink)) - return false; - } -#elif defined(JS_CODEGEN_X64) - // Global data accesses on x64 use rip-relative addressing and thus we can - // patch here, now that we know the final codeLength. - for (GlobalAccess a : masm_.globalAccesses()) { - void* from = code.begin() + a.patchAt.offset(); - void* to = code.end() + a.globalDataOffset; - X86Encoding::SetRel32(from, to); - } -#else - // Global access is performed using the GlobalReg and requires no patching. - MOZ_ASSERT(masm_.globalAccesses().length() == 0); -#endif - return true; } diff --git a/js/src/wasm/WasmInstance.cpp b/js/src/wasm/WasmInstance.cpp index 1b94d943c0..8b2cc9fad7 100644 --- a/js/src/wasm/WasmInstance.cpp +++ b/js/src/wasm/WasmInstance.cpp @@ -106,22 +106,19 @@ js::wasm::ShutDownInstanceStaticData() const void** Instance::addressOfSigId(const SigIdDesc& sigId) const { - MOZ_ASSERT(sigId.globalDataOffset() >= InitialGlobalDataBytes); - return (const void**)(codeSegment().globalData() + sigId.globalDataOffset()); + return (const void**)(globalSegment().globalData() + sigId.globalDataOffset()); } FuncImportTls& Instance::funcImportTls(const FuncImport& fi) { - MOZ_ASSERT(fi.tlsDataOffset() >= InitialGlobalDataBytes); - return *(FuncImportTls*)(codeSegment().globalData() + fi.tlsDataOffset()); + return *(FuncImportTls*)(globalSegment().globalData() + fi.tlsDataOffset()); } TableTls& Instance::tableTls(const TableDesc& td) const { - MOZ_ASSERT(td.globalDataOffset >= InitialGlobalDataBytes); - return *(TableTls*)(codeSegment().globalData() + td.globalDataOffset); + return *(TableTls*)(globalSegment().globalData() + td.globalDataOffset); } bool @@ -293,7 +290,7 @@ Instance::growMemory_i32(Instance* instance, uint32_t delta) uint32_t ret = WasmMemoryObject::grow(memory, delta, cx); // If there has been a moving grow, this Instance should have been notified. - MOZ_RELEASE_ASSERT(instance->tlsData_.memoryBase == + MOZ_RELEASE_ASSERT(instance->tlsData()->memoryBase == instance->memory_->buffer().dataPointerEither()); return ret; @@ -310,6 +307,7 @@ Instance::currentMemory_i32(Instance* instance) Instance::Instance(JSContext* cx, Handle object, UniqueCode code, + UniqueGlobalSegment globals, HandleWasmMemoryObject memory, SharedTableVector&& tables, Handle funcImports, @@ -317,6 +315,7 @@ Instance::Instance(JSContext* cx, : compartment_(cx->compartment()), object_(object), code_(Move(code)), + globals_(Move(globals)), memory_(memory), tables_(Move(tables)), enterFrameTrapsEnabled_(false) @@ -324,11 +323,11 @@ Instance::Instance(JSContext* cx, MOZ_ASSERT(funcImports.length() == metadata().funcImports.length()); MOZ_ASSERT(tables_.length() == metadata().tables.length()); - tlsData_.cx = cx; - tlsData_.instance = this; - tlsData_.globalData = code_->segment().globalData(); - tlsData_.memoryBase = memory ? memory->buffer().dataPointerEither().unwrap() : nullptr; - tlsData_.stackLimit = *(void**)cx->stackLimitAddressForJitCode(StackForUntrustedScript); + tlsData()->cx = cx; + tlsData()->instance = this; + tlsData()->globalData = globals_->globalData(); + tlsData()->memoryBase = memory ? memory->buffer().dataPointerEither().unwrap() : nullptr; + tlsData()->stackLimit = *(void**)cx->stackLimitAddressForJitCode(JS::StackForUntrustedScript); for (size_t i = 0; i < metadata().funcImports.length(); i++) { HandleFunction f = funcImports[i]; @@ -338,12 +337,12 @@ Instance::Instance(JSContext* cx, WasmInstanceObject* calleeInstanceObj = ExportedFunctionToInstanceObject(f); const CodeRange& codeRange = calleeInstanceObj->getExportedFunctionCodeRange(f); Instance& calleeInstance = calleeInstanceObj->instance(); - import.tls = &calleeInstance.tlsData_; + import.tls = calleeInstance.tlsData(); import.code = calleeInstance.codeSegment().base() + codeRange.funcNonProfilingEntry(); import.baselineScript = nullptr; import.obj = calleeInstanceObj; } else { - import.tls = &tlsData_; + import.tls = tlsData(); import.code = codeBase() + fi.interpExitCodeOffset(); import.baselineScript = nullptr; import.obj = f; @@ -357,7 +356,7 @@ Instance::Instance(JSContext* cx, table.base = tables_[i]->base(); } - uint8_t* globalData = code_->segment().globalData(); + uint8_t* globalData = globals_->globalData(); for (size_t i = 0; i < metadata().globals.length(); i++) { const GlobalDesc& global = metadata().globals[i]; @@ -497,7 +496,7 @@ SharedMem Instance::memoryBase() const { MOZ_ASSERT(metadata().usesMemory()); - MOZ_ASSERT(tlsData_.memoryBase == memory_->buffer().dataPointerEither()); + MOZ_ASSERT(tlsData()->memoryBase == memory_->buffer().dataPointerEither()); return memory_->buffer().dataPointerEither(); } @@ -523,7 +522,7 @@ bool Instance::callExport(JSContext* cx, uint32_t funcIndex, CallArgs args) { // If there has been a moving grow, this Instance should have been notified. - MOZ_RELEASE_ASSERT(!memory_ || tlsData_.memoryBase == memory_->buffer().dataPointerEither()); + MOZ_RELEASE_ASSERT(!memory_ || tlsData()->memoryBase == memory_->buffer().dataPointerEither()); if (!cx->compartment()->wasm.ensureProfilingState(cx)) return false; @@ -590,7 +589,7 @@ Instance::callExport(JSContext* cx, uint32_t funcIndex, CallArgs args) // Call the per-exported-function trampoline created by GenerateEntry. auto funcPtr = JS_DATA_TO_FUNC_PTR(ExportFuncPtr, codeBase() + func.entryOffset()); - if (!CALL_GENERATED_2(funcPtr, exportArgs.begin(), &tlsData_)) + if (!CALL_GENERATED_2(funcPtr, exportArgs.begin(), tlsData())) return false; } @@ -657,7 +656,7 @@ Instance::onMovingGrowMemory(uint8_t* prevMemoryBase) { MOZ_ASSERT(!isAsmJS()); ArrayBufferObject& buffer = memory_->buffer().as(); - tlsData_.memoryBase = buffer.dataPointer(); + tlsData()->memoryBase = buffer.dataPointer(); code_->segment().onMovingGrow(prevMemoryBase, metadata(), buffer); } @@ -753,10 +752,48 @@ Instance::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code, size_t* data) const { - *data += mallocSizeOf(this); + *data += mallocSizeOf(this) + globals_->sizeOfMisc(mallocSizeOf); code_->addSizeOfMisc(mallocSizeOf, seenMetadata, seenBytes, code, data); for (const SharedTable& table : tables_) *data += table->sizeOfIncludingThisIfNotSeen(mallocSizeOf, seenTables); } + +/* static */ UniqueGlobalSegment +GlobalSegment::create(uint32_t globalDataLength) +{ + MOZ_ASSERT(globalDataLength % gc::SystemPageSize() == 0); + + auto gs = MakeUnique(); + if (!gs) + return nullptr; + + TlsData* tlsData = + reinterpret_cast(js_calloc(offsetof(TlsData, globalArea) + globalDataLength)); + if (!tlsData) + return nullptr; + +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // We will emit SIMD memory accesses that require 16-byte alignment. + MOZ_RELEASE_ASSERT((uintptr_t(tlsData) % 16) == 0); +#endif + + gs->tlsData_ = tlsData; + gs->globalDataLength_ = globalDataLength; + + return gs; +} + +GlobalSegment::~GlobalSegment() +{ + js_free(tlsData_); +} + +size_t +GlobalSegment::sizeOfMisc(MallocSizeOf mallocSizeOf) const +{ + // Note, once the GlobalSegment is shared among instances, we will have to + // take that sharing into account. + return mallocSizeOf(this) + mallocSizeOf(tlsData_); +} diff --git a/js/src/wasm/WasmInstance.h b/js/src/wasm/WasmInstance.h index a17c6eaf33..f5c294e4e0 100644 --- a/js/src/wasm/WasmInstance.h +++ b/js/src/wasm/WasmInstance.h @@ -25,6 +25,35 @@ namespace js { namespace wasm { +// A wasm GlobalSegment owns the allocated global data for a wasm module. A +// module may be compiled multiple times (at multiple tiers) but the compiled +// representations share the same GlobalSegment. + +class GlobalSegment +{ + uint32_t globalDataLength_; + TlsData* tlsData_; + + GlobalSegment(const GlobalSegment&) = delete; + GlobalSegment(GlobalSegment&&) = delete; + void operator=(const GlobalSegment&) = delete; + void operator=(GlobalSegment&&) = delete; + + public: + static UniquePtr create(uint32_t globalDataLength); + + GlobalSegment() { PodZero(this); } + ~GlobalSegment(); + + TlsData* tlsData() const { return tlsData_; } + uint8_t* globalData() const { return (uint8_t*)&tlsData_->globalArea; } + uint32_t globalDataLength() const { return globalDataLength_; } + + size_t sizeOfMisc(MallocSizeOf mallocSizeOf) const; +}; + +typedef UniquePtr UniqueGlobalSegment; + // Instance represents a wasm instance and provides all the support for runtime // execution of code in the instance. Instances share various immutable data // structures with the Module from which they were instantiated and other @@ -37,9 +66,9 @@ class Instance JSCompartment* const compartment_; ReadBarrieredWasmInstanceObject object_; const UniqueCode code_; + const UniqueGlobalSegment globals_; GCPtrWasmMemoryObject memory_; SharedTableVector tables_; - TlsData tlsData_; bool enterFrameTrapsEnabled_; // Internal helpers: @@ -66,6 +95,7 @@ class Instance Instance(JSContext* cx, HandleWasmInstanceObject object, UniqueCode code, + UniqueGlobalSegment globals, HandleWasmMemoryObject memory, SharedTableVector&& tables, Handle funcImports, @@ -74,11 +104,12 @@ class Instance bool init(JSContext* cx); void trace(JSTracer* trc); - JSContext* cx() const { return tlsData_.cx; } + JSContext* cx() const { return tlsData()->cx; } JSCompartment* compartment() const { return compartment_; } Code& code() { return *code_; } const Code& code() const { return *code_; } const CodeSegment& codeSegment() const { return code_->segment(); } + const GlobalSegment& globalSegment() const { return *globals_; } uint8_t* codeBase() const { return code_->segment().base(); } const Metadata& metadata() const { return code_->metadata(); } bool isAsmJS() const { return metadata().isAsmJS(); } @@ -87,7 +118,7 @@ class Instance size_t memoryLength() const; size_t memoryMappedSize() const; bool memoryAccessInGuardRegion(uint8_t* addr, unsigned numBytes) const; - TlsData& tlsData() { return tlsData_; } + TlsData* tlsData() const { return globals_->tlsData(); } // This method returns a pointer to the GC object that owns this Instance. // Instances may be reached via weak edges (e.g., Compartment::instances_) diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp index 866cdc993f..e25cbefa43 100644 --- a/js/src/wasm/WasmIonCompile.cpp +++ b/js/src/wasm/WasmIonCompile.cpp @@ -635,7 +635,7 @@ class FunctionCompiler if (inDeadCode()) return nullptr; - auto* load = MWasmLoadGlobalVar::New(alloc(), type, globalDataOffset, isConst); + auto* load = MWasmLoadGlobalVar::New(alloc(), type, globalDataOffset, isConst, tlsPointer_); curBlock_->add(load); return load; } @@ -644,7 +644,7 @@ class FunctionCompiler { if (inDeadCode()) return; - curBlock_->add(MWasmStoreGlobalVar::New(alloc(), globalDataOffset, v)); + curBlock_->add(MWasmStoreGlobalVar::New(alloc(), globalDataOffset, v, tlsPointer_)); } void addInterruptCheck() diff --git a/js/src/wasm/WasmJS.cpp b/js/src/wasm/WasmJS.cpp index b853dd7e91..52addf774b 100644 --- a/js/src/wasm/WasmJS.cpp +++ b/js/src/wasm/WasmJS.cpp @@ -919,6 +919,7 @@ WasmInstanceObject::trace(JSTracer* trc, JSObject* obj) /* static */ WasmInstanceObject* WasmInstanceObject::create(JSContext* cx, UniqueCode code, + UniqueGlobalSegment globals, HandleWasmMemoryObject memory, SharedTableVector&& tables, Handle funcImports, @@ -953,6 +954,7 @@ WasmInstanceObject::create(JSContext* cx, auto* instance = cx->new_(cx, obj, Move(code), + Move(globals), memory, Move(tables), funcImports, diff --git a/js/src/wasm/WasmJS.h b/js/src/wasm/WasmJS.h index eac771f201..7e19136c43 100644 --- a/js/src/wasm/WasmJS.h +++ b/js/src/wasm/WasmJS.h @@ -184,6 +184,7 @@ class WasmInstanceObject : public NativeObject static WasmInstanceObject* create(JSContext* cx, UniquePtr code, + UniquePtr globals, HandleWasmMemoryObject memory, Vector, 0, SystemAllocPolicy>&& tables, Handle funcImports, diff --git a/js/src/wasm/WasmModule.cpp b/js/src/wasm/WasmModule.cpp index afd950eb94..4bf54fbdbe 100644 --- a/js/src/wasm/WasmModule.cpp +++ b/js/src/wasm/WasmModule.cpp @@ -896,6 +896,12 @@ Module::instantiate(JSContext* cx, return false; } + auto globalSegment = GlobalSegment::create(linkData_.globalDataLength); + if (!globalSegment) { + ReportOutOfMemory(cx); + return false; + } + auto code = cx->make_unique(Move(codeSegment), *metadata_, maybeBytecode); if (!code) { ReportOutOfMemory(cx); @@ -904,6 +910,7 @@ Module::instantiate(JSContext* cx, instance.set(WasmInstanceObject::create(cx, Move(code), + Move(globalSegment), memory, Move(tables), funcImports, diff --git a/js/src/wasm/WasmStubs.cpp b/js/src/wasm/WasmStubs.cpp index 60fa8f020f..cbe305a105 100644 --- a/js/src/wasm/WasmStubs.cpp +++ b/js/src/wasm/WasmStubs.cpp @@ -570,11 +570,6 @@ wasm::GenerateImportInterpExit(MacroAssembler& masm, const FuncImport& fi, uint3 defined(JS_CODEGEN_LOONGARCH64) MOZ_ASSERT(NonVolatileRegs.has(HeapReg)); #endif -#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ - defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ - defined(JS_CODEGEN_LOONGARCH64) - MOZ_ASSERT(NonVolatileRegs.has(GlobalReg)); -#endif GenerateExitEpilogue(masm, framePushed, ExitReason::ImportInterp, &offsets); diff --git a/js/src/wasm/WasmTable.cpp b/js/src/wasm/WasmTable.cpp index d41055b704..c0ae73d042 100644 --- a/js/src/wasm/WasmTable.cpp +++ b/js/src/wasm/WasmTable.cpp @@ -117,7 +117,7 @@ Table::set(uint32_t index, void* code, Instance& instance) JSObject::writeBarrierPre(elem.tls->instance->objectUnbarriered()); elem.code = code; - elem.tls = &instance.tlsData(); + elem.tls = instance.tlsData(); MOZ_ASSERT(elem.tls->instance->objectUnbarriered()->isTenured(), "no writeBarrierPost"); } else { diff --git a/js/src/wasm/WasmTypes.h b/js/src/wasm/WasmTypes.h index 4ab6e0c734..6676b92c52 100644 --- a/js/src/wasm/WasmTypes.h +++ b/js/src/wasm/WasmTypes.h @@ -19,6 +19,7 @@ #ifndef wasm_types_h #define wasm_types_h +#include "mozilla/Alignment.h" #include "mozilla/EnumeratedArray.h" #include "mozilla/HashFunctions.h" #include "mozilla/Maybe.h" @@ -86,6 +87,7 @@ typedef Vector Bytes; class Code; class CodeRange; +class GlobalSegment; class Memory; class Module; class Instance; @@ -996,8 +998,15 @@ struct TlsData // stack pointer in the prologue of functions that allocate stack space. See // `CodeGenerator::generateWasm`. void* stackLimit; + + // The globalArea must be the last field. Globals for the module start here + // and are inline in this structure. 16-byte alignment is required for SIMD + // data. + MOZ_ALIGNED_DECL(char globalArea, 16); }; +static_assert(offsetof(TlsData, globalArea) % 16 == 0, "aligned"); + typedef int32_t (*ExportFuncPtr)(ExportArg* args, TlsData* tls); // FuncImportTls describes the region of wasm global memory allocated in the