1339089 - Inline floor/ceil/trunc/nearest for floating-point values when we have adequate SSE support

1339089: Inline floor/ceil/trunc/nearest in Ion when we have sse4.

1339089: Inline wasm::floor/trunc/nearest/round in the baseline compiler too;
This commit is contained in:
win7-7 2025-12-30 13:45:26 +02:00 committed by wuggy
commit 0b212e1fe4
20 changed files with 379 additions and 50 deletions

View file

@ -806,6 +806,14 @@ enum class BarrierKind : uint32_t {
enum ReprotectCode { Reprotect = true, DontReprotect = false };
// Rounding modes for round instructions.
enum class RoundingMode {
Down,
Up,
NearestTiesToEven,
TowardsZero
};
} // namespace jit
} // namespace js

View file

@ -1405,6 +1405,22 @@ LIRGenerator::visitRound(MRound* ins)
define(lir, ins);
}
void
LIRGenerator::visitNearbyInt(MNearbyInt* ins)
{
MIRType inputType = ins->input()->type();
MOZ_ASSERT(IsFloatingPointType(inputType));
MOZ_ASSERT(ins->type() == inputType);
LInstructionHelper<1, 1, 0>* lir;
if (inputType == MIRType::Double)
lir = new(alloc()) LNearbyInt(useRegisterAtStart(ins->input()));
else
lir = new(alloc()) LNearbyIntF(useRegisterAtStart(ins->input()));
define(lir, ins);
}
void
LIRGenerator::visitMinMax(MMinMax* ins)
{

View file

@ -137,6 +137,7 @@ class LIRGenerator : public LIRGeneratorSpecific
void visitFloor(MFloor* ins);
void visitCeil(MCeil* ins);
void visitRound(MRound* ins);
void visitNearbyInt(MNearbyInt* ins);
void visitMinMax(MMinMax* ins);
void visitAbs(MAbs* ins);
void visitClz(MClz* ins);

View file

@ -923,20 +923,30 @@ IonBuilder::inlineMathFloor(CallInfo& callInfo)
return InliningStatus_Inlined;
}
if (IsFloatingPointType(argType) && returnType == MIRType::Int32) {
callInfo.setImplicitlyUsedUnchecked();
MFloor* ins = MFloor::New(alloc(), callInfo.getArg(0));
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;
}
if (IsFloatingPointType(argType)) {
if (returnType == MIRType::Int32) {
callInfo.setImplicitlyUsedUnchecked();
MFloor* ins = MFloor::New(alloc(), callInfo.getArg(0));
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;
}
if (IsFloatingPointType(argType) && returnType == MIRType::Double) {
callInfo.setImplicitlyUsedUnchecked();
MMathFunction* ins = MMathFunction::New(alloc(), callInfo.getArg(0), MMathFunction::Floor, nullptr);
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;
if (returnType == MIRType::Double) {
callInfo.setImplicitlyUsedUnchecked();
MInstruction* ins = nullptr;
if (MNearbyInt::HasAssemblerSupport(RoundingMode::Down)) {
ins = MNearbyInt::New(alloc(), callInfo.getArg(0), argType, RoundingMode::Down);
} else {
ins = MMathFunction::New(alloc(), callInfo.getArg(0), MMathFunction::Floor,
/* cache */ nullptr);
}
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;
}
}
return InliningStatus_NotInlined;
@ -967,20 +977,30 @@ IonBuilder::inlineMathCeil(CallInfo& callInfo)
return InliningStatus_Inlined;
}
if (IsFloatingPointType(argType) && returnType == MIRType::Int32) {
callInfo.setImplicitlyUsedUnchecked();
MCeil* ins = MCeil::New(alloc(), callInfo.getArg(0));
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;
}
if (IsFloatingPointType(argType)) {
if (returnType == MIRType::Int32) {
callInfo.setImplicitlyUsedUnchecked();
MCeil* ins = MCeil::New(alloc(), callInfo.getArg(0));
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;
}
if (IsFloatingPointType(argType) && returnType == MIRType::Double) {
callInfo.setImplicitlyUsedUnchecked();
MMathFunction* ins = MMathFunction::New(alloc(), callInfo.getArg(0), MMathFunction::Ceil, nullptr);
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;
if (returnType == MIRType::Double) {
callInfo.setImplicitlyUsedUnchecked();
MInstruction* ins = nullptr;
if (MNearbyInt::HasAssemblerSupport(RoundingMode::Up)) {
ins = MNearbyInt::New(alloc(), callInfo.getArg(0), argType, RoundingMode::Up);
} else {
ins = MMathFunction::New(alloc(), callInfo.getArg(0), MMathFunction::Ceil,
/* cache */ nullptr);
}
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;
}
}
return InliningStatus_NotInlined;
@ -1045,7 +1065,8 @@ IonBuilder::inlineMathRound(CallInfo& callInfo)
if (IsFloatingPointType(argType) && returnType == MIRType::Double) {
callInfo.setImplicitlyUsedUnchecked();
MMathFunction* ins = MMathFunction::New(alloc(), callInfo.getArg(0), MMathFunction::Round, nullptr);
MMathFunction* ins = MMathFunction::New(alloc(), callInfo.getArg(0), MMathFunction::Round,
/* cache */ nullptr);
current->add(ins);
current->push(ins);
return InliningStatus_Inlined;

View file

@ -1272,6 +1272,18 @@ MAssertRange::printOpcode(GenericPrinter& out) const
assertedRange()->dump(out);
}
void MNearbyInt::printOpcode(GenericPrinter& out) const
{
MDefinition::printOpcode(out);
const char* roundingModeStr = nullptr;
switch (roundingMode_) {
case RoundingMode::Up: roundingModeStr = "(up)"; break;
case RoundingMode::Down: roundingModeStr = "(down)"; break;
case RoundingMode::NearestTiesToEven: roundingModeStr = "(nearest ties even)"; break;
case RoundingMode::TowardsZero: roundingModeStr = "(towards zero)"; break;
}
out.printf(" %s", roundingModeStr);
}
const char*
MMathFunction::FunctionName(Function function)
{
@ -1647,6 +1659,12 @@ MRound::trySpecializeFloat32(TempAllocator& alloc)
specialization_ = MIRType::Float32;
}
void
MNearbyInt::trySpecializeFloat32(TempAllocator& alloc)
{
if (EnsureFloatInputOrConvert(this, alloc))
specialization_ = MIRType::Float32;
}
MTableSwitch*
MTableSwitch::New(TempAllocator& alloc, MDefinition* ins, int32_t low, int32_t high)
{

View file

@ -11064,7 +11064,7 @@ class MStringLength
ALLOW_CLONE(MStringLength)
};
// Inlined version of Math.floor().
// Inlined assembly for Math.floor(double | float32) -> int32.
class MFloor
: public MUnaryInstruction,
public FloatingPointPolicy<0>::Data
@ -11105,7 +11105,7 @@ class MFloor
ALLOW_CLONE(MFloor)
};
// Inlined version of Math.ceil().
// Inlined version of Math.round(double | float32) -> int32.
class MCeil
: public MUnaryInstruction,
public FloatingPointPolicy<0>::Data
@ -11146,7 +11146,7 @@ class MCeil
ALLOW_CLONE(MCeil)
};
// Inlined version of Math.round().
// Inlined version of Math.round(double | float32) -> int32.
class MRound
: public MUnaryInstruction,
public FloatingPointPolicy<0>::Data
@ -11188,6 +11188,62 @@ class MRound
ALLOW_CLONE(MRound)
};
// NearbyInt rounds the floating-point input to the nearest integer, according
// to the RoundingMode.
class MNearbyInt
: public MUnaryInstruction,
public FloatingPointPolicy<0>::Data
{
RoundingMode roundingMode_;
explicit MNearbyInt(MDefinition* num, MIRType resultType, RoundingMode roundingMode)
: MUnaryInstruction(classOpcode, num),
roundingMode_(roundingMode)
{
MOZ_ASSERT(HasAssemblerSupport(roundingMode));
MOZ_ASSERT(IsFloatingPointType(resultType));
setResultType(resultType);
specialization_ = resultType;
setMovable();
}
public:
INSTRUCTION_HEADER(NearbyInt)
TRIVIAL_NEW_WRAPPERS
static bool HasAssemblerSupport(RoundingMode mode) {
return Assembler::HasRoundInstruction(mode);
}
RoundingMode roundingMode() const { return roundingMode_; }
AliasSet getAliasSet() const override {
return AliasSet::None();
}
bool isFloat32Commutative() const override {
return true;
}
void trySpecializeFloat32(TempAllocator& alloc) override;
#ifdef DEBUG
bool isConsistentFloat32Use(MUse* use) const override {
return true;
}
#endif
bool congruentTo(const MDefinition* ins) const override {
return congruentIfOperandsEqual(ins) &&
ins->toNearbyInt()->roundingMode() == roundingMode_;
}
void printOpcode(GenericPrinter& out) const override;
ALLOW_CLONE(MNearbyInt)
};
class MIteratorStart
: public MUnaryInstruction,
public BoxExceptPolicy<0, MIRType::Object>::Data

View file

@ -242,7 +242,9 @@ namespace jit {
_(Floor) \
_(Ceil) \
_(Round) \
_(In) \
_(NearbyInt) \
_(InCache) \
_(HasOwnCache) \
_(InstanceOf) \
_(CallInstanceOf) \
_(InterruptCheck) \

View file

@ -1725,6 +1725,8 @@ class Assembler : public AssemblerShared
return js::jit::SupportsSimd;
}
static bool HasRoundInstruction(RoundingMode mode) { return false; }
protected:
void addPendingJump(BufferOffset src, ImmPtr target, Relocation::Kind kind) {
enoughMemory_ &= jumps_.append(RelativePatch(target.value, kind));

View file

@ -290,6 +290,8 @@ class Assembler : public vixl::Assembler
static bool SupportsUnalignedAccesses() { return true; }
static bool SupportsSimd() { return js::jit::SupportsSimd; }
static bool HasRoundInstruction(RoundingMode mode) { return false; }
// Tracks a jump that is patchable after finalization.
void addJumpRelocation(BufferOffset src, Relocation::Kind reloc);

View file

@ -1247,6 +1247,10 @@ class AssemblerMIPSShared : public AssemblerShared
return js::jit::SupportsSimd;
}
static bool HasRoundInstruction(RoundingMode mode) {
return false;
}
protected:
InstImm invertBranch(InstImm branch, BOffImm16 skipOffset);
void addPendingJump(BufferOffset src, ImmPtr target, Relocation::Kind kind) {

View file

@ -192,6 +192,8 @@ class MacroAssemblerNone : public Assembler
static bool SupportsSimd() { return false; }
static bool SupportsUnalignedAccesses() { return false; }
static bool HasRoundInstruction(RoundingMode) { return false; }
void executableCopy(void*) { MOZ_CRASH(); }
void copyJumpRelocationTable(uint8_t*) { MOZ_CRASH(); }
void copyDataRelocationTable(uint8_t*) { MOZ_CRASH(); }

View file

@ -6260,7 +6260,8 @@ class LStringLength : public LInstructionHelper<1, 1, 0>
}
};
// Take the floor of a double precision number. Implements Math.floor().
// Take the floor of a double precision number and converts it to an int32.
// Implements Math.floor().
class LFloor : public LInstructionHelper<1, 1, 0>
{
public:
@ -6271,7 +6272,9 @@ class LFloor : public LInstructionHelper<1, 1, 0>
}
};
// Take the floor of a single precision number. Implements Math.floor().
// Take the floor of a single precision number and converts it to an int32.
// Implements Math.floor().
class LFloorF : public LInstructionHelper<1, 1, 0>
{
public:
@ -6282,7 +6285,8 @@ class LFloorF : public LInstructionHelper<1, 1, 0>
}
};
// Take the ceiling of a double precision number. Implements Math.ceil().
// Take the ceiling of a double precision number and converts it to an int32.
// Implements Math.ceil().
class LCeil : public LInstructionHelper<1, 1, 0>
{
public:
@ -6293,7 +6297,8 @@ class LCeil : public LInstructionHelper<1, 1, 0>
}
};
// Take the ceiling of a single precision number. Implements Math.ceil().
// Take the ceiling of a single precision number and converts it to an int32.
// Implements Math.ceil().
class LCeilF : public LInstructionHelper<1, 1, 0>
{
public:
@ -6304,7 +6309,8 @@ class LCeilF : public LInstructionHelper<1, 1, 0>
}
};
// Round a double precision number. Implements Math.round().
// Round a double precision number and converts it to an int32.
// Implements Math.round().
class LRound : public LInstructionHelper<1, 1, 1>
{
public:
@ -6323,7 +6329,8 @@ class LRound : public LInstructionHelper<1, 1, 1>
}
};
// Round a single precision number. Implements Math.round().
// Round a single precision number and converts it to an int32.
// Implements Math.round().
class LRoundF : public LInstructionHelper<1, 1, 1>
{
public:
@ -6342,6 +6349,36 @@ class LRoundF : public LInstructionHelper<1, 1, 1>
}
};
// Rounds a double precision number accordingly to mir()->roundingMode(),
// and keeps a double output.
class LNearbyInt : public LInstructionHelper<1, 1, 0>
{
public:
LIR_HEADER(NearbyInt)
explicit LNearbyInt(const LAllocation& num) {
setOperand(0, num);
}
MNearbyInt* mir() const {
return mir_->toNearbyInt();
}
};
// Rounds a single precision number accordingly to mir()->roundingMode(),
// and keeps a single output.
class LNearbyIntF : public LInstructionHelper<1, 1, 0>
{
public:
LIR_HEADER(NearbyIntF)
explicit LNearbyIntF(const LAllocation& num) {
setOperand(0, num);
}
MNearbyInt* mir() const {
return mir_->toNearbyInt();
}
};
// Load a function's call environment.
class LFunctionEnvironment : public LInstructionHelper<1, 1, 0>
{

View file

@ -329,7 +329,9 @@
_(CeilF) \
_(Round) \
_(RoundF) \
_(In) \
_(NearbyInt) \
_(NearbyIntF) \
_(InCache) \
_(InArray) \
_(InstanceOfO) \
_(InstanceOfV) \

View file

@ -1111,6 +1111,17 @@ class AssemblerX86Shared : public AssemblerShared
static bool SupportsSimd() { return CPUInfo::IsSSE2Present(); }
static bool HasAVX() { return CPUInfo::IsAVXPresent(); }
static bool HasRoundInstruction(RoundingMode mode) {
switch (mode) {
case RoundingMode::Up:
case RoundingMode::Down:
case RoundingMode::NearestTiesToEven:
case RoundingMode::TowardsZero:
return CPUInfo::IsSSE41Present();
}
MOZ_CRASH("unexpected mode");
}
void cmpl(Register rhs, Register lhs) {
masm.cmpl_rr(rhs.encoding(), lhs.encoding());
}
@ -3349,6 +3360,22 @@ class AssemblerX86Shared : public AssemblerShared
MOZ_ASSERT(HasSSE2());
masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
static X86Encoding::RoundingMode
ToX86RoundingMode(RoundingMode mode) {
switch (mode) {
case RoundingMode::Up:
return X86Encoding::RoundUp;
case RoundingMode::Down:
return X86Encoding::RoundDown;
case RoundingMode::NearestTiesToEven:
return X86Encoding::RoundToNearest;
case RoundingMode::TowardsZero:
return X86Encoding::RoundToZero;
}
MOZ_CRASH("unexpected mode");
}
void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vroundsd_irr(mode, src1.encoding(), src0.encoding(), dest.encoding());

View file

@ -2330,6 +2330,26 @@ CodeGeneratorX86Shared::visitRoundF(LRoundF* lir)
masm.bind(&end);
}
void
CodeGeneratorX86Shared::visitNearbyInt(LNearbyInt* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
FloatRegister output = ToFloatRegister(lir->output());
RoundingMode roundingMode = lir->mir()->roundingMode();
masm.vroundsd(Assembler::ToX86RoundingMode(roundingMode), input, output, output);
}
void
CodeGeneratorX86Shared::visitNearbyIntF(LNearbyIntF* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
FloatRegister output = ToFloatRegister(lir->output());
RoundingMode roundingMode = lir->mir()->roundingMode();
masm.vroundss(Assembler::ToX86RoundingMode(roundingMode), input, output, output);
}
void
CodeGeneratorX86Shared::visitGuardShape(LGuardShape* guard)
{

View file

@ -204,6 +204,8 @@ class CodeGeneratorX86Shared : public CodeGeneratorShared
virtual void visitCeilF(LCeilF* lir);
virtual void visitRound(LRound* lir);
virtual void visitRoundF(LRoundF* lir);
virtual void visitNearbyInt(LNearbyInt* lir);
virtual void visitNearbyIntF(LNearbyIntF* lir);
virtual void visitGuardShape(LGuardShape* guard);
virtual void visitGuardObjectGroup(LGuardObjectGroup* guard);
virtual void visitGuardClass(LGuardClass* guard);

View file

@ -3777,6 +3777,36 @@ ScratchI32 tmp(*this);
return true;
}
[[nodiscard]] bool
supportsRoundInstruction(RoundingMode mode)
{
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
return Assembler::HasRoundInstruction(mode);
#else
return false;
#endif
}
void
roundF32(RoundingMode roundingMode, RegF32 f0)
{
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
masm.vroundss(Assembler::ToX86RoundingMode(roundingMode), f0, f0, f0);
#else
MOZ_CRASH("NYI");
#endif
}
void
roundF64(RoundingMode roundingMode, RegF64 f0)
{
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
masm.vroundsd(Assembler::ToX86RoundingMode(roundingMode), f0, f0, f0);
#else
MOZ_CRASH("NYI");
#endif
}
////////////////////////////////////////////////////////////
// Generally speaking, ABOVE this point there should be no value
@ -4028,13 +4058,7 @@ ScratchI32 tmp(*this);
#endif
void emitReinterpretI32AsF32();
void emitReinterpretI64AsF64();
#if defined(JS_CODEGEN_LOONGARCH64)
[[nodiscard]] bool emitAtomicLoad();
[[nodiscard]] bool emitAtomicStore();
[[nodiscard]] bool emitAtomicBinOp();
[[nodiscard]] bool emitAtomicCompareExchange();
[[nodiscard]] bool emitAtomicExchange();
#endif
void emitRound(RoundingMode roundingMode, ValType operandType);
[[nodiscard]] bool emitGrowMemory();
[[nodiscard]] bool emitCurrentMemory();
};
@ -6069,6 +6093,22 @@ BaseCompiler::emitCommonMathCall(uint32_t lineOrBytecode, SymbolicAddress callee
return true;
}
void
BaseCompiler::emitRound(RoundingMode roundingMode, ValType operandType)
{
if (operandType == ValType::F32) {
RegF32 f0 = popF32();
roundF32(roundingMode, f0);
pushF32(f0);
} else if (operandType == ValType::F64) {
RegF64 f0 = popF64();
roundF64(roundingMode, f0);
pushF64(f0);
} else {
MOZ_CRASH("unexpected type");
}
}
bool
BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee, ValType operandType)
{
@ -6077,6 +6117,12 @@ BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee, ValType operandTy
if (deadCode_)
return true;
RoundingMode roundingMode;
if (IsRoundingFunction(callee, &roundingMode) && supportsRoundInstruction(roundingMode)) {
emitRound(roundingMode, operandType);
return true;
}
return emitCommonMathCall(lineOrBytecode, callee,
operandType == ValType::F32 ? SigF_ : SigD_,
operandType == ValType::F32 ? ExprType::F32 : ExprType::F64);

View file

@ -385,6 +385,16 @@ class FunctionCompiler
return ins;
}
MDefinition* nearbyInt(MDefinition* input, RoundingMode roundingMode)
{
if (inDeadCode())
return nullptr;
auto* ins = MNearbyInt::New(alloc(), input, input->type(), roundingMode);
curBlock_->add(ins);
return ins;
}
MDefinition* minMax(MDefinition* lhs, MDefinition* rhs, MIRType type, bool isMax) {
if (inDeadCode())
return nullptr;
@ -2188,19 +2198,41 @@ EmitTeeStoreWithCoercion(FunctionCompiler& f, ValType resultType, Scalar::Type v
return true;
}
static bool
TryInlineUnaryBuiltin(FunctionCompiler& f, SymbolicAddress callee, MDefinition* input)
{
if (!input)
return false;
MOZ_ASSERT(IsFloatingPointType(input->type()));
RoundingMode mode;
if (!IsRoundingFunction(callee, &mode))
return false;
if (!MNearbyInt::HasAssemblerSupport(mode))
return false;
f.iter().setResult(f.nearbyInt(input, mode));
return true;
}
static bool
EmitUnaryMathBuiltinCall(FunctionCompiler& f, SymbolicAddress callee, ValType operandType)
{
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
CallCompileState call(f, lineOrBytecode);
if (!f.startCall(&call))
return false;
MDefinition* input;
if (!f.iter().readUnary(operandType, &input))
return false;
if (TryInlineUnaryBuiltin(f, callee, input))
return true;
CallCompileState call(f, lineOrBytecode);
if (!f.startCall(&call))
return false;
if (!f.passArg(input, operandType, &call))
return false;

View file

@ -329,6 +329,31 @@ FuncCast(F* pf, ABIFunctionType type)
return pv;
}
bool
wasm::IsRoundingFunction(SymbolicAddress callee, jit::RoundingMode* mode)
{
switch (callee) {
case SymbolicAddress::FloorD:
case SymbolicAddress::FloorF:
*mode = jit::RoundingMode::Down;
return true;
case SymbolicAddress::CeilD:
case SymbolicAddress::CeilF:
*mode = jit::RoundingMode::Up;
return true;
case SymbolicAddress::TruncD:
case SymbolicAddress::TruncF:
*mode = jit::RoundingMode::TowardsZero;
return true;
case SymbolicAddress::NearbyIntD:
case SymbolicAddress::NearbyIntF:
*mode = jit::RoundingMode::NearestTiesToEven;
return true;
default:
return false;
}
}
void*
wasm::AddressOf(SymbolicAddress imm, ExclusiveContext* cx)
{

View file

@ -41,7 +41,10 @@
namespace js {
class PropertyName;
namespace jit { struct BaselineScript; }
namespace jit {
struct BaselineScript;
enum class RoundingMode;
}
// This is a widespread header, so lets keep out the core wasm impl types.
@ -885,6 +888,9 @@ enum class SymbolicAddress
Limit
};
bool
IsRoundingFunction(SymbolicAddress callee, jit::RoundingMode* mode);
void*
AddressOf(SymbolicAddress imm, ExclusiveContext* cx);