void CodeGen::genFloatMath(GenTree *tree, RegSet::RegisterPreference *pref)
{
assert(tree->OperGet() == GT_INTRINSIC);
GenTreePtr op1 = tree->gtOp.gtOp1;
// get tree into a register
genCodeForTreeFloat(op1, pref);
instruction ins;
switch (tree->gtIntrinsic.gtIntrinsicId)
{
case CORINFO_INTRINSIC_Sin:
ins = INS_invalid;
break;
case CORINFO_INTRINSIC_Cos:
ins = INS_invalid;
break;
case CORINFO_INTRINSIC_Sqrt:
ins = INS_vsqrt;
break;
case CORINFO_INTRINSIC_Abs:
ins = INS_vabs;
break;
case CORINFO_INTRINSIC_Round:
{
regNumber reg = regSet.PickRegFloat(tree->TypeGet(), pref);
genMarkTreeInReg(tree, reg);
// convert it to a long and back
inst_RV_RV(ins_FloatConv(TYP_LONG,tree->TypeGet()),
reg, op1->gtRegNum, tree->TypeGet());
inst_RV_RV(ins_FloatConv(tree->TypeGet(), TYP_LONG), reg, reg);
genCodeForTreeFloat_DONE(tree, op1->gtRegNum);
return;
}
break;
default:
unreached();
}
if (ins != INS_invalid)
{
regNumber reg = regSet.PickRegFloat(tree->TypeGet(), pref);
genMarkTreeInReg(tree, reg);
inst_RV_RV(ins, reg, op1->gtRegNum, tree->TypeGet());
// mark register that holds tree
genCodeForTreeFloat_DONE(tree, reg);
}
else
{
unreached();
// If unreached is removed, mark register that holds tree
// genCodeForTreeFloat_DONE(tree, op1->gtRegNum);
}
return;
}
regNumber CodeGen::genAssignArithFloat(genTreeOps oper,
GenTreePtr dst, regNumber dstreg,
GenTreePtr src, regNumber srcreg)
{
regNumber result;
// dst should be a regvar or memory
if (dst->IsRegVar())
{
regNumber reg = dst->gtRegNum;
if (src->IsRegVar())
{
inst_RV_RV(ins_MathOp(oper, dst->gtType), reg, src->gtRegNum, dst->gtType);
}
else
{
inst_RV_TT(ins_MathOp(oper, dst->gtType), reg, src, 0, EmitSize(dst));
}
result = reg;
}
else // dst in memory
{
// since this is an asgop the ACTUAL destination is memory
// but it is also one of the sources and SSE ops do not allow mem dests
// so we have loaded it into a reg, and that is what dstreg represents
assert(dstreg != REG_NA);
if ( (src->InReg()))
{
inst_RV_RV(ins_MathOp(oper, dst->gtType), dstreg, src->gtRegNum, dst->gtType);
}
else
{
//mem mem operation
inst_RV_TT(ins_MathOp(oper, dst->gtType), dstreg, src, 0, EmitSize(dst));
}
dst->gtFlags &= ~GTF_REG_VAL; // ???
inst_TT_RV(ins_FloatStore(dst->gtType), dst, dstreg, 0, EmitSize(dst));
result = REG_NA;
}
return result;
}
//------------------------------------------------------------------------
// genSSE42Intrinsic: Generates the code for an SSE4.2 hardware intrinsic node
//
// Arguments:
// node - The hardware intrinsic node
//
void CodeGen::genSSE42Intrinsic(GenTreeHWIntrinsic* node)
{
NamedIntrinsic intrinsicID = node->gtHWIntrinsicId;
GenTree* op1 = node->gtGetOp1();
GenTree* op2 = node->gtGetOp2();
regNumber targetReg = node->gtRegNum;
assert(targetReg != REG_NA);
var_types targetType = node->TypeGet();
var_types baseType = node->gtSIMDBaseType;
regNumber op1Reg = op1->gtRegNum;
regNumber op2Reg = op2->gtRegNum;
genConsumeOperands(node);
switch (intrinsicID)
{
case NI_SSE42_Crc32:
if (op1Reg != targetReg)
{
inst_RV_RV(INS_mov, targetReg, op1Reg, targetType, emitTypeSize(targetType));
}
if (baseType == TYP_UBYTE || baseType == TYP_USHORT) // baseType is the type of the second argument
{
assert(targetType == TYP_INT);
inst_RV_RV(INS_crc32, targetReg, op2Reg, baseType, emitTypeSize(baseType));
}
else
{
assert(op1->TypeGet() == op2->TypeGet());
assert(targetType == TYP_INT || targetType == TYP_LONG);
inst_RV_RV(INS_crc32, targetReg, op2Reg, targetType, emitTypeSize(targetType));
}
break;
default:
unreached();
break;
}
genProduceReg(node);
}
regNumber CodeGen::genArithmFloat(genTreeOps oper,
GenTreePtr dst, regNumber dstreg,
GenTreePtr src, regNumber srcreg,
bool bReverse)
{
regNumber result = REG_NA;
assert(dstreg != REG_NA);
if (bReverse)
{
GenTree *temp = src;
regNumber tempreg = srcreg;
src = dst;
srcreg = dstreg;
dst = temp;
dstreg = tempreg;
}
if (srcreg == REG_NA)
{
if (src->IsRegVar())
{
inst_RV_RV(ins_MathOp(oper, dst->gtType), dst->gtRegNum, src->gtRegNum, dst->gtType);
}
else
{
inst_RV_TT(ins_MathOp(oper, dst->gtType), dst->gtRegNum, src);
}
}
else
{
inst_RV_RV(ins_MathOp(oper, dst->gtType), dstreg, srcreg, dst->gtType);
}
result = dstreg;
assert (result != REG_NA);
return result;
}
//------------------------------------------------------------------------
// genPOPCNTIntrinsic: Generates the code for a POPCNT hardware intrinsic node
//
// Arguments:
// node - The hardware intrinsic node
//
void CodeGen::genPOPCNTIntrinsic(GenTreeHWIntrinsic* node)
{
NamedIntrinsic intrinsicID = node->gtHWIntrinsicId;
GenTree* op1 = node->gtGetOp1();
regNumber targetReg = node->gtRegNum;
assert(targetReg != REG_NA);
var_types targetType = node->TypeGet();
regNumber op1Reg = op1->gtRegNum;
genConsumeOperands(node);
assert(intrinsicID == NI_POPCNT_PopCount);
inst_RV_RV(INS_popcnt, targetReg, op1Reg, targetType, emitTypeSize(targetType));
genProduceReg(node);
}
void CodeGen::genLoadFloat(GenTreePtr tree, regNumber reg)
{
if (tree->IsRegVar())
{
// if it has been spilled, unspill it.%
LclVarDsc * varDsc = &compiler->lvaTable[tree->gtLclVarCommon.gtLclNum];
if (varDsc->lvSpilled)
{
UnspillFloat(varDsc);
}
inst_RV_RV(ins_FloatCopy(tree->TypeGet()), reg, tree->gtRegNum, tree->TypeGet());
}
else
{
bool unalignedLoad = false;
switch (tree->OperGet())
{
case GT_IND:
case GT_CLS_VAR:
if (tree->gtFlags & GTF_IND_UNALIGNED)
unalignedLoad = true;
break;
case GT_LCL_FLD:
// Check for a misalignment on a Floating Point field
//
if (varTypeIsFloating(tree->TypeGet()))
{
if ((tree->gtLclFld.gtLclOffs % emitTypeSize(tree->TypeGet())) != 0)
{
unalignedLoad = true;
}
}
break;
default:
break;
}
if (unalignedLoad)
{
// Make the target addressable
//
regMaskTP addrReg = genMakeAddressable(tree, 0, RegSet::KEEP_REG, true);
regSet.rsLockUsedReg(addrReg); // Must prevent regSet.rsGrabReg from choosing an addrReg
var_types loadType = tree->TypeGet();
assert(loadType == TYP_DOUBLE || loadType == TYP_FLOAT);
// Unaligned Floating-Point Loads must be loaded into integer register(s)
// and then moved over to the Floating-Point register
regNumber intRegLo = regSet.rsGrabReg(RBM_ALLINT);
regNumber intRegHi = REG_NA;
regMaskTP tmpLockMask = genRegMask(intRegLo);
if (loadType == TYP_DOUBLE)
{
intRegHi = regSet.rsGrabReg(RBM_ALLINT & ~genRegMask(intRegLo));
tmpLockMask |= genRegMask(intRegHi);
}
regSet.rsLockReg(tmpLockMask); // Temporarily lock the intRegs
tree->gtType = TYP_INT; // Temporarily change the type to TYP_INT
inst_RV_TT(ins_Load(TYP_INT), intRegLo, tree);
regTracker.rsTrackRegTrash(intRegLo);
if (loadType == TYP_DOUBLE)
{
inst_RV_TT(ins_Load(TYP_INT), intRegHi, tree, 4);
regTracker.rsTrackRegTrash(intRegHi);
}
tree->gtType = loadType; // Change the type back to the floating point type
regSet.rsUnlockReg(tmpLockMask); // Unlock the intRegs
// move the integer register(s) over to the FP register
//
if (loadType == TYP_DOUBLE)
getEmitter()->emitIns_R_R_R(INS_vmov_i2d, EA_8BYTE, reg, intRegLo, intRegHi);
else
getEmitter()->emitIns_R_R(INS_vmov_i2f, EA_4BYTE, reg, intRegLo);
// Free up anything that was tied up by genMakeAddressable
//
regSet.rsUnlockUsedReg(addrReg);
genDoneAddressable(tree, addrReg, RegSet::KEEP_REG);
}
else
{
inst_RV_TT(ins_FloatLoad(tree->TypeGet()), reg, tree);
}
if (((tree->OperGet() == GT_CLS_VAR) || (tree->OperGet() == GT_IND)) &&
(tree->gtFlags & GTF_IND_VOLATILE))
{
// Emit a memory barrier instruction after the load
instGen_MemoryBarrier();
}
}
}
void CodeGen::genFloatAssign(GenTree *tree)
{
var_types type = tree->TypeGet();
GenTreePtr op1 = tree->gtGetOp1();
GenTreePtr op2 = tree->gtGetOp2();
regMaskTP needRegOp1 = RBM_ALLINT;
regMaskTP addrReg = RBM_NONE;
bool volat = false; // Is this a volatile store
bool unaligned = false; // Is this an unaligned store
regNumber op2reg = REG_NA;
#ifdef DEBUGGING_SUPPORT
unsigned lclVarNum = compiler->lvaCount;
unsigned lclILoffs = DUMMY_INIT(0);
#endif
noway_assert(tree->OperGet() == GT_ASG);
// Is the target a floating-point local variable?
// possibly even an enregistered floating-point local variable?
//
switch (op1->gtOper)
{
unsigned varNum;
LclVarDsc * varDsc;
case GT_LCL_FLD:
// Check for a misalignment on a Floating Point field
//
if (varTypeIsFloating(op1->TypeGet()))
{
if ((op1->gtLclFld.gtLclOffs % emitTypeSize(op1->TypeGet())) != 0)
{
unaligned = true;
}
}
break;
case GT_LCL_VAR:
varNum = op1->gtLclVarCommon.gtLclNum;
noway_assert(varNum < compiler->lvaCount);
varDsc = compiler->lvaTable + varNum;
#ifdef DEBUGGING_SUPPORT
// For non-debuggable code, every definition of a lcl-var has
// to be checked to see if we need to open a new scope for it.
// Remember the local var info to call siCheckVarScope
// AFTER code generation of the assignment.
//
if (compiler->opts.compScopeInfo && !compiler->opts.compDbgCode && (compiler->info.compVarScopesCount > 0))
{
lclVarNum = varNum;
lclILoffs = op1->gtLclVar.gtLclILoffs;
}
#endif
// Dead Store assert (with min opts we may have dead stores)
//
noway_assert(!varDsc->lvTracked || compiler->opts.MinOpts() || !(op1->gtFlags & GTF_VAR_DEATH));
// Does this variable live in a register?
//
if (genMarkLclVar(op1))
{
noway_assert(!compiler->opts.compDbgCode); // We don't enregister any floats with debug codegen
// Get hold of the target register
//
regNumber op1Reg = op1->gtRegVar.gtRegNum;
// the variable being assigned should be dead in op2
assert(!varDsc->lvTracked || !VarSetOps::IsMember(compiler, genUpdateLiveSetForward(op2), varDsc->lvVarIndex));
// Setup register preferencing, so that we try to target the op1 enregistered variable
//
regMaskTP bestMask = genRegMask(op1Reg);
if (type==TYP_DOUBLE)
{
assert((bestMask & RBM_DBL_REGS) != 0);
bestMask |= genRegMask(REG_NEXT(op1Reg));
}
RegSet::RegisterPreference pref(RBM_ALLFLOAT, bestMask);
// Evaluate op2 into a floating point register
//
genCodeForTreeFloat(op2, &pref);
noway_assert(op2->gtFlags & GTF_REG_VAL);
// Make sure the value ends up in the right place ...
// For example if op2 is a call that returns a result
// in REG_F0, we will need to do a move instruction here
//
if ((op2->gtRegNum != op1Reg) || (op2->TypeGet() != type))
{
regMaskTP spillRegs = regSet.rsMaskUsed & genRegMaskFloat(op1Reg, op1->TypeGet());
if (spillRegs != 0)
regSet.rsSpillRegs(spillRegs);
assert(type == op1->TypeGet());
inst_RV_RV(ins_FloatConv(type, op2->TypeGet()), op1Reg, op2->gtRegNum, type);
}
genUpdateLife(op1);
goto DONE_ASG;
}
break;
case GT_CLS_VAR:
case GT_IND:
// Check for a volatile/unaligned store
//
assert((op1->OperGet() == GT_CLS_VAR) || (op1->OperGet() == GT_IND)); // Required for GTF_IND_VOLATILE flag to be valid
if (op1->gtFlags & GTF_IND_VOLATILE)
volat = true;
if (op1->gtFlags & GTF_IND_UNALIGNED)
unaligned = true;
break;
default:
break;
}
// Is the value being assigned an enregistered floating-point local variable?
//
switch (op2->gtOper)
{
case GT_LCL_VAR:
if (!genMarkLclVar(op2))
break;
__fallthrough;
case GT_REG_VAR:
// We must honor the order evalauation in case op1 reassigns our op2 register
//
if (tree->gtFlags & GTF_REVERSE_OPS)
break;
// Is there an implicit conversion that we have to insert?
// Handle this case with the normal cases below.
//
if (type != op2->TypeGet())
break;
// Make the target addressable
//
addrReg = genMakeAddressable(op1, needRegOp1, RegSet::KEEP_REG, true);
noway_assert(op2->gtFlags & GTF_REG_VAL);
noway_assert(op2->IsRegVar());
op2reg = op2->gtRegVar.gtRegNum;
genUpdateLife(op2);
goto CHK_VOLAT_UNALIGN;
default:
break;
}
// Is the op2 (RHS) more complex than op1 (LHS)?
//
if (tree->gtFlags & GTF_REVERSE_OPS)
{
regMaskTP bestRegs = regSet.rsNarrowHint(RBM_ALLFLOAT, ~op1->gtRsvdRegs);
RegSet::RegisterPreference pref(RBM_ALLFLOAT, bestRegs);
// Generate op2 (RHS) into a floating point register
//
genCodeForTreeFloat(op2, &pref);
regSet.SetUsedRegFloat(op2, true);
// Make the target addressable
//
addrReg = genMakeAddressable(op1,
needRegOp1,
RegSet::KEEP_REG, true);
genRecoverReg(op2, RBM_ALLFLOAT, RegSet::KEEP_REG);
noway_assert(op2->gtFlags & GTF_REG_VAL);
regSet.SetUsedRegFloat(op2, false);
}
else
{
needRegOp1 = regSet.rsNarrowHint(needRegOp1, ~op2->gtRsvdRegs);
// Make the target addressable
//
addrReg = genMakeAddressable(op1,
needRegOp1,
RegSet::KEEP_REG, true);
// Generate the RHS into any floating point register
genCodeForTreeFloat(op2);
}
noway_assert(op2->gtFlags & GTF_REG_VAL);
op2reg = op2->gtRegNum;
// Is there an implicit conversion that we have to insert?
//
if (type != op2->TypeGet())
{
regMaskTP bestMask = genRegMask(op2reg);
if (type==TYP_DOUBLE)
{
if (bestMask & RBM_DBL_REGS)
{
bestMask |= genRegMask(REG_NEXT(op2reg));
}
else
{
bestMask |= genRegMask(REG_PREV(op2reg));
}
}
RegSet::RegisterPreference op2Pref(RBM_ALLFLOAT, bestMask);
op2reg = regSet.PickRegFloat(type, &op2Pref);
inst_RV_RV(ins_FloatConv(type, op2->TypeGet()), op2reg, op2->gtRegNum, type);
}
// Make sure the LHS is still addressable
//
addrReg = genKeepAddressable(op1, addrReg);
CHK_VOLAT_UNALIGN:
regSet.rsLockUsedReg(addrReg); // Must prevent unaligned regSet.rsGrabReg from choosing an addrReg
if (volat)
{
// Emit a memory barrier instruction before the store
instGen_MemoryBarrier();
}
if (unaligned)
{
var_types storeType = op1->TypeGet();
assert(storeType == TYP_DOUBLE || storeType == TYP_FLOAT);
// Unaligned Floating-Point Stores must be done using the integer register(s)
regNumber intRegLo = regSet.rsGrabReg(RBM_ALLINT);
regNumber intRegHi = REG_NA;
regMaskTP tmpLockMask = genRegMask(intRegLo);
if (storeType == TYP_DOUBLE)
{
intRegHi = regSet.rsGrabReg(RBM_ALLINT & ~genRegMask(intRegLo));
tmpLockMask |= genRegMask(intRegHi);
}
// move the FP register over to the integer register(s)
//
if (storeType == TYP_DOUBLE)
{
getEmitter()->emitIns_R_R_R(INS_vmov_d2i, EA_8BYTE, intRegLo, intRegHi, op2reg);
regTracker.rsTrackRegTrash(intRegHi);
}
else
{
getEmitter()->emitIns_R_R(INS_vmov_f2i, EA_4BYTE, intRegLo, op2reg);
}
regTracker.rsTrackRegTrash(intRegLo);
regSet.rsLockReg(tmpLockMask); // Temporarily lock the intRegs
op1->gtType = TYP_INT; // Temporarily change the type to TYP_INT
inst_TT_RV(ins_Store(TYP_INT), op1, intRegLo);
if (storeType == TYP_DOUBLE)
{
inst_TT_RV(ins_Store(TYP_INT), op1, intRegHi, 4);
}
op1->gtType = storeType; // Change the type back to the floating point type
regSet.rsUnlockReg(tmpLockMask); // Unlock the intRegs
}
else
{
// Move the value into the target
//
inst_TT_RV(ins_Store(op1->TypeGet()), op1, op2reg);
}
// Free up anything that was tied up by the LHS
//
regSet.rsUnlockUsedReg(addrReg);
genDoneAddressable(op1, addrReg, RegSet::KEEP_REG);
DONE_ASG:
genUpdateLife(tree);
#ifdef DEBUGGING_SUPPORT
/* For non-debuggable code, every definition of a lcl-var has
* to be checked to see if we need to open a new scope for it.
*/
if (lclVarNum < compiler->lvaCount)
siCheckVarScope(lclVarNum, lclILoffs);
#endif
}
void CodeGen::genFloatSimple(GenTree *tree, RegSet::RegisterPreference *pref)
{
assert(tree->OperKind() & GTK_SMPOP);
var_types type = tree->TypeGet();
RegSet::RegisterPreference defaultPref(RBM_ALLFLOAT, RBM_NONE);
if (pref == NULL)
{
pref = &defaultPref;
}
switch (tree->OperGet())
{
// Assignment
case GT_ASG:
{
genFloatAssign(tree);
break;
}
// Arithmetic binops
case GT_ADD:
case GT_SUB:
case GT_MUL:
case GT_DIV:
{
genFloatArith(tree, pref);
break;
}
case GT_NEG:
{
GenTreePtr op1 = tree->gtOp.gtOp1;
// get the tree into a register
genCodeForTreeFloat(op1, pref);
// change the sign
regNumber reg = regSet.PickRegFloat(type, pref);
genMarkTreeInReg(tree, reg);
inst_RV_RV(ins_MathOp(tree->OperGet(), type),
reg, op1->gtRegNum, type);
// mark register that holds tree
genCodeForTreeFloat_DONE(tree, reg);
return;
}
case GT_IND:
{
regMaskTP addrReg;
// Make sure the address value is 'addressable' */
addrReg = genMakeAddressable(tree, 0, RegSet::FREE_REG);
// Load the value onto the FP stack
regNumber reg = regSet.PickRegFloat(type, pref);
genLoadFloat(tree, reg);
genDoneAddressable(tree, addrReg, RegSet::FREE_REG);
genCodeForTreeFloat_DONE(tree, reg);
break;
}
case GT_CAST:
{
genCodeForTreeCastFloat(tree, pref);
break;
}
// Asg-Arithmetic ops
case GT_ASG_ADD:
case GT_ASG_SUB:
case GT_ASG_MUL:
case GT_ASG_DIV:
{
genFloatAsgArith(tree);
break;
}
case GT_INTRINSIC:
genFloatMath(tree, pref);
break;
case GT_RETURN:
{
GenTreePtr op1 = tree->gtOp.gtOp1;
assert(op1);
pref->best = (type==TYP_DOUBLE) ? RBM_DOUBLERET : RBM_FLOATRET;
// Compute the result
genCodeForTreeFloat(op1, pref);
inst_RV_TT(ins_FloatConv(tree->TypeGet(), op1->TypeGet()), REG_FLOATRET, op1);
if (compiler->info.compIsVarArgs)
{
if (tree->TypeGet() == TYP_FLOAT)
{
inst_RV_RV(INS_vmov_f2i, REG_INTRET, REG_FLOATRET, TYP_FLOAT, EA_4BYTE);
}
else
{
assert(tree->TypeGet() == TYP_DOUBLE);
inst_RV_RV_RV(INS_vmov_d2i, REG_INTRET, REG_NEXT(REG_INTRET), REG_FLOATRET, EA_8BYTE);
}
}
break;
}
case GT_ARGPLACE:
break;
case GT_COMMA:
{
GenTreePtr op1 = tree->gtOp.gtOp1;
GenTreePtr op2 = tree->gtGetOp2();
if (tree->gtFlags & GTF_REVERSE_OPS)
{
genCodeForTreeFloat(op2, pref);
regSet.SetUsedRegFloat(op2, true);
genEvalSideEffects(op1);
regSet.SetUsedRegFloat(op2, false);
}
else
{
genEvalSideEffects(op1);
genCodeForTreeFloat(op2, pref);
}
genCodeForTreeFloat_DONE(tree, op2->gtRegNum);
break;
}
case GT_CKFINITE:
genFloatCheckFinite(tree, pref);
break;
default:
NYI("Unhandled register FP codegen");
}
}