void CodeGen::genKeepAddressableFloat(GenTreePtr tree, regMaskTP * regMaskIntPtr, regMaskTP * regMaskFltPtr)
{
regMaskTP regMaskInt, regMaskFlt;
regMaskInt = *regMaskIntPtr;
regMaskFlt = *regMaskFltPtr;
*regMaskIntPtr = *regMaskFltPtr = 0;
switch (tree->OperGet())
{
case GT_REG_VAR:
// If register has been spilled, unspill it
if (tree->gtFlags & GTF_SPILLED)
{
UnspillFloat(&compiler->lvaTable[tree->gtLclVarCommon.gtLclNum]);
}
break;
case GT_CNS_DBL:
if (tree->gtFlags & GTF_SPILLED)
{
UnspillFloat(tree);
}
*regMaskFltPtr = genRegMaskFloat(tree->gtRegNum, tree->TypeGet());
break;
case GT_LCL_FLD:
case GT_LCL_VAR:
case GT_CLS_VAR:
break;
case GT_IND:
if (regMaskFlt == RBM_NONE)
{
*regMaskIntPtr = genKeepAddressable(tree, regMaskInt, 0);
*regMaskFltPtr = 0;
return;
}
__fallthrough;
default:
*regMaskIntPtr = 0;
if (tree->gtFlags & GTF_SPILLED)
{
UnspillFloat(tree);
}
*regMaskFltPtr = genRegMaskFloat(tree->gtRegNum, tree->TypeGet());
break;
}
}
void RegSet::SetLockedRegFloat(GenTree * tree, bool bValue)
{
regNumber reg = tree->gtRegNum;
var_types type = tree->TypeGet(); assert(varTypeIsFloating(type));
regMaskTP regMask = genRegMaskFloat(reg, tree->TypeGet());
if (bValue)
{
JITDUMP("locking register %s\n", getRegNameFloat(reg, type));
assert((rsGetMaskUsed() & regMask) == regMask);
assert((rsGetMaskLock() & regMask) == 0);
rsSetMaskLock( (rsGetMaskLock() | regMask) );
}
else
{
JITDUMP("unlocking register %s\n", getRegNameFloat(reg, type));
assert((rsGetMaskUsed() & regMask) == regMask);
assert((rsGetMaskLock() & regMask) == regMask);
rsSetMaskLock( (rsGetMaskLock() & ~regMask) );
}
}
void CodeGen::genDoneAddressableFloat(GenTreePtr tree,
regMaskTP addrRegInt,
regMaskTP addrRegFlt,
RegSet::KeepReg keptReg)
{
assert(!(addrRegInt && addrRegFlt));
if (addrRegInt)
{
return genDoneAddressable(tree, addrRegInt, keptReg);
}
else if (addrRegFlt)
{
if (keptReg == RegSet::KEEP_REG)
{
for (regNumber r = REG_FP_FIRST; r != REG_NA; r = regNextOfType(r, tree->TypeGet()))
{
regMaskTP mask = genRegMaskFloat(r, tree->TypeGet());
// some masks take up more than one bit
if ((mask & addrRegFlt) == mask)
{
regSet.SetUsedRegFloat(tree, false);
}
}
}
}
}
GenTreePtr CodeGen::genMakeAddressableFloat(GenTreePtr tree,
regMaskTP * regMaskIntPtr,
regMaskTP * regMaskFltPtr,
bool bCollapseConstantDoubles)
{
*regMaskIntPtr = *regMaskFltPtr = 0;
switch (tree->OperGet())
{
case GT_LCL_VAR:
genMarkLclVar(tree);
__fallthrough;
case GT_REG_VAR:
case GT_LCL_FLD:
case GT_CLS_VAR:
return tree;
case GT_IND:
// Try to make the address directly addressable
if (genMakeIndAddrMode(tree->gtOp.gtOp1,
tree,
false,
RBM_ALLFLOAT,
RegSet::KEEP_REG,
regMaskIntPtr,
false))
{
genUpdateLife(tree);
return tree;
}
else
{
GenTreePtr addr = tree;
tree = tree->gtOp.gtOp1;
genCodeForTree(tree, 0);
regSet.rsMarkRegUsed(tree, addr);
*regMaskIntPtr = genRegMask(tree->gtRegNum);
return addr;
}
// fall through
default:
genCodeForTreeFloat(tree);
regSet.SetUsedRegFloat(tree, true);
// update mask
*regMaskFltPtr = genRegMaskFloat(tree->gtRegNum, tree->TypeGet());
return tree;
break;
}
}
bool RegSet::IsLockedRegFloat(GenTreePtr tree)
{
/* The value must be sitting in a register */
assert(tree);
assert(tree->gtFlags & GTF_REG_VAL);
assert(varTypeIsFloating(tree->TypeGet()));
regMaskTP regMask = genRegMaskFloat(tree->gtRegNum, tree->TypeGet());
return (rsGetMaskLock() & regMask) == regMask;
}
void CodeGen::genComputeAddressableFloat(GenTreePtr tree,
regMaskTP addrRegInt,
regMaskTP addrRegFlt,
RegSet::KeepReg keptReg,
regMaskTP needReg,
RegSet::KeepReg keepReg,
bool freeOnly /* = false */)
{
noway_assert(genStillAddressable(tree));
noway_assert(varTypeIsFloating(tree->TypeGet()));
genDoneAddressableFloat(tree, addrRegInt, addrRegFlt, keptReg);
regNumber reg;
if (tree->gtFlags & GTF_REG_VAL)
{
reg = tree->gtRegNum;
if (freeOnly && !(genRegMaskFloat(reg, tree->TypeGet()) & regSet.RegFreeFloat()))
{
goto LOAD_REG;
}
}
else
{
LOAD_REG:
RegSet::RegisterPreference pref(needReg, RBM_NONE);
reg = regSet.PickRegFloat(tree->TypeGet(), &pref);
genLoadFloat(tree, reg);
}
genMarkTreeInReg(tree, reg);
if (keepReg == RegSet::KEEP_REG)
{
regSet.SetUsedRegFloat(tree, true);
}
}
void RegSet::SetUsedRegFloat(GenTreePtr tree, bool bValue)
{
/* The value must be sitting in a register */
assert(tree);
assert(tree->gtFlags & GTF_REG_VAL);
var_types type = tree->TypeGet();
#ifdef _TARGET_ARM_
if (type == TYP_STRUCT)
{
assert(m_rsCompiler->IsHfa(tree));
type = TYP_FLOAT;
}
#endif
regNumber regNum = tree->gtRegNum;
regMaskTP regMask = genRegMaskFloat(regNum, type);
if (bValue)
{
// Mark as used
#ifdef DEBUG
if (m_rsCompiler->verbose)
{
printf("\t\t\t\t\t\t\tThe register %s currently holds ",
getRegNameFloat(regNum, type));
Compiler::printTreeID(tree);
printf("\n");
}
#endif
assert((rsGetMaskLock() & regMask) == 0);
#if FEATURE_STACK_FP_X87
assert((rsGetMaskUsed() & regMask) == 0);
#else
/* Is the register used by two different values simultaneously? */
if (regMask & rsGetMaskUsed())
{
/* Save the preceding use information */
rsRecMultiReg(regNum, type);
}
#endif
/* Set the register's bit in the 'used' bitset */
rsSetMaskUsed( (rsGetMaskUsed() | regMask) );
// Assign slot
rsSetUsedTree(regNum, tree);
}
else
{
// Mark as free
#ifdef DEBUG
if (m_rsCompiler->verbose)
{
printf("\t\t\t\t\t\t\tThe register %s no longer holds ",
getRegNameFloat(regNum, type));
Compiler::printTreeID(tree);
printf("\n");
}
#endif
assert((rsGetMaskUsed() & regMask) == regMask);
// Are we freeing a multi-use registers?
if (regMask & rsGetMaskMult())
{
// Free any multi-use registers
rsMultRegFree(regMask);
return;
}
rsSetMaskUsed( (rsGetMaskUsed() & ~regMask) );
// Free slot
rsFreeUsedTree(regNum, tree);
}
}
regNumber RegSet::PickRegFloat(var_types type, RegisterPreference *pref, bool bUsed)
{
regMaskTP wantedMask;
bool tryBest = true;
bool tryOk = true;
bool bSpill = false;
regNumber reg = REG_NA;
while (tryOk)
{
if (pref)
{
if (tryBest)
{
wantedMask = pref->best;
tryBest = false;
}
else
{
assert(tryOk);
wantedMask = pref->ok;
tryOk = false;
}
}
else// pref is NULL
{
wantedMask = RBM_ALLFLOAT;
tryBest = false;
tryOk = false;
}
// better not have asked for a non-fp register
assert((wantedMask & ~RBM_ALLFLOAT) == 0);
regMaskTP availMask = RegFreeFloat();
regMaskTP OKmask = availMask & wantedMask;
if (OKmask == 0)
{
if (tryOk)
{
// the pref->best mask doesn't work so try the pref->ok mask next
continue;
}
if (bUsed)
{
// Allow used registers to be picked
OKmask |= rsGetMaskUsed() & ~rsGetMaskLock();
bSpill = true;
}
}
#if FEATURE_FP_REGALLOC
regMaskTP restrictMask = (m_rsCompiler->raConfigRestrictMaskFP() | RBM_FLT_CALLEE_TRASH);
#endif
for (unsigned i=0; i<ArrLen(pickOrder); i++)
{
regNumber r = pickOrder[i];
if (!floatRegCanHoldType(r, type))
continue;
regMaskTP mask = genRegMaskFloat(r, type);
#if FEATURE_FP_REGALLOC
if ((mask & restrictMask) != mask)
continue;
#endif
if ((OKmask & mask) == mask)
{
reg = r;
goto RET;
}
}
if (tryOk)
{
// We couldn't find a register using tryBest
continue;
}
assert(!"Unable to find a free FP virtual register");
NO_WAY("FP register allocator was too optimistic!");
}
RET:
if (bSpill)
{
m_rsCompiler->codeGen->SpillFloat(reg);
}
#if FEATURE_FP_REGALLOC
rsSetRegsModified(genRegMaskFloat(reg, type));
#endif
return reg;
}
void CodeGen::genFloatArith (GenTreePtr tree,
RegSet::RegisterPreference *tgtPref)
{
var_types type = tree->TypeGet();
genTreeOps oper = tree->OperGet();
GenTreePtr op1 = tree->gtGetOp1();
GenTreePtr op2 = tree->gtGetOp2();
regNumber tgtReg;
unsigned varNum;
LclVarDsc * varDsc;
VARSET_TP varBit;
assert(oper == GT_ADD ||
oper == GT_SUB ||
oper == GT_MUL ||
oper == GT_DIV);
RegSet::RegisterPreference defaultPref(RBM_ALLFLOAT, RBM_NONE);
if (tgtPref == NULL)
{
tgtPref = &defaultPref;
}
// Is the op2 (RHS)more complex than op1 (LHS)?
//
if (tree->gtFlags & GTF_REVERSE_OPS)
{
regMaskTP bestRegs = regSet.rsNarrowHint(RBM_ALLFLOAT, ~op1->gtRsvdRegs);
RegSet::RegisterPreference pref(RBM_ALLFLOAT, bestRegs);
// Evaluate op2 into a floating point register
//
genCodeForTreeFloat(op2, &pref);
regSet.SetUsedRegFloat(op2, true);
// Evaluate op1 into any floating point register
//
genCodeForTreeFloat(op1);
regSet.SetUsedRegFloat(op1, true);
regNumber op1Reg = op1->gtRegNum;
regMaskTP op1Mask = genRegMaskFloat(op1Reg, type);
// Fix 388445 ARM JitStress WP7
regSet.rsLockUsedReg(op1Mask);
genRecoverReg(op2, RBM_ALLFLOAT, RegSet::KEEP_REG);
noway_assert(op2->gtFlags & GTF_REG_VAL);
regSet.rsUnlockUsedReg(op1Mask);
regSet.SetUsedRegFloat(op1, false);
regSet.SetUsedRegFloat(op2, false);
}
else
{
regMaskTP bestRegs = regSet.rsNarrowHint(RBM_ALLFLOAT, ~op2->gtRsvdRegs);
RegSet::RegisterPreference pref(RBM_ALLFLOAT, bestRegs);
// Evaluate op1 into a floating point register
//
genCodeForTreeFloat(op1, &pref);
regSet.SetUsedRegFloat(op1, true);
// Evaluate op2 into any floating point register
//
genCodeForTreeFloat(op2);
regSet.SetUsedRegFloat(op2, true);
regNumber op2Reg = op2->gtRegNum;
regMaskTP op2Mask = genRegMaskFloat(op2Reg, type);
// Fix 388445 ARM JitStress WP7
regSet.rsLockUsedReg(op2Mask);
genRecoverReg(op1, RBM_ALLFLOAT, RegSet::KEEP_REG);
noway_assert(op1->gtFlags & GTF_REG_VAL);
regSet.rsUnlockUsedReg(op2Mask);
regSet.SetUsedRegFloat(op2, false);
regSet.SetUsedRegFloat(op1, false);
}
tgtReg = regSet.PickRegFloat(type, tgtPref, true);
noway_assert(op1->gtFlags & GTF_REG_VAL);
noway_assert(op2->gtFlags & GTF_REG_VAL);
inst_RV_RV_RV(ins_MathOp(oper, type), tgtReg, op1->gtRegNum, op2->gtRegNum, emitActualTypeSize(type));
genCodeForTreeFloat_DONE(tree, tgtReg);
}
void CodeGen::genFloatAssign(GenTree *tree)
{
var_types type = tree->TypeGet();
GenTreePtr op1 = tree->gtGetOp1();
GenTreePtr op2 = tree->gtGetOp2();
regMaskTP needRegOp1 = RBM_ALLINT;
regMaskTP addrReg = RBM_NONE;
bool volat = false; // Is this a volatile store
bool unaligned = false; // Is this an unaligned store
regNumber op2reg = REG_NA;
#ifdef DEBUGGING_SUPPORT
unsigned lclVarNum = compiler->lvaCount;
unsigned lclILoffs = DUMMY_INIT(0);
#endif
noway_assert(tree->OperGet() == GT_ASG);
// Is the target a floating-point local variable?
// possibly even an enregistered floating-point local variable?
//
switch (op1->gtOper)
{
unsigned varNum;
LclVarDsc * varDsc;
case GT_LCL_FLD:
// Check for a misalignment on a Floating Point field
//
if (varTypeIsFloating(op1->TypeGet()))
{
if ((op1->gtLclFld.gtLclOffs % emitTypeSize(op1->TypeGet())) != 0)
{
unaligned = true;
}
}
break;
case GT_LCL_VAR:
varNum = op1->gtLclVarCommon.gtLclNum;
noway_assert(varNum < compiler->lvaCount);
varDsc = compiler->lvaTable + varNum;
#ifdef DEBUGGING_SUPPORT
// For non-debuggable code, every definition of a lcl-var has
// to be checked to see if we need to open a new scope for it.
// Remember the local var info to call siCheckVarScope
// AFTER code generation of the assignment.
//
if (compiler->opts.compScopeInfo && !compiler->opts.compDbgCode && (compiler->info.compVarScopesCount > 0))
{
lclVarNum = varNum;
lclILoffs = op1->gtLclVar.gtLclILoffs;
}
#endif
// Dead Store assert (with min opts we may have dead stores)
//
noway_assert(!varDsc->lvTracked || compiler->opts.MinOpts() || !(op1->gtFlags & GTF_VAR_DEATH));
// Does this variable live in a register?
//
if (genMarkLclVar(op1))
{
noway_assert(!compiler->opts.compDbgCode); // We don't enregister any floats with debug codegen
// Get hold of the target register
//
regNumber op1Reg = op1->gtRegVar.gtRegNum;
// the variable being assigned should be dead in op2
assert(!varDsc->lvTracked || !VarSetOps::IsMember(compiler, genUpdateLiveSetForward(op2), varDsc->lvVarIndex));
// Setup register preferencing, so that we try to target the op1 enregistered variable
//
regMaskTP bestMask = genRegMask(op1Reg);
if (type==TYP_DOUBLE)
{
assert((bestMask & RBM_DBL_REGS) != 0);
bestMask |= genRegMask(REG_NEXT(op1Reg));
}
RegSet::RegisterPreference pref(RBM_ALLFLOAT, bestMask);
// Evaluate op2 into a floating point register
//
genCodeForTreeFloat(op2, &pref);
noway_assert(op2->gtFlags & GTF_REG_VAL);
// Make sure the value ends up in the right place ...
// For example if op2 is a call that returns a result
// in REG_F0, we will need to do a move instruction here
//
if ((op2->gtRegNum != op1Reg) || (op2->TypeGet() != type))
{
regMaskTP spillRegs = regSet.rsMaskUsed & genRegMaskFloat(op1Reg, op1->TypeGet());
if (spillRegs != 0)
regSet.rsSpillRegs(spillRegs);
assert(type == op1->TypeGet());
inst_RV_RV(ins_FloatConv(type, op2->TypeGet()), op1Reg, op2->gtRegNum, type);
}
genUpdateLife(op1);
goto DONE_ASG;
}
break;
case GT_CLS_VAR:
case GT_IND:
// Check for a volatile/unaligned store
//
assert((op1->OperGet() == GT_CLS_VAR) || (op1->OperGet() == GT_IND)); // Required for GTF_IND_VOLATILE flag to be valid
if (op1->gtFlags & GTF_IND_VOLATILE)
volat = true;
if (op1->gtFlags & GTF_IND_UNALIGNED)
unaligned = true;
break;
default:
break;
}
// Is the value being assigned an enregistered floating-point local variable?
//
switch (op2->gtOper)
{
case GT_LCL_VAR:
if (!genMarkLclVar(op2))
break;
__fallthrough;
case GT_REG_VAR:
// We must honor the order evalauation in case op1 reassigns our op2 register
//
if (tree->gtFlags & GTF_REVERSE_OPS)
break;
// Is there an implicit conversion that we have to insert?
// Handle this case with the normal cases below.
//
if (type != op2->TypeGet())
break;
// Make the target addressable
//
addrReg = genMakeAddressable(op1, needRegOp1, RegSet::KEEP_REG, true);
noway_assert(op2->gtFlags & GTF_REG_VAL);
noway_assert(op2->IsRegVar());
op2reg = op2->gtRegVar.gtRegNum;
genUpdateLife(op2);
goto CHK_VOLAT_UNALIGN;
default:
break;
}
// Is the op2 (RHS) more complex than op1 (LHS)?
//
if (tree->gtFlags & GTF_REVERSE_OPS)
{
regMaskTP bestRegs = regSet.rsNarrowHint(RBM_ALLFLOAT, ~op1->gtRsvdRegs);
RegSet::RegisterPreference pref(RBM_ALLFLOAT, bestRegs);
// Generate op2 (RHS) into a floating point register
//
genCodeForTreeFloat(op2, &pref);
regSet.SetUsedRegFloat(op2, true);
// Make the target addressable
//
addrReg = genMakeAddressable(op1,
needRegOp1,
RegSet::KEEP_REG, true);
genRecoverReg(op2, RBM_ALLFLOAT, RegSet::KEEP_REG);
noway_assert(op2->gtFlags & GTF_REG_VAL);
regSet.SetUsedRegFloat(op2, false);
}
else
{
needRegOp1 = regSet.rsNarrowHint(needRegOp1, ~op2->gtRsvdRegs);
// Make the target addressable
//
addrReg = genMakeAddressable(op1,
needRegOp1,
RegSet::KEEP_REG, true);
// Generate the RHS into any floating point register
genCodeForTreeFloat(op2);
}
noway_assert(op2->gtFlags & GTF_REG_VAL);
op2reg = op2->gtRegNum;
// Is there an implicit conversion that we have to insert?
//
if (type != op2->TypeGet())
{
regMaskTP bestMask = genRegMask(op2reg);
if (type==TYP_DOUBLE)
{
if (bestMask & RBM_DBL_REGS)
{
bestMask |= genRegMask(REG_NEXT(op2reg));
}
else
{
bestMask |= genRegMask(REG_PREV(op2reg));
}
}
RegSet::RegisterPreference op2Pref(RBM_ALLFLOAT, bestMask);
op2reg = regSet.PickRegFloat(type, &op2Pref);
inst_RV_RV(ins_FloatConv(type, op2->TypeGet()), op2reg, op2->gtRegNum, type);
}
// Make sure the LHS is still addressable
//
addrReg = genKeepAddressable(op1, addrReg);
CHK_VOLAT_UNALIGN:
regSet.rsLockUsedReg(addrReg); // Must prevent unaligned regSet.rsGrabReg from choosing an addrReg
if (volat)
{
// Emit a memory barrier instruction before the store
instGen_MemoryBarrier();
}
if (unaligned)
{
var_types storeType = op1->TypeGet();
assert(storeType == TYP_DOUBLE || storeType == TYP_FLOAT);
// Unaligned Floating-Point Stores must be done using the integer register(s)
regNumber intRegLo = regSet.rsGrabReg(RBM_ALLINT);
regNumber intRegHi = REG_NA;
regMaskTP tmpLockMask = genRegMask(intRegLo);
if (storeType == TYP_DOUBLE)
{
intRegHi = regSet.rsGrabReg(RBM_ALLINT & ~genRegMask(intRegLo));
tmpLockMask |= genRegMask(intRegHi);
}
// move the FP register over to the integer register(s)
//
if (storeType == TYP_DOUBLE)
{
getEmitter()->emitIns_R_R_R(INS_vmov_d2i, EA_8BYTE, intRegLo, intRegHi, op2reg);
regTracker.rsTrackRegTrash(intRegHi);
}
else
{
getEmitter()->emitIns_R_R(INS_vmov_f2i, EA_4BYTE, intRegLo, op2reg);
}
regTracker.rsTrackRegTrash(intRegLo);
regSet.rsLockReg(tmpLockMask); // Temporarily lock the intRegs
op1->gtType = TYP_INT; // Temporarily change the type to TYP_INT
inst_TT_RV(ins_Store(TYP_INT), op1, intRegLo);
if (storeType == TYP_DOUBLE)
{
inst_TT_RV(ins_Store(TYP_INT), op1, intRegHi, 4);
}
op1->gtType = storeType; // Change the type back to the floating point type
regSet.rsUnlockReg(tmpLockMask); // Unlock the intRegs
}
else
{
// Move the value into the target
//
inst_TT_RV(ins_Store(op1->TypeGet()), op1, op2reg);
}
// Free up anything that was tied up by the LHS
//
regSet.rsUnlockUsedReg(addrReg);
genDoneAddressable(op1, addrReg, RegSet::KEEP_REG);
DONE_ASG:
genUpdateLife(tree);
#ifdef DEBUGGING_SUPPORT
/* For non-debuggable code, every definition of a lcl-var has
* to be checked to see if we need to open a new scope for it.
*/
if (lclVarNum < compiler->lvaCount)
siCheckVarScope(lclVarNum, lclILoffs);
#endif
}
