void CodeGen::genFloatAsgArith (GenTreePtr tree)
{
// Set Flowgraph.cpp, line 13750
// arm VFP has tons of regs, 3-op instructions, and no addressing modes
// so asg ops are kind of pointless
noway_assert(!"Not Reachable for _TARGET_ARM_");
}
/**
* Returns a SSA count number for a local variable from top of the stack.
*
* @params lclNum The local variable def for which a count has to be returned.
* @return the current variable name for the "use".
*
* @remarks If the stack is empty, then we have an use before a def. To handle this
* special case, we need to initialize the count with 'default+1', so the
* next definition will always use 'default+1' but return 'default' for
* all uses until a definition.
*
*/
unsigned SsaRenameState::CountForUse(unsigned lclNum)
{
EnsureStacks();
DBG_SSA_JITDUMP("[SsaRenameState::CountForUse] V%02u\n", lclNum);
Stack* stack = stacks[lclNum];
noway_assert((stack != nullptr) && !stack->empty());
return stack->back().m_count;
}
//------------------------------------------------------------------------
// LowerStoreLoc: Lower a store of a lclVar
//
// Arguments:
// storeLoc - the local store (GT_STORE_LCL_FLD or GT_STORE_LCL_VAR)
//
// Notes:
// This involves:
// - Widening operations of unsigneds.
//
void Lowering::LowerStoreLoc(GenTreeLclVarCommon* storeLoc)
{
// Try to widen the ops if they are going into a local var.
GenTree* op1 = storeLoc->gtGetOp1();
if ((storeLoc->gtOper == GT_STORE_LCL_VAR) && (op1->gtOper == GT_CNS_INT))
{
GenTreeIntCon* con = op1->AsIntCon();
ssize_t ival = con->gtIconVal;
unsigned varNum = storeLoc->gtLclNum;
LclVarDsc* varDsc = comp->lvaTable + varNum;
if (varDsc->lvIsSIMDType())
{
noway_assert(storeLoc->gtType != TYP_STRUCT);
}
unsigned size = genTypeSize(storeLoc);
// If we are storing a constant into a local variable
// we extend the size of the store here
if ((size < 4) && !varTypeIsStruct(varDsc))
{
if (!varTypeIsUnsigned(varDsc))
{
if (genTypeSize(storeLoc) == 1)
{
if ((ival & 0x7f) != ival)
{
ival = ival | 0xffffff00;
}
}
else
{
assert(genTypeSize(storeLoc) == 2);
if ((ival & 0x7fff) != ival)
{
ival = ival | 0xffff0000;
}
}
}
// A local stack slot is at least 4 bytes in size, regardless of
// what the local var is typed as, so auto-promote it here
// unless it is a field of a promoted struct
// TODO-CQ: if the field is promoted shouldn't we also be able to do this?
if (!varDsc->lvIsStructField)
{
storeLoc->gtType = TYP_INT;
con->SetIconValue(ival);
}
}
}
if (storeLoc->OperIs(GT_STORE_LCL_FLD))
{
// We should only encounter this for lclVars that are lvDoNotEnregister.
verifyLclFldDoNotEnregister(storeLoc->gtLclNum);
}
ContainCheckStoreLoc(storeLoc);
}
// Rewrite GT_OBJ of SIMD Vector as GT_IND(GT_LEA(obj.op1)) of a SIMD type.
//
// Arguments:
// ppTree - A pointer-to-a-pointer for the GT_OBJ
// fgWalkData - A pointer to tree walk data providing the context
//
// Return Value:
// None.
//
// TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer
// be required.
//
void Rationalizer::RewriteObj(LIR::Use& use)
{
#ifdef FEATURE_SIMD
GenTreeObj* obj = use.Def()->AsObj();
// For UNIX struct passing, we can have Obj nodes for arguments.
// For other cases, we should never see a non-SIMD type here.
#ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
if (!varTypeIsSIMD(obj))
{
return;
}
#endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
// Should come here only if featureSIMD is enabled
noway_assert(comp->featureSIMD);
// We should only call this with a SIMD type.
noway_assert(varTypeIsSIMD(obj));
var_types simdType = obj->TypeGet();
// If the operand of obj is a GT_ADDR(GT_LCL_VAR) and LclVar is known to be a SIMD type,
// replace obj by GT_LCL_VAR.
GenTree* srcAddr = obj->gtGetOp1();
if (srcAddr->OperIsLocalAddr() && comp->isAddrOfSIMDType(srcAddr))
{
BlockRange().Remove(obj);
srcAddr->SetOper(loadForm(srcAddr->OperGet()));
srcAddr->gtType = simdType;
use.ReplaceWith(comp, srcAddr);
}
else
{
obj->SetOper(GT_IND);
obj->gtType = simdType;
}
#else
// we should never reach without feature SIMD
assert(!"Unexpected obj during rationalization\n");
unreached();
#endif
}
void Compiler::unwindAllocStackCFI(unsigned size)
{
assert(compGeneratingProlog);
FuncInfoDsc* func = funCurrentFunc();
unsigned int cbProlog = unwindGetCurrentOffset(func);
noway_assert((BYTE)cbProlog == cbProlog);
createCfiCode(func, cbProlog, CFI_ADJUST_CFA_OFFSET, DWARF_REG_ILLEGAL, size);
}
void CodeGen::genFloatLeaf(GenTree *tree, RegSet::RegisterPreference *pref)
{
regNumber reg = REG_NA;
switch (tree->OperGet())
{
case GT_LCL_VAR:
// Does the variable live in a register?
//
if (!genMarkLclVar(tree))
goto MEM_LEAF;
__fallthrough;
case GT_REG_VAR:
noway_assert(tree->gtFlags & GTF_REG_VAL);
reg = tree->gtRegVar.gtRegNum;
break;
case GT_LCL_FLD:
// We only use GT_LCL_FLD for lvAddrTaken vars, so we don't have
// to worry about it being enregistered.
noway_assert(compiler->lvaTable[tree->gtLclFld.gtLclNum].lvRegister == 0);
__fallthrough;
case GT_CLS_VAR:
MEM_LEAF:
reg = regSet.PickRegFloat(tree->TypeGet(), pref);
genLoadFloat(tree, reg);
break;
default:
DISPTREE(tree);
assert(!"unexpected leaf");
}
genCodeForTreeFloat_DONE (tree, reg);
return;
}
void Compiler::unwindSaveRegCFI(regNumber reg, unsigned offset)
{
assert(compGeneratingProlog);
if (RBM_CALLEE_SAVED & genRegMask(reg))
{
FuncInfoDsc* func = funCurrentFunc();
unsigned int cbProlog = unwindGetCurrentOffset(func);
noway_assert((BYTE)cbProlog == cbProlog);
createCfiCode(func, cbProlog, CFI_REL_OFFSET, mapRegNumToDwarfReg(reg), offset);
}
}
// return op that is the load equivalent of the given addr opcode
genTreeOps loadForm(genTreeOps addrForm)
{
switch (addrForm)
{
case GT_LCL_VAR_ADDR:
return GT_LCL_VAR;
case GT_LCL_FLD_ADDR:
return GT_LCL_FLD;
default:
noway_assert(!"not a local address opcode\n");
unreached();
}
}
//------------------------------------------------------------------------
// DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeArith(LIR::Use& use)
{
assert(use.IsInitialized());
GenTree* tree = use.Def();
genTreeOps oper = tree->OperGet();
assert((oper == GT_ADD) || (oper == GT_SUB) || (oper == GT_OR) || (oper == GT_XOR) || (oper == GT_AND));
GenTree* op1 = tree->gtGetOp1();
GenTree* op2 = tree->gtGetOp2();
// Both operands must have already been decomposed into GT_LONG operators.
noway_assert((op1->OperGet() == GT_LONG) && (op2->OperGet() == GT_LONG));
// Capture the lo and hi halves of op1 and op2.
GenTree* loOp1 = op1->gtGetOp1();
GenTree* hiOp1 = op1->gtGetOp2();
GenTree* loOp2 = op2->gtGetOp1();
GenTree* hiOp2 = op2->gtGetOp2();
// Now, remove op1 and op2 from the node list.
Range().Remove(op1);
Range().Remove(op2);
// We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands
// will be the lo halves of op1 from above.
GenTree* loResult = tree;
loResult->SetOper(GetLoOper(oper));
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
loResult->gtOp.gtOp2 = loOp2;
GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2);
Range().InsertAfter(loResult, hiResult);
if ((oper == GT_ADD) || (oper == GT_SUB))
{
if (loResult->gtOverflow())
{
hiResult->gtFlags |= GTF_OVERFLOW;
loResult->gtFlags &= ~GTF_OVERFLOW;
}
if (loResult->gtFlags & GTF_UNSIGNED)
{
hiResult->gtFlags |= GTF_UNSIGNED;
}
}
return FinalizeDecomposition(use, loResult, hiResult);
}
void Compiler::unwindSetFrameRegCFI(regNumber reg, unsigned offset)
{
assert(compGeneratingProlog);
FuncInfoDsc* func = funCurrentFunc();
unsigned int cbProlog = unwindGetCurrentOffset(func);
noway_assert((BYTE)cbProlog == cbProlog);
createCfiCode(func, cbProlog, CFI_DEF_CFA_REGISTER, mapRegNumToDwarfReg(reg));
if (offset != 0)
{
createCfiCode(func, cbProlog, CFI_ADJUST_CFA_OFFSET, DWARF_REG_ILLEGAL, offset);
}
}
void CodeGen::genComputeAddressableFloat(GenTreePtr tree,
regMaskTP addrRegInt,
regMaskTP addrRegFlt,
RegSet::KeepReg keptReg,
regMaskTP needReg,
RegSet::KeepReg keepReg,
bool freeOnly /* = false */)
{
noway_assert(genStillAddressable(tree));
noway_assert(varTypeIsFloating(tree->TypeGet()));
genDoneAddressableFloat(tree, addrRegInt, addrRegFlt, keptReg);
regNumber reg;
if (tree->gtFlags & GTF_REG_VAL)
{
reg = tree->gtRegNum;
if (freeOnly && !(genRegMaskFloat(reg, tree->TypeGet()) & regSet.RegFreeFloat()))
{
goto LOAD_REG;
}
}
else
{
LOAD_REG:
RegSet::RegisterPreference pref(needReg, RBM_NONE);
reg = regSet.PickRegFloat(tree->TypeGet(), &pref);
genLoadFloat(tree, reg);
}
genMarkTreeInReg(tree, reg);
if (keepReg == RegSet::KEEP_REG)
{
regSet.SetUsedRegFloat(tree, true);
}
}
void Compiler::unwindAllocStackCFI(unsigned size)
{
#if defined(_TARGET_ARM_)
assert(compGeneratingEpilog);
#else
assert(compGeneratingProlog);
#endif
FuncInfoDsc* func = funCurrentFunc();
unsigned int cbProlog = 0;
if (compGeneratingProlog)
{
cbProlog = unwindGetCurrentOffset(func);
noway_assert((BYTE)cbProlog == cbProlog);
}
createCfiCode(func, cbProlog, CFI_ADJUST_CFA_OFFSET, DWARF_REG_ILLEGAL, size);
}
InlineContext* InlineStrategy::NewSuccess(InlineInfo* inlineInfo)
{
InlineContext* calleeContext = new (m_Compiler, CMK_Inlining) InlineContext(this);
GenTree* stmt = inlineInfo->iciStmt;
BYTE* calleeIL = inlineInfo->inlineCandidateInfo->methInfo.ILCode;
unsigned calleeILSize = inlineInfo->inlineCandidateInfo->methInfo.ILCodeSize;
InlineContext* parentContext = stmt->gtStmt.gtInlineContext;
noway_assert(parentContext != nullptr);
calleeContext->m_Code = calleeIL;
calleeContext->m_ILSize = calleeILSize;
calleeContext->m_Parent = parentContext;
// Push on front here will put siblings in reverse lexical
// order which we undo in the dumper
calleeContext->m_Sibling = parentContext->m_Child;
parentContext->m_Child = calleeContext;
calleeContext->m_Child = nullptr;
calleeContext->m_Offset = stmt->AsStmt()->gtStmtILoffsx;
calleeContext->m_Observation = inlineInfo->inlineResult->GetObservation();
calleeContext->m_Success = true;
#if defined(DEBUG) || defined(INLINE_DATA)
InlinePolicy* policy = inlineInfo->inlineResult->GetPolicy();
calleeContext->m_Policy = policy;
calleeContext->m_CodeSizeEstimate = policy->CodeSizeEstimate();
calleeContext->m_Callee = inlineInfo->fncHandle;
// +1 here since we set this before calling NoteOutcome.
calleeContext->m_Ordinal = m_InlineCount + 1;
// Update offset with more accurate info
calleeContext->m_Offset = inlineInfo->inlineResult->GetCall()->gtRawILOffset;
#endif // defined(DEBUG) || defined(INLINE_DATA)
#if defined(DEBUG)
calleeContext->m_TreeID = inlineInfo->inlineResult->GetCall()->gtTreeID;
#endif // defined(DEBUG)
NoteOutcome(calleeContext);
return calleeContext;
}
int Compiler::mapRegNumToDwarfReg(regNumber reg)
{
int dwarfReg = DWARF_REG_ILLEGAL;
switch (reg)
{
case REG_RAX: dwarfReg = 0; break;
case REG_RCX: dwarfReg = 2; break;
case REG_RDX: dwarfReg = 1; break;
case REG_RBX: dwarfReg = 3; break;
case REG_RSP: dwarfReg = 7; break;
case REG_RBP: dwarfReg = 6; break;
case REG_RSI: dwarfReg = 4; break;
case REG_RDI: dwarfReg = 5; break;
case REG_R8: dwarfReg = 8; break;
case REG_R9: dwarfReg = 9; break;
case REG_R10: dwarfReg = 10; break;
case REG_R11: dwarfReg = 11; break;
case REG_R12: dwarfReg = 12; break;
case REG_R13: dwarfReg = 13; break;
case REG_R14: dwarfReg = 14; break;
case REG_R15: dwarfReg = 15; break;
case REG_XMM0: dwarfReg = 17; break;
case REG_XMM1: dwarfReg = 18; break;
case REG_XMM2: dwarfReg = 19; break;
case REG_XMM3: dwarfReg = 20; break;
case REG_XMM4: dwarfReg = 21; break;
case REG_XMM5: dwarfReg = 22; break;
case REG_XMM6: dwarfReg = 23; break;
case REG_XMM7: dwarfReg = 24; break;
case REG_XMM8: dwarfReg = 25; break;
case REG_XMM9: dwarfReg = 26; break;
case REG_XMM10:dwarfReg = 27; break;
case REG_XMM11:dwarfReg = 28; break;
case REG_XMM12:dwarfReg = 29; break;
case REG_XMM13:dwarfReg = 30; break;
case REG_XMM14:dwarfReg = 31; break;
case REG_XMM15:dwarfReg = 32; break;
default:
noway_assert(!"unexpected REG_NUM");
}
return dwarfReg;
}
//------------------------------------------------------------------------
// Compiler::unwindSaveReg: Record a register save.
//
// Arguments:
// reg - The register being saved.
// offset - The offset from the current stack pointer where the register is being saved.
//
void Compiler::unwindSaveReg(regNumber reg, unsigned offset)
{
assert(compGeneratingProlog);
FuncInfoDsc* func = funCurrentFunc();
assert(func->unwindHeader.Version == 1); // Can't call this before unwindBegProlog
assert(func->unwindHeader.CountOfUnwindCodes == 0); // Can't call this after unwindReserve
if (RBM_CALLEE_SAVED & genRegMask(reg))
{
UNWIND_CODE * code;
if (offset < 0x80000)
{
assert(func->unwindCodeSlot > (sizeof(UNWIND_CODE) + sizeof(USHORT)));
USHORT * codedSize = (USHORT*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(USHORT)];
code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
// As per AMD64 ABI, if saving entire xmm reg, then offset need to be scaled by 16.
if (genIsValidFloatReg(reg))
{
*codedSize = (USHORT) (offset/16);
code->UnwindOp = UWOP_SAVE_XMM128;
}
else
{
*codedSize = (USHORT) (offset/8);
code->UnwindOp = UWOP_SAVE_NONVOL;
}
}
else
{
assert(func->unwindCodeSlot > (sizeof(UNWIND_CODE) + sizeof(ULONG)));
ULONG * codedSize = (ULONG*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(ULONG)];
*codedSize = offset;
code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
code->UnwindOp = (genIsValidFloatReg(reg)) ? UWOP_SAVE_XMM128_FAR : UWOP_SAVE_NONVOL_FAR;
}
code->OpInfo = (BYTE)reg;
unsigned int cbProlog = unwindGetCurrentOffset(func);
noway_assert((BYTE)cbProlog == cbProlog);
code->CodeOffset = (BYTE)cbProlog;
}
}
//------------------------------------------------------------------------
// DecomposeLclVar: Decompose GT_LCL_VAR.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeLclVar(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_LCL_VAR);
GenTree* tree = use.Def();
unsigned varNum = tree->AsLclVarCommon()->gtLclNum;
LclVarDsc* varDsc = m_compiler->lvaTable + varNum;
m_compiler->lvaDecRefCnts(tree);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
GenTree* hiResult = m_compiler->gtNewLclLNode(varNum, TYP_INT);
hiResult->CopyCosts(loResult);
BlockRange().InsertAfter(loResult, hiResult);
if (varDsc->lvPromoted)
{
assert(varDsc->lvFieldCnt == 2);
unsigned loVarNum = varDsc->lvFieldLclStart;
unsigned hiVarNum = loVarNum + 1;
loResult->AsLclVarCommon()->SetLclNum(loVarNum);
hiResult->AsLclVarCommon()->SetLclNum(hiVarNum);
}
else
{
noway_assert(varDsc->lvLRACandidate == false);
loResult->SetOper(GT_LCL_FLD);
loResult->AsLclFld()->gtLclOffs = 0;
loResult->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
hiResult->SetOper(GT_LCL_FLD);
hiResult->AsLclFld()->gtLclOffs = 4;
hiResult->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
}
m_compiler->lvaIncRefCnts(loResult);
m_compiler->lvaIncRefCnts(hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeLclVar: Decompose GT_LCL_VAR.
//
// Arguments:
// ppTree - the tree to decompose
// data - tree walk context
//
// Return Value:
// None.
//
void DecomposeLongs::DecomposeLclVar(GenTree** ppTree, Compiler::fgWalkData* data)
{
assert(ppTree != nullptr);
assert(*ppTree != nullptr);
assert(data != nullptr);
assert((*ppTree)->OperGet() == GT_LCL_VAR);
GenTree* tree = *ppTree;
unsigned varNum = tree->AsLclVarCommon()->gtLclNum;
LclVarDsc* varDsc = m_compiler->lvaTable + varNum;
m_compiler->lvaDecRefCnts(tree);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
GenTree* hiResult = m_compiler->gtNewLclLNode(varNum, TYP_INT);
if (varDsc->lvPromoted)
{
assert(varDsc->lvFieldCnt == 2);
unsigned loVarNum = varDsc->lvFieldLclStart;
unsigned hiVarNum = loVarNum + 1;
loResult->AsLclVarCommon()->SetLclNum(loVarNum);
hiResult->AsLclVarCommon()->SetLclNum(hiVarNum);
}
else
{
noway_assert(varDsc->lvLRACandidate == false);
loResult->SetOper(GT_LCL_FLD);
loResult->AsLclFld()->gtLclOffs = 0;
loResult->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
hiResult->SetOper(GT_LCL_FLD);
hiResult->AsLclFld()->gtLclOffs = 4;
hiResult->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
}
m_compiler->lvaIncRefCnts(loResult);
m_compiler->lvaIncRefCnts(hiResult);
FinalizeDecomposition(ppTree, data, loResult, hiResult);
}
//------------------------------------------------------------------------
// Compiler::unwindSetFrameReg: Record a frame register.
//
// Arguments:
// reg - The register being set as the frame register.
// offset - The offset from the current stack pointer that the frame pointer will point at.
//
void Compiler::unwindSetFrameReg(regNumber reg, unsigned offset)
{
assert(compGeneratingProlog);
FuncInfoDsc* func = funCurrentFunc();
assert(func->unwindHeader.Version == 1); // Can't call this before unwindBegProlog
assert(func->unwindHeader.CountOfUnwindCodes == 0); // Can't call this after unwindReserve
assert(func->unwindCodeSlot > sizeof(UNWIND_CODE));
UNWIND_CODE * code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
unsigned int cbProlog = unwindGetCurrentOffset(func);
noway_assert((BYTE)cbProlog == cbProlog);
code->CodeOffset = (BYTE)cbProlog;
code->UnwindOp = UWOP_SET_FPREG;
code->OpInfo = 0;
func->unwindHeader.FrameRegister = (BYTE)reg;
assert(offset <= 240);
assert(offset % 16 == 0);
func->unwindHeader.FrameOffset = offset / 16;
}
//------------------------------------------------------------------------
// DecomposeNeg: Decompose GT_NEG.
//
// Arguments:
// ppTree - the tree to decompose
// data - tree walk context
//
// Return Value:
// None.
//
void DecomposeLongs::DecomposeNeg(GenTree** ppTree, Compiler::fgWalkData* data)
{
assert(ppTree != nullptr);
assert(*ppTree != nullptr);
assert(data != nullptr);
assert((*ppTree)->OperGet() == GT_NEG);
assert(m_compiler->compCurStmt != nullptr);
GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt();
GenTree* tree = *ppTree;
GenTree* op1 = tree->gtGetOp1();
noway_assert(op1->OperGet() == GT_LONG);
CreateTemporary(&(op1->gtOp.gtOp1));
CreateTemporary(&(op1->gtOp.gtOp2));
// Neither GT_NEG nor the introduced temporaries have side effects.
tree->gtFlags &= ~GTF_ALL_EFFECT;
GenTree* loOp1 = op1->gtGetOp1();
GenTree* hiOp1 = op1->gtGetOp2();
Compiler::fgSnipNode(curStmt, op1);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
GenTree* zero = m_compiler->gtNewZeroConNode(TYP_INT);
GenTree* hiAdjust = m_compiler->gtNewOperNode(GT_ADD_HI, TYP_INT, hiOp1, zero);
GenTree* hiResult = m_compiler->gtNewOperNode(GT_NEG, TYP_INT, hiAdjust);
hiResult->gtFlags = tree->gtFlags;
Compiler::fgSnipNode(curStmt, hiOp1);
// fgSnipNode doesn't clear gtNext/gtPrev...
hiOp1->gtNext = nullptr;
hiOp1->gtPrev = nullptr;
SimpleLinkNodeAfter(hiOp1, zero);
SimpleLinkNodeAfter(zero, hiAdjust);
SimpleLinkNodeAfter(hiAdjust, hiResult);
FinalizeDecomposition(ppTree, data, loResult, hiResult);
}
//------------------------------------------------------------------------
// LowerCast: Lower GT_CAST(srcType, DstType) nodes.
//
// Arguments:
// tree - GT_CAST node to be lowered
//
// Return Value:
// None.
//
// Notes:
// Casts from float/double to a smaller int type are transformed as follows:
// GT_CAST(float/double, byte) = GT_CAST(GT_CAST(float/double, int32), byte)
// GT_CAST(float/double, sbyte) = GT_CAST(GT_CAST(float/double, int32), sbyte)
// GT_CAST(float/double, int16) = GT_CAST(GT_CAST(double/double, int32), int16)
// GT_CAST(float/double, uint16) = GT_CAST(GT_CAST(double/double, int32), uint16)
//
// Note that for the overflow conversions we still depend on helper calls and
// don't expect to see them here.
// i) GT_CAST(float/double, int type with overflow detection)
//
void Lowering::LowerCast(GenTree* tree)
{
assert(tree->OperGet() == GT_CAST);
JITDUMP("LowerCast for: ");
DISPNODE(tree);
JITDUMP("\n");
GenTreePtr op1 = tree->gtOp.gtOp1;
var_types dstType = tree->CastToType();
var_types srcType = genActualType(op1->TypeGet());
var_types tmpType = TYP_UNDEF;
if (varTypeIsFloating(srcType))
{
noway_assert(!tree->gtOverflow());
}
assert(!varTypeIsSmall(srcType));
// case of src is a floating point type and dst is a small type.
if (varTypeIsFloating(srcType) && varTypeIsSmall(dstType))
{
NYI_ARM("Lowering for cast from float to small type"); // Not tested yet.
tmpType = TYP_INT;
}
if (tmpType != TYP_UNDEF)
{
GenTreePtr tmp = comp->gtNewCastNode(tmpType, op1, tmpType);
tmp->gtFlags |= (tree->gtFlags & (GTF_UNSIGNED | GTF_OVERFLOW | GTF_EXCEPT));
tree->gtFlags &= ~GTF_UNSIGNED;
tree->gtOp.gtOp1 = tmp;
BlockRange().InsertAfter(op1, tmp);
}
// Now determine if we have operands that should be contained.
ContainCheckCast(tree->AsCast());
}
//------------------------------------------------------------------------
// DecomposeNeg: Decompose GT_NEG.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeNeg(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_NEG);
GenTree* tree = use.Def();
GenTree* gtLong = tree->gtGetOp1();
noway_assert(gtLong->OperGet() == GT_LONG);
unsigned blockWeight = m_block->getBBWeight(m_compiler);
LIR::Use op1(BlockRange(), >Long->gtOp.gtOp1, gtLong);
op1.ReplaceWithLclVar(m_compiler, blockWeight);
LIR::Use op2(BlockRange(), >Long->gtOp.gtOp2, gtLong);
op2.ReplaceWithLclVar(m_compiler, blockWeight);
// Neither GT_NEG nor the introduced temporaries have side effects.
tree->gtFlags &= ~GTF_ALL_EFFECT;
GenTree* loOp1 = gtLong->gtGetOp1();
GenTree* hiOp1 = gtLong->gtGetOp2();
BlockRange().Remove(gtLong);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
GenTree* zero = m_compiler->gtNewZeroConNode(TYP_INT);
GenTree* hiAdjust = m_compiler->gtNewOperNode(GT_ADD_HI, TYP_INT, hiOp1, zero);
GenTree* hiResult = m_compiler->gtNewOperNode(GT_NEG, TYP_INT, hiAdjust);
hiResult->gtFlags = tree->gtFlags;
// Annotate new nodes with costs. This will re-cost the hiOp1 tree as well.
m_compiler->gtPrepareCost(hiResult);
BlockRange().InsertAfter(loResult, zero, hiAdjust, hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// Compiler::unwindAllocStack: Record a stack frame allocation (sub sp, X).
//
// Arguments:
// size - The size of the stack frame allocation (the amount subtracted from the stack pointer).
//
void Compiler::unwindAllocStack(unsigned size)
{
assert(compGeneratingProlog);
FuncInfoDsc* func = funCurrentFunc();
assert(func->unwindHeader.Version == 1); // Can't call this before unwindBegProlog
assert(func->unwindHeader.CountOfUnwindCodes == 0); // Can't call this after unwindReserve
assert(size % 8 == 0); // Stack size is *always* 8 byte aligned
UNWIND_CODE * code;
if (size <= 128)
{
assert(func->unwindCodeSlot > sizeof(UNWIND_CODE));
code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
code->UnwindOp = UWOP_ALLOC_SMALL;
code->OpInfo = (size - 8) / 8;
}
else if (size <= 0x7FFF8)
{
assert(func->unwindCodeSlot > (sizeof(UNWIND_CODE) + sizeof(USHORT)));
USHORT * codedSize = (USHORT*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(USHORT)];
*codedSize = (USHORT)(size / 8);
code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
code->UnwindOp = UWOP_ALLOC_LARGE;
code->OpInfo = 0;
}
else
{
assert(func->unwindCodeSlot > (sizeof(UNWIND_CODE) + sizeof(ULONG)));
ULONG * codedSize = (ULONG*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(ULONG)];
*codedSize = size;
code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
code->UnwindOp = UWOP_ALLOC_LARGE;
code->OpInfo = 1;
}
unsigned int cbProlog = unwindGetCurrentOffset(func);
noway_assert((BYTE)cbProlog == cbProlog);
code->CodeOffset = (BYTE)cbProlog;
}
//------------------------------------------------------------------------
// DecomposeNot: Decompose GT_NOT.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeNot(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_NOT);
GenTree* tree = use.Def();
GenTree* gtLong = tree->gtGetOp1();
noway_assert(gtLong->OperGet() == GT_LONG);
GenTree* loOp1 = gtLong->gtGetOp1();
GenTree* hiOp1 = gtLong->gtGetOp2();
Range().Remove(gtLong);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
GenTree* hiResult = new (m_compiler, GT_NOT) GenTreeOp(GT_NOT, TYP_INT, hiOp1, nullptr);
Range().InsertAfter(loResult, hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
void Compiler::unwindPushCFI(regNumber reg)
{
assert(compGeneratingProlog);
FuncInfoDsc* func = funCurrentFunc();
unsigned int cbProlog = unwindGetCurrentOffset(func);
noway_assert((BYTE)cbProlog == cbProlog);
createCfiCode(func, cbProlog, CFI_ADJUST_CFA_OFFSET, DWARF_REG_ILLEGAL, 8);
if ((RBM_CALLEE_SAVED & genRegMask(reg))
#if ETW_EBP_FRAMED
// In case of ETW_EBP_FRAMED defined the REG_FPBASE (RBP)
// is excluded from the callee-save register list.
// Make sure the register gets PUSH unwind info in this case,
// since it is pushed as a frame register.
|| (reg == REG_FPBASE)
#endif // ETW_EBP_FRAMED
)
{
createCfiCode(func, cbProlog, CFI_REL_OFFSET, mapRegNumToDwarfReg(reg));
}
}
//------------------------------------------------------------------------
// TreeNodeInfoInitBlockStore: Set the NodeInfo for a block store.
//
// Arguments:
// blkNode - The block store node of interest
//
// Return Value:
// None.
//
void Lowering::TreeNodeInfoInitBlockStore(GenTreeBlk* blkNode)
{
GenTree* dstAddr = blkNode->Addr();
unsigned size = blkNode->gtBlkSize;
GenTree* source = blkNode->Data();
LinearScan* l = m_lsra;
Compiler* compiler = comp;
// Sources are dest address and initVal or source.
// We may require an additional source or temp register for the size.
blkNode->gtLsraInfo.srcCount = 2;
blkNode->gtLsraInfo.dstCount = 0;
GenTreePtr srcAddrOrFill = nullptr;
bool isInitBlk = blkNode->OperIsInitBlkOp();
if (!isInitBlk)
{
// CopyObj or CopyBlk
if (source->gtOper == GT_IND)
{
srcAddrOrFill = blkNode->Data()->gtGetOp1();
// We're effectively setting source as contained, but can't call MakeSrcContained, because the
// "inheritance" of the srcCount is to a child not a parent - it would "just work" but could be misleading.
// If srcAddr is already non-contained, we don't need to change it.
if (srcAddrOrFill->gtLsraInfo.getDstCount() == 0)
{
srcAddrOrFill->gtLsraInfo.setDstCount(1);
srcAddrOrFill->gtLsraInfo.setSrcCount(source->gtLsraInfo.srcCount);
}
m_lsra->clearOperandCounts(source);
source->SetContained();
source->AsIndir()->Addr()->ClearContained();
}
else if (!source->IsMultiRegCall() && !source->OperIsSIMD())
{
assert(source->IsLocal());
MakeSrcContained(blkNode, source);
blkNode->gtLsraInfo.srcCount--;
}
}
if (isInitBlk)
{
GenTreePtr initVal = source;
if (initVal->OperIsInitVal())
{
initVal->SetContained();
initVal = initVal->gtGetOp1();
}
srcAddrOrFill = initVal;
if (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindUnroll)
{
// TODO-ARM-CQ: Currently we generate a helper call for every
// initblk we encounter. Later on we should implement loop unrolling
// code sequences to improve CQ.
// For reference see the code in lsraxarch.cpp.
NYI_ARM("initblk loop unrolling is currently not implemented.");
#ifdef _TARGET_ARM64_
// No additional temporaries required
ssize_t fill = initVal->gtIntCon.gtIconVal & 0xFF;
if (fill == 0)
{
MakeSrcContained(blkNode, source);
blkNode->gtLsraInfo.srcCount--;
}
#endif // _TARGET_ARM64_
}
else
{
assert(blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindHelper);
// The helper follows the regular ABI.
dstAddr->gtLsraInfo.setSrcCandidates(l, RBM_ARG_0);
initVal->gtLsraInfo.setSrcCandidates(l, RBM_ARG_1);
if (size != 0)
{
// Reserve a temp register for the block size argument.
blkNode->gtLsraInfo.setInternalCandidates(l, RBM_ARG_2);
blkNode->gtLsraInfo.internalIntCount = 1;
}
else
{
// The block size argument is a third argument to GT_STORE_DYN_BLK
noway_assert(blkNode->gtOper == GT_STORE_DYN_BLK);
blkNode->gtLsraInfo.setSrcCount(3);
GenTree* sizeNode = blkNode->AsDynBlk()->gtDynamicSize;
sizeNode->gtLsraInfo.setSrcCandidates(l, RBM_ARG_2);
}
}
}
else
{
// CopyObj or CopyBlk
// Sources are src and dest and size if not constant.
if (blkNode->OperGet() == GT_STORE_OBJ)
{
// CopyObj
// We don't need to materialize the struct size but we still need
// a temporary register to perform the sequence of loads and stores.
blkNode->gtLsraInfo.internalIntCount = 1;
if (size >= 2 * REGSIZE_BYTES)
{
// We will use ldp/stp to reduce code size and improve performance
// so we need to reserve an extra internal register
blkNode->gtLsraInfo.internalIntCount++;
}
// We can't use the special Write Barrier registers, so exclude them from the mask
regMaskTP internalIntCandidates = RBM_ALLINT & ~(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF);
blkNode->gtLsraInfo.setInternalCandidates(l, internalIntCandidates);
// If we have a dest address we want it in RBM_WRITE_BARRIER_DST_BYREF.
dstAddr->gtLsraInfo.setSrcCandidates(l, RBM_WRITE_BARRIER_DST_BYREF);
// If we have a source address we want it in REG_WRITE_BARRIER_SRC_BYREF.
// Otherwise, if it is a local, codegen will put its address in REG_WRITE_BARRIER_SRC_BYREF,
// which is killed by a StoreObj (and thus needn't be reserved).
if (srcAddrOrFill != nullptr)
{
srcAddrOrFill->gtLsraInfo.setSrcCandidates(l, RBM_WRITE_BARRIER_SRC_BYREF);
}
}
else
{
// CopyBlk
short internalIntCount = 0;
regMaskTP internalIntCandidates = RBM_NONE;
if (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindUnroll)
{
// TODO-ARM-CQ: cpblk loop unrolling is currently not implemented.
// In case of a CpBlk with a constant size and less than CPBLK_UNROLL_LIMIT size
// we should unroll the loop to improve CQ.
// For reference see the code in lsraxarch.cpp.
NYI_ARM("cpblk loop unrolling is currently not implemented.");
#ifdef _TARGET_ARM64_
internalIntCount = 1;
internalIntCandidates = RBM_ALLINT;
if (size >= 2 * REGSIZE_BYTES)
{
// We will use ldp/stp to reduce code size and improve performance
// so we need to reserve an extra internal register
internalIntCount++;
}
#endif // _TARGET_ARM64_
}
else
{
assert(blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindHelper);
dstAddr->gtLsraInfo.setSrcCandidates(l, RBM_ARG_0);
// The srcAddr goes in arg1.
if (srcAddrOrFill != nullptr)
{
srcAddrOrFill->gtLsraInfo.setSrcCandidates(l, RBM_ARG_1);
}
if (size != 0)
{
// Reserve a temp register for the block size argument.
internalIntCandidates |= RBM_ARG_2;
internalIntCount++;
}
else
{
// The block size argument is a third argument to GT_STORE_DYN_BLK
noway_assert(blkNode->gtOper == GT_STORE_DYN_BLK);
blkNode->gtLsraInfo.setSrcCount(3);
GenTree* blockSize = blkNode->AsDynBlk()->gtDynamicSize;
blockSize->gtLsraInfo.setSrcCandidates(l, RBM_ARG_2);
}
}
if (internalIntCount != 0)
{
blkNode->gtLsraInfo.internalIntCount = internalIntCount;
blkNode->gtLsraInfo.setInternalCandidates(l, internalIntCandidates);
}
}
}
}
//------------------------------------------------------------------------
// DecomposeStoreLclVar: Decompose GT_STORE_LCL_VAR.
//
// Arguments:
// ppTree - the tree to decompose
// data - tree walk context
//
// Return Value:
// None.
//
void DecomposeLongs::DecomposeStoreLclVar(GenTree** ppTree, Compiler::fgWalkData* data)
{
assert(ppTree != nullptr);
assert(*ppTree != nullptr);
assert(data != nullptr);
assert((*ppTree)->OperGet() == GT_STORE_LCL_VAR);
assert(m_compiler->compCurStmt != nullptr);
GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt();
GenTree* tree = *ppTree;
GenTree* nextTree = tree->gtNext;
GenTree* rhs = tree->gtGetOp1();
if ((rhs->OperGet() == GT_PHI) || (rhs->OperGet() == GT_CALL))
{
// GT_CALLs are not decomposed, so will not be converted to GT_LONG
// GT_STORE_LCL_VAR = GT_CALL are handled in genMultiRegCallStoreToLocal
return;
}
noway_assert(rhs->OperGet() == GT_LONG);
unsigned varNum = tree->AsLclVarCommon()->gtLclNum;
LclVarDsc* varDsc = m_compiler->lvaTable + varNum;
m_compiler->lvaDecRefCnts(tree);
GenTree* loRhs = rhs->gtGetOp1();
GenTree* hiRhs = rhs->gtGetOp2();
GenTree* hiStore = m_compiler->gtNewLclLNode(varNum, TYP_INT);
if (varDsc->lvPromoted)
{
assert(varDsc->lvFieldCnt == 2);
unsigned loVarNum = varDsc->lvFieldLclStart;
unsigned hiVarNum = loVarNum + 1;
tree->AsLclVarCommon()->SetLclNum(loVarNum);
hiStore->SetOper(GT_STORE_LCL_VAR);
hiStore->AsLclVarCommon()->SetLclNum(hiVarNum);
}
else
{
noway_assert(varDsc->lvLRACandidate == false);
tree->SetOper(GT_STORE_LCL_FLD);
tree->AsLclFld()->gtLclOffs = 0;
tree->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
hiStore->SetOper(GT_STORE_LCL_FLD);
hiStore->AsLclFld()->gtLclOffs = 4;
hiStore->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
}
tree->gtOp.gtOp1 = loRhs;
tree->gtType = TYP_INT;
loRhs->gtNext = tree;
tree->gtPrev = loRhs;
hiStore->gtOp.gtOp1 = hiRhs;
hiStore->CopyCosts(tree);
hiStore->gtFlags |= GTF_VAR_DEF;
m_compiler->lvaIncRefCnts(tree);
m_compiler->lvaIncRefCnts(hiStore);
tree->gtNext = hiRhs;
hiRhs->gtPrev = tree;
hiRhs->gtNext = hiStore;
hiStore->gtPrev = hiRhs;
hiStore->gtNext = nextTree;
if (nextTree != nullptr)
{
nextTree->gtPrev = hiStore;
}
nextTree = hiRhs;
bool isEmbeddedStmt = !curStmt->gtStmtIsTopLevel();
if (!isEmbeddedStmt)
{
tree->gtNext = nullptr;
hiRhs->gtPrev = nullptr;
}
InsertNodeAsStmt(hiStore);
}
//------------------------------------------------------------------------
// DecomposeNode: Decompose long-type trees into lower and upper halves.
//
// Arguments:
// *ppTree - A node that may or may not require decomposition.
// data - The tree-walk data that provides the context.
//
// Return Value:
// None. It the tree at *ppTree is of TYP_LONG, it will generally be replaced.
//
void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data)
{
GenTree* tree = *ppTree;
// Handle the case where we are implicitly using the lower half of a long lclVar.
if ((tree->TypeGet() == TYP_INT) && tree->OperIsLocal())
{
LclVarDsc* varDsc = m_compiler->lvaTable + tree->AsLclVarCommon()->gtLclNum;
if (varTypeIsLong(varDsc) && varDsc->lvPromoted)
{
#ifdef DEBUG
if (m_compiler->verbose)
{
printf("Changing implicit reference to lo half of long lclVar to an explicit reference of its promoted half:\n");
m_compiler->gtDispTree(tree);
}
#endif // DEBUG
m_compiler->lvaDecRefCnts(tree);
unsigned loVarNum = varDsc->lvFieldLclStart;
tree->AsLclVarCommon()->SetLclNum(loVarNum);
m_compiler->lvaIncRefCnts(tree);
return;
}
}
if (tree->TypeGet() != TYP_LONG)
{
return;
}
#ifdef DEBUG
if (m_compiler->verbose)
{
printf("Decomposing TYP_LONG tree. BEFORE:\n");
m_compiler->gtDispTree(tree);
}
#endif // DEBUG
switch (tree->OperGet())
{
case GT_PHI:
case GT_PHI_ARG:
break;
case GT_LCL_VAR:
DecomposeLclVar(ppTree, data);
break;
case GT_LCL_FLD:
DecomposeLclFld(ppTree, data);
break;
case GT_STORE_LCL_VAR:
DecomposeStoreLclVar(ppTree, data);
break;
case GT_CAST:
DecomposeCast(ppTree, data);
break;
case GT_CNS_LNG:
DecomposeCnsLng(ppTree, data);
break;
case GT_CALL:
DecomposeCall(ppTree, data);
break;
case GT_RETURN:
assert(tree->gtOp.gtOp1->OperGet() == GT_LONG);
break;
case GT_STOREIND:
DecomposeStoreInd(ppTree, data);
break;
case GT_STORE_LCL_FLD:
assert(tree->gtOp.gtOp1->OperGet() == GT_LONG);
NYI("st.lclFld of of TYP_LONG");
break;
case GT_IND:
DecomposeInd(ppTree, data);
break;
case GT_NOT:
DecomposeNot(ppTree, data);
break;
case GT_NEG:
DecomposeNeg(ppTree, data);
break;
// Binary operators. Those that require different computation for upper and lower half are
// handled by the use of GetHiOper().
case GT_ADD:
case GT_SUB:
case GT_OR:
case GT_XOR:
case GT_AND:
DecomposeArith(ppTree, data);
break;
case GT_MUL:
NYI("Arithmetic binary operators on TYP_LONG - GT_MUL");
break;
case GT_DIV:
NYI("Arithmetic binary operators on TYP_LONG - GT_DIV");
break;
case GT_MOD:
NYI("Arithmetic binary operators on TYP_LONG - GT_MOD");
break;
case GT_UDIV:
NYI("Arithmetic binary operators on TYP_LONG - GT_UDIV");
break;
case GT_UMOD:
NYI("Arithmetic binary operators on TYP_LONG - GT_UMOD");
break;
case GT_LSH:
case GT_RSH:
case GT_RSZ:
NYI("Arithmetic binary operators on TYP_LONG - SHIFT");
break;
case GT_ROL:
case GT_ROR:
NYI("Arithmetic binary operators on TYP_LONG - ROTATE");
break;
case GT_MULHI:
NYI("Arithmetic binary operators on TYP_LONG - MULHI");
break;
case GT_LOCKADD:
case GT_XADD:
case GT_XCHG:
case GT_CMPXCHG:
NYI("Interlocked operations on TYP_LONG");
break;
default:
{
JITDUMP("Illegal TYP_LONG node %s in Decomposition.", GenTree::NodeName(tree->OperGet()));
noway_assert(!"Illegal TYP_LONG node in Decomposition.");
break;
}
}
#ifdef DEBUG
if (m_compiler->verbose)
{
printf(" AFTER:\n");
m_compiler->gtDispTree(*ppTree);
}
#endif
}
//------------------------------------------------------------------------
// DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND.
//
// Arguments:
// ppTree - the tree to decompose
// data - tree walk context
//
// Return Value:
// None.
//
void DecomposeLongs::DecomposeArith(GenTree** ppTree, Compiler::fgWalkData* data)
{
assert(ppTree != nullptr);
assert(*ppTree != nullptr);
assert(data != nullptr);
assert(m_compiler->compCurStmt != nullptr);
GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt();
GenTree* tree = *ppTree;
genTreeOps oper = tree->OperGet();
assert((oper == GT_ADD) ||
(oper == GT_SUB) ||
(oper == GT_OR) ||
(oper == GT_XOR) ||
(oper == GT_AND));
NYI_IF((tree->gtFlags & GTF_REVERSE_OPS) != 0, "Binary operator with GTF_REVERSE_OPS");
GenTree* op1 = tree->gtGetOp1();
GenTree* op2 = tree->gtGetOp2();
// Both operands must have already been decomposed into GT_LONG operators.
noway_assert((op1->OperGet() == GT_LONG) && (op2->OperGet() == GT_LONG));
// Capture the lo and hi halves of op1 and op2.
GenTree* loOp1 = op1->gtGetOp1();
GenTree* hiOp1 = op1->gtGetOp2();
GenTree* loOp2 = op2->gtGetOp1();
GenTree* hiOp2 = op2->gtGetOp2();
// We don't have support to decompose a TYP_LONG node that already has a child that has
// been decomposed into parts, where the high part depends on the value generated by the
// low part (via the flags register). For example, if we have:
// +(gt_long(+(lo3, lo4), +Hi(hi3, hi4)), gt_long(lo2, hi2))
// We would decompose it here to:
// gt_long(+(+(lo3, lo4), lo2), +Hi(+Hi(hi3, hi4), hi2))
// But this would generate incorrect code, because the "+Hi(hi3, hi4)" code generation
// needs to immediately follow the "+(lo3, lo4)" part. Also, if this node is one that
// requires a unique high operator, and the child nodes are not simple locals (e.g.,
// they are decomposed nodes), then we also can't decompose the node, as we aren't
// guaranteed the high and low parts will be executed immediately after each other.
NYI_IF(hiOp1->OperIsHigh() ||
hiOp2->OperIsHigh() ||
(GenTree::OperIsHigh(GetHiOper(oper)) &&
(!loOp1->OperIsLeaf() ||
!hiOp1->OperIsLeaf() ||
!loOp1->OperIsLeaf() ||
!hiOp2->OperIsLeaf())),
"Can't decompose expression tree TYP_LONG node");
// Now, remove op1 and op2 from the node list.
m_compiler->fgSnipNode(curStmt, op1);
m_compiler->fgSnipNode(curStmt, op2);
// We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands
// will be the lo halves of op1 from above.
GenTree* loResult = tree;
loResult->SetOper(GetLoOper(loResult->OperGet()));
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
loResult->gtOp.gtOp2 = loOp2;
// The various halves will be correctly threaded internally. We simply need to
// relink them into the proper order, i.e. loOp1 is followed by loOp2, and then
// the loResult node.
// (This rethreading, and that below, are where we need to address the reverse ops case).
// The current order is (after snipping op1 and op2):
// ... loOp1-> ... hiOp1->loOp2First ... loOp2->hiOp2First ... hiOp2
// The order we want is:
// ... loOp1->loOp2First ... loOp2->loResult
// ... hiOp1->hiOp2First ... hiOp2->hiResult
// i.e. we swap hiOp1 and loOp2, and create (for now) separate loResult and hiResult trees
GenTree* loOp2First = hiOp1->gtNext;
GenTree* hiOp2First = loOp2->gtNext;
// First, we will NYI if both hiOp1 and loOp2 have side effects.
NYI_IF(((loOp2->gtFlags & GTF_ALL_EFFECT) != 0) && ((hiOp1->gtFlags & GTF_ALL_EFFECT) != 0),
"Binary long operator with non-reorderable sub expressions");
// Now, we reorder the loOps and the loResult.
loOp1->gtNext = loOp2First;
loOp2First->gtPrev = loOp1;
loOp2->gtNext = loResult;
loResult->gtPrev = loOp2;
// Next, reorder the hiOps and the hiResult.
GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2);
hiOp1->gtNext = hiOp2First;
hiOp2First->gtPrev = hiOp1;
hiOp2->gtNext = hiResult;
hiResult->gtPrev = hiOp2;
if ((oper == GT_ADD) || (oper == GT_SUB))
{
if (loResult->gtOverflow())
{
hiResult->gtFlags |= GTF_OVERFLOW;
loResult->gtFlags &= ~GTF_OVERFLOW;
}
if (loResult->gtFlags & GTF_UNSIGNED)
{
hiResult->gtFlags |= GTF_UNSIGNED;
}
}
FinalizeDecomposition(ppTree, data, loResult, hiResult);
}
// Merge assertions on the edge flowing into the block about a variable.
void RangeCheck::MergeEdgeAssertions(GenTreePtr tree, EXPSET_TP assertions, Range* pRange)
{
if (assertions == 0)
{
return;
}
GenTreeLclVarCommon* lcl = (GenTreeLclVarCommon*) tree;
if (lcl->gtSsaNum == SsaConfig::RESERVED_SSA_NUM)
{
return;
}
// Walk through the "assertions" to check if the apply.
unsigned index = 1;
for (EXPSET_TP mask = 1; index <= m_pCompiler->GetAssertionCount(); index++, mask <<= 1)
{
if ((assertions & mask) == 0)
{
continue;
}
Compiler::AssertionDsc* curAssertion = m_pCompiler->optGetAssertion(index);
// Current assertion is about array length.
if (!curAssertion->IsArrLenArithBound() &&
!curAssertion->IsArrLenBound())
{
continue;
}
#ifdef DEBUG
if (m_pCompiler->verbose)
{
m_pCompiler->optPrintAssertion(curAssertion, index);
}
#endif
assert(m_pCompiler->vnStore->IsVNArrLenArithBound(curAssertion->op1.vn) ||
m_pCompiler->vnStore->IsVNArrLenBound(curAssertion->op1.vn));
ValueNumStore::ArrLenArithBoundInfo info;
Limit limit(Limit::keUndef);
// Current assertion is of the form (i < a.len - cns) != 0
if (curAssertion->IsArrLenArithBound())
{
// Get i, a.len, cns and < as "info."
m_pCompiler->vnStore->GetArrLenArithBoundInfo(curAssertion->op1.vn, &info);
if (m_pCompiler->lvaTable[lcl->gtLclNum].GetPerSsaData(lcl->gtSsaNum)->m_vnPair.GetConservative()
!= info.cmpOp)
{
continue;
}
switch (info.arrOper)
{
case GT_SUB:
case GT_ADD:
{
// If the operand that operates on the array is not constant, then done.
if (!m_pCompiler->vnStore->IsVNConstant(info.arrOp) || m_pCompiler->vnStore->TypeOfVN(info.arrOp) != TYP_INT)
{
break;
}
int cons = m_pCompiler->vnStore->ConstantValue<int>(info.arrOp);
limit = Limit(Limit::keBinOpArray, info.vnArray, info.arrOper == GT_SUB ? -cons : cons);
}
}
}
// Current assertion is of the form (i < a.len) != 0
else if (curAssertion->IsArrLenBound())
{
// Get the info as "i", "<" and "a.len"
m_pCompiler->vnStore->GetArrLenBoundInfo(curAssertion->op1.vn, &info);
ValueNum lclVn = m_pCompiler->lvaTable[lcl->gtLclNum].GetPerSsaData(lcl->gtSsaNum)->m_vnPair.GetConservative();
// If we don't have the same variable we are comparing against, bail.
if (lclVn != info.cmpOp)
{
continue;
}
limit.type = Limit::keArray;
limit.vn = info.vnArray;
}
else
{
noway_assert(false);
}
if (limit.IsUndef())
{
continue;
}
// Make sure the assertion is of the form != 0 or == 0.
if (curAssertion->op2.vn != m_pCompiler->vnStore->VNZeroForType(TYP_INT))
{
continue;
}
#ifdef DEBUG
if (m_pCompiler->verbose) m_pCompiler->optPrintAssertion(curAssertion, index);
#endif
noway_assert(limit.IsBinOpArray() || limit.IsArray());
ValueNum arrLenVN = m_pCurBndsChk->gtArrLen->gtVNPair.GetConservative();
ValueNum arrRefVN = m_pCompiler->vnStore->GetArrForLenVn(arrLenVN);
// During assertion prop we add assertions of the form:
//
// (i < a.Length) == 0
// (i < a.Length) != 0
//
// At this point, we have detected that op1.vn is (i < a.Length) or (i < a.Length + cns),
// and the op2.vn is 0.
//
// Now, let us check if we are == 0 (i.e., op1 assertion is false) or != 0 (op1 assertion
// is true.),
//
// If we have an assertion of the form == 0 (i.e., equals false), then reverse relop.
// The relop has to be reversed because we have: (i < a.Length) is false which is the same
// as (i >= a.Length).
genTreeOps cmpOper = (genTreeOps) info.cmpOper;
if (curAssertion->assertionKind == Compiler::OAK_EQUAL)
{
cmpOper = GenTree::ReverseRelop(cmpOper);
}
// Bounds are inclusive, so add -1 for upper bound when "<". But make sure we won't overflow.
if (cmpOper == GT_LT && !limit.AddConstant(-1))
{
continue;
}
// Bounds are inclusive, so add +1 for lower bound when ">". But make sure we won't overflow.
if (cmpOper == GT_GT && !limit.AddConstant(1))
{
continue;
}
// Doesn't tighten the current bound. So skip.
if (pRange->uLimit.IsConstant() && limit.vn != arrRefVN)
{
continue;
}
// Check if the incoming limit from assertions tightens the existing upper limit.
if ((pRange->uLimit.IsArray() || pRange->uLimit.IsBinOpArray()) && pRange->uLimit.vn == arrRefVN)
{
// We have checked the current range's (pRange's) upper limit is either of the form:
// a.Length
// a.Length + cns
// and a == the bndsChkCandidate's arrRef
//
// We want to check if the incoming limit tightens the bound, and for that the
// we need to make sure that incoming limit is also on a.Length or a.Length + cns
// and not b.Length or some c.Length.
if (limit.vn != arrRefVN)
{
JITDUMP("Array ref did not match cur=$%x, assert=$%x\n", arrRefVN, limit.vn);
continue;
}
int curCns = (pRange->uLimit.IsBinOpArray()) ? pRange->uLimit.cns : 0;
int limCns = (limit.IsBinOpArray()) ? limit.cns : 0;
// Incoming limit doesn't tighten the existing upper limit.
if (limCns >= curCns)
{
JITDUMP("Bound limit %d doesn't tighten current bound %d\n", limCns, curCns);
continue;
}
}
else
{
// Current range's upper bound is not "a.Length or a.Length + cns" and the
// incoming limit is not on the same arrRef as the bounds check candidate.
// So we could skip this assertion. But in cases, of Dependent or Unknown
// type of upper limit, the incoming assertion still tightens the upper
// bound to a saner value. So do not skip the assertion.
}
// cmpOp (loop index i) cmpOper a.len +/- cns
switch (cmpOper)
{
case GT_LT:
pRange->uLimit = limit;
break;
case GT_GT:
pRange->lLimit = limit;
break;
case GT_GE:
pRange->lLimit = limit;
break;
case GT_LE:
pRange->uLimit = limit;
break;
}
JITDUMP("The range after edge merging:");
JITDUMP(pRange->ToString(m_pCompiler->getAllocatorDebugOnly()));
JITDUMP("\n");
}
}
void Compiler::raMarkStkVars()
{
unsigned lclNum;
LclVarDsc* varDsc;
for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
{
// lvOnFrame is set by LSRA, except in the case of zero-ref, which is set below.
if (lvaIsFieldOfDependentlyPromotedStruct(varDsc))
{
noway_assert(!varDsc->lvRegister);
goto ON_STK;
}
/* Fully enregistered variables don't need any frame space */
if (varDsc->lvRegister)
{
goto NOT_STK;
}
/* Unused variables typically don't get any frame space */
else if (varDsc->lvRefCnt() == 0)
{
bool needSlot = false;
bool stkFixedArgInVarArgs =
info.compIsVarArgs && varDsc->lvIsParam && !varDsc->lvIsRegArg && lclNum != lvaVarargsHandleArg;
// If its address has been exposed, ignore lvRefCnt. However, exclude
// fixed arguments in varargs method as lvOnFrame shouldn't be set
// for them as we don't want to explicitly report them to GC.
if (!stkFixedArgInVarArgs)
{
needSlot |= varDsc->lvAddrExposed;
}
#if FEATURE_FIXED_OUT_ARGS
/* Is this the dummy variable representing GT_LCLBLK ? */
needSlot |= (lclNum == lvaOutgoingArgSpaceVar);
#endif // FEATURE_FIXED_OUT_ARGS
#ifdef DEBUG
/* For debugging, note that we have to reserve space even for
unused variables if they are ever in scope. However, this is not
an issue as fgExtendDbgLifetimes() adds an initialization and
variables in scope will not have a zero ref-cnt.
*/
if (opts.compDbgCode && !varDsc->lvIsParam && varDsc->lvTracked)
{
for (unsigned scopeNum = 0; scopeNum < info.compVarScopesCount; scopeNum++)
{
noway_assert(info.compVarScopes[scopeNum].vsdVarNum != lclNum);
}
}
#endif
/*
For Debug Code, we have to reserve space even if the variable is never
in scope. We will also need to initialize it if it is a GC var.
So we set lvMustInit and verify it has a nonzero ref-cnt.
*/
if (opts.compDbgCode && !stkFixedArgInVarArgs && lclNum < info.compLocalsCount)
{
if (varDsc->lvRefCnt() == 0)
{
assert(!"unreferenced local in debug codegen");
varDsc->lvImplicitlyReferenced = 1;
}
needSlot |= true;
if (!varDsc->lvIsParam)
{
varDsc->lvMustInit = true;
}
}
varDsc->lvOnFrame = needSlot;
if (!needSlot)
{
/* Clear the lvMustInit flag in case it is set */
varDsc->lvMustInit = false;
goto NOT_STK;
}
}
if (!varDsc->lvOnFrame)
{
goto NOT_STK;
}
ON_STK:
/* The variable (or part of it) lives on the stack frame */
noway_assert((varDsc->lvType != TYP_UNDEF) && (varDsc->lvType != TYP_VOID) && (varDsc->lvType != TYP_UNKNOWN));
#if FEATURE_FIXED_OUT_ARGS
noway_assert((lclNum == lvaOutgoingArgSpaceVar) || lvaLclSize(lclNum) != 0);
#else // FEATURE_FIXED_OUT_ARGS
noway_assert(lvaLclSize(lclNum) != 0);
#endif // FEATURE_FIXED_OUT_ARGS
varDsc->lvOnFrame = true; // Our prediction is that the final home for this local variable will be in the
// stack frame
NOT_STK:;
varDsc->lvFramePointerBased = codeGen->isFramePointerUsed();
#if DOUBLE_ALIGN
if (codeGen->doDoubleAlign())
{
noway_assert(codeGen->isFramePointerUsed() == false);
/* All arguments are off of EBP with double-aligned frames */
if (varDsc->lvIsParam && !varDsc->lvIsRegArg)
{
varDsc->lvFramePointerBased = true;
}
}
#endif
/* Some basic checks */
// It must be in a register, on frame, or have zero references.
noway_assert(varDsc->lvIsInReg() || varDsc->lvOnFrame || varDsc->lvRefCnt() == 0);
// We can't have both lvRegister and lvOnFrame
noway_assert(!varDsc->lvRegister || !varDsc->lvOnFrame);
#ifdef DEBUG
// For varargs functions, there should be no direct references to
// parameter variables except for 'this' (because these were morphed
// in the importer) and the 'arglist' parameter (which is not a GC
// pointer). and the return buffer argument (if we are returning a
// struct).
// This is important because we don't want to try to report them
// to the GC, as the frame offsets in these local varables would
// not be correct.
if (varDsc->lvIsParam && raIsVarargsStackArg(lclNum))
{
if (!varDsc->lvPromoted && !varDsc->lvIsStructField)
{
noway_assert(varDsc->lvRefCnt() == 0 && !varDsc->lvRegister && !varDsc->lvOnFrame);
}
}
#endif
}
}
