void Rationalizer::RewriteAssignment(LIR::Use& use)
{
assert(use.IsInitialized());
GenTreeOp* assignment = use.Def()->AsOp();
assert(assignment->OperGet() == GT_ASG);
GenTree* location = assignment->gtGetOp1();
GenTree* value = assignment->gtGetOp2();
genTreeOps locationOp = location->OperGet();
switch (locationOp)
{
case GT_LCL_VAR:
case GT_LCL_FLD:
case GT_REG_VAR:
case GT_PHI_ARG:
RewriteAssignmentIntoStoreLclCore(assignment, location, value, locationOp);
BlockRange().Remove(location);
break;
case GT_IND:
{
GenTreeStoreInd* store =
new (comp, GT_STOREIND) GenTreeStoreInd(location->TypeGet(), location->gtGetOp1(), value);
copyFlags(store, assignment, GTF_ALL_EFFECT);
copyFlags(store, location, GTF_IND_FLAGS);
if (assignment->IsReverseOp())
{
store->gtFlags |= GTF_REVERSE_OPS;
}
// TODO: JIT dump
// Remove the GT_IND node and replace the assignment node with the store
BlockRange().Remove(location);
BlockRange().InsertBefore(assignment, store);
use.ReplaceWith(comp, store);
BlockRange().Remove(assignment);
}
break;
case GT_CLS_VAR:
{
location->SetOper(GT_CLS_VAR_ADDR);
location->gtType = TYP_BYREF;
assignment->SetOper(GT_STOREIND);
// TODO: JIT dump
}
break;
default:
unreached();
break;
}
}
//------------------------------------------------------------------------
// ContainCheckStoreIndir: determine whether the sources of a STOREIND node should be contained.
//
// Arguments:
// node - pointer to the node
//
void Lowering::ContainCheckStoreIndir(GenTreeIndir* node)
{
#ifdef _TARGET_ARM64_
GenTree* src = node->gtOp.gtOp2;
if (!varTypeIsFloating(src->TypeGet()) && src->IsIntegralConst(0))
{
// an integer zero for 'src' can be contained.
MakeSrcContained(node, src);
}
#endif // _TARGET_ARM64_
ContainCheckIndir(node);
}
//------------------------------------------------------------------------
// ContainCheckCast: determine whether the source of a CAST node should be contained.
//
// Arguments:
// node - pointer to the node
//
void Lowering::ContainCheckCast(GenTreeCast* node)
{
#ifdef _TARGET_ARM_
GenTree* castOp = node->CastOp();
var_types castToType = node->CastToType();
var_types srcType = castOp->TypeGet();
if (varTypeIsLong(castOp))
{
assert(castOp->OperGet() == GT_LONG);
MakeSrcContained(node, castOp);
}
#endif // _TARGET_ARM_
}
// Rewrite a SIMD indirection as GT_IND(GT_LEA(obj.op1)), or as a simple
// lclVar if possible.
//
// Arguments:
// use - A use reference for a block node
// keepBlk - True if this should remain a block node if it is not a lclVar
//
// Return Value:
// None.
//
// TODO-1stClassStructs: These should be eliminated earlier, once we can handle
// lclVars in all the places that used to have GT_OBJ.
//
void Rationalizer::RewriteSIMDOperand(LIR::Use& use, bool keepBlk)
{
#ifdef FEATURE_SIMD
// No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
GenTree* tree = use.Def();
if (!tree->OperIsIndir())
{
return;
}
var_types simdType = tree->TypeGet();
if (!varTypeIsSIMD(simdType))
{
return;
}
// If we have GT_IND(GT_LCL_VAR_ADDR) and the GT_LCL_VAR_ADDR is TYP_BYREF/TYP_I_IMPL,
// and the var is a SIMD type, replace the expression by GT_LCL_VAR.
GenTree* addr = tree->AsIndir()->Addr();
if (addr->OperIsLocalAddr() && comp->isAddrOfSIMDType(addr))
{
BlockRange().Remove(tree);
addr->SetOper(loadForm(addr->OperGet()));
addr->gtType = simdType;
use.ReplaceWith(comp, addr);
}
else if ((addr->OperGet() == GT_ADDR) && (addr->gtGetOp1()->OperGet() == GT_SIMD))
{
// if we have GT_IND(GT_ADDR(GT_SIMD)), remove the GT_IND(GT_ADDR()), leaving just the GT_SIMD.
BlockRange().Remove(tree);
BlockRange().Remove(addr);
use.ReplaceWith(comp, addr->gtGetOp1());
}
else if (!keepBlk)
{
tree->SetOper(GT_IND);
tree->gtType = simdType;
}
#endif // FEATURE_SIMD
}
// Rewrite a SIMD indirection as GT_IND(GT_LEA(obj.op1)), or as a simple
// lclVar if possible.
//
// Arguments:
// use - A use reference for a block node
// keepBlk - True if this should remain a block node if it is not a lclVar
//
// Return Value:
// None.
//
// TODO-1stClassStructs: These should be eliminated earlier, once we can handle
// lclVars in all the places that used to have GT_OBJ.
//
void Rationalizer::RewriteSIMDOperand(LIR::Use& use, bool keepBlk)
{
#ifdef FEATURE_SIMD
// No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
GenTree* tree = use.Def();
if (!tree->OperIsIndir())
{
return;
}
var_types simdType = tree->TypeGet();
if (!varTypeIsSIMD(simdType))
{
return;
}
// If the operand of is a GT_ADDR(GT_LCL_VAR) and LclVar is known to be of simdType,
// replace obj by GT_LCL_VAR.
GenTree* addr = tree->AsIndir()->Addr();
if (addr->OperIsLocalAddr() && comp->isAddrOfSIMDType(addr))
{
BlockRange().Remove(tree);
addr->SetOper(loadForm(addr->OperGet()));
addr->gtType = simdType;
use.ReplaceWith(comp, addr);
}
else if (!keepBlk)
{
tree->SetOper(GT_IND);
tree->gtType = simdType;
}
#endif // FEATURE_SIMD
}
//------------------------------------------------------------------------
// DecomposeNode: Decompose long-type trees into lower and upper halves.
//
// Arguments:
// *ppTree - A node that may or may not require decomposition.
// data - The tree-walk data that provides the context.
//
// Return Value:
// None. It the tree at *ppTree is of TYP_LONG, it will generally be replaced.
//
void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data)
{
GenTree* tree = *ppTree;
// Handle the case where we are implicitly using the lower half of a long lclVar.
if ((tree->TypeGet() == TYP_INT) && tree->OperIsLocal())
{
LclVarDsc* varDsc = m_compiler->lvaTable + tree->AsLclVarCommon()->gtLclNum;
if (varTypeIsLong(varDsc) && varDsc->lvPromoted)
{
#ifdef DEBUG
if (m_compiler->verbose)
{
printf("Changing implicit reference to lo half of long lclVar to an explicit reference of its promoted half:\n");
m_compiler->gtDispTree(tree);
}
#endif // DEBUG
m_compiler->lvaDecRefCnts(tree);
unsigned loVarNum = varDsc->lvFieldLclStart;
tree->AsLclVarCommon()->SetLclNum(loVarNum);
m_compiler->lvaIncRefCnts(tree);
return;
}
}
if (tree->TypeGet() != TYP_LONG)
{
return;
}
#ifdef DEBUG
if (m_compiler->verbose)
{
printf("Decomposing TYP_LONG tree. BEFORE:\n");
m_compiler->gtDispTree(tree);
}
#endif // DEBUG
switch (tree->OperGet())
{
case GT_PHI:
case GT_PHI_ARG:
break;
case GT_LCL_VAR:
DecomposeLclVar(ppTree, data);
break;
case GT_LCL_FLD:
DecomposeLclFld(ppTree, data);
break;
case GT_STORE_LCL_VAR:
DecomposeStoreLclVar(ppTree, data);
break;
case GT_CAST:
DecomposeCast(ppTree, data);
break;
case GT_CNS_LNG:
DecomposeCnsLng(ppTree, data);
break;
case GT_CALL:
DecomposeCall(ppTree, data);
break;
case GT_RETURN:
assert(tree->gtOp.gtOp1->OperGet() == GT_LONG);
break;
case GT_STOREIND:
DecomposeStoreInd(ppTree, data);
break;
case GT_STORE_LCL_FLD:
assert(tree->gtOp.gtOp1->OperGet() == GT_LONG);
NYI("st.lclFld of of TYP_LONG");
break;
case GT_IND:
DecomposeInd(ppTree, data);
break;
case GT_NOT:
DecomposeNot(ppTree, data);
break;
case GT_NEG:
DecomposeNeg(ppTree, data);
break;
// Binary operators. Those that require different computation for upper and lower half are
// handled by the use of GetHiOper().
case GT_ADD:
case GT_SUB:
case GT_OR:
case GT_XOR:
case GT_AND:
DecomposeArith(ppTree, data);
break;
case GT_MUL:
NYI("Arithmetic binary operators on TYP_LONG - GT_MUL");
break;
case GT_DIV:
NYI("Arithmetic binary operators on TYP_LONG - GT_DIV");
break;
case GT_MOD:
NYI("Arithmetic binary operators on TYP_LONG - GT_MOD");
break;
case GT_UDIV:
NYI("Arithmetic binary operators on TYP_LONG - GT_UDIV");
break;
case GT_UMOD:
NYI("Arithmetic binary operators on TYP_LONG - GT_UMOD");
break;
case GT_LSH:
case GT_RSH:
case GT_RSZ:
NYI("Arithmetic binary operators on TYP_LONG - SHIFT");
break;
case GT_ROL:
case GT_ROR:
NYI("Arithmetic binary operators on TYP_LONG - ROTATE");
break;
case GT_MULHI:
NYI("Arithmetic binary operators on TYP_LONG - MULHI");
break;
case GT_LOCKADD:
case GT_XADD:
case GT_XCHG:
case GT_CMPXCHG:
NYI("Interlocked operations on TYP_LONG");
break;
default:
{
JITDUMP("Illegal TYP_LONG node %s in Decomposition.", GenTree::NodeName(tree->OperGet()));
noway_assert(!"Illegal TYP_LONG node in Decomposition.");
break;
}
}
#ifdef DEBUG
if (m_compiler->verbose)
{
printf(" AFTER:\n");
m_compiler->gtDispTree(*ppTree);
}
#endif
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding list nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_FIELD_LIST head nodes, and GT_LIST nodes used by phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperIsAnyList() && !(prev->OperIsFieldListHead());
prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node that is not an aggregate.
if (node->OperIsAnyList())
{
GenTreeArgList* list = node->AsArgList();
if (!list->OperIsFieldListHead())
{
BlockRange().Remove(list);
}
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_IND:
// Clear the `GTF_IND_ASG_LHS` flag, which overlaps with `GTF_IND_REQ_ADDR_IN_REG`.
node->gtFlags &= ~GTF_IND_ASG_LHS;
if (varTypeIsSIMD(node))
{
RewriteSIMDOperand(use, false);
}
else
{
// Due to promotion of structs containing fields of type struct with a
// single scalar type field, we could potentially see IR nodes of the
// form GT_IND(GT_ADD(lclvarAddr, 0)) where 0 is an offset representing
// a field-seq. These get folded here.
//
// TODO: This code can be removed once JIT implements recursive struct
// promotion instead of lying about the type of struct field as the type
// of its single scalar field.
GenTree* addr = node->AsIndir()->Addr();
if (addr->OperGet() == GT_ADD && addr->gtGetOp1()->OperGet() == GT_LCL_VAR_ADDR &&
addr->gtGetOp2()->IsIntegralConst(0))
{
GenTreeLclVarCommon* lclVarNode = addr->gtGetOp1()->AsLclVarCommon();
unsigned lclNum = lclVarNode->GetLclNum();
LclVarDsc* varDsc = comp->lvaTable + lclNum;
if (node->TypeGet() == varDsc->TypeGet())
{
JITDUMP("Rewriting GT_IND(GT_ADD(LCL_VAR_ADDR,0)) to LCL_VAR\n");
lclVarNode->SetOper(GT_LCL_VAR);
lclVarNode->gtType = node->TypeGet();
use.ReplaceWith(comp, lclVarNode);
BlockRange().Remove(addr);
BlockRange().Remove(addr->gtGetOp2());
BlockRange().Remove(node);
}
}
}
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#if defined(_TARGET_XARCH_) || defined(_TARGET_ARM_)
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_BLK:
case GT_OBJ:
{
// TODO-1stClassStructs: These should have been transformed to GT_INDs, but in order
// to preserve existing behavior, we will keep this as a block node if this is the
// lhs of a block assignment, and either:
// - It is a "generic" TYP_STRUCT assignment, OR
// - It is an initblk, OR
// - Neither the lhs or rhs are known to be of SIMD type.
GenTree* parent = use.User();
bool keepBlk = false;
if ((parent->OperGet() == GT_ASG) && (node == parent->gtGetOp1()))
{
if ((node->TypeGet() == TYP_STRUCT) || parent->OperIsInitBlkOp())
{
keepBlk = true;
}
else if (!comp->isAddrOfSIMDType(node->AsBlk()->Addr()))
{
GenTree* dataSrc = parent->gtGetOp2();
if (!dataSrc->IsLocal() && (dataSrc->OperGet() != GT_SIMD))
{
noway_assert(dataSrc->OperIsIndir());
keepBlk = !comp->isAddrOfSIMDType(dataSrc->AsIndir()->Addr());
}
}
}
RewriteSIMDOperand(use, keepBlk);
}
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2IfPresent();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
// JCC nodes should not be present in HIR.
assert(node->OperGet() != GT_JCC);
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (node->OperIsLocalRead())
{
if (use.IsDummyUse())
{
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
else
{
// Local reads are side-effect-free; clear any flags leftover from frontend transformations.
node->gtFlags &= ~GTF_ALL_EFFECT;
}
}
assert(isLateArg == ((use.Def()->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}
void Rationalizer::RewriteAssignment(LIR::Use& use)
{
assert(use.IsInitialized());
GenTreeOp* assignment = use.Def()->AsOp();
assert(assignment->OperGet() == GT_ASG);
GenTree* location = assignment->gtGetOp1();
GenTree* value = assignment->gtGetOp2();
genTreeOps locationOp = location->OperGet();
if (assignment->OperIsBlkOp())
{
#ifdef FEATURE_SIMD
if (varTypeIsSIMD(location) && assignment->OperIsInitBlkOp())
{
if (location->OperGet() == GT_LCL_VAR)
{
var_types simdType = location->TypeGet();
GenTree* initVal = assignment->gtOp.gtOp2;
var_types baseType = comp->getBaseTypeOfSIMDLocal(location);
if (baseType != TYP_UNKNOWN)
{
GenTreeSIMD* simdTree = new (comp, GT_SIMD)
GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, genTypeSize(simdType));
assignment->gtOp.gtOp2 = simdTree;
value = simdTree;
initVal->gtNext = simdTree;
simdTree->gtPrev = initVal;
simdTree->gtNext = location;
location->gtPrev = simdTree;
}
}
}
#endif // FEATURE_SIMD
if ((location->TypeGet() == TYP_STRUCT) && !assignment->IsPhiDefn() && !value->IsMultiRegCall())
{
if ((location->OperGet() == GT_LCL_VAR))
{
// We need to construct a block node for the location.
// Modify lcl to be the address form.
location->SetOper(addrForm(locationOp));
LclVarDsc* varDsc = &(comp->lvaTable[location->AsLclVarCommon()->gtLclNum]);
location->gtType = TYP_BYREF;
GenTreeBlk* storeBlk = nullptr;
unsigned int size = varDsc->lvExactSize;
if (varDsc->lvStructGcCount != 0)
{
CORINFO_CLASS_HANDLE structHnd = varDsc->lvVerTypeInfo.GetClassHandle();
GenTreeObj* objNode = comp->gtNewObjNode(structHnd, location)->AsObj();
unsigned int slots = (unsigned)(roundUp(size, TARGET_POINTER_SIZE) / TARGET_POINTER_SIZE);
objNode->SetGCInfo(varDsc->lvGcLayout, varDsc->lvStructGcCount, slots);
objNode->ChangeOper(GT_STORE_OBJ);
objNode->SetData(value);
comp->fgMorphUnsafeBlk(objNode);
storeBlk = objNode;
}
else
{
storeBlk = new (comp, GT_STORE_BLK) GenTreeBlk(GT_STORE_BLK, TYP_STRUCT, location, value, size);
}
storeBlk->gtFlags |= (GTF_REVERSE_OPS | GTF_ASG);
storeBlk->gtFlags |= ((location->gtFlags | value->gtFlags) & GTF_ALL_EFFECT);
GenTree* insertionPoint = location->gtNext;
BlockRange().InsertBefore(insertionPoint, storeBlk);
use.ReplaceWith(comp, storeBlk);
BlockRange().Remove(assignment);
JITDUMP("After transforming local struct assignment into a block op:\n");
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
return;
}
else
{
assert(location->OperIsBlk());
}
}
}
switch (locationOp)
{
case GT_LCL_VAR:
case GT_LCL_FLD:
case GT_REG_VAR:
case GT_PHI_ARG:
RewriteAssignmentIntoStoreLclCore(assignment, location, value, locationOp);
BlockRange().Remove(location);
break;
case GT_IND:
{
GenTreeStoreInd* store =
new (comp, GT_STOREIND) GenTreeStoreInd(location->TypeGet(), location->gtGetOp1(), value);
copyFlags(store, assignment, GTF_ALL_EFFECT);
copyFlags(store, location, GTF_IND_FLAGS);
if (assignment->IsReverseOp())
{
store->gtFlags |= GTF_REVERSE_OPS;
}
// TODO: JIT dump
// Remove the GT_IND node and replace the assignment node with the store
BlockRange().Remove(location);
BlockRange().InsertBefore(assignment, store);
use.ReplaceWith(comp, store);
BlockRange().Remove(assignment);
}
break;
case GT_CLS_VAR:
{
location->SetOper(GT_CLS_VAR_ADDR);
location->gtType = TYP_BYREF;
assignment->SetOper(GT_STOREIND);
// TODO: JIT dump
}
break;
case GT_BLK:
case GT_OBJ:
case GT_DYN_BLK:
{
assert(varTypeIsStruct(location));
GenTreeBlk* storeBlk = location->AsBlk();
genTreeOps storeOper;
switch (location->gtOper)
{
case GT_BLK:
storeOper = GT_STORE_BLK;
break;
case GT_OBJ:
storeOper = GT_STORE_OBJ;
break;
case GT_DYN_BLK:
storeOper = GT_STORE_DYN_BLK;
break;
default:
unreached();
}
JITDUMP("Rewriting GT_ASG(%s(X), Y) to %s(X,Y):\n", GenTree::NodeName(location->gtOper),
GenTree::NodeName(storeOper));
storeBlk->SetOperRaw(storeOper);
storeBlk->gtFlags &= ~GTF_DONT_CSE;
storeBlk->gtFlags |= (assignment->gtFlags & (GTF_ALL_EFFECT | GTF_REVERSE_OPS | GTF_BLK_VOLATILE |
GTF_BLK_UNALIGNED | GTF_DONT_CSE));
storeBlk->gtBlk.Data() = value;
// Replace the assignment node with the store
use.ReplaceWith(comp, storeBlk);
BlockRange().Remove(assignment);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
break;
default:
unreached();
break;
}
}
//------------------------------------------------------------------------
// BuildSIMD: Set the NodeInfo for a GT_SIMD tree.
//
// Arguments:
// tree - The GT_SIMD node of interest
//
// Return Value:
// The number of sources consumed by this node.
//
int LinearScan::BuildSIMD(GenTreeSIMD* simdTree)
{
int srcCount = 0;
// Only SIMDIntrinsicInit can be contained
if (simdTree->isContained())
{
assert(simdTree->gtSIMDIntrinsicID == SIMDIntrinsicInit);
}
int dstCount = simdTree->IsValue() ? 1 : 0;
assert(dstCount == 1);
bool buildUses = true;
GenTree* op1 = simdTree->gtGetOp1();
GenTree* op2 = simdTree->gtGetOp2();
switch (simdTree->gtSIMDIntrinsicID)
{
case SIMDIntrinsicInit:
case SIMDIntrinsicCast:
case SIMDIntrinsicSqrt:
case SIMDIntrinsicAbs:
case SIMDIntrinsicConvertToSingle:
case SIMDIntrinsicConvertToInt32:
case SIMDIntrinsicConvertToDouble:
case SIMDIntrinsicConvertToInt64:
case SIMDIntrinsicWidenLo:
case SIMDIntrinsicWidenHi:
// No special handling required.
break;
case SIMDIntrinsicGetItem:
{
op1 = simdTree->gtGetOp1();
op2 = simdTree->gtGetOp2();
// We have an object and an index, either of which may be contained.
bool setOp2DelayFree = false;
if (!op2->IsCnsIntOrI() && (!op1->isContained() || op1->OperIsLocal()))
{
// If the index is not a constant and the object is not contained or is a local
// we will need a general purpose register to calculate the address
// internal register must not clobber input index
// TODO-Cleanup: An internal register will never clobber a source; this code actually
// ensures that the index (op2) doesn't interfere with the target.
buildInternalIntRegisterDefForNode(simdTree);
setOp2DelayFree = true;
}
srcCount += BuildOperandUses(op1);
if (!op2->isContained())
{
RefPosition* op2Use = BuildUse(op2);
if (setOp2DelayFree)
{
setDelayFree(op2Use);
}
srcCount++;
}
if (!op2->IsCnsIntOrI() && (!op1->isContained()))
{
// If vector is not already in memory (contained) and the index is not a constant,
// we will use the SIMD temp location to store the vector.
compiler->getSIMDInitTempVarNum();
}
buildUses = false;
}
break;
case SIMDIntrinsicAdd:
case SIMDIntrinsicSub:
case SIMDIntrinsicMul:
case SIMDIntrinsicDiv:
case SIMDIntrinsicBitwiseAnd:
case SIMDIntrinsicBitwiseAndNot:
case SIMDIntrinsicBitwiseOr:
case SIMDIntrinsicBitwiseXor:
case SIMDIntrinsicMin:
case SIMDIntrinsicMax:
case SIMDIntrinsicEqual:
case SIMDIntrinsicLessThan:
case SIMDIntrinsicGreaterThan:
case SIMDIntrinsicLessThanOrEqual:
case SIMDIntrinsicGreaterThanOrEqual:
// No special handling required.
break;
case SIMDIntrinsicSetX:
case SIMDIntrinsicSetY:
case SIMDIntrinsicSetZ:
case SIMDIntrinsicSetW:
case SIMDIntrinsicNarrow:
{
// Op1 will write to dst before Op2 is free
BuildUse(op1);
RefPosition* op2Use = BuildUse(op2);
setDelayFree(op2Use);
srcCount = 2;
buildUses = false;
break;
}
case SIMDIntrinsicInitN:
{
var_types baseType = simdTree->gtSIMDBaseType;
srcCount = (short)(simdTree->gtSIMDSize / genTypeSize(baseType));
if (varTypeIsFloating(simdTree->gtSIMDBaseType))
{
// Need an internal register to stitch together all the values into a single vector in a SIMD reg.
buildInternalFloatRegisterDefForNode(simdTree);
}
int initCount = 0;
for (GenTree* list = op1; list != nullptr; list = list->gtGetOp2())
{
assert(list->OperGet() == GT_LIST);
GenTree* listItem = list->gtGetOp1();
assert(listItem->TypeGet() == baseType);
assert(!listItem->isContained());
BuildUse(listItem);
initCount++;
}
assert(initCount == srcCount);
buildUses = false;
break;
}
case SIMDIntrinsicInitArray:
// We have an array and an index, which may be contained.
break;
case SIMDIntrinsicOpEquality:
case SIMDIntrinsicOpInEquality:
buildInternalFloatRegisterDefForNode(simdTree);
break;
case SIMDIntrinsicDotProduct:
buildInternalFloatRegisterDefForNode(simdTree);
break;
case SIMDIntrinsicSelect:
// TODO-ARM64-CQ Allow lowering to see SIMDIntrinsicSelect so we can generate BSL VC, VA, VB
// bsl target register must be VC. Reserve a temp in case we need to shuffle things.
// This will require a different approach, as GenTreeSIMD has only two operands.
assert(!"SIMDIntrinsicSelect not yet supported");
buildInternalFloatRegisterDefForNode(simdTree);
break;
case SIMDIntrinsicInitArrayX:
case SIMDIntrinsicInitFixed:
case SIMDIntrinsicCopyToArray:
case SIMDIntrinsicCopyToArrayX:
case SIMDIntrinsicNone:
case SIMDIntrinsicGetCount:
case SIMDIntrinsicGetOne:
case SIMDIntrinsicGetZero:
case SIMDIntrinsicGetAllOnes:
case SIMDIntrinsicGetX:
case SIMDIntrinsicGetY:
case SIMDIntrinsicGetZ:
case SIMDIntrinsicGetW:
case SIMDIntrinsicInstEquals:
case SIMDIntrinsicHWAccel:
case SIMDIntrinsicWiden:
case SIMDIntrinsicInvalid:
assert(!"These intrinsics should not be seen during register allocation");
__fallthrough;
default:
noway_assert(!"Unimplemented SIMD node type.");
unreached();
}
if (buildUses)
{
assert(!op1->OperIs(GT_LIST));
assert(srcCount == 0);
srcCount = BuildOperandUses(op1);
if ((op2 != nullptr) && !op2->isContained())
{
srcCount += BuildOperandUses(op2);
}
}
assert(internalCount <= MaxInternalCount);
buildInternalRegisterUses();
if (dstCount == 1)
{
BuildDef(simdTree);
}
else
{
assert(dstCount == 0);
}
return srcCount;
}
void Rationalizer::RewriteAssignment(LIR::Use& use)
{
assert(use.IsInitialized());
GenTreeOp* assignment = use.Def()->AsOp();
assert(assignment->OperGet() == GT_ASG);
GenTree* location = assignment->gtGetOp1();
GenTree* value = assignment->gtGetOp2();
genTreeOps locationOp = location->OperGet();
#ifdef FEATURE_SIMD
if (varTypeIsSIMD(location) && assignment->OperIsInitBlkOp())
{
if (location->OperGet() == GT_LCL_VAR)
{
var_types simdType = location->TypeGet();
GenTree* initVal = assignment->gtOp.gtOp2;
var_types baseType = comp->getBaseTypeOfSIMDLocal(location);
if (baseType != TYP_UNKNOWN)
{
GenTreeSIMD* simdTree = new (comp, GT_SIMD)
GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, genTypeSize(simdType));
assignment->gtOp.gtOp2 = simdTree;
value = simdTree;
initVal->gtNext = simdTree;
simdTree->gtPrev = initVal;
simdTree->gtNext = location;
location->gtPrev = simdTree;
}
}
else
{
assert(location->OperIsBlk());
}
}
#endif // FEATURE_SIMD
switch (locationOp)
{
case GT_LCL_VAR:
case GT_LCL_FLD:
case GT_REG_VAR:
case GT_PHI_ARG:
RewriteAssignmentIntoStoreLclCore(assignment, location, value, locationOp);
BlockRange().Remove(location);
break;
case GT_IND:
{
GenTreeStoreInd* store =
new (comp, GT_STOREIND) GenTreeStoreInd(location->TypeGet(), location->gtGetOp1(), value);
copyFlags(store, assignment, GTF_ALL_EFFECT);
copyFlags(store, location, GTF_IND_FLAGS);
if (assignment->IsReverseOp())
{
store->gtFlags |= GTF_REVERSE_OPS;
}
// TODO: JIT dump
// Remove the GT_IND node and replace the assignment node with the store
BlockRange().Remove(location);
BlockRange().InsertBefore(assignment, store);
use.ReplaceWith(comp, store);
BlockRange().Remove(assignment);
}
break;
case GT_CLS_VAR:
{
location->SetOper(GT_CLS_VAR_ADDR);
location->gtType = TYP_BYREF;
assignment->SetOper(GT_STOREIND);
// TODO: JIT dump
}
break;
case GT_BLK:
case GT_OBJ:
case GT_DYN_BLK:
{
assert(varTypeIsStruct(location));
GenTreeBlk* storeBlk = location->AsBlk();
genTreeOps storeOper;
switch (location->gtOper)
{
case GT_BLK:
storeOper = GT_STORE_BLK;
break;
case GT_OBJ:
storeOper = GT_STORE_OBJ;
break;
case GT_DYN_BLK:
storeOper = GT_STORE_DYN_BLK;
break;
default:
unreached();
}
JITDUMP("Rewriting GT_ASG(%s(X), Y) to %s(X,Y):\n", GenTree::NodeName(location->gtOper),
GenTree::NodeName(storeOper));
storeBlk->SetOperRaw(storeOper);
storeBlk->gtFlags &= ~GTF_DONT_CSE;
storeBlk->gtFlags |= (assignment->gtFlags & (GTF_ALL_EFFECT | GTF_REVERSE_OPS | GTF_BLK_VOLATILE |
GTF_BLK_UNALIGNED | GTF_BLK_INIT | GTF_DONT_CSE));
storeBlk->gtBlk.Data() = value;
// Replace the assignment node with the store
use.ReplaceWith(comp, storeBlk);
BlockRange().Remove(assignment);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
break;
default:
unreached();
break;
}
}
//------------------------------------------------------------------------
// DecomposeStoreInd: Decompose GT_STOREIND.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeStoreInd(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_STOREIND);
GenTree* tree = use.Def();
assert(tree->gtOp.gtOp2->OperGet() == GT_LONG);
// Example input (address expression omitted):
//
// t51 = const int 0x37C05E7D
// t154 = const int 0x2A0A3C80
// / --* t51 int
// + --* t154 int
// t155 = *gt_long long
// / --* t52 byref
// + --* t155 long
// * storeIndir long
GenTree* gtLong = tree->gtOp.gtOp2;
// Save address to a temp. It is used in storeIndLow and storeIndHigh trees.
LIR::Use address(Range(), &tree->gtOp.gtOp1, tree);
address.ReplaceWithLclVar(m_compiler, m_blockWeight);
JITDUMP("[DecomposeStoreInd]: Saving address tree to a temp var:\n");
DISPTREERANGE(Range(), address.Def());
if (!gtLong->gtOp.gtOp1->OperIsLeaf())
{
LIR::Use op1(Range(), >Long->gtOp.gtOp1, gtLong);
op1.ReplaceWithLclVar(m_compiler, m_blockWeight);
JITDUMP("[DecomposeStoreInd]: Saving low data tree to a temp var:\n");
DISPTREERANGE(Range(), op1.Def());
}
if (!gtLong->gtOp.gtOp2->OperIsLeaf())
{
LIR::Use op2(Range(), >Long->gtOp.gtOp2, gtLong);
op2.ReplaceWithLclVar(m_compiler, m_blockWeight);
JITDUMP("[DecomposeStoreInd]: Saving high data tree to a temp var:\n");
DISPTREERANGE(Range(), op2.Def());
}
GenTree* addrBase = tree->gtOp.gtOp1;
GenTree* dataHigh = gtLong->gtOp.gtOp2;
GenTree* dataLow = gtLong->gtOp.gtOp1;
GenTree* storeIndLow = tree;
Range().Remove(gtLong);
Range().Remove(dataHigh);
storeIndLow->gtOp.gtOp2 = dataLow;
storeIndLow->gtType = TYP_INT;
GenTree* addrBaseHigh = new (m_compiler, GT_LCL_VAR)
GenTreeLclVar(GT_LCL_VAR, addrBase->TypeGet(), addrBase->AsLclVarCommon()->GetLclNum(), BAD_IL_OFFSET);
GenTree* addrHigh =
new (m_compiler, GT_LEA) GenTreeAddrMode(TYP_REF, addrBaseHigh, nullptr, 0, genTypeSize(TYP_INT));
GenTree* storeIndHigh = new (m_compiler, GT_STOREIND) GenTreeStoreInd(TYP_INT, addrHigh, dataHigh);
storeIndHigh->gtFlags = (storeIndLow->gtFlags & (GTF_ALL_EFFECT | GTF_LIVENESS_MASK));
storeIndHigh->gtFlags |= GTF_REVERSE_OPS;
m_compiler->lvaIncRefCnts(addrBaseHigh);
Range().InsertAfter(storeIndLow, dataHigh, addrBaseHigh, addrHigh, storeIndHigh);
return storeIndHigh;
// Example final output:
//
// /--* t52 byref
// * st.lclVar byref V07 rat0
// t158 = lclVar byref V07 rat0
// t51 = const int 0x37C05E7D
// /--* t158 byref
// +--* t51 int
// * storeIndir int
// t154 = const int 0x2A0A3C80
// t159 = lclVar byref V07 rat0
// /--* t159 byref
// t160 = * lea(b + 4) ref
// /--* t154 int
// +--* t160 ref
// * storeIndir int
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding GT_LIST nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_LIST nodes that are used by block ops and phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperGet() == GT_LIST; prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node.
if ((*useEdge)->OperGet() == GT_LIST)
{
BlockRange().Remove(*useEdge);
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#ifdef _TARGET_XARCH_
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_INITBLK:
RewriteInitBlk(use);
break;
case GT_COPYBLK:
RewriteCopyBlk(use);
break;
case GT_OBJ:
RewriteObj(use);
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_STOREIND:
case GT_IND:
if (node->gtType == TYP_STRUCT)
{
GenTree* addr = node->AsIndir()->Addr();
assert(addr->TypeGet() == TYP_BYREF);
if (addr->OperIsLocal())
{
LclVarDsc* varDsc = &(comp->lvaTable[addr->AsLclVarCommon()->gtLclNum]);
assert(varDsc->lvSIMDType);
unsigned simdSize = (unsigned int)roundUp(varDsc->lvExactSize, TARGET_POINTER_SIZE);
node->gtType = comp->getSIMDTypeForSize(simdSize);
}
#if DEBUG
else
{
// If the address is not a local var, assert that the user of this IND is an ADDR node.
assert((use.User()->OperGet() == GT_ADDR) || use.User()->OperIsLocalAddr());
}
#endif
}
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
else if (simdNode->gtType == TYP_STRUCT || varTypeIsSIMD(simdNode))
{
node->gtType = simdType;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (use.IsDummyUse() && node->OperIsLocalRead())
{
assert((node->gtFlags & GTF_ALL_EFFECT) == 0);
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
assert(isLateArg == ((node->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}
//------------------------------------------------------------------------
// MorphAllocObjNodes: Morph each GT_ALLOCOBJ node either into an
// allocation helper call or stack allocation.
//
// Notes:
// Runs only over the blocks having bbFlags BBF_HAS_NEWOBJ set.
void ObjectAllocator::MorphAllocObjNodes()
{
BasicBlock* block;
foreach_block(comp, block)
{
const bool basicBlockHasNewObj = (block->bbFlags & BBF_HAS_NEWOBJ) == BBF_HAS_NEWOBJ;
#ifndef DEBUG
if (!basicBlockHasNewObj)
{
continue;
}
#endif // DEBUG
for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt)
{
GenTree* stmtExpr = stmt->gtStmtExpr;
GenTree* op2 = nullptr;
bool canonicalAllocObjFound = false;
if (stmtExpr->OperGet() == GT_ASG && stmtExpr->TypeGet() == TYP_REF)
{
op2 = stmtExpr->gtGetOp2();
if (op2->OperGet() == GT_ALLOCOBJ)
{
canonicalAllocObjFound = true;
}
}
if (canonicalAllocObjFound)
{
assert(basicBlockHasNewObj);
//------------------------------------------------------------------------
// We expect the following expression tree at this point
// * GT_STMT void (top level)
// | /--* GT_ALLOCOBJ ref
// \--* GT_ASG ref
// \--* GT_LCL_VAR ref
//------------------------------------------------------------------------
GenTree* op1 = stmtExpr->gtGetOp1();
assert(op1->OperGet() == GT_LCL_VAR);
assert(op1->TypeGet() == TYP_REF);
assert(op2 != nullptr);
assert(op2->OperGet() == GT_ALLOCOBJ);
GenTreeAllocObj* asAllocObj = op2->AsAllocObj();
unsigned int lclNum = op1->AsLclVar()->GetLclNum();
if (IsObjectStackAllocationEnabled() && CanAllocateLclVarOnStack(lclNum))
{
op2 = MorphAllocObjNodeIntoStackAlloc(asAllocObj, block, stmt);
}
else
{
op2 = MorphAllocObjNodeIntoHelperCall(asAllocObj);
}
// Propagate flags of op2 to its parent.
stmtExpr->gtOp.gtOp2 = op2;
stmtExpr->gtFlags |= op2->gtFlags & GTF_ALL_EFFECT;
}
#ifdef DEBUG
else
{
// We assume that GT_ALLOCOBJ nodes are always present in the
// canonical form.
comp->fgWalkTreePre(&stmt->gtStmtExpr, AssertWhenAllocObjFoundVisitor);
}
#endif // DEBUG
}
}
}
void Compiler::optCopyProp(BasicBlock* block, GenTree* stmt, GenTree* tree, LclNumToGenTreePtrStack* curSsaName)
{
// TODO-Review: EH successor/predecessor iteration seems broken.
if (block->bbCatchTyp == BBCT_FINALLY || block->bbCatchTyp == BBCT_FAULT)
{
return;
}
// If not local nothing to do.
if (!tree->IsLocal())
{
return;
}
if (tree->OperGet() == GT_PHI_ARG || tree->OperGet() == GT_LCL_FLD)
{
return;
}
// Propagate only on uses.
if (tree->gtFlags & GTF_VAR_DEF)
{
return;
}
unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
// Skip non-SSA variables.
if (!lvaInSsa(lclNum))
{
return;
}
assert(tree->gtVNPair.GetConservative() != ValueNumStore::NoVN);
for (LclNumToGenTreePtrStack::KeyIterator iter = curSsaName->Begin(); !iter.Equal(curSsaName->End()); ++iter)
{
unsigned newLclNum = iter.Get();
GenTree* op = iter.GetValue()->Index(0);
// Nothing to do if same.
if (lclNum == newLclNum)
{
continue;
}
// Skip variables with assignments embedded in the statement (i.e., with a comma). Because we
// are not currently updating their SSA names as live in the copy-prop pass of the stmt.
if (VarSetOps::IsMember(this, optCopyPropKillSet, lvaTable[newLclNum].lvVarIndex))
{
continue;
}
if (op->gtFlags & GTF_VAR_CAST)
{
continue;
}
if (gsShadowVarInfo != nullptr && lvaTable[newLclNum].lvIsParam &&
gsShadowVarInfo[newLclNum].shadowCopy == lclNum)
{
continue;
}
ValueNum opVN = GetUseAsgDefVNOrTreeVN(op);
if (opVN == ValueNumStore::NoVN)
{
continue;
}
if (op->TypeGet() != tree->TypeGet())
{
continue;
}
if (opVN != tree->gtVNPair.GetConservative())
{
continue;
}
if (optCopyProp_LclVarScore(&lvaTable[lclNum], &lvaTable[newLclNum], true) <= 0)
{
continue;
}
// Check whether the newLclNum is live before being substituted. Otherwise, we could end
// up in a situation where there must've been a phi node that got pruned because the variable
// is not live anymore. For example,
// if
// x0 = 1
// else
// x1 = 2
// print(c) <-- x is not live here. Let's say 'c' shares the value number with "x0."
//
// If we simply substituted 'c' with "x0", we would be wrong. Ideally, there would be a phi
// node x2 = phi(x0, x1) which can then be used to substitute 'c' with. But because of pruning
// there would be no such phi node. To solve this we'll check if 'x' is live, before replacing
// 'c' with 'x.'
if (!lvaTable[newLclNum].lvVerTypeInfo.IsThisPtr())
{
if (lvaTable[newLclNum].lvAddrExposed)
{
continue;
}
// We compute liveness only on tracked variables. So skip untracked locals.
if (!lvaTable[newLclNum].lvTracked)
{
continue;
}
// Because of this dependence on live variable analysis, CopyProp phase is immediately
// after Liveness, SSA and VN.
if (!VarSetOps::IsMember(this, compCurLife, lvaTable[newLclNum].lvVarIndex))
{
continue;
}
}
unsigned newSsaNum = SsaConfig::RESERVED_SSA_NUM;
if (op->gtFlags & GTF_VAR_DEF)
{
newSsaNum = GetSsaNumForLocalVarDef(op);
}
else // parameters, this pointer etc.
{
newSsaNum = op->AsLclVarCommon()->GetSsaNum();
}
if (newSsaNum == SsaConfig::RESERVED_SSA_NUM)
{
continue;
}
#ifdef DEBUG
if (verbose)
{
JITDUMP("VN based copy assertion for ");
printTreeID(tree);
printf(" V%02d @%08X by ", lclNum, tree->GetVN(VNK_Conservative));
printTreeID(op);
printf(" V%02d @%08X.\n", newLclNum, op->GetVN(VNK_Conservative));
gtDispTree(tree, nullptr, nullptr, true);
}
#endif
tree->gtLclVarCommon.SetLclNum(newLclNum);
tree->AsLclVarCommon()->SetSsaNum(newSsaNum);
gtUpdateSideEffects(stmt, tree);
#ifdef DEBUG
if (verbose)
{
printf("copy propagated to:\n");
gtDispTree(tree, nullptr, nullptr, true);
}
#endif
break;
}
return;
}
//------------------------------------------------------------------------
// DecomposeStoreInd: Decompose GT_STOREIND.
//
// Arguments:
// tree - the tree to decompose
//
// Return Value:
// None.
//
void DecomposeLongs::DecomposeStoreInd(GenTree** ppTree, Compiler::fgWalkData* data)
{
assert(ppTree != nullptr);
assert(*ppTree != nullptr);
assert(data != nullptr);
assert((*ppTree)->OperGet() == GT_STOREIND);
assert(m_compiler->compCurStmt != nullptr);
GenTree* tree = *ppTree;
assert(tree->gtOp.gtOp2->OperGet() == GT_LONG);
GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt();
bool isEmbeddedStmt = !curStmt->gtStmtIsTopLevel();
// Example input trees (a nested embedded statement case)
//
// <linkBegin Node>
// * stmtExpr void (top level) (IL ???... ???)
// | /--* argPlace ref $280
// | +--* argPlace int $4a
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar ref V11 tmp9 u:3 $21c
// | | { | +--* const int 4 $44
// | | { | /--* + byref $2c8
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclFld long V01 arg1 u:2[+8] Fseq[i] $380
// | | { | | { \--* st.lclVar long (P) V21 cse8
// | | { | | { \--* int V21.hi (offs=0x00) -> V22 rat0
// | | { | | { \--* int V21.hi (offs=0x04) -> V23 rat1
// | | { | | /--* lclVar int V22 rat0 $380
// | | { | | +--* lclVar int V23 rat1
// | | { | +--* gt_long long
// | | { \--* storeIndir long
// | +--* lclVar ref V11 tmp9 u:3 (last use) $21c
// | +--* lclVar ref V02 tmp0 u:3 $280
// | +--* const int 8 $4a
// \--* call help void HELPER.CORINFO_HELP_ARRADDR_ST $205
// <linkEndNode>
//
// (editor brace matching compensation: }}}}}}}}}}}}}}}}}})
GenTree* linkBegin = m_compiler->fgGetFirstNode(tree)->gtPrev;
GenTree* linkEnd = tree->gtNext;
GenTree* gtLong = tree->gtOp.gtOp2;
// Save address to a temp. It is used in storeIndLow and storeIndHigh trees.
GenTreeStmt* addrStmt = CreateTemporary(&tree->gtOp.gtOp1);
JITDUMP("[DecomposeStoreInd]: Saving address tree to a temp var:\n");
DISPTREE(addrStmt);
if (!gtLong->gtOp.gtOp1->OperIsLeaf())
{
GenTreeStmt* dataLowStmt = CreateTemporary(>Long->gtOp.gtOp1);
JITDUMP("[DecomposeStoreInd]: Saving low data tree to a temp var:\n");
DISPTREE(dataLowStmt);
}
if (!gtLong->gtOp.gtOp2->OperIsLeaf())
{
GenTreeStmt* dataHighStmt = CreateTemporary(>Long->gtOp.gtOp2);
JITDUMP("[DecomposeStoreInd]: Saving high data tree to a temp var:\n");
DISPTREE(dataHighStmt);
}
// Example trees after embedded statements for address and data are added.
// This example saves all address and data trees into temp variables
// to show how those embedded statements are created.
//
// * stmtExpr void (top level) (IL ???... ???)
// | /--* argPlace ref $280
// | +--* argPlace int $4a
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar ref V11 tmp9 u:3 $21c
// | | { | +--* const int 4 $44
// | | { | /--* + byref $2c8
// | | { \--* st.lclVar byref V24 rat2
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar byref V24 rat2
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclFld long V01 arg1 u:2[+8] Fseq[i] $380380
// | | { | | { \--* st.lclVar long (P) V21 cse8
// | | { | | { \--* int V21.hi (offs=0x00) -> V22 rat0
// | | { | | { \--* int V21.hi (offs=0x04) -> V23 rat1
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclVar int V22 rat0 $380
// | | { | | { \--* st.lclVar int V25 rat3
// | | { | | /--* lclVar int V25 rat3
// | | { | | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | | { | /--* lclVar int V23 rat1
// | | { | | | { \--* st.lclVar int V26 rat4
// | | { | | +--* lclVar int V26 rat4
// | | { | +--* gt_long long
// | | { \--* storeIndir long
// | +--* lclVar ref V11 tmp9 u:3 (last use) $21c
// | +--* lclVar ref V02 tmp0 u:3 $280
// | +--* const int 8 $4a
// \--* call help void HELPER.CORINFO_HELP_ARRADDR_ST $205
//
// (editor brace matching compensation: }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}})
GenTree* addrBase = tree->gtOp.gtOp1;
GenTree* dataHigh = gtLong->gtOp.gtOp2;
GenTree* dataLow = gtLong->gtOp.gtOp1;
GenTree* storeIndLow = tree;
// Rewrite storeIndLow tree to save only lower 32-bit data.
//
// | | { | /--* lclVar byref V24 rat2 (address)
// ...
// | | { | +--* lclVar int V25 rat3 (lower 32-bit data)
// | | { | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | { | /--* lclVar int V23 rat1
// | | { | { \--* st.lclVar int V26 rat4
// | | { \--* storeIndir int
//
// (editor brace matching compensation: }}}}}}}}})
m_compiler->fgSnipNode(curStmt, gtLong);
m_compiler->fgSnipNode(curStmt, dataHigh);
storeIndLow->gtOp.gtOp2 = dataLow;
storeIndLow->gtType = TYP_INT;
// Construct storeIndHigh tree
//
// | | { *stmtExpr void (embedded)(IL ? ? ? ... ? ? ? )
// | | { | / --* lclVar int V26 rat4
// | | { | | / --* lclVar byref V24 rat2
// | | { | +--* lea(b + 4) ref
// | | { \--* storeIndir int
//
// (editor brace matching compensation: }}}}})
GenTree* addrBaseHigh = new(m_compiler, GT_LCL_VAR) GenTreeLclVar(GT_LCL_VAR,
addrBase->TypeGet(), addrBase->AsLclVarCommon()->GetLclNum(), BAD_IL_OFFSET);
GenTree* addrHigh = new(m_compiler, GT_LEA) GenTreeAddrMode(TYP_REF, addrBaseHigh, nullptr, 0, genTypeSize(TYP_INT));
GenTree* storeIndHigh = new(m_compiler, GT_STOREIND) GenTreeStoreInd(TYP_INT, addrHigh, dataHigh);
storeIndHigh->gtFlags = (storeIndLow->gtFlags & (GTF_ALL_EFFECT | GTF_LIVENESS_MASK));
storeIndHigh->gtFlags |= GTF_REVERSE_OPS;
storeIndHigh->CopyCosts(storeIndLow);
// Internal links of storeIndHigh tree
dataHigh->gtPrev = nullptr;
dataHigh->gtNext = nullptr;
SimpleLinkNodeAfter(dataHigh, addrBaseHigh);
SimpleLinkNodeAfter(addrBaseHigh, addrHigh);
SimpleLinkNodeAfter(addrHigh, storeIndHigh);
// External links of storeIndHigh tree
//dataHigh->gtPrev = nullptr;
if (isEmbeddedStmt)
{
// If storeIndTree is an embedded statement, connect storeIndLow
// and dataHigh
storeIndLow->gtNext = dataHigh;
dataHigh->gtPrev = storeIndLow;
}
storeIndHigh->gtNext = linkEnd;
if (linkEnd != nullptr)
{
linkEnd->gtPrev = storeIndHigh;
}
InsertNodeAsStmt(storeIndHigh);
// Example final output
//
// * stmtExpr void (top level) (IL ???... ???)
// | /--* argPlace ref $280
// | +--* argPlace int $4a
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar ref V11 tmp9 u:3 $21c
// | | { | +--* const int 4 $44
// | | { | /--* + byref $2c8
// | | { \--* st.lclVar byref V24 rat2
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar byref V24 rat2
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclFld int V01 arg1 u:2[+8] Fseq[i] $380
// | | { | | { | +--* lclFld int V01 arg1 [+12]
// | | { | | { | /--* gt_long long
// | | { | | { \--* st.lclVar long (P) V21 cse8
// | | { | | { \--* int V21.hi (offs=0x00) -> V22 rat0
// | | { | | { \--* int V21.hi (offs=0x04) -> V23 rat1
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclVar int V22 rat0 $380
// | | { | | { \--* st.lclVar int V25 rat3
// | | { | +--* lclVar int V25 rat3
// | | { | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | { | /--* lclVar int V23 rat1
// | | { | { \--* st.lclVar int V26 rat4
// | | { \--* storeIndir int
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar int V26 rat4
// | | { | | /--* lclVar byref V24 rat2
// | | { | +--* lea(b+4) ref
// | | { \--* storeIndir int
// | | /--* lclVar ref V11 tmp9 u:3 (last use) $21c
// | +--* putarg_stk [+0x00] ref
// | | /--* lclVar ref V02 tmp0 u:3 $280
// | +--* putarg_reg ref
// | | /--* const int 8 $4a
// | +--* putarg_reg int
// \--* call help void HELPER.CORINFO_HELP_ARRADDR_ST $205
//
// (editor brace matching compensation: }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}})
}
//------------------------------------------------------------------------
// TreeNodeInfoInitCall: Set the NodeInfo for a call.
//
// Arguments:
// call - The call node of interest
//
// Return Value:
// None.
//
void Lowering::TreeNodeInfoInitCall(GenTreeCall* call)
{
TreeNodeInfo* info = &(call->gtLsraInfo);
LinearScan* l = m_lsra;
Compiler* compiler = comp;
bool hasMultiRegRetVal = false;
ReturnTypeDesc* retTypeDesc = nullptr;
info->srcCount = 0;
if (call->TypeGet() != TYP_VOID)
{
hasMultiRegRetVal = call->HasMultiRegRetVal();
if (hasMultiRegRetVal)
{
// dst count = number of registers in which the value is returned by call
retTypeDesc = call->GetReturnTypeDesc();
info->dstCount = retTypeDesc->GetReturnRegCount();
}
else
{
info->dstCount = 1;
}
}
else
{
info->dstCount = 0;
}
GenTree* ctrlExpr = call->gtControlExpr;
if (call->gtCallType == CT_INDIRECT)
{
// either gtControlExpr != null or gtCallAddr != null.
// Both cannot be non-null at the same time.
assert(ctrlExpr == nullptr);
assert(call->gtCallAddr != nullptr);
ctrlExpr = call->gtCallAddr;
}
// set reg requirements on call target represented as control sequence.
if (ctrlExpr != nullptr)
{
// we should never see a gtControlExpr whose type is void.
assert(ctrlExpr->TypeGet() != TYP_VOID);
info->srcCount++;
// In case of fast tail implemented as jmp, make sure that gtControlExpr is
// computed into a register.
if (call->IsFastTailCall())
{
NYI_ARM("tail call");
#ifdef _TARGET_ARM64_
// Fast tail call - make sure that call target is always computed in IP0
// so that epilog sequence can generate "br xip0" to achieve fast tail call.
ctrlExpr->gtLsraInfo.setSrcCandidates(l, genRegMask(REG_IP0));
#endif // _TARGET_ARM64_
}
}
#ifdef _TARGET_ARM_
else
{
info->internalIntCount = 1;
}
#endif // _TARGET_ARM_
RegisterType registerType = call->TypeGet();
// Set destination candidates for return value of the call.
#ifdef _TARGET_ARM_
if (call->IsHelperCall(compiler, CORINFO_HELP_INIT_PINVOKE_FRAME))
{
// The ARM CORINFO_HELP_INIT_PINVOKE_FRAME helper uses a custom calling convention that returns with
// TCB in REG_PINVOKE_TCB. fgMorphCall() sets the correct argument registers.
info->setDstCandidates(l, RBM_PINVOKE_TCB);
}
else
#endif // _TARGET_ARM_
if (hasMultiRegRetVal)
{
assert(retTypeDesc != nullptr);
info->setDstCandidates(l, retTypeDesc->GetABIReturnRegs());
}
else if (varTypeIsFloating(registerType))
{
info->setDstCandidates(l, RBM_FLOATRET);
}
else if (registerType == TYP_LONG)
{
info->setDstCandidates(l, RBM_LNGRET);
}
else
{
info->setDstCandidates(l, RBM_INTRET);
}
// If there is an explicit this pointer, we don't want that node to produce anything
// as it is redundant
if (call->gtCallObjp != nullptr)
{
GenTreePtr thisPtrNode = call->gtCallObjp;
if (thisPtrNode->gtOper == GT_PUTARG_REG)
{
l->clearOperandCounts(thisPtrNode);
thisPtrNode->SetContained();
l->clearDstCount(thisPtrNode->gtOp.gtOp1);
}
else
{
l->clearDstCount(thisPtrNode);
}
}
// First, count reg args
bool callHasFloatRegArgs = false;
for (GenTreePtr list = call->gtCallLateArgs; list; list = list->MoveNext())
{
assert(list->OperIsList());
GenTreePtr argNode = list->Current();
fgArgTabEntryPtr curArgTabEntry = compiler->gtArgEntryByNode(call, argNode);
assert(curArgTabEntry);
if (curArgTabEntry->regNum == REG_STK)
{
// late arg that is not passed in a register
assert(argNode->gtOper == GT_PUTARG_STK);
TreeNodeInfoInitPutArgStk(argNode->AsPutArgStk(), curArgTabEntry);
continue;
}
// A GT_FIELD_LIST has a TYP_VOID, but is used to represent a multireg struct
if (argNode->OperGet() == GT_FIELD_LIST)
{
argNode->SetContained();
// There could be up to 2-4 PUTARG_REGs in the list (3 or 4 can only occur for HFAs)
regNumber argReg = curArgTabEntry->regNum;
for (GenTreeFieldList* entry = argNode->AsFieldList(); entry != nullptr; entry = entry->Rest())
{
TreeNodeInfoInitPutArgReg(entry->Current()->AsUnOp(), argReg, *info, false, &callHasFloatRegArgs);
// Update argReg for the next putarg_reg (if any)
argReg = genRegArgNext(argReg);
#if defined(_TARGET_ARM_)
// A double register is modelled as an even-numbered single one
if (entry->Current()->TypeGet() == TYP_DOUBLE)
{
argReg = genRegArgNext(argReg);
}
#endif // _TARGET_ARM_
}
}
#ifdef _TARGET_ARM_
else if (argNode->OperGet() == GT_PUTARG_SPLIT)
{
fgArgTabEntryPtr curArgTabEntry = compiler->gtArgEntryByNode(call, argNode);
TreeNodeInfoInitPutArgSplit(argNode->AsPutArgSplit(), *info, curArgTabEntry);
}
#endif
else
{
TreeNodeInfoInitPutArgReg(argNode->AsUnOp(), curArgTabEntry->regNum, *info, false, &callHasFloatRegArgs);
}
}
// Now, count stack args
// Note that these need to be computed into a register, but then
// they're just stored to the stack - so the reg doesn't
// need to remain live until the call. In fact, it must not
// because the code generator doesn't actually consider it live,
// so it can't be spilled.
GenTreePtr args = call->gtCallArgs;
while (args)
{
GenTreePtr arg = args->gtOp.gtOp1;
// Skip arguments that have been moved to the Late Arg list
if (!(args->gtFlags & GTF_LATE_ARG))
{
if (arg->gtOper == GT_PUTARG_STK)
{
fgArgTabEntryPtr curArgTabEntry = compiler->gtArgEntryByNode(call, arg);
assert(curArgTabEntry);
assert(curArgTabEntry->regNum == REG_STK);
TreeNodeInfoInitPutArgStk(arg->AsPutArgStk(), curArgTabEntry);
}
#ifdef _TARGET_ARM_
else if (arg->OperGet() == GT_PUTARG_SPLIT)
{
fgArgTabEntryPtr curArgTabEntry = compiler->gtArgEntryByNode(call, arg);
TreeNodeInfoInitPutArgSplit(arg->AsPutArgSplit(), *info, curArgTabEntry);
}
#endif
else
{
TreeNodeInfo* argInfo = &(arg->gtLsraInfo);
if (argInfo->dstCount != 0)
{
argInfo->isLocalDefUse = true;
}
argInfo->dstCount = 0;
}
}
args = args->gtOp.gtOp2;
}
// If it is a fast tail call, it is already preferenced to use IP0.
// Therefore, no need set src candidates on call tgt again.
if (call->IsVarargs() && callHasFloatRegArgs && !call->IsFastTailCall() && (ctrlExpr != nullptr))
{
NYI_ARM("float reg varargs");
// Don't assign the call target to any of the argument registers because
// we will use them to also pass floating point arguments as required
// by Arm64 ABI.
ctrlExpr->gtLsraInfo.setSrcCandidates(l, l->allRegs(TYP_INT) & ~(RBM_ARG_REGS));
}
#ifdef _TARGET_ARM_
if (call->NeedsNullCheck())
{
info->internalIntCount++;
}
#endif // _TARGET_ARM_
}
//------------------------------------------------------------------------
// DecomposeStoreInd: Decompose GT_STOREIND.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
// TODO-LIR: replace comments below that use embedded statements with ones that do not.
GenTree* DecomposeLongs::DecomposeStoreInd(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_STOREIND);
GenTree* tree = use.Def();
assert(tree->gtOp.gtOp2->OperGet() == GT_LONG);
// Example input trees (a nested embedded statement case)
//
// <linkBegin Node>
// * stmtExpr void (top level) (IL ???... ???)
// | /--* argPlace ref $280
// | +--* argPlace int $4a
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar ref V11 tmp9 u:3 $21c
// | | { | +--* const int 4 $44
// | | { | /--* + byref $2c8
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclFld long V01 arg1 u:2[+8] Fseq[i] $380
// | | { | | { \--* st.lclVar long (P) V21 cse8
// | | { | | { \--* int V21.hi (offs=0x00) -> V22 rat0
// | | { | | { \--* int V21.hi (offs=0x04) -> V23 rat1
// | | { | | /--* lclVar int V22 rat0 $380
// | | { | | +--* lclVar int V23 rat1
// | | { | +--* gt_long long
// | | { \--* storeIndir long
// | +--* lclVar ref V11 tmp9 u:3 (last use) $21c
// | +--* lclVar ref V02 tmp0 u:3 $280
// | +--* const int 8 $4a
// \--* call help void HELPER.CORINFO_HELP_ARRADDR_ST $205
// <linkEndNode>
//
// (editor brace matching compensation: }}}}}}}}}}}}}}}}}})
GenTree* gtLong = tree->gtOp.gtOp2;
unsigned blockWeight = m_block->getBBWeight(m_compiler);
// Save address to a temp. It is used in storeIndLow and storeIndHigh trees.
LIR::Use address(BlockRange(), &tree->gtOp.gtOp1, tree);
address.ReplaceWithLclVar(m_compiler, blockWeight);
JITDUMP("[DecomposeStoreInd]: Saving address tree to a temp var:\n");
DISPTREERANGE(BlockRange(), address.Def());
if (!gtLong->gtOp.gtOp1->OperIsLeaf())
{
LIR::Use op1(BlockRange(), >Long->gtOp.gtOp1, gtLong);
op1.ReplaceWithLclVar(m_compiler, blockWeight);
JITDUMP("[DecomposeStoreInd]: Saving low data tree to a temp var:\n");
DISPTREERANGE(BlockRange(), op1.Def());
}
if (!gtLong->gtOp.gtOp2->OperIsLeaf())
{
LIR::Use op2(BlockRange(), >Long->gtOp.gtOp2, gtLong);
op2.ReplaceWithLclVar(m_compiler, blockWeight);
JITDUMP("[DecomposeStoreInd]: Saving high data tree to a temp var:\n");
DISPTREERANGE(BlockRange(), op2.Def());
}
// Example trees after embedded statements for address and data are added.
// This example saves all address and data trees into temp variables
// to show how those embedded statements are created.
//
// * stmtExpr void (top level) (IL ???... ???)
// | /--* argPlace ref $280
// | +--* argPlace int $4a
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar ref V11 tmp9 u:3 $21c
// | | { | +--* const int 4 $44
// | | { | /--* + byref $2c8
// | | { \--* st.lclVar byref V24 rat2
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar byref V24 rat2
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclFld long V01 arg1 u:2[+8] Fseq[i] $380380
// | | { | | { \--* st.lclVar long (P) V21 cse8
// | | { | | { \--* int V21.hi (offs=0x00) -> V22 rat0
// | | { | | { \--* int V21.hi (offs=0x04) -> V23 rat1
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclVar int V22 rat0 $380
// | | { | | { \--* st.lclVar int V25 rat3
// | | { | | /--* lclVar int V25 rat3
// | | { | | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | | { | /--* lclVar int V23 rat1
// | | { | | | { \--* st.lclVar int V26 rat4
// | | { | | +--* lclVar int V26 rat4
// | | { | +--* gt_long long
// | | { \--* storeIndir long
// | +--* lclVar ref V11 tmp9 u:3 (last use) $21c
// | +--* lclVar ref V02 tmp0 u:3 $280
// | +--* const int 8 $4a
// \--* call help void HELPER.CORINFO_HELP_ARRADDR_ST $205
//
// (editor brace matching compensation: }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}})
GenTree* addrBase = tree->gtOp.gtOp1;
GenTree* dataHigh = gtLong->gtOp.gtOp2;
GenTree* dataLow = gtLong->gtOp.gtOp1;
GenTree* storeIndLow = tree;
// Rewrite storeIndLow tree to save only lower 32-bit data.
//
// | | { | /--* lclVar byref V24 rat2 (address)
// ...
// | | { | +--* lclVar int V25 rat3 (lower 32-bit data)
// | | { | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | { | /--* lclVar int V23 rat1
// | | { | { \--* st.lclVar int V26 rat4
// | | { \--* storeIndir int
//
// (editor brace matching compensation: }}}}}}}}})
BlockRange().Remove(gtLong);
BlockRange().Remove(dataHigh);
storeIndLow->gtOp.gtOp2 = dataLow;
storeIndLow->gtType = TYP_INT;
// Construct storeIndHigh tree
//
// | | { *stmtExpr void (embedded)(IL ? ? ? ... ? ? ? )
// | | { | / --* lclVar int V26 rat4
// | | { | | / --* lclVar byref V24 rat2
// | | { | +--* lea(b + 4) ref
// | | { \--* storeIndir int
//
// (editor brace matching compensation: }}}}})
GenTree* addrBaseHigh = new (m_compiler, GT_LCL_VAR)
GenTreeLclVar(GT_LCL_VAR, addrBase->TypeGet(), addrBase->AsLclVarCommon()->GetLclNum(), BAD_IL_OFFSET);
GenTree* addrHigh =
new (m_compiler, GT_LEA) GenTreeAddrMode(TYP_REF, addrBaseHigh, nullptr, 0, genTypeSize(TYP_INT));
GenTree* storeIndHigh = new (m_compiler, GT_STOREIND) GenTreeStoreInd(TYP_INT, addrHigh, dataHigh);
storeIndHigh->gtFlags = (storeIndLow->gtFlags & (GTF_ALL_EFFECT | GTF_LIVENESS_MASK));
storeIndHigh->gtFlags |= GTF_REVERSE_OPS;
m_compiler->gtPrepareCost(storeIndHigh);
BlockRange().InsertAfter(storeIndLow, dataHigh, addrBaseHigh, addrHigh, storeIndHigh);
return storeIndHigh;
// Example final output
//
// * stmtExpr void (top level) (IL ???... ???)
// | /--* argPlace ref $280
// | +--* argPlace int $4a
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar ref V11 tmp9 u:3 $21c
// | | { | +--* const int 4 $44
// | | { | /--* + byref $2c8
// | | { \--* st.lclVar byref V24 rat2
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar byref V24 rat2
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclFld int V01 arg1 u:2[+8] Fseq[i] $380
// | | { | | { | +--* lclFld int V01 arg1 [+12]
// | | { | | { | /--* gt_long long
// | | { | | { \--* st.lclVar long (P) V21 cse8
// | | { | | { \--* int V21.hi (offs=0x00) -> V22 rat0
// | | { | | { \--* int V21.hi (offs=0x04) -> V23 rat1
// | | { | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | | { | /--* lclVar int V22 rat0 $380
// | | { | | { \--* st.lclVar int V25 rat3
// | | { | +--* lclVar int V25 rat3
// | | { | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | { | /--* lclVar int V23 rat1
// | | { | { \--* st.lclVar int V26 rat4
// | | { \--* storeIndir int
// | | { * stmtExpr void (embedded) (IL ???... ???)
// | | { | /--* lclVar int V26 rat4
// | | { | | /--* lclVar byref V24 rat2
// | | { | +--* lea(b+4) ref
// | | { \--* storeIndir int
// | | /--* lclVar ref V11 tmp9 u:3 (last use) $21c
// | +--* putarg_stk [+0x00] ref
// | | /--* lclVar ref V02 tmp0 u:3 $280
// | +--* putarg_reg ref
// | | /--* const int 8 $4a
// | +--* putarg_reg int
// \--* call help void HELPER.CORINFO_HELP_ARRADDR_ST $205
//
// (editor brace matching compensation: }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}})
}