// sanity checks that apply to all kinds of IR
void Rationalizer::SanityCheck()
{
// TODO: assert(!IsLIR());
BasicBlock* block;
foreach_block(comp, block)
{
for (GenTree* statement = block->bbTreeList; statement != nullptr; statement = statement->gtNext)
{
ValidateStatement(statement, block);
for (GenTree* tree = statement->gtStmt.gtStmtList; tree; tree = tree->gtNext)
{
// QMARK nodes should have been removed before this phase.
assert(tree->OperGet() != GT_QMARK);
if (tree->OperGet() == GT_ASG)
{
if (tree->gtGetOp1()->OperGet() == GT_LCL_VAR)
{
assert(tree->gtGetOp1()->gtFlags & GTF_VAR_DEF);
}
else if (tree->gtGetOp2()->OperGet() == GT_LCL_VAR)
{
assert(!(tree->gtGetOp2()->gtFlags & GTF_VAR_DEF));
}
}
}
}
}
}
//------------------------------------------------------------------------
// DecomposeMul: Decompose GT_MUL. The only GT_MULs that make it to decompose are
// those with the GTF_MUL_64RSLT flag set. These muls result in a mul instruction that
// returns its result in two registers like GT_CALLs do. Additionally, these muls are
// guaranteed to be in the form long = (long)int * (long)int. Therefore, to decompose
// these nodes, we convert them into GT_MUL_LONGs, undo the cast from int to long by
// stripping out the lo ops, and force them into the form var = mul, as we do for
// GT_CALLs. In codegen, we then produce a mul instruction that produces the result
// in edx:eax, and store those registers on the stack in genStoreLongLclVar.
//
// All other GT_MULs have been converted to helper calls in morph.cpp
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeMul(LIR::Use& use)
{
assert(use.IsInitialized());
GenTree* tree = use.Def();
genTreeOps oper = tree->OperGet();
assert(oper == GT_MUL);
assert((tree->gtFlags & GTF_MUL_64RSLT) != 0);
GenTree* op1 = tree->gtGetOp1();
GenTree* op2 = tree->gtGetOp2();
GenTree* loOp1 = op1->gtGetOp1();
GenTree* hiOp1 = op1->gtGetOp2();
GenTree* loOp2 = op2->gtGetOp1();
GenTree* hiOp2 = op2->gtGetOp2();
Range().Remove(hiOp1);
Range().Remove(hiOp2);
Range().Remove(op1);
Range().Remove(op2);
// Get rid of the hi ops. We don't need them.
tree->gtOp.gtOp1 = loOp1;
tree->gtOp.gtOp2 = loOp2;
tree->gtOper = GT_MUL_LONG;
return StoreNodeToVar(use);
}
//------------------------------------------------------------------------
// DecomposeNeg: Decompose GT_NEG.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeNeg(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_NEG);
GenTree* tree = use.Def();
GenTree* gtLong = tree->gtGetOp1();
noway_assert(gtLong->OperGet() == GT_LONG);
LIR::Use op1(Range(), >Long->gtOp.gtOp1, gtLong);
op1.ReplaceWithLclVar(m_compiler, m_blockWeight);
LIR::Use op2(Range(), >Long->gtOp.gtOp2, gtLong);
op2.ReplaceWithLclVar(m_compiler, m_blockWeight);
// Neither GT_NEG nor the introduced temporaries have side effects.
tree->gtFlags &= ~GTF_ALL_EFFECT;
GenTree* loOp1 = gtLong->gtGetOp1();
GenTree* hiOp1 = gtLong->gtGetOp2();
Range().Remove(gtLong);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
GenTree* zero = m_compiler->gtNewZeroConNode(TYP_INT);
GenTree* hiAdjust = m_compiler->gtNewOperNode(GT_ADD_HI, TYP_INT, hiOp1, zero);
GenTree* hiResult = m_compiler->gtNewOperNode(GT_NEG, TYP_INT, hiAdjust);
hiResult->gtFlags = tree->gtFlags;
Range().InsertAfter(loResult, zero, hiAdjust, hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeArith(LIR::Use& use)
{
assert(use.IsInitialized());
GenTree* tree = use.Def();
genTreeOps oper = tree->OperGet();
assert((oper == GT_ADD) || (oper == GT_SUB) || (oper == GT_OR) || (oper == GT_XOR) || (oper == GT_AND));
GenTree* op1 = tree->gtGetOp1();
GenTree* op2 = tree->gtGetOp2();
// Both operands must have already been decomposed into GT_LONG operators.
noway_assert((op1->OperGet() == GT_LONG) && (op2->OperGet() == GT_LONG));
// Capture the lo and hi halves of op1 and op2.
GenTree* loOp1 = op1->gtGetOp1();
GenTree* hiOp1 = op1->gtGetOp2();
GenTree* loOp2 = op2->gtGetOp1();
GenTree* hiOp2 = op2->gtGetOp2();
// Now, remove op1 and op2 from the node list.
Range().Remove(op1);
Range().Remove(op2);
// We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands
// will be the lo halves of op1 from above.
GenTree* loResult = tree;
loResult->SetOper(GetLoOper(oper));
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
loResult->gtOp.gtOp2 = loOp2;
GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2);
Range().InsertAfter(loResult, hiResult);
if ((oper == GT_ADD) || (oper == GT_SUB))
{
if (loResult->gtOverflow())
{
hiResult->gtFlags |= GTF_OVERFLOW;
loResult->gtFlags &= ~GTF_OVERFLOW;
}
if (loResult->gtFlags & GTF_UNSIGNED)
{
hiResult->gtFlags |= GTF_UNSIGNED;
}
}
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeNot: Decompose GT_NOT.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeNot(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_NOT);
GenTree* tree = use.Def();
GenTree* gtLong = tree->gtGetOp1();
noway_assert(gtLong->OperGet() == GT_LONG);
GenTree* loOp1 = gtLong->gtGetOp1();
GenTree* hiOp1 = gtLong->gtGetOp2();
Range().Remove(gtLong);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
GenTree* hiResult = new (m_compiler, GT_NOT) GenTreeOp(GT_NOT, TYP_INT, hiOp1, nullptr);
Range().InsertAfter(loResult, hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeStoreLclVar: Decompose GT_STORE_LCL_VAR.
//
// Arguments:
// ppTree - the tree to decompose
// data - tree walk context
//
// Return Value:
// None.
//
void DecomposeLongs::DecomposeStoreLclVar(GenTree** ppTree, Compiler::fgWalkData* data)
{
assert(ppTree != nullptr);
assert(*ppTree != nullptr);
assert(data != nullptr);
assert((*ppTree)->OperGet() == GT_STORE_LCL_VAR);
assert(m_compiler->compCurStmt != nullptr);
GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt();
GenTree* tree = *ppTree;
GenTree* nextTree = tree->gtNext;
GenTree* rhs = tree->gtGetOp1();
if ((rhs->OperGet() == GT_PHI) || (rhs->OperGet() == GT_CALL))
{
// GT_CALLs are not decomposed, so will not be converted to GT_LONG
// GT_STORE_LCL_VAR = GT_CALL are handled in genMultiRegCallStoreToLocal
return;
}
noway_assert(rhs->OperGet() == GT_LONG);
unsigned varNum = tree->AsLclVarCommon()->gtLclNum;
LclVarDsc* varDsc = m_compiler->lvaTable + varNum;
m_compiler->lvaDecRefCnts(tree);
GenTree* loRhs = rhs->gtGetOp1();
GenTree* hiRhs = rhs->gtGetOp2();
GenTree* hiStore = m_compiler->gtNewLclLNode(varNum, TYP_INT);
if (varDsc->lvPromoted)
{
assert(varDsc->lvFieldCnt == 2);
unsigned loVarNum = varDsc->lvFieldLclStart;
unsigned hiVarNum = loVarNum + 1;
tree->AsLclVarCommon()->SetLclNum(loVarNum);
hiStore->SetOper(GT_STORE_LCL_VAR);
hiStore->AsLclVarCommon()->SetLclNum(hiVarNum);
}
else
{
noway_assert(varDsc->lvLRACandidate == false);
tree->SetOper(GT_STORE_LCL_FLD);
tree->AsLclFld()->gtLclOffs = 0;
tree->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
hiStore->SetOper(GT_STORE_LCL_FLD);
hiStore->AsLclFld()->gtLclOffs = 4;
hiStore->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
}
tree->gtOp.gtOp1 = loRhs;
tree->gtType = TYP_INT;
loRhs->gtNext = tree;
tree->gtPrev = loRhs;
hiStore->gtOp.gtOp1 = hiRhs;
hiStore->CopyCosts(tree);
hiStore->gtFlags |= GTF_VAR_DEF;
m_compiler->lvaIncRefCnts(tree);
m_compiler->lvaIncRefCnts(hiStore);
tree->gtNext = hiRhs;
hiRhs->gtPrev = tree;
hiRhs->gtNext = hiStore;
hiStore->gtPrev = hiRhs;
hiStore->gtNext = nextTree;
if (nextTree != nullptr)
{
nextTree->gtPrev = hiStore;
}
nextTree = hiRhs;
bool isEmbeddedStmt = !curStmt->gtStmtIsTopLevel();
if (!isEmbeddedStmt)
{
tree->gtNext = nullptr;
hiRhs->gtPrev = nullptr;
}
InsertNodeAsStmt(hiStore);
}
//------------------------------------------------------------------------
// DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND.
//
// Arguments:
// ppTree - the tree to decompose
// data - tree walk context
//
// Return Value:
// None.
//
void DecomposeLongs::DecomposeArith(GenTree** ppTree, Compiler::fgWalkData* data)
{
assert(ppTree != nullptr);
assert(*ppTree != nullptr);
assert(data != nullptr);
assert(m_compiler->compCurStmt != nullptr);
GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt();
GenTree* tree = *ppTree;
genTreeOps oper = tree->OperGet();
assert((oper == GT_ADD) ||
(oper == GT_SUB) ||
(oper == GT_OR) ||
(oper == GT_XOR) ||
(oper == GT_AND));
NYI_IF((tree->gtFlags & GTF_REVERSE_OPS) != 0, "Binary operator with GTF_REVERSE_OPS");
GenTree* op1 = tree->gtGetOp1();
GenTree* op2 = tree->gtGetOp2();
// Both operands must have already been decomposed into GT_LONG operators.
noway_assert((op1->OperGet() == GT_LONG) && (op2->OperGet() == GT_LONG));
// Capture the lo and hi halves of op1 and op2.
GenTree* loOp1 = op1->gtGetOp1();
GenTree* hiOp1 = op1->gtGetOp2();
GenTree* loOp2 = op2->gtGetOp1();
GenTree* hiOp2 = op2->gtGetOp2();
// We don't have support to decompose a TYP_LONG node that already has a child that has
// been decomposed into parts, where the high part depends on the value generated by the
// low part (via the flags register). For example, if we have:
// +(gt_long(+(lo3, lo4), +Hi(hi3, hi4)), gt_long(lo2, hi2))
// We would decompose it here to:
// gt_long(+(+(lo3, lo4), lo2), +Hi(+Hi(hi3, hi4), hi2))
// But this would generate incorrect code, because the "+Hi(hi3, hi4)" code generation
// needs to immediately follow the "+(lo3, lo4)" part. Also, if this node is one that
// requires a unique high operator, and the child nodes are not simple locals (e.g.,
// they are decomposed nodes), then we also can't decompose the node, as we aren't
// guaranteed the high and low parts will be executed immediately after each other.
NYI_IF(hiOp1->OperIsHigh() ||
hiOp2->OperIsHigh() ||
(GenTree::OperIsHigh(GetHiOper(oper)) &&
(!loOp1->OperIsLeaf() ||
!hiOp1->OperIsLeaf() ||
!loOp1->OperIsLeaf() ||
!hiOp2->OperIsLeaf())),
"Can't decompose expression tree TYP_LONG node");
// Now, remove op1 and op2 from the node list.
m_compiler->fgSnipNode(curStmt, op1);
m_compiler->fgSnipNode(curStmt, op2);
// We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands
// will be the lo halves of op1 from above.
GenTree* loResult = tree;
loResult->SetOper(GetLoOper(loResult->OperGet()));
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
loResult->gtOp.gtOp2 = loOp2;
// The various halves will be correctly threaded internally. We simply need to
// relink them into the proper order, i.e. loOp1 is followed by loOp2, and then
// the loResult node.
// (This rethreading, and that below, are where we need to address the reverse ops case).
// The current order is (after snipping op1 and op2):
// ... loOp1-> ... hiOp1->loOp2First ... loOp2->hiOp2First ... hiOp2
// The order we want is:
// ... loOp1->loOp2First ... loOp2->loResult
// ... hiOp1->hiOp2First ... hiOp2->hiResult
// i.e. we swap hiOp1 and loOp2, and create (for now) separate loResult and hiResult trees
GenTree* loOp2First = hiOp1->gtNext;
GenTree* hiOp2First = loOp2->gtNext;
// First, we will NYI if both hiOp1 and loOp2 have side effects.
NYI_IF(((loOp2->gtFlags & GTF_ALL_EFFECT) != 0) && ((hiOp1->gtFlags & GTF_ALL_EFFECT) != 0),
"Binary long operator with non-reorderable sub expressions");
// Now, we reorder the loOps and the loResult.
loOp1->gtNext = loOp2First;
loOp2First->gtPrev = loOp1;
loOp2->gtNext = loResult;
loResult->gtPrev = loOp2;
// Next, reorder the hiOps and the hiResult.
GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2);
hiOp1->gtNext = hiOp2First;
hiOp2First->gtPrev = hiOp1;
hiOp2->gtNext = hiResult;
hiResult->gtPrev = hiOp2;
if ((oper == GT_ADD) || (oper == GT_SUB))
{
if (loResult->gtOverflow())
{
hiResult->gtFlags |= GTF_OVERFLOW;
loResult->gtFlags &= ~GTF_OVERFLOW;
}
if (loResult->gtFlags & GTF_UNSIGNED)
{
hiResult->gtFlags |= GTF_UNSIGNED;
}
}
FinalizeDecomposition(ppTree, data, loResult, hiResult);
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding list nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_FIELD_LIST head nodes, and GT_LIST nodes used by phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperIsAnyList() && !(prev->OperIsFieldListHead());
prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node that is not an aggregate.
if (node->OperIsAnyList())
{
GenTreeArgList* list = node->AsArgList();
if (!list->OperIsFieldListHead())
{
BlockRange().Remove(list);
}
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_IND:
// Clear the `GTF_IND_ASG_LHS` flag, which overlaps with `GTF_IND_REQ_ADDR_IN_REG`.
node->gtFlags &= ~GTF_IND_ASG_LHS;
if (varTypeIsSIMD(node))
{
RewriteSIMDOperand(use, false);
}
else
{
// Due to promotion of structs containing fields of type struct with a
// single scalar type field, we could potentially see IR nodes of the
// form GT_IND(GT_ADD(lclvarAddr, 0)) where 0 is an offset representing
// a field-seq. These get folded here.
//
// TODO: This code can be removed once JIT implements recursive struct
// promotion instead of lying about the type of struct field as the type
// of its single scalar field.
GenTree* addr = node->AsIndir()->Addr();
if (addr->OperGet() == GT_ADD && addr->gtGetOp1()->OperGet() == GT_LCL_VAR_ADDR &&
addr->gtGetOp2()->IsIntegralConst(0))
{
GenTreeLclVarCommon* lclVarNode = addr->gtGetOp1()->AsLclVarCommon();
unsigned lclNum = lclVarNode->GetLclNum();
LclVarDsc* varDsc = comp->lvaTable + lclNum;
if (node->TypeGet() == varDsc->TypeGet())
{
JITDUMP("Rewriting GT_IND(GT_ADD(LCL_VAR_ADDR,0)) to LCL_VAR\n");
lclVarNode->SetOper(GT_LCL_VAR);
lclVarNode->gtType = node->TypeGet();
use.ReplaceWith(comp, lclVarNode);
BlockRange().Remove(addr);
BlockRange().Remove(addr->gtGetOp2());
BlockRange().Remove(node);
}
}
}
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#if defined(_TARGET_XARCH_) || defined(_TARGET_ARM_)
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_BLK:
case GT_OBJ:
{
// TODO-1stClassStructs: These should have been transformed to GT_INDs, but in order
// to preserve existing behavior, we will keep this as a block node if this is the
// lhs of a block assignment, and either:
// - It is a "generic" TYP_STRUCT assignment, OR
// - It is an initblk, OR
// - Neither the lhs or rhs are known to be of SIMD type.
GenTree* parent = use.User();
bool keepBlk = false;
if ((parent->OperGet() == GT_ASG) && (node == parent->gtGetOp1()))
{
if ((node->TypeGet() == TYP_STRUCT) || parent->OperIsInitBlkOp())
{
keepBlk = true;
}
else if (!comp->isAddrOfSIMDType(node->AsBlk()->Addr()))
{
GenTree* dataSrc = parent->gtGetOp2();
if (!dataSrc->IsLocal() && (dataSrc->OperGet() != GT_SIMD))
{
noway_assert(dataSrc->OperIsIndir());
keepBlk = !comp->isAddrOfSIMDType(dataSrc->AsIndir()->Addr());
}
}
}
RewriteSIMDOperand(use, keepBlk);
}
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2IfPresent();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
// JCC nodes should not be present in HIR.
assert(node->OperGet() != GT_JCC);
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (node->OperIsLocalRead())
{
if (use.IsDummyUse())
{
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
else
{
// Local reads are side-effect-free; clear any flags leftover from frontend transformations.
node->gtFlags &= ~GTF_ALL_EFFECT;
}
}
assert(isLateArg == ((use.Def()->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}
//------------------------------------------------------------------------
// DecomposeShift: Decompose GT_LSH, GT_RSH, GT_RSZ. For shift nodes, we need to use
// the shift helper functions, so we here convert the shift into a helper call by
// pulling its arguments out of linear order and making them the args to a call, then
// replacing the original node with the new call.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeShift(LIR::Use& use)
{
assert(use.IsInitialized());
GenTree* tree = use.Def();
GenTree* gtLong = tree->gtGetOp1();
genTreeOps oper = tree->OperGet();
assert((oper == GT_LSH) || (oper == GT_RSH) || (oper == GT_RSZ));
LIR::Use loOp1Use(Range(), >Long->gtOp.gtOp1, gtLong);
loOp1Use.ReplaceWithLclVar(m_compiler, m_blockWeight);
LIR::Use hiOp1Use(Range(), >Long->gtOp.gtOp2, gtLong);
hiOp1Use.ReplaceWithLclVar(m_compiler, m_blockWeight);
LIR::Use shiftWidthUse(Range(), &tree->gtOp.gtOp2, tree);
shiftWidthUse.ReplaceWithLclVar(m_compiler, m_blockWeight);
GenTree* loOp1 = gtLong->gtGetOp1();
GenTree* hiOp1 = gtLong->gtGetOp2();
GenTree* shiftWidthOp = tree->gtGetOp2();
Range().Remove(gtLong);
Range().Remove(loOp1);
Range().Remove(hiOp1);
Range().Remove(shiftWidthOp);
// TODO-X86-CQ: If the shift operand is a GT_CNS_INT, we should pipe the instructions through to codegen
// and generate the shift instructions ourselves there, rather than replacing it with a helper call.
unsigned helper;
switch (oper)
{
case GT_LSH:
helper = CORINFO_HELP_LLSH;
break;
case GT_RSH:
helper = CORINFO_HELP_LRSH;
break;
case GT_RSZ:
helper = CORINFO_HELP_LRSZ;
break;
default:
unreached();
}
GenTreeArgList* argList = m_compiler->gtNewArgList(loOp1, hiOp1, shiftWidthOp);
GenTree* call = m_compiler->gtNewHelperCallNode(helper, TYP_LONG, 0, argList);
GenTreeCall* callNode = call->AsCall();
ReturnTypeDesc* retTypeDesc = callNode->GetReturnTypeDesc();
retTypeDesc->InitializeLongReturnType(m_compiler);
call = m_compiler->fgMorphArgs(callNode);
Range().InsertAfter(tree, LIR::SeqTree(m_compiler, call));
Range().Remove(tree);
use.ReplaceWith(m_compiler, call);
return call;
}
//------------------------------------------------------------------------
// DecomposeCast: Decompose GT_CAST.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_CAST);
GenTree* cast = use.Def()->AsCast();
GenTree* loResult = nullptr;
GenTree* hiResult = nullptr;
var_types srcType = cast->CastFromType();
var_types dstType = cast->CastToType();
if ((cast->gtFlags & GTF_UNSIGNED) != 0)
{
srcType = genUnsignedType(srcType);
}
if (varTypeIsLong(srcType))
{
if (cast->gtOverflow() && (varTypeIsUnsigned(srcType) != varTypeIsUnsigned(dstType)))
{
GenTree* srcOp = cast->gtGetOp1();
noway_assert(srcOp->OperGet() == GT_LONG);
GenTree* loSrcOp = srcOp->gtGetOp1();
GenTree* hiSrcOp = srcOp->gtGetOp2();
//
// When casting between long types an overflow check is needed only if the types
// have different signedness. In both cases (long->ulong and ulong->long) we only
// need to check if the high part is negative or not. Use the existing cast node
// to perform a int->uint cast of the high part to take advantage of the overflow
// check provided by codegen.
//
loResult = loSrcOp;
hiResult = cast;
hiResult->gtType = TYP_INT;
hiResult->AsCast()->gtCastType = TYP_UINT;
hiResult->gtFlags &= ~GTF_UNSIGNED;
hiResult->gtOp.gtOp1 = hiSrcOp;
Range().Remove(cast);
Range().Remove(srcOp);
Range().InsertAfter(hiSrcOp, hiResult);
}
else
{
NYI("Unimplemented long->long no-op cast decomposition");
}
}
else if (varTypeIsIntegralOrI(srcType))
{
if (cast->gtOverflow() && !varTypeIsUnsigned(srcType) && varTypeIsUnsigned(dstType))
{
//
// An overflow check is needed only when casting from a signed type to ulong.
// Change the cast type to uint to take advantage of the overflow check provided
// by codegen and then zero extend the resulting uint to ulong.
//
loResult = cast;
loResult->AsCast()->gtCastType = TYP_UINT;
loResult->gtType = TYP_INT;
hiResult = m_compiler->gtNewZeroConNode(TYP_INT);
Range().InsertAfter(loResult, hiResult);
}
else
{
if (varTypeIsUnsigned(srcType))
{
loResult = cast->gtGetOp1();
hiResult = m_compiler->gtNewZeroConNode(TYP_INT);
Range().Remove(cast);
Range().InsertAfter(loResult, hiResult);
}
else
{
LIR::Use src(Range(), &(cast->gtOp.gtOp1), cast);
unsigned lclNum = src.ReplaceWithLclVar(m_compiler, m_blockWeight);
loResult = src.Def();
GenTree* loCopy = m_compiler->gtNewLclvNode(lclNum, TYP_INT);
GenTree* shiftBy = m_compiler->gtNewIconNode(31, TYP_INT);
hiResult = m_compiler->gtNewOperNode(GT_RSH, TYP_INT, loCopy, shiftBy);
Range().Remove(cast);
Range().InsertAfter(loResult, loCopy, shiftBy, hiResult);
m_compiler->lvaIncRefCnts(loCopy);
}
}
}
else
{
NYI("Unimplemented cast decomposition");
}
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeStoreLclVar: Decompose GT_STORE_LCL_VAR.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeStoreLclVar(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_STORE_LCL_VAR);
GenTree* tree = use.Def();
GenTree* rhs = tree->gtGetOp1();
if ((rhs->OperGet() == GT_PHI) || (rhs->OperGet() == GT_CALL) ||
((rhs->OperGet() == GT_MUL_LONG) && (rhs->gtFlags & GTF_MUL_64RSLT) != 0))
{
// GT_CALLs are not decomposed, so will not be converted to GT_LONG
// GT_STORE_LCL_VAR = GT_CALL are handled in genMultiRegCallStoreToLocal
// GT_MULs are not decomposed, so will not be converted to GT_LONG
return tree->gtNext;
}
noway_assert(rhs->OperGet() == GT_LONG);
unsigned varNum = tree->AsLclVarCommon()->gtLclNum;
LclVarDsc* varDsc = m_compiler->lvaTable + varNum;
m_compiler->lvaDecRefCnts(tree);
GenTree* loRhs = rhs->gtGetOp1();
GenTree* hiRhs = rhs->gtGetOp2();
GenTree* hiStore = m_compiler->gtNewLclLNode(varNum, TYP_INT);
if (varDsc->lvPromoted)
{
assert(varDsc->lvFieldCnt == 2);
unsigned loVarNum = varDsc->lvFieldLclStart;
unsigned hiVarNum = loVarNum + 1;
tree->AsLclVarCommon()->SetLclNum(loVarNum);
hiStore->SetOper(GT_STORE_LCL_VAR);
hiStore->AsLclVarCommon()->SetLclNum(hiVarNum);
}
else
{
noway_assert(varDsc->lvLRACandidate == false);
tree->SetOper(GT_STORE_LCL_FLD);
tree->AsLclFld()->gtLclOffs = 0;
tree->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
hiStore->SetOper(GT_STORE_LCL_FLD);
hiStore->AsLclFld()->gtLclOffs = 4;
hiStore->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
}
// 'tree' is going to steal the loRhs node for itself, so we need to remove the
// GT_LONG node from the threading.
Range().Remove(rhs);
tree->gtOp.gtOp1 = loRhs;
tree->gtType = TYP_INT;
hiStore->gtOp.gtOp1 = hiRhs;
hiStore->gtFlags |= GTF_VAR_DEF;
m_compiler->lvaIncRefCnts(tree);
m_compiler->lvaIncRefCnts(hiStore);
Range().InsertAfter(tree, hiStore);
return hiStore->gtNext;
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding GT_LIST nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_LIST nodes that are used by block ops and phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperGet() == GT_LIST; prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node.
if ((*useEdge)->OperGet() == GT_LIST)
{
BlockRange().Remove(*useEdge);
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#ifdef _TARGET_XARCH_
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_INITBLK:
RewriteInitBlk(use);
break;
case GT_COPYBLK:
RewriteCopyBlk(use);
break;
case GT_OBJ:
RewriteObj(use);
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_STOREIND:
case GT_IND:
if (node->gtType == TYP_STRUCT)
{
GenTree* addr = node->AsIndir()->Addr();
assert(addr->TypeGet() == TYP_BYREF);
if (addr->OperIsLocal())
{
LclVarDsc* varDsc = &(comp->lvaTable[addr->AsLclVarCommon()->gtLclNum]);
assert(varDsc->lvSIMDType);
unsigned simdSize = (unsigned int)roundUp(varDsc->lvExactSize, TARGET_POINTER_SIZE);
node->gtType = comp->getSIMDTypeForSize(simdSize);
}
#if DEBUG
else
{
// If the address is not a local var, assert that the user of this IND is an ADDR node.
assert((use.User()->OperGet() == GT_ADDR) || use.User()->OperIsLocalAddr());
}
#endif
}
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
else if (simdNode->gtType == TYP_STRUCT || varTypeIsSIMD(simdNode))
{
node->gtType = simdType;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (use.IsDummyUse() && node->OperIsLocalRead())
{
assert((node->gtFlags & GTF_ALL_EFFECT) == 0);
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
assert(isLateArg == ((node->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}
//------------------------------------------------------------------------
// MorphAllocObjNodes: Morph each GT_ALLOCOBJ node either into an
// allocation helper call or stack allocation.
//
// Notes:
// Runs only over the blocks having bbFlags BBF_HAS_NEWOBJ set.
void ObjectAllocator::MorphAllocObjNodes()
{
BasicBlock* block;
foreach_block(comp, block)
{
const bool basicBlockHasNewObj = (block->bbFlags & BBF_HAS_NEWOBJ) == BBF_HAS_NEWOBJ;
#ifndef DEBUG
if (!basicBlockHasNewObj)
{
continue;
}
#endif // DEBUG
for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt)
{
GenTree* stmtExpr = stmt->gtStmtExpr;
GenTree* op2 = nullptr;
bool canonicalAllocObjFound = false;
if (stmtExpr->OperGet() == GT_ASG && stmtExpr->TypeGet() == TYP_REF)
{
op2 = stmtExpr->gtGetOp2();
if (op2->OperGet() == GT_ALLOCOBJ)
{
canonicalAllocObjFound = true;
}
}
if (canonicalAllocObjFound)
{
assert(basicBlockHasNewObj);
//------------------------------------------------------------------------
// We expect the following expression tree at this point
// * GT_STMT void (top level)
// | /--* GT_ALLOCOBJ ref
// \--* GT_ASG ref
// \--* GT_LCL_VAR ref
//------------------------------------------------------------------------
GenTree* op1 = stmtExpr->gtGetOp1();
assert(op1->OperGet() == GT_LCL_VAR);
assert(op1->TypeGet() == TYP_REF);
assert(op2 != nullptr);
assert(op2->OperGet() == GT_ALLOCOBJ);
GenTreeAllocObj* asAllocObj = op2->AsAllocObj();
unsigned int lclNum = op1->AsLclVar()->GetLclNum();
if (IsObjectStackAllocationEnabled() && CanAllocateLclVarOnStack(lclNum))
{
op2 = MorphAllocObjNodeIntoStackAlloc(asAllocObj, block, stmt);
}
else
{
op2 = MorphAllocObjNodeIntoHelperCall(asAllocObj);
}
// Propagate flags of op2 to its parent.
stmtExpr->gtOp.gtOp2 = op2;
stmtExpr->gtFlags |= op2->gtFlags & GTF_ALL_EFFECT;
}
#ifdef DEBUG
else
{
// We assume that GT_ALLOCOBJ nodes are always present in the
// canonical form.
comp->fgWalkTreePre(&stmt->gtStmtExpr, AssertWhenAllocObjFoundVisitor);
}
#endif // DEBUG
}
}
}
//------------------------------------------------------------------------
// BuildSIMD: Set the NodeInfo for a GT_SIMD tree.
//
// Arguments:
// tree - The GT_SIMD node of interest
//
// Return Value:
// The number of sources consumed by this node.
//
int LinearScan::BuildSIMD(GenTreeSIMD* simdTree)
{
int srcCount = 0;
// Only SIMDIntrinsicInit can be contained
if (simdTree->isContained())
{
assert(simdTree->gtSIMDIntrinsicID == SIMDIntrinsicInit);
}
int dstCount = simdTree->IsValue() ? 1 : 0;
assert(dstCount == 1);
bool buildUses = true;
GenTree* op1 = simdTree->gtGetOp1();
GenTree* op2 = simdTree->gtGetOp2();
switch (simdTree->gtSIMDIntrinsicID)
{
case SIMDIntrinsicInit:
case SIMDIntrinsicCast:
case SIMDIntrinsicSqrt:
case SIMDIntrinsicAbs:
case SIMDIntrinsicConvertToSingle:
case SIMDIntrinsicConvertToInt32:
case SIMDIntrinsicConvertToDouble:
case SIMDIntrinsicConvertToInt64:
case SIMDIntrinsicWidenLo:
case SIMDIntrinsicWidenHi:
// No special handling required.
break;
case SIMDIntrinsicGetItem:
{
op1 = simdTree->gtGetOp1();
op2 = simdTree->gtGetOp2();
// We have an object and an index, either of which may be contained.
bool setOp2DelayFree = false;
if (!op2->IsCnsIntOrI() && (!op1->isContained() || op1->OperIsLocal()))
{
// If the index is not a constant and the object is not contained or is a local
// we will need a general purpose register to calculate the address
// internal register must not clobber input index
// TODO-Cleanup: An internal register will never clobber a source; this code actually
// ensures that the index (op2) doesn't interfere with the target.
buildInternalIntRegisterDefForNode(simdTree);
setOp2DelayFree = true;
}
srcCount += BuildOperandUses(op1);
if (!op2->isContained())
{
RefPosition* op2Use = BuildUse(op2);
if (setOp2DelayFree)
{
setDelayFree(op2Use);
}
srcCount++;
}
if (!op2->IsCnsIntOrI() && (!op1->isContained()))
{
// If vector is not already in memory (contained) and the index is not a constant,
// we will use the SIMD temp location to store the vector.
compiler->getSIMDInitTempVarNum();
}
buildUses = false;
}
break;
case SIMDIntrinsicAdd:
case SIMDIntrinsicSub:
case SIMDIntrinsicMul:
case SIMDIntrinsicDiv:
case SIMDIntrinsicBitwiseAnd:
case SIMDIntrinsicBitwiseAndNot:
case SIMDIntrinsicBitwiseOr:
case SIMDIntrinsicBitwiseXor:
case SIMDIntrinsicMin:
case SIMDIntrinsicMax:
case SIMDIntrinsicEqual:
case SIMDIntrinsicLessThan:
case SIMDIntrinsicGreaterThan:
case SIMDIntrinsicLessThanOrEqual:
case SIMDIntrinsicGreaterThanOrEqual:
// No special handling required.
break;
case SIMDIntrinsicSetX:
case SIMDIntrinsicSetY:
case SIMDIntrinsicSetZ:
case SIMDIntrinsicSetW:
case SIMDIntrinsicNarrow:
{
// Op1 will write to dst before Op2 is free
BuildUse(op1);
RefPosition* op2Use = BuildUse(op2);
setDelayFree(op2Use);
srcCount = 2;
buildUses = false;
break;
}
case SIMDIntrinsicInitN:
{
var_types baseType = simdTree->gtSIMDBaseType;
srcCount = (short)(simdTree->gtSIMDSize / genTypeSize(baseType));
if (varTypeIsFloating(simdTree->gtSIMDBaseType))
{
// Need an internal register to stitch together all the values into a single vector in a SIMD reg.
buildInternalFloatRegisterDefForNode(simdTree);
}
int initCount = 0;
for (GenTree* list = op1; list != nullptr; list = list->gtGetOp2())
{
assert(list->OperGet() == GT_LIST);
GenTree* listItem = list->gtGetOp1();
assert(listItem->TypeGet() == baseType);
assert(!listItem->isContained());
BuildUse(listItem);
initCount++;
}
assert(initCount == srcCount);
buildUses = false;
break;
}
case SIMDIntrinsicInitArray:
// We have an array and an index, which may be contained.
break;
case SIMDIntrinsicOpEquality:
case SIMDIntrinsicOpInEquality:
buildInternalFloatRegisterDefForNode(simdTree);
break;
case SIMDIntrinsicDotProduct:
buildInternalFloatRegisterDefForNode(simdTree);
break;
case SIMDIntrinsicSelect:
// TODO-ARM64-CQ Allow lowering to see SIMDIntrinsicSelect so we can generate BSL VC, VA, VB
// bsl target register must be VC. Reserve a temp in case we need to shuffle things.
// This will require a different approach, as GenTreeSIMD has only two operands.
assert(!"SIMDIntrinsicSelect not yet supported");
buildInternalFloatRegisterDefForNode(simdTree);
break;
case SIMDIntrinsicInitArrayX:
case SIMDIntrinsicInitFixed:
case SIMDIntrinsicCopyToArray:
case SIMDIntrinsicCopyToArrayX:
case SIMDIntrinsicNone:
case SIMDIntrinsicGetCount:
case SIMDIntrinsicGetOne:
case SIMDIntrinsicGetZero:
case SIMDIntrinsicGetAllOnes:
case SIMDIntrinsicGetX:
case SIMDIntrinsicGetY:
case SIMDIntrinsicGetZ:
case SIMDIntrinsicGetW:
case SIMDIntrinsicInstEquals:
case SIMDIntrinsicHWAccel:
case SIMDIntrinsicWiden:
case SIMDIntrinsicInvalid:
assert(!"These intrinsics should not be seen during register allocation");
__fallthrough;
default:
noway_assert(!"Unimplemented SIMD node type.");
unreached();
}
if (buildUses)
{
assert(!op1->OperIs(GT_LIST));
assert(srcCount == 0);
srcCount = BuildOperandUses(op1);
if ((op2 != nullptr) && !op2->isContained())
{
srcCount += BuildOperandUses(op2);
}
}
assert(internalCount <= MaxInternalCount);
buildInternalRegisterUses();
if (dstCount == 1)
{
BuildDef(simdTree);
}
else
{
assert(dstCount == 0);
}
return srcCount;
}
//------------------------------------------------------------------------
// DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeArith(LIR::Use& use)
{
assert(use.IsInitialized());
GenTree* tree = use.Def();
genTreeOps oper = tree->OperGet();
assert((oper == GT_ADD) || (oper == GT_SUB) || (oper == GT_OR) || (oper == GT_XOR) || (oper == GT_AND));
GenTree* op1 = tree->gtGetOp1();
GenTree* op2 = tree->gtGetOp2();
// Both operands must have already been decomposed into GT_LONG operators.
noway_assert((op1->OperGet() == GT_LONG) && (op2->OperGet() == GT_LONG));
// Capture the lo and hi halves of op1 and op2.
GenTree* loOp1 = op1->gtGetOp1();
GenTree* hiOp1 = op1->gtGetOp2();
GenTree* loOp2 = op2->gtGetOp1();
GenTree* hiOp2 = op2->gtGetOp2();
// We don't have support to decompose a TYP_LONG node that already has a child that has
// been decomposed into parts, where the high part depends on the value generated by the
// low part (via the flags register). For example, if we have:
// +(gt_long(+(lo3, lo4), +Hi(hi3, hi4)), gt_long(lo2, hi2))
// We would decompose it here to:
// gt_long(+(+(lo3, lo4), lo2), +Hi(+Hi(hi3, hi4), hi2))
// But this would generate incorrect code, because the "+Hi(hi3, hi4)" code generation
// needs to immediately follow the "+(lo3, lo4)" part. Also, if this node is one that
// requires a unique high operator, and the child nodes are not simple locals (e.g.,
// they are decomposed nodes), then we also can't decompose the node, as we aren't
// guaranteed the high and low parts will be executed immediately after each other.
NYI_IF(hiOp1->OperIsHigh() || hiOp2->OperIsHigh() ||
(GenTree::OperIsHigh(GetHiOper(oper)) &&
(!loOp1->OperIsLeaf() || !hiOp1->OperIsLeaf() || !loOp1->OperIsLeaf() || !hiOp2->OperIsLeaf())),
"Can't decompose expression tree TYP_LONG node");
// Now, remove op1 and op2 from the node list.
BlockRange().Remove(op1);
BlockRange().Remove(op2);
// We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands
// will be the lo halves of op1 from above.
GenTree* loResult = tree;
loResult->SetOper(GetLoOper(loResult->OperGet()));
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
loResult->gtOp.gtOp2 = loOp2;
GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2);
hiResult->CopyCosts(loResult);
BlockRange().InsertAfter(loResult, hiResult);
if ((oper == GT_ADD) || (oper == GT_SUB))
{
if (loResult->gtOverflow())
{
hiResult->gtFlags |= GTF_OVERFLOW;
loResult->gtFlags &= ~GTF_OVERFLOW;
}
if (loResult->gtFlags & GTF_UNSIGNED)
{
hiResult->gtFlags |= GTF_UNSIGNED;
}
}
return FinalizeDecomposition(use, loResult, hiResult);
}
