// Rewrite a SIMD indirection as GT_IND(GT_LEA(obj.op1)), or as a simple
// lclVar if possible.
//
// Arguments:
// use - A use reference for a block node
// keepBlk - True if this should remain a block node if it is not a lclVar
//
// Return Value:
// None.
//
// TODO-1stClassStructs: These should be eliminated earlier, once we can handle
// lclVars in all the places that used to have GT_OBJ.
//
void Rationalizer::RewriteSIMDOperand(LIR::Use& use, bool keepBlk)
{
#ifdef FEATURE_SIMD
// No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
GenTree* tree = use.Def();
if (!tree->OperIsIndir())
{
return;
}
var_types simdType = tree->TypeGet();
if (!varTypeIsSIMD(simdType))
{
return;
}
// If we have GT_IND(GT_LCL_VAR_ADDR) and the GT_LCL_VAR_ADDR is TYP_BYREF/TYP_I_IMPL,
// and the var is a SIMD type, replace the expression by GT_LCL_VAR.
GenTree* addr = tree->AsIndir()->Addr();
if (addr->OperIsLocalAddr() && comp->isAddrOfSIMDType(addr))
{
BlockRange().Remove(tree);
addr->SetOper(loadForm(addr->OperGet()));
addr->gtType = simdType;
use.ReplaceWith(comp, addr);
}
else if ((addr->OperGet() == GT_ADDR) && (addr->gtGetOp1()->OperGet() == GT_SIMD))
{
// if we have GT_IND(GT_ADDR(GT_SIMD)), remove the GT_IND(GT_ADDR()), leaving just the GT_SIMD.
BlockRange().Remove(tree);
BlockRange().Remove(addr);
use.ReplaceWith(comp, addr->gtGetOp1());
}
else if (!keepBlk)
{
tree->SetOper(GT_IND);
tree->gtType = simdType;
}
#endif // FEATURE_SIMD
}
// Rewrite a SIMD indirection as GT_IND(GT_LEA(obj.op1)), or as a simple
// lclVar if possible.
//
// Arguments:
// use - A use reference for a block node
// keepBlk - True if this should remain a block node if it is not a lclVar
//
// Return Value:
// None.
//
// TODO-1stClassStructs: These should be eliminated earlier, once we can handle
// lclVars in all the places that used to have GT_OBJ.
//
void Rationalizer::RewriteSIMDOperand(LIR::Use& use, bool keepBlk)
{
#ifdef FEATURE_SIMD
// No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
GenTree* tree = use.Def();
if (!tree->OperIsIndir())
{
return;
}
var_types simdType = tree->TypeGet();
if (!varTypeIsSIMD(simdType))
{
return;
}
// If the operand of is a GT_ADDR(GT_LCL_VAR) and LclVar is known to be of simdType,
// replace obj by GT_LCL_VAR.
GenTree* addr = tree->AsIndir()->Addr();
if (addr->OperIsLocalAddr() && comp->isAddrOfSIMDType(addr))
{
BlockRange().Remove(tree);
addr->SetOper(loadForm(addr->OperGet()));
addr->gtType = simdType;
use.ReplaceWith(comp, addr);
}
else if (!keepBlk)
{
tree->SetOper(GT_IND);
tree->gtType = simdType;
}
#endif // FEATURE_SIMD
}
//------------------------------------------------------------------------
// TreeNodeInfoInitBlockStore: Set the NodeInfo for a block store.
//
// Arguments:
// blkNode - The block store node of interest
//
// Return Value:
// None.
//
void Lowering::TreeNodeInfoInitBlockStore(GenTreeBlk* blkNode)
{
GenTree* dstAddr = blkNode->Addr();
unsigned size = blkNode->gtBlkSize;
GenTree* source = blkNode->Data();
LinearScan* l = m_lsra;
Compiler* compiler = comp;
// Sources are dest address and initVal or source.
// We may require an additional source or temp register for the size.
blkNode->gtLsraInfo.srcCount = 2;
blkNode->gtLsraInfo.dstCount = 0;
GenTreePtr srcAddrOrFill = nullptr;
bool isInitBlk = blkNode->OperIsInitBlkOp();
if (!isInitBlk)
{
// CopyObj or CopyBlk
if (source->gtOper == GT_IND)
{
srcAddrOrFill = blkNode->Data()->gtGetOp1();
// We're effectively setting source as contained, but can't call MakeSrcContained, because the
// "inheritance" of the srcCount is to a child not a parent - it would "just work" but could be misleading.
// If srcAddr is already non-contained, we don't need to change it.
if (srcAddrOrFill->gtLsraInfo.getDstCount() == 0)
{
srcAddrOrFill->gtLsraInfo.setDstCount(1);
srcAddrOrFill->gtLsraInfo.setSrcCount(source->gtLsraInfo.srcCount);
}
m_lsra->clearOperandCounts(source);
source->SetContained();
source->AsIndir()->Addr()->ClearContained();
}
else if (!source->IsMultiRegCall() && !source->OperIsSIMD())
{
assert(source->IsLocal());
MakeSrcContained(blkNode, source);
blkNode->gtLsraInfo.srcCount--;
}
}
if (isInitBlk)
{
GenTreePtr initVal = source;
if (initVal->OperIsInitVal())
{
initVal->SetContained();
initVal = initVal->gtGetOp1();
}
srcAddrOrFill = initVal;
if (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindUnroll)
{
// TODO-ARM-CQ: Currently we generate a helper call for every
// initblk we encounter. Later on we should implement loop unrolling
// code sequences to improve CQ.
// For reference see the code in lsraxarch.cpp.
NYI_ARM("initblk loop unrolling is currently not implemented.");
#ifdef _TARGET_ARM64_
// No additional temporaries required
ssize_t fill = initVal->gtIntCon.gtIconVal & 0xFF;
if (fill == 0)
{
MakeSrcContained(blkNode, source);
blkNode->gtLsraInfo.srcCount--;
}
#endif // _TARGET_ARM64_
}
else
{
assert(blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindHelper);
// The helper follows the regular ABI.
dstAddr->gtLsraInfo.setSrcCandidates(l, RBM_ARG_0);
initVal->gtLsraInfo.setSrcCandidates(l, RBM_ARG_1);
if (size != 0)
{
// Reserve a temp register for the block size argument.
blkNode->gtLsraInfo.setInternalCandidates(l, RBM_ARG_2);
blkNode->gtLsraInfo.internalIntCount = 1;
}
else
{
// The block size argument is a third argument to GT_STORE_DYN_BLK
noway_assert(blkNode->gtOper == GT_STORE_DYN_BLK);
blkNode->gtLsraInfo.setSrcCount(3);
GenTree* sizeNode = blkNode->AsDynBlk()->gtDynamicSize;
sizeNode->gtLsraInfo.setSrcCandidates(l, RBM_ARG_2);
}
}
}
else
{
// CopyObj or CopyBlk
// Sources are src and dest and size if not constant.
if (blkNode->OperGet() == GT_STORE_OBJ)
{
// CopyObj
// We don't need to materialize the struct size but we still need
// a temporary register to perform the sequence of loads and stores.
blkNode->gtLsraInfo.internalIntCount = 1;
if (size >= 2 * REGSIZE_BYTES)
{
// We will use ldp/stp to reduce code size and improve performance
// so we need to reserve an extra internal register
blkNode->gtLsraInfo.internalIntCount++;
}
// We can't use the special Write Barrier registers, so exclude them from the mask
regMaskTP internalIntCandidates = RBM_ALLINT & ~(RBM_WRITE_BARRIER_DST_BYREF | RBM_WRITE_BARRIER_SRC_BYREF);
blkNode->gtLsraInfo.setInternalCandidates(l, internalIntCandidates);
// If we have a dest address we want it in RBM_WRITE_BARRIER_DST_BYREF.
dstAddr->gtLsraInfo.setSrcCandidates(l, RBM_WRITE_BARRIER_DST_BYREF);
// If we have a source address we want it in REG_WRITE_BARRIER_SRC_BYREF.
// Otherwise, if it is a local, codegen will put its address in REG_WRITE_BARRIER_SRC_BYREF,
// which is killed by a StoreObj (and thus needn't be reserved).
if (srcAddrOrFill != nullptr)
{
srcAddrOrFill->gtLsraInfo.setSrcCandidates(l, RBM_WRITE_BARRIER_SRC_BYREF);
}
}
else
{
// CopyBlk
short internalIntCount = 0;
regMaskTP internalIntCandidates = RBM_NONE;
if (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindUnroll)
{
// TODO-ARM-CQ: cpblk loop unrolling is currently not implemented.
// In case of a CpBlk with a constant size and less than CPBLK_UNROLL_LIMIT size
// we should unroll the loop to improve CQ.
// For reference see the code in lsraxarch.cpp.
NYI_ARM("cpblk loop unrolling is currently not implemented.");
#ifdef _TARGET_ARM64_
internalIntCount = 1;
internalIntCandidates = RBM_ALLINT;
if (size >= 2 * REGSIZE_BYTES)
{
// We will use ldp/stp to reduce code size and improve performance
// so we need to reserve an extra internal register
internalIntCount++;
}
#endif // _TARGET_ARM64_
}
else
{
assert(blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindHelper);
dstAddr->gtLsraInfo.setSrcCandidates(l, RBM_ARG_0);
// The srcAddr goes in arg1.
if (srcAddrOrFill != nullptr)
{
srcAddrOrFill->gtLsraInfo.setSrcCandidates(l, RBM_ARG_1);
}
if (size != 0)
{
// Reserve a temp register for the block size argument.
internalIntCandidates |= RBM_ARG_2;
internalIntCount++;
}
else
{
// The block size argument is a third argument to GT_STORE_DYN_BLK
noway_assert(blkNode->gtOper == GT_STORE_DYN_BLK);
blkNode->gtLsraInfo.setSrcCount(3);
GenTree* blockSize = blkNode->AsDynBlk()->gtDynamicSize;
blockSize->gtLsraInfo.setSrcCandidates(l, RBM_ARG_2);
}
}
if (internalIntCount != 0)
{
blkNode->gtLsraInfo.internalIntCount = internalIntCount;
blkNode->gtLsraInfo.setInternalCandidates(l, internalIntCandidates);
}
}
}
}
//------------------------------------------------------------------------
// LowerBlockStore: Set block store type
//
// Arguments:
// blkNode - The block store node of interest
//
// Return Value:
// None.
//
void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
{
GenTree* dstAddr = blkNode->Addr();
unsigned size = blkNode->gtBlkSize;
GenTree* source = blkNode->Data();
Compiler* compiler = comp;
// Sources are dest address and initVal or source.
GenTree* srcAddrOrFill = nullptr;
bool isInitBlk = blkNode->OperIsInitBlkOp();
if (!isInitBlk)
{
// CopyObj or CopyBlk
if ((blkNode->OperGet() == GT_STORE_OBJ) && ((blkNode->AsObj()->gtGcPtrCount == 0) || blkNode->gtBlkOpGcUnsafe))
{
blkNode->SetOper(GT_STORE_BLK);
}
if (source->gtOper == GT_IND)
{
srcAddrOrFill = blkNode->Data()->gtGetOp1();
}
}
if (isInitBlk)
{
GenTree* initVal = source;
if (initVal->OperIsInitVal())
{
initVal->SetContained();
initVal = initVal->gtGetOp1();
}
srcAddrOrFill = initVal;
#ifdef _TARGET_ARM64_
if ((size != 0) && (size <= INITBLK_UNROLL_LIMIT) && initVal->IsCnsIntOrI())
{
// TODO-ARM-CQ: Currently we generate a helper call for every
// initblk we encounter. Later on we should implement loop unrolling
// code sequences to improve CQ.
// For reference see the code in LowerXArch.cpp.
NYI_ARM("initblk loop unrolling is currently not implemented.");
// The fill value of an initblk is interpreted to hold a
// value of (unsigned int8) however a constant of any size
// may practically reside on the evaluation stack. So extract
// the lower byte out of the initVal constant and replicate
// it to a larger constant whose size is sufficient to support
// the largest width store of the desired inline expansion.
ssize_t fill = initVal->gtIntCon.gtIconVal & 0xFF;
if (fill == 0)
{
MakeSrcContained(blkNode, source);
}
else if (size < REGSIZE_BYTES)
{
initVal->gtIntCon.gtIconVal = 0x01010101 * fill;
}
else
{
initVal->gtIntCon.gtIconVal = 0x0101010101010101LL * fill;
initVal->gtType = TYP_LONG;
}
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll;
}
else
#endif // _TARGET_ARM64_
{
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper;
}
}
else
{
// CopyObj or CopyBlk
// Sources are src and dest and size if not constant.
if (blkNode->OperGet() == GT_STORE_OBJ)
{
// CopyObj
GenTreeObj* objNode = blkNode->AsObj();
unsigned slots = objNode->gtSlots;
#ifdef DEBUG
// CpObj must always have at least one GC-Pointer as a member.
assert(objNode->gtGcPtrCount > 0);
assert(dstAddr->gtType == TYP_BYREF || dstAddr->gtType == TYP_I_IMPL);
CORINFO_CLASS_HANDLE clsHnd = objNode->gtClass;
size_t classSize = compiler->info.compCompHnd->getClassSize(clsHnd);
size_t blkSize = roundUp(classSize, TARGET_POINTER_SIZE);
// Currently, the EE always round up a class data structure so
// we are not handling the case where we have a non multiple of pointer sized
// struct. This behavior may change in the future so in order to keeps things correct
// let's assert it just to be safe. Going forward we should simply
// handle this case.
assert(classSize == blkSize);
assert((blkSize / TARGET_POINTER_SIZE) == slots);
assert(objNode->HasGCPtr());
#endif
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll;
}
else // CopyBlk
{
// In case of a CpBlk with a constant size and less than CPBLK_UNROLL_LIMIT size
// we should unroll the loop to improve CQ.
// For reference see the code in lowerxarch.cpp.
if ((size != 0) && (size <= CPBLK_UNROLL_LIMIT))
{
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll;
}
else
{
// In case we have a constant integer this means we went beyond
// CPBLK_UNROLL_LIMIT bytes of size, still we should never have the case of
// any GC-Pointers in the src struct.
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper;
}
}
// CopyObj or CopyBlk
if (source->gtOper == GT_IND)
{
MakeSrcContained(blkNode, source);
GenTree* addr = source->AsIndir()->Addr();
if (!addr->OperIsLocalAddr())
{
addr->ClearContained();
}
}
else if (!source->IsMultiRegCall() && !source->OperIsSIMD())
{
assert(source->IsLocal());
MakeSrcContained(blkNode, source);
}
}
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding list nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_FIELD_LIST head nodes, and GT_LIST nodes used by phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperIsAnyList() && !(prev->OperIsFieldListHead());
prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node that is not an aggregate.
if (node->OperIsAnyList())
{
GenTreeArgList* list = node->AsArgList();
if (!list->OperIsFieldListHead())
{
BlockRange().Remove(list);
}
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_IND:
// Clear the `GTF_IND_ASG_LHS` flag, which overlaps with `GTF_IND_REQ_ADDR_IN_REG`.
node->gtFlags &= ~GTF_IND_ASG_LHS;
if (varTypeIsSIMD(node))
{
RewriteSIMDOperand(use, false);
}
else
{
// Due to promotion of structs containing fields of type struct with a
// single scalar type field, we could potentially see IR nodes of the
// form GT_IND(GT_ADD(lclvarAddr, 0)) where 0 is an offset representing
// a field-seq. These get folded here.
//
// TODO: This code can be removed once JIT implements recursive struct
// promotion instead of lying about the type of struct field as the type
// of its single scalar field.
GenTree* addr = node->AsIndir()->Addr();
if (addr->OperGet() == GT_ADD && addr->gtGetOp1()->OperGet() == GT_LCL_VAR_ADDR &&
addr->gtGetOp2()->IsIntegralConst(0))
{
GenTreeLclVarCommon* lclVarNode = addr->gtGetOp1()->AsLclVarCommon();
unsigned lclNum = lclVarNode->GetLclNum();
LclVarDsc* varDsc = comp->lvaTable + lclNum;
if (node->TypeGet() == varDsc->TypeGet())
{
JITDUMP("Rewriting GT_IND(GT_ADD(LCL_VAR_ADDR,0)) to LCL_VAR\n");
lclVarNode->SetOper(GT_LCL_VAR);
lclVarNode->gtType = node->TypeGet();
use.ReplaceWith(comp, lclVarNode);
BlockRange().Remove(addr);
BlockRange().Remove(addr->gtGetOp2());
BlockRange().Remove(node);
}
}
}
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#if defined(_TARGET_XARCH_) || defined(_TARGET_ARM_)
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_BLK:
case GT_OBJ:
{
// TODO-1stClassStructs: These should have been transformed to GT_INDs, but in order
// to preserve existing behavior, we will keep this as a block node if this is the
// lhs of a block assignment, and either:
// - It is a "generic" TYP_STRUCT assignment, OR
// - It is an initblk, OR
// - Neither the lhs or rhs are known to be of SIMD type.
GenTree* parent = use.User();
bool keepBlk = false;
if ((parent->OperGet() == GT_ASG) && (node == parent->gtGetOp1()))
{
if ((node->TypeGet() == TYP_STRUCT) || parent->OperIsInitBlkOp())
{
keepBlk = true;
}
else if (!comp->isAddrOfSIMDType(node->AsBlk()->Addr()))
{
GenTree* dataSrc = parent->gtGetOp2();
if (!dataSrc->IsLocal() && (dataSrc->OperGet() != GT_SIMD))
{
noway_assert(dataSrc->OperIsIndir());
keepBlk = !comp->isAddrOfSIMDType(dataSrc->AsIndir()->Addr());
}
}
}
RewriteSIMDOperand(use, keepBlk);
}
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2IfPresent();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
// JCC nodes should not be present in HIR.
assert(node->OperGet() != GT_JCC);
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (node->OperIsLocalRead())
{
if (use.IsDummyUse())
{
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
else
{
// Local reads are side-effect-free; clear any flags leftover from frontend transformations.
node->gtFlags &= ~GTF_ALL_EFFECT;
}
}
assert(isLateArg == ((use.Def()->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding GT_LIST nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_LIST nodes that are used by block ops and phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperGet() == GT_LIST; prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node.
if ((*useEdge)->OperGet() == GT_LIST)
{
BlockRange().Remove(*useEdge);
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#ifdef _TARGET_XARCH_
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_INITBLK:
RewriteInitBlk(use);
break;
case GT_COPYBLK:
RewriteCopyBlk(use);
break;
case GT_OBJ:
RewriteObj(use);
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_STOREIND:
case GT_IND:
if (node->gtType == TYP_STRUCT)
{
GenTree* addr = node->AsIndir()->Addr();
assert(addr->TypeGet() == TYP_BYREF);
if (addr->OperIsLocal())
{
LclVarDsc* varDsc = &(comp->lvaTable[addr->AsLclVarCommon()->gtLclNum]);
assert(varDsc->lvSIMDType);
unsigned simdSize = (unsigned int)roundUp(varDsc->lvExactSize, TARGET_POINTER_SIZE);
node->gtType = comp->getSIMDTypeForSize(simdSize);
}
#if DEBUG
else
{
// If the address is not a local var, assert that the user of this IND is an ADDR node.
assert((use.User()->OperGet() == GT_ADDR) || use.User()->OperIsLocalAddr());
}
#endif
}
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
else if (simdNode->gtType == TYP_STRUCT || varTypeIsSIMD(simdNode))
{
node->gtType = simdType;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (use.IsDummyUse() && node->OperIsLocalRead())
{
assert((node->gtFlags & GTF_ALL_EFFECT) == 0);
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
assert(isLateArg == ((node->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}