// Transform CopyBlk involving SIMD vectors into stlclvar or stind of a SIMD type.
// Transformation is done if either src or dst are known to be SIMD vectors.
//
// Arguments:
// ppTree - A pointer-to-a-pointer for the GT_COPYBLK
// fgWalkData - A pointer to tree walk data providing the context
//
// Return Value:
// None.
//
// If either the source or the dst are known to be SIMD (a lclVar or SIMD intrinsic),
// get the simdType (TYP_DOUBLE or a SIMD type for SSE2) from the size of the SIMD node.
//
// For the source:
// - If it is a SIMD intrinsic or a lvSIMDType lclVar, change the node type to simdType.
// - Otherwise, add a GT_IND of simdType.
// For the dst:
// - If it is a lclVar of a SIMD type, chanage the node type to simdType.
// - Otherwise, change it to a GT_STORE_IND of simdType
//
// TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer
// be required.
//
void Rationalizer::RewriteCopyBlk(LIR::Use& use)
{
#ifdef FEATURE_SIMD
// No need to transofrm non-SIMD nodes, if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
// See if this is a SIMD copyBlk
GenTreeCpBlk* cpBlk = use.Def()->AsCpBlk();
GenTreePtr dstAddr = cpBlk->Dest();
GenTree* srcAddr = cpBlk->Source();
const bool srcIsSIMDAddr = comp->isAddrOfSIMDType(srcAddr);
const bool dstIsSIMDAddr = comp->isAddrOfSIMDType(dstAddr);
// Do not transform if neither src or dst is known to be a SIMD type.
// If src tree type is something we cannot reason but if dst is known to be of a SIMD type
// we will treat src tree as a SIMD type and vice versa.
if (!srcIsSIMDAddr && !dstIsSIMDAddr)
{
return;
}
// At this point it is known to be a copyblk of SIMD vectors and we can
// start transforming the original tree. Prior to this point do not perform
// any modifications to the original tree.
JITDUMP("\nRewriting SIMD CopyBlk\n");
DISPTREERANGE(BlockRange(), cpBlk);
// There are currently only three sizes supported: 8 bytes, 12 bytes, 16 bytes or the vector register length.
GenTreeIntConCommon* sizeNode = cpBlk->Size()->AsIntConCommon();
var_types simdType = comp->getSIMDTypeForSize((unsigned int)sizeNode->IconValue());
// Remove 'size' from execution order
BlockRange().Remove(sizeNode);
// Is destination a lclVar which is not an arg?
// If yes then we can turn it to a stlcl.var, otherwise turn into stind.
GenTree* simdDst = nullptr;
genTreeOps oper = GT_NONE;
if (dstIsSIMDAddr && dstAddr->OperIsLocalAddr())
{
simdDst = dstAddr;
simdDst->gtType = simdType;
oper = GT_STORE_LCL_VAR;
// For structs that are padded (e.g. Vector3f, Vector3i), the morpher will have marked them
// as GTF_VAR_USEASG. Unmark them.
simdDst->gtFlags &= ~(GTF_VAR_USEASG);
}
else
{
// Address of a non-local var
simdDst = dstAddr;
oper = GT_STOREIND;
}
GenTree* simdSrc = nullptr;
if ((srcAddr->OperGet() == GT_ADDR) && varTypeIsSIMD(srcAddr->gtGetOp1()))
{
// Get rid of parent node of GT_ADDR(..) if its child happens to be of a SIMD type.
BlockRange().Remove(srcAddr);
simdSrc = srcAddr->gtGetOp1();
}
else if (srcIsSIMDAddr && srcAddr->OperIsLocalAddr())
{
// If the source has been rewritten into a local addr node, rewrite it back into a
// local var node.
simdSrc = srcAddr;
simdSrc->SetOper(loadForm(srcAddr->OperGet()));
}
else
{
// Since destination is known to be a SIMD type, src must be a SIMD type too
// though we cannot figure it out easily enough. Transform src into
// GT_IND(src) of simdType.
GenTree* indir = comp->gtNewOperNode(GT_IND, simdType, srcAddr);
BlockRange().InsertAfter(srcAddr, indir);
cpBlk->gtGetOp1()->gtOp.gtOp2 = indir;
simdSrc = indir;
}
simdSrc->gtType = simdType;
// Change cpblk to either a st.lclvar or st.ind.
// At this point we are manipulating cpblk node with the knowledge of
// its internals (i.e. op1 is the size node, and the src & dst are in a GT_LIST on op2).
// This logic might need to be changed if we ever restructure cpblk node.
assert(simdDst != nullptr);
assert(simdSrc != nullptr);
GenTree* newNode = nullptr;
if (oper == GT_STORE_LCL_VAR)
{
newNode = simdDst;
newNode->SetOper(oper);
GenTreeLclVar* store = newNode->AsLclVar();
store->gtOp1 = simdSrc;
store->gtType = simdType;
store->gtFlags |= ((simdSrc->gtFlags & GTF_ALL_EFFECT) | GTF_ASG);
BlockRange().Remove(simdDst);
BlockRange().InsertAfter(simdSrc, store);
}
else
{
assert(oper == GT_STOREIND);
newNode = cpBlk->gtGetOp1();
newNode->SetOper(oper);
GenTreeStoreInd* storeInd = newNode->AsStoreInd();
storeInd->gtType = simdType;
storeInd->gtFlags |= ((simdSrc->gtFlags & GTF_ALL_EFFECT) | GTF_ASG);
storeInd->gtOp1 = simdDst;
storeInd->gtOp2 = simdSrc;
BlockRange().InsertBefore(cpBlk, storeInd);
}
use.ReplaceWith(comp, newNode);
BlockRange().Remove(cpBlk);
JITDUMP("After rewriting SIMD CopyBlk:\n");
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
#endif // FEATURE_SIMD
}
// Rewrite InitBlk involving SIMD vector into stlcl.var of a SIMD type.
//
// Arguments:
// ppTree - A pointer-to-a-pointer for the GT_INITBLK
// fgWalkData - A pointer to tree walk data providing the context
//
// Return Value:
// None.
//
// TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer
// be required.
//
void Rationalizer::RewriteInitBlk(LIR::Use& use)
{
#ifdef FEATURE_SIMD
// No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
// See if this is a SIMD initBlk that needs to be changed to a simple st.lclVar.
GenTreeInitBlk* initBlk = use.Def()->AsInitBlk();
// Is the dstAddr is addr of a SIMD type lclVar?
GenTree* dstAddr = initBlk->Dest();
if (!comp->isAddrOfSIMDType(dstAddr) || !dstAddr->OperIsLocalAddr())
{
return;
}
unsigned lclNum = dstAddr->AsLclVarCommon()->gtLclNum;
if (!comp->lvaTable[lclNum].lvSIMDType)
{
return;
}
var_types baseType = comp->lvaTable[lclNum].lvBaseType;
CORINFO_CLASS_HANDLE typeHnd = comp->lvaTable[lclNum].lvVerTypeInfo.GetClassHandle();
unsigned simdLocalSize = comp->getSIMDTypeSizeInBytes(typeHnd);
JITDUMP("Rewriting SIMD InitBlk\n");
DISPTREERANGE(BlockRange(), initBlk);
assert((dstAddr->gtFlags & GTF_VAR_USEASG) == 0);
// There are currently only three sizes supported: 8 bytes, 16 bytes or the vector register length.
GenTreeIntConCommon* sizeNode = initBlk->Size()->AsIntConCommon();
unsigned int size = (unsigned int)roundUp(sizeNode->IconValue(), TARGET_POINTER_SIZE);
var_types simdType = comp->getSIMDTypeForSize(size);
assert(roundUp(simdLocalSize, TARGET_POINTER_SIZE) == size);
GenTree* initVal = initBlk->InitVal();
GenTreeSIMD* simdNode = new (comp, GT_SIMD)
GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, (unsigned)sizeNode->IconValue());
dstAddr->SetOper(GT_STORE_LCL_VAR);
GenTreeLclVar* store = dstAddr->AsLclVar();
store->gtType = simdType;
store->gtOp.gtOp1 = simdNode;
store->gtFlags |= ((simdNode->gtFlags & GTF_ALL_EFFECT) | GTF_ASG);
BlockRange().Remove(store);
// Insert the new nodes into the block
BlockRange().InsertAfter(initVal, simdNode, store);
use.ReplaceWith(comp, store);
// Remove the old size and GT_INITBLK nodes.
BlockRange().Remove(sizeNode);
BlockRange().Remove(initBlk);
JITDUMP("After rewriting SIMD InitBlk:\n");
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
#endif // FEATURE_SIMD
}