// Transform CopyBlk involving SIMD vectors into stlclvar or stind of a SIMD type. // Transformation is done if either src or dst are known to be SIMD vectors. // // Arguments: // ppTree - A pointer-to-a-pointer for the GT_COPYBLK // fgWalkData - A pointer to tree walk data providing the context // // Return Value: // None. // // If either the source or the dst are known to be SIMD (a lclVar or SIMD intrinsic), // get the simdType (TYP_DOUBLE or a SIMD type for SSE2) from the size of the SIMD node. // // For the source: // - If it is a SIMD intrinsic or a lvSIMDType lclVar, change the node type to simdType. // - Otherwise, add a GT_IND of simdType. // For the dst: // - If it is a lclVar of a SIMD type, chanage the node type to simdType. // - Otherwise, change it to a GT_STORE_IND of simdType // // TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer // be required. // void Rationalizer::RewriteCopyBlk(LIR::Use& use) { #ifdef FEATURE_SIMD // No need to transofrm non-SIMD nodes, if featureSIMD is not enabled. if (!comp->featureSIMD) { return; } // See if this is a SIMD copyBlk GenTreeCpBlk* cpBlk = use.Def()->AsCpBlk(); GenTreePtr dstAddr = cpBlk->Dest(); GenTree* srcAddr = cpBlk->Source(); const bool srcIsSIMDAddr = comp->isAddrOfSIMDType(srcAddr); const bool dstIsSIMDAddr = comp->isAddrOfSIMDType(dstAddr); // Do not transform if neither src or dst is known to be a SIMD type. // If src tree type is something we cannot reason but if dst is known to be of a SIMD type // we will treat src tree as a SIMD type and vice versa. if (!srcIsSIMDAddr && !dstIsSIMDAddr) { return; } // At this point it is known to be a copyblk of SIMD vectors and we can // start transforming the original tree. Prior to this point do not perform // any modifications to the original tree. JITDUMP("\nRewriting SIMD CopyBlk\n"); DISPTREERANGE(BlockRange(), cpBlk); // There are currently only three sizes supported: 8 bytes, 12 bytes, 16 bytes or the vector register length. GenTreeIntConCommon* sizeNode = cpBlk->Size()->AsIntConCommon(); var_types simdType = comp->getSIMDTypeForSize((unsigned int)sizeNode->IconValue()); // Remove 'size' from execution order BlockRange().Remove(sizeNode); // Is destination a lclVar which is not an arg? // If yes then we can turn it to a stlcl.var, otherwise turn into stind. GenTree* simdDst = nullptr; genTreeOps oper = GT_NONE; if (dstIsSIMDAddr && dstAddr->OperIsLocalAddr()) { simdDst = dstAddr; simdDst->gtType = simdType; oper = GT_STORE_LCL_VAR; // For structs that are padded (e.g. Vector3f, Vector3i), the morpher will have marked them // as GTF_VAR_USEASG. Unmark them. simdDst->gtFlags &= ~(GTF_VAR_USEASG); } else { // Address of a non-local var simdDst = dstAddr; oper = GT_STOREIND; } GenTree* simdSrc = nullptr; if ((srcAddr->OperGet() == GT_ADDR) && varTypeIsSIMD(srcAddr->gtGetOp1())) { // Get rid of parent node of GT_ADDR(..) if its child happens to be of a SIMD type. BlockRange().Remove(srcAddr); simdSrc = srcAddr->gtGetOp1(); } else if (srcIsSIMDAddr && srcAddr->OperIsLocalAddr()) { // If the source has been rewritten into a local addr node, rewrite it back into a // local var node. simdSrc = srcAddr; simdSrc->SetOper(loadForm(srcAddr->OperGet())); } else { // Since destination is known to be a SIMD type, src must be a SIMD type too // though we cannot figure it out easily enough. Transform src into // GT_IND(src) of simdType. GenTree* indir = comp->gtNewOperNode(GT_IND, simdType, srcAddr); BlockRange().InsertAfter(srcAddr, indir); cpBlk->gtGetOp1()->gtOp.gtOp2 = indir; simdSrc = indir; } simdSrc->gtType = simdType; // Change cpblk to either a st.lclvar or st.ind. // At this point we are manipulating cpblk node with the knowledge of // its internals (i.e. op1 is the size node, and the src & dst are in a GT_LIST on op2). // This logic might need to be changed if we ever restructure cpblk node. assert(simdDst != nullptr); assert(simdSrc != nullptr); GenTree* newNode = nullptr; if (oper == GT_STORE_LCL_VAR) { newNode = simdDst; newNode->SetOper(oper); GenTreeLclVar* store = newNode->AsLclVar(); store->gtOp1 = simdSrc; store->gtType = simdType; store->gtFlags |= ((simdSrc->gtFlags & GTF_ALL_EFFECT) | GTF_ASG); BlockRange().Remove(simdDst); BlockRange().InsertAfter(simdSrc, store); } else { assert(oper == GT_STOREIND); newNode = cpBlk->gtGetOp1(); newNode->SetOper(oper); GenTreeStoreInd* storeInd = newNode->AsStoreInd(); storeInd->gtType = simdType; storeInd->gtFlags |= ((simdSrc->gtFlags & GTF_ALL_EFFECT) | GTF_ASG); storeInd->gtOp1 = simdDst; storeInd->gtOp2 = simdSrc; BlockRange().InsertBefore(cpBlk, storeInd); } use.ReplaceWith(comp, newNode); BlockRange().Remove(cpBlk); JITDUMP("After rewriting SIMD CopyBlk:\n"); DISPTREERANGE(BlockRange(), use.Def()); JITDUMP("\n"); #endif // FEATURE_SIMD }
// Rewrite InitBlk involving SIMD vector into stlcl.var of a SIMD type. // // Arguments: // ppTree - A pointer-to-a-pointer for the GT_INITBLK // fgWalkData - A pointer to tree walk data providing the context // // Return Value: // None. // // TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer // be required. // void Rationalizer::RewriteInitBlk(LIR::Use& use) { #ifdef FEATURE_SIMD // No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled. if (!comp->featureSIMD) { return; } // See if this is a SIMD initBlk that needs to be changed to a simple st.lclVar. GenTreeInitBlk* initBlk = use.Def()->AsInitBlk(); // Is the dstAddr is addr of a SIMD type lclVar? GenTree* dstAddr = initBlk->Dest(); if (!comp->isAddrOfSIMDType(dstAddr) || !dstAddr->OperIsLocalAddr()) { return; } unsigned lclNum = dstAddr->AsLclVarCommon()->gtLclNum; if (!comp->lvaTable[lclNum].lvSIMDType) { return; } var_types baseType = comp->lvaTable[lclNum].lvBaseType; CORINFO_CLASS_HANDLE typeHnd = comp->lvaTable[lclNum].lvVerTypeInfo.GetClassHandle(); unsigned simdLocalSize = comp->getSIMDTypeSizeInBytes(typeHnd); JITDUMP("Rewriting SIMD InitBlk\n"); DISPTREERANGE(BlockRange(), initBlk); assert((dstAddr->gtFlags & GTF_VAR_USEASG) == 0); // There are currently only three sizes supported: 8 bytes, 16 bytes or the vector register length. GenTreeIntConCommon* sizeNode = initBlk->Size()->AsIntConCommon(); unsigned int size = (unsigned int)roundUp(sizeNode->IconValue(), TARGET_POINTER_SIZE); var_types simdType = comp->getSIMDTypeForSize(size); assert(roundUp(simdLocalSize, TARGET_POINTER_SIZE) == size); GenTree* initVal = initBlk->InitVal(); GenTreeSIMD* simdNode = new (comp, GT_SIMD) GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, (unsigned)sizeNode->IconValue()); dstAddr->SetOper(GT_STORE_LCL_VAR); GenTreeLclVar* store = dstAddr->AsLclVar(); store->gtType = simdType; store->gtOp.gtOp1 = simdNode; store->gtFlags |= ((simdNode->gtFlags & GTF_ALL_EFFECT) | GTF_ASG); BlockRange().Remove(store); // Insert the new nodes into the block BlockRange().InsertAfter(initVal, simdNode, store); use.ReplaceWith(comp, store); // Remove the old size and GT_INITBLK nodes. BlockRange().Remove(sizeNode); BlockRange().Remove(initBlk); JITDUMP("After rewriting SIMD InitBlk:\n"); DISPTREERANGE(BlockRange(), use.Def()); JITDUMP("\n"); #endif // FEATURE_SIMD }