// sanity checks that apply to all kinds of IR void Rationalizer::SanityCheck() { // TODO: assert(!IsLIR()); BasicBlock* block; foreach_block(comp, block) { for (GenTree* statement = block->bbTreeList; statement != nullptr; statement = statement->gtNext) { ValidateStatement(statement, block); for (GenTree* tree = statement->gtStmt.gtStmtList; tree; tree = tree->gtNext) { // QMARK nodes should have been removed before this phase. assert(tree->OperGet() != GT_QMARK); if (tree->OperGet() == GT_ASG) { if (tree->gtGetOp1()->OperGet() == GT_LCL_VAR) { assert(tree->gtGetOp1()->gtFlags & GTF_VAR_DEF); } else if (tree->gtGetOp2()->OperGet() == GT_LCL_VAR) { assert(!(tree->gtGetOp2()->gtFlags & GTF_VAR_DEF)); } } } } } }
//------------------------------------------------------------------------ // DecomposeMul: Decompose GT_MUL. The only GT_MULs that make it to decompose are // those with the GTF_MUL_64RSLT flag set. These muls result in a mul instruction that // returns its result in two registers like GT_CALLs do. Additionally, these muls are // guaranteed to be in the form long = (long)int * (long)int. Therefore, to decompose // these nodes, we convert them into GT_MUL_LONGs, undo the cast from int to long by // stripping out the lo ops, and force them into the form var = mul, as we do for // GT_CALLs. In codegen, we then produce a mul instruction that produces the result // in edx:eax, and store those registers on the stack in genStoreLongLclVar. // // All other GT_MULs have been converted to helper calls in morph.cpp // // Arguments: // use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: // The next node to process. // GenTree* DecomposeLongs::DecomposeMul(LIR::Use& use) { assert(use.IsInitialized()); GenTree* tree = use.Def(); genTreeOps oper = tree->OperGet(); assert(oper == GT_MUL); assert((tree->gtFlags & GTF_MUL_64RSLT) != 0); GenTree* op1 = tree->gtGetOp1(); GenTree* op2 = tree->gtGetOp2(); GenTree* loOp1 = op1->gtGetOp1(); GenTree* hiOp1 = op1->gtGetOp2(); GenTree* loOp2 = op2->gtGetOp1(); GenTree* hiOp2 = op2->gtGetOp2(); Range().Remove(hiOp1); Range().Remove(hiOp2); Range().Remove(op1); Range().Remove(op2); // Get rid of the hi ops. We don't need them. tree->gtOp.gtOp1 = loOp1; tree->gtOp.gtOp2 = loOp2; tree->gtOper = GT_MUL_LONG; return StoreNodeToVar(use); }
//------------------------------------------------------------------------ // DecomposeNeg: Decompose GT_NEG. // // Arguments: // use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: // The next node to process. // GenTree* DecomposeLongs::DecomposeNeg(LIR::Use& use) { assert(use.IsInitialized()); assert(use.Def()->OperGet() == GT_NEG); GenTree* tree = use.Def(); GenTree* gtLong = tree->gtGetOp1(); noway_assert(gtLong->OperGet() == GT_LONG); LIR::Use op1(Range(), >Long->gtOp.gtOp1, gtLong); op1.ReplaceWithLclVar(m_compiler, m_blockWeight); LIR::Use op2(Range(), >Long->gtOp.gtOp2, gtLong); op2.ReplaceWithLclVar(m_compiler, m_blockWeight); // Neither GT_NEG nor the introduced temporaries have side effects. tree->gtFlags &= ~GTF_ALL_EFFECT; GenTree* loOp1 = gtLong->gtGetOp1(); GenTree* hiOp1 = gtLong->gtGetOp2(); Range().Remove(gtLong); GenTree* loResult = tree; loResult->gtType = TYP_INT; loResult->gtOp.gtOp1 = loOp1; GenTree* zero = m_compiler->gtNewZeroConNode(TYP_INT); GenTree* hiAdjust = m_compiler->gtNewOperNode(GT_ADD_HI, TYP_INT, hiOp1, zero); GenTree* hiResult = m_compiler->gtNewOperNode(GT_NEG, TYP_INT, hiAdjust); hiResult->gtFlags = tree->gtFlags; Range().InsertAfter(loResult, zero, hiAdjust, hiResult); return FinalizeDecomposition(use, loResult, hiResult); }
//------------------------------------------------------------------------ // DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND. // // Arguments: // use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: // The next node to process. // GenTree* DecomposeLongs::DecomposeArith(LIR::Use& use) { assert(use.IsInitialized()); GenTree* tree = use.Def(); genTreeOps oper = tree->OperGet(); assert((oper == GT_ADD) || (oper == GT_SUB) || (oper == GT_OR) || (oper == GT_XOR) || (oper == GT_AND)); GenTree* op1 = tree->gtGetOp1(); GenTree* op2 = tree->gtGetOp2(); // Both operands must have already been decomposed into GT_LONG operators. noway_assert((op1->OperGet() == GT_LONG) && (op2->OperGet() == GT_LONG)); // Capture the lo and hi halves of op1 and op2. GenTree* loOp1 = op1->gtGetOp1(); GenTree* hiOp1 = op1->gtGetOp2(); GenTree* loOp2 = op2->gtGetOp1(); GenTree* hiOp2 = op2->gtGetOp2(); // Now, remove op1 and op2 from the node list. Range().Remove(op1); Range().Remove(op2); // We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands // will be the lo halves of op1 from above. GenTree* loResult = tree; loResult->SetOper(GetLoOper(oper)); loResult->gtType = TYP_INT; loResult->gtOp.gtOp1 = loOp1; loResult->gtOp.gtOp2 = loOp2; GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2); Range().InsertAfter(loResult, hiResult); if ((oper == GT_ADD) || (oper == GT_SUB)) { if (loResult->gtOverflow()) { hiResult->gtFlags |= GTF_OVERFLOW; loResult->gtFlags &= ~GTF_OVERFLOW; } if (loResult->gtFlags & GTF_UNSIGNED) { hiResult->gtFlags |= GTF_UNSIGNED; } } return FinalizeDecomposition(use, loResult, hiResult); }
//------------------------------------------------------------------------ // DecomposeNot: Decompose GT_NOT. // // Arguments: // use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: // The next node to process. // GenTree* DecomposeLongs::DecomposeNot(LIR::Use& use) { assert(use.IsInitialized()); assert(use.Def()->OperGet() == GT_NOT); GenTree* tree = use.Def(); GenTree* gtLong = tree->gtGetOp1(); noway_assert(gtLong->OperGet() == GT_LONG); GenTree* loOp1 = gtLong->gtGetOp1(); GenTree* hiOp1 = gtLong->gtGetOp2(); Range().Remove(gtLong); GenTree* loResult = tree; loResult->gtType = TYP_INT; loResult->gtOp.gtOp1 = loOp1; GenTree* hiResult = new (m_compiler, GT_NOT) GenTreeOp(GT_NOT, TYP_INT, hiOp1, nullptr); Range().InsertAfter(loResult, hiResult); return FinalizeDecomposition(use, loResult, hiResult); }
//------------------------------------------------------------------------ // DecomposeStoreLclVar: Decompose GT_STORE_LCL_VAR. // // Arguments: // ppTree - the tree to decompose // data - tree walk context // // Return Value: // None. // void DecomposeLongs::DecomposeStoreLclVar(GenTree** ppTree, Compiler::fgWalkData* data) { assert(ppTree != nullptr); assert(*ppTree != nullptr); assert(data != nullptr); assert((*ppTree)->OperGet() == GT_STORE_LCL_VAR); assert(m_compiler->compCurStmt != nullptr); GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); GenTree* tree = *ppTree; GenTree* nextTree = tree->gtNext; GenTree* rhs = tree->gtGetOp1(); if ((rhs->OperGet() == GT_PHI) || (rhs->OperGet() == GT_CALL)) { // GT_CALLs are not decomposed, so will not be converted to GT_LONG // GT_STORE_LCL_VAR = GT_CALL are handled in genMultiRegCallStoreToLocal return; } noway_assert(rhs->OperGet() == GT_LONG); unsigned varNum = tree->AsLclVarCommon()->gtLclNum; LclVarDsc* varDsc = m_compiler->lvaTable + varNum; m_compiler->lvaDecRefCnts(tree); GenTree* loRhs = rhs->gtGetOp1(); GenTree* hiRhs = rhs->gtGetOp2(); GenTree* hiStore = m_compiler->gtNewLclLNode(varNum, TYP_INT); if (varDsc->lvPromoted) { assert(varDsc->lvFieldCnt == 2); unsigned loVarNum = varDsc->lvFieldLclStart; unsigned hiVarNum = loVarNum + 1; tree->AsLclVarCommon()->SetLclNum(loVarNum); hiStore->SetOper(GT_STORE_LCL_VAR); hiStore->AsLclVarCommon()->SetLclNum(hiVarNum); } else { noway_assert(varDsc->lvLRACandidate == false); tree->SetOper(GT_STORE_LCL_FLD); tree->AsLclFld()->gtLclOffs = 0; tree->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField(); hiStore->SetOper(GT_STORE_LCL_FLD); hiStore->AsLclFld()->gtLclOffs = 4; hiStore->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField(); } tree->gtOp.gtOp1 = loRhs; tree->gtType = TYP_INT; loRhs->gtNext = tree; tree->gtPrev = loRhs; hiStore->gtOp.gtOp1 = hiRhs; hiStore->CopyCosts(tree); hiStore->gtFlags |= GTF_VAR_DEF; m_compiler->lvaIncRefCnts(tree); m_compiler->lvaIncRefCnts(hiStore); tree->gtNext = hiRhs; hiRhs->gtPrev = tree; hiRhs->gtNext = hiStore; hiStore->gtPrev = hiRhs; hiStore->gtNext = nextTree; if (nextTree != nullptr) { nextTree->gtPrev = hiStore; } nextTree = hiRhs; bool isEmbeddedStmt = !curStmt->gtStmtIsTopLevel(); if (!isEmbeddedStmt) { tree->gtNext = nullptr; hiRhs->gtPrev = nullptr; } InsertNodeAsStmt(hiStore); }
//------------------------------------------------------------------------ // DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND. // // Arguments: // ppTree - the tree to decompose // data - tree walk context // // Return Value: // None. // void DecomposeLongs::DecomposeArith(GenTree** ppTree, Compiler::fgWalkData* data) { assert(ppTree != nullptr); assert(*ppTree != nullptr); assert(data != nullptr); assert(m_compiler->compCurStmt != nullptr); GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); GenTree* tree = *ppTree; genTreeOps oper = tree->OperGet(); assert((oper == GT_ADD) || (oper == GT_SUB) || (oper == GT_OR) || (oper == GT_XOR) || (oper == GT_AND)); NYI_IF((tree->gtFlags & GTF_REVERSE_OPS) != 0, "Binary operator with GTF_REVERSE_OPS"); GenTree* op1 = tree->gtGetOp1(); GenTree* op2 = tree->gtGetOp2(); // Both operands must have already been decomposed into GT_LONG operators. noway_assert((op1->OperGet() == GT_LONG) && (op2->OperGet() == GT_LONG)); // Capture the lo and hi halves of op1 and op2. GenTree* loOp1 = op1->gtGetOp1(); GenTree* hiOp1 = op1->gtGetOp2(); GenTree* loOp2 = op2->gtGetOp1(); GenTree* hiOp2 = op2->gtGetOp2(); // We don't have support to decompose a TYP_LONG node that already has a child that has // been decomposed into parts, where the high part depends on the value generated by the // low part (via the flags register). For example, if we have: // +(gt_long(+(lo3, lo4), +Hi(hi3, hi4)), gt_long(lo2, hi2)) // We would decompose it here to: // gt_long(+(+(lo3, lo4), lo2), +Hi(+Hi(hi3, hi4), hi2)) // But this would generate incorrect code, because the "+Hi(hi3, hi4)" code generation // needs to immediately follow the "+(lo3, lo4)" part. Also, if this node is one that // requires a unique high operator, and the child nodes are not simple locals (e.g., // they are decomposed nodes), then we also can't decompose the node, as we aren't // guaranteed the high and low parts will be executed immediately after each other. NYI_IF(hiOp1->OperIsHigh() || hiOp2->OperIsHigh() || (GenTree::OperIsHigh(GetHiOper(oper)) && (!loOp1->OperIsLeaf() || !hiOp1->OperIsLeaf() || !loOp1->OperIsLeaf() || !hiOp2->OperIsLeaf())), "Can't decompose expression tree TYP_LONG node"); // Now, remove op1 and op2 from the node list. m_compiler->fgSnipNode(curStmt, op1); m_compiler->fgSnipNode(curStmt, op2); // We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands // will be the lo halves of op1 from above. GenTree* loResult = tree; loResult->SetOper(GetLoOper(loResult->OperGet())); loResult->gtType = TYP_INT; loResult->gtOp.gtOp1 = loOp1; loResult->gtOp.gtOp2 = loOp2; // The various halves will be correctly threaded internally. We simply need to // relink them into the proper order, i.e. loOp1 is followed by loOp2, and then // the loResult node. // (This rethreading, and that below, are where we need to address the reverse ops case). // The current order is (after snipping op1 and op2): // ... loOp1-> ... hiOp1->loOp2First ... loOp2->hiOp2First ... hiOp2 // The order we want is: // ... loOp1->loOp2First ... loOp2->loResult // ... hiOp1->hiOp2First ... hiOp2->hiResult // i.e. we swap hiOp1 and loOp2, and create (for now) separate loResult and hiResult trees GenTree* loOp2First = hiOp1->gtNext; GenTree* hiOp2First = loOp2->gtNext; // First, we will NYI if both hiOp1 and loOp2 have side effects. NYI_IF(((loOp2->gtFlags & GTF_ALL_EFFECT) != 0) && ((hiOp1->gtFlags & GTF_ALL_EFFECT) != 0), "Binary long operator with non-reorderable sub expressions"); // Now, we reorder the loOps and the loResult. loOp1->gtNext = loOp2First; loOp2First->gtPrev = loOp1; loOp2->gtNext = loResult; loResult->gtPrev = loOp2; // Next, reorder the hiOps and the hiResult. GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2); hiOp1->gtNext = hiOp2First; hiOp2First->gtPrev = hiOp1; hiOp2->gtNext = hiResult; hiResult->gtPrev = hiOp2; if ((oper == GT_ADD) || (oper == GT_SUB)) { if (loResult->gtOverflow()) { hiResult->gtFlags |= GTF_OVERFLOW; loResult->gtFlags &= ~GTF_OVERFLOW; } if (loResult->gtFlags & GTF_UNSIGNED) { hiResult->gtFlags |= GTF_UNSIGNED; } } FinalizeDecomposition(ppTree, data, loResult, hiResult); }
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack) { assert(useEdge != nullptr); GenTree* node = *useEdge; assert(node != nullptr); #ifdef DEBUG const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0; #endif // First, remove any preceeding list nodes, which are not otherwise visited by the tree walk. // // NOTE: GT_FIELD_LIST head nodes, and GT_LIST nodes used by phi nodes will in fact be visited. for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperIsAnyList() && !(prev->OperIsFieldListHead()); prev = node->gtPrev) { BlockRange().Remove(prev); } // In addition, remove the current node if it is a GT_LIST node that is not an aggregate. if (node->OperIsAnyList()) { GenTreeArgList* list = node->AsArgList(); if (!list->OperIsFieldListHead()) { BlockRange().Remove(list); } return Compiler::WALK_CONTINUE; } LIR::Use use; if (parentStack.Height() < 2) { use = LIR::Use::GetDummyUse(BlockRange(), *useEdge); } else { use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1)); } assert(node == use.Def()); switch (node->OperGet()) { case GT_ASG: RewriteAssignment(use); break; case GT_BOX: // GT_BOX at this level just passes through so get rid of it use.ReplaceWith(comp, node->gtGetOp1()); BlockRange().Remove(node); break; case GT_ADDR: RewriteAddress(use); break; case GT_IND: // Clear the `GTF_IND_ASG_LHS` flag, which overlaps with `GTF_IND_REQ_ADDR_IN_REG`. node->gtFlags &= ~GTF_IND_ASG_LHS; if (varTypeIsSIMD(node)) { RewriteSIMDOperand(use, false); } else { // Due to promotion of structs containing fields of type struct with a // single scalar type field, we could potentially see IR nodes of the // form GT_IND(GT_ADD(lclvarAddr, 0)) where 0 is an offset representing // a field-seq. These get folded here. // // TODO: This code can be removed once JIT implements recursive struct // promotion instead of lying about the type of struct field as the type // of its single scalar field. GenTree* addr = node->AsIndir()->Addr(); if (addr->OperGet() == GT_ADD && addr->gtGetOp1()->OperGet() == GT_LCL_VAR_ADDR && addr->gtGetOp2()->IsIntegralConst(0)) { GenTreeLclVarCommon* lclVarNode = addr->gtGetOp1()->AsLclVarCommon(); unsigned lclNum = lclVarNode->GetLclNum(); LclVarDsc* varDsc = comp->lvaTable + lclNum; if (node->TypeGet() == varDsc->TypeGet()) { JITDUMP("Rewriting GT_IND(GT_ADD(LCL_VAR_ADDR,0)) to LCL_VAR\n"); lclVarNode->SetOper(GT_LCL_VAR); lclVarNode->gtType = node->TypeGet(); use.ReplaceWith(comp, lclVarNode); BlockRange().Remove(addr); BlockRange().Remove(addr->gtGetOp2()); BlockRange().Remove(node); } } } break; case GT_NOP: // fgMorph sometimes inserts NOP nodes between defs and uses // supposedly 'to prevent constant folding'. In this case, remove the // NOP. if (node->gtGetOp1() != nullptr) { use.ReplaceWith(comp, node->gtGetOp1()); BlockRange().Remove(node); } break; case GT_COMMA: { GenTree* op1 = node->gtGetOp1(); if ((op1->gtFlags & GTF_ALL_EFFECT) == 0) { // The LHS has no side effects. Remove it. bool isClosed = false; unsigned sideEffects = 0; LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects); // None of the transforms performed herein violate tree order, so these // should always be true. assert(isClosed); assert((sideEffects & GTF_ALL_EFFECT) == 0); BlockRange().Delete(comp, m_block, std::move(lhsRange)); } GenTree* replacement = node->gtGetOp2(); if (!use.IsDummyUse()) { use.ReplaceWith(comp, replacement); } else { // This is a top-level comma. If the RHS has no side effects we can remove // it as well. if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0) { bool isClosed = false; unsigned sideEffects = 0; LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects); // None of the transforms performed herein violate tree order, so these // should always be true. assert(isClosed); assert((sideEffects & GTF_ALL_EFFECT) == 0); BlockRange().Delete(comp, m_block, std::move(rhsRange)); } } BlockRange().Remove(node); } break; case GT_ARGPLACE: // Remove argplace and list nodes from the execution order. // // TODO: remove phi args and phi nodes as well? BlockRange().Remove(node); break; #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM_) case GT_CLS_VAR: { // Class vars that are the target of an assignment will get rewritten into // GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is // not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would // otherwise be generated would also be picked up by RewriteAssignment--but // skipping the rewrite here saves an allocation and a bit of extra work. const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node); if (!isLHSOfAssignment) { GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node); node->SetOper(GT_CLS_VAR_ADDR); node->gtType = TYP_BYREF; BlockRange().InsertAfter(node, ind); use.ReplaceWith(comp, ind); // TODO: JIT dump } } break; #endif // _TARGET_XARCH_ case GT_INTRINSIC: // Non-target intrinsics should have already been rewritten back into user calls. assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId)); break; #ifdef FEATURE_SIMD case GT_BLK: case GT_OBJ: { // TODO-1stClassStructs: These should have been transformed to GT_INDs, but in order // to preserve existing behavior, we will keep this as a block node if this is the // lhs of a block assignment, and either: // - It is a "generic" TYP_STRUCT assignment, OR // - It is an initblk, OR // - Neither the lhs or rhs are known to be of SIMD type. GenTree* parent = use.User(); bool keepBlk = false; if ((parent->OperGet() == GT_ASG) && (node == parent->gtGetOp1())) { if ((node->TypeGet() == TYP_STRUCT) || parent->OperIsInitBlkOp()) { keepBlk = true; } else if (!comp->isAddrOfSIMDType(node->AsBlk()->Addr())) { GenTree* dataSrc = parent->gtGetOp2(); if (!dataSrc->IsLocal() && (dataSrc->OperGet() != GT_SIMD)) { noway_assert(dataSrc->OperIsIndir()); keepBlk = !comp->isAddrOfSIMDType(dataSrc->AsIndir()->Addr()); } } } RewriteSIMDOperand(use, keepBlk); } break; case GT_LCL_FLD: case GT_STORE_LCL_FLD: // TODO-1stClassStructs: Eliminate this. FixupIfSIMDLocal(node->AsLclVarCommon()); break; case GT_SIMD: { noway_assert(comp->featureSIMD); GenTreeSIMD* simdNode = node->AsSIMD(); unsigned simdSize = simdNode->gtSIMDSize; var_types simdType = comp->getSIMDTypeForSize(simdSize); // TODO-1stClassStructs: This should be handled more generally for enregistered or promoted // structs that are passed or returned in a different register type than their enregistered // type(s). if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE) { // This happens when it is consumed by a GT_RET_EXPR. // It can only be a Vector2f or Vector2i. assert(genTypeSize(simdNode->gtSIMDBaseType) == 4); simdNode->gtType = TYP_SIMD8; } // Certain SIMD trees require rationalizing. if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray) { // Rewrite this as an explicit load. JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n"); unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType); GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2, baseTypeSize, offsetof(CORINFO_Array, u1Elems)); GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address); BlockRange().InsertBefore(simdNode, address, ind); use.ReplaceWith(comp, ind); BlockRange().Remove(simdNode); DISPTREERANGE(BlockRange(), use.Def()); JITDUMP("\n"); } else { // This code depends on the fact that NONE of the SIMD intrinsics take vector operands // of a different width. If that assumption changes, we will EITHER have to make these type // transformations during importation, and plumb the types all the way through the JIT, // OR add a lot of special handling here. GenTree* op1 = simdNode->gtGetOp1(); if (op1 != nullptr && op1->gtType == TYP_STRUCT) { op1->gtType = simdType; } GenTree* op2 = simdNode->gtGetOp2IfPresent(); if (op2 != nullptr && op2->gtType == TYP_STRUCT) { op2->gtType = simdType; } } } break; #endif // FEATURE_SIMD default: // JCC nodes should not be present in HIR. assert(node->OperGet() != GT_JCC); break; } // Do some extra processing on top-level nodes to remove unused local reads. if (node->OperIsLocalRead()) { if (use.IsDummyUse()) { comp->lvaDecRefCnts(node); BlockRange().Remove(node); } else { // Local reads are side-effect-free; clear any flags leftover from frontend transformations. node->gtFlags &= ~GTF_ALL_EFFECT; } } assert(isLateArg == ((use.Def()->gtFlags & GTF_LATE_ARG) != 0)); return Compiler::WALK_CONTINUE; }
//------------------------------------------------------------------------ // DecomposeShift: Decompose GT_LSH, GT_RSH, GT_RSZ. For shift nodes, we need to use // the shift helper functions, so we here convert the shift into a helper call by // pulling its arguments out of linear order and making them the args to a call, then // replacing the original node with the new call. // // Arguments: // use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: // The next node to process. // GenTree* DecomposeLongs::DecomposeShift(LIR::Use& use) { assert(use.IsInitialized()); GenTree* tree = use.Def(); GenTree* gtLong = tree->gtGetOp1(); genTreeOps oper = tree->OperGet(); assert((oper == GT_LSH) || (oper == GT_RSH) || (oper == GT_RSZ)); LIR::Use loOp1Use(Range(), >Long->gtOp.gtOp1, gtLong); loOp1Use.ReplaceWithLclVar(m_compiler, m_blockWeight); LIR::Use hiOp1Use(Range(), >Long->gtOp.gtOp2, gtLong); hiOp1Use.ReplaceWithLclVar(m_compiler, m_blockWeight); LIR::Use shiftWidthUse(Range(), &tree->gtOp.gtOp2, tree); shiftWidthUse.ReplaceWithLclVar(m_compiler, m_blockWeight); GenTree* loOp1 = gtLong->gtGetOp1(); GenTree* hiOp1 = gtLong->gtGetOp2(); GenTree* shiftWidthOp = tree->gtGetOp2(); Range().Remove(gtLong); Range().Remove(loOp1); Range().Remove(hiOp1); Range().Remove(shiftWidthOp); // TODO-X86-CQ: If the shift operand is a GT_CNS_INT, we should pipe the instructions through to codegen // and generate the shift instructions ourselves there, rather than replacing it with a helper call. unsigned helper; switch (oper) { case GT_LSH: helper = CORINFO_HELP_LLSH; break; case GT_RSH: helper = CORINFO_HELP_LRSH; break; case GT_RSZ: helper = CORINFO_HELP_LRSZ; break; default: unreached(); } GenTreeArgList* argList = m_compiler->gtNewArgList(loOp1, hiOp1, shiftWidthOp); GenTree* call = m_compiler->gtNewHelperCallNode(helper, TYP_LONG, 0, argList); GenTreeCall* callNode = call->AsCall(); ReturnTypeDesc* retTypeDesc = callNode->GetReturnTypeDesc(); retTypeDesc->InitializeLongReturnType(m_compiler); call = m_compiler->fgMorphArgs(callNode); Range().InsertAfter(tree, LIR::SeqTree(m_compiler, call)); Range().Remove(tree); use.ReplaceWith(m_compiler, call); return call; }
//------------------------------------------------------------------------ // DecomposeCast: Decompose GT_CAST. // // Arguments: // use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: // The next node to process. // GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use) { assert(use.IsInitialized()); assert(use.Def()->OperGet() == GT_CAST); GenTree* cast = use.Def()->AsCast(); GenTree* loResult = nullptr; GenTree* hiResult = nullptr; var_types srcType = cast->CastFromType(); var_types dstType = cast->CastToType(); if ((cast->gtFlags & GTF_UNSIGNED) != 0) { srcType = genUnsignedType(srcType); } if (varTypeIsLong(srcType)) { if (cast->gtOverflow() && (varTypeIsUnsigned(srcType) != varTypeIsUnsigned(dstType))) { GenTree* srcOp = cast->gtGetOp1(); noway_assert(srcOp->OperGet() == GT_LONG); GenTree* loSrcOp = srcOp->gtGetOp1(); GenTree* hiSrcOp = srcOp->gtGetOp2(); // // When casting between long types an overflow check is needed only if the types // have different signedness. In both cases (long->ulong and ulong->long) we only // need to check if the high part is negative or not. Use the existing cast node // to perform a int->uint cast of the high part to take advantage of the overflow // check provided by codegen. // loResult = loSrcOp; hiResult = cast; hiResult->gtType = TYP_INT; hiResult->AsCast()->gtCastType = TYP_UINT; hiResult->gtFlags &= ~GTF_UNSIGNED; hiResult->gtOp.gtOp1 = hiSrcOp; Range().Remove(cast); Range().Remove(srcOp); Range().InsertAfter(hiSrcOp, hiResult); } else { NYI("Unimplemented long->long no-op cast decomposition"); } } else if (varTypeIsIntegralOrI(srcType)) { if (cast->gtOverflow() && !varTypeIsUnsigned(srcType) && varTypeIsUnsigned(dstType)) { // // An overflow check is needed only when casting from a signed type to ulong. // Change the cast type to uint to take advantage of the overflow check provided // by codegen and then zero extend the resulting uint to ulong. // loResult = cast; loResult->AsCast()->gtCastType = TYP_UINT; loResult->gtType = TYP_INT; hiResult = m_compiler->gtNewZeroConNode(TYP_INT); Range().InsertAfter(loResult, hiResult); } else { if (varTypeIsUnsigned(srcType)) { loResult = cast->gtGetOp1(); hiResult = m_compiler->gtNewZeroConNode(TYP_INT); Range().Remove(cast); Range().InsertAfter(loResult, hiResult); } else { LIR::Use src(Range(), &(cast->gtOp.gtOp1), cast); unsigned lclNum = src.ReplaceWithLclVar(m_compiler, m_blockWeight); loResult = src.Def(); GenTree* loCopy = m_compiler->gtNewLclvNode(lclNum, TYP_INT); GenTree* shiftBy = m_compiler->gtNewIconNode(31, TYP_INT); hiResult = m_compiler->gtNewOperNode(GT_RSH, TYP_INT, loCopy, shiftBy); Range().Remove(cast); Range().InsertAfter(loResult, loCopy, shiftBy, hiResult); m_compiler->lvaIncRefCnts(loCopy); } } } else { NYI("Unimplemented cast decomposition"); } return FinalizeDecomposition(use, loResult, hiResult); }
//------------------------------------------------------------------------ // DecomposeStoreLclVar: Decompose GT_STORE_LCL_VAR. // // Arguments: // use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: // The next node to process. // GenTree* DecomposeLongs::DecomposeStoreLclVar(LIR::Use& use) { assert(use.IsInitialized()); assert(use.Def()->OperGet() == GT_STORE_LCL_VAR); GenTree* tree = use.Def(); GenTree* rhs = tree->gtGetOp1(); if ((rhs->OperGet() == GT_PHI) || (rhs->OperGet() == GT_CALL) || ((rhs->OperGet() == GT_MUL_LONG) && (rhs->gtFlags & GTF_MUL_64RSLT) != 0)) { // GT_CALLs are not decomposed, so will not be converted to GT_LONG // GT_STORE_LCL_VAR = GT_CALL are handled in genMultiRegCallStoreToLocal // GT_MULs are not decomposed, so will not be converted to GT_LONG return tree->gtNext; } noway_assert(rhs->OperGet() == GT_LONG); unsigned varNum = tree->AsLclVarCommon()->gtLclNum; LclVarDsc* varDsc = m_compiler->lvaTable + varNum; m_compiler->lvaDecRefCnts(tree); GenTree* loRhs = rhs->gtGetOp1(); GenTree* hiRhs = rhs->gtGetOp2(); GenTree* hiStore = m_compiler->gtNewLclLNode(varNum, TYP_INT); if (varDsc->lvPromoted) { assert(varDsc->lvFieldCnt == 2); unsigned loVarNum = varDsc->lvFieldLclStart; unsigned hiVarNum = loVarNum + 1; tree->AsLclVarCommon()->SetLclNum(loVarNum); hiStore->SetOper(GT_STORE_LCL_VAR); hiStore->AsLclVarCommon()->SetLclNum(hiVarNum); } else { noway_assert(varDsc->lvLRACandidate == false); tree->SetOper(GT_STORE_LCL_FLD); tree->AsLclFld()->gtLclOffs = 0; tree->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField(); hiStore->SetOper(GT_STORE_LCL_FLD); hiStore->AsLclFld()->gtLclOffs = 4; hiStore->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField(); } // 'tree' is going to steal the loRhs node for itself, so we need to remove the // GT_LONG node from the threading. Range().Remove(rhs); tree->gtOp.gtOp1 = loRhs; tree->gtType = TYP_INT; hiStore->gtOp.gtOp1 = hiRhs; hiStore->gtFlags |= GTF_VAR_DEF; m_compiler->lvaIncRefCnts(tree); m_compiler->lvaIncRefCnts(hiStore); Range().InsertAfter(tree, hiStore); return hiStore->gtNext; }
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack) { assert(useEdge != nullptr); GenTree* node = *useEdge; assert(node != nullptr); #ifdef DEBUG const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0; #endif // First, remove any preceeding GT_LIST nodes, which are not otherwise visited by the tree walk. // // NOTE: GT_LIST nodes that are used by block ops and phi nodes will in fact be visited. for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperGet() == GT_LIST; prev = node->gtPrev) { BlockRange().Remove(prev); } // In addition, remove the current node if it is a GT_LIST node. if ((*useEdge)->OperGet() == GT_LIST) { BlockRange().Remove(*useEdge); return Compiler::WALK_CONTINUE; } LIR::Use use; if (parentStack.Height() < 2) { use = LIR::Use::GetDummyUse(BlockRange(), *useEdge); } else { use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1)); } assert(node == use.Def()); switch (node->OperGet()) { case GT_ASG: RewriteAssignment(use); break; case GT_BOX: // GT_BOX at this level just passes through so get rid of it use.ReplaceWith(comp, node->gtGetOp1()); BlockRange().Remove(node); break; case GT_ADDR: RewriteAddress(use); break; case GT_NOP: // fgMorph sometimes inserts NOP nodes between defs and uses // supposedly 'to prevent constant folding'. In this case, remove the // NOP. if (node->gtGetOp1() != nullptr) { use.ReplaceWith(comp, node->gtGetOp1()); BlockRange().Remove(node); } break; case GT_COMMA: { GenTree* op1 = node->gtGetOp1(); if ((op1->gtFlags & GTF_ALL_EFFECT) == 0) { // The LHS has no side effects. Remove it. bool isClosed = false; unsigned sideEffects = 0; LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects); // None of the transforms performed herein violate tree order, so these // should always be true. assert(isClosed); assert((sideEffects & GTF_ALL_EFFECT) == 0); BlockRange().Delete(comp, m_block, std::move(lhsRange)); } GenTree* replacement = node->gtGetOp2(); if (!use.IsDummyUse()) { use.ReplaceWith(comp, replacement); } else { // This is a top-level comma. If the RHS has no side effects we can remove // it as well. if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0) { bool isClosed = false; unsigned sideEffects = 0; LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects); // None of the transforms performed herein violate tree order, so these // should always be true. assert(isClosed); assert((sideEffects & GTF_ALL_EFFECT) == 0); BlockRange().Delete(comp, m_block, std::move(rhsRange)); } } BlockRange().Remove(node); } break; case GT_ARGPLACE: // Remove argplace and list nodes from the execution order. // // TODO: remove phi args and phi nodes as well? BlockRange().Remove(node); break; #ifdef _TARGET_XARCH_ case GT_CLS_VAR: { // Class vars that are the target of an assignment will get rewritten into // GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is // not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would // otherwise be generated would also be picked up by RewriteAssignment--but // skipping the rewrite here saves an allocation and a bit of extra work. const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node); if (!isLHSOfAssignment) { GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node); node->SetOper(GT_CLS_VAR_ADDR); node->gtType = TYP_BYREF; BlockRange().InsertAfter(node, ind); use.ReplaceWith(comp, ind); // TODO: JIT dump } } break; #endif // _TARGET_XARCH_ case GT_INTRINSIC: // Non-target intrinsics should have already been rewritten back into user calls. assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId)); break; #ifdef FEATURE_SIMD case GT_INITBLK: RewriteInitBlk(use); break; case GT_COPYBLK: RewriteCopyBlk(use); break; case GT_OBJ: RewriteObj(use); break; case GT_LCL_FLD: case GT_STORE_LCL_FLD: // TODO-1stClassStructs: Eliminate this. FixupIfSIMDLocal(node->AsLclVarCommon()); break; case GT_STOREIND: case GT_IND: if (node->gtType == TYP_STRUCT) { GenTree* addr = node->AsIndir()->Addr(); assert(addr->TypeGet() == TYP_BYREF); if (addr->OperIsLocal()) { LclVarDsc* varDsc = &(comp->lvaTable[addr->AsLclVarCommon()->gtLclNum]); assert(varDsc->lvSIMDType); unsigned simdSize = (unsigned int)roundUp(varDsc->lvExactSize, TARGET_POINTER_SIZE); node->gtType = comp->getSIMDTypeForSize(simdSize); } #if DEBUG else { // If the address is not a local var, assert that the user of this IND is an ADDR node. assert((use.User()->OperGet() == GT_ADDR) || use.User()->OperIsLocalAddr()); } #endif } break; case GT_SIMD: { noway_assert(comp->featureSIMD); GenTreeSIMD* simdNode = node->AsSIMD(); unsigned simdSize = simdNode->gtSIMDSize; var_types simdType = comp->getSIMDTypeForSize(simdSize); // TODO-1stClassStructs: This should be handled more generally for enregistered or promoted // structs that are passed or returned in a different register type than their enregistered // type(s). if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE) { // This happens when it is consumed by a GT_RET_EXPR. // It can only be a Vector2f or Vector2i. assert(genTypeSize(simdNode->gtSIMDBaseType) == 4); simdNode->gtType = TYP_SIMD8; } else if (simdNode->gtType == TYP_STRUCT || varTypeIsSIMD(simdNode)) { node->gtType = simdType; } // Certain SIMD trees require rationalizing. if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray) { // Rewrite this as an explicit load. JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n"); unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType); GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2, baseTypeSize, offsetof(CORINFO_Array, u1Elems)); GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address); BlockRange().InsertBefore(simdNode, address, ind); use.ReplaceWith(comp, ind); BlockRange().Remove(simdNode); DISPTREERANGE(BlockRange(), use.Def()); JITDUMP("\n"); } else { // This code depends on the fact that NONE of the SIMD intrinsics take vector operands // of a different width. If that assumption changes, we will EITHER have to make these type // transformations during importation, and plumb the types all the way through the JIT, // OR add a lot of special handling here. GenTree* op1 = simdNode->gtGetOp1(); if (op1 != nullptr && op1->gtType == TYP_STRUCT) { op1->gtType = simdType; } GenTree* op2 = simdNode->gtGetOp2(); if (op2 != nullptr && op2->gtType == TYP_STRUCT) { op2->gtType = simdType; } } } break; #endif // FEATURE_SIMD default: break; } // Do some extra processing on top-level nodes to remove unused local reads. if (use.IsDummyUse() && node->OperIsLocalRead()) { assert((node->gtFlags & GTF_ALL_EFFECT) == 0); comp->lvaDecRefCnts(node); BlockRange().Remove(node); } assert(isLateArg == ((node->gtFlags & GTF_LATE_ARG) != 0)); return Compiler::WALK_CONTINUE; }
//------------------------------------------------------------------------ // MorphAllocObjNodes: Morph each GT_ALLOCOBJ node either into an // allocation helper call or stack allocation. // // Notes: // Runs only over the blocks having bbFlags BBF_HAS_NEWOBJ set. void ObjectAllocator::MorphAllocObjNodes() { BasicBlock* block; foreach_block(comp, block) { const bool basicBlockHasNewObj = (block->bbFlags & BBF_HAS_NEWOBJ) == BBF_HAS_NEWOBJ; #ifndef DEBUG if (!basicBlockHasNewObj) { continue; } #endif // DEBUG for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt) { GenTree* stmtExpr = stmt->gtStmtExpr; GenTree* op2 = nullptr; bool canonicalAllocObjFound = false; if (stmtExpr->OperGet() == GT_ASG && stmtExpr->TypeGet() == TYP_REF) { op2 = stmtExpr->gtGetOp2(); if (op2->OperGet() == GT_ALLOCOBJ) { canonicalAllocObjFound = true; } } if (canonicalAllocObjFound) { assert(basicBlockHasNewObj); //------------------------------------------------------------------------ // We expect the following expression tree at this point // * GT_STMT void (top level) // | /--* GT_ALLOCOBJ ref // \--* GT_ASG ref // \--* GT_LCL_VAR ref //------------------------------------------------------------------------ GenTree* op1 = stmtExpr->gtGetOp1(); assert(op1->OperGet() == GT_LCL_VAR); assert(op1->TypeGet() == TYP_REF); assert(op2 != nullptr); assert(op2->OperGet() == GT_ALLOCOBJ); GenTreeAllocObj* asAllocObj = op2->AsAllocObj(); unsigned int lclNum = op1->AsLclVar()->GetLclNum(); if (IsObjectStackAllocationEnabled() && CanAllocateLclVarOnStack(lclNum)) { op2 = MorphAllocObjNodeIntoStackAlloc(asAllocObj, block, stmt); } else { op2 = MorphAllocObjNodeIntoHelperCall(asAllocObj); } // Propagate flags of op2 to its parent. stmtExpr->gtOp.gtOp2 = op2; stmtExpr->gtFlags |= op2->gtFlags & GTF_ALL_EFFECT; } #ifdef DEBUG else { // We assume that GT_ALLOCOBJ nodes are always present in the // canonical form. comp->fgWalkTreePre(&stmt->gtStmtExpr, AssertWhenAllocObjFoundVisitor); } #endif // DEBUG } } }
//------------------------------------------------------------------------ // BuildSIMD: Set the NodeInfo for a GT_SIMD tree. // // Arguments: // tree - The GT_SIMD node of interest // // Return Value: // The number of sources consumed by this node. // int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) { int srcCount = 0; // Only SIMDIntrinsicInit can be contained if (simdTree->isContained()) { assert(simdTree->gtSIMDIntrinsicID == SIMDIntrinsicInit); } int dstCount = simdTree->IsValue() ? 1 : 0; assert(dstCount == 1); bool buildUses = true; GenTree* op1 = simdTree->gtGetOp1(); GenTree* op2 = simdTree->gtGetOp2(); switch (simdTree->gtSIMDIntrinsicID) { case SIMDIntrinsicInit: case SIMDIntrinsicCast: case SIMDIntrinsicSqrt: case SIMDIntrinsicAbs: case SIMDIntrinsicConvertToSingle: case SIMDIntrinsicConvertToInt32: case SIMDIntrinsicConvertToDouble: case SIMDIntrinsicConvertToInt64: case SIMDIntrinsicWidenLo: case SIMDIntrinsicWidenHi: // No special handling required. break; case SIMDIntrinsicGetItem: { op1 = simdTree->gtGetOp1(); op2 = simdTree->gtGetOp2(); // We have an object and an index, either of which may be contained. bool setOp2DelayFree = false; if (!op2->IsCnsIntOrI() && (!op1->isContained() || op1->OperIsLocal())) { // If the index is not a constant and the object is not contained or is a local // we will need a general purpose register to calculate the address // internal register must not clobber input index // TODO-Cleanup: An internal register will never clobber a source; this code actually // ensures that the index (op2) doesn't interfere with the target. buildInternalIntRegisterDefForNode(simdTree); setOp2DelayFree = true; } srcCount += BuildOperandUses(op1); if (!op2->isContained()) { RefPosition* op2Use = BuildUse(op2); if (setOp2DelayFree) { setDelayFree(op2Use); } srcCount++; } if (!op2->IsCnsIntOrI() && (!op1->isContained())) { // If vector is not already in memory (contained) and the index is not a constant, // we will use the SIMD temp location to store the vector. compiler->getSIMDInitTempVarNum(); } buildUses = false; } break; case SIMDIntrinsicAdd: case SIMDIntrinsicSub: case SIMDIntrinsicMul: case SIMDIntrinsicDiv: case SIMDIntrinsicBitwiseAnd: case SIMDIntrinsicBitwiseAndNot: case SIMDIntrinsicBitwiseOr: case SIMDIntrinsicBitwiseXor: case SIMDIntrinsicMin: case SIMDIntrinsicMax: case SIMDIntrinsicEqual: case SIMDIntrinsicLessThan: case SIMDIntrinsicGreaterThan: case SIMDIntrinsicLessThanOrEqual: case SIMDIntrinsicGreaterThanOrEqual: // No special handling required. break; case SIMDIntrinsicSetX: case SIMDIntrinsicSetY: case SIMDIntrinsicSetZ: case SIMDIntrinsicSetW: case SIMDIntrinsicNarrow: { // Op1 will write to dst before Op2 is free BuildUse(op1); RefPosition* op2Use = BuildUse(op2); setDelayFree(op2Use); srcCount = 2; buildUses = false; break; } case SIMDIntrinsicInitN: { var_types baseType = simdTree->gtSIMDBaseType; srcCount = (short)(simdTree->gtSIMDSize / genTypeSize(baseType)); if (varTypeIsFloating(simdTree->gtSIMDBaseType)) { // Need an internal register to stitch together all the values into a single vector in a SIMD reg. buildInternalFloatRegisterDefForNode(simdTree); } int initCount = 0; for (GenTree* list = op1; list != nullptr; list = list->gtGetOp2()) { assert(list->OperGet() == GT_LIST); GenTree* listItem = list->gtGetOp1(); assert(listItem->TypeGet() == baseType); assert(!listItem->isContained()); BuildUse(listItem); initCount++; } assert(initCount == srcCount); buildUses = false; break; } case SIMDIntrinsicInitArray: // We have an array and an index, which may be contained. break; case SIMDIntrinsicOpEquality: case SIMDIntrinsicOpInEquality: buildInternalFloatRegisterDefForNode(simdTree); break; case SIMDIntrinsicDotProduct: buildInternalFloatRegisterDefForNode(simdTree); break; case SIMDIntrinsicSelect: // TODO-ARM64-CQ Allow lowering to see SIMDIntrinsicSelect so we can generate BSL VC, VA, VB // bsl target register must be VC. Reserve a temp in case we need to shuffle things. // This will require a different approach, as GenTreeSIMD has only two operands. assert(!"SIMDIntrinsicSelect not yet supported"); buildInternalFloatRegisterDefForNode(simdTree); break; case SIMDIntrinsicInitArrayX: case SIMDIntrinsicInitFixed: case SIMDIntrinsicCopyToArray: case SIMDIntrinsicCopyToArrayX: case SIMDIntrinsicNone: case SIMDIntrinsicGetCount: case SIMDIntrinsicGetOne: case SIMDIntrinsicGetZero: case SIMDIntrinsicGetAllOnes: case SIMDIntrinsicGetX: case SIMDIntrinsicGetY: case SIMDIntrinsicGetZ: case SIMDIntrinsicGetW: case SIMDIntrinsicInstEquals: case SIMDIntrinsicHWAccel: case SIMDIntrinsicWiden: case SIMDIntrinsicInvalid: assert(!"These intrinsics should not be seen during register allocation"); __fallthrough; default: noway_assert(!"Unimplemented SIMD node type."); unreached(); } if (buildUses) { assert(!op1->OperIs(GT_LIST)); assert(srcCount == 0); srcCount = BuildOperandUses(op1); if ((op2 != nullptr) && !op2->isContained()) { srcCount += BuildOperandUses(op2); } } assert(internalCount <= MaxInternalCount); buildInternalRegisterUses(); if (dstCount == 1) { BuildDef(simdTree); } else { assert(dstCount == 0); } return srcCount; }
//------------------------------------------------------------------------ // DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND. // // Arguments: // use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: // The next node to process. // GenTree* DecomposeLongs::DecomposeArith(LIR::Use& use) { assert(use.IsInitialized()); GenTree* tree = use.Def(); genTreeOps oper = tree->OperGet(); assert((oper == GT_ADD) || (oper == GT_SUB) || (oper == GT_OR) || (oper == GT_XOR) || (oper == GT_AND)); GenTree* op1 = tree->gtGetOp1(); GenTree* op2 = tree->gtGetOp2(); // Both operands must have already been decomposed into GT_LONG operators. noway_assert((op1->OperGet() == GT_LONG) && (op2->OperGet() == GT_LONG)); // Capture the lo and hi halves of op1 and op2. GenTree* loOp1 = op1->gtGetOp1(); GenTree* hiOp1 = op1->gtGetOp2(); GenTree* loOp2 = op2->gtGetOp1(); GenTree* hiOp2 = op2->gtGetOp2(); // We don't have support to decompose a TYP_LONG node that already has a child that has // been decomposed into parts, where the high part depends on the value generated by the // low part (via the flags register). For example, if we have: // +(gt_long(+(lo3, lo4), +Hi(hi3, hi4)), gt_long(lo2, hi2)) // We would decompose it here to: // gt_long(+(+(lo3, lo4), lo2), +Hi(+Hi(hi3, hi4), hi2)) // But this would generate incorrect code, because the "+Hi(hi3, hi4)" code generation // needs to immediately follow the "+(lo3, lo4)" part. Also, if this node is one that // requires a unique high operator, and the child nodes are not simple locals (e.g., // they are decomposed nodes), then we also can't decompose the node, as we aren't // guaranteed the high and low parts will be executed immediately after each other. NYI_IF(hiOp1->OperIsHigh() || hiOp2->OperIsHigh() || (GenTree::OperIsHigh(GetHiOper(oper)) && (!loOp1->OperIsLeaf() || !hiOp1->OperIsLeaf() || !loOp1->OperIsLeaf() || !hiOp2->OperIsLeaf())), "Can't decompose expression tree TYP_LONG node"); // Now, remove op1 and op2 from the node list. BlockRange().Remove(op1); BlockRange().Remove(op2); // We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands // will be the lo halves of op1 from above. GenTree* loResult = tree; loResult->SetOper(GetLoOper(loResult->OperGet())); loResult->gtType = TYP_INT; loResult->gtOp.gtOp1 = loOp1; loResult->gtOp.gtOp2 = loOp2; GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2); hiResult->CopyCosts(loResult); BlockRange().InsertAfter(loResult, hiResult); if ((oper == GT_ADD) || (oper == GT_SUB)) { if (loResult->gtOverflow()) { hiResult->gtFlags |= GTF_OVERFLOW; loResult->gtFlags &= ~GTF_OVERFLOW; } if (loResult->gtFlags & GTF_UNSIGNED) { hiResult->gtFlags |= GTF_UNSIGNED; } } return FinalizeDecomposition(use, loResult, hiResult); }