void Rationalizer::RewriteAssignment(LIR::Use& use)
{
assert(use.IsInitialized());
GenTreeOp* assignment = use.Def()->AsOp();
assert(assignment->OperGet() == GT_ASG);
GenTree* location = assignment->gtGetOp1();
GenTree* value = assignment->gtGetOp2();
genTreeOps locationOp = location->OperGet();
switch (locationOp)
{
case GT_LCL_VAR:
case GT_LCL_FLD:
case GT_REG_VAR:
case GT_PHI_ARG:
RewriteAssignmentIntoStoreLclCore(assignment, location, value, locationOp);
BlockRange().Remove(location);
break;
case GT_IND:
{
GenTreeStoreInd* store =
new (comp, GT_STOREIND) GenTreeStoreInd(location->TypeGet(), location->gtGetOp1(), value);
copyFlags(store, assignment, GTF_ALL_EFFECT);
copyFlags(store, location, GTF_IND_FLAGS);
if (assignment->IsReverseOp())
{
store->gtFlags |= GTF_REVERSE_OPS;
}
// TODO: JIT dump
// Remove the GT_IND node and replace the assignment node with the store
BlockRange().Remove(location);
BlockRange().InsertBefore(assignment, store);
use.ReplaceWith(comp, store);
BlockRange().Remove(assignment);
}
break;
case GT_CLS_VAR:
{
location->SetOper(GT_CLS_VAR_ADDR);
location->gtType = TYP_BYREF;
assignment->SetOper(GT_STOREIND);
// TODO: JIT dump
}
break;
default:
unreached();
break;
}
}
//------------------------------------------------------------------------
// DecomposeInd: Decompose GT_IND.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeInd(LIR::Use& use)
{
GenTree* indLow = use.Def();
LIR::Use address(BlockRange(), &indLow->gtOp.gtOp1, indLow);
address.ReplaceWithLclVar(m_compiler, m_block->getBBWeight(m_compiler));
JITDUMP("[DecomposeInd]: Saving addr tree to a temp var:\n");
DISPTREERANGE(BlockRange(), address.Def());
// Change the type of lower ind.
indLow->gtType = TYP_INT;
// Create tree of ind(addr+4)
GenTreePtr addrBase = indLow->gtGetOp1();
GenTreePtr addrBaseHigh = new (m_compiler, GT_LCL_VAR)
GenTreeLclVar(GT_LCL_VAR, addrBase->TypeGet(), addrBase->AsLclVarCommon()->GetLclNum(), BAD_IL_OFFSET);
GenTreePtr addrHigh =
new (m_compiler, GT_LEA) GenTreeAddrMode(TYP_REF, addrBaseHigh, nullptr, 0, genTypeSize(TYP_INT));
GenTreePtr indHigh = new (m_compiler, GT_IND) GenTreeIndir(GT_IND, TYP_INT, addrHigh, nullptr);
m_compiler->gtPrepareCost(indHigh);
BlockRange().InsertAfter(indLow, addrBaseHigh, addrHigh, indHigh);
return FinalizeDecomposition(use, indLow, indHigh);
}
void Rationalizer::RewriteNodeAsCall(GenTree** use,
ArrayStack<GenTree*>& parents,
CORINFO_METHOD_HANDLE callHnd,
#ifdef FEATURE_READYTORUN_COMPILER
CORINFO_CONST_LOOKUP entryPoint,
#endif
GenTreeArgList* args)
{
GenTree* const tree = *use;
GenTree* const treeFirstNode = comp->fgGetFirstNode(tree);
GenTree* const insertionPoint = treeFirstNode->gtPrev;
BlockRange().Remove(treeFirstNode, tree);
// Create the call node
GenTreeCall* call = comp->gtNewCallNode(CT_USER_FUNC, callHnd, tree->gtType, args);
#if DEBUG
CORINFO_SIG_INFO sig;
comp->eeGetMethodSig(callHnd, &sig);
assert(JITtype2varType(sig.retType) == tree->gtType);
#endif // DEBUG
call = comp->fgMorphArgs(call);
// Determine if this call has changed any codegen requirements.
comp->fgCheckArgCnt();
#ifdef FEATURE_READYTORUN_COMPILER
call->gtCall.setEntryPoint(entryPoint);
#endif
// Replace "tree" with "call"
if (parents.Height() > 1)
{
parents.Index(1)->ReplaceOperand(use, call);
}
else
{
// If there's no parent, the tree being replaced is the root of the
// statement (and no special handling is necessary).
*use = call;
}
comp->gtSetEvalOrder(call);
BlockRange().InsertAfter(insertionPoint, LIR::Range(comp->fgSetTreeSeq(call), call));
// Propagate flags of "call" to its parents.
// 0 is current node, so start at 1
for (int i = 1; i < parents.Height(); i++)
{
parents.Index(i)->gtFlags |= (call->gtFlags & GTF_ALL_EFFECT) | GTF_CALL;
}
// Since "tree" is replaced with "call", pop "tree" node (i.e the current node)
// and replace it with "call" on parent stack.
assert(parents.Top() == tree);
(void)parents.Pop();
parents.Push(call);
}
void Rationalizer::RewriteAddress(LIR::Use& use)
{
assert(use.IsInitialized());
GenTreeUnOp* address = use.Def()->AsUnOp();
assert(address->OperGet() == GT_ADDR);
GenTree* location = address->gtGetOp1();
genTreeOps locationOp = location->OperGet();
if (location->IsLocal())
{
// We are changing the child from GT_LCL_VAR TO GT_LCL_VAR_ADDR.
// Therefore gtType of the child needs to be changed to a TYP_BYREF
#ifdef DEBUG
if (locationOp == GT_LCL_VAR)
{
JITDUMP("Rewriting GT_ADDR(GT_LCL_VAR) to GT_LCL_VAR_ADDR:\n");
}
else
{
assert(locationOp == GT_LCL_FLD);
JITDUMP("Rewriting GT_ADDR(GT_LCL_FLD) to GT_LCL_FLD_ADDR:\n");
}
#endif // DEBUG
location->SetOper(addrForm(locationOp));
location->gtType = TYP_BYREF;
copyFlags(location, address, GTF_ALL_EFFECT);
use.ReplaceWith(comp, location);
BlockRange().Remove(address);
}
else if (locationOp == GT_CLS_VAR)
{
location->SetOper(GT_CLS_VAR_ADDR);
location->gtType = TYP_BYREF;
copyFlags(location, address, GTF_ALL_EFFECT);
use.ReplaceWith(comp, location);
BlockRange().Remove(address);
JITDUMP("Rewriting GT_ADDR(GT_CLS_VAR) to GT_CLS_VAR_ADDR:\n");
}
else if (location->OperIsIndir())
{
use.ReplaceWith(comp, location->gtGetOp1());
BlockRange().Remove(location);
BlockRange().Remove(address);
JITDUMP("Rewriting GT_ADDR(GT_IND(X)) to X:\n");
}
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
//----------------------------------------------------------------------------------------------
// Lowering::LowerHWIntrinsic: Perform containment analysis for a hardware intrinsic node.
//
// Arguments:
// node - The hardware intrinsic node.
//
void Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node)
{
auto intrinsicID = node->gtHWIntrinsicId;
auto intrinsicInfo = HWIntrinsicInfo::lookup(node->gtHWIntrinsicId);
//
// Lower unsupported Unsigned Compare Zero intrinsics to their trivial transformations
//
// ARM64 does not support most forms of compare zero for Unsigned values
// This is because some are non-sensical, and the rest are trivial transformations of other operators
//
if ((intrinsicInfo.flags & HWIntrinsicInfo::LowerCmpUZero) && varTypeIsUnsigned(node->gtSIMDBaseType))
{
auto setAllVector = node->gtSIMDSize > 8 ? NI_ARM64_SIMD_SetAllVector128 : NI_ARM64_SIMD_SetAllVector64;
auto origOp1 = node->gtOp.gtOp1;
switch (intrinsicID)
{
case NI_ARM64_SIMD_GT_ZERO:
// Unsigned > 0 ==> !(Unsigned == 0)
node->gtOp.gtOp1 =
comp->gtNewSimdHWIntrinsicNode(node->TypeGet(), node->gtOp.gtOp1, NI_ARM64_SIMD_EQ_ZERO,
node->gtSIMDBaseType, node->gtSIMDSize);
node->gtHWIntrinsicId = NI_ARM64_SIMD_BitwiseNot;
BlockRange().InsertBefore(node, node->gtOp.gtOp1);
break;
case NI_ARM64_SIMD_LE_ZERO:
// Unsigned <= 0 ==> Unsigned == 0
node->gtHWIntrinsicId = NI_ARM64_SIMD_EQ_ZERO;
break;
case NI_ARM64_SIMD_GE_ZERO:
case NI_ARM64_SIMD_LT_ZERO:
// Unsigned >= 0 ==> Always true
// Unsigned < 0 ==> Always false
node->gtHWIntrinsicId = setAllVector;
node->gtOp.gtOp1 = comp->gtNewLconNode((intrinsicID == NI_ARM64_SIMD_GE_ZERO) ? ~0ULL : 0ULL);
BlockRange().InsertBefore(node, node->gtOp.gtOp1);
if ((origOp1->gtFlags & GTF_ALL_EFFECT) == 0)
{
BlockRange().Remove(origOp1, true);
}
else
{
origOp1->SetUnusedValue();
}
break;
default:
assert(!"Unhandled LowerCmpUZero case");
}
}
ContainCheckHWIntrinsic(node);
}
//------------------------------------------------------------------------
// FinalizeDecomposition: A helper function to finalize LONG decomposition by
// taking the resulting two halves of the decomposition, and tie them together
// with a new GT_LONG node that will replace the original node.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
// loResult - the decomposed low part
// hiResult - the decomposed high part. This must follow loResult in the linear order,
// as the new GT_LONG node will be inserted immediately after it.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::FinalizeDecomposition(LIR::Use& use, GenTree* loResult, GenTree* hiResult)
{
assert(use.IsInitialized());
assert(loResult != nullptr);
assert(hiResult != nullptr);
assert(BlockRange().Contains(loResult));
assert(BlockRange().Contains(hiResult));
assert(loResult->Precedes(hiResult));
GenTree* gtLong = new (m_compiler, GT_LONG) GenTreeOp(GT_LONG, TYP_LONG, loResult, hiResult);
BlockRange().InsertAfter(hiResult, gtLong);
use.ReplaceWith(m_compiler, gtLong);
return gtLong->gtNext;
}
//------------------------------------------------------------------------
// LowerCast: Lower GT_CAST(srcType, DstType) nodes.
//
// Arguments:
// tree - GT_CAST node to be lowered
//
// Return Value:
// None.
//
// Notes:
// Casts from float/double to a smaller int type are transformed as follows:
// GT_CAST(float/double, byte) = GT_CAST(GT_CAST(float/double, int32), byte)
// GT_CAST(float/double, sbyte) = GT_CAST(GT_CAST(float/double, int32), sbyte)
// GT_CAST(float/double, int16) = GT_CAST(GT_CAST(double/double, int32), int16)
// GT_CAST(float/double, uint16) = GT_CAST(GT_CAST(double/double, int32), uint16)
//
// Note that for the overflow conversions we still depend on helper calls and
// don't expect to see them here.
// i) GT_CAST(float/double, int type with overflow detection)
//
void Lowering::LowerCast(GenTree* tree)
{
assert(tree->OperGet() == GT_CAST);
JITDUMP("LowerCast for: ");
DISPNODE(tree);
JITDUMP("\n");
GenTree* op1 = tree->gtOp.gtOp1;
var_types dstType = tree->CastToType();
var_types srcType = genActualType(op1->TypeGet());
var_types tmpType = TYP_UNDEF;
if (varTypeIsFloating(srcType))
{
noway_assert(!tree->gtOverflow());
assert(!varTypeIsSmall(dstType)); // fgMorphCast creates intermediate casts when converting from float to small
// int.
}
assert(!varTypeIsSmall(srcType));
if (tmpType != TYP_UNDEF)
{
GenTree* tmp = comp->gtNewCastNode(tmpType, op1, tree->IsUnsigned(), tmpType);
tmp->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW | GTF_EXCEPT));
tree->gtFlags &= ~GTF_UNSIGNED;
tree->gtOp.gtOp1 = tmp;
BlockRange().InsertAfter(op1, tmp);
}
// Now determine if we have operands that should be contained.
ContainCheckCast(tree->AsCast());
}
//------------------------------------------------------------------------
// LowerRotate: Lower GT_ROL and GT_ROR nodes.
//
// Arguments:
// tree - the node to lower
//
// Return Value:
// None.
//
void Lowering::LowerRotate(GenTree* tree)
{
if (tree->OperGet() == GT_ROL)
{
// There is no ROL instruction on ARM. Convert ROL into ROR.
GenTree* rotatedValue = tree->gtOp.gtOp1;
unsigned rotatedValueBitSize = genTypeSize(rotatedValue->gtType) * 8;
GenTree* rotateLeftIndexNode = tree->gtOp.gtOp2;
if (rotateLeftIndexNode->IsCnsIntOrI())
{
ssize_t rotateLeftIndex = rotateLeftIndexNode->gtIntCon.gtIconVal;
ssize_t rotateRightIndex = rotatedValueBitSize - rotateLeftIndex;
rotateLeftIndexNode->gtIntCon.gtIconVal = rotateRightIndex;
}
else
{
GenTree* tmp = comp->gtNewOperNode(GT_NEG, genActualType(rotateLeftIndexNode->gtType), rotateLeftIndexNode);
BlockRange().InsertAfter(rotateLeftIndexNode, tmp);
tree->gtOp.gtOp2 = tmp;
}
tree->ChangeOper(GT_ROR);
}
ContainCheckShiftRotate(tree->AsOp());
}
// Rewrite a SIMD indirection as GT_IND(GT_LEA(obj.op1)), or as a simple
// lclVar if possible.
//
// Arguments:
// use - A use reference for a block node
// keepBlk - True if this should remain a block node if it is not a lclVar
//
// Return Value:
// None.
//
// TODO-1stClassStructs: These should be eliminated earlier, once we can handle
// lclVars in all the places that used to have GT_OBJ.
//
void Rationalizer::RewriteSIMDOperand(LIR::Use& use, bool keepBlk)
{
#ifdef FEATURE_SIMD
// No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
GenTree* tree = use.Def();
if (!tree->OperIsIndir())
{
return;
}
var_types simdType = tree->TypeGet();
if (!varTypeIsSIMD(simdType))
{
return;
}
// If we have GT_IND(GT_LCL_VAR_ADDR) and the GT_LCL_VAR_ADDR is TYP_BYREF/TYP_I_IMPL,
// and the var is a SIMD type, replace the expression by GT_LCL_VAR.
GenTree* addr = tree->AsIndir()->Addr();
if (addr->OperIsLocalAddr() && comp->isAddrOfSIMDType(addr))
{
BlockRange().Remove(tree);
addr->SetOper(loadForm(addr->OperGet()));
addr->gtType = simdType;
use.ReplaceWith(comp, addr);
}
else if ((addr->OperGet() == GT_ADDR) && (addr->gtGetOp1()->OperGet() == GT_SIMD))
{
// if we have GT_IND(GT_ADDR(GT_SIMD)), remove the GT_IND(GT_ADDR()), leaving just the GT_SIMD.
BlockRange().Remove(tree);
BlockRange().Remove(addr);
use.ReplaceWith(comp, addr->gtGetOp1());
}
else if (!keepBlk)
{
tree->SetOper(GT_IND);
tree->gtType = simdType;
}
#endif // FEATURE_SIMD
}
//------------------------------------------------------------------------
// DecomposeBlock: Do LONG decomposition to all the statements in the given block.
// This must be done before lowering the block, as decomposition can insert
// additional statements.
//
// Decomposition is done as a post-order tree walk. Lower levels of the tree can
// create new nodes that need to be further decomposed at higher levels. That is,
// the decomposition "bubbles up" the tree.
//
// Arguments:
// block - the block to process
//
// Return Value:
// None.
//
void DecomposeLongs::DecomposeBlock(BasicBlock* block)
{
assert(block == m_compiler->compCurBB); // compCurBB must already be set.
assert(block->isEmpty() || block->IsLIR());
m_block = block;
GenTree* node = BlockRange().FirstNonPhiNode();
while (node != nullptr)
{
LIR::Use use;
if (!BlockRange().TryGetUse(node, &use))
{
use = LIR::Use::GetDummyUse(BlockRange(), node);
}
node = DecomposeNode(use);
}
assert(BlockRange().CheckLIR(m_compiler));
}
//------------------------------------------------------------------------
// DecomposeNeg: Decompose GT_NEG.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeNeg(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_NEG);
GenTree* tree = use.Def();
GenTree* gtLong = tree->gtGetOp1();
noway_assert(gtLong->OperGet() == GT_LONG);
unsigned blockWeight = m_block->getBBWeight(m_compiler);
LIR::Use op1(BlockRange(), >Long->gtOp.gtOp1, gtLong);
op1.ReplaceWithLclVar(m_compiler, blockWeight);
LIR::Use op2(BlockRange(), >Long->gtOp.gtOp2, gtLong);
op2.ReplaceWithLclVar(m_compiler, blockWeight);
// Neither GT_NEG nor the introduced temporaries have side effects.
tree->gtFlags &= ~GTF_ALL_EFFECT;
GenTree* loOp1 = gtLong->gtGetOp1();
GenTree* hiOp1 = gtLong->gtGetOp2();
BlockRange().Remove(gtLong);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
GenTree* zero = m_compiler->gtNewZeroConNode(TYP_INT);
GenTree* hiAdjust = m_compiler->gtNewOperNode(GT_ADD_HI, TYP_INT, hiOp1, zero);
GenTree* hiResult = m_compiler->gtNewOperNode(GT_NEG, TYP_INT, hiAdjust);
hiResult->gtFlags = tree->gtFlags;
// Annotate new nodes with costs. This will re-cost the hiOp1 tree as well.
m_compiler->gtPrepareCost(hiResult);
BlockRange().InsertAfter(loResult, zero, hiAdjust, hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeNot: Decompose GT_NOT.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeNot(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_NOT);
GenTree* tree = use.Def();
GenTree* gtLong = tree->gtGetOp1();
noway_assert(gtLong->OperGet() == GT_LONG);
GenTree* loOp1 = gtLong->gtGetOp1();
GenTree* hiOp1 = gtLong->gtGetOp2();
BlockRange().Remove(gtLong);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
loResult->gtOp.gtOp1 = loOp1;
GenTree* hiResult = new (m_compiler, GT_NOT) GenTreeOp(GT_NOT, TYP_INT, hiOp1, nullptr);
hiResult->CopyCosts(loResult);
BlockRange().InsertAfter(loResult, hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeCast: Decompose GT_CAST.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_CAST);
GenTree* tree = use.Def();
GenTree* loResult = nullptr;
GenTree* hiResult = nullptr;
assert(tree->gtPrev == tree->gtGetOp1());
NYI_IF(tree->gtOverflow(), "TYP_LONG cast with overflow");
switch (tree->AsCast()->CastFromType())
{
case TYP_INT:
if (tree->gtFlags & GTF_UNSIGNED)
{
loResult = tree->gtGetOp1();
BlockRange().Remove(tree);
hiResult = new (m_compiler, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
hiResult->CopyCosts(loResult);
BlockRange().InsertAfter(loResult, hiResult);
}
else
{
NYI("Lowering of signed cast TYP_INT->TYP_LONG");
}
break;
default:
NYI("Unimplemented type for Lowering of cast to TYP_LONG");
break;
}
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeLclFld: Decompose GT_LCL_FLD.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeLclFld(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_LCL_FLD);
GenTree* tree = use.Def();
GenTreeLclFld* loResult = tree->AsLclFld();
loResult->gtType = TYP_INT;
GenTree* hiResult = m_compiler->gtNewLclFldNode(loResult->gtLclNum, TYP_INT, loResult->gtLclOffs + 4);
hiResult->CopyCosts(loResult);
BlockRange().InsertAfter(loResult, hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// DecomposeCnsLng: Decompose GT_CNS_LNG.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeCnsLng(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_CNS_LNG);
GenTree* tree = use.Def();
INT32 hiVal = tree->AsLngCon()->HiVal();
GenTree* loResult = tree;
loResult->ChangeOperConst(GT_CNS_INT);
loResult->gtType = TYP_INT;
GenTree* hiResult = new (m_compiler, GT_CNS_INT) GenTreeIntCon(TYP_INT, hiVal);
hiResult->CopyCosts(loResult);
BlockRange().InsertAfter(loResult, hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
// Rewrite GT_OBJ of SIMD Vector as GT_IND(GT_LEA(obj.op1)) of a SIMD type.
//
// Arguments:
// ppTree - A pointer-to-a-pointer for the GT_OBJ
// fgWalkData - A pointer to tree walk data providing the context
//
// Return Value:
// None.
//
// TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer
// be required.
//
void Rationalizer::RewriteObj(LIR::Use& use)
{
#ifdef FEATURE_SIMD
GenTreeObj* obj = use.Def()->AsObj();
// For UNIX struct passing, we can have Obj nodes for arguments.
// For other cases, we should never see a non-SIMD type here.
#ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
if (!varTypeIsSIMD(obj))
{
return;
}
#endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
// Should come here only if featureSIMD is enabled
noway_assert(comp->featureSIMD);
// We should only call this with a SIMD type.
noway_assert(varTypeIsSIMD(obj));
var_types simdType = obj->TypeGet();
// If the operand of obj is a GT_ADDR(GT_LCL_VAR) and LclVar is known to be a SIMD type,
// replace obj by GT_LCL_VAR.
GenTree* srcAddr = obj->gtGetOp1();
if (srcAddr->OperIsLocalAddr() && comp->isAddrOfSIMDType(srcAddr))
{
BlockRange().Remove(obj);
srcAddr->SetOper(loadForm(srcAddr->OperGet()));
srcAddr->gtType = simdType;
use.ReplaceWith(comp, srcAddr);
}
else
{
obj->SetOper(GT_IND);
obj->gtType = simdType;
}
#else
// we should never reach without feature SIMD
assert(!"Unexpected obj during rationalization\n");
unreached();
#endif
}
//------------------------------------------------------------------------
// DecomposeLclVar: Decompose GT_LCL_VAR.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeLclVar(LIR::Use& use)
{
assert(use.IsInitialized());
assert(use.Def()->OperGet() == GT_LCL_VAR);
GenTree* tree = use.Def();
unsigned varNum = tree->AsLclVarCommon()->gtLclNum;
LclVarDsc* varDsc = m_compiler->lvaTable + varNum;
m_compiler->lvaDecRefCnts(tree);
GenTree* loResult = tree;
loResult->gtType = TYP_INT;
GenTree* hiResult = m_compiler->gtNewLclLNode(varNum, TYP_INT);
hiResult->CopyCosts(loResult);
BlockRange().InsertAfter(loResult, hiResult);
if (varDsc->lvPromoted)
{
assert(varDsc->lvFieldCnt == 2);
unsigned loVarNum = varDsc->lvFieldLclStart;
unsigned hiVarNum = loVarNum + 1;
loResult->AsLclVarCommon()->SetLclNum(loVarNum);
hiResult->AsLclVarCommon()->SetLclNum(hiVarNum);
}
else
{
noway_assert(varDsc->lvLRACandidate == false);
loResult->SetOper(GT_LCL_FLD);
loResult->AsLclFld()->gtLclOffs = 0;
loResult->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
hiResult->SetOper(GT_LCL_FLD);
hiResult->AsLclFld()->gtLclOffs = 4;
hiResult->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
}
m_compiler->lvaIncRefCnts(loResult);
m_compiler->lvaIncRefCnts(hiResult);
return FinalizeDecomposition(use, loResult, hiResult);
}
//------------------------------------------------------------------------
// LowerCast: Lower GT_CAST(srcType, DstType) nodes.
//
// Arguments:
// tree - GT_CAST node to be lowered
//
// Return Value:
// None.
//
// Notes:
// Casts from float/double to a smaller int type are transformed as follows:
// GT_CAST(float/double, byte) = GT_CAST(GT_CAST(float/double, int32), byte)
// GT_CAST(float/double, sbyte) = GT_CAST(GT_CAST(float/double, int32), sbyte)
// GT_CAST(float/double, int16) = GT_CAST(GT_CAST(double/double, int32), int16)
// GT_CAST(float/double, uint16) = GT_CAST(GT_CAST(double/double, int32), uint16)
//
// Note that for the overflow conversions we still depend on helper calls and
// don't expect to see them here.
// i) GT_CAST(float/double, int type with overflow detection)
//
void Lowering::LowerCast(GenTree* tree)
{
assert(tree->OperGet() == GT_CAST);
JITDUMP("LowerCast for: ");
DISPNODE(tree);
JITDUMP("\n");
GenTreePtr op1 = tree->gtOp.gtOp1;
var_types dstType = tree->CastToType();
var_types srcType = genActualType(op1->TypeGet());
var_types tmpType = TYP_UNDEF;
if (varTypeIsFloating(srcType))
{
noway_assert(!tree->gtOverflow());
}
assert(!varTypeIsSmall(srcType));
// case of src is a floating point type and dst is a small type.
if (varTypeIsFloating(srcType) && varTypeIsSmall(dstType))
{
NYI_ARM("Lowering for cast from float to small type"); // Not tested yet.
tmpType = TYP_INT;
}
if (tmpType != TYP_UNDEF)
{
GenTreePtr tmp = comp->gtNewCastNode(tmpType, op1, tmpType);
tmp->gtFlags |= (tree->gtFlags & (GTF_UNSIGNED | GTF_OVERFLOW | GTF_EXCEPT));
tree->gtFlags &= ~GTF_UNSIGNED;
tree->gtOp.gtOp1 = tmp;
BlockRange().InsertAfter(op1, tmp);
}
// Now determine if we have operands that should be contained.
ContainCheckCast(tree->AsCast());
}
// Rewrite a SIMD indirection as GT_IND(GT_LEA(obj.op1)), or as a simple
// lclVar if possible.
//
// Arguments:
// use - A use reference for a block node
// keepBlk - True if this should remain a block node if it is not a lclVar
//
// Return Value:
// None.
//
// TODO-1stClassStructs: These should be eliminated earlier, once we can handle
// lclVars in all the places that used to have GT_OBJ.
//
void Rationalizer::RewriteSIMDOperand(LIR::Use& use, bool keepBlk)
{
#ifdef FEATURE_SIMD
// No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
GenTree* tree = use.Def();
if (!tree->OperIsIndir())
{
return;
}
var_types simdType = tree->TypeGet();
if (!varTypeIsSIMD(simdType))
{
return;
}
// If the operand of is a GT_ADDR(GT_LCL_VAR) and LclVar is known to be of simdType,
// replace obj by GT_LCL_VAR.
GenTree* addr = tree->AsIndir()->Addr();
if (addr->OperIsLocalAddr() && comp->isAddrOfSIMDType(addr))
{
BlockRange().Remove(tree);
addr->SetOper(loadForm(addr->OperGet()));
addr->gtType = simdType;
use.ReplaceWith(comp, addr);
}
else if (!keepBlk)
{
tree->SetOper(GT_IND);
tree->gtType = simdType;
}
#endif // FEATURE_SIMD
}
//------------------------------------------------------------------------
// DecomposeNode: Decompose long-type trees into lower and upper halves.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
//
// Return Value:
// The next node to process.
//
GenTree* DecomposeLongs::DecomposeNode(LIR::Use& use)
{
GenTree* tree = use.Def();
// Handle the case where we are implicitly using the lower half of a long lclVar.
if ((tree->TypeGet() == TYP_INT) && tree->OperIsLocal())
{
LclVarDsc* varDsc = m_compiler->lvaTable + tree->AsLclVarCommon()->gtLclNum;
if (varTypeIsLong(varDsc) && varDsc->lvPromoted)
{
#ifdef DEBUG
if (m_compiler->verbose)
{
printf("Changing implicit reference to lo half of long lclVar to an explicit reference of its promoted "
"half:\n");
m_compiler->gtDispTreeRange(BlockRange(), tree);
}
#endif // DEBUG
m_compiler->lvaDecRefCnts(tree);
unsigned loVarNum = varDsc->lvFieldLclStart;
tree->AsLclVarCommon()->SetLclNum(loVarNum);
m_compiler->lvaIncRefCnts(tree);
return tree->gtNext;
}
}
if (tree->TypeGet() != TYP_LONG)
{
return tree->gtNext;
}
#ifdef DEBUG
if (m_compiler->verbose)
{
printf("Decomposing TYP_LONG tree. BEFORE:\n");
m_compiler->gtDispTreeRange(BlockRange(), tree);
}
#endif // DEBUG
GenTree* nextNode = nullptr;
switch (tree->OperGet())
{
case GT_PHI:
case GT_PHI_ARG:
nextNode = tree->gtNext;
break;
case GT_LCL_VAR:
nextNode = DecomposeLclVar(use);
break;
case GT_LCL_FLD:
nextNode = DecomposeLclFld(use);
break;
case GT_STORE_LCL_VAR:
nextNode = DecomposeStoreLclVar(use);
break;
case GT_CAST:
nextNode = DecomposeCast(use);
break;
case GT_CNS_LNG:
nextNode = DecomposeCnsLng(use);
break;
case GT_CALL:
nextNode = DecomposeCall(use);
break;
case GT_RETURN:
assert(tree->gtOp.gtOp1->OperGet() == GT_LONG);
break;
case GT_STOREIND:
nextNode = DecomposeStoreInd(use);
break;
case GT_STORE_LCL_FLD:
assert(tree->gtOp.gtOp1->OperGet() == GT_LONG);
NYI("st.lclFld of of TYP_LONG");
break;
case GT_IND:
nextNode = DecomposeInd(use);
break;
case GT_NOT:
nextNode = DecomposeNot(use);
break;
case GT_NEG:
nextNode = DecomposeNeg(use);
break;
// Binary operators. Those that require different computation for upper and lower half are
// handled by the use of GetHiOper().
case GT_ADD:
case GT_SUB:
case GT_OR:
case GT_XOR:
case GT_AND:
nextNode = DecomposeArith(use);
break;
case GT_MUL:
NYI("Arithmetic binary operators on TYP_LONG - GT_MUL");
break;
case GT_DIV:
NYI("Arithmetic binary operators on TYP_LONG - GT_DIV");
break;
case GT_MOD:
NYI("Arithmetic binary operators on TYP_LONG - GT_MOD");
break;
case GT_UDIV:
NYI("Arithmetic binary operators on TYP_LONG - GT_UDIV");
break;
case GT_UMOD:
NYI("Arithmetic binary operators on TYP_LONG - GT_UMOD");
break;
case GT_LSH:
case GT_RSH:
case GT_RSZ:
NYI("Arithmetic binary operators on TYP_LONG - SHIFT");
break;
case GT_ROL:
case GT_ROR:
NYI("Arithmetic binary operators on TYP_LONG - ROTATE");
break;
case GT_MULHI:
NYI("Arithmetic binary operators on TYP_LONG - MULHI");
break;
case GT_LOCKADD:
case GT_XADD:
case GT_XCHG:
case GT_CMPXCHG:
NYI("Interlocked operations on TYP_LONG");
break;
default:
{
JITDUMP("Illegal TYP_LONG node %s in Decomposition.", GenTree::NodeName(tree->OperGet()));
noway_assert(!"Illegal TYP_LONG node in Decomposition.");
break;
}
}
#ifdef DEBUG
if (m_compiler->verbose)
{
// NOTE: st_lcl_var doesn't dump properly afterwards.
printf("Decomposing TYP_LONG tree. AFTER:\n");
m_compiler->gtDispTreeRange(BlockRange(), use.Def());
}
#endif
return nextNode;
}
// Rewrite InitBlk involving SIMD vector into stlcl.var of a SIMD type.
//
// Arguments:
// ppTree - A pointer-to-a-pointer for the GT_INITBLK
// fgWalkData - A pointer to tree walk data providing the context
//
// Return Value:
// None.
//
// TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer
// be required.
//
void Rationalizer::RewriteInitBlk(LIR::Use& use)
{
#ifdef FEATURE_SIMD
// No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
// See if this is a SIMD initBlk that needs to be changed to a simple st.lclVar.
GenTreeInitBlk* initBlk = use.Def()->AsInitBlk();
// Is the dstAddr is addr of a SIMD type lclVar?
GenTree* dstAddr = initBlk->Dest();
if (!comp->isAddrOfSIMDType(dstAddr) || !dstAddr->OperIsLocalAddr())
{
return;
}
unsigned lclNum = dstAddr->AsLclVarCommon()->gtLclNum;
if (!comp->lvaTable[lclNum].lvSIMDType)
{
return;
}
var_types baseType = comp->lvaTable[lclNum].lvBaseType;
CORINFO_CLASS_HANDLE typeHnd = comp->lvaTable[lclNum].lvVerTypeInfo.GetClassHandle();
unsigned simdLocalSize = comp->getSIMDTypeSizeInBytes(typeHnd);
JITDUMP("Rewriting SIMD InitBlk\n");
DISPTREERANGE(BlockRange(), initBlk);
assert((dstAddr->gtFlags & GTF_VAR_USEASG) == 0);
// There are currently only three sizes supported: 8 bytes, 16 bytes or the vector register length.
GenTreeIntConCommon* sizeNode = initBlk->Size()->AsIntConCommon();
unsigned int size = (unsigned int)roundUp(sizeNode->IconValue(), TARGET_POINTER_SIZE);
var_types simdType = comp->getSIMDTypeForSize(size);
assert(roundUp(simdLocalSize, TARGET_POINTER_SIZE) == size);
GenTree* initVal = initBlk->InitVal();
GenTreeSIMD* simdNode = new (comp, GT_SIMD)
GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, (unsigned)sizeNode->IconValue());
dstAddr->SetOper(GT_STORE_LCL_VAR);
GenTreeLclVar* store = dstAddr->AsLclVar();
store->gtType = simdType;
store->gtOp.gtOp1 = simdNode;
store->gtFlags |= ((simdNode->gtFlags & GTF_ALL_EFFECT) | GTF_ASG);
BlockRange().Remove(store);
// Insert the new nodes into the block
BlockRange().InsertAfter(initVal, simdNode, store);
use.ReplaceWith(comp, store);
// Remove the old size and GT_INITBLK nodes.
BlockRange().Remove(sizeNode);
BlockRange().Remove(initBlk);
JITDUMP("After rewriting SIMD InitBlk:\n");
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
#endif // FEATURE_SIMD
}
void Rationalizer::DoPhase()
{
DBEXEC(TRUE, SanityCheck());
comp->compCurBB = nullptr;
comp->fgOrder = Compiler::FGOrderLinear;
BasicBlock* firstBlock = comp->fgFirstBB;
for (BasicBlock* block = comp->fgFirstBB; block != nullptr; block = block->bbNext)
{
comp->compCurBB = block;
m_block = block;
// Establish the first and last nodes for the block. This is necessary in order for the LIR
// utilities that hang off the BasicBlock type to work correctly.
GenTreeStmt* firstStatement = block->firstStmt();
if (firstStatement == nullptr)
{
// No statements in this block; skip it.
block->MakeLIR(nullptr, nullptr);
continue;
}
GenTreeStmt* lastStatement = block->lastStmt();
// Rewrite intrinsics that are not supported by the target back into user calls.
// This needs to be done before the transition to LIR because it relies on the use
// of fgMorphArgs, which is designed to operate on HIR. Once this is done for a
// particular statement, link that statement's nodes into the current basic block.
//
// This walk also clears the GTF_VAR_USEDEF bit on locals, which is not necessary
// in the backend.
GenTree* lastNodeInPreviousStatement = nullptr;
for (GenTreeStmt* statement = firstStatement; statement != nullptr; statement = statement->getNextStmt())
{
assert(statement->gtStmtList != nullptr);
assert(statement->gtStmtList->gtPrev == nullptr);
assert(statement->gtStmtExpr != nullptr);
assert(statement->gtStmtExpr->gtNext == nullptr);
SplitData splitData;
splitData.root = statement;
splitData.block = block;
splitData.thisPhase = this;
comp->fgWalkTreePost(&statement->gtStmtExpr,
[](GenTree** use, Compiler::fgWalkData* walkData) -> Compiler::fgWalkResult {
GenTree* node = *use;
if (node->OperGet() == GT_INTRINSIC &&
Compiler::IsIntrinsicImplementedByUserCall(node->gtIntrinsic.gtIntrinsicId))
{
RewriteIntrinsicAsUserCall(use, walkData);
}
else if (node->OperIsLocal())
{
node->gtFlags &= ~GTF_VAR_USEDEF;
}
return Compiler::WALK_CONTINUE;
},
&splitData, true);
GenTree* firstNodeInStatement = statement->gtStmtList;
if (lastNodeInPreviousStatement != nullptr)
{
lastNodeInPreviousStatement->gtNext = firstNodeInStatement;
}
firstNodeInStatement->gtPrev = lastNodeInPreviousStatement;
lastNodeInPreviousStatement = statement->gtStmtExpr;
}
block->MakeLIR(firstStatement->gtStmtList, lastStatement->gtStmtExpr);
// Rewrite HIR nodes into LIR nodes.
for (GenTreeStmt *statement = firstStatement, *nextStatement; statement != nullptr; statement = nextStatement)
{
nextStatement = statement->getNextStmt();
// If this statement has correct offset information, change it into an IL offset
// node and insert it into the LIR.
if (statement->gtStmtILoffsx != BAD_IL_OFFSET)
{
assert(!statement->IsPhiDefnStmt());
statement->SetOper(GT_IL_OFFSET);
statement->gtNext = nullptr;
statement->gtPrev = nullptr;
BlockRange().InsertBefore(statement->gtStmtList, statement);
}
m_statement = statement;
comp->fgWalkTreePost(&statement->gtStmtExpr,
[](GenTree** use, Compiler::fgWalkData* walkData) -> Compiler::fgWalkResult {
return reinterpret_cast<Rationalizer*>(walkData->pCallbackData)
->RewriteNode(use, *walkData->parentStack);
},
this, true);
}
assert(BlockRange().CheckLIR(comp));
}
comp->compRationalIRForm = true;
}
// Transform CopyBlk involving SIMD vectors into stlclvar or stind of a SIMD type.
// Transformation is done if either src or dst are known to be SIMD vectors.
//
// Arguments:
// ppTree - A pointer-to-a-pointer for the GT_COPYBLK
// fgWalkData - A pointer to tree walk data providing the context
//
// Return Value:
// None.
//
// If either the source or the dst are known to be SIMD (a lclVar or SIMD intrinsic),
// get the simdType (TYP_DOUBLE or a SIMD type for SSE2) from the size of the SIMD node.
//
// For the source:
// - If it is a SIMD intrinsic or a lvSIMDType lclVar, change the node type to simdType.
// - Otherwise, add a GT_IND of simdType.
// For the dst:
// - If it is a lclVar of a SIMD type, chanage the node type to simdType.
// - Otherwise, change it to a GT_STORE_IND of simdType
//
// TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer
// be required.
//
void Rationalizer::RewriteCopyBlk(LIR::Use& use)
{
#ifdef FEATURE_SIMD
// No need to transofrm non-SIMD nodes, if featureSIMD is not enabled.
if (!comp->featureSIMD)
{
return;
}
// See if this is a SIMD copyBlk
GenTreeCpBlk* cpBlk = use.Def()->AsCpBlk();
GenTreePtr dstAddr = cpBlk->Dest();
GenTree* srcAddr = cpBlk->Source();
const bool srcIsSIMDAddr = comp->isAddrOfSIMDType(srcAddr);
const bool dstIsSIMDAddr = comp->isAddrOfSIMDType(dstAddr);
// Do not transform if neither src or dst is known to be a SIMD type.
// If src tree type is something we cannot reason but if dst is known to be of a SIMD type
// we will treat src tree as a SIMD type and vice versa.
if (!srcIsSIMDAddr && !dstIsSIMDAddr)
{
return;
}
// At this point it is known to be a copyblk of SIMD vectors and we can
// start transforming the original tree. Prior to this point do not perform
// any modifications to the original tree.
JITDUMP("\nRewriting SIMD CopyBlk\n");
DISPTREERANGE(BlockRange(), cpBlk);
// There are currently only three sizes supported: 8 bytes, 12 bytes, 16 bytes or the vector register length.
GenTreeIntConCommon* sizeNode = cpBlk->Size()->AsIntConCommon();
var_types simdType = comp->getSIMDTypeForSize((unsigned int)sizeNode->IconValue());
// Remove 'size' from execution order
BlockRange().Remove(sizeNode);
// Is destination a lclVar which is not an arg?
// If yes then we can turn it to a stlcl.var, otherwise turn into stind.
GenTree* simdDst = nullptr;
genTreeOps oper = GT_NONE;
if (dstIsSIMDAddr && dstAddr->OperIsLocalAddr())
{
simdDst = dstAddr;
simdDst->gtType = simdType;
oper = GT_STORE_LCL_VAR;
// For structs that are padded (e.g. Vector3f, Vector3i), the morpher will have marked them
// as GTF_VAR_USEASG. Unmark them.
simdDst->gtFlags &= ~(GTF_VAR_USEASG);
}
else
{
// Address of a non-local var
simdDst = dstAddr;
oper = GT_STOREIND;
}
GenTree* simdSrc = nullptr;
if ((srcAddr->OperGet() == GT_ADDR) && varTypeIsSIMD(srcAddr->gtGetOp1()))
{
// Get rid of parent node of GT_ADDR(..) if its child happens to be of a SIMD type.
BlockRange().Remove(srcAddr);
simdSrc = srcAddr->gtGetOp1();
}
else if (srcIsSIMDAddr && srcAddr->OperIsLocalAddr())
{
// If the source has been rewritten into a local addr node, rewrite it back into a
// local var node.
simdSrc = srcAddr;
simdSrc->SetOper(loadForm(srcAddr->OperGet()));
}
else
{
// Since destination is known to be a SIMD type, src must be a SIMD type too
// though we cannot figure it out easily enough. Transform src into
// GT_IND(src) of simdType.
GenTree* indir = comp->gtNewOperNode(GT_IND, simdType, srcAddr);
BlockRange().InsertAfter(srcAddr, indir);
cpBlk->gtGetOp1()->gtOp.gtOp2 = indir;
simdSrc = indir;
}
simdSrc->gtType = simdType;
// Change cpblk to either a st.lclvar or st.ind.
// At this point we are manipulating cpblk node with the knowledge of
// its internals (i.e. op1 is the size node, and the src & dst are in a GT_LIST on op2).
// This logic might need to be changed if we ever restructure cpblk node.
assert(simdDst != nullptr);
assert(simdSrc != nullptr);
GenTree* newNode = nullptr;
if (oper == GT_STORE_LCL_VAR)
{
newNode = simdDst;
newNode->SetOper(oper);
GenTreeLclVar* store = newNode->AsLclVar();
store->gtOp1 = simdSrc;
store->gtType = simdType;
store->gtFlags |= ((simdSrc->gtFlags & GTF_ALL_EFFECT) | GTF_ASG);
BlockRange().Remove(simdDst);
BlockRange().InsertAfter(simdSrc, store);
}
else
{
assert(oper == GT_STOREIND);
newNode = cpBlk->gtGetOp1();
newNode->SetOper(oper);
GenTreeStoreInd* storeInd = newNode->AsStoreInd();
storeInd->gtType = simdType;
storeInd->gtFlags |= ((simdSrc->gtFlags & GTF_ALL_EFFECT) | GTF_ASG);
storeInd->gtOp1 = simdDst;
storeInd->gtOp2 = simdSrc;
BlockRange().InsertBefore(cpBlk, storeInd);
}
use.ReplaceWith(comp, newNode);
BlockRange().Remove(cpBlk);
JITDUMP("After rewriting SIMD CopyBlk:\n");
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
#endif // FEATURE_SIMD
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding list nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_FIELD_LIST head nodes, and GT_LIST nodes used by phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperIsAnyList() && !(prev->OperIsFieldListHead());
prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node that is not an aggregate.
if (node->OperIsAnyList())
{
GenTreeArgList* list = node->AsArgList();
if (!list->OperIsFieldListHead())
{
BlockRange().Remove(list);
}
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_IND:
// Clear the `GTF_IND_ASG_LHS` flag, which overlaps with `GTF_IND_REQ_ADDR_IN_REG`.
node->gtFlags &= ~GTF_IND_ASG_LHS;
if (varTypeIsSIMD(node))
{
RewriteSIMDOperand(use, false);
}
else
{
// Due to promotion of structs containing fields of type struct with a
// single scalar type field, we could potentially see IR nodes of the
// form GT_IND(GT_ADD(lclvarAddr, 0)) where 0 is an offset representing
// a field-seq. These get folded here.
//
// TODO: This code can be removed once JIT implements recursive struct
// promotion instead of lying about the type of struct field as the type
// of its single scalar field.
GenTree* addr = node->AsIndir()->Addr();
if (addr->OperGet() == GT_ADD && addr->gtGetOp1()->OperGet() == GT_LCL_VAR_ADDR &&
addr->gtGetOp2()->IsIntegralConst(0))
{
GenTreeLclVarCommon* lclVarNode = addr->gtGetOp1()->AsLclVarCommon();
unsigned lclNum = lclVarNode->GetLclNum();
LclVarDsc* varDsc = comp->lvaTable + lclNum;
if (node->TypeGet() == varDsc->TypeGet())
{
JITDUMP("Rewriting GT_IND(GT_ADD(LCL_VAR_ADDR,0)) to LCL_VAR\n");
lclVarNode->SetOper(GT_LCL_VAR);
lclVarNode->gtType = node->TypeGet();
use.ReplaceWith(comp, lclVarNode);
BlockRange().Remove(addr);
BlockRange().Remove(addr->gtGetOp2());
BlockRange().Remove(node);
}
}
}
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#if defined(_TARGET_XARCH_) || defined(_TARGET_ARM_)
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_BLK:
case GT_OBJ:
{
// TODO-1stClassStructs: These should have been transformed to GT_INDs, but in order
// to preserve existing behavior, we will keep this as a block node if this is the
// lhs of a block assignment, and either:
// - It is a "generic" TYP_STRUCT assignment, OR
// - It is an initblk, OR
// - Neither the lhs or rhs are known to be of SIMD type.
GenTree* parent = use.User();
bool keepBlk = false;
if ((parent->OperGet() == GT_ASG) && (node == parent->gtGetOp1()))
{
if ((node->TypeGet() == TYP_STRUCT) || parent->OperIsInitBlkOp())
{
keepBlk = true;
}
else if (!comp->isAddrOfSIMDType(node->AsBlk()->Addr()))
{
GenTree* dataSrc = parent->gtGetOp2();
if (!dataSrc->IsLocal() && (dataSrc->OperGet() != GT_SIMD))
{
noway_assert(dataSrc->OperIsIndir());
keepBlk = !comp->isAddrOfSIMDType(dataSrc->AsIndir()->Addr());
}
}
}
RewriteSIMDOperand(use, keepBlk);
}
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2IfPresent();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
// JCC nodes should not be present in HIR.
assert(node->OperGet() != GT_JCC);
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (node->OperIsLocalRead())
{
if (use.IsDummyUse())
{
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
else
{
// Local reads are side-effect-free; clear any flags leftover from frontend transformations.
node->gtFlags &= ~GTF_ALL_EFFECT;
}
}
assert(isLateArg == ((use.Def()->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}
//virtual
int DVDStream::safe_read(void *data, uint size)
{
uint32_t lb = size / DVD_VIDEO_LB_LEN;
if (lb < 1)
{
LOG(VB_GENERAL, LOG_ERR, "DVDStream::safe_read too small");
return -1;
}
if (!m_reader)
return -1;
int ret = 0;
// Are any blocks in the range encrypted?
list_t::const_iterator it = qBinaryFind(m_list, BlockRange(m_pos, lb, -1));
uint32_t b = it == m_list.end() ? lb : m_pos < it->Start() ? it->Start() - m_pos : 0;
if (b)
{
// Read the beginning unencrypted blocks
ret = UDFReadBlocksRaw(m_reader, m_pos, b, (unsigned char*)data, DVDINPUT_NOFLAGS);
if (ret == -1)
{
LOG(VB_GENERAL, LOG_ERR, "DVDStream::safe_read DVDReadBlocks error");
return -1;
}
m_pos += ret;
lb -= ret;
if (it == m_list.end())
return ret * DVD_VIDEO_LB_LEN;
data = (unsigned char*)data + ret * DVD_VIDEO_LB_LEN;
}
b = it->End() - m_pos;
if (b > lb)
b = lb;
// Request new key if change in title
int flags = DVDINPUT_READ_DECRYPT;
if (it->Title() != m_title)
{
m_title = it->Title();
flags |= DVDCSS_SEEK_KEY;
}
// Read the encrypted blocks
int ret2 = UDFReadBlocksRaw(m_reader, m_pos + m_start, b, (unsigned char*)data, flags);
if (ret2 == -1)
{
LOG(VB_GENERAL, LOG_ERR, "DVDStream::safe_read DVDReadBlocks error");
m_title = -1;
return -1;
}
m_pos += ret2;
ret += ret2;
lb -= ret2;
data = (unsigned char*)data + ret2 * DVD_VIDEO_LB_LEN;
if (lb > 0 && m_start == 0)
{
// Read the last unencrypted blocks
ret2 = UDFReadBlocksRaw(m_reader, m_pos, lb, (unsigned char*)data, DVDINPUT_NOFLAGS);
if (ret2 == -1)
{
LOG(VB_GENERAL, LOG_ERR, "DVDStream::safe_read DVDReadBlocks error");
return -1;
}
m_pos += ret2;
ret += ret2;;
}
return ret * DVD_VIDEO_LB_LEN;
}
bool DVDStream::OpenFile(const QString &filename, uint /*retry_ms*/)
{
rwlock.lockForWrite();
const QString root = filename.section("/VIDEO_TS/", 0, 0);
const QString path = filename.section(root, 1);
if (m_reader)
DVDClose(m_reader);
m_reader = DVDOpen(qPrintable(root));
if (!m_reader)
{
LOG(VB_GENERAL, LOG_ERR, QString("DVDStream DVDOpen(%1) failed").arg(filename));
rwlock.unlock();
return false;
}
if (!path.isEmpty())
{
// Locate the start block of the requested title
uint32_t len;
m_start = UDFFindFile(m_reader, const_cast<char*>(qPrintable(path)), &len);
if (m_start == 0)
{
LOG(VB_GENERAL, LOG_ERR, QString("DVDStream(%1) UDFFindFile(%2) failed").
arg(root).arg(path));
DVDClose(m_reader);
m_reader = 0;
rwlock.unlock();
return false;
}
else
{
m_list.append(BlockRange(0, Len2Blocks(len), 0));
}
}
else
{
// Create a list of the possibly encrypted files
uint32_t len, start;
// Root menu
char name[64] = "VIDEO_TS/VIDEO_TS.VOB";
start = UDFFindFile(m_reader, name, &len);
if( start != 0 && len != 0 )
m_list.append(BlockRange(start, Len2Blocks(len), 0));
const int kTitles = 100;
for ( int title = 1; title < kTitles; ++title)
{
// Menu
snprintf(name, sizeof name, "/VIDEO_TS/VTS_%02d_0.VOB", title);
start = UDFFindFile(m_reader, name, &len);
if( start != 0 && len != 0 )
m_list.append(BlockRange(start, Len2Blocks(len), title));
for ( int part = 1; part < 10; ++part)
{
// A/V track
snprintf(name, sizeof name, "/VIDEO_TS/VTS_%02d_%d.VOB", title, part);
start = UDFFindFile(m_reader, name, &len);
if( start != 0 && len != 0 )
m_list.append(BlockRange(start, Len2Blocks(len), title + part * kTitles));
}
}
qSort( m_list);
// Open the root menu so that CSS keys are generated now
dvd_file_t *file = DVDOpenFile(m_reader, 0, DVD_READ_MENU_VOBS);
if (file)
DVDCloseFile(file);
else
LOG(VB_GENERAL, LOG_ERR, "DVDStream DVDOpenFile(VOBS_1) failed");
}
rwlock.unlock();
return true;
}
void Rationalizer::RewriteAssignment(LIR::Use& use)
{
assert(use.IsInitialized());
GenTreeOp* assignment = use.Def()->AsOp();
assert(assignment->OperGet() == GT_ASG);
GenTree* location = assignment->gtGetOp1();
GenTree* value = assignment->gtGetOp2();
genTreeOps locationOp = location->OperGet();
if (assignment->OperIsBlkOp())
{
#ifdef FEATURE_SIMD
if (varTypeIsSIMD(location) && assignment->OperIsInitBlkOp())
{
if (location->OperGet() == GT_LCL_VAR)
{
var_types simdType = location->TypeGet();
GenTree* initVal = assignment->gtOp.gtOp2;
var_types baseType = comp->getBaseTypeOfSIMDLocal(location);
if (baseType != TYP_UNKNOWN)
{
GenTreeSIMD* simdTree = new (comp, GT_SIMD)
GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, genTypeSize(simdType));
assignment->gtOp.gtOp2 = simdTree;
value = simdTree;
initVal->gtNext = simdTree;
simdTree->gtPrev = initVal;
simdTree->gtNext = location;
location->gtPrev = simdTree;
}
}
}
#endif // FEATURE_SIMD
if ((location->TypeGet() == TYP_STRUCT) && !assignment->IsPhiDefn() && !value->IsMultiRegCall())
{
if ((location->OperGet() == GT_LCL_VAR))
{
// We need to construct a block node for the location.
// Modify lcl to be the address form.
location->SetOper(addrForm(locationOp));
LclVarDsc* varDsc = &(comp->lvaTable[location->AsLclVarCommon()->gtLclNum]);
location->gtType = TYP_BYREF;
GenTreeBlk* storeBlk = nullptr;
unsigned int size = varDsc->lvExactSize;
if (varDsc->lvStructGcCount != 0)
{
CORINFO_CLASS_HANDLE structHnd = varDsc->lvVerTypeInfo.GetClassHandle();
GenTreeObj* objNode = comp->gtNewObjNode(structHnd, location)->AsObj();
unsigned int slots = (unsigned)(roundUp(size, TARGET_POINTER_SIZE) / TARGET_POINTER_SIZE);
objNode->SetGCInfo(varDsc->lvGcLayout, varDsc->lvStructGcCount, slots);
objNode->ChangeOper(GT_STORE_OBJ);
objNode->SetData(value);
comp->fgMorphUnsafeBlk(objNode);
storeBlk = objNode;
}
else
{
storeBlk = new (comp, GT_STORE_BLK) GenTreeBlk(GT_STORE_BLK, TYP_STRUCT, location, value, size);
}
storeBlk->gtFlags |= (GTF_REVERSE_OPS | GTF_ASG);
storeBlk->gtFlags |= ((location->gtFlags | value->gtFlags) & GTF_ALL_EFFECT);
GenTree* insertionPoint = location->gtNext;
BlockRange().InsertBefore(insertionPoint, storeBlk);
use.ReplaceWith(comp, storeBlk);
BlockRange().Remove(assignment);
JITDUMP("After transforming local struct assignment into a block op:\n");
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
return;
}
else
{
assert(location->OperIsBlk());
}
}
}
switch (locationOp)
{
case GT_LCL_VAR:
case GT_LCL_FLD:
case GT_REG_VAR:
case GT_PHI_ARG:
RewriteAssignmentIntoStoreLclCore(assignment, location, value, locationOp);
BlockRange().Remove(location);
break;
case GT_IND:
{
GenTreeStoreInd* store =
new (comp, GT_STOREIND) GenTreeStoreInd(location->TypeGet(), location->gtGetOp1(), value);
copyFlags(store, assignment, GTF_ALL_EFFECT);
copyFlags(store, location, GTF_IND_FLAGS);
if (assignment->IsReverseOp())
{
store->gtFlags |= GTF_REVERSE_OPS;
}
// TODO: JIT dump
// Remove the GT_IND node and replace the assignment node with the store
BlockRange().Remove(location);
BlockRange().InsertBefore(assignment, store);
use.ReplaceWith(comp, store);
BlockRange().Remove(assignment);
}
break;
case GT_CLS_VAR:
{
location->SetOper(GT_CLS_VAR_ADDR);
location->gtType = TYP_BYREF;
assignment->SetOper(GT_STOREIND);
// TODO: JIT dump
}
break;
case GT_BLK:
case GT_OBJ:
case GT_DYN_BLK:
{
assert(varTypeIsStruct(location));
GenTreeBlk* storeBlk = location->AsBlk();
genTreeOps storeOper;
switch (location->gtOper)
{
case GT_BLK:
storeOper = GT_STORE_BLK;
break;
case GT_OBJ:
storeOper = GT_STORE_OBJ;
break;
case GT_DYN_BLK:
storeOper = GT_STORE_DYN_BLK;
break;
default:
unreached();
}
JITDUMP("Rewriting GT_ASG(%s(X), Y) to %s(X,Y):\n", GenTree::NodeName(location->gtOper),
GenTree::NodeName(storeOper));
storeBlk->SetOperRaw(storeOper);
storeBlk->gtFlags &= ~GTF_DONT_CSE;
storeBlk->gtFlags |= (assignment->gtFlags & (GTF_ALL_EFFECT | GTF_REVERSE_OPS | GTF_BLK_VOLATILE |
GTF_BLK_UNALIGNED | GTF_DONT_CSE));
storeBlk->gtBlk.Data() = value;
// Replace the assignment node with the store
use.ReplaceWith(comp, storeBlk);
BlockRange().Remove(assignment);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
break;
default:
unreached();
break;
}
}
void Rationalizer::DoPhase()
{
class RationalizeVisitor final : public GenTreeVisitor<RationalizeVisitor>
{
Rationalizer& m_rationalizer;
public:
enum
{
ComputeStack = true,
DoPreOrder = true,
DoPostOrder = true,
UseExecutionOrder = true,
};
RationalizeVisitor(Rationalizer& rationalizer)
: GenTreeVisitor<RationalizeVisitor>(rationalizer.comp), m_rationalizer(rationalizer)
{
}
// Rewrite intrinsics that are not supported by the target back into user calls.
// This needs to be done before the transition to LIR because it relies on the use
// of fgMorphArgs, which is designed to operate on HIR. Once this is done for a
// particular statement, link that statement's nodes into the current basic block.
fgWalkResult PreOrderVisit(GenTree** use, GenTree* user)
{
GenTree* const node = *use;
if (node->OperGet() == GT_INTRINSIC &&
Compiler::IsIntrinsicImplementedByUserCall(node->gtIntrinsic.gtIntrinsicId))
{
m_rationalizer.RewriteIntrinsicAsUserCall(use, this->m_ancestors);
}
return Compiler::WALK_CONTINUE;
}
// Rewrite HIR nodes into LIR nodes.
fgWalkResult PostOrderVisit(GenTree** use, GenTree* user)
{
return m_rationalizer.RewriteNode(use, this->m_ancestors);
}
};
DBEXEC(TRUE, SanityCheck());
comp->compCurBB = nullptr;
comp->fgOrder = Compiler::FGOrderLinear;
RationalizeVisitor visitor(*this);
for (BasicBlock* block = comp->fgFirstBB; block != nullptr; block = block->bbNext)
{
comp->compCurBB = block;
m_block = block;
GenTreeStmt* firstStatement = block->firstStmt();
block->MakeLIR(nullptr, nullptr);
// Establish the first and last nodes for the block. This is necessary in order for the LIR
// utilities that hang off the BasicBlock type to work correctly.
if (firstStatement == nullptr)
{
// No statements in this block; skip it.
continue;
}
for (GenTreeStmt *statement = firstStatement, *nextStatement; statement != nullptr; statement = nextStatement)
{
assert(statement->gtStmtList != nullptr);
assert(statement->gtStmtList->gtPrev == nullptr);
assert(statement->gtStmtExpr != nullptr);
assert(statement->gtStmtExpr->gtNext == nullptr);
BlockRange().InsertAtEnd(LIR::Range(statement->gtStmtList, statement->gtStmtExpr));
nextStatement = statement->getNextStmt();
statement->gtNext = nullptr;
statement->gtPrev = nullptr;
// If this statement has correct offset information, change it into an IL offset
// node and insert it into the LIR.
if (statement->gtStmtILoffsx != BAD_IL_OFFSET)
{
assert(!statement->IsPhiDefnStmt());
statement->SetOper(GT_IL_OFFSET);
BlockRange().InsertBefore(statement->gtStmtList, statement);
}
m_block = block;
visitor.WalkTree(&statement->gtStmtExpr, nullptr);
}
assert(BlockRange().CheckLIR(comp, true));
}
comp->compRationalIRForm = true;
}
void Rationalizer::RewriteAssignment(LIR::Use& use)
{
assert(use.IsInitialized());
GenTreeOp* assignment = use.Def()->AsOp();
assert(assignment->OperGet() == GT_ASG);
GenTree* location = assignment->gtGetOp1();
GenTree* value = assignment->gtGetOp2();
genTreeOps locationOp = location->OperGet();
#ifdef FEATURE_SIMD
if (varTypeIsSIMD(location) && assignment->OperIsInitBlkOp())
{
if (location->OperGet() == GT_LCL_VAR)
{
var_types simdType = location->TypeGet();
GenTree* initVal = assignment->gtOp.gtOp2;
var_types baseType = comp->getBaseTypeOfSIMDLocal(location);
if (baseType != TYP_UNKNOWN)
{
GenTreeSIMD* simdTree = new (comp, GT_SIMD)
GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, genTypeSize(simdType));
assignment->gtOp.gtOp2 = simdTree;
value = simdTree;
initVal->gtNext = simdTree;
simdTree->gtPrev = initVal;
simdTree->gtNext = location;
location->gtPrev = simdTree;
}
}
else
{
assert(location->OperIsBlk());
}
}
#endif // FEATURE_SIMD
switch (locationOp)
{
case GT_LCL_VAR:
case GT_LCL_FLD:
case GT_REG_VAR:
case GT_PHI_ARG:
RewriteAssignmentIntoStoreLclCore(assignment, location, value, locationOp);
BlockRange().Remove(location);
break;
case GT_IND:
{
GenTreeStoreInd* store =
new (comp, GT_STOREIND) GenTreeStoreInd(location->TypeGet(), location->gtGetOp1(), value);
copyFlags(store, assignment, GTF_ALL_EFFECT);
copyFlags(store, location, GTF_IND_FLAGS);
if (assignment->IsReverseOp())
{
store->gtFlags |= GTF_REVERSE_OPS;
}
// TODO: JIT dump
// Remove the GT_IND node and replace the assignment node with the store
BlockRange().Remove(location);
BlockRange().InsertBefore(assignment, store);
use.ReplaceWith(comp, store);
BlockRange().Remove(assignment);
}
break;
case GT_CLS_VAR:
{
location->SetOper(GT_CLS_VAR_ADDR);
location->gtType = TYP_BYREF;
assignment->SetOper(GT_STOREIND);
// TODO: JIT dump
}
break;
case GT_BLK:
case GT_OBJ:
case GT_DYN_BLK:
{
assert(varTypeIsStruct(location));
GenTreeBlk* storeBlk = location->AsBlk();
genTreeOps storeOper;
switch (location->gtOper)
{
case GT_BLK:
storeOper = GT_STORE_BLK;
break;
case GT_OBJ:
storeOper = GT_STORE_OBJ;
break;
case GT_DYN_BLK:
storeOper = GT_STORE_DYN_BLK;
break;
default:
unreached();
}
JITDUMP("Rewriting GT_ASG(%s(X), Y) to %s(X,Y):\n", GenTree::NodeName(location->gtOper),
GenTree::NodeName(storeOper));
storeBlk->SetOperRaw(storeOper);
storeBlk->gtFlags &= ~GTF_DONT_CSE;
storeBlk->gtFlags |= (assignment->gtFlags & (GTF_ALL_EFFECT | GTF_REVERSE_OPS | GTF_BLK_VOLATILE |
GTF_BLK_UNALIGNED | GTF_BLK_INIT | GTF_DONT_CSE));
storeBlk->gtBlk.Data() = value;
// Replace the assignment node with the store
use.ReplaceWith(comp, storeBlk);
BlockRange().Remove(assignment);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
break;
default:
unreached();
break;
}
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding GT_LIST nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_LIST nodes that are used by block ops and phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperGet() == GT_LIST; prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node.
if ((*useEdge)->OperGet() == GT_LIST)
{
BlockRange().Remove(*useEdge);
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#ifdef _TARGET_XARCH_
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_INITBLK:
RewriteInitBlk(use);
break;
case GT_COPYBLK:
RewriteCopyBlk(use);
break;
case GT_OBJ:
RewriteObj(use);
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_STOREIND:
case GT_IND:
if (node->gtType == TYP_STRUCT)
{
GenTree* addr = node->AsIndir()->Addr();
assert(addr->TypeGet() == TYP_BYREF);
if (addr->OperIsLocal())
{
LclVarDsc* varDsc = &(comp->lvaTable[addr->AsLclVarCommon()->gtLclNum]);
assert(varDsc->lvSIMDType);
unsigned simdSize = (unsigned int)roundUp(varDsc->lvExactSize, TARGET_POINTER_SIZE);
node->gtType = comp->getSIMDTypeForSize(simdSize);
}
#if DEBUG
else
{
// If the address is not a local var, assert that the user of this IND is an ADDR node.
assert((use.User()->OperGet() == GT_ADDR) || use.User()->OperIsLocalAddr());
}
#endif
}
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
else if (simdNode->gtType == TYP_STRUCT || varTypeIsSIMD(simdNode))
{
node->gtType = simdType;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (use.IsDummyUse() && node->OperIsLocalRead())
{
assert((node->gtFlags & GTF_ALL_EFFECT) == 0);
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
assert(isLateArg == ((node->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}
