//------------------------------------------------------------------------
// LowerStoreLoc: Lower a store of a lclVar
//
// Arguments:
// storeLoc - the local store (GT_STORE_LCL_FLD or GT_STORE_LCL_VAR)
//
// Notes:
// This involves:
// - Widening operations of unsigneds.
//
void Lowering::LowerStoreLoc(GenTreeLclVarCommon* storeLoc)
{
// Try to widen the ops if they are going into a local var.
GenTree* op1 = storeLoc->gtGetOp1();
if ((storeLoc->gtOper == GT_STORE_LCL_VAR) && (op1->gtOper == GT_CNS_INT))
{
GenTreeIntCon* con = op1->AsIntCon();
ssize_t ival = con->gtIconVal;
unsigned varNum = storeLoc->gtLclNum;
LclVarDsc* varDsc = comp->lvaTable + varNum;
if (varDsc->lvIsSIMDType())
{
noway_assert(storeLoc->gtType != TYP_STRUCT);
}
unsigned size = genTypeSize(storeLoc);
// If we are storing a constant into a local variable
// we extend the size of the store here
if ((size < 4) && !varTypeIsStruct(varDsc))
{
if (!varTypeIsUnsigned(varDsc))
{
if (genTypeSize(storeLoc) == 1)
{
if ((ival & 0x7f) != ival)
{
ival = ival | 0xffffff00;
}
}
else
{
assert(genTypeSize(storeLoc) == 2);
if ((ival & 0x7fff) != ival)
{
ival = ival | 0xffff0000;
}
}
}
// A local stack slot is at least 4 bytes in size, regardless of
// what the local var is typed as, so auto-promote it here
// unless it is a field of a promoted struct
// TODO-CQ: if the field is promoted shouldn't we also be able to do this?
if (!varDsc->lvIsStructField)
{
storeLoc->gtType = TYP_INT;
con->SetIconValue(ival);
}
}
}
if (storeLoc->OperIs(GT_STORE_LCL_FLD))
{
// We should only encounter this for lclVars that are lvDoNotEnregister.
verifyLclFldDoNotEnregister(storeLoc->gtLclNum);
}
ContainCheckStoreLoc(storeLoc);
}
/*****************************************************************************
* gsParamsToShadows
* Copy each vulnerable param ptr or buffer to a local shadow copy and replace
* uses of the param by the shadow copy
*/
void Compiler::gsParamsToShadows()
{
// Cache old count since we'll add new variables, and
// gsShadowVarInfo will not grow to accomodate the new ones.
UINT lvaOldCount = lvaCount;
// Create shadow copy for each param candidate
for (UINT lclNum = 0; lclNum < lvaOldCount; lclNum++)
{
LclVarDsc *varDsc = &lvaTable[lclNum];
gsShadowVarInfo[lclNum].shadowCopy = NO_SHADOW_COPY;
// Only care about params whose values are on the stack
if (!ShadowParamVarInfo::mayNeedShadowCopy(varDsc))
{
continue;
}
if (!varDsc->lvIsPtr && !varDsc->lvIsUnsafeBuffer)
{
continue;
}
int shadowVar = lvaGrabTemp(false DEBUGARG("shadowVar"));
// Copy some info
var_types type = varTypeIsSmall(varDsc->TypeGet()) ? TYP_INT : varDsc->TypeGet();
lvaTable[shadowVar].lvType = type;
lvaTable[shadowVar].lvAddrExposed = varDsc->lvAddrExposed;
lvaTable[shadowVar].lvDoNotEnregister = varDsc->lvDoNotEnregister;
#ifdef DEBUG
lvaTable[shadowVar].lvVMNeedsStackAddr = varDsc->lvVMNeedsStackAddr;
lvaTable[shadowVar].lvLiveInOutOfHndlr = varDsc->lvLiveInOutOfHndlr;
lvaTable[shadowVar].lvLclFieldExpr = varDsc->lvLclFieldExpr;
lvaTable[shadowVar].lvLiveAcrossUCall = varDsc->lvLiveAcrossUCall;
#endif
lvaTable[shadowVar].lvVerTypeInfo = varDsc->lvVerTypeInfo;
lvaTable[shadowVar].lvGcLayout = varDsc->lvGcLayout;
lvaTable[shadowVar].lvIsUnsafeBuffer = varDsc->lvIsUnsafeBuffer;
lvaTable[shadowVar].lvIsPtr = varDsc->lvIsPtr;
#ifdef DEBUG
if (verbose)
{
printf("Var V%02u is shadow param candidate. Shadow copy is V%02u.\n", lclNum, shadowVar);
}
#endif
gsShadowVarInfo[lclNum].shadowCopy = shadowVar;
}
// Replace param uses with shadow copy
fgWalkAllTreesPre(gsReplaceShadowParams, (void *)this);
// Now insert code to copy the params to their shadow copy.
for (UINT lclNum = 0; lclNum < lvaOldCount; lclNum++)
{
LclVarDsc *varDsc = &lvaTable[lclNum];
unsigned shadowVar = gsShadowVarInfo[lclNum].shadowCopy;
if (shadowVar == NO_SHADOW_COPY)
{
continue;
}
var_types type = lvaTable[shadowVar].TypeGet();
GenTreePtr src = gtNewLclvNode(lclNum, varDsc->TypeGet());
GenTreePtr dst = gtNewLclvNode(shadowVar, type);
src->gtFlags |= GTF_DONT_CSE;
dst->gtFlags |= GTF_DONT_CSE;
GenTreePtr opAssign = NULL;
if (type == TYP_STRUCT)
{
CORINFO_CLASS_HANDLE clsHnd = varDsc->lvVerTypeInfo.GetClassHandle();
// We don't need unsafe value cls check here since we are copying the params and this flag
// would have been set on the original param before reaching here.
lvaSetStruct(shadowVar, clsHnd, false);
src = gtNewOperNode(GT_ADDR, TYP_BYREF, src);
dst = gtNewOperNode(GT_ADDR, TYP_BYREF, dst);
opAssign = gtNewCpObjNode(dst, src, clsHnd, false);
#if FEATURE_MULTIREG_ARGS_OR_RET
lvaTable[shadowVar].lvIsMultiRegArgOrRet = lvaTable[lclNum].lvIsMultiRegArgOrRet;
#endif // FEATURE_MULTIREG_ARGS_OR_RET
}
else
{
opAssign = gtNewAssignNode(dst, src);
}
fgEnsureFirstBBisScratch();
(void) fgInsertStmtAtBeg(fgFirstBB, fgMorphTree(opAssign));
}
// If the method has "Jmp CalleeMethod", then we need to copy shadow params back to original
// params before "jmp" to CalleeMethod.
if (compJmpOpUsed)
{
// There could be more than one basic block ending with a "Jmp" type tail call.
// We would have to insert assignments in all such blocks, just before GT_JMP stmnt.
for (BasicBlock * block = fgFirstBB; block; block = block->bbNext)
{
if (block->bbJumpKind != BBJ_RETURN)
{
continue;
}
if ((block->bbFlags & BBF_HAS_JMP) == 0)
{
continue;
}
for (UINT lclNum = 0; lclNum < info.compArgsCount; lclNum++)
{
LclVarDsc *varDsc = &lvaTable[lclNum];
unsigned shadowVar = gsShadowVarInfo[lclNum].shadowCopy;
if (shadowVar == NO_SHADOW_COPY)
{
continue;
}
GenTreePtr src = gtNewLclvNode(shadowVar, lvaTable[shadowVar].TypeGet());
GenTreePtr dst = gtNewLclvNode(lclNum, varDsc->TypeGet());
src->gtFlags |= GTF_DONT_CSE;
dst->gtFlags |= GTF_DONT_CSE;
GenTreePtr opAssign = nullptr;
if (varDsc->TypeGet() == TYP_STRUCT)
{
CORINFO_CLASS_HANDLE clsHnd = varDsc->lvVerTypeInfo.GetClassHandle();
src = gtNewOperNode(GT_ADDR, TYP_BYREF, src);
dst = gtNewOperNode(GT_ADDR, TYP_BYREF, dst);
opAssign = gtNewCpObjNode(dst, src, clsHnd, false);
}
else
{
opAssign = gtNewAssignNode(dst, src);
}
(void) fgInsertStmtNearEnd(block, fgMorphTree(opAssign));
}
}
}
}
void Compiler::raMarkStkVars()
{
unsigned lclNum;
LclVarDsc* varDsc;
for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
{
// lvOnFrame is set by LSRA, except in the case of zero-ref, which is set below.
if (lvaIsFieldOfDependentlyPromotedStruct(varDsc))
{
noway_assert(!varDsc->lvRegister);
goto ON_STK;
}
/* Fully enregistered variables don't need any frame space */
if (varDsc->lvRegister)
{
goto NOT_STK;
}
/* Unused variables typically don't get any frame space */
else if (varDsc->lvRefCnt() == 0)
{
bool needSlot = false;
bool stkFixedArgInVarArgs =
info.compIsVarArgs && varDsc->lvIsParam && !varDsc->lvIsRegArg && lclNum != lvaVarargsHandleArg;
// If its address has been exposed, ignore lvRefCnt. However, exclude
// fixed arguments in varargs method as lvOnFrame shouldn't be set
// for them as we don't want to explicitly report them to GC.
if (!stkFixedArgInVarArgs)
{
needSlot |= varDsc->lvAddrExposed;
}
#if FEATURE_FIXED_OUT_ARGS
/* Is this the dummy variable representing GT_LCLBLK ? */
needSlot |= (lclNum == lvaOutgoingArgSpaceVar);
#endif // FEATURE_FIXED_OUT_ARGS
#ifdef DEBUG
/* For debugging, note that we have to reserve space even for
unused variables if they are ever in scope. However, this is not
an issue as fgExtendDbgLifetimes() adds an initialization and
variables in scope will not have a zero ref-cnt.
*/
if (opts.compDbgCode && !varDsc->lvIsParam && varDsc->lvTracked)
{
for (unsigned scopeNum = 0; scopeNum < info.compVarScopesCount; scopeNum++)
{
noway_assert(info.compVarScopes[scopeNum].vsdVarNum != lclNum);
}
}
#endif
/*
For Debug Code, we have to reserve space even if the variable is never
in scope. We will also need to initialize it if it is a GC var.
So we set lvMustInit and verify it has a nonzero ref-cnt.
*/
if (opts.compDbgCode && !stkFixedArgInVarArgs && lclNum < info.compLocalsCount)
{
if (varDsc->lvRefCnt() == 0)
{
assert(!"unreferenced local in debug codegen");
varDsc->lvImplicitlyReferenced = 1;
}
needSlot |= true;
if (!varDsc->lvIsParam)
{
varDsc->lvMustInit = true;
}
}
varDsc->lvOnFrame = needSlot;
if (!needSlot)
{
/* Clear the lvMustInit flag in case it is set */
varDsc->lvMustInit = false;
goto NOT_STK;
}
}
if (!varDsc->lvOnFrame)
{
goto NOT_STK;
}
ON_STK:
/* The variable (or part of it) lives on the stack frame */
noway_assert((varDsc->lvType != TYP_UNDEF) && (varDsc->lvType != TYP_VOID) && (varDsc->lvType != TYP_UNKNOWN));
#if FEATURE_FIXED_OUT_ARGS
noway_assert((lclNum == lvaOutgoingArgSpaceVar) || lvaLclSize(lclNum) != 0);
#else // FEATURE_FIXED_OUT_ARGS
noway_assert(lvaLclSize(lclNum) != 0);
#endif // FEATURE_FIXED_OUT_ARGS
varDsc->lvOnFrame = true; // Our prediction is that the final home for this local variable will be in the
// stack frame
NOT_STK:;
varDsc->lvFramePointerBased = codeGen->isFramePointerUsed();
#if DOUBLE_ALIGN
if (codeGen->doDoubleAlign())
{
noway_assert(codeGen->isFramePointerUsed() == false);
/* All arguments are off of EBP with double-aligned frames */
if (varDsc->lvIsParam && !varDsc->lvIsRegArg)
{
varDsc->lvFramePointerBased = true;
}
}
#endif
/* Some basic checks */
// It must be in a register, on frame, or have zero references.
noway_assert(varDsc->lvIsInReg() || varDsc->lvOnFrame || varDsc->lvRefCnt() == 0);
// We can't have both lvRegister and lvOnFrame
noway_assert(!varDsc->lvRegister || !varDsc->lvOnFrame);
#ifdef DEBUG
// For varargs functions, there should be no direct references to
// parameter variables except for 'this' (because these were morphed
// in the importer) and the 'arglist' parameter (which is not a GC
// pointer). and the return buffer argument (if we are returning a
// struct).
// This is important because we don't want to try to report them
// to the GC, as the frame offsets in these local varables would
// not be correct.
if (varDsc->lvIsParam && raIsVarargsStackArg(lclNum))
{
if (!varDsc->lvPromoted && !varDsc->lvIsStructField)
{
noway_assert(varDsc->lvRefCnt() == 0 && !varDsc->lvRegister && !varDsc->lvOnFrame);
}
}
#endif
}
}
GCInfo::WriteBarrierForm GCInfo::gcWriteBarrierFormFromTargetAddress(GenTreePtr tgtAddr)
{
GCInfo::WriteBarrierForm result = GCInfo::WBF_BarrierUnknown; // Default case, we have no information.
// If we store through an int to a GC_REF field, we'll assume that needs to use a checked barriers.
if (tgtAddr->TypeGet() == TYP_I_IMPL)
{
return GCInfo::WBF_BarrierChecked; // Why isn't this GCInfo::WBF_BarrierUnknown?
}
// Otherwise...
assert(tgtAddr->TypeGet() == TYP_BYREF);
bool simplifiedExpr = true;
while (simplifiedExpr)
{
simplifiedExpr = false;
tgtAddr = tgtAddr->gtSkipReloadOrCopy();
while (tgtAddr->OperGet() == GT_ADDR && tgtAddr->gtOp.gtOp1->OperGet() == GT_IND)
{
tgtAddr = tgtAddr->gtOp.gtOp1->gtOp.gtOp1;
simplifiedExpr = true;
assert(tgtAddr->TypeGet() == TYP_BYREF);
}
// For additions, one of the operands is a byref or a ref (and the other is not). Follow this down to its
// source.
while (tgtAddr->OperGet() == GT_ADD || tgtAddr->OperGet() == GT_LEA)
{
if (tgtAddr->OperGet() == GT_ADD)
{
if (tgtAddr->gtOp.gtOp1->TypeGet() == TYP_BYREF || tgtAddr->gtOp.gtOp1->TypeGet() == TYP_REF)
{
assert(!(tgtAddr->gtOp.gtOp2->TypeGet() == TYP_BYREF || tgtAddr->gtOp.gtOp2->TypeGet() == TYP_REF));
tgtAddr = tgtAddr->gtOp.gtOp1;
simplifiedExpr = true;
}
else if (tgtAddr->gtOp.gtOp2->TypeGet() == TYP_BYREF || tgtAddr->gtOp.gtOp2->TypeGet() == TYP_REF)
{
tgtAddr = tgtAddr->gtOp.gtOp2;
simplifiedExpr = true;
}
else
{
// We might have a native int. For example:
// const int 0
// + byref
// lclVar int V06 loc5 // this is a local declared "valuetype VType*"
return GCInfo::WBF_BarrierUnknown;
}
}
else
{
// Must be an LEA (i.e., an AddrMode)
assert(tgtAddr->OperGet() == GT_LEA);
tgtAddr = tgtAddr->AsAddrMode()->Base();
if (tgtAddr->TypeGet() == TYP_BYREF || tgtAddr->TypeGet() == TYP_REF)
{
simplifiedExpr = true;
}
else
{
// We might have a native int.
return GCInfo::WBF_BarrierUnknown;
}
}
}
}
if (tgtAddr->IsLocalAddrExpr() != nullptr)
{
// No need for a GC barrier when writing to a local variable.
return GCInfo::WBF_NoBarrier;
}
if (tgtAddr->OperGet() == GT_LCL_VAR || tgtAddr->OperGet() == GT_REG_VAR)
{
unsigned lclNum = 0;
if (tgtAddr->gtOper == GT_LCL_VAR)
{
lclNum = tgtAddr->gtLclVar.gtLclNum;
}
else
{
assert(tgtAddr->gtOper == GT_REG_VAR);
lclNum = tgtAddr->gtRegVar.gtLclNum;
}
LclVarDsc* varDsc = &compiler->lvaTable[lclNum];
// Instead of marking LclVar with 'lvStackByref',
// Consider decomposing the Value Number given to this LclVar to see if it was
// created using a GT_ADDR(GT_LCLVAR) or a GT_ADD( GT_ADDR(GT_LCLVAR), Constant)
// We may have an internal compiler temp created in fgMorphCopyBlock() that we know
// points at one of our stack local variables, it will have lvStackByref set to true.
//
if (varDsc->lvStackByref)
{
assert(varDsc->TypeGet() == TYP_BYREF);
return GCInfo::WBF_NoBarrier;
}
// We don't eliminate for inlined methods, where we (can) know where the "retBuff" points.
if (!compiler->compIsForInlining() && lclNum == compiler->info.compRetBuffArg)
{
assert(compiler->info.compRetType == TYP_STRUCT); // Else shouldn't have a ret buff.
// Are we assured that the ret buff pointer points into the stack of a caller?
if (compiler->info.compRetBuffDefStack)
{
#if 0
// This is an optional debugging mode. If the #if 0 above is changed to #if 1,
// every barrier we remove for stores to GC ref fields of a retbuff use a special
// helper that asserts that the target is not in the heap.
#ifdef DEBUG
return WBF_NoBarrier_CheckNotHeapInDebug;
#else
return WBF_NoBarrier;
#endif
#else // 0
return GCInfo::WBF_NoBarrier;
#endif // 0
}
}
}
if (tgtAddr->TypeGet() == TYP_REF)
{
return GCInfo::WBF_BarrierUnchecked;
}
// Otherwise, we have no information.
return GCInfo::WBF_BarrierUnknown;
}
void GCInfo::gcCountForHeader(UNALIGNED unsigned int* untrackedCount, UNALIGNED unsigned int* varPtrTableSize)
{
unsigned varNum;
LclVarDsc* varDsc;
varPtrDsc* varTmp;
bool thisKeptAliveIsInUntracked = false; // did we track "this" in a synchronized method?
unsigned int count = 0;
/* Count the untracked locals and non-enregistered args */
for (varNum = 0, varDsc = compiler->lvaTable; varNum < compiler->lvaCount; varNum++, varDsc++)
{
if (varTypeIsGC(varDsc->TypeGet()))
{
if (compiler->lvaIsFieldOfDependentlyPromotedStruct(varDsc))
{
// Field local of a PROMOTION_TYPE_DEPENDENT struct must have been
// reported through its parent local
continue;
}
/* Do we have an argument or local variable? */
if (!varDsc->lvIsParam)
{
if (varDsc->lvTracked || !varDsc->lvOnFrame)
{
continue;
}
}
else
{
/* Stack-passed arguments which are not enregistered
* are always reported in this "untracked stack
* pointers" section of the GC info even if lvTracked==true
*/
/* Has this argument been fully enregistered? */
CLANG_FORMAT_COMMENT_ANCHOR;
#ifndef LEGACY_BACKEND
if (!varDsc->lvOnFrame)
#else // LEGACY_BACKEND
if (varDsc->lvRegister)
#endif // LEGACY_BACKEND
{
/* if a CEE_JMP has been used, then we need to report all the arguments
even if they are enregistered, since we will be using this value
in JMP call. Note that this is subtle as we require that
argument offsets are always fixed up properly even if lvRegister
is set */
if (!compiler->compJmpOpUsed)
{
continue;
}
}
else
{
if (!varDsc->lvOnFrame)
{
/* If this non-enregistered pointer arg is never
* used, we don't need to report it
*/
assert(varDsc->lvRefCnt == 0);
continue;
}
else if (varDsc->lvIsRegArg && varDsc->lvTracked)
{
/* If this register-passed arg is tracked, then
* it has been allocated space near the other
* pointer variables and we have accurate life-
* time info. It will be reported with
* gcVarPtrList in the "tracked-pointer" section
*/
continue;
}
}
}
#if !defined(JIT32_GCENCODER) || !defined(WIN64EXCEPTIONS)
// For x86/WIN64EXCEPTIONS, "this" must always be in untracked variables
// so we cannot have "this" in variable lifetimes
if (compiler->lvaIsOriginalThisArg(varNum) && compiler->lvaKeepAliveAndReportThis())
{
// Encoding of untracked variables does not support reporting
// "this". So report it as a tracked variable with a liveness
// extending over the entire method.
thisKeptAliveIsInUntracked = true;
continue;
}
#endif
#ifdef DEBUG
if (compiler->verbose)
{
int offs = varDsc->lvStkOffs;
printf("GCINFO: untrckd %s lcl at [%s", varTypeGCstring(varDsc->TypeGet()),
compiler->genEmitter->emitGetFrameReg());
if (offs < 0)
{
printf("-%02XH", -offs);
}
else if (offs > 0)
{
printf("+%02XH", +offs);
}
printf("]\n");
}
#endif
count++;
}
else if (varDsc->lvType == TYP_STRUCT && varDsc->lvOnFrame && (varDsc->lvExactSize >= TARGET_POINTER_SIZE))
{
unsigned slots = compiler->lvaLclSize(varNum) / sizeof(void*);
BYTE* gcPtrs = compiler->lvaGetGcLayout(varNum);
// walk each member of the array
for (unsigned i = 0; i < slots; i++)
{
if (gcPtrs[i] != TYPE_GC_NONE)
{ // count only gc slots
count++;
}
}
}
}
/* Also count spill temps that hold pointers */
assert(compiler->tmpAllFree());
for (TempDsc* tempThis = compiler->tmpListBeg(); tempThis != nullptr; tempThis = compiler->tmpListNxt(tempThis))
{
if (varTypeIsGC(tempThis->tdTempType()) == false)
{
continue;
}
#ifdef DEBUG
if (compiler->verbose)
{
int offs = tempThis->tdTempOffs();
printf("GCINFO: untrck %s Temp at [%s", varTypeGCstring(varDsc->TypeGet()),
compiler->genEmitter->emitGetFrameReg());
if (offs < 0)
{
printf("-%02XH", -offs);
}
else if (offs > 0)
{
printf("+%02XH", +offs);
}
printf("]\n");
}
#endif
count++;
}
#ifdef DEBUG
if (compiler->verbose)
{
printf("GCINFO: untrckVars = %u\n", count);
}
#endif
*untrackedCount = count;
/* Count the number of entries in the table of non-register pointer
variable lifetimes. */
count = 0;
if (thisKeptAliveIsInUntracked)
{
count++;
}
if (gcVarPtrList)
{
/* We'll use a delta encoding for the lifetime offsets */
for (varTmp = gcVarPtrList; varTmp; varTmp = varTmp->vpdNext)
{
/* Special case: skip any 0-length lifetimes */
if (varTmp->vpdBegOfs == varTmp->vpdEndOfs)
{
continue;
}
count++;
}
}
#ifdef DEBUG
if (compiler->verbose)
{
printf("GCINFO: trackdLcls = %u\n", count);
}
#endif
*varPtrTableSize = count;
}
Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack)
{
assert(useEdge != nullptr);
GenTree* node = *useEdge;
assert(node != nullptr);
#ifdef DEBUG
const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0;
#endif
// First, remove any preceeding list nodes, which are not otherwise visited by the tree walk.
//
// NOTE: GT_FIELD_LIST head nodes, and GT_LIST nodes used by phi nodes will in fact be visited.
for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperIsAnyList() && !(prev->OperIsFieldListHead());
prev = node->gtPrev)
{
BlockRange().Remove(prev);
}
// In addition, remove the current node if it is a GT_LIST node that is not an aggregate.
if (node->OperIsAnyList())
{
GenTreeArgList* list = node->AsArgList();
if (!list->OperIsFieldListHead())
{
BlockRange().Remove(list);
}
return Compiler::WALK_CONTINUE;
}
LIR::Use use;
if (parentStack.Height() < 2)
{
use = LIR::Use::GetDummyUse(BlockRange(), *useEdge);
}
else
{
use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1));
}
assert(node == use.Def());
switch (node->OperGet())
{
case GT_ASG:
RewriteAssignment(use);
break;
case GT_BOX:
// GT_BOX at this level just passes through so get rid of it
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
break;
case GT_ADDR:
RewriteAddress(use);
break;
case GT_IND:
// Clear the `GTF_IND_ASG_LHS` flag, which overlaps with `GTF_IND_REQ_ADDR_IN_REG`.
node->gtFlags &= ~GTF_IND_ASG_LHS;
if (varTypeIsSIMD(node))
{
RewriteSIMDOperand(use, false);
}
else
{
// Due to promotion of structs containing fields of type struct with a
// single scalar type field, we could potentially see IR nodes of the
// form GT_IND(GT_ADD(lclvarAddr, 0)) where 0 is an offset representing
// a field-seq. These get folded here.
//
// TODO: This code can be removed once JIT implements recursive struct
// promotion instead of lying about the type of struct field as the type
// of its single scalar field.
GenTree* addr = node->AsIndir()->Addr();
if (addr->OperGet() == GT_ADD && addr->gtGetOp1()->OperGet() == GT_LCL_VAR_ADDR &&
addr->gtGetOp2()->IsIntegralConst(0))
{
GenTreeLclVarCommon* lclVarNode = addr->gtGetOp1()->AsLclVarCommon();
unsigned lclNum = lclVarNode->GetLclNum();
LclVarDsc* varDsc = comp->lvaTable + lclNum;
if (node->TypeGet() == varDsc->TypeGet())
{
JITDUMP("Rewriting GT_IND(GT_ADD(LCL_VAR_ADDR,0)) to LCL_VAR\n");
lclVarNode->SetOper(GT_LCL_VAR);
lclVarNode->gtType = node->TypeGet();
use.ReplaceWith(comp, lclVarNode);
BlockRange().Remove(addr);
BlockRange().Remove(addr->gtGetOp2());
BlockRange().Remove(node);
}
}
}
break;
case GT_NOP:
// fgMorph sometimes inserts NOP nodes between defs and uses
// supposedly 'to prevent constant folding'. In this case, remove the
// NOP.
if (node->gtGetOp1() != nullptr)
{
use.ReplaceWith(comp, node->gtGetOp1());
BlockRange().Remove(node);
}
break;
case GT_COMMA:
{
GenTree* op1 = node->gtGetOp1();
if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
{
// The LHS has no side effects. Remove it.
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(lhsRange));
}
GenTree* replacement = node->gtGetOp2();
if (!use.IsDummyUse())
{
use.ReplaceWith(comp, replacement);
}
else
{
// This is a top-level comma. If the RHS has no side effects we can remove
// it as well.
if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0)
{
bool isClosed = false;
unsigned sideEffects = 0;
LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects);
// None of the transforms performed herein violate tree order, so these
// should always be true.
assert(isClosed);
assert((sideEffects & GTF_ALL_EFFECT) == 0);
BlockRange().Delete(comp, m_block, std::move(rhsRange));
}
}
BlockRange().Remove(node);
}
break;
case GT_ARGPLACE:
// Remove argplace and list nodes from the execution order.
//
// TODO: remove phi args and phi nodes as well?
BlockRange().Remove(node);
break;
#if defined(_TARGET_XARCH_) || defined(_TARGET_ARM_)
case GT_CLS_VAR:
{
// Class vars that are the target of an assignment will get rewritten into
// GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is
// not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would
// otherwise be generated would also be picked up by RewriteAssignment--but
// skipping the rewrite here saves an allocation and a bit of extra work.
const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node);
if (!isLHSOfAssignment)
{
GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node);
node->SetOper(GT_CLS_VAR_ADDR);
node->gtType = TYP_BYREF;
BlockRange().InsertAfter(node, ind);
use.ReplaceWith(comp, ind);
// TODO: JIT dump
}
}
break;
#endif // _TARGET_XARCH_
case GT_INTRINSIC:
// Non-target intrinsics should have already been rewritten back into user calls.
assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId));
break;
#ifdef FEATURE_SIMD
case GT_BLK:
case GT_OBJ:
{
// TODO-1stClassStructs: These should have been transformed to GT_INDs, but in order
// to preserve existing behavior, we will keep this as a block node if this is the
// lhs of a block assignment, and either:
// - It is a "generic" TYP_STRUCT assignment, OR
// - It is an initblk, OR
// - Neither the lhs or rhs are known to be of SIMD type.
GenTree* parent = use.User();
bool keepBlk = false;
if ((parent->OperGet() == GT_ASG) && (node == parent->gtGetOp1()))
{
if ((node->TypeGet() == TYP_STRUCT) || parent->OperIsInitBlkOp())
{
keepBlk = true;
}
else if (!comp->isAddrOfSIMDType(node->AsBlk()->Addr()))
{
GenTree* dataSrc = parent->gtGetOp2();
if (!dataSrc->IsLocal() && (dataSrc->OperGet() != GT_SIMD))
{
noway_assert(dataSrc->OperIsIndir());
keepBlk = !comp->isAddrOfSIMDType(dataSrc->AsIndir()->Addr());
}
}
}
RewriteSIMDOperand(use, keepBlk);
}
break;
case GT_LCL_FLD:
case GT_STORE_LCL_FLD:
// TODO-1stClassStructs: Eliminate this.
FixupIfSIMDLocal(node->AsLclVarCommon());
break;
case GT_SIMD:
{
noway_assert(comp->featureSIMD);
GenTreeSIMD* simdNode = node->AsSIMD();
unsigned simdSize = simdNode->gtSIMDSize;
var_types simdType = comp->getSIMDTypeForSize(simdSize);
// TODO-1stClassStructs: This should be handled more generally for enregistered or promoted
// structs that are passed or returned in a different register type than their enregistered
// type(s).
if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE)
{
// This happens when it is consumed by a GT_RET_EXPR.
// It can only be a Vector2f or Vector2i.
assert(genTypeSize(simdNode->gtSIMDBaseType) == 4);
simdNode->gtType = TYP_SIMD8;
}
// Certain SIMD trees require rationalizing.
if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray)
{
// Rewrite this as an explicit load.
JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n");
unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType);
GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2,
baseTypeSize, offsetof(CORINFO_Array, u1Elems));
GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address);
BlockRange().InsertBefore(simdNode, address, ind);
use.ReplaceWith(comp, ind);
BlockRange().Remove(simdNode);
DISPTREERANGE(BlockRange(), use.Def());
JITDUMP("\n");
}
else
{
// This code depends on the fact that NONE of the SIMD intrinsics take vector operands
// of a different width. If that assumption changes, we will EITHER have to make these type
// transformations during importation, and plumb the types all the way through the JIT,
// OR add a lot of special handling here.
GenTree* op1 = simdNode->gtGetOp1();
if (op1 != nullptr && op1->gtType == TYP_STRUCT)
{
op1->gtType = simdType;
}
GenTree* op2 = simdNode->gtGetOp2IfPresent();
if (op2 != nullptr && op2->gtType == TYP_STRUCT)
{
op2->gtType = simdType;
}
}
}
break;
#endif // FEATURE_SIMD
default:
// JCC nodes should not be present in HIR.
assert(node->OperGet() != GT_JCC);
break;
}
// Do some extra processing on top-level nodes to remove unused local reads.
if (node->OperIsLocalRead())
{
if (use.IsDummyUse())
{
comp->lvaDecRefCnts(node);
BlockRange().Remove(node);
}
else
{
// Local reads are side-effect-free; clear any flags leftover from frontend transformations.
node->gtFlags &= ~GTF_ALL_EFFECT;
}
}
assert(isLateArg == ((use.Def()->gtFlags & GTF_LATE_ARG) != 0));
return Compiler::WALK_CONTINUE;
}
void TreeLifeUpdater<ForCodeGen>::UpdateLifeVar(GenTree* tree)
{
GenTree* indirAddrLocal = compiler->fgIsIndirOfAddrOfLocal(tree);
assert(tree->OperIsNonPhiLocal() || indirAddrLocal != nullptr);
// Get the local var tree -- if "tree" is "Ldobj(addr(x))", or "ind(addr(x))" this is "x", else it's "tree".
GenTree* lclVarTree = indirAddrLocal;
if (lclVarTree == nullptr)
{
lclVarTree = tree;
}
unsigned int lclNum = lclVarTree->gtLclVarCommon.gtLclNum;
LclVarDsc* varDsc = compiler->lvaTable + lclNum;
#ifdef DEBUG
#if !defined(_TARGET_AMD64_)
// There are no addr nodes on ARM and we are experimenting with encountering vars in 'random' order.
// Struct fields are not traversed in a consistent order, so ignore them when
// verifying that we see the var nodes in execution order
if (ForCodeGen)
{
if (tree->OperIsIndir())
{
assert(indirAddrLocal != NULL);
}
else if (tree->gtNext != NULL && tree->gtNext->gtOper == GT_ADDR &&
((tree->gtNext->gtNext == NULL || !tree->gtNext->gtNext->OperIsIndir())))
{
assert(tree->IsLocal()); // Can only take the address of a local.
// The ADDR might occur in a context where the address it contributes is eventually
// dereferenced, so we can't say that this is not a use or def.
}
}
#endif // !_TARGET_AMD64_
#endif // DEBUG
compiler->compCurLifeTree = tree;
VarSetOps::Assign(compiler, newLife, compiler->compCurLife);
// By codegen, a struct may not be TYP_STRUCT, so we have to
// check lvPromoted, for the case where the fields are being
// tracked.
if (!varDsc->lvTracked && !varDsc->lvPromoted)
{
return;
}
// if it's a partial definition then variable "x" must have had a previous, original, site to be born.
bool isBorn = ((tree->gtFlags & GTF_VAR_DEF) != 0 && (tree->gtFlags & GTF_VAR_USEASG) == 0);
bool isDying = ((tree->gtFlags & GTF_VAR_DEATH) != 0);
bool spill = ((tree->gtFlags & GTF_SPILL) != 0);
// Since all tracked vars are register candidates, but not all are in registers at all times,
// we maintain two separate sets of variables - the total set of variables that are either
// born or dying here, and the subset of those that are on the stack
VarSetOps::ClearD(compiler, stackVarDeltaSet);
if (isBorn || isDying)
{
VarSetOps::ClearD(compiler, varDeltaSet);
if (varDsc->lvTracked)
{
VarSetOps::AddElemD(compiler, varDeltaSet, varDsc->lvVarIndex);
if (ForCodeGen)
{
if (isBorn && varDsc->lvIsRegCandidate() && tree->gtHasReg())
{
compiler->codeGen->genUpdateVarReg(varDsc, tree);
}
if (varDsc->lvIsInReg() && tree->gtRegNum != REG_NA)
{
compiler->codeGen->genUpdateRegLife(varDsc, isBorn, isDying DEBUGARG(tree));
}
else
{
VarSetOps::AddElemD(compiler, stackVarDeltaSet, varDsc->lvVarIndex);
}
}
}
else if (varDsc->lvPromoted)
{
// If hasDeadTrackedFieldVars is true, then, for a LDOBJ(ADDR(<promoted struct local>)),
// *deadTrackedFieldVars indicates which tracked field vars are dying.
bool hasDeadTrackedFieldVars = false;
if (indirAddrLocal != nullptr && isDying)
{
assert(!isBorn); // GTF_VAR_DEATH only set for LDOBJ last use.
VARSET_TP* deadTrackedFieldVars = nullptr;
hasDeadTrackedFieldVars =
compiler->LookupPromotedStructDeathVars(indirAddrLocal, &deadTrackedFieldVars);
if (hasDeadTrackedFieldVars)
{
VarSetOps::Assign(compiler, varDeltaSet, *deadTrackedFieldVars);
}
}
for (unsigned i = varDsc->lvFieldLclStart; i < varDsc->lvFieldLclStart + varDsc->lvFieldCnt; ++i)
{
LclVarDsc* fldVarDsc = &(compiler->lvaTable[i]);
noway_assert(fldVarDsc->lvIsStructField);
if (fldVarDsc->lvTracked)
{
unsigned fldVarIndex = fldVarDsc->lvVarIndex;
noway_assert(fldVarIndex < compiler->lvaTrackedCount);
if (!hasDeadTrackedFieldVars)
{
VarSetOps::AddElemD(compiler, varDeltaSet, fldVarIndex);
if (ForCodeGen)
{
// We repeat this call here and below to avoid the VarSetOps::IsMember
// test in this, the common case, where we have no deadTrackedFieldVars.
if (fldVarDsc->lvIsInReg())
{
if (isBorn)
{
compiler->codeGen->genUpdateVarReg(fldVarDsc, tree);
}
compiler->codeGen->genUpdateRegLife(fldVarDsc, isBorn, isDying DEBUGARG(tree));
}
else
{
VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex);
}
}
}
else if (ForCodeGen && VarSetOps::IsMember(compiler, varDeltaSet, fldVarIndex))
{
if (compiler->lvaTable[i].lvIsInReg())
{
if (isBorn)
{
compiler->codeGen->genUpdateVarReg(fldVarDsc, tree);
}
compiler->codeGen->genUpdateRegLife(fldVarDsc, isBorn, isDying DEBUGARG(tree));
}
else
{
VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex);
}
}
}
}
}
// First, update the live set
if (isDying)
{
// We'd like to be able to assert the following, however if we are walking
// through a qmark/colon tree, we may encounter multiple last-use nodes.
// assert (VarSetOps::IsSubset(compiler, regVarDeltaSet, newLife));
VarSetOps::DiffD(compiler, newLife, varDeltaSet);
}
else
{
// This shouldn't be in newLife, unless this is debug code, in which
// case we keep vars live everywhere, OR the variable is address-exposed,
// OR this block is part of a try block, in which case it may be live at the handler
// Could add a check that, if it's in newLife, that it's also in
// fgGetHandlerLiveVars(compCurBB), but seems excessive
//
// For a dead store, it can be the case that we set both isBorn and isDying to true.
// (We don't eliminate dead stores under MinOpts, so we can't assume they're always
// eliminated.) If it's both, we handled it above.
VarSetOps::UnionD(compiler, newLife, varDeltaSet);
}
}
if (!VarSetOps::Equal(compiler, compiler->compCurLife, newLife))
{
#ifdef DEBUG
if (compiler->verbose)
{
printf("\t\t\t\t\t\t\tLive vars: ");
dumpConvertedVarSet(compiler, compiler->compCurLife);
printf(" => ");
dumpConvertedVarSet(compiler, newLife);
printf("\n");
}
#endif // DEBUG
VarSetOps::Assign(compiler, compiler->compCurLife, newLife);
if (ForCodeGen)
{
// Only add vars to the gcInfo.gcVarPtrSetCur if they are currently on stack, since the
// gcInfo.gcTrkStkPtrLcls
// includes all TRACKED vars that EVER live on the stack (i.e. are not always in a register).
VarSetOps::Assign(compiler, gcTrkStkDeltaSet, compiler->codeGen->gcInfo.gcTrkStkPtrLcls);
VarSetOps::IntersectionD(compiler, gcTrkStkDeltaSet, stackVarDeltaSet);
if (!VarSetOps::IsEmpty(compiler, gcTrkStkDeltaSet))
{
#ifdef DEBUG
if (compiler->verbose)
{
printf("\t\t\t\t\t\t\tGCvars: ");
dumpConvertedVarSet(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur);
printf(" => ");
}
#endif // DEBUG
if (isBorn)
{
VarSetOps::UnionD(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur, gcTrkStkDeltaSet);
}
else
{
VarSetOps::DiffD(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur, gcTrkStkDeltaSet);
}
#ifdef DEBUG
if (compiler->verbose)
{
dumpConvertedVarSet(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur);
printf("\n");
}
#endif // DEBUG
}
#ifdef USING_SCOPE_INFO
compiler->codeGen->siUpdate();
#endif // USING_SCOPE_INFO
}
}
if (ForCodeGen && spill)
{
assert(!varDsc->lvPromoted);
compiler->codeGen->genSpillVar(tree);
if (VarSetOps::IsMember(compiler, compiler->codeGen->gcInfo.gcTrkStkPtrLcls, varDsc->lvVarIndex))
{
if (!VarSetOps::IsMember(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex))
{
VarSetOps::AddElemD(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur, varDsc->lvVarIndex);
#ifdef DEBUG
if (compiler->verbose)
{
printf("\t\t\t\t\t\t\tVar V%02u becoming live\n", varDsc - compiler->lvaTable);
}
#endif // DEBUG
}
}
}
}