//----------------------------------------------------------------------------------------------
// ContainCheckHWIntrinsic: Perform containment analysis for a hardware intrinsic node.
//
// Arguments:
// node - The hardware intrinsic node.
//
void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
{
NamedIntrinsic intrinsicID = node->gtHWIntrinsicId;
GenTreeArgList* argList = nullptr;
GenTree* op1 = node->gtOp.gtOp1;
GenTree* op2 = node->gtOp.gtOp2;
if (op1->OperIs(GT_LIST))
{
argList = op1->AsArgList();
op1 = argList->Current();
op2 = argList->Rest()->Current();
}
switch (HWIntrinsicInfo::lookup(node->gtHWIntrinsicId).form)
{
case HWIntrinsicInfo::SimdExtractOp:
if (op2->IsCnsIntOrI())
{
MakeSrcContained(node, op2);
}
break;
case HWIntrinsicInfo::SimdInsertOp:
if (op2->IsCnsIntOrI())
{
MakeSrcContained(node, op2);
GenTree* op3 = argList->Rest()->Rest()->Current();
// In the HW intrinsics C# API there is no direct way to specify a vector element to element mov
// VX[a] = VY[b]
// In C# this would naturally be expressed by
// Insert(VX, a, Extract(VY, b))
// If both a & b are immediate constants contain the extract/getItem so that we can emit
// the single instruction mov Vx[a], Vy[b]
if (op3->OperIs(GT_HWIntrinsic) && (op3->AsHWIntrinsic()->gtHWIntrinsicId == NI_ARM64_SIMD_GetItem))
{
ContainCheckHWIntrinsic(op3->AsHWIntrinsic());
if (op3->gtOp.gtOp2->isContained())
{
MakeSrcContained(node, op3);
}
}
}
break;
default:
break;
}
}
/*****************************************************************************
* gsMarkPtrsAndAssignGroups
* Walk a tree looking for assignment groups, variables whose value is used
* in a *p store or use, and variable passed to calls. This info is then used
* to determine parameters which are vulnerable.
* This function carries a state to know if it is under an assign node, call node
* or indirection node. It starts a new tree walk for it's subtrees when the state
* changes.
*/
Compiler::fgWalkResult Compiler::gsMarkPtrsAndAssignGroups(GenTreePtr *pTree, fgWalkData *data)
{
struct MarkPtrsInfo *pState= (MarkPtrsInfo *)data->pCallbackData;
struct MarkPtrsInfo newState = *pState;
Compiler *comp = data->compiler;
GenTreePtr tree = *pTree;
ShadowParamVarInfo *shadowVarInfo = pState->comp->gsShadowVarInfo;
assert(shadowVarInfo);
bool fIsBlk = false;
unsigned lclNum;
assert(!pState->isAssignSrc || pState->lvAssignDef != (unsigned)-1);
if (pState->skipNextNode)
{
pState->skipNextNode = false;
return WALK_CONTINUE;
}
switch (tree->OperGet())
{
// Indirections - look for *p uses and defs
case GT_INITBLK:
case GT_COPYOBJ:
case GT_COPYBLK:
fIsBlk = true;
// fallthrough
case GT_IND:
case GT_LDOBJ:
case GT_ARR_ELEM:
case GT_ARR_INDEX:
case GT_ARR_OFFSET:
case GT_FIELD:
newState.isUnderIndir = true;
{
if (fIsBlk)
{
// Blk nodes have implicit indirections.
comp->fgWalkTreePre(&tree->gtOp.gtOp1, comp->gsMarkPtrsAndAssignGroups, (void *)&newState);
if (tree->OperGet() == GT_INITBLK)
{
newState.isUnderIndir = false;
}
comp->fgWalkTreePre(&tree->gtOp.gtOp2, comp->gsMarkPtrsAndAssignGroups, (void *)&newState);
}
else
{
newState.skipNextNode = true; // Don't have to worry about which kind of node we're dealing with
comp->fgWalkTreePre(&tree, comp->gsMarkPtrsAndAssignGroups, (void *)&newState);
}
}
return WALK_SKIP_SUBTREES;
// local vars and param uses
case GT_LCL_VAR:
case GT_LCL_FLD:
lclNum = tree->gtLclVarCommon.gtLclNum;
if (pState->isUnderIndir)
{
// The variable is being dereferenced for a read or a write.
comp->lvaTable[lclNum].lvIsPtr = 1;
}
if (pState->isAssignSrc)
{
//
// Add lvAssignDef and lclNum to a common assign group
if (shadowVarInfo[pState->lvAssignDef].assignGroup)
{
if (shadowVarInfo[lclNum].assignGroup)
{
// OR both bit vector
shadowVarInfo[pState->lvAssignDef].assignGroup->bitVectOr(shadowVarInfo[lclNum].assignGroup);
}
else
{
shadowVarInfo[pState->lvAssignDef].assignGroup->bitVectSet(lclNum);
}
// Point both to the same bit vector
shadowVarInfo[lclNum].assignGroup = shadowVarInfo[pState->lvAssignDef].assignGroup;
}
else if (shadowVarInfo[lclNum].assignGroup)
{
shadowVarInfo[lclNum].assignGroup->bitVectSet(pState->lvAssignDef);
// Point both to the same bit vector
shadowVarInfo[pState->lvAssignDef].assignGroup = shadowVarInfo[lclNum].assignGroup;
}
else
{
FixedBitVect *bv = FixedBitVect::bitVectInit(pState->comp->lvaCount, pState->comp);
// (shadowVarInfo[pState->lvAssignDef] == NULL && shadowVarInfo[lclNew] == NULL);
// Neither of them has an assign group yet. Make a new one.
shadowVarInfo[pState->lvAssignDef].assignGroup = bv;
shadowVarInfo[lclNum].assignGroup = bv;
bv->bitVectSet(pState->lvAssignDef);
bv->bitVectSet(lclNum);
}
}
return WALK_CONTINUE;
// Calls - Mark arg variables
case GT_CALL:
newState.isUnderIndir = false;
newState.isAssignSrc = false;
{
if (tree->gtCall.gtCallObjp)
{
newState.isUnderIndir = true;
comp->fgWalkTreePre(&tree->gtCall.gtCallObjp, gsMarkPtrsAndAssignGroups, (void *)&newState);
}
for (GenTreeArgList* args = tree->gtCall.gtCallArgs; args; args = args->Rest())
{
comp->fgWalkTreePre(&args->Current(), gsMarkPtrsAndAssignGroups, (void *)&newState);
}
for (GenTreeArgList* args = tree->gtCall.gtCallLateArgs; args; args = args->Rest())
{
comp->fgWalkTreePre(&args->Current(), gsMarkPtrsAndAssignGroups, (void *)&newState);
}
if (tree->gtCall.gtCallType == CT_INDIRECT)
{
newState.isUnderIndir = true;
// A function pointer is treated like a write-through pointer since
// it controls what code gets executed, and so indirectly can cause
// a write to memory.
comp->fgWalkTreePre(&tree->gtCall.gtCallAddr, gsMarkPtrsAndAssignGroups, (void *)&newState);
}
}
return WALK_SKIP_SUBTREES;
case GT_ADDR:
newState.isUnderIndir = false;
// We'll assume p in "**p = " can be vulnerable because by changing 'p', someone
// could control where **p stores to.
{
comp->fgWalkTreePre(&tree->gtOp.gtOp1, comp->gsMarkPtrsAndAssignGroups, (void *)&newState);
}
return WALK_SKIP_SUBTREES;
default:
// Assignments - track assign groups and *p defs.
if (tree->OperIsAssignment())
{
bool isLocVar;
bool isLocFld;
// Walk dst side
comp->fgWalkTreePre(&tree->gtOp.gtOp1, comp->gsMarkPtrsAndAssignGroups, (void *)&newState);
// Now handle src side
isLocVar = tree->gtOp.gtOp1->OperGet() == GT_LCL_VAR;
isLocFld = tree->gtOp.gtOp1->OperGet() == GT_LCL_FLD;
if ((isLocVar || isLocFld) && tree->gtOp.gtOp2)
{
lclNum = tree->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
newState.lvAssignDef = lclNum;
newState.isAssignSrc = true;
}
comp->fgWalkTreePre(&tree->gtOp.gtOp2, comp->gsMarkPtrsAndAssignGroups, (void *)&newState);
return WALK_SKIP_SUBTREES;
}
}
return WALK_CONTINUE;
}
bool RangeCheck::IsMonotonicallyIncreasing(GenTreePtr expr, SearchPath* path)
{
JITDUMP("[RangeCheck::IsMonotonicallyIncreasing] %p\n", dspPtr(expr));
if (path->Lookup(expr))
{
return true;
}
// Add hashtable entry for expr.
path->Set(expr, NULL);
// Remove hashtable entry for expr when we exit the present scope.
auto code = [&] { path->Remove(expr); };
jitstd::utility::scoped_code<decltype(code)> finally(code);
// If the rhs expr is constant, then it is not part of the dependency
// loop which has to increase monotonically.
ValueNum vn = expr->gtVNPair.GetConservative();
if (m_pCompiler->vnStore->IsVNConstant(vn))
{
return true;
}
// If the rhs expr is local, then try to find the def of the local.
else if (expr->IsLocal())
{
Location* loc = GetDef(expr);
if (loc == nullptr)
{
return false;
}
GenTreePtr asg = loc->parent;
assert(asg->OperKind() & GTK_ASGOP);
switch (asg->OperGet())
{
case GT_ASG:
return IsMonotonicallyIncreasing(asg->gtGetOp2(), path);
case GT_ASG_ADD:
return IsBinOpMonotonicallyIncreasing(asg->gtGetOp1(), asg->gtGetOp2(), GT_ADD, path);
}
JITDUMP("Unknown local definition type\n");
return false;
}
else if (expr->OperGet() == GT_ADD)
{
return IsBinOpMonotonicallyIncreasing(expr->gtGetOp1(), expr->gtGetOp2(), GT_ADD, path);
}
else if (expr->OperGet() == GT_PHI)
{
for (GenTreeArgList* args = expr->gtOp.gtOp1->AsArgList();
args != nullptr; args = args->Rest())
{
// If the arg is already in the path, skip.
if (path->Lookup(args->Current()))
{
continue;
}
if (!IsMonotonicallyIncreasing(args->Current(), path))
{
JITDUMP("Phi argument not monotonic\n");
return false;
}
}
return true;
}
JITDUMP("Unknown tree type\n");
return false;
}
// The parameter rejectNegativeConst is true when we are adding two local vars (see above)
bool RangeCheck::IsMonotonicallyIncreasing(GenTree* expr, bool rejectNegativeConst)
{
JITDUMP("[RangeCheck::IsMonotonicallyIncreasing] [%06d]\n", Compiler::dspTreeID(expr));
// Add hashtable entry for expr.
bool alreadyPresent = !m_pSearchPath->Set(expr, nullptr, SearchPath::Overwrite);
if (alreadyPresent)
{
return true;
}
// Remove hashtable entry for expr when we exit the present scope.
auto code = [this, expr] { m_pSearchPath->Remove(expr); };
jitstd::utility::scoped_code<decltype(code)> finally(code);
if (m_pSearchPath->GetCount() > MAX_SEARCH_DEPTH)
{
return false;
}
// If expr is constant, then it is not part of the dependency
// loop which has to increase monotonically.
ValueNum vn = expr->gtVNPair.GetConservative();
if (m_pCompiler->vnStore->IsVNInt32Constant(vn))
{
if (rejectNegativeConst)
{
int cons = m_pCompiler->vnStore->ConstantValue<int>(vn);
return (cons >= 0);
}
else
{
return true;
}
}
// If the rhs expr is local, then try to find the def of the local.
else if (expr->IsLocal())
{
BasicBlock* asgBlock;
GenTreeOp* asg = GetSsaDefAsg(expr->AsLclVarCommon(), &asgBlock);
return (asg != nullptr) && IsMonotonicallyIncreasing(asg->gtGetOp2(), rejectNegativeConst);
}
else if (expr->OperGet() == GT_ADD)
{
return IsBinOpMonotonicallyIncreasing(expr->AsOp());
}
else if (expr->OperGet() == GT_PHI)
{
for (GenTreeArgList* args = expr->gtOp.gtOp1->AsArgList(); args != nullptr; args = args->Rest())
{
// If the arg is already in the path, skip.
if (m_pSearchPath->Lookup(args->Current()))
{
continue;
}
if (!IsMonotonicallyIncreasing(args->Current(), rejectNegativeConst))
{
JITDUMP("Phi argument not monotonic\n");
return false;
}
}
return true;
}
JITDUMP("Unknown tree type\n");
return false;
}
//------------------------------------------------------------------------
// BuildHWIntrinsic: Set the NodeInfo for a GT_HWIntrinsic tree.
//
// Arguments:
// tree - The GT_HWIntrinsic node of interest
//
// Return Value:
// The number of sources consumed by this node.
//
int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree)
{
NamedIntrinsic intrinsicID = intrinsicTree->gtHWIntrinsicId;
int numArgs = HWIntrinsicInfo::lookupNumArgs(intrinsicTree);
GenTree* op1 = intrinsicTree->gtGetOp1();
GenTree* op2 = intrinsicTree->gtGetOp2();
GenTree* op3 = nullptr;
int srcCount = 0;
if ((op1 != nullptr) && op1->OperIsList())
{
// op2 must be null, and there must be at least two more arguments.
assert(op2 == nullptr);
noway_assert(op1->AsArgList()->Rest() != nullptr);
noway_assert(op1->AsArgList()->Rest()->Rest() != nullptr);
assert(op1->AsArgList()->Rest()->Rest()->Rest() == nullptr);
op2 = op1->AsArgList()->Rest()->Current();
op3 = op1->AsArgList()->Rest()->Rest()->Current();
op1 = op1->AsArgList()->Current();
}
int dstCount = intrinsicTree->IsValue() ? 1 : 0;
bool op2IsDelayFree = false;
bool op3IsDelayFree = false;
// Create internal temps, and handle any other special requirements.
switch (HWIntrinsicInfo::lookup(intrinsicID).form)
{
case HWIntrinsicInfo::Sha1HashOp:
assert((numArgs == 3) && (op2 != nullptr) && (op3 != nullptr));
if (!op2->isContained())
{
assert(!op3->isContained());
op2IsDelayFree = true;
op3IsDelayFree = true;
setInternalRegsDelayFree = true;
}
buildInternalFloatRegisterDefForNode(intrinsicTree);
break;
case HWIntrinsicInfo::SimdTernaryRMWOp:
assert((numArgs == 3) && (op2 != nullptr) && (op3 != nullptr));
if (!op2->isContained())
{
assert(!op3->isContained());
op2IsDelayFree = true;
op3IsDelayFree = true;
}
break;
case HWIntrinsicInfo::Sha1RotateOp:
buildInternalFloatRegisterDefForNode(intrinsicTree);
break;
case HWIntrinsicInfo::SimdExtractOp:
case HWIntrinsicInfo::SimdInsertOp:
if (!op2->isContained())
{
// We need a temp to create a switch table
buildInternalIntRegisterDefForNode(intrinsicTree);
}
break;
default:
break;
}
// Next, build uses
if (numArgs > 3)
{
srcCount = 0;
assert(!op2IsDelayFree && !op3IsDelayFree);
assert(op1->OperIs(GT_LIST));
{
for (GenTreeArgList* list = op1->AsArgList(); list != nullptr; list = list->Rest())
{
srcCount += BuildOperandUses(list->Current());
}
}
assert(srcCount == numArgs);
}
else
{
if (op1 != nullptr)
{
srcCount += BuildOperandUses(op1);
if (op2 != nullptr)
{
srcCount += (op2IsDelayFree) ? BuildDelayFreeUses(op2) : BuildOperandUses(op2);
if (op3 != nullptr)
{
srcCount += (op3IsDelayFree) ? BuildDelayFreeUses(op3) : BuildOperandUses(op3);
}
}
}
}
buildInternalRegisterUses();
// Now defs
if (intrinsicTree->IsValue())
{
BuildDef(intrinsicTree);
}
return srcCount;
}
//------------------------------------------------------------------------
// genHWIntrinsic: Generates the code for a given hardware intrinsic node.
//
// Arguments:
// node - The hardware intrinsic node
//
void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
{
NamedIntrinsic intrinsicID = node->gtHWIntrinsicId;
InstructionSet isa = Compiler::isaOfHWIntrinsic(intrinsicID);
HWIntrinsicCategory category = Compiler::categoryOfHWIntrinsic(intrinsicID);
HWIntrinsicFlag flags = Compiler::flagsOfHWIntrinsic(intrinsicID);
int ival = Compiler::ivalOfHWIntrinsic(intrinsicID);
int numArgs = Compiler::numArgsOfHWIntrinsic(node);
assert((flags & HW_Flag_NoCodeGen) == 0);
if (genIsTableDrivenHWIntrinsic(category, flags))
{
GenTree* op1 = node->gtGetOp1();
GenTree* op2 = node->gtGetOp2();
regNumber targetReg = node->gtRegNum;
var_types targetType = node->TypeGet();
var_types baseType = node->gtSIMDBaseType;
regNumber op1Reg = REG_NA;
regNumber op2Reg = REG_NA;
emitter* emit = getEmitter();
assert(numArgs >= 0);
instruction ins = Compiler::insOfHWIntrinsic(intrinsicID, baseType);
assert(ins != INS_invalid);
emitAttr simdSize = EA_ATTR(node->gtSIMDSize);
assert(simdSize != 0);
switch (numArgs)
{
case 1:
genConsumeOperands(node);
op1Reg = op1->gtRegNum;
if (category == HW_Category_MemoryLoad)
{
emit->emitIns_R_AR(ins, simdSize, targetReg, op1Reg, 0);
}
else if (category == HW_Category_SIMDScalar && (flags & HW_Flag_CopyUpperBits) != 0)
{
emit->emitIns_SIMD_R_R_R(ins, simdSize, targetReg, op1Reg, op1Reg);
}
else if ((ival != -1) && varTypeIsFloating(baseType))
{
emit->emitIns_R_R_I(ins, simdSize, targetReg, op1Reg, ival);
}
else
{
emit->emitIns_R_R(ins, simdSize, targetReg, op1Reg);
}
break;
case 2:
genConsumeOperands(node);
op1Reg = op1->gtRegNum;
op2Reg = op2->gtRegNum;
if (category == HW_Category_MemoryStore)
{
emit->emitIns_AR_R(ins, simdSize, op2Reg, op1Reg, 0);
}
else if ((ival != -1) && varTypeIsFloating(baseType))
{
genHWIntrinsic_R_R_RM_I(node, ins);
}
else if (category == HW_Category_MemoryLoad)
{
emit->emitIns_SIMD_R_R_AR(ins, simdSize, targetReg, op1Reg, op2Reg);
}
else if (Compiler::isImmHWIntrinsic(intrinsicID, op2))
{
if (intrinsicID == NI_SSE2_Extract)
{
// extract instructions return to GP-registers, so it needs int size as the emitsize
simdSize = emitTypeSize(TYP_INT);
}
auto emitSwCase = [&](unsigned i) {
emit->emitIns_SIMD_R_R_I(ins, simdSize, targetReg, op1Reg, (int)i);
};
if (op2->IsCnsIntOrI())
{
ssize_t ival = op2->AsIntCon()->IconValue();
emitSwCase((unsigned)ival);
}
else
{
// We emit a fallback case for the scenario when the imm-op is not a constant. This should
// normally happen when the intrinsic is called indirectly, such as via Reflection. However, it
// can also occur if the consumer calls it directly and just doesn't pass a constant value.
regNumber baseReg = node->ExtractTempReg();
regNumber offsReg = node->GetSingleTempReg();
genHWIntrinsicJumpTableFallback(intrinsicID, op2Reg, baseReg, offsReg, emitSwCase);
}
}
else
{
genHWIntrinsic_R_R_RM(node, ins);
}
break;
case 3:
{
assert(op1->OperIsList());
assert(op1->gtGetOp2()->OperIsList());
assert(op1->gtGetOp2()->gtGetOp2()->OperIsList());
GenTreeArgList* argList = op1->AsArgList();
op1 = argList->Current();
genConsumeRegs(op1);
op1Reg = op1->gtRegNum;
argList = argList->Rest();
op2 = argList->Current();
genConsumeRegs(op2);
op2Reg = op2->gtRegNum;
argList = argList->Rest();
GenTree* op3 = argList->Current();
genConsumeRegs(op3);
regNumber op3Reg = op3->gtRegNum;
if (Compiler::isImmHWIntrinsic(intrinsicID, op3))
{
auto emitSwCase = [&](unsigned i) {
emit->emitIns_SIMD_R_R_R_I(ins, simdSize, targetReg, op1Reg, op2Reg, (int)i);
};
if (op3->IsCnsIntOrI())
{
ssize_t ival = op3->AsIntCon()->IconValue();
emitSwCase((unsigned)ival);
}
else
{
// We emit a fallback case for the scenario when the imm-op is not a constant. This should
// normally happen when the intrinsic is called indirectly, such as via Reflection. However, it
// can also occur if the consumer calls it directly and just doesn't pass a constant value.
regNumber baseReg = node->ExtractTempReg();
regNumber offsReg = node->GetSingleTempReg();
genHWIntrinsicJumpTableFallback(intrinsicID, op3Reg, baseReg, offsReg, emitSwCase);
}
}
else if (category == HW_Category_MemoryStore)
{
assert(intrinsicID == NI_SSE2_MaskMove);
assert(targetReg == REG_NA);
// SSE2 MaskMove hardcodes the destination (op3) in DI/EDI/RDI
if (op3Reg != REG_EDI)
{
emit->emitIns_R_R(INS_mov, EA_PTRSIZE, REG_EDI, op3Reg);
}
emit->emitIns_R_R(ins, simdSize, op1Reg, op2Reg);
}
else
{
emit->emitIns_SIMD_R_R_R_R(ins, simdSize, targetReg, op1Reg, op2Reg, op3Reg);
}
break;
}
default:
unreached();
break;
}
genProduceReg(node);
return;
}
switch (isa)
{
case InstructionSet_SSE:
genSSEIntrinsic(node);
break;
case InstructionSet_SSE2:
genSSE2Intrinsic(node);
break;
case InstructionSet_SSE41:
genSSE41Intrinsic(node);
break;
case InstructionSet_SSE42:
genSSE42Intrinsic(node);
break;
case InstructionSet_AVX:
genAVXIntrinsic(node);
break;
case InstructionSet_AVX2:
genAVX2Intrinsic(node);
break;
case InstructionSet_AES:
genAESIntrinsic(node);
break;
case InstructionSet_BMI1:
genBMI1Intrinsic(node);
break;
case InstructionSet_BMI2:
genBMI2Intrinsic(node);
break;
case InstructionSet_FMA:
genFMAIntrinsic(node);
break;
case InstructionSet_LZCNT:
genLZCNTIntrinsic(node);
break;
case InstructionSet_PCLMULQDQ:
genPCLMULQDQIntrinsic(node);
break;
case InstructionSet_POPCNT:
genPOPCNTIntrinsic(node);
break;
default:
unreached();
break;
}
}