void AMDGPUTargetInfo::fillValidCPUList(
SmallVectorImpl<StringRef> &Values) const {
if (isAMDGCN(getTriple()))
llvm::for_each(AMDGCNGPUs, [&Values](const GPUInfo &GPU) {
Values.emplace_back(GPU.Name);});
else
llvm::for_each(R600GPUs, [&Values](const GPUInfo &GPU) {
Values.emplace_back(GPU.Name);});
}
// TODO: This should really be an operation on type lowering.
void SILBuilder::emitDestructureValueOperation(
SILLocation loc, SILValue v, SmallVectorImpl<SILValue> &results) {
// Once destructure is allowed everywhere, remove the projection code.
// If we do not have a tuple or a struct, add to our results list and return.
SILType type = v->getType();
if (!(type.is<TupleType>() || type.getStructOrBoundGenericStruct())) {
results.emplace_back(v);
return;
}
// Otherwise, we want to destructure add the destructure and return.
if (getFunction().hasOwnership()) {
auto *i = emitDestructureValueOperation(loc, v);
copy(i->getResults(), std::back_inserter(results));
return;
}
// In non qualified ownership SIL, drop back to using projection code.
SmallVector<Projection, 16> projections;
Projection::getFirstLevelProjections(v->getType(), getModule(), projections);
transform(projections, std::back_inserter(results),
[&](const Projection &p) -> SILValue {
return p.createObjectProjection(*this, loc, v).get();
});
}
void Scheduler::promoteToReadyQueue(SmallVectorImpl<InstRef> &Ready) {
// Scan the set of waiting instructions and promote them to the
// ready queue if operands are all ready.
for (auto I = WaitQueue.begin(), E = WaitQueue.end(); I != E;) {
const unsigned IID = I->first;
Instruction *IS = I->second;
// Check if this instruction is now ready. In case, force
// a transition in state using method 'update()'.
IS->update();
const InstrDesc &Desc = IS->getDesc();
bool IsMemOp = Desc.MayLoad || Desc.MayStore;
if (!IS->isReady() || (IsMemOp && !LSU->isReady({IID, IS}))) {
++I;
continue;
}
Ready.emplace_back(IID, IS);
ReadyQueue[IID] = IS;
auto ToRemove = I;
++I;
WaitQueue.erase(ToRemove);
}
}
void ResourceManager::issueInstruction(
const InstrDesc &Desc,
SmallVectorImpl<std::pair<ResourceRef, double>> &Pipes) {
for (const std::pair<uint64_t, ResourceUsage> &R : Desc.Resources) {
const CycleSegment &CS = R.second.CS;
if (!CS.size()) {
releaseResource(R.first);
continue;
}
assert(CS.begin() == 0 && "Invalid {Start, End} cycles!");
if (!R.second.isReserved()) {
ResourceRef Pipe = selectPipe(R.first);
use(Pipe);
BusyResources[Pipe] += CS.size();
// Replace the resource mask with a valid processor resource index.
const ResourceState &RS = *Resources[Pipe.first];
Pipe.first = RS.getProcResourceID();
Pipes.emplace_back(
std::pair<ResourceRef, double>(Pipe, static_cast<double>(CS.size())));
} else {
assert((countPopulation(R.first) > 1) && "Expected a group!");
// Mark this group as reserved.
assert(R.second.isReserved());
reserveResource(R.first);
BusyResources[ResourceRef(R.first, R.first)] += CS.size();
}
}
}
void CounterExpressionBuilder::extractTerms(Counter C, int Factor,
SmallVectorImpl<Term> &Terms) {
switch (C.getKind()) {
case Counter::Zero:
break;
case Counter::CounterValueReference:
Terms.emplace_back(C.getCounterID(), Factor);
break;
case Counter::Expression:
const auto &E = Expressions[C.getExpressionID()];
extractTerms(E.LHS, Factor, Terms);
extractTerms(
E.RHS, E.Kind == CounterExpression::Subtract ? -Factor : Factor, Terms);
break;
}
}
// TODO: Can we put this on type lowering? It would take a little bit of work
// since we would need to be able to handle aggregate trivial types which is not
// represented today in TypeLowering.
void SILBuilder::emitDestructureAddressOperation(
SILLocation loc, SILValue v, SmallVectorImpl<SILValue> &results) {
// If we do not have a tuple or a struct, add to our results list.
SILType type = v->getType();
if (!(type.is<TupleType>() || type.getStructOrBoundGenericStruct())) {
results.emplace_back(v);
return;
}
SmallVector<Projection, 16> projections;
Projection::getFirstLevelProjections(v->getType(), getModule(), projections);
transform(projections, std::back_inserter(results),
[&](const Projection &p) -> SILValue {
return p.createAddressProjection(*this, loc, v).get();
});
}
bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
SmallVectorImpl<ArgInfo> &SplitArgs,
const DataLayout &DL,
MachineRegisterInfo &MRI,
SplitArgTy PerformArgSplit) const {
const X86TargetLowering &TLI = *getTLI<X86TargetLowering>();
LLVMContext &Context = OrigArg.Ty->getContext();
SmallVector<EVT, 4> SplitVTs;
SmallVector<uint64_t, 4> Offsets;
ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0);
if (SplitVTs.size() != 1) {
// TODO: support struct/array split
return false;
}
EVT VT = SplitVTs[0];
unsigned NumParts = TLI.getNumRegisters(Context, VT);
if (NumParts == 1) {
// replace the original type ( pointer -> GPR ).
SplitArgs.emplace_back(OrigArg.Reg, VT.getTypeForEVT(Context),
OrigArg.Flags, OrigArg.IsFixed);
return true;
}
SmallVector<unsigned, 8> SplitRegs;
EVT PartVT = TLI.getRegisterType(Context, VT);
Type *PartTy = PartVT.getTypeForEVT(Context);
for (unsigned i = 0; i < NumParts; ++i) {
ArgInfo Info =
ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*PartTy, DL)),
PartTy, OrigArg.Flags};
SplitArgs.push_back(Info);
SplitRegs.push_back(Info.Reg);
}
PerformArgSplit(SplitRegs);
return true;
}
void AArch64CallLowering::splitToValueTypes(
const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs,
const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv,
const SplitArgTy &PerformArgSplit) const {
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
LLVMContext &Ctx = OrigArg.Ty->getContext();
if (OrigArg.Ty->isVoidTy())
return;
SmallVector<EVT, 4> SplitVTs;
SmallVector<uint64_t, 4> Offsets;
ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0);
if (SplitVTs.size() == 1) {
// No splitting to do, but we want to replace the original type (e.g. [1 x
// double] -> double).
SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx),
OrigArg.Flags, OrigArg.IsFixed);
return;
}
unsigned FirstRegIdx = SplitArgs.size();
bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
OrigArg.Ty, CallConv, false);
for (auto SplitVT : SplitVTs) {
Type *SplitTy = SplitVT.getTypeForEVT(Ctx);
SplitArgs.push_back(
ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*SplitTy, DL)),
SplitTy, OrigArg.Flags, OrigArg.IsFixed});
if (NeedsRegBlock)
SplitArgs.back().Flags.setInConsecutiveRegs();
}
SplitArgs.back().Flags.setInConsecutiveRegsLast();
for (unsigned i = 0; i < Offsets.size(); ++i)
PerformArgSplit(SplitArgs[FirstRegIdx + i].Reg, Offsets[i] * 8);
}
void AArch64CallLowering::splitToValueTypes(
const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs,
const DataLayout &DL, MachineRegisterInfo &MRI,
const SplitArgTy &PerformArgSplit) const {
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
LLVMContext &Ctx = OrigArg.Ty->getContext();
SmallVector<EVT, 4> SplitVTs;
SmallVector<uint64_t, 4> Offsets;
ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0);
if (SplitVTs.size() == 1) {
// No splitting to do, but we want to replace the original type (e.g. [1 x
// double] -> double).
SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx),
OrigArg.Flags, OrigArg.IsFixed);
return;
}
unsigned FirstRegIdx = SplitArgs.size();
for (auto SplitVT : SplitVTs) {
// FIXME: set split flags if they're actually used (e.g. i128 on AAPCS).
Type *SplitTy = SplitVT.getTypeForEVT(Ctx);
SplitArgs.push_back(
ArgInfo{MRI.createGenericVirtualRegister(LLT{*SplitTy, DL}), SplitTy,
OrigArg.Flags, OrigArg.IsFixed});
}
SmallVector<uint64_t, 4> BitOffsets;
for (auto Offset : Offsets)
BitOffsets.push_back(Offset * 8);
SmallVector<unsigned, 8> SplitRegs;
for (auto I = &SplitArgs[FirstRegIdx]; I != SplitArgs.end(); ++I)
SplitRegs.push_back(I->Reg);
PerformArgSplit(SplitRegs, BitOffsets);
}
bool GuardWideningImpl::combineRangeChecks(
SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks,
SmallVectorImpl<GuardWideningImpl::RangeCheck> &RangeChecksOut) {
unsigned OldCount = Checks.size();
while (!Checks.empty()) {
// Pick all of the range checks with a specific base and length, and try to
// merge them.
Value *CurrentBase = Checks.front().getBase();
Value *CurrentLength = Checks.front().getLength();
SmallVector<GuardWideningImpl::RangeCheck, 3> CurrentChecks;
auto IsCurrentCheck = [&](GuardWideningImpl::RangeCheck &RC) {
return RC.getBase() == CurrentBase && RC.getLength() == CurrentLength;
};
copy_if(Checks, std::back_inserter(CurrentChecks), IsCurrentCheck);
Checks.erase(remove_if(Checks, IsCurrentCheck), Checks.end());
assert(CurrentChecks.size() != 0 && "We know we have at least one!");
if (CurrentChecks.size() < 3) {
RangeChecksOut.insert(RangeChecksOut.end(), CurrentChecks.begin(),
CurrentChecks.end());
continue;
}
// CurrentChecks.size() will typically be 3 here, but so far there has been
// no need to hard-code that fact.
std::sort(CurrentChecks.begin(), CurrentChecks.end(),
[&](const GuardWideningImpl::RangeCheck &LHS,
const GuardWideningImpl::RangeCheck &RHS) {
return LHS.getOffsetValue().slt(RHS.getOffsetValue());
});
// Note: std::sort should not invalidate the ChecksStart iterator.
ConstantInt *MinOffset = CurrentChecks.front().getOffset(),
*MaxOffset = CurrentChecks.back().getOffset();
unsigned BitWidth = MaxOffset->getValue().getBitWidth();
if ((MaxOffset->getValue() - MinOffset->getValue())
.ugt(APInt::getSignedMinValue(BitWidth)))
return false;
APInt MaxDiff = MaxOffset->getValue() - MinOffset->getValue();
const APInt &HighOffset = MaxOffset->getValue();
auto OffsetOK = [&](const GuardWideningImpl::RangeCheck &RC) {
return (HighOffset - RC.getOffsetValue()).ult(MaxDiff);
};
if (MaxDiff.isMinValue() ||
!std::all_of(std::next(CurrentChecks.begin()), CurrentChecks.end(),
OffsetOK))
return false;
// We have a series of f+1 checks as:
//
// I+k_0 u< L ... Chk_0
// I_k_1 u< L ... Chk_1
// ...
// I_k_f u< L ... Chk_(f+1)
//
// with forall i in [0,f): k_f-k_i u< k_f-k_0 ... Precond_0
// k_f-k_0 u< INT_MIN+k_f ... Precond_1
// k_f != k_0 ... Precond_2
//
// Claim:
// Chk_0 AND Chk_(f+1) implies all the other checks
//
// Informal proof sketch:
//
// We will show that the integer range [I+k_0,I+k_f] does not unsigned-wrap
// (i.e. going from I+k_0 to I+k_f does not cross the -1,0 boundary) and
// thus I+k_f is the greatest unsigned value in that range.
//
// This combined with Ckh_(f+1) shows that everything in that range is u< L.
// Via Precond_0 we know that all of the indices in Chk_0 through Chk_(f+1)
// lie in [I+k_0,I+k_f], this proving our claim.
//
// To see that [I+k_0,I+k_f] is not a wrapping range, note that there are
// two possibilities: I+k_0 u< I+k_f or I+k_0 >u I+k_f (they can't be equal
// since k_0 != k_f). In the former case, [I+k_0,I+k_f] is not a wrapping
// range by definition, and the latter case is impossible:
//
// 0-----I+k_f---I+k_0----L---INT_MAX,INT_MIN------------------(-1)
// xxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
//
// For Chk_0 to succeed, we'd have to have k_f-k_0 (the range highlighted
// with 'x' above) to be at least >u INT_MIN.
RangeChecksOut.emplace_back(CurrentChecks.front());
RangeChecksOut.emplace_back(CurrentChecks.back());
}
assert(RangeChecksOut.size() <= OldCount && "We pessimized!");
return RangeChecksOut.size() != OldCount;
}
/// AnalyzeAsmString - Analyze the asm string of the current asm, decomposing
/// it into pieces. If the asm string is erroneous, emit errors and return
/// true, otherwise return false.
unsigned GCCAsmStmt::AnalyzeAsmString(SmallVectorImpl<AsmStringPiece>&Pieces,
const ASTContext &C, unsigned &DiagOffs) const {
StringRef Str = getAsmString()->getString();
const char *StrStart = Str.begin();
const char *StrEnd = Str.end();
const char *CurPtr = StrStart;
// "Simple" inline asms have no constraints or operands, just convert the asm
// string to escape $'s.
if (isSimple()) {
std::string Result;
for (; CurPtr != StrEnd; ++CurPtr) {
switch (*CurPtr) {
case '$':
Result += "$$";
break;
default:
Result += *CurPtr;
break;
}
}
Pieces.push_back(AsmStringPiece(Result));
return 0;
}
// CurStringPiece - The current string that we are building up as we scan the
// asm string.
std::string CurStringPiece;
bool HasVariants = !C.getTargetInfo().hasNoAsmVariants();
while (1) {
// Done with the string?
if (CurPtr == StrEnd) {
if (!CurStringPiece.empty())
Pieces.push_back(AsmStringPiece(CurStringPiece));
return 0;
}
char CurChar = *CurPtr++;
switch (CurChar) {
case '$': CurStringPiece += "$$"; continue;
case '{': CurStringPiece += (HasVariants ? "$(" : "{"); continue;
case '|': CurStringPiece += (HasVariants ? "$|" : "|"); continue;
case '}': CurStringPiece += (HasVariants ? "$)" : "}"); continue;
case '%':
break;
default:
CurStringPiece += CurChar;
continue;
}
// Escaped "%" character in asm string.
if (CurPtr == StrEnd) {
// % at end of string is invalid (no escape).
DiagOffs = CurPtr-StrStart-1;
return diag::err_asm_invalid_escape;
}
char EscapedChar = *CurPtr++;
if (EscapedChar == '%') { // %% -> %
// Escaped percentage sign.
CurStringPiece += '%';
continue;
}
if (EscapedChar == '=') { // %= -> Generate an unique ID.
CurStringPiece += "${:uid}";
continue;
}
// Otherwise, we have an operand. If we have accumulated a string so far,
// add it to the Pieces list.
if (!CurStringPiece.empty()) {
Pieces.push_back(AsmStringPiece(CurStringPiece));
CurStringPiece.clear();
}
// Handle operands that have asmSymbolicName (e.g., %x[foo]) and those that
// don't (e.g., %x4). 'x' following the '%' is the constraint modifier.
const char *Begin = CurPtr - 1; // Points to the character following '%'.
const char *Percent = Begin - 1; // Points to '%'.
if (isLetter(EscapedChar)) {
if (CurPtr == StrEnd) { // Premature end.
DiagOffs = CurPtr-StrStart-1;
return diag::err_asm_invalid_escape;
}
EscapedChar = *CurPtr++;
}
const TargetInfo &TI = C.getTargetInfo();
const SourceManager &SM = C.getSourceManager();
const LangOptions &LO = C.getLangOpts();
// Handle operands that don't have asmSymbolicName (e.g., %x4).
if (isDigit(EscapedChar)) {
// %n - Assembler operand n
unsigned N = 0;
--CurPtr;
while (CurPtr != StrEnd && isDigit(*CurPtr))
N = N*10 + ((*CurPtr++)-'0');
unsigned NumOperands =
getNumOutputs() + getNumPlusOperands() + getNumInputs();
if (N >= NumOperands) {
DiagOffs = CurPtr-StrStart-1;
return diag::err_asm_invalid_operand_number;
}
// Str contains "x4" (Operand without the leading %).
std::string Str(Begin, CurPtr - Begin);
// (BeginLoc, EndLoc) represents the range of the operand we are currently
// processing. Unlike Str, the range includes the leading '%'.
SourceLocation BeginLoc =
getAsmString()->getLocationOfByte(Percent - StrStart, SM, LO, TI);
SourceLocation EndLoc =
getAsmString()->getLocationOfByte(CurPtr - StrStart, SM, LO, TI);
Pieces.emplace_back(N, std::move(Str), BeginLoc, EndLoc);
continue;
}
// Handle operands that have asmSymbolicName (e.g., %x[foo]).
if (EscapedChar == '[') {
DiagOffs = CurPtr-StrStart-1;
// Find the ']'.
const char *NameEnd = (const char*)memchr(CurPtr, ']', StrEnd-CurPtr);
if (NameEnd == nullptr)
return diag::err_asm_unterminated_symbolic_operand_name;
if (NameEnd == CurPtr)
return diag::err_asm_empty_symbolic_operand_name;
StringRef SymbolicName(CurPtr, NameEnd - CurPtr);
int N = getNamedOperand(SymbolicName);
if (N == -1) {
// Verify that an operand with that name exists.
DiagOffs = CurPtr-StrStart;
return diag::err_asm_unknown_symbolic_operand_name;
}
// Str contains "x[foo]" (Operand without the leading %).
std::string Str(Begin, NameEnd + 1 - Begin);
// (BeginLoc, EndLoc) represents the range of the operand we are currently
// processing. Unlike Str, the range includes the leading '%'.
SourceLocation BeginLoc =
getAsmString()->getLocationOfByte(Percent - StrStart, SM, LO, TI);
SourceLocation EndLoc =
getAsmString()->getLocationOfByte(NameEnd + 1 - StrStart, SM, LO, TI);
Pieces.emplace_back(N, std::move(Str), BeginLoc, EndLoc);
CurPtr = NameEnd+1;
continue;
}
DiagOffs = CurPtr-StrStart-1;
return diag::err_asm_invalid_escape;
}
}
bool SILValueOwnershipChecker::gatherUsers(
SmallVectorImpl<BranchPropagatedUser> &lifetimeEndingUsers,
SmallVectorImpl<BranchPropagatedUser> &nonLifetimeEndingUsers,
SmallVectorImpl<BranchPropagatedUser> &implicitRegularUsers) {
// See if Value is guaranteed. If we are guaranteed and not forwarding, then
// we need to look through subobject uses for more uses. Otherwise, if we are
// forwarding, we do not create any lifetime ending users/non lifetime ending
// users since we verify against our base.
auto ownershipKind = value.getOwnershipKind();
bool isGuaranteed = ownershipKind == ValueOwnershipKind::Guaranteed;
bool isOwned = ownershipKind == ValueOwnershipKind::Owned;
if (isGuaranteed && isGuaranteedForwardingValue(value))
return true;
// Then gather up our initial list of users.
SmallVector<Operand *, 8> users;
std::copy(value->use_begin(), value->use_end(), std::back_inserter(users));
auto addCondBranchToList = [](SmallVectorImpl<BranchPropagatedUser> &list,
CondBranchInst *cbi, unsigned operandIndex) {
if (cbi->isConditionOperandIndex(operandIndex)) {
list.emplace_back(cbi);
return;
}
bool isTrueOperand = cbi->isTrueOperandIndex(operandIndex);
list.emplace_back(cbi, isTrueOperand ? CondBranchInst::TrueIdx
: CondBranchInst::FalseIdx);
};
bool foundError = false;
while (!users.empty()) {
Operand *op = users.pop_back_val();
SILInstruction *user = op->getUser();
// If this op is a type dependent operand, skip it. It is not interesting
// from an ownership perspective.
if (user->isTypeDependentOperand(*op))
continue;
bool isGuaranteedSubValue = false;
if (isGuaranteed && isGuaranteedForwardingInst(op->getUser())) {
isGuaranteedSubValue = true;
}
auto opOwnershipKindMap = op->getOwnershipKindMap(isGuaranteedSubValue);
// If our ownership kind doesn't match, track that we found an error, emit
// an error message optionally and then continue.
if (!opOwnershipKindMap.canAcceptKind(ownershipKind)) {
foundError = true;
// If we did not support /any/ ownership kind, it means that we found a
// conflicting answer so the kind map that was returned is the empty
// map. Put out a more specific error here.
if (!opOwnershipKindMap.data.any()) {
handleError([&]() {
llvm::errs() << "Function: '" << user->getFunction()->getName()
<< "'\n"
<< "Ill-formed SIL! Unable to compute ownership kind "
"map for user?!\n"
<< "For terminator users, check that successors have "
"compatible ownership kinds.\n"
<< "Value: " << op->get() << "User: "******"Operand Number: " << op->getOperandNumber() << '\n'
<< "Conv: " << ownershipKind << "\n\n";
});
continue;
}
handleError([&]() {
llvm::errs() << "Function: '" << user->getFunction()->getName() << "'\n"
<< "Have operand with incompatible ownership?!\n"
<< "Value: " << op->get() << "User: "******"Operand Number: " << op->getOperandNumber() << '\n'
<< "Conv: " << ownershipKind << '\n'
<< "OwnershipMap:\n"
<< opOwnershipKindMap << '\n';
});
continue;
}
auto lifetimeConstraint =
opOwnershipKindMap.getLifetimeConstraint(ownershipKind);
if (lifetimeConstraint == UseLifetimeConstraint::MustBeInvalidated) {
LLVM_DEBUG(llvm::dbgs() << " Lifetime Ending User: "******" Regular User: "******"Our value is guaranteed and this is a forwarding instruction. "
"Should have guaranteed ownership as well.");
copy(result->getUses(), std::back_inserter(users));
}
continue;
}
assert(user->getResults().empty());
auto *ti = dyn_cast<TermInst>(user);
if (!ti) {
continue;
}
// Otherwise if we have a terminator, add any as uses any end_borrow to
// ensure that the subscope is completely enclsed within the super scope. We
// require all of our arguments to be either trivial or guaranteed.
for (auto &succ : ti->getSuccessors()) {
auto *succBlock = succ.getBB();
// If we do not have any arguments, then continue.
if (succBlock->args_empty())
continue;
// Otherwise, make sure that all arguments are trivial or guaranteed. If
// we fail, emit an error.
//
// TODO: We could ignore this error and emit a more specific error on the
// actual terminator.
for (auto *succArg : succBlock->getPhiArguments()) {
// *NOTE* We do not emit an error here since we want to allow for more
// specific errors to be found during use_verification.
//
// TODO: Add a flag that associates the terminator instruction with
// needing to be verified. If it isn't verified appropriately, assert
// when the verifier is destroyed.
auto succArgOwnershipKind = succArg->getOwnershipKind();
if (!succArgOwnershipKind.isCompatibleWith(ownershipKind)) {
// This is where the error would go.
continue;
}
// If we have an any value, just continue.
if (succArgOwnershipKind == ValueOwnershipKind::Any)
continue;
// Otherwise add all end_borrow users for this BBArg to the
// implicit regular user list. We know that BBArg must be
// completely joint post-dominated by these users, so we use
// them to ensure that all of BBArg's uses are completely
// enclosed within the end_borrow of this argument.
for (auto *op : succArg->getUses()) {
if (auto *ebi = dyn_cast<EndBorrowInst>(op->getUser())) {
implicitRegularUsers.push_back(ebi);
}
}
}
}
}
// Return true if we did not have an error and false if we did find an error.
//
// The reason why we use this extra variable is to make sure that when we are
// testing, we print out all mismatching pairs rather than just the first.
return !foundError;
}
/// Analyze MBB to check if its terminating branch can be turned into an
/// implicit null check. If yes, append a description of the said null check to
/// NullCheckList and return true, else return false.
bool ImplicitNullChecks::analyzeBlockForNullChecks(
MachineBasicBlock &MBB, SmallVectorImpl<NullCheck> &NullCheckList) {
typedef TargetInstrInfo::MachineBranchPredicate MachineBranchPredicate;
MDNode *BranchMD = nullptr;
if (auto *BB = MBB.getBasicBlock())
BranchMD = BB->getTerminator()->getMetadata(LLVMContext::MD_make_implicit);
if (!BranchMD)
return false;
MachineBranchPredicate MBP;
if (TII->analyzeBranchPredicate(MBB, MBP, true))
return false;
// Is the predicate comparing an integer to zero?
if (!(MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
(MBP.Predicate == MachineBranchPredicate::PRED_NE ||
MBP.Predicate == MachineBranchPredicate::PRED_EQ)))
return false;
// If we cannot erase the test instruction itself, then making the null check
// implicit does not buy us much.
if (!MBP.SingleUseCondition)
return false;
MachineBasicBlock *NotNullSucc, *NullSucc;
if (MBP.Predicate == MachineBranchPredicate::PRED_NE) {
NotNullSucc = MBP.TrueDest;
NullSucc = MBP.FalseDest;
} else {
NotNullSucc = MBP.FalseDest;
NullSucc = MBP.TrueDest;
}
// We handle the simplest case for now. We can potentially do better by using
// the machine dominator tree.
if (NotNullSucc->pred_size() != 1)
return false;
// Starting with a code fragment like:
//
// test %RAX, %RAX
// jne LblNotNull
//
// LblNull:
// callq throw_NullPointerException
//
// LblNotNull:
// Inst0
// Inst1
// ...
// Def = Load (%RAX + <offset>)
// ...
//
//
// we want to end up with
//
// Def = FaultingLoad (%RAX + <offset>), LblNull
// jmp LblNotNull ;; explicit or fallthrough
//
// LblNotNull:
// Inst0
// Inst1
// ...
//
// LblNull:
// callq throw_NullPointerException
//
//
// To see why this is legal, consider the two possibilities:
//
// 1. %RAX is null: since we constrain <offset> to be less than PageSize, the
// load instruction dereferences the null page, causing a segmentation
// fault.
//
// 2. %RAX is not null: in this case we know that the load cannot fault, as
// otherwise the load would've faulted in the original program too and the
// original program would've been undefined.
//
// This reasoning cannot be extended to justify hoisting through arbitrary
// control flow. For instance, in the example below (in pseudo-C)
//
// if (ptr == null) { throw_npe(); unreachable; }
// if (some_cond) { return 42; }
// v = ptr->field; // LD
// ...
//
// we cannot (without code duplication) use the load marked "LD" to null check
// ptr -- clause (2) above does not apply in this case. In the above program
// the safety of ptr->field can be dependent on some_cond; and, for instance,
// ptr could be some non-null invalid reference that never gets loaded from
// because some_cond is always true.
const unsigned PointerReg = MBP.LHS.getReg();
SmallVector<MachineInstr *, 8> InstsSeenSoFar;
for (auto &MI : *NotNullSucc) {
if (!canHandle(&MI) || InstsSeenSoFar.size() >= MaxInstsToConsider)
return false;
MachineInstr *Dependence;
SuitabilityResult SR = isSuitableMemoryOp(MI, PointerReg, InstsSeenSoFar);
if (SR == SR_Impossible)
return false;
if (SR == SR_Suitable &&
canHoistInst(&MI, PointerReg, InstsSeenSoFar, NullSucc, Dependence)) {
NullCheckList.emplace_back(&MI, MBP.ConditionDef, &MBB, NotNullSucc,
NullSucc, Dependence);
return true;
}
InstsSeenSoFar.push_back(&MI);
}
return false;
}
/// Analyze MBB to check if its terminating branch can be turned into an
/// implicit null check. If yes, append a description of the said null check to
/// NullCheckList and return true, else return false.
bool ImplicitNullChecks::analyzeBlockForNullChecks(
MachineBasicBlock &MBB, SmallVectorImpl<NullCheck> &NullCheckList) {
typedef TargetInstrInfo::MachineBranchPredicate MachineBranchPredicate;
MDNode *BranchMD = nullptr;
if (auto *BB = MBB.getBasicBlock())
BranchMD = BB->getTerminator()->getMetadata(LLVMContext::MD_make_implicit);
if (!BranchMD)
return false;
MachineBranchPredicate MBP;
if (TII->AnalyzeBranchPredicate(MBB, MBP, true))
return false;
// Is the predicate comparing an integer to zero?
if (!(MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
(MBP.Predicate == MachineBranchPredicate::PRED_NE ||
MBP.Predicate == MachineBranchPredicate::PRED_EQ)))
return false;
// If we cannot erase the test instruction itself, then making the null check
// implicit does not buy us much.
if (!MBP.SingleUseCondition)
return false;
MachineBasicBlock *NotNullSucc, *NullSucc;
if (MBP.Predicate == MachineBranchPredicate::PRED_NE) {
NotNullSucc = MBP.TrueDest;
NullSucc = MBP.FalseDest;
} else {
NotNullSucc = MBP.FalseDest;
NullSucc = MBP.TrueDest;
}
// We handle the simplest case for now. We can potentially do better by using
// the machine dominator tree.
if (NotNullSucc->pred_size() != 1)
return false;
// Starting with a code fragment like:
//
// test %RAX, %RAX
// jne LblNotNull
//
// LblNull:
// callq throw_NullPointerException
//
// LblNotNull:
// Inst0
// Inst1
// ...
// Def = Load (%RAX + <offset>)
// ...
//
//
// we want to end up with
//
// Def = FaultingLoad (%RAX + <offset>), LblNull
// jmp LblNotNull ;; explicit or fallthrough
//
// LblNotNull:
// Inst0
// Inst1
// ...
//
// LblNull:
// callq throw_NullPointerException
//
//
// To see why this is legal, consider the two possibilities:
//
// 1. %RAX is null: since we constrain <offset> to be less than PageSize, the
// load instruction dereferences the null page, causing a segmentation
// fault.
//
// 2. %RAX is not null: in this case we know that the load cannot fault, as
// otherwise the load would've faulted in the original program too and the
// original program would've been undefined.
//
// This reasoning cannot be extended to justify hoisting through arbitrary
// control flow. For instance, in the example below (in pseudo-C)
//
// if (ptr == null) { throw_npe(); unreachable; }
// if (some_cond) { return 42; }
// v = ptr->field; // LD
// ...
//
// we cannot (without code duplication) use the load marked "LD" to null check
// ptr -- clause (2) above does not apply in this case. In the above program
// the safety of ptr->field can be dependent on some_cond; and, for instance,
// ptr could be some non-null invalid reference that never gets loaded from
// because some_cond is always true.
unsigned PointerReg = MBP.LHS.getReg();
HazardDetector HD(*TRI, *AA);
for (auto MII = NotNullSucc->begin(), MIE = NotNullSucc->end(); MII != MIE;
++MII) {
MachineInstr *MI = &*MII;
unsigned BaseReg;
int64_t Offset;
MachineInstr *Dependency = nullptr;
if (TII->getMemOpBaseRegImmOfs(MI, BaseReg, Offset, TRI))
if (MI->mayLoad() && !MI->isPredicable() && BaseReg == PointerReg &&
Offset < PageSize && MI->getDesc().getNumDefs() <= 1 &&
HD.isSafeToHoist(MI, Dependency)) {
auto DependencyOperandIsOk = [&](MachineOperand &MO) {
assert(!(MO.isReg() && MO.isUse()) &&
"No transitive dependendencies please!");
if (!MO.isReg() || !MO.getReg() || !MO.isDef())
return true;
// Make sure that we won't clobber any live ins to the sibling block
// by hoisting Dependency. For instance, we can't hoist INST to
// before the null check (even if it safe, and does not violate any
// dependencies in the non_null_block) if %rdx is live in to
// _null_block.
//
// test %rcx, %rcx
// je _null_block
// _non_null_block:
// %rdx<def> = INST
// ...
if (AnyAliasLiveIn(TRI, NullSucc, MO.getReg()))
return false;
// Make sure Dependency isn't re-defining the base register. Then we
// won't get the memory operation on the address we want.
if (TRI->regsOverlap(MO.getReg(), BaseReg))
return false;
return true;
};
bool DependencyOperandsAreOk =
!Dependency ||
all_of(Dependency->operands(), DependencyOperandIsOk);
if (DependencyOperandsAreOk) {
NullCheckList.emplace_back(MI, MBP.ConditionDef, &MBB, NotNullSucc,
NullSucc, Dependency);
return true;
}
}
HD.rememberInstruction(MI);
if (HD.isClobbered())
return false;
}
return false;
}
void IDFCalculator<NodeTy>::calculate(
SmallVectorImpl<BasicBlock *> &PHIBlocks) {
// Use a priority queue keyed on dominator tree level so that inserted nodes
// are handled from the bottom of the dominator tree upwards.
typedef std::pair<DomTreeNode *, unsigned> DomTreeNodePair;
typedef std::priority_queue<DomTreeNodePair, SmallVector<DomTreeNodePair, 32>,
less_second> IDFPriorityQueue;
IDFPriorityQueue PQ;
for (BasicBlock *BB : *DefBlocks) {
if (DomTreeNode *Node = DT.getNode(BB))
PQ.push({Node, Node->getLevel()});
}
SmallVector<DomTreeNode *, 32> Worklist;
SmallPtrSet<DomTreeNode *, 32> VisitedPQ;
SmallPtrSet<DomTreeNode *, 32> VisitedWorklist;
while (!PQ.empty()) {
DomTreeNodePair RootPair = PQ.top();
PQ.pop();
DomTreeNode *Root = RootPair.first;
unsigned RootLevel = RootPair.second;
// Walk all dominator tree children of Root, inspecting their CFG edges with
// targets elsewhere on the dominator tree. Only targets whose level is at
// most Root's level are added to the iterated dominance frontier of the
// definition set.
Worklist.clear();
Worklist.push_back(Root);
VisitedWorklist.insert(Root);
while (!Worklist.empty()) {
DomTreeNode *Node = Worklist.pop_back_val();
BasicBlock *BB = Node->getBlock();
// Succ is the successor in the direction we are calculating IDF, so it is
// successor for IDF, and predecessor for Reverse IDF.
for (auto *Succ : children<NodeTy>(BB)) {
DomTreeNode *SuccNode = DT.getNode(Succ);
// Quickly skip all CFG edges that are also dominator tree edges instead
// of catching them below.
if (SuccNode->getIDom() == Node)
continue;
const unsigned SuccLevel = SuccNode->getLevel();
if (SuccLevel > RootLevel)
continue;
if (!VisitedPQ.insert(SuccNode).second)
continue;
BasicBlock *SuccBB = SuccNode->getBlock();
if (useLiveIn && !LiveInBlocks->count(SuccBB))
continue;
PHIBlocks.emplace_back(SuccBB);
if (!DefBlocks->count(SuccBB))
PQ.push(std::make_pair(SuccNode, SuccLevel));
}
for (auto DomChild : *Node) {
if (VisitedWorklist.insert(DomChild).second)
Worklist.push_back(DomChild);
}
}
}
}
/// Analyze MBB to check if its terminating branch can be turned into an
/// implicit null check. If yes, append a description of the said null check to
/// NullCheckList and return true, else return false.
bool ImplicitNullChecks::analyzeBlockForNullChecks(
MachineBasicBlock &MBB, SmallVectorImpl<NullCheck> &NullCheckList) {
typedef TargetInstrInfo::MachineBranchPredicate MachineBranchPredicate;
MDNode *BranchMD =
MBB.getBasicBlock()
? MBB.getBasicBlock()->getTerminator()->getMetadata(LLVMContext::MD_make_implicit)
: nullptr;
if (!BranchMD)
return false;
MachineBranchPredicate MBP;
if (TII->AnalyzeBranchPredicate(MBB, MBP, true))
return false;
// Is the predicate comparing an integer to zero?
if (!(MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
(MBP.Predicate == MachineBranchPredicate::PRED_NE ||
MBP.Predicate == MachineBranchPredicate::PRED_EQ)))
return false;
// If we cannot erase the test instruction itself, then making the null check
// implicit does not buy us much.
if (!MBP.SingleUseCondition)
return false;
MachineBasicBlock *NotNullSucc, *NullSucc;
if (MBP.Predicate == MachineBranchPredicate::PRED_NE) {
NotNullSucc = MBP.TrueDest;
NullSucc = MBP.FalseDest;
} else {
NotNullSucc = MBP.FalseDest;
NullSucc = MBP.TrueDest;
}
// We handle the simplest case for now. We can potentially do better by using
// the machine dominator tree.
if (NotNullSucc->pred_size() != 1)
return false;
// Starting with a code fragment like:
//
// test %RAX, %RAX
// jne LblNotNull
//
// LblNull:
// callq throw_NullPointerException
//
// LblNotNull:
// Inst0
// Inst1
// ...
// Def = Load (%RAX + <offset>)
// ...
//
//
// we want to end up with
//
// Def = TrappingLoad (%RAX + <offset>), LblNull
// jmp LblNotNull ;; explicit or fallthrough
//
// LblNotNull:
// Inst0
// Inst1
// ...
//
// LblNull:
// callq throw_NullPointerException
//
unsigned PointerReg = MBP.LHS.getReg();
// As we scan NotNullSucc for a suitable load instruction, we keep track of
// the registers defined and used by the instructions we scan past. This bit
// of information lets us decide if it is legal to hoist the load instruction
// we find (if we do find such an instruction) to before NotNullSucc.
DenseSet<unsigned> RegDefs, RegUses;
// Returns true if it is safe to reorder MI to before NotNullSucc.
auto IsSafeToHoist = [&](MachineInstr *MI) {
// Right now we don't want to worry about LLVM's memory model. This can be
// made more precise later.
for (auto *MMO : MI->memoperands())
if (!MMO->isUnordered())
return false;
for (auto &MO : MI->operands()) {
if (MO.isReg() && MO.getReg()) {
for (unsigned Reg : RegDefs)
if (TRI->regsOverlap(Reg, MO.getReg()))
return false; // We found a write-after-write or read-after-write
if (MO.isDef())
for (unsigned Reg : RegUses)
if (TRI->regsOverlap(Reg, MO.getReg()))
return false; // We found a write-after-read
}
}
return true;
};
for (auto MII = NotNullSucc->begin(), MIE = NotNullSucc->end(); MII != MIE;
++MII) {
MachineInstr *MI = &*MII;
unsigned BaseReg, Offset;
if (TII->getMemOpBaseRegImmOfs(MI, BaseReg, Offset, TRI))
if (MI->mayLoad() && !MI->isPredicable() && BaseReg == PointerReg &&
Offset < PageSize && MI->getDesc().getNumDefs() <= 1 &&
IsSafeToHoist(MI)) {
NullCheckList.emplace_back(MI, MBP.ConditionDef, &MBB, NotNullSucc,
NullSucc);
return true;
}
// MI did not match our criteria for conversion to a trapping load. Check
// if we can continue looking.
if (MI->mayStore() || MI->hasUnmodeledSideEffects())
return false;
for (auto *MMO : MI->memoperands())
// Right now we don't want to worry about LLVM's memory model.
if (!MMO->isUnordered())
return false;
// It _may_ be okay to reorder a later load instruction across MI. Make a
// note of its operands so that we can make the legality check if we find a
// suitable load instruction:
for (auto &MO : MI->operands()) {
if (!MO.isReg() || !MO.getReg())
continue;
if (MO.isDef())
RegDefs.insert(MO.getReg());
else
RegUses.insert(MO.getReg());
}
}
return false;
}
