/// Replaces the given call site (Call or Invoke) with a gc.statepoint
/// intrinsic with an empty deoptimization arguments list. This does
/// NOT do explicit relocation for GC support.
static Value *ReplaceWithStatepoint(const CallSite &CS /* to replace */) {
assert(CS.getInstruction()->getModule() && "must be set");
// TODO: technically, a pass is not allowed to get functions from within a
// function pass since it might trigger a new function addition. Refactor
// this logic out to the initialization of the pass. Doesn't appear to
// matter in practice.
// Then go ahead and use the builder do actually do the inserts. We insert
// immediately before the previous instruction under the assumption that all
// arguments will be available here. We can't insert afterwards since we may
// be replacing a terminator.
IRBuilder<> Builder(CS.getInstruction());
// Note: The gc args are not filled in at this time, that's handled by
// RewriteStatepointsForGC (which is currently under review).
// Create the statepoint given all the arguments
Instruction *Token = nullptr;
uint64_t ID;
uint32_t NumPatchBytes;
AttributeSet OriginalAttrs = CS.getAttributes();
Attribute AttrID =
OriginalAttrs.getAttribute(AttributeSet::FunctionIndex, "statepoint-id");
Attribute AttrNumPatchBytes = OriginalAttrs.getAttribute(
AttributeSet::FunctionIndex, "statepoint-num-patch-bytes");
AttrBuilder AttrsToRemove;
bool HasID = AttrID.isStringAttribute() &&
!AttrID.getValueAsString().getAsInteger(10, ID);
if (HasID)
AttrsToRemove.addAttribute("statepoint-id");
else
ID = 0xABCDEF00;
bool HasNumPatchBytes =
AttrNumPatchBytes.isStringAttribute() &&
!AttrNumPatchBytes.getValueAsString().getAsInteger(10, NumPatchBytes);
if (HasNumPatchBytes)
AttrsToRemove.addAttribute("statepoint-num-patch-bytes");
else
NumPatchBytes = 0;
OriginalAttrs = OriginalAttrs.removeAttributes(
CS.getInstruction()->getContext(), AttributeSet::FunctionIndex,
AttrsToRemove);
if (CS.isCall()) {
CallInst *ToReplace = cast<CallInst>(CS.getInstruction());
CallInst *Call = Builder.CreateGCStatepointCall(
ID, NumPatchBytes, CS.getCalledValue(),
makeArrayRef(CS.arg_begin(), CS.arg_end()), None, None,
"safepoint_token");
Call->setTailCall(ToReplace->isTailCall());
Call->setCallingConv(ToReplace->getCallingConv());
// In case if we can handle this set of attributes - set up function
// attributes directly on statepoint and return attributes later for
// gc_result intrinsic.
Call->setAttributes(OriginalAttrs.getFnAttributes());
Token = Call;
// Put the following gc_result and gc_relocate calls immediately after
// the old call (which we're about to delete).
assert(ToReplace->getNextNode() && "not a terminator, must have next");
Builder.SetInsertPoint(ToReplace->getNextNode());
Builder.SetCurrentDebugLocation(ToReplace->getNextNode()->getDebugLoc());
} else if (CS.isInvoke()) {
InvokeInst *ToReplace = cast<InvokeInst>(CS.getInstruction());
// Insert the new invoke into the old block. We'll remove the old one in a
// moment at which point this will become the new terminator for the
// original block.
Builder.SetInsertPoint(ToReplace->getParent());
InvokeInst *Invoke = Builder.CreateGCStatepointInvoke(
ID, NumPatchBytes, CS.getCalledValue(), ToReplace->getNormalDest(),
ToReplace->getUnwindDest(), makeArrayRef(CS.arg_begin(), CS.arg_end()),
None, None, "safepoint_token");
Invoke->setCallingConv(ToReplace->getCallingConv());
// In case if we can handle this set of attributes - set up function
// attributes directly on statepoint and return attributes later for
// gc_result intrinsic.
Invoke->setAttributes(OriginalAttrs.getFnAttributes());
Token = Invoke;
// We'll insert the gc.result into the normal block
BasicBlock *NormalDest = ToReplace->getNormalDest();
// Can not insert gc.result in case of phi nodes preset.
// Should have removed this cases prior to running this function
assert(!isa<PHINode>(NormalDest->begin()));
Instruction *IP = &*(NormalDest->getFirstInsertionPt());
Builder.SetInsertPoint(IP);
} else {
llvm_unreachable("unexpect type of CallSite");
}
assert(Token);
// Handle the return value of the original call - update all uses to use a
// gc_result hanging off the statepoint node we just inserted
// Only add the gc_result iff there is actually a used result
if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) {
std::string TakenName =
CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : "";
CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), TakenName);
GCResult->setAttributes(OriginalAttrs.getRetAttributes());
return GCResult;
} else {
// No return value for the call.
return nullptr;
}
}
CallInst* ArgumentRecovery::createCallSite(TargetInfo& targetInfo, const CallInformation& ci, Value& callee, Value& callerRegisters, Instruction& insertionPoint)
{
LLVMContext& ctx = insertionPoint.getContext();
unsigned pointerSize = targetInfo.getPointerSize() * CHAR_BIT;
Type* integer = Type::getIntNTy(ctx, pointerSize);
Type* integerPtr = Type::getIntNPtrTy(ctx, pointerSize, 1);
// Create GEPs in caller for each value that we need.
// Load SP first since we might need it.
auto spPtr = targetInfo.getRegister(&callerRegisters, *targetInfo.getStackPointer());
spPtr->insertBefore(&insertionPoint);
auto spValue = new LoadInst(spPtr, "sp", &insertionPoint);
// Fix parameters
SmallVector<Value*, 8> arguments;
for (const auto& vi : ci.parameters())
{
if (vi.type == ValueInformation::IntegerRegister)
{
auto registerPtr = targetInfo.getRegister(&callerRegisters, *vi.registerInfo);
registerPtr->insertBefore(&insertionPoint);
auto registerValue = new LoadInst(registerPtr, vi.registerInfo->name, &insertionPoint);
arguments.push_back(registerValue);
}
else if (vi.type == ValueInformation::Stack)
{
// assume one pointer-sized word
auto offsetConstant = ConstantInt::get(integer, vi.frameBaseOffset);
auto offset = BinaryOperator::Create(BinaryOperator::Add, spValue, offsetConstant, "", &insertionPoint);
auto casted = new IntToPtrInst(offset, integerPtr, "", &insertionPoint);
auto loaded = new LoadInst(casted, "", &insertionPoint);
arguments.push_back(loaded);
}
else
{
llvm_unreachable("not implemented");
}
}
CallInst* newCall = CallInst::Create(&callee, arguments, "", &insertionPoint);
// Fix return value(s)
unsigned i = 0;
Instruction* returnInsertionPoint = newCall->getNextNode();
for (const auto& vi : ci.returns())
{
if (vi.type == ValueInformation::IntegerRegister)
{
Value* registerValue = ci.returns_size() == 1
? static_cast<Value*>(newCall)
: ExtractValueInst::Create(newCall, {i}, vi.registerInfo->name, returnInsertionPoint);
auto registerPtr = targetInfo.getRegister(&callerRegisters, *vi.registerInfo);
registerPtr->insertBefore(returnInsertionPoint);
new StoreInst(registerValue, registerPtr, returnInsertionPoint);
}
else
{
llvm_unreachable("not implemented");
}
i++;
}
return newCall;
}
/// \brief Recursively handle the condition leading to a loop
Value *SIAnnotateControlFlow::handleLoopCondition(
Value *Cond, PHINode *Broken, llvm::Loop *L, BranchInst *Term,
SmallVectorImpl<WeakTrackingVH> &LoopPhiConditions) {
// Only search through PHI nodes which are inside the loop. If we try this
// with PHI nodes that are outside of the loop, we end up inserting new PHI
// nodes outside of the loop which depend on values defined inside the loop.
// This will break the module with
// 'Instruction does not dominate all users!' errors.
PHINode *Phi = nullptr;
if ((Phi = dyn_cast<PHINode>(Cond)) && L->contains(Phi)) {
BasicBlock *Parent = Phi->getParent();
PHINode *NewPhi = PHINode::Create(Int64, 0, "loop.phi", &Parent->front());
Value *Ret = NewPhi;
// Handle all non-constant incoming values first
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
Value *Incoming = Phi->getIncomingValue(i);
BasicBlock *From = Phi->getIncomingBlock(i);
if (isa<ConstantInt>(Incoming)) {
NewPhi->addIncoming(Broken, From);
continue;
}
Phi->setIncomingValue(i, BoolFalse);
Value *PhiArg = handleLoopCondition(Incoming, Broken, L,
Term, LoopPhiConditions);
NewPhi->addIncoming(PhiArg, From);
}
BasicBlock *IDom = DT->getNode(Parent)->getIDom()->getBlock();
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
Value *Incoming = Phi->getIncomingValue(i);
if (Incoming != BoolTrue)
continue;
BasicBlock *From = Phi->getIncomingBlock(i);
if (From == IDom) {
// We're in the following situation:
// IDom/From
// | \
// | If-block
// | /
// Parent
// where we want to break out of the loop if the If-block is not taken.
// Due to the depth-first traversal, there should be an end.cf
// intrinsic in Parent, and we insert an else.break before it.
//
// Note that the end.cf need not be the first non-phi instruction
// of parent, particularly when we're dealing with a multi-level
// break, but it should occur within a group of intrinsic calls
// at the beginning of the block.
CallInst *OldEnd = dyn_cast<CallInst>(Parent->getFirstInsertionPt());
while (OldEnd && OldEnd->getCalledFunction() != EndCf)
OldEnd = dyn_cast<CallInst>(OldEnd->getNextNode());
if (OldEnd && OldEnd->getCalledFunction() == EndCf) {
Value *Args[] = { OldEnd->getArgOperand(0), NewPhi };
Ret = CallInst::Create(ElseBreak, Args, "", OldEnd);
continue;
}
}
TerminatorInst *Insert = From->getTerminator();
Value *PhiArg = CallInst::Create(Break, Broken, "", Insert);
NewPhi->setIncomingValue(i, PhiArg);
}
LoopPhiConditions.push_back(WeakTrackingVH(Phi));
return Ret;
}
if (Instruction *Inst = dyn_cast<Instruction>(Cond)) {
BasicBlock *Parent = Inst->getParent();
Instruction *Insert;
if (L->contains(Inst)) {
Insert = Parent->getTerminator();
} else {
Insert = L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
}
Value *Args[] = { Cond, Broken };
return CallInst::Create(IfBreak, Args, "", Insert);
}
// Insert IfBreak in the loop header TERM for constant COND other than true.
if (isa<Constant>(Cond)) {
Instruction *Insert = Cond == BoolTrue ?
Term : L->getHeader()->getTerminator();
Value *Args[] = { Cond, Broken };
return CallInst::Create(IfBreak, Args, "", Insert);
}
llvm_unreachable("Unhandled loop condition!");
}
void DuettoNativeRewriter::rewriteConstructorImplementation(Module& M, Function& F)
{
//Copy the code in a function with the right signature
Function* newFunc=getReturningConstructor(M, &F);
if(!newFunc->empty())
return;
//Visit each instruction and take note of the ones that needs to be replaced
Function::const_iterator B=F.begin();
Function::const_iterator BE=F.end();
ValueToValueMapTy valueMap;
CallInst* lowerConstructor = NULL;
const CallInst* oldLowerConstructor = NULL;
for(;B!=BE;++B)
{
BasicBlock::const_iterator I=B->begin();
BasicBlock::const_iterator IE=B->end();
for(;I!=IE;++I)
{
if(I->getOpcode()!=Instruction::Call)
continue;
const CallInst* callInst=cast<CallInst>(&(*I));
Function* f=callInst->getCalledFunction();
if(!f)
continue;
const char* startOfType;
const char* endOfType;
if(!DuettoNativeRewriter::isBuiltinConstructor(f->getName().data(), startOfType, endOfType))
continue;
//Check that the constructor is for 'this'
if(callInst->getOperand(0)!=F.arg_begin())
continue;
//If this is another constructor for the same type, change it to a
//returning constructor and use it as the 'this' argument
Function* newFunc = getReturningConstructor(M, f);
llvm::SmallVector<Value*, 4> newArgs;
for(unsigned i=1;i<callInst->getNumArgOperands();i++)
newArgs.push_back(callInst->getArgOperand(i));
lowerConstructor = CallInst::Create(newFunc, newArgs);
oldLowerConstructor = callInst;
break;
}
if(lowerConstructor)
break;
}
//Clone the linkage first
newFunc->setLinkage(F.getLinkage());
Function::arg_iterator origArg=++F.arg_begin();
Function::arg_iterator newArg=newFunc->arg_begin();
valueMap.insert(make_pair(F.arg_begin(), lowerConstructor));
for(unsigned i=1;i<F.arg_size();i++)
{
valueMap.insert(make_pair(&(*origArg), &(*newArg)));
++origArg;
++newArg;
}
SmallVector<ReturnInst*, 4> returns;
CloneFunctionInto(newFunc, &F, valueMap, false, returns);
//Find the right place to add the base construtor call
assert(lowerConstructor->getNumArgOperands()<=1 && "Native constructors with multiple args are not supported");
Instruction* callPred = NULL;
if (lowerConstructor->getNumArgOperands()==1 && Instruction::classof(lowerConstructor->getArgOperand(0)))
{
//Switch the argument to the one in the new func
lowerConstructor->setArgOperand(0, valueMap[lowerConstructor->getArgOperand(0)]);
callPred = cast<Instruction>(lowerConstructor->getArgOperand(0));
}
else
callPred = &newFunc->getEntryBlock().front();
//Add add it
lowerConstructor->insertAfter(callPred);
//Override the returs values
for(unsigned i=0;i<returns.size();i++)
{
Instruction* newInst = ReturnInst::Create(M.getContext(),lowerConstructor);
newInst->insertBefore(returns[i]);
returns[i]->removeFromParent();
}
//Recursively move all the users of the lower constructor after the call itself
Instruction* insertPoint = lowerConstructor->getNextNode();
SmallVector<Value*, 4> usersQueue(lowerConstructor->getNumUses());
unsigned int i;
Value::use_iterator it;
for(i=usersQueue.size()-1,it=lowerConstructor->use_begin();it!=lowerConstructor->use_end();++it,i--)
usersQueue[i]=it->getUser();
SmallSet<Instruction*, 4> movedInstructions;
while(!usersQueue.empty())
{
Instruction* cur=dyn_cast<Instruction>(usersQueue.pop_back_val());
if(!cur)
continue;
if(movedInstructions.count(cur))
continue;
movedInstructions.insert(cur);
cur->moveBefore(insertPoint);
//Add users of this instrucution as well
usersQueue.resize(usersQueue.size()+cur->getNumUses());
for(i=usersQueue.size()-1,it=cur->use_begin();it!=cur->use_end();++it,i--)
usersQueue[i]=it->getUser();
}
cast<Instruction>(valueMap[oldLowerConstructor])->eraseFromParent();
}
/// Replaces the given call site (Call or Invoke) with a gc.statepoint
/// intrinsic with an empty deoptimization arguments list. This does
/// NOT do explicit relocation for GC support.
static Value *ReplaceWithStatepoint(const CallSite &CS, /* to replace */
Pass *P) {
assert(CS.getInstruction()->getParent()->getParent()->getParent() &&
"must be set");
// TODO: technically, a pass is not allowed to get functions from within a
// function pass since it might trigger a new function addition. Refactor
// this logic out to the initialization of the pass. Doesn't appear to
// matter in practice.
// Then go ahead and use the builder do actually do the inserts. We insert
// immediately before the previous instruction under the assumption that all
// arguments will be available here. We can't insert afterwards since we may
// be replacing a terminator.
IRBuilder<> Builder(CS.getInstruction());
// Note: The gc args are not filled in at this time, that's handled by
// RewriteStatepointsForGC (which is currently under review).
// Create the statepoint given all the arguments
Instruction *Token = nullptr;
AttributeSet OriginalAttrs;
if (CS.isCall()) {
CallInst *ToReplace = cast<CallInst>(CS.getInstruction());
CallInst *Call = Builder.CreateGCStatepointCall(
CS.getCalledValue(), makeArrayRef(CS.arg_begin(), CS.arg_end()), None,
None, "safepoint_token");
Call->setTailCall(ToReplace->isTailCall());
Call->setCallingConv(ToReplace->getCallingConv());
// Before we have to worry about GC semantics, all attributes are legal
// TODO: handle param attributes
OriginalAttrs = ToReplace->getAttributes();
// In case if we can handle this set of attributes - set up function
// attributes directly on statepoint and return attributes later for
// gc_result intrinsic.
Call->setAttributes(OriginalAttrs.getFnAttributes());
Token = Call;
// Put the following gc_result and gc_relocate calls immediately after the
// the old call (which we're about to delete).
assert(ToReplace->getNextNode() && "not a terminator, must have next");
Builder.SetInsertPoint(ToReplace->getNextNode());
Builder.SetCurrentDebugLocation(ToReplace->getNextNode()->getDebugLoc());
} else if (CS.isInvoke()) {
InvokeInst *ToReplace = cast<InvokeInst>(CS.getInstruction());
// Insert the new invoke into the old block. We'll remove the old one in a
// moment at which point this will become the new terminator for the
// original block.
Builder.SetInsertPoint(ToReplace->getParent());
InvokeInst *Invoke = Builder.CreateGCStatepointInvoke(
CS.getCalledValue(), ToReplace->getNormalDest(),
ToReplace->getUnwindDest(), makeArrayRef(CS.arg_begin(), CS.arg_end()),
Builder.getInt32(0), None, "safepoint_token");
// Currently we will fail on parameter attributes and on certain
// function attributes.
OriginalAttrs = ToReplace->getAttributes();
// In case if we can handle this set of attributes - set up function
// attributes directly on statepoint and return attributes later for
// gc_result intrinsic.
Invoke->setAttributes(OriginalAttrs.getFnAttributes());
Token = Invoke;
// We'll insert the gc.result into the normal block
BasicBlock *NormalDest = normalizeBBForInvokeSafepoint(
ToReplace->getNormalDest(), Invoke->getParent());
Builder.SetInsertPoint(NormalDest->getFirstInsertionPt());
} else {
llvm_unreachable("unexpect type of CallSite");
}
assert(Token);
// Handle the return value of the original call - update all uses to use a
// gc_result hanging off the statepoint node we just inserted
// Only add the gc_result iff there is actually a used result
if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) {
std::string TakenName =
CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : "";
CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), TakenName);
GCResult->setAttributes(OriginalAttrs.getRetAttributes());
return GCResult;
} else {
// No return value for the call.
return nullptr;
}
}