static uint32_t _AllocUnits(PPMdSubAllocatorVariantH *self,int index)
{
if(self->GlueCount==0)
{
self->GlueCount=255;
GlueFreeBlocks(self);
if(self->FreeList[index].next) return PointerToOffset(self,RemoveNode(self,index));
}
for(int i=index+1;i<N_INDEXES;i++)
{
if(self->FreeList[i].next)
{
void *units=RemoveNode(self,i);
SplitBlock(self,units,i,index);
return PointerToOffset(self,units);
}
}
self->GlueCount--;
int i=I2B(self,index);
if(self->UnitsStart-self->pText>i)
{
self->UnitsStart-=i;
return PointerToOffset(self,self->UnitsStart);
}
return 0;
}
static uint32_t _AllocUnits(PPMdSubAllocatorVariantI *self,int index)
{
if(self->GlueCount==0)
{
GlueFreeBlocks(self);
if(AreBlocksAvailable(&self->BList[index])) return PointerToOffset(self,RemoveBlockAfter(&self->BList[index],self));
}
for(int i=index+1;i<N_INDEXES;i++)
{
if(AreBlocksAvailable(&self->BList[i]))
{
void *units=RemoveBlockAfter(&self->BList[i],self);
SplitBlock(self,units,i,index);
return PointerToOffset(self,units);
}
}
self->GlueCount--;
int i=I2B(self,index);
if(self->UnitsStart-self->pText>i)
{
self->UnitsStart-=i;
return PointerToOffset(self,self->UnitsStart);
}
return 0;
}
bool SplitBBAtCondBr::runOnFunction(Function &F) {
bool Changed = false;
std::vector<BasicBlock*> BBInvolved;
for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI)
{
BasicBlock *BB = &*BI;
BBInvolved.push_back(BB);
}
for (std::vector<BasicBlock*>::iterator BI = BBInvolved.begin(), BE = BBInvolved.end(); BI != BE; ++BI) {
BasicBlock *BB = *BI;
TerminatorInst *TI = BB->getTerminator();
if (BranchInst *Br = dyn_cast<BranchInst>(TI)) {
if (Br->isConditional() && !BarrierInst::endsWithBarrier(BB)) {
BasicBlock *NewBB = SplitBlock(BB, Br, this);
NewBB->setName(BB->getName() + ".splitcondbr");
Changed = true;
}
}
}
return Changed;
}
void* SubAllocator::AllocUnitsRare(int indx)
{
if ( !GlueCount )
{
GlueCount = 255;
GlueFreeBlocks();
if ( FreeList[indx].next )
return RemoveNode(indx);
}
int i=indx;
do
{
if (++i == N_INDEXES)
{
GlueCount--;
i=U2B(Indx2Units[indx]);
int j=12*Indx2Units[indx];
if (FakeUnitsStart-pText > j)
{
FakeUnitsStart-=j;
UnitsStart -= i;
return(UnitsStart);
}
return(NULL);
}
} while ( !FreeList[i].next );
void* RetVal=RemoveNode(i);
SplitBlock(RetVal,i,indx);
return RetVal;
}
CallSite GNUstep::IMPCacher::SplitSend(CallSite msgSend)
{
BasicBlock *lookupBB = msgSend->getParent();
Function *F = lookupBB->getParent();
Module *M = F->getParent();
Function *send = M->getFunction("objc_msgSend");
Function *send_stret = M->getFunction("objc_msgSend_stret");
Function *send_fpret = M->getFunction("objc_msgSend_fpret");
Value *self;
Value *cmd;
int selfIndex = 0;
if ((msgSend.getCalledFunction() == send) ||
(msgSend.getCalledFunction() == send_fpret)) {
self = msgSend.getArgument(0);
cmd = msgSend.getArgument(1);
} else if (msgSend.getCalledFunction() == send_stret) {
selfIndex = 1;
self = msgSend.getArgument(1);
cmd = msgSend.getArgument(2);
} else {
abort();
return CallSite();
}
CGBuilder B(&F->getEntryBlock(), F->getEntryBlock().begin());
Value *selfPtr = B.CreateAlloca(self->getType());
B.SetInsertPoint(msgSend.getInstruction());
B.CreateStore(self, selfPtr, true);
LLVMType *impTy = msgSend.getCalledValue()->getType();
LLVMType *slotTy = PointerType::getUnqual(StructType::get(PtrTy, PtrTy, PtrTy,
IntTy, impTy, PtrTy, NULL));
Value *slot;
Constant *lookupFn = M->getOrInsertFunction("objc_msg_lookup_sender",
slotTy, selfPtr->getType(), cmd->getType(), PtrTy, NULL);
if (msgSend.isCall()) {
slot = B.CreateCall3(lookupFn, selfPtr, cmd, Constant::getNullValue(PtrTy));
} else {
InvokeInst *inv = cast<InvokeInst>(msgSend.getInstruction());
BasicBlock *callBB = SplitBlock(lookupBB, msgSend.getInstruction(), Owner);
removeTerminator(lookupBB);
B.SetInsertPoint(lookupBB);
slot = B.CreateInvoke3(lookupFn, callBB, inv->getUnwindDest(), selfPtr, cmd,
Constant::getNullValue(PtrTy));
addPredecssor(inv->getUnwindDest(), msgSend->getParent(), lookupBB);
B.SetInsertPoint(msgSend.getInstruction());
}
Value *imp = B.CreateLoad(B.CreateStructGEP(slot, 4));
msgSend.setArgument(selfIndex, B.CreateLoad(selfPtr, true));
msgSend.setCalledFunction(imp);
return CallSite(slot);
}
void* SubAllocator::ShrinkUnits(void* OldPtr,int OldNU,int NewNU)
{
int i0=Units2Indx[OldNU-1], i1=Units2Indx[NewNU-1];
if (i0 == i1)
return OldPtr;
if ( FreeList[i1].next )
{
void* ptr=RemoveNode(i1);
memcpy(ptr,OldPtr,U2B(NewNU));
InsertNode(OldPtr,i0);
return ptr;
}
else
{
SplitBlock(OldPtr,i0,i1);
return OldPtr;
}
}
static uint32_t ShrinkUnitsVariantI(PPMdSubAllocatorVariantI *self,uint32_t oldoffs,int oldnum,int newnum)
{
void *oldptr=OffsetToPointer(self,oldoffs);
int oldindex=self->Units2Index[oldnum-1];
int newindex=self->Units2Index[newnum-1];
if(oldindex==newindex) return oldoffs;
if(AreBlocksAvailable(&self->BList[newindex]))
{
void *ptr=RemoveBlockAfter(&self->BList[newindex],self);
memcpy(ptr,oldptr,newnum*UNIT_SIZE);
InsertBlockAfter(&self->BList[oldindex],oldptr,self->Index2Units[oldindex],self);
return PointerToOffset(self,ptr);
}
else
{
SplitBlock(self,oldptr,oldindex,newindex);
return oldoffs;
}
}
static uint32_t ShrinkUnitsBrimstone(PPMdSubAllocatorBrimstone *self,uint32_t oldoffs,int oldnum,int newnum)
{
void *oldptr=OffsetToPointer(self,oldoffs);
int oldindex=self->Units2Index[oldnum-1];
int newindex=self->Units2Index[newnum-1];
if(oldindex==newindex) return oldoffs;
if(self->FreeList[newindex].next)
{
void *ptr=RemoveNode(self,newindex);
memcpy(ptr,oldptr,I2B(self,newindex));
InsertNode(self,oldptr,oldindex);
return PointerToOffset(self,ptr);
}
else
{
SplitBlock(self,oldptr,oldindex,newindex);
return oldoffs;
}
}
static uint32_t AllocUnitsBrimstone(PPMdSubAllocatorBrimstone *self,int num)
{
int index=self->Units2Index[num-1];
if(self->FreeList[index].next) return PointerToOffset(self,RemoveNode(self,index));
void *units=self->LowUnit;
self->LowUnit+=I2B(self,index);
if(self->LowUnit<=self->HighUnit) return PointerToOffset(self,units);
self->LowUnit-=I2B(self,index);
for(int i=index+1;i<N_INDEXES;i++)
{
if(self->FreeList[i].next)
{
void *units=RemoveNode(self,i);
SplitBlock(self,units,i,index);
return PointerToOffset(self,units);
}
}
return 0;
}
bool
LoopBarriers::ProcessLoop(Loop *L, LPPassManager &LPM)
{
bool isBLoop = false;
bool changed = false;
for (Loop::block_iterator i = L->block_begin(), e = L->block_end();
i != e && !isBLoop; ++i) {
for (BasicBlock::iterator j = (*i)->begin(), e = (*i)->end();
j != e; ++j) {
if (isa<BarrierInst>(j)) {
isBLoop = true;
break;
}
}
}
LLVMContext &LC = getGlobalContext();
IntegerType * IntTy = IntegerType::get(LC, 32);
Value *Args = ConstantInt::get(IntTy, 0);
for (Loop::block_iterator i = L->block_begin(), e = L->block_end();
i != e && isBLoop; ++i) {
for (BasicBlock::iterator j = (*i)->begin(), e = (*i)->end();
j != e; ++j) {
if (isa<BarrierInst>(j)) {
BasicBlock *preheader = L->getLoopPreheader();
assert((preheader != NULL) && "Non-canonicalized loop found!\n");
Instruction *PhdrBarrierInst =
BarrierInst::createBarrier(Args, preheader->getTerminator());
MDNode* PhdrAuxBarrierInfo =
MDNode::get(LC, MDString::get(LC, "auxiliary phdr barrier"));
PhdrBarrierInst->setMetadata("aux.phdr.barrier", PhdrAuxBarrierInfo);
preheader->setName(preheader->getName() + ".loopbarrier");
BasicBlock *header = L->getHeader();
if (header->getFirstNonPHI() != &header->front()) {
Instruction *HdrBarrierInst =
BarrierInst::createBarrier(Args, header->getFirstNonPHI());
MDNode* HdrAuxBarrierInfo =
MDNode::get(LC, MDString::get(LC, "auxiliary phihdr barrier"));
HdrBarrierInst->setMetadata("aux.phihdr.barrier", HdrAuxBarrierInfo);
header->setName(header->getName() + ".phibarrier");
}
/*
SmallVector<BasicBlock*, 8> ExitingBlocks;
L->getExitingBlocks(ExitingBlocks);
*/
BasicBlock *brexit = L->getExitingBlock();
if (brexit != NULL) {
Instruction *ExitingBarrierInst =
BarrierInst::createBarrier(Args, brexit->getTerminator());
MDNode* ExitingAuxBarrierInfo =
MDNode::get(LC, MDString::get(LC, "auxiliary exiting barrier"));
ExitingBarrierInst->setMetadata("aux.exiting.barrier",
ExitingAuxBarrierInfo);
brexit->setName(brexit->getName() + ".brexitbarrier");
}
BasicBlock *latch = L->getLoopLatch();
if (latch != NULL && brexit != latch) {
Instruction *LatchBarrierInst =
BarrierInst::createBarrier(Args, latch->getTerminator());
MDNode* LatchAuxBarrierInfo =
MDNode::get(LC, MDString::get(LC, "auxiliary latch barrier"));
LatchBarrierInst->setMetadata("aux.latch.barrier",
LatchAuxBarrierInfo);
latch->setName(latch->getName() + ".latchbarrier");
return changed;
}
BasicBlock *Header = L->getHeader();
typedef GraphTraits<Inverse<BasicBlock *> > InvBlockTraits;
InvBlockTraits::ChildIteratorType PI = InvBlockTraits::child_begin(Header);
InvBlockTraits::ChildIteratorType PE = InvBlockTraits::child_end(Header);
BasicBlock *Latch = NULL;
for (; PI != PE; ++PI) {
InvBlockTraits::NodeType *N = *PI;
if (L->contains(N)) {
Latch = N;
if (DT->dominates(j->getParent(), Latch)) {
BarrierInst::createBarrier(Args, Latch->getTerminator());
Latch->setName(Latch->getName() + ".latchbarrier");
}
}
}
return true;
}
}
}
BasicBlock *preheader = L->getLoopPreheader();
assert((preheader != NULL) && "Non-canonicalized loop found!\n");
TerminatorInst *t = preheader->getTerminator();
Instruction *prev = NULL;
if (&preheader->front() != t) {
// If t is not the first/only instruction in the block, get the previous
// instruction.
prev = t->getPrevNode();
}
if (prev && isa<BarrierInst>(prev)) {
BasicBlock *new_b = SplitBlock(preheader, t, this);
new_b->setName(preheader->getName() + ".postbarrier_dummy");
return true;
}
return changed;
}
void GNUstep::IMPCacher::CacheLookup(Instruction *lookup, Value *slot, Value
*version, bool isSuperMessage) {
// If this IMP is already cached, don't cache it again.
if (lookup->getMetadata(IMPCacheFlagKind)) {
return;
}
lookup->setMetadata(IMPCacheFlagKind, AlreadyCachedFlag);
bool isInvoke = false;
BasicBlock *beforeLookupBB = lookup->getParent();
BasicBlock *lookupBB = SplitBlock(beforeLookupBB, lookup, Owner);
BasicBlock *lookupFinishedBB = lookupBB;
BasicBlock *afterLookupBB;
if (InvokeInst *inv = dyn_cast<InvokeInst>(lookup)) {
afterLookupBB = inv->getNormalDest();
lookupFinishedBB =
BasicBlock::Create(Context, "done_lookup", lookupBB->getParent());
CGBuilder B(lookupFinishedBB);
B.CreateBr(afterLookupBB);
inv->setNormalDest(lookupFinishedBB);
isInvoke = true;
} else {
BasicBlock::iterator iter = lookup;
iter++;
afterLookupBB = SplitBlock(iter->getParent(), iter, Owner);
}
removeTerminator(beforeLookupBB);
CGBuilder B = CGBuilder(beforeLookupBB);
// Load the slot and check that neither it nor the version is 0.
Value *versionValue = B.CreateLoad(version);
Value *receiverPtr = lookup->getOperand(0);
Value *receiver = receiverPtr;
if (!isSuperMessage) {
receiver = B.CreateLoad(receiverPtr);
}
// For small objects, we skip the cache entirely.
// FIXME: Class messages are never to small objects...
bool is64Bit = llvm::Module::Pointer64 ==
B.GetInsertBlock()->getParent()->getParent()->getPointerSize();
LLVMType *intPtrTy = is64Bit ? Type::getInt64Ty(Context) :
Type::getInt32Ty(Context);
// Receiver as an integer
Value *receiverSmallObject = B.CreatePtrToInt(receiver, intPtrTy);
// Receiver is a small object...
receiverSmallObject =
B.CreateAnd(receiverSmallObject, is64Bit ? 7 : 1);
// Receiver is not a small object.
receiverSmallObject =
B.CreateICmpNE(receiverSmallObject, Constant::getNullValue(intPtrTy));
// Ideally, we'd call objc_msgSend() here, but for now just skip the cache
// lookup
Value *isCacheEmpty =
B.CreateICmpEQ(versionValue, Constant::getNullValue(IntTy));
Value *receiverNil =
B.CreateICmpEQ(receiver, Constant::getNullValue(receiver->getType()));
isCacheEmpty = B.CreateOr(isCacheEmpty, receiverNil);
isCacheEmpty = B.CreateOr(isCacheEmpty, receiverSmallObject);
BasicBlock *cacheLookupBB = BasicBlock::Create(Context, "cache_check",
lookupBB->getParent());
B.CreateCondBr(isCacheEmpty, lookupBB, cacheLookupBB);
// Check the cache node is current
B.SetInsertPoint(cacheLookupBB);
Value *slotValue = B.CreateLoad(slot, "slot_value");
Value *slotVersion = B.CreateStructGEP(slotValue, 3);
// Note: Volatile load because the slot version might have changed in
// another thread.
slotVersion = B.CreateLoad(slotVersion, true, "slot_version");
Value *slotCachedFor = B.CreateStructGEP(slotValue, 1);
slotCachedFor = B.CreateLoad(slotCachedFor, true, "slot_owner");
Value *cls = B.CreateLoad(B.CreateBitCast(receiver, IdTy));
Value *isVersionCorrect = B.CreateICmpEQ(slotVersion, versionValue);
Value *isOwnerCorrect = B.CreateICmpEQ(slotCachedFor, cls);
Value *isSlotValid = B.CreateAnd(isVersionCorrect, isOwnerCorrect);
// If this slot is still valid, skip the lookup.
B.CreateCondBr(isSlotValid, afterLookupBB, lookupBB);
// Perform the real lookup and cache the result
removeTerminator(lookupFinishedBB);
// Replace the looked up slot with the loaded one
B.SetInsertPoint(afterLookupBB, afterLookupBB->begin());
PHINode *newLookup = IRBuilderCreatePHI(&B, lookup->getType(), 3, "new_lookup");
// Not volatile, so a redundant load elimination pass can do some phi
// magic with this later.
lookup->replaceAllUsesWith(newLookup);
B.SetInsertPoint(lookupFinishedBB);
Value * newReceiver = receiver;
if (!isSuperMessage) {
newReceiver = B.CreateLoad(receiverPtr);
}
BasicBlock *storeCacheBB = BasicBlock::Create(Context, "cache_store",
lookupBB->getParent());
// Don't store the cached lookup if we are doing forwarding tricks.
// Also skip caching small object messages for now
Value *skipCacheWrite =
B.CreateOr(B.CreateICmpNE(receiver, newReceiver), receiverSmallObject);
skipCacheWrite = B.CreateOr(skipCacheWrite, receiverNil);
B.CreateCondBr(skipCacheWrite, afterLookupBB, storeCacheBB);
B.SetInsertPoint(storeCacheBB);
// Store it even if the version is 0, because we always check that the
// version is not 0 at the start and an occasional redundant store is
// probably better than a branch every time.
B.CreateStore(lookup, slot);
B.CreateStore(B.CreateLoad(B.CreateStructGEP(lookup, 3)), version);
cls = B.CreateLoad(B.CreateBitCast(receiver, IdTy));
B.CreateStore(cls, B.CreateStructGEP(lookup, 1));
B.CreateBr(afterLookupBB);
newLookup->addIncoming(lookup, lookupFinishedBB);
newLookup->addIncoming(slotValue, cacheLookupBB);
newLookup->addIncoming(lookup, storeCacheBB);
}
void GNUstep::IMPCacher::SpeculativelyInline(Instruction *call, Function
*function) {
BasicBlock *beforeCallBB = call->getParent();
BasicBlock *callBB = SplitBlock(beforeCallBB, call, Owner);
BasicBlock *inlineBB = BasicBlock::Create(Context, "inline",
callBB->getParent());
BasicBlock::iterator iter = call;
iter++;
BasicBlock *afterCallBB = SplitBlock(iter->getParent(), iter, Owner);
removeTerminator(beforeCallBB);
// Put a branch before the call, testing whether the callee really is the
// function
IRBuilder<> B = IRBuilder<>(beforeCallBB);
Value *callee = isa<CallInst>(call) ? cast<CallInst>(call)->getCalledValue()
: cast<InvokeInst>(call)->getCalledValue();
const FunctionType *FTy = function->getFunctionType();
const FunctionType *calleeTy = cast<FunctionType>(
cast<PointerType>(callee->getType())->getElementType());
if (calleeTy != FTy) {
callee = B.CreateBitCast(callee, function->getType());
}
Value *isInlineValid = B.CreateICmpEQ(callee, function);
B.CreateCondBr(isInlineValid, inlineBB, callBB);
// In the inline BB, add a copy of the call, but this time calling the real
// version.
Instruction *inlineCall = call->clone();
Value *inlineResult= inlineCall;
inlineBB->getInstList().push_back(inlineCall);
B.SetInsertPoint(inlineBB);
if (calleeTy != FTy) {
for (unsigned i=0 ; i<FTy->getNumParams() ; i++) {
LLVMType *callType = calleeTy->getParamType(i);
LLVMType *argType = FTy->getParamType(i);
if (callType != argType) {
inlineCall->setOperand(i, new
BitCastInst(inlineCall->getOperand(i), argType, "", inlineCall));
}
}
if (FTy->getReturnType() != calleeTy->getReturnType()) {
if (FTy->getReturnType() == Type::getVoidTy(Context)) {
inlineResult = Constant::getNullValue(calleeTy->getReturnType());
} else {
inlineResult =
new BitCastInst(inlineCall, calleeTy->getReturnType(), "", inlineBB);
}
}
}
B.CreateBr(afterCallBB);
// Unify the return values
if (call->getType() != Type::getVoidTy(Context)) {
PHINode *phi = CreatePHI(call->getType(), 2, "", afterCallBB->begin());
call->replaceAllUsesWith(phi);
phi->addIncoming(call, callBB);
phi->addIncoming(inlineResult, inlineBB);
}
// Really do the real inlining
InlineFunctionInfo IFI(0, 0);
if (CallInst *c = dyn_cast<CallInst>(inlineCall)) {
c->setCalledFunction(function);
InlineFunction(c, IFI);
} else if (InvokeInst *c = dyn_cast<InvokeInst>(inlineCall)) {
c->setCalledFunction(function);
InlineFunction(c, IFI);
}
}
