bool PPCLoopDataPrefetch::runOnLoop(Loop *L) { bool MadeChange = false; // Only prefetch in the inner-most loop if (!L->empty()) return MadeChange; SmallPtrSet<const Value *, 32> EphValues; CodeMetrics::collectEphemeralValues(L, AC, EphValues); // Calculate the number of iterations ahead to prefetch CodeMetrics Metrics; for (Loop::block_iterator I = L->block_begin(), IE = L->block_end(); I != IE; ++I) { // If the loop already has prefetches, then assume that the user knows // what he or she is doing and don't add any more. for (BasicBlock::iterator J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) if (CallInst *CI = dyn_cast<CallInst>(J)) if (Function *F = CI->getCalledFunction()) if (F->getIntrinsicID() == Intrinsic::prefetch) return MadeChange; Metrics.analyzeBasicBlock(*I, *TTI, EphValues); } unsigned LoopSize = Metrics.NumInsts; if (!LoopSize) LoopSize = 1; unsigned ItersAhead = PrefDist/LoopSize; if (!ItersAhead) ItersAhead = 1; SmallVector<std::pair<Instruction *, const SCEVAddRecExpr *>, 16> PrefLoads; for (Loop::block_iterator I = L->block_begin(), IE = L->block_end(); I != IE; ++I) { for (BasicBlock::iterator J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) { Value *PtrValue; Instruction *MemI; if (LoadInst *LMemI = dyn_cast<LoadInst>(J)) { MemI = LMemI; PtrValue = LMemI->getPointerOperand(); } else if (StoreInst *SMemI = dyn_cast<StoreInst>(J)) { if (!PrefetchWrites) continue; MemI = SMemI; PtrValue = SMemI->getPointerOperand(); } else continue; unsigned PtrAddrSpace = PtrValue->getType()->getPointerAddressSpace(); if (PtrAddrSpace) continue; if (L->isLoopInvariant(PtrValue)) continue; const SCEV *LSCEV = SE->getSCEV(PtrValue); const SCEVAddRecExpr *LSCEVAddRec = dyn_cast<SCEVAddRecExpr>(LSCEV); if (!LSCEVAddRec) continue; // We don't want to double prefetch individual cache lines. If this load // is known to be within one cache line of some other load that has // already been prefetched, then don't prefetch this one as well. bool DupPref = false; for (SmallVector<std::pair<Instruction *, const SCEVAddRecExpr *>, 16>::iterator K = PrefLoads.begin(), KE = PrefLoads.end(); K != KE; ++K) { const SCEV *PtrDiff = SE->getMinusSCEV(LSCEVAddRec, K->second); if (const SCEVConstant *ConstPtrDiff = dyn_cast<SCEVConstant>(PtrDiff)) { int64_t PD = std::abs(ConstPtrDiff->getValue()->getSExtValue()); if (PD < (int64_t) CacheLineSize) { DupPref = true; break; } } } if (DupPref) continue; const SCEV *NextLSCEV = SE->getAddExpr(LSCEVAddRec, SE->getMulExpr( SE->getConstant(LSCEVAddRec->getType(), ItersAhead), LSCEVAddRec->getStepRecurrence(*SE))); if (!isSafeToExpand(NextLSCEV, *SE)) continue; PrefLoads.push_back(std::make_pair(MemI, LSCEVAddRec)); Type *I8Ptr = Type::getInt8PtrTy((*I)->getContext(), PtrAddrSpace); SCEVExpander SCEVE(*SE, J->getModule()->getDataLayout(), "prefaddr"); Value *PrefPtrValue = SCEVE.expandCodeFor(NextLSCEV, I8Ptr, MemI); IRBuilder<> Builder(MemI); Module *M = (*I)->getParent()->getParent(); Type *I32 = Type::getInt32Ty((*I)->getContext()); Value *PrefetchFunc = Intrinsic::getDeclaration(M, Intrinsic::prefetch); Builder.CreateCall4(PrefetchFunc, PrefPtrValue, ConstantInt::get(I32, MemI->mayReadFromMemory() ? 0 : 1), ConstantInt::get(I32, 3), ConstantInt::get(I32, 1)); MadeChange = true; } } return MadeChange; }