void BlockState::mergePredecessorAvailSetAndValue(RLEContext &Ctx) {
// Clear the state if the basic block has no predecessor.
if (BB->getPreds().begin() == BB->getPreds().end()) {
ForwardSetIn.reset();
ForwardValIn.clear();
return;
}
auto Iter = BB->pred_begin();
ForwardSetIn = Ctx.getBlockState(*Iter).ForwardSetOut;
ForwardValIn = Ctx.getBlockState(*Iter).ForwardValOut;
Iter = std::next(Iter);
for (auto EndIter = BB->pred_end(); Iter != EndIter; ++Iter) {
BlockState &OtherState = Ctx.getBlockState(*Iter);
ForwardSetIn &= OtherState.ForwardSetOut;
// Merge in the predecessor state.
for (unsigned i = 0; i < LocationNum; ++i) {
if (OtherState.ForwardSetOut[i]) {
// There are multiple values from multiple predecessors, set this as
// a covering value. We do not need to track the value itself, as we
// can always go to the predecessors BlockState to find it.
ForwardValIn[i] = Ctx.getValueBit(LSValue(true));
continue;
}
// If this location does have an available value, then clear it.
stopTrackingValue(ForwardValIn, i);
stopTrackingLocation(ForwardSetIn, i);
}
}
}
void BlockState::mergePredecessorsAvailSetMax(RLEContext &Ctx) {
if (BB->pred_empty()) {
ForwardSetMax.reset();
return;
}
auto Iter = BB->pred_begin();
ForwardSetMax = Ctx.getBlockState(*Iter).ForwardSetMax;
Iter = std::next(Iter);
for (auto EndIter = BB->pred_end(); Iter != EndIter; ++Iter) {
ForwardSetMax &= Ctx.getBlockState(*Iter).ForwardSetMax;
}
}
void BlockState::mergePredecessorAvailSet(RLEContext &Ctx) {
// Clear the state if the basic block has no predecessor.
if (BB->getPreds().begin() == BB->getPreds().end()) {
ForwardSetIn.reset();
return;
}
auto Iter = BB->pred_begin();
ForwardSetIn = Ctx.getBlockState(*Iter).ForwardSetOut;
Iter = std::next(Iter);
for (auto EndIter = BB->pred_end(); Iter != EndIter; ++Iter) {
ForwardSetIn &= Ctx.getBlockState(*Iter).ForwardSetOut;
}
}
void BBState::mergePredecessorStates(RLEContext &Ctx) {
// Clear the state if the basic block has no predecessor.
if (BB->getPreds().begin() == BB->getPreds().end()) {
clearMemLocations();
return;
}
// We initialize the state with the first predecessor's state and merge
// in states of other predecessors.
bool HasAtLeastOnePred = false;
// For each predecessor of BB...
for (auto Pred : BB->getPreds()) {
BBState &Other = Ctx.getBBLocState(Pred);
// If we have not had at least one predecessor, initialize BBState
// with the state of the initial predecessor.
// If BB is also a predecessor of itself, we should not initialize.
if (!HasAtLeastOnePred) {
ForwardSetIn = Other.ForwardSetOut;
ForwardValIn = Other.ForwardValOut;
} else {
mergePredecessorState(Other);
}
HasAtLeastOnePred = true;
}
#ifndef NDEBUG
for (auto &X : ForwardValIn) {
(void)X;
assert(X.second.isValid() && "Invalid load store value");
}
#endif
}
void BBState::processUnknownWriteInst(RLEContext &Ctx, SILInstruction *I) {
auto *AA = Ctx.getAA();
for (unsigned i = 0; i < ForwardSetIn.size(); ++i) {
if (!isTrackingMemLocation(i))
continue;
// Invalidate any location this instruction may write to.
//
// TODO: checking may alias with Base is overly conservative,
// we should check may alias with base plus projection path.
MemLocation &R = Ctx.getMemLocation(i);
if (!AA->mayWriteToMemory(I, R.getBase()))
continue;
// MayAlias.
stopTrackingMemLocation(i);
}
}
void BlockState::mergePredecessorState(RLEContext &Ctx, BlockState &OtherState,
RLEKind Kind) {
// Are we computing the available set ?
if (isComputeAvailSet(Kind)) {
ForwardSetIn &= OtherState.ForwardSetOut;
return;
}
// Are we computing the available value ?
if (isComputeAvailValue(Kind) || isPerformRLE(Kind)) {
// Merge in the predecessor state.
llvm::SmallVector<unsigned, 8> LocDeleteList;
for (unsigned i = 0; i < ForwardSetIn.size(); ++i) {
if (OtherState.ForwardSetOut[i]) {
// There are multiple values from multiple predecessors, set this as
// a covering value. We do not need to track the value itself, as we
// can always go to the predecessors BlockState to find it.
ForwardValIn[i] = Ctx.getLSValueBit(LSValue(true));
continue;
}
// If this location does have an available value, then clear it.
stopTrackingLSLocation(i);
}
}
}
void BlockState::mergePredecessorStates(RLEContext &Ctx, RLEKind Kind) {
// Clear the state if the basic block has no predecessor.
if (BB->getPreds().begin() == BB->getPreds().end()) {
clearLSLocations();
return;
}
// We initialize the state with the first predecessor's state and merge
// in states of other predecessors.
bool HasAtLeastOnePred = false;
// For each predecessor of BB...
for (auto Pred : BB->getPreds()) {
BlockState &Other = Ctx.getBlockState(Pred);
// If we have not had at least one predecessor, initialize BlockState
// with the state of the initial predecessor.
// If BB is also a predecessor of itself, we should not initialize.
if (!HasAtLeastOnePred) {
if (isComputeAvailSet(Kind)) {
ForwardSetIn = Other.ForwardSetOut;
}
if (isComputeAvailValue(Kind) || isPerformRLE(Kind)) {
ForwardSetIn = Other.ForwardSetOut;
ForwardValIn = Other.ForwardValOut;
}
} else {
mergePredecessorState(Ctx, Other, Kind);
}
HasAtLeastOnePred = true;
}
}
SILValue BBState::computeForwardingValues(RLEContext &Ctx, MemLocation &L,
SILInstruction *InsertPt,
bool UseForwardValOut) {
SILBasicBlock *ParentBB = InsertPt->getParent();
bool IsTerminator = (InsertPt == ParentBB->getTerminator());
// We do not have a SILValue for the current MemLocation, try to construct
// one.
//
// Collect the locations and their corresponding values into a map.
// First, collect current available locations and their corresponding values
// into a map.
MemLocationValueMap Values;
if (!Ctx.gatherValues(ParentBB, L, Values, UseForwardValOut))
return SILValue();
// If the InsertPt is the terminator instruction of the basic block, we
// *refresh* it as terminator instruction could be deleted as a result
// of adding new edge values to the terminator instruction.
if (IsTerminator)
InsertPt = ParentBB->getTerminator();
// Second, reduce the available values into a single SILValue we can use to
// forward.
SILValue TheForwardingValue;
TheForwardingValue = MemLocation::reduceWithValues(L, &ParentBB->getModule(),
Values, InsertPt);
/// Return the forwarding value.
return TheForwardingValue;
}
void BlockState::updateForwardSetForWrite(RLEContext &Ctx, unsigned bit) {
// This is a store, invalidate any location that this location may alias, as
// their values can no longer be forwarded.
LSLocation &R = Ctx.getLSLocation(bit);
for (unsigned i = 0; i < ForwardSetIn.size(); ++i) {
if (!isTrackingLSLocation(i))
continue;
LSLocation &L = Ctx.getLSLocation(i);
if (!L.isMayAliasLSLocation(R, Ctx.getAA()))
continue;
// MayAlias, invalidate the LSLocation.
stopTrackingLSLocation(i);
}
// Start tracking this LSLocation.
startTrackingLSLocation(bit);
}
void BlockState::processUnknownWriteInstForGenKillSet(RLEContext &Ctx,
SILInstruction *I) {
auto *AA = Ctx.getAA();
for (unsigned i = 0; i < LocationNum; ++i) {
if (!isTrackingLocation(ForwardSetMax, i))
continue;
// Invalidate any location this instruction may write to.
//
// TODO: checking may alias with Base is overly conservative,
// we should check may alias with base plus projection path.
LSLocation &R = Ctx.getLocation(i);
if (!AA->mayWriteToMemory(I, R.getBase()))
continue;
// MayAlias.
stopTrackingLocation(BBGenSet, i);
startTrackingLocation(BBKillSet, i);
}
}
void BBState::updateForwardSetForWrite(RLEContext &Ctx, unsigned bit,
LoadStoreValue Val) {
// This is a store. Invalidate any Memlocation that this location may
// alias, as their value can no longer be forwarded.
MemLocation &R = Ctx.getMemLocation(bit);
for (unsigned i = 0; i < ForwardSetIn.size(); ++i) {
if (!isTrackingMemLocation(i))
continue;
MemLocation &L = Ctx.getMemLocation(i);
if (!L.isMayAliasMemLocation(R, Ctx.getAA()))
continue;
// MayAlias, invaliate the MemLocation.
stopTrackingMemLocation(i);
}
// Start tracking this MemLocation.
startTrackingMemLocation(bit, Val);
}
void BlockState::processWrite(RLEContext &Ctx, SILInstruction *I, SILValue Mem,
SILValue Val, RLEKind Kind) {
// Initialize the LSLocation.
LSLocation L(Mem);
// If we cant figure out the Base or Projection Path for the write,
// process it as an unknown memory instruction.
if (!L.isValid()) {
// we can ignore unknown store instructions if we are computing the
// AvailSetMax.
if (!isComputeAvailSetMax(Kind)) {
processUnknownWriteInst(Ctx, I, Kind);
}
return;
}
// Expand the given location and val into individual fields and process
// them as separate writes.
LSLocationList Locs;
LSLocation::expand(L, &I->getModule(), Locs, Ctx.getTE());
if (isComputeAvailSetMax(Kind)) {
for (unsigned i = 0; i < Locs.size(); ++i) {
updateMaxAvailForwardSetForWrite(Ctx, Ctx.getLocationBit(Locs[i]));
}
return;
}
// Are we computing the genset and killset ?
if (isComputeAvailGenKillSet(Kind)) {
for (unsigned i = 0; i < Locs.size(); ++i) {
updateGenKillSetForWrite(Ctx, Ctx.getLocationBit(Locs[i]));
}
return;
}
// Are we computing available set ?
if (isComputeAvailSet(Kind)) {
for (unsigned i = 0; i < Locs.size(); ++i) {
updateForwardSetForWrite(Ctx, Ctx.getLocationBit(Locs[i]));
}
return;
}
// Are we computing available value or performing RLE?
if (isComputeAvailValue(Kind) || isPerformingRLE(Kind)) {
LSValueList Vals;
LSValue::expand(Val, &I->getModule(), Vals, Ctx.getTE());
for (unsigned i = 0; i < Locs.size(); ++i) {
updateForwardSetAndValForWrite(Ctx, Ctx.getLocationBit(Locs[i]),
Ctx.getValueBit(Vals[i]));
}
return;
}
llvm_unreachable("Unknown RLE compute kind");
}
bool BlockState::setupRLE(RLEContext &Ctx, SILInstruction *I, SILValue Mem) {
// Try to construct a SILValue for the current LSLocation.
//
// Collect the locations and their corresponding values into a map.
LSLocation L(Mem);
LSLocationValueMap Values;
// Use the ForwardValIn as we are currently processing the basic block.
if (!Ctx.gatherLocationValues(I->getParent(), L, Values, getForwardValIn()))
return false;
// Reduce the available values into a single SILValue we can use to forward.
SILModule *Mod = &I->getModule();
SILValue TheForwardingValue;
TheForwardingValue = LSValue::reduce(L, Mod, Values, I, Ctx.getTE());
if (!TheForwardingValue)
return false;
// Now we have the forwarding value, record it for forwarding!.
//
// NOTE: we do not perform the RLE right here because doing so could introduce
// new LSLocations.
//
// e.g.
// %0 = load %x
// %1 = load %x
// %2 = extract_struct %1, #a
// %3 = load %2
//
// If we perform the RLE and replace %1 with %0, we end up having a memory
// location we do not have before, i.e. Base == %0, and Path == #a.
//
// We may be able to add the LSLocation to the vault, but it gets
// complicated very quickly, e.g. we need to resize the bit vectors size,
// etc.
//
// However, since we already know the instruction to replace and the value to
// replace it with, we can record it for now and forwarded it after all the
// forwardable values are recorded in the function.
//
RedundantLoads[I] = TheForwardingValue;
return true;
}
void BlockState::updateForwardSetAndValForWrite(RLEContext &Ctx, unsigned lbit,
unsigned vbit) {
// This is a store, invalidate any location that this location may alias, as
// their values can no longer be forwarded.
LSLocation &R = Ctx.getLocation(lbit);
for (unsigned i = 0; i < LocationNum; ++i) {
if (!isTrackingLocation(ForwardSetIn, i))
continue;
LSLocation &L = Ctx.getLocation(i);
if (!L.isMayAliasLSLocation(R, Ctx.getAA()))
continue;
// MayAlias, invalidate the location and value.
stopTrackingValue(ForwardValIn, i);
stopTrackingLocation(ForwardSetIn, i);
}
// Start tracking this location and value.
startTrackingLocation(ForwardSetIn, lbit);
startTrackingValue(ForwardValIn, lbit, vbit);
}
void BBState::processRead(RLEContext &Ctx, SILInstruction *I, SILValue Mem,
SILValue Val, bool PF) {
// Initialize the MemLocation.
MemLocation L(Mem);
// If we cant figure out the Base or Projection Path for the read, simply
// ignore it for now.
if (!L.isValid())
return;
// Expand the given MemLocation and Val into individual fields and process
// them as separate reads.
MemLocationList Locs;
LoadStoreValueList Vals;
MemLocation::expandWithValues(L, Val, &I->getModule(), Locs, Vals);
bool CanForward = true;
for (auto &X : Locs) {
CanForward &= isTrackingMemLocation(Ctx.getMemLocationBit(X));
}
// We do not have every location available, track the MemLocations and
// their values from this instruction, and return.
if (!CanForward) {
for (unsigned i = 0; i < Locs.size(); ++i) {
updateForwardSetForRead(Ctx, Ctx.getMemLocationBit(Locs[i]), Vals[i]);
}
return;
}
// At this point, we have all the MemLocations and their values available.
//
// If we are not doing forwarding just yet, simply return.
if (!PF)
return;
// Lastly, forward value to the load.
setupRLE(Ctx, I, Mem);
}
SILValue BlockState::reduceValuesAtEndOfBlock(RLEContext &Ctx, LSLocation &L) {
// First, collect current available locations and their corresponding values
// into a map.
LSLocationValueMap Values;
LSLocationList Locs;
LSLocation::expand(L, &BB->getModule(), Locs, Ctx.getTE());
// Find the values that this basic block defines and the locations which
// we do not have a concrete value in the current basic block.
ValueTableMap &OTM = getForwardValOut();
for (unsigned i = 0; i < Locs.size(); ++i) {
Values[Locs[i]] = Ctx.getLSValue(OTM[Ctx.getLSLocationBit(Locs[i])]);
}
// Second, reduce the available values into a single SILValue we can use to
// forward.
SILValue TheForwardingValue;
TheForwardingValue = LSValue::reduce(L, &BB->getModule(), Values,
BB->getTerminator(), Ctx.getTE());
/// Return the forwarding value.
return TheForwardingValue;
}
BlockState::ValueState BlockState::getValueStateAtEndOfBlock(RLEContext &Ctx,
LSLocation &L) {
LSLocationList Locs;
LSLocation::expand(L, &BB->getModule(), Locs, Ctx.getTE());
// Find number of covering value and concrete values for the locations
// expanded from the given location.
unsigned CSCount = 0, CTCount = 0;
ValueTableMap &OTM = getForwardValOut();
for (auto &X : Locs) {
LSValue &V = Ctx.getLSValue(OTM[Ctx.getLSLocationBit(X)]);
if (V.isCoveringValue()) {
++CSCount;
continue;
}
++CTCount;
}
if (CSCount == Locs.size())
return ValueState::CoverValues;
if (CTCount == Locs.size())
return ValueState::ConcreteValues;
return ValueState::CoverAndConcreteValues;
}
void BlockState::processRead(RLEContext &Ctx, SILInstruction *I, SILValue Mem,
SILValue Val, RLEKind Kind) {
// Initialize the LSLocation.
LSLocation L(Mem);
// If we cant figure out the Base or Projection Path for the read, simply
// ignore it for now.
if (!L.isValid())
return;
// Expand the given LSLocation and Val into individual fields and process
// them as separate reads.
LSLocationList Locs;
LSLocation::expand(L, &I->getModule(), Locs, Ctx.getTE());
// Are we computing available set ?.
if (isComputeAvailSet(Kind)) {
for (auto &X : Locs) {
if (isTrackingLSLocation(Ctx.getLSLocationBit(X)))
continue;
updateForwardSetForRead(Ctx, Ctx.getLSLocationBit(X));
}
return;
}
// Are we computing available values ?.
bool CanForward = true;
if (isComputeAvailValue(Kind) || isPerformRLE(Kind)) {
LSValueList Vals;
LSValue::expand(Val, &I->getModule(), Vals, Ctx.getTE());
for (unsigned i = 0; i < Locs.size(); ++i) {
if (isTrackingLSLocation(Ctx.getLSLocationBit(Locs[i])))
continue;
updateForwardValForRead(Ctx, Ctx.getLSLocationBit(Locs[i]),
Ctx.getLSValueBit(Vals[i]));
CanForward = false;
}
}
// Simply return if we are not performing RLE or we do not have all the
// values available to perform RLE.
if (!isPerformRLE(Kind) || !CanForward)
return;
// Lastly, forward value to the load.
setupRLE(Ctx, I, Mem);
}
void BBState::processWrite(RLEContext &Ctx, SILInstruction *I, SILValue Mem,
SILValue Val) {
// Initialize the MemLocation.
MemLocation L(Mem);
// If we cant figure out the Base or Projection Path for the write,
// process it as an unknown memory instruction.
if (!L.isValid()) {
processUnknownWriteInst(Ctx, I);
return;
}
// Expand the given MemLocation and Val into individual fields and process
// them as separate writes.
MemLocationList Locs;
LoadStoreValueList Vals;
MemLocation::expandWithValues(L, Val, &I->getModule(), Locs, Vals);
for (unsigned i = 0; i < Locs.size(); ++i) {
updateForwardSetForWrite(Ctx, Ctx.getMemLocationBit(Locs[i]), Vals[i]);
}
}