void DebugReporter::generateDebugReport(
AstTranslationUnit& translationUnit, std::string id, std::string title) {
std::stringstream datalogSpec;
translationUnit.getProgram()->print(datalogSpec);
DebugReportSection datalogSection = getCodeSection(id + "-dl", "Datalog", datalogSpec.str());
std::stringstream precGraphDot;
translationUnit.getAnalysis<PrecedenceGraph>()->outputPrecedenceGraph(precGraphDot);
DebugReportSection precedenceGraphSection =
getDotGraphSection(id + "-prec-graph", "Precedence Graph", precGraphDot.str());
std::stringstream sccGraphDot;
translationUnit.getAnalysis<SCCGraph>()->outputSCCGraph(sccGraphDot);
DebugReportSection sccGraphSection =
getDotGraphSection(id + "-scc-graph", "SCC Graph", sccGraphDot.str());
std::stringstream topsortSCCGraph;
translationUnit.getAnalysis<TopologicallySortedSCCGraph>()->outputTopologicallySortedSCCGraph(
topsortSCCGraph);
DebugReportSection topsortSCCGraphSection =
getCodeSection(id + "-topsort-scc-graph", "SCC Topological Sort Order", topsortSCCGraph.str());
std::stringstream adornment;
translationUnit.getAnalysis<Adornment>()->outputAdornment(adornment);
DebugReportSection adornmentSection =
getCodeSection(id + "-adornment-display", "Adornment", adornment.str());
translationUnit.getDebugReport().addSection(
DebugReportSection(id, title, {datalogSection, precedenceGraphSection, sccGraphSection,
topsortSCCGraphSection, adornmentSection},
""));
}
void PrecedenceGraph::run(const AstTranslationUnit& translationUnit) {
/* Get relations */
std::vector<AstRelation *> relations = translationUnit.getProgram()->getRelations();
for (AstRelation* r : relations) {
precedenceGraph.addNode(r);
for (size_t i = 0; i < r->clauseSize(); i++) {
AstClause *c = r->getClause(i);
const std::set<const AstRelation *> &dependencies = getBodyRelations(c, translationUnit.getProgram());
for(std::set<const AstRelation *>::const_iterator irs = dependencies.begin(); irs != dependencies.end(); ++irs) {
const AstRelation *source = (*irs);
precedenceGraph.addEdge(r, source);
}
}
}
}
void RecursiveClauses::run(const AstTranslationUnit& translationUnit) {
visitDepthFirst(*translationUnit.getProgram(), [&](const AstClause& clause) {
if (computeIsRecursive(clause, translationUnit)) {
recursiveClauses.insert(&clause);
}
});
}
bool AutoScheduleTransformer::transform(AstTranslationUnit& translationUnit) {
bool changed = false;
if (!Global::config().get("debug-report").empty()) {
std::stringstream report;
changed = autotune(translationUnit, &report);
translationUnit.getDebugReport().addSection(
DebugReporter::getCodeSection("auto-schedule", "Auto Schedule Report", report.str()));
} else {
changed = autotune(translationUnit, nullptr);
}
return changed;
}
bool DebugReporter::transform(AstTranslationUnit& translationUnit) {
auto start = std::chrono::high_resolution_clock::now();
bool changed = applySubtransformer(translationUnit, wrappedTransformer.get());
auto end = std::chrono::high_resolution_clock::now();
std::string runtimeStr = "(" + std::to_string(std::chrono::duration<double>(end - start).count()) + "s)";
if (changed) {
generateDebugReport(translationUnit, wrappedTransformer->getName(),
"After " + wrappedTransformer->getName() + " " + runtimeStr);
} else {
translationUnit.getDebugReport().addSection(DebugReportSection(wrappedTransformer->getName(),
"After " + wrappedTransformer->getName() + " " + runtimeStr + " (unchanged)", {}, ""));
}
return changed;
}
void ComponentLookup::run(const AstTranslationUnit& translationUnit) {
const AstProgram *program = translationUnit.getProgram();
for (AstComponent *component : program->getComponents()) {
globalScopeComponents.insert(component);
enclosingComponent[component] = nullptr;
}
visitDepthFirst(*program, [&](const AstComponent& cur) {
nestedComponents[&cur];
for (AstComponent *nestedComponent : cur.getComponents()) {
nestedComponents[&cur].insert(nestedComponent);
enclosingComponent[nestedComponent] = &cur;
}
});
}
void DebugReporter::generateDebugReport(
AstTranslationUnit& translationUnit, const std::string& id, std::string title) {
std::stringstream datalogSpec;
translationUnit.getProgram()->print(datalogSpec);
DebugReportSection datalogSection = getCodeSection(id + "-dl", "Datalog", datalogSpec.str());
std::stringstream typeAnalysis;
translationUnit.getAnalysis<TypeAnalysis>()->print(typeAnalysis);
DebugReportSection typeAnalysisSection = getCodeSection(id + "-ta", "Type Analysis", typeAnalysis.str());
std::stringstream typeEnvironmentAnalysis;
translationUnit.getAnalysis<TypeEnvironmentAnalysis>()->print(typeEnvironmentAnalysis);
DebugReportSection typeEnvironmentAnalysisSection =
getCodeSection(id + "-tea", "Type Environment Analysis", typeEnvironmentAnalysis.str());
std::stringstream precGraphDot;
translationUnit.getAnalysis<PrecedenceGraph>()->print(precGraphDot);
DebugReportSection precedenceGraphSection =
getDotGraphSection(id + "-prec-graph", "Precedence Graph", precGraphDot.str());
std::stringstream sccGraphDot;
translationUnit.getAnalysis<SCCGraph>()->print(sccGraphDot);
DebugReportSection sccGraphSection =
getDotGraphSection(id + "-scc-graph", "SCC Graph", sccGraphDot.str());
std::stringstream topsortSCCGraph;
translationUnit.getAnalysis<TopologicallySortedSCCGraph>()->print(topsortSCCGraph);
DebugReportSection topsortSCCGraphSection =
getCodeSection(id + "-topsort-scc-graph", "SCC Topological Sort Order", topsortSCCGraph.str());
translationUnit.getDebugReport().addSection(DebugReportSection(id, std::move(title),
{datalogSection, typeAnalysisSection, typeEnvironmentAnalysisSection, precedenceGraphSection,
sccGraphSection, topsortSCCGraphSection},
""));
}
void SCCGraph::run(const AstTranslationUnit& translationUnit) {
precedenceGraph = translationUnit.getAnalysis<PrecedenceGraph>();
SCC.clear();
nodeToSCC.clear();
predSCC.clear();
succSCC.clear();
/* Compute SCC */
std::vector<AstRelation *> relations = translationUnit.getProgram()->getRelations();
unsigned int counter = 0;
int numSCCs = 0;
std::stack<const AstRelation *> S, P;
std::map<const AstRelation *, int> preOrder; // Pre-order number of a node (for Gabow's Algo)
for (const AstRelation *relation : relations) {
nodeToSCC[relation] = preOrder[relation] = -1;
}
for (const AstRelation *relation : relations) {
if (preOrder[relation] == -1) {
scR(relation, preOrder, counter, S, P, numSCCs);
}
}
/* Build SCC graph */
succSCC.resize(numSCCs);
predSCC.resize(numSCCs);
for (const AstRelation *u : relations) {
for (const AstRelation *v : precedenceGraph->getPredecessors(u)) {
int scc_u = nodeToSCC[u];
int scc_v = nodeToSCC[v];
ASSERT(scc_u >= 0 && scc_u < numSCCs && "Wrong range");
ASSERT(scc_v >= 0 && scc_v < numSCCs && "Wrong range");
if (scc_u != scc_v) {
predSCC[scc_u].insert(scc_v);
succSCC[scc_v].insert(scc_u);
}
}
}
/* Store the relations for each SCC */
SCC.resize(numSCCs);
for (const AstRelation *relation : relations) {
SCC[nodeToSCC[relation]].insert(relation);
}
}
bool ComponentInstantiationTransformer::transform(AstTranslationUnit& translationUnit) {
// TODO: Do this without being a friend class of AstProgram
// unbound clauses with no relation defined
std::vector<std::unique_ptr<AstClause>> unbound;
AstProgram &program = *translationUnit.getProgram();
ComponentLookup *componentLookup = translationUnit.getAnalysis<ComponentLookup>();
for(const auto& cur : program.instantiations) {
std::vector<std::unique_ptr<AstClause>> orphans;
ComponentContent content = getInstantiatedContent(*cur, nullptr, *componentLookup, orphans, translationUnit.getErrorReport());
for(auto& type : content.types) {
program.types.insert(std::make_pair(type->getName(), std::move(type)));
}
for(auto& rel : content.relations) {
program.relations.insert(std::make_pair(rel->getName(), std::move(rel)));
}
for(auto& cur : orphans) {
auto pos = program.relations.find(cur->getHead()->getName());
if (pos != program.relations.end()) {
pos->second->addClause(std::move(cur));
} else {
unbound.push_back(std::move(cur));
}
}
}
// add clauses
for(auto& cur : program.clauses) {
auto pos = program.relations.find(cur->getHead()->getName());
if (pos != program.relations.end()) {
pos->second->addClause(std::move(cur));
} else {
unbound.push_back(std::move(cur));
}
}
// remember the remaining orphan clauses
program.clauses.clear();
program.clauses.swap(unbound);
return true;
}
void RedundantRelations::run(const AstTranslationUnit& translationUnit) {
precedenceGraph = translationUnit.getAnalysis<PrecedenceGraph>();
std::set<const AstRelation *> work;
std::set<const AstRelation *> notRedundant;
const std::vector<AstRelation *> &relations = translationUnit.getProgram()->getRelations();
/* Add all output relations to the work set */
for (const AstRelation* r : relations) {
if(r->isComputed()) {
work.insert(r);
}
}
/* Find all relations which are not redundant for the computations of the
output relations. */
while(!work.empty()){
/* Chose one element in the work set and add it to notRedundant */
const AstRelation *u = *(work.begin());
work.erase(work.begin());
notRedundant.insert(u);
/* Find all predecessors of u and add them to the worklist
if they are not in the set notRedundant */
for (const AstRelation *predecessor : precedenceGraph->getPredecessors(u)) {
if (!notRedundant.count(predecessor)) {
work.insert(predecessor);
}
}
}
/* All remaining relations are redundant. */
redundantRelations.clear();
for (const AstRelation *r : relations) {
if (!notRedundant.count(r)) {
redundantRelations.insert(r);
}
}
}
bool RecursiveClauses::computeIsRecursive(const AstClause& clause, const AstTranslationUnit& translationUnit) const {
const AstProgram &program = *translationUnit.getProgram();
// we want to reach the atom of the head through the body
const AstRelation* trg = getHeadRelation(&clause, &program);
std::set<const AstRelation*> reached;
std::vector<const AstRelation*> worklist;
// set up start list
for(const AstAtom* cur : clause.getAtoms()) {
auto rel = program.getRelation(cur->getName());
if (rel == trg) return true;
worklist.push_back(rel);
}
// process remaining elements
while(!worklist.empty()) {
// get next to process
const AstRelation* cur = worklist.back();
worklist.pop_back();
// skip null pointers (errors in the input code)
if (!cur) continue;
// check whether this one has been checked before
if (!reached.insert(cur).second) continue;
// check all atoms in the relations
for(const AstClause* cl : cur->getClauses()) {
for(const AstAtom* at : cl->getAtoms()) {
auto rel = program.getRelation(at->getName());
if (rel == trg) return true;
worklist.push_back(rel);
}
}
}
// no cycles found
return false;
}
std::vector<std::set<const AstRelation *>> RelationSchedule::computeRelationExpirySchedule(const AstTranslationUnit &translationUnit) {
std::vector<std::set<const AstRelation *>> relationExpirySchedule;
/* Compute for each step in the reverse topological order
of evaluating the SCC the set of alive relations. */
int numSCCs = topsortSCCGraph->getSCCOrder().size();
/* Alive set for each step */
std::vector< std::set<const AstRelation *> > alive(numSCCs);
/* Resize expired relations sets */
relationExpirySchedule.resize(numSCCs);
/* Mark the output relations as alive in the first step */
for (const AstRelation *relation : translationUnit.getProgram()->getRelations()) {
if (relation->isComputed()) {
alive[0].insert(relation);
}
}
/* Compute all alive relations by iterating over all steps in reverse order
determine the dependencies */
for(int orderedSCC = 1; orderedSCC < numSCCs; orderedSCC++) {
/* Add alive set of previous step */
alive[orderedSCC].insert(alive[orderedSCC-1].begin(), alive[orderedSCC-1].end());
/* Add predecessors of relations computed in this step */
for (const AstRelation *r : topsortSCCGraph->getSCCGraph()->getRelationsForSCC(numSCCs - orderedSCC)) {
for (const AstRelation *predecessor : precedenceGraph->getPredecessors(r)) {
alive[orderedSCC].insert(predecessor);
}
}
/* Compute expired relations in reverse topological order using the set difference of the alive sets
between steps. */
std::set_difference(alive[orderedSCC].begin(), alive[orderedSCC].end(),
alive[orderedSCC-1].begin(), alive[orderedSCC-1].end(),
std::inserter(relationExpirySchedule[numSCCs-orderedSCC], relationExpirySchedule[numSCCs-orderedSCC].end()));
}
return relationExpirySchedule;
}
bool AutoScheduleTransformer::autotune(AstTranslationUnit& translationUnit, std::ostream* report) {
const QueryExecutionStrategy& strategy = ScheduledExecution;
bool verbose = Global::config().has("verbose");
// start with status message
if (verbose) {
std::cout << "\n";
}
if (verbose) {
std::cout << "----------------- Auto-Scheduling Started -----------------\n";
}
// step 1 - translate to RAM program
if (verbose) {
std::cout << "[ Converting to RAM Program ... ]\n";
}
std::unique_ptr<RamStatement> stmt = RamTranslator().translateProgram(translationUnit);
// check whether there is something to tune
if (!stmt) {
if (verbose) {
std::cout << "[ No Rules in Program ]\n";
std::cout << "---------------- Auto-Scheduling Completed ----------------\n";
}
return false;
}
if (verbose) {
std::cout << "[ Done ]\n";
}
// step 2 - run in interpreted mode, collect decisions
if (verbose) {
std::cout << "[ Profiling RAM Program ... ]\n";
}
Profiler::Data data;
Profiler profiler(strategy, data);
// create a copy of the symbol table
souffle::SymbolTable table = translationUnit.getSymbolTable();
// create interpreter instance
RamGuidedInterpreter interpreter(profiler);
if (report && verbose) {
SplitStream splitStream(report, &std::cout);
interpreter.setReportTarget(splitStream);
} else if (report) {
interpreter.setReportTarget(*report);
} else if (verbose) {
interpreter.setReportTarget(std::cout);
}
// run interpreter
interpreter.execute(table, *stmt);
if (verbose) {
std::cout << "[ Done ]\n";
}
if (verbose) {
std::cout << "Data:\n";
for (const auto& cur : data) {
std::cout << "Clause @ " << cur.first << "\n";
for (const ExecutionSummary& instance : cur.second) {
std::cout << "\t" << instance.order << " in " << instance.time << "ms\n";
}
}
}
// step 3 - process collected data ..
if (verbose) {
std::cout << "[ Selecting most significant schedules ... ]\n";
}
std::map<AstSrcLocation, const AstClause*> clauses;
visitDepthFirst(*translationUnit.getProgram(),
[&](const AstClause& clause) { clauses[clause.getSrcLoc()] = &clause; });
std::map<const AstClause*, long> longestTime;
std::map<const AstClause*, Order> bestOrders;
// extract best order for each clause
for (const auto& cur : data) {
const AstClause* clause = clauses[cur.first];
assert(clause && "Unknown clause discovered!");
for (const ExecutionSummary& instance : cur.second) {
if (longestTime[clause] < instance.time) {
longestTime[clause] = instance.time;
bestOrders[clause] = instance.order;
}
}
}
if (verbose) {
for (const auto& cur : bestOrders) {
std::cout << *cur.first << "\n Best Order: " << cur.second
<< "\n Time: " << longestTime[cur.first] << "\n\n";
}
}
if (verbose) {
std::cout << "[ Done ]\n";
}
// step 4 - apply transformations
if (verbose) {
std::cout << "[ Re-scheduling rules ... ]\n";
}
bool changed = false;
for (const auto& cur : bestOrders) {
AstClause* clause = const_cast<AstClause*>(cur.first);
bool orderChanged = false;
const std::vector<unsigned>& newOrder = cur.second.getOrder();
// Check whether best order is different to the original order
for (unsigned int i = 0; i < clause->getAtoms().size(); i++) {
if (newOrder[i] != i) {
orderChanged = true;
break;
}
}
if (orderChanged) {
clause->reorderAtoms(newOrder);
changed = true;
}
}
if (verbose) {
std::cout << "[ Done ]\n";
}
// end with status message
if (verbose) {
std::cout << "---------------- Auto-Scheduling Completed -----------------\n";
}
return changed;
}
