// MS: will possibly be replaced with an implementation from the BOOST graph library
LabelSet Flow::reachableNodesButNotBeyondTargetNode(Label start, Label target) {
LabelSet reachableNodes;
LabelSet toVisitSet=succ(start);
size_t oldSize=0;
size_t newSize=0;
do {
LabelSet newToVisitSet;
for(LabelSet::iterator i=toVisitSet.begin();i!=toVisitSet.end();++i) {
LabelSet succSet=succ(*i);
for(LabelSet::iterator j=succSet.begin();j!=succSet.end();++j) {
if(reachableNodes.find(*j)==reachableNodes.end())
newToVisitSet.insert(*j);
}
}
toVisitSet=newToVisitSet;
oldSize=reachableNodes.size();
reachableNodes+=toVisitSet;
newSize=reachableNodes.size();
} while(oldSize!=newSize);
return reachableNodes;
}
static void get_labelset(BindingsPtr bindings, NodePtr root, LabelSet& labels) {
assert(root->get_op() == Op::cfg_edge_condition);
if (root->size() == 2) {
get_labelset(bindings, root->get_operand(0), labels);
root = root->get_operand(1);
}
std::string label = root->get_operand(0)->get_token().get_text();
std::size_t index = label_by_name(bindings, label);
if (index >= labels.size()) {
labels.resize(index + 1);
}
labels.set(index);
}
void Train(const std::string& dataset, const std::string& model) {
NGramMaker ngram;
LabelSet ls;
Document doc = Parse(dataset, ngram, ls);
SequenceTaggerParams tagger(ngram.dict().size(), ls.size());
std::cout << tagger.Train(doc) << "% accuracy FINAL" << std::endl;
std::ofstream out(model);
out << ngram.Serialize();
out << ls.Serialize();
out << tagger.Serialize();
InteractiveShell(tagger, ngram, ls);
}
//Generates SSA form numbers for the variables contained in the CFG node labeled *label and attaches them as AstValueAttributes to the related SgNodes
//Generates phi statements for the node if it is an if node; Collects those phi statements in a phi attribute and attaches it to the related SgNode
//Continues the traversal of the CFG; The CFG nodes are traversed in the topological order that treats if nodes as follows:
//if node -> true branch -> false branch -> (phi statements for the if node are now finished) -> if node's associated continue node and its successors
//Assumption: The node is located in in the inTrueBranch branch of the if node labeled *condLabel (These two arguments are required to generate the SSA form numbers)
void SSAGenerator::processNode(Label* label, Label* condLabel, bool inTrueBranch)
{
SgNode* node = labeler->getNode(*label);
LabelSet successors = flow->succ(*label);
assert(successors.size() <= 2);
//Skip the current node if it is just a return node for a called function
if(labeler->isFunctionCallReturnLabel(*label))
{
logger[Sawyer::Message::DEBUG] << "- - - - - - - - - - - - - - - - - - - - - - - - -" << endl;
logger[Sawyer::Message::DEBUG] << "Ignoring function call return node " << *label << endl;
assert(successors.size() == 1);
Label nextLabel = *successors.begin();
//If the next node is not the continue node to any of the enclosing condition nodes and not the exit node: Process it
if(!isExitOrContOfPred(&nextLabel, label)) processNode(&nextLabel, condLabel, inTrueBranch);
return;
}
logger[Sawyer::Message::DEBUG] << "- - - - - - - - - - - - - - - - - - - - - - - - -" << endl;
logger[Sawyer::Message::DEBUG] << "processNode() called for label " << *label << endl;
SgVariableDeclaration* varDec = dynamic_cast<SgVariableDeclaration*>(node);
SgExprStatement* exprStmt = dynamic_cast<SgExprStatement*>(node);
if(varDec) //Variable declaration
{
SgInitializedNamePtrList varNames = varDec->get_variables();
SgInitializedNamePtrList::iterator i = varNames.begin();
while(i != varNames.end())
{
string name = (*i)->get_qualified_name();
//Update the varsDeclaredInTrueBranch/varsDeclaredInFalseBranch attribute in the PhiAttribute of the condition node
if(condLabel != NULL)
{
SgNode* condNode = labeler->getNode(*condLabel);
AstAttribute* condAtt = condNode->getAttribute("PHI");
assert(condAtt != NULL);
PhiAttribute* condPhiAtt = dynamic_cast<PhiAttribute*>(condAtt);
assert(condPhiAtt != NULL);
if(inTrueBranch) condPhiAtt->varsDeclaredInTrueBranch.insert(name);
else condPhiAtt->varsDeclaredInFalseBranch.insert(name);
}
SgAssignInitializer* assignInit = dynamic_cast<SgAssignInitializer*>((*i)->get_initializer());
if(assignInit) //Declaration with initialization
{
SgExpression* ex = assignInit->get_operand();
processSgExpression(ex, condLabel, inTrueBranch);
}
else //Declaration without initialization
{
assert((*i)->get_initializer() == NULL);
}
//Assign number to declared variable
int number = nextNumber(name, condLabel, inTrueBranch);
logger[Sawyer::Message::DEBUG] << "Next number for variable " << name << ": " << number << endl;
AstValueAttribute<int>* numberAtt = new AstValueAttribute<int>(number);
(*i)->setAttribute("SSA_NUMBER", numberAtt);
i++;
}
}
else if(exprStmt) //Assignment to variable or if statement or function call or ...
{
SgExpression* ex = exprStmt->get_expression();
processSgExpression(ex, condLabel, inTrueBranch);
}
else //CFG node that is not a variable declaration and not an expression statement; Should only be the case for the first node (Entry), the last node (Exit) and return nodes
{
logger[Sawyer::Message::DEBUG] << "Node is not a variable declaration and not an expression statement" << endl;
SgReturnStmt* retStmt = dynamic_cast<SgReturnStmt*>(node);
assert(labeler->isFunctionEntryLabel(*label) || labeler->isFunctionExitLabel(*label) || retStmt != NULL);
}
//Continue traversal of CFG
if(successors.size() == 1) //Current node is a regular node (not an if node)
{
Label nextLabel = *successors.begin();
//If the next node is not the continue node to any of the enclosing condition nodes and not the exit node: Process it
if(!isExitOrContOfPred(&nextLabel, label)) processNode(&nextLabel, condLabel, inTrueBranch);
}
else if(successors.size() == 2) //Current node is an if node
{
assert(exprStmt != NULL);
//Attach PhiAttribute to node that (for now) only includes its reaching variable numbers
map<string, int> reachingNumbers = currentNumberMap;
if(condLabel != NULL)
{
SgNode* condNode = labeler->getNode(*condLabel);
AstAttribute* condAtt = condNode->getAttribute("PHI");
assert(condAtt != NULL);
PhiAttribute* condPhiAtt = dynamic_cast<PhiAttribute*>(condAtt);
assert(condPhiAtt != NULL);
map<string, int>::iterator m = reachingNumbers.begin();
while(m != reachingNumbers.end())
{
//m->first is in scope at the current node only if it is in scope at the condition node or it is declared locally in the current node's branch of the condition node
bool inScope = (condPhiAtt->reachingNumbers.find(m->first) != condPhiAtt->reachingNumbers.end()) ||
(inTrueBranch && condPhiAtt->varsDeclaredInTrueBranch.find(m->first) != condPhiAtt->varsDeclaredInTrueBranch.end()) ||
(!inTrueBranch && condPhiAtt->varsDeclaredInFalseBranch.find(m->first) != condPhiAtt->varsDeclaredInFalseBranch.end());
if(!inScope)
{
m = reachingNumbers.erase(m);
continue;
}
//Reaching number for m->first has to be updated //TODO: Makes no sense to take reaching numbers from current numbers in the first place
m->second = currentNumber(m->first, condLabel, inTrueBranch);
m++;
}
}
CondAtomic* cond = new CondAtomic(*label);
PhiAttribute* phiAtt = new PhiAttribute(reachingNumbers, cond);
exprStmt->setAttribute("PHI", phiAtt);
//Identify true successor, false successor and continue node
Flow trueOutEdges = flow->outEdgesOfType(*label, EDGE_TRUE);
Flow falseOutEdges = flow->outEdgesOfType(*label, EDGE_FALSE);
assert( (trueOutEdges.size() == 1) && (falseOutEdges.size() == 1) );
Edge outTrue = *trueOutEdges.begin();
Edge outFalse = *falseOutEdges.begin();
Label nextLabelTrue = outTrue.target();
Label nextLabelFalse = outFalse.target();
Label* contLabel = getContinueLabel(*label);
//Process true and false branch
ContNodeAttribute* contAttr = getContinueNodeAttr(*label);
bool commitPhiStatements = true; //"Hack":
//If one or both of the two branches definitely return there will be phi statements created for the current node although the SSA form that is being created here does not require them in that case
//They are however required to find out current variable numbers in the branch/branches that definitely return
//Therefore in that case the phi statements will be created but not committed
if (contAttr == NULL) //Both branches definitely return
{
if(!isExitOrContOrContOfPred(&nextLabelTrue, label)) processNode(&nextLabelTrue, label, true); //"Hack"
if(!isExitOrContOrContOfPred(&nextLabelFalse, label)) processNode(&nextLabelFalse, label, false); //"Hack"
commitPhiStatements = false;
}
else if (contAttr->trueBranchReturns == YES && contAttr->falseBranchReturns != YES) //Only the true branch definitely returns
{
if(!isExitOrContOrContOfPred(&nextLabelTrue, label)) processNode(&nextLabelTrue, label, true); //"Hack"
if(condLabel == NULL) //No enclosing condition node exists
{
//"Hack"-phi-statement needs to be used to determine current variable numbers because processing the true branch modified currentNumberMap
if(!isExitOrContOrContOfPred(&nextLabelFalse, label)) processNode(&nextLabelFalse, label, inTrueBranch);
}
else if(!isExitOrContOrContOfPred(&nextLabelFalse, label)) processNode(&nextLabelFalse, condLabel, inTrueBranch);
commitPhiStatements = false;
}
else if (contAttr->trueBranchReturns != YES && contAttr->falseBranchReturns == YES) //Only the false branch definitely returns
{
if(!isExitOrContOrContOfPred(&nextLabelTrue, label)) processNode(&nextLabelTrue, condLabel, inTrueBranch);
if(!isExitOrContOrContOfPred(&nextLabelFalse, label)) processNode(&nextLabelFalse, label, false); //"Hack"
commitPhiStatements = false;
}
else //Neither of the two branches definitely returns
{
assert(!(contAttr->trueBranchReturns == YES && contAttr->falseBranchReturns == YES));
if(!isExitOrContOrContOfPred(&nextLabelTrue, label)) processNode(&nextLabelTrue, label, true);
if(!isExitOrContOrContOfPred(&nextLabelFalse, label)) processNode(&nextLabelFalse, label, false);
commitPhiStatements = true;
}
if(commitPhiStatements) //Commit phi statements: Generate a new number for the variable of each phi statement and save that number in its respective newNumber attribute
{
vector<PhiStatement*>::iterator i = phiAtt->phiStatements.begin();
logger[Sawyer::Message::DEBUG] << "- - - - - - - - - - - - - - - - - - - - - - - - -" << endl;
logger[Sawyer::Message::DEBUG] << "Phi statements created for node with label " << *label << ":" << endl;
while (i != phiAtt->phiStatements.end())
{
if((*i)->trueNumber != (*i)->falseNumber)
{
//Generate new number for phi statement's variable
int newNumber = nextNumber((*i)->varName, condLabel, inTrueBranch);
(*i)->newNumber = newNumber;
logger[Sawyer::Message::DEBUG] << (*i)->toString() << endl;
}
i++;
}
logger[Sawyer::Message::DEBUG] << "- - - - - - - - - - - - - - - - - - - - - - - - -" << endl;
logger[Sawyer::Message::DEBUG] << "COMPLETE PHI ATTRIBUTE:" << endl << phiAtt->toString() << endl;
}
else //Delete phi statements ("Hack")
{
phiAtt->phiStatements.clear();
}
//If the continue node exists and is not the continue node to any of the enclosing condition nodes and not the exit node: Process it
if(contLabel != NULL && !isExitOrContOfPred(contLabel, label))
{
processNode(contLabel, condLabel, inTrueBranch);
}
}
}
void SSAGenerator::findReachingConditions(Label* label, Label* condLabel, bool inTrueBranch, bool updateContNode)
{
logger[Sawyer::Message::DEBUG] << "- - - - - - - - - - - - - - - - - - - - - - - - -" << endl;
logger[Sawyer::Message::DEBUG] << "findReachingConditions() called for label " << *label << endl;
SgNode* node = labeler->getNode(*label);
LabelSet successors = flow->succ(*label);
assert(successors.size() <= 2);
if(successors.size() == 1) //Current node is a regular node (not an if node)
{
Label nextLabel = *successors.begin();
if(labeler->isFunctionExitLabel(nextLabel)) //Next node is exit node
{
return;
}
else if(isCont(&nextLabel, condLabel)) //Next node is the continue node of the condition node
{
if(updateContNode) updateReachCond(&nextLabel, getReachCond(label));
else return;
}
else //Next node is neither the continue node of the condition node nor the exit node
{
updateReachCond(&nextLabel, getReachCond(label));
findReachingConditions(&nextLabel, condLabel, inTrueBranch, updateContNode);
return;
}
}
else if(successors.size() == 2) //Current node is an if node
{
//Identify true successor, false successor, and continue node
Flow trueOutEdges = flow->outEdgesOfType(*label, EDGE_TRUE);
Flow falseOutEdges = flow->outEdgesOfType(*label, EDGE_FALSE);
assert( (trueOutEdges.size() == 1) && (falseOutEdges.size() == 1) );
Edge outTrue = *trueOutEdges.begin();
Edge outFalse = *falseOutEdges.begin();
Label nextLabelTrue = outTrue.target();
Label nextLabelFalse = outFalse.target();
Label* contLabel = NULL;
ContNodeAttribute* contAttr = getContinueNodeAttr(*label);
if(contAttr != NULL) contLabel = &(contAttr->contLabel);
//Identify condition
AstAttribute* condAtt = node->getAttribute("PHI");
assert(condAtt != NULL);
PhiAttribute* phiAtt = dynamic_cast<PhiAttribute*>(condAtt);
assert(phiAtt != NULL);
Condition* cond = phiAtt->condition;
//Determine whether the reaching condition of current node's continue node has to be updated by the recoursive calls
bool updateContNodeRek = true;
if (contAttr != NULL && contAttr->trueBranchReturns == NO && contAttr->falseBranchReturns == NO) //None of the two branches might return; Thus they both definitely end at the continue node
{
updateContNodeRek = false;
}
//Update both successors' reaching condition and process both branches
Condition* reachCond = getReachCond(label);
if(!labeler->isFunctionExitLabel(nextLabelTrue)) //True successor is not the exit node
{
if(isCont(&nextLabelTrue, condLabel)) //True successor is the condition node's continue node
{
if(updateContNode) updateReachCond(&nextLabelTrue, *reachCond && *cond);
}
else //True successor is actually in the condition node's inTrueBranch branch
{
if(!isCont(&nextLabelTrue, label)) //True successor is not the current node's continue node but actually in its true branch
{
updateReachCond(&nextLabelTrue, *reachCond && *cond);
findReachingConditions(&nextLabelTrue, label, true, updateContNodeRek);
}
}
}
if(!labeler->isFunctionExitLabel(nextLabelFalse)) //False successor is not the exit node
{
if(isCont(&nextLabelFalse, condLabel)) //False successor is the condition node's continue node
{
if(updateContNode) updateReachCond(&nextLabelFalse, (*reachCond && *!*cond));
}
else //False successor is actually in the condition node's false branch
{
if(!isCont(&nextLabelFalse, label)) //False successor is not the current node's continue node but actually in its false branch
{
updateReachCond(&nextLabelFalse, (*reachCond && *!*cond));
findReachingConditions(&nextLabelFalse, label, false, updateContNodeRek);
}
}
}
//Update the continue node's reaching condition if it has not been updated by the recoursive calls and process it
if(contLabel != NULL)
{
assert(!labeler->isFunctionExitLabel(*contLabel));
if(isCont(contLabel, condLabel)) //Continue node is also the condition node's continue node
{
if(updateContNode && !updateContNodeRek) updateReachCond(contLabel, reachCond);
}
else //Continue node is actually in the condition node's inTrueBranch branch
{
if(!updateContNodeRek) updateReachCond(contLabel, reachCond);
findReachingConditions(contLabel, condLabel, inTrueBranch, updateContNode);
}
}
}
}
//Calculates the continue node for each condition node among the CFG nodes that are reachable from the CFG node marked with *label (the "current node")
//Identifies branches that definitely or maybe end with a return statement
//Attaches continue attributes that contain this information to the related condition nodes; Exception: No such attribute is added to condition nodes for which both branches definitely return
//*callingCondLabel denotes the current node's recoursive predecessor (It is a condition node that contains the current node in one of its two branches)
//Definition continue node:
//The continue node cx for a condition node x is defined as the direct successor of x's true and false branch (the two branches' cont node) if that node is uniquely defined
//If there is not one unique direct successor of the two branches (possible due to return statements), there are two: An inner CFG node and the CFG's exit node
//In that case cx is defined as the inner CFG node
pair<Label, BranchReturns> SSAGenerator::findContinueNodes(Label* label, Label* callingCondLabel)
{
SgNode* node = labeler->getNode(*label);
//Identify the current node's actual condition node if it exists
Label* condLabel = NULL;
map<Label, Label>::iterator i = conditionMap.find(*label);
if(i != conditionMap.end())
{
condLabel = &(i->second);
}
//Do not process the current node and its successors in the CFG if *callingCondLabel does not denote the current node's actual condition node; Return the current node's label as continue node label; The current branch does not return
bool process = (condLabel == NULL && callingCondLabel == NULL) || (condLabel != NULL && callingCondLabel != NULL && *condLabel == *callingCondLabel);
if(!process)
{
return pair<Label, BranchReturns>(*label, NO);
}
//Do not process the current node and its successors in the CFG if *label denotes a return statement; Return the current node's label as continue node label; The current branch definitely returns
SgReturnStmt* retStmt = dynamic_cast<SgReturnStmt*>(node);
if(retStmt != NULL)
{
return pair<Label, BranchReturns>(*label, YES);
}
//Process the current node's successors if they exist
LabelSet successors = flow->succ(*label);
assert(successors.size() <= 2);
if(successors.size() == 0) //Current node is the exit node; Return its label; Current branch does not return
{
assert(labeler->isFunctionExitLabel(*label));
return pair<Label, BranchReturns>(*label, NO);
}
if(successors.size() == 1) //Current node is a regular node (not an if node or the exit node); Process its successor
{
Label succLabel = *(successors.begin());
return findContinueNodes(&succLabel, callingCondLabel);
}
if(successors.size() == 2) //Current node is an if node
{
//Identify true successor and false successor
Flow trueOutEdges = flow->outEdgesOfType(*label, EDGE_TRUE);
Flow falseOutEdges = flow->outEdgesOfType(*label, EDGE_FALSE);
assert( (trueOutEdges.size() == 1) && (falseOutEdges.size() == 1) );
Edge outTrue = *trueOutEdges.begin();
Edge outFalse = *falseOutEdges.begin();
Label nextLabelTrue = outTrue.target();
Label nextLabelFalse = outFalse.target();
//Process true successor and false successor
pair<Label, BranchReturns> trueRet = findContinueNodes(&nextLabelTrue, label);
pair<Label, BranchReturns> falseRet = findContinueNodes(&nextLabelFalse, label);
//If necessary: Attach a continue node attribute to the current node, determine its continue node, and process that continue node
ContNodeAttribute* contNodeAtt;
if(trueRet.second == YES && falseRet.second == YES) //Both branches definitely return
{
logger[Sawyer::Message::DEBUG] << "Continue node for node " << *label << ": None" << endl;
return pair<Label, BranchReturns>(*label, YES); //The current branch definitely returns
}
else if(trueRet.second == YES) //Only the true branch definitely returns
{
contNodeAtt = new ContNodeAttribute(falseRet.first, trueRet.second, falseRet.second);
node->setAttribute("CONTINUE_NODE", contNodeAtt);
logger[Sawyer::Message::DEBUG] << "Continue node for node " << *label << ": " << falseRet.first << endl;
logger[Sawyer::Message::DEBUG] << "trueBranchReturns: " << trueRet.second << "; falseBranchReturns: " << falseRet.second << endl;
pair<Label, BranchReturns> contRet = findContinueNodes(&falseRet.first, callingCondLabel); //Process the continue node and its successors
if(contRet.second == YES) return pair<Label, BranchReturns>(contRet.first, YES); //The current branch definitely returns
else return pair<Label, BranchReturns>(contRet.first, MAYBE); //The current branch may return
}
else if(falseRet.second == YES) //Only the false branch definitely returns
{
contNodeAtt = new ContNodeAttribute(trueRet.first, trueRet.second, falseRet.second);
node->setAttribute("CONTINUE_NODE", contNodeAtt);
logger[Sawyer::Message::DEBUG] << "Continue node for node " << *label << ": " << trueRet.first << endl;
logger[Sawyer::Message::DEBUG] << "trueBranchReturns: " << trueRet.second << "; falseBranchReturns: " << falseRet.second << endl;
pair<Label, BranchReturns> contRet = findContinueNodes(&trueRet.first, callingCondLabel);
if(contRet.second == YES) return pair<Label, BranchReturns>(contRet.first, YES); //The current branch definitely returns
else return pair<Label, BranchReturns>(contRet.first, MAYBE); //The current branch may return
}
else if(trueRet.second == NO && falseRet.second == NO) //Neither the true branch nor the false branch may return
{
assert(trueRet.first == falseRet.first); //The calls for both branches should yield the same continue node
contNodeAtt = new ContNodeAttribute(trueRet.first, trueRet.second, falseRet.second);
node->setAttribute("CONTINUE_NODE", contNodeAtt);
logger[Sawyer::Message::DEBUG] << "Continue node for node " << *label << ": " << trueRet.first << endl;
logger[Sawyer::Message::DEBUG] << "trueBranchReturns: " << trueRet.second << "; falseBranchReturns: " << falseRet.second << endl;
return findContinueNodes(&trueRet.first, callingCondLabel); //Process the continue node and its successors; Whether the current branch returns depends on the outcome
}
else //Neither the true branch nor the false branch definitely returns but at least one of them may return
{
assert(trueRet.first == falseRet.first); //The calls for both branches should yield the same continue node
contNodeAtt = new ContNodeAttribute(trueRet.first, trueRet.second, falseRet.second);
node->setAttribute("CONTINUE_NODE", contNodeAtt);
logger[Sawyer::Message::DEBUG] << "Continue node for node " << *label << ": " << trueRet.first << endl;
logger[Sawyer::Message::DEBUG] << "trueBranchReturns: " << trueRet.second << "; falseBranchReturns: " << falseRet.second << endl;
pair<Label, BranchReturns> contRet = findContinueNodes(&trueRet.first, callingCondLabel); //Process the continue node and its successors
if(contRet.second == YES) return pair<Label, BranchReturns>(contRet.first, YES); //The current branch definitely returns
return pair<Label, BranchReturns>(contRet.first, MAYBE); //The current branch may return
}
}
ROSE_ASSERT(false);
}