void dDeterministicFiniteAutonata::EmptyTransitionClosure (const dTree<dAutomataState*,dAutomataState*>& set, dTree<dAutomataState*,dAutomataState*>& closureStates) const
{
int stack = 0;
dAutomataState* stackPool[2048];
dTree<dAutomataState*,dAutomataState*>::Iterator iter (set);
for (iter.Begin(); iter; iter ++) {
dAutomataState* const state = iter.GetNode()->GetInfo();
stackPool[stack] = state;
stack ++;
dAssert (stack < sizeof (stackPool) / sizeof (stackPool[0]));
closureStates.Insert(state, state);
}
while(stack) {
stack --;
dAutomataState* const state = stackPool[stack];
for (dList<dAutomataState::dTransition>::dListNode* node = state->m_transtions.GetFirst(); node; node = node->GetNext()) {
dAutomataState::dTransition& transition = node->GetInfo();
dAutomataState::dCharacter ch (transition.GetCharater());
if (ch.m_symbol == 0) {
dAutomataState* const targetState = transition.GetState();
if(!closureStates.Find(targetState)) {
closureStates.Insert(targetState, targetState);
stackPool[stack] = targetState;
stack ++;
dAssert (stack < sizeof (stackPool) / sizeof (stackPool[0]));
}
}
}
}
}
void dParserCompiler::First (
dCRCTYPE symbol,
dTree<int, dCRCTYPE>& symbolListMark,
const dTree<dTokenInfo, dCRCTYPE>& symbolList,
const dTree<dList<void*>, dCRCTYPE>& ruleMap,
dTree<int, dCRCTYPE>& firstSetOut) const
{
if (symbolListMark.Find(symbol)) {
return;
}
symbolListMark.Insert(0, symbol);
dTree<dTokenInfo, dCRCTYPE>::dTreeNode* const node = symbolList.Find(symbol);
dAssert (node);
if (node->GetInfo().m_type == TERMINAL) {
firstSetOut.Insert(0, symbol);
} else if (DoesSymbolDeriveEmpty (symbol, ruleMap)) {
firstSetOut.Insert(0, 0);
} else {
dTree<dList<void*>, dCRCTYPE>::dTreeNode* const ruleNodes = ruleMap.Find(symbol);
if (ruleNodes) {
const dList<void*>& matchingRulesList = ruleNodes->GetInfo();
for (dList<void*>::dListNode* node = matchingRulesList.GetFirst(); node; node = node->GetNext()) {
dProductionRule::dListNode* const ruleInfoNode = (dProductionRule::dListNode*) node->GetInfo();
const dRuleInfo& info = ruleInfoNode->GetInfo();
bool allDeriveEmpty = true;
for (dRuleInfo::dListNode* sentenceSymbolNode = info.GetFirst(); sentenceSymbolNode; sentenceSymbolNode = sentenceSymbolNode->GetNext()) {
const dSymbol& sentenceSymnol = sentenceSymbolNode->GetInfo();
if (!DoesSymbolDeriveEmpty (sentenceSymnol.m_nameCRC, ruleMap)) {
allDeriveEmpty = false;
dTree<int, dCRCTYPE> newFirstSetOut;
First (sentenceSymnol.m_nameCRC, symbolListMark, symbolList, ruleMap, newFirstSetOut);
dTree<int, dCRCTYPE>::Iterator iter (newFirstSetOut);
for (iter.Begin(); iter; iter ++) {
dCRCTYPE symbol = iter.GetKey();
dAssert (symbol != 0);
dAssert (symbolList.Find(symbol)->GetInfo().m_type == TERMINAL);
firstSetOut.Insert(0, symbol);
}
break;
}
}
if (allDeriveEmpty) {
dTrace (("this could be a bug here, I am not sure if I should closure with the accepting state or not, need more debugging\n"))
// firstSetOut.Insert(0, 0);
}
}
}
}
}
void dParserCompiler::First (
const dList<dCRCTYPE>& symbolSet,
const dTree<dTokenInfo, dCRCTYPE>& symbolList,
const dTree<dList<void*>, dCRCTYPE>& ruleMap,
dTree<int, dCRCTYPE>& firstSetOut) const
{
if (symbolSet.GetCount() > 1) {
dList<dCRCTYPE>::dListNode* node = symbolSet.GetFirst();
bool deriveEmpty = true;
while ((deriveEmpty) && node) {
dCRCTYPE symbol = node->GetInfo();
node = node->GetNext();
dTree<int, dCRCTYPE> tmpFirst;
dTree<int, dCRCTYPE> symbolListMark;
First (symbol, symbolListMark, symbolList, ruleMap, tmpFirst);
dTree<int, dCRCTYPE>::Iterator iter (tmpFirst);
deriveEmpty = false;
for (iter.Begin(); iter; iter ++) {
dCRCTYPE symbol = iter.GetKey();
if (symbol == 0) {
deriveEmpty = true;
} else {
firstSetOut.Insert(0, symbol);
}
}
}
if (deriveEmpty) {
firstSetOut.Insert(0, 0);
}
} else {
dCRCTYPE symbol = symbolSet.GetFirst()->GetInfo();
dTree<int, dCRCTYPE> symbolListMark;
First (symbol, symbolListMark, symbolList, ruleMap, firstSetOut);
}
}
void dDeterministicFiniteAutonata::MoveSymbol (int symbol, const dAutomataState* const state, dTree<dAutomataState*,dAutomataState*>& ouput) const
{
for (dList<dAutomataState*>::dListNode* stateNode = state->m_myNFANullStates.GetFirst(); stateNode; stateNode = stateNode->GetNext()) {
const dAutomataState* const state = stateNode->GetInfo();
for (dList<dAutomataState::dTransition>::dListNode* transitionNode = state->m_transtions.GetFirst(); transitionNode; transitionNode = transitionNode->GetNext()) {
dAutomataState::dTransition& thans = transitionNode->GetInfo();
dAutomataState::dCharacter ch (thans.GetCharater());
if (ch.m_symbol == symbol) {
dAutomataState* const target = thans.GetState();
ouput.Insert(target, target);
}
}
}
}
void dDataFlowGraph::FindNodesInPathway(dCIL::dListNode* const source, dCIL::dListNode* const destination, dTree<int, dCIL::dListNode*>& pathOut) const
{
m_mark ++;
dList<dDataFlowPoint*> queue;
queue.Append(&m_dataFlowGraph.Find(source)->GetInfo());
dAssert (queue.GetFirst()->GetInfo()->m_statement == source);
while (queue.GetCount()) {
dDataFlowPoint* const rootNode = queue.GetFirst()->GetInfo();
queue.Remove(queue.GetFirst());
if (rootNode->m_mark != m_mark) {
rootNode->m_mark = m_mark;
pathOut.Insert(rootNode->m_statement);
for (dList<dDataFlowPoint*>::dListNode* successorsNode = rootNode->m_successors.GetFirst(); successorsNode; successorsNode = successorsNode->GetNext()) {
dDataFlowPoint* const successor = successorsNode->GetInfo();
if (successor->m_statement != destination) {
queue.Append(successor);
}
}
}
}
// pathOut.Remove(source);
}
void dBasicBlocksGraph::GetStatementsWorklist(dTree <int, dCIL::dListNode*>& list) const
{
for (dCIL::dListNode* node = m_begin; node != m_end; node = node->GetNext()) {
list.Insert(0, node);
}
}
// generates the canonical Items set for a LR(1) grammar
void dParserCompiler::CanonicalItemSets (
dTree<dState*, dCRCTYPE>& stateMap,
const dProductionRule& ruleList,
const dTree<dTokenInfo, dCRCTYPE>& symbolList,
const dOperatorsPrecedence& operatorPrecedence,
FILE* const debugFile)
{
dList<dItem> itemSet;
dList<dState*> stateList;
// start by building an item set with only the first rule
dItem& item = itemSet.Append()->GetInfo();
item.m_indexMarker = 0;
item.m_lookAheadSymbolCRC = dCRC64 (DACCEPT_SYMBOL);
item.m_lookAheadSymbolName = DACCEPT_SYMBOL;
item.m_ruleNode = ruleList.GetFirst();
// build a rule info map
dTree<dList<void*>, dCRCTYPE> ruleMap;
for (dProductionRule::dListNode* ruleNode = ruleList.GetFirst(); ruleNode; ruleNode = ruleNode->GetNext()) {
dRuleInfo& info = ruleNode->GetInfo();
dTree<dList<void*>, dCRCTYPE>::dTreeNode* node = ruleMap.Find(info.m_nameCRC);
if (!node) {
node = ruleMap.Insert(info.m_nameCRC);
}
dList<void*>& entry = node->GetInfo();
entry.Append(ruleNode);
}
// find the closure for the first this item set with only the first rule
dState* const state = Closure (itemSet, symbolList, ruleMap);
operatorPrecedence.SaveLastOperationSymbol (state);
stateMap.Insert(state, state->GetKey());
stateList.Append(state);
state->Trace(debugFile);
// now for each state found
int stateNumber = 1;
for (dList<dState*>::dListNode* node = stateList.GetFirst(); node; node = node->GetNext()) {
dState* const state = node->GetInfo();
dTree<dTokenInfo, dCRCTYPE>::Iterator iter (symbolList);
for (iter.Begin(); iter; iter ++) {
dCRCTYPE symbol = iter.GetKey();
dState* const newState = Goto (state, symbol, symbolList, ruleMap);
if (newState->GetCount()) {
const dTokenInfo& tokenInfo = iter.GetNode()->GetInfo();
dTransition& transition = state->m_transitions.Append()->GetInfo();
transition.m_symbol = symbol;
transition.m_name = tokenInfo.m_name;
transition.m_type = tokenInfo.m_type;
dAssert (transition.m_symbol == dCRC64(transition.m_name.GetStr()));
transition.m_targetState = newState;
dTree<dState*, dCRCTYPE>::dTreeNode* const targetStateNode = stateMap.Find(newState->GetKey());
if (!targetStateNode) {
newState->m_number = stateNumber;
stateNumber ++;
stateMap.Insert(newState, newState->GetKey());
newState->Trace(debugFile);
stateList.Append(newState);
operatorPrecedence.SaveLastOperationSymbol (newState);
} else {
transition.m_targetState = targetStateNode->GetInfo();
delete newState;
}
} else {
delete newState;
}
}
dTrace (("state#:%d items: %d transitions: %d\n", state->m_number, state->GetCount(), state->m_transitions.GetCount()));
}
}
dParserCompiler::dToken dParserCompiler::ScanGrammarRule(
dParserLexical& lexical,
dProductionRule& rules,
dTree<dTokenInfo, dCRCTYPE>& symbolList,
int& ruleNumber,
int& tokenEnumeration,
const dOperatorsPrecedence& operatorPrecedence)
{
dRuleInfo* currentRule = &rules.GetLast()->GetInfo();
dToken token = dToken(lexical.NextToken());
do {
dList<dTokenStringPair> ruleTokens;
for (token = dToken(lexical.NextToken()); !((token == SIMICOLOM) || (token == OR)); token = dToken(lexical.NextToken())) {
dAssert (token != -1);
dTokenStringPair& pair = ruleTokens.Append()->GetInfo();
pair.m_token = token;
pair.m_info = lexical.GetTokenString();
}
dList<dTokenStringPair>::dListNode* lastNode = ruleTokens.GetLast();
if (lastNode) {
if (lastNode->GetInfo().m_token != SEMANTIC_ACTION) {
lastNode = NULL;
} else {
currentRule->m_semanticActionCode = lastNode->GetInfo().m_info;
}
}
for (dList<dTokenStringPair>::dListNode* node = ruleTokens.GetFirst(); node != lastNode; node = node->GetNext()) {
dTokenStringPair& pair = node->GetInfo();
if (pair.m_token == LITERAL) {
dSymbol& symbol = currentRule->Append()->GetInfo();
symbol.m_token = pair.m_token;
symbol.m_name = pair.m_info;
symbol.m_nameCRC = dCRC64 (symbol.m_name.GetStr());
dTree<dTokenInfo, dCRCTYPE>::dTreeNode* symbolNode = symbolList.Find(symbol.m_nameCRC);
if (!symbolNode) {
symbolNode = symbolList.Insert(dTokenInfo (tokenEnumeration, NONTERMINAL, symbol.m_name), symbol.m_nameCRC);
tokenEnumeration ++;
}
symbol.m_type = symbolNode->GetInfo().m_type;
} else if (pair.m_token < 256) {
dAssert (pair.m_info.Size() == 1);
dSymbol& symbol = currentRule->Append()->GetInfo();
symbol.m_name = pair.m_info;
symbol.m_nameCRC = dCRC64 (symbol.m_name.GetStr());
symbol.m_type = TERMINAL;
symbol.m_token = LITERAL;
symbolList.Insert(dTokenInfo (pair.m_token, TERMINAL, symbol.m_name), symbol.m_nameCRC);
} else if (pair.m_token == PREC) {
node = node->GetNext();
for (dRuleInfo::dListNode* ruleNode = currentRule->GetLast(); ruleNode; ruleNode = ruleNode->GetPrev()) {
dSymbol& symbol = ruleNode->GetInfo();
if (operatorPrecedence.FindAssociation (symbol.m_nameCRC)) {
dTokenStringPair& pair = node->GetInfo();
symbol.m_operatorPrecendeceOverright = pair.m_info;
break;
}
}
// } else if (pair.m_token != SEMANTIC_ACTION) {
// // no user action allowed in the middle of a sentence
// _ASSERTE (pair.m_token == SEMANTIC_ACTION);
// } else {
// _ASSERTE (0);
}
}
if (token == OR) {
// this is a rule with multiples sentences alternates, add new rule with the same name Non terminal
dRuleInfo& rule = rules.Append()->GetInfo();
rule.m_ruleNumber = ruleNumber;
ruleNumber ++;
rule.m_ruleId = currentRule->m_ruleId;
rule.m_token = currentRule->m_token;
rule.m_type = NONTERMINAL;
//rule.m_name += currentRule->m_name;
rule.m_name = currentRule->m_name;
rule.m_nameCRC = currentRule->m_nameCRC;
currentRule = &rule;
}
} while (token != SIMICOLOM);
return token;
}
void dParserCompiler::ScanGrammarFile(
const dString& inputRules,
dProductionRule& ruleList,
dTree<dTokenInfo, dCRCTYPE>& symbolList,
dOperatorsPrecedence& operatorPrecedence,
dString& userCodeBlock,
dString& userVariableClass,
dString& endUserCode,
int& lastTokenEnum)
{
dString startSymbol ("");
int tokenEnumeration = 256;
int operatorPrecedencePriority = 0;
dParserLexical lexical (inputRules.GetStr());
LoadTemplateFile("dParserUserVariableTemplate_cpp.txt", userVariableClass);
// scan the definition segment
for (dToken token = dToken(lexical.NextToken()); token != GRAMMAR_SEGMENT; ) {
switch (int (token))
{
case START:
{
token = dToken(lexical.NextToken());
startSymbol = lexical.GetTokenString();
token = dToken(lexical.NextToken());
break;
}
case TOKEN:
{
for (token = dToken(lexical.NextToken()); token == LITERAL; token = dToken(lexical.NextToken())) {
const char* const name = lexical.GetTokenString();
symbolList.Insert(dTokenInfo (tokenEnumeration, TERMINAL, name), dCRC64 (name));
tokenEnumeration ++;
}
break;
}
case LEFT:
case RIGHT:
{
dOperatorsAssociation& association = operatorPrecedence.Append()->GetInfo();
association.m_prioprity = operatorPrecedencePriority;
operatorPrecedencePriority ++;
switch (int (token))
{
case LEFT:
association.m_associativity = dOperatorsAssociation::m_left;
break;
case RIGHT:
association.m_associativity = dOperatorsAssociation::m_right;
break;
}
for (token = dToken(lexical.NextToken()); (token == LITERAL) || ((token < 256) && !isalnum (token)); token = dToken(lexical.NextToken())) {
association.Append(dCRC64 (lexical.GetTokenString()));
}
break;
}
case UNION:
{
token = dToken(lexical.NextToken());
dAssert (token == SEMANTIC_ACTION);
userVariableClass = lexical.GetTokenString() + 1;
userVariableClass.Replace(userVariableClass.Size() - 1, 1, "");
token = dToken(lexical.NextToken());
break;
}
case CODE_BLOCK:
{
userCodeBlock += lexical.GetTokenString();
token = dToken(lexical.NextToken());
break;
}
case EXPECT:
{
token = dToken(lexical.NextToken());
dAssert (token == INTEGER);
m_shiftReduceExpectedWarnings = atoi (lexical.GetTokenString());
token = dToken(lexical.NextToken());
break;
}
default:;
{
dAssert (0);
token = dToken(lexical.NextToken());
}
}
}
int ruleNumber = 1;
lastTokenEnum = tokenEnumeration;
// scan the production rules segment
dToken token1 = dToken(lexical.NextToken());
for (; (token1 != GRAMMAR_SEGMENT) && (token1 != -1); token1 = dToken(lexical.NextToken())) {
//dTrace (("%s\n", lexical.GetTokenString()));
switch (int (token1))
{
case LITERAL:
{
// add the first Rule;
dRuleInfo& rule = ruleList.Append()->GetInfo();
rule.m_token = token1;
rule.m_type = NONTERMINAL;
rule.m_name = lexical.GetTokenString();
rule.m_nameCRC = dCRC64 (lexical.GetTokenString());
dTree<dTokenInfo, dCRCTYPE>::dTreeNode* nonTerminalIdNode = symbolList.Find(rule.m_nameCRC);
if (!nonTerminalIdNode) {
nonTerminalIdNode = symbolList.Insert(dTokenInfo (tokenEnumeration, NONTERMINAL, rule.m_name), rule.m_nameCRC);
tokenEnumeration ++;
}
rule.m_ruleId = nonTerminalIdNode->GetInfo().m_tokenId;
rule.m_ruleNumber = ruleNumber;
ruleNumber ++;
token1 = ScanGrammarRule(lexical, ruleList, symbolList, ruleNumber, tokenEnumeration, operatorPrecedence);
break;
}
default:
dAssert (0);
}
}
dProductionRule::dListNode* firtRuleNode = ruleList.GetFirst();
if (startSymbol != "") {
firtRuleNode = ruleList.Find (dCRC64 (startSymbol.GetStr()));
}
//Expand the Grammar Rule by adding an empty start Rule;
const dRuleInfo& firstRule = firtRuleNode->GetInfo();
dRuleInfo& rule = ruleList.Addtop()->GetInfo();
rule.m_ruleNumber = 0;
rule.m_ruleId = tokenEnumeration;
rule.m_token = firstRule.m_token;
rule.m_type = NONTERMINAL;
rule.m_name = firstRule.m_name + dString("__");
rule.m_nameCRC = dCRC64 (rule.m_name.GetStr());
symbolList.Insert(dTokenInfo (tokenEnumeration, rule.m_type, rule.m_name), rule.m_nameCRC);
tokenEnumeration ++;
dSymbol& symbol = rule.Append()->GetInfo();
symbol.m_token = firstRule.m_token;
symbol.m_type = firstRule.m_type;
symbol.m_name = firstRule.m_name;
symbol.m_nameCRC = firstRule.m_nameCRC;
// scan literal use code
if (token1 == GRAMMAR_SEGMENT) {
endUserCode = lexical.GetNextBuffer();
//endUserCode += "\n";
}
}
