Token* getAssignmentStatements(TokenList &tokenList) { Token* ptr = tokenList.getFirst(); //create a pointer pointing to the head of the full list of tokens from our cpp file while(ptr) //until the pointer is at the end { //if statement for checking any assignment / compound assignment operators if (ptr -> getStringRep() == "=" || ptr -> getStringRep() == "+=" || ptr -> getStringRep() == "-=" || ptr -> getStringRep() == "*=" || ptr -> getStringRep() == "/=" || ptr -> getStringRep() == "%=" || ptr -> getStringRep() == "^=" || ptr -> getStringRep() == "<<=" || ptr -> getStringRep() == ">>=" || ptr -> getStringRep() == "&=" || ptr -> getStringRep() == "|=") { numAssignmentStatements++; //since form is (Variable = Expression), only one token before the assignment operator would be taken into account ptr = ptr -> getPrev(); while (ptr -> getStringRep() != ";") //until a semicolon is found { //append the token string (part of the assignment statement) to the assigstats TokenList assigstats.append(ptr->getStringRep()); ptr = ptr -> getNext(); //move pointer to next token } assigstats.append(ptr->getStringRep()); //append semicolon to assigstats } ptr = ptr -> getNext(); //move pointer to next token } return assigstats.getFirst(); //return the head pointer so we can begin traversing through the list }
//Creates a new TokenList, and returns a pointer to this list //Searches for all conditional expressions in tokenList and appends them to the new list //Format is as follows: //Each token that is part of a condtional expression is appended sequentially //At the end of a conditional expression a newline character is appened //Example: if (a = true) then //Your list should include "(", "a", "=", "true", ")" and "\n" //tokenList is NOT modified TokenList* findAllConditionalExpressions(const TokenList &tokenList) { TokenList* conditionalExpressionTokenList = new TokenList(); Token* t = tokenList.getFirst(); while (t!= nullptr && t->getNext()!= nullptr) { if (t!= nullptr && (t->getStringRep() == "if" || t->getStringRep() == "elsif") && t->getPrev()->getStringRep()!="end" && !t->isComment()&& t->getNext()!= nullptr) { t = t->getNext(); while (t!= nullptr && !(t->getStringRep() == "then" && !t->isComment()) && t->getNext()!= nullptr){ conditionalExpressionTokenList->append(t->getStringRep()); t = t->getNext();} conditionalExpressionTokenList ->append("\n"); } if (t!= nullptr && t->getStringRep() == "when" && !t->isComment() && t->getNext()!= nullptr) { t = t->getNext(); while (t!=nullptr && !(t->getStringRep() == "else" && !t->isComment())&& t->getNext()!= nullptr) { conditionalExpressionTokenList->append(t->getStringRep()); t = t->getNext();} conditionalExpressionTokenList ->append("\n"); } t = t->getNext();} return conditionalExpressionTokenList; }
Token* getFunctionDeclarations(TokenList &tokenList) { int typeindex = 12; int numoffuncdetectors = getTypes(tokenList, typeindex); Token* ptr = tokenList.getFirst(); //create a pointer pointing to the head of the full list of tokens from our cpp file while(ptr) //until the pointer is at the end { if (ptr->getStringRep() == "(") { for (int i = 0; i < numoffuncdetectors; i++) { if (ptr->getNext()->getStringRep() == tokenList.tableOfFunctionDetectors[i]) { Token *checkingPtr = ptr; bool isPrototype = false; while(checkingPtr->getStringRep() != ";" && checkingPtr->getStringRep() != "{") //while not at end of declaration { checkingPtr = checkingPtr->getNext(); } if (checkingPtr->getStringRep() == ";") { isPrototype = true; //if the function header is a prototype and not the beginning of a definition } ///////////////////Append Line (token by token) to funcdecs Linked List//////////////////// if (isPrototype) { while (ptr->getStringRep() != ";" && ptr->getStringRep() != "\n" && ptr->getStringRep() != "}") { ptr = ptr->getPrev(); //go back as far as within same function declaration } ptr = ptr->getNext(); while (ptr->getStringRep() != ";" && ptr->getStringRep() != "\n") { funcdecs.append(ptr->getStringRep()); //append the tokens relating to the function declaration to our funcdecs linked list ptr = ptr->getNext(); } funcdecs.append(ptr->getStringRep()); numFunctionDeclarations++; //for statistics at end of code break; } else { ptr = checkingPtr; break; } /////////////////////////////////////////////////////////////////////////////////////////// } } } ptr = ptr -> getNext(); //move pointer to next token } return funcdecs.getFirst(); }
//Example Test code for interacting with your Token, TokenList, and Tokenizer classes //Add your own code to further test the operation of your Token, TokenList, and Tokenizer classes int main() { ifstream sourceFile; TokenList tokens; Tokenizer tokenizer; //Read in a file line-by-line and tokenize each line sourceFile.open("test.cpp"); if (!sourceFile.is_open()) { cout << "Failed to open file" << endl; return 1; } while (!sourceFile.eof()) { string lineA, lineB; getline(sourceFile, lineA); //while the current line ends with a line-continuation \ append the next line to the current line while (lineA.length() > 0 && lineA[lineA.length() - 1] == '\\') { lineA.erase(lineA.length() - 1, 1); getline(sourceFile, lineB); lineA += lineB; } tokenizer.setString(&lineA); while (!tokenizer.isComplete()) { tokens.append(tokenizer.getNextToken()); } //Re-insert newline that was removed by the getline function tokens.append("\n"); } cout << "Inline comments removed: " << removeInlineComments(tokens) << endl; cout << "Block comments removed: " << removeBlockComments(tokens) << endl; /*Test your tokenization of the file by traversing the tokens list and printing out the tokens*/ Token *t = tokens.getFirst(); // while (t) { // cout << t->getStringRep() << endl; // t = t->getNext(); // } for (int i = 0; t; i++) { cout << t->getStringRep() << " "; t = t->getNext(); } return 0; }
TokenList KoEnhancedPathFormula::scan( const QString &formula ) const { // parsing state enum { Start, Finish, Bad, InNumber, InDecimal, InExpIndicator, InExponent, InString, InIdentifier } state; TokenList tokens; int i = 0; state = Start; int tokenStart = 0; QString tokenText; QString expr = formula + QChar(); // main loop while( (state != Bad) && (state != Finish) && (i < expr.length() ) ) { QChar ch = expr[i]; switch( state ) { case Start: tokenStart = i; // skip any whitespaces if( ch.isSpace() ) { i++; } // check for number else if( ch.isDigit() ) { state = InNumber; } // beginning with alphanumeric ? // could be identifier, function, function reference, modifier reference else if( isIdentifier( ch ) ) { state = InIdentifier; } // decimal dot ? else if ( ch == '.' ) { tokenText.append( expr[i++] ); state = InDecimal; } // terminator character else if ( ch == QChar::Null ) { state = Finish; } // look for operator match else { QString opString( ch ); int op = matchOperator( opString ); // any matched operator ? if( op != FormulaToken::OperatorInvalid ) { i++; tokens.append( FormulaToken( FormulaToken::TypeOperator, opString, tokenStart ) ); } else state = Bad; } break; case InIdentifier: // consume as long as alpha, dollar sign, question mark, or digit if( isIdentifier( ch ) || ch.isDigit() ) { tokenText.append( expr[i++] ); } // a '(' ? then this must be a function identifier else if( ch == '(' ) { tokens.append( FormulaToken( FormulaToken::TypeFunction, tokenText, tokenStart) ); tokenStart = i; tokenText = ""; state = Start; } // we're done with identifier else { tokens.append( FormulaToken( FormulaToken::TypeReference, tokenText, tokenStart) ); tokenStart = i; tokenText = ""; state = Start; } break; case InNumber: // consume as long as it's digit if( ch.isDigit() ) { tokenText.append( expr[i++] ); } // decimal dot ? else if( ch == '.' ) { tokenText.append( '.' ); i++; state = InDecimal; } // exponent ? else if( ch.toUpper() == 'E' ) { tokenText.append( 'E' ); i++; state = InExpIndicator; } // we're done with integer number else { tokens.append( FormulaToken( FormulaToken::TypeNumber, tokenText, tokenStart ) ); tokenText = ""; state = Start; }; break; case InDecimal: // consume as long as it's digit if( ch.isDigit() ) { tokenText.append( expr[i++] ); } // exponent ? else if( ch.toUpper() == 'E' ) { tokenText.append( 'E' ); i++; state = InExpIndicator; } // we're done with floating-point number else { tokens.append( FormulaToken( FormulaToken::TypeNumber, tokenText, tokenStart ) ); tokenText = ""; state = Start; }; break; case InExpIndicator: // possible + or - right after E, e.g 1.23E+12 or 4.67E-8 if( ( ch == '+' ) || ( ch == '-' ) ) { tokenText.append( expr[i++] ); } // consume as long as it's digit else if( ch.isDigit() ) { state = InExponent; } // invalid thing here else state = Bad; break; case InExponent: // consume as long as it's digit if( ch.isDigit() ) { tokenText.append( expr[i++] ); } // we're done with floating-point number else { tokens.append( FormulaToken( FormulaToken::TypeNumber, tokenText, tokenStart ) ); tokenText = ""; state = Start; }; break; case Bad: default: break; } } return tokens; }
//Example Test code for interacting with your Token, TokenList, and Tokenizer classes //Add your own code to further test the operation of your Token, TokenList, and Tokenizer classes int main(int argc, char *argv[]) { ifstream sourceFile; TokenList tokens; //Lists for types of tokens TokenList operatorTokens; TokenList identifierTokens; TokenList literalTokens; TokenList commentBodyTokens; TokenList otherTokens; Tokenizer tokenizer; //Read in a file line-by-line and tokenize each line cout << "File: " << argv[1] << endl; sourceFile.open(argv[1]); if (!sourceFile.is_open()) { cout << "Failed to open file" << endl; return 1; } while(!sourceFile.eof()) { string line; getline(sourceFile, line); tokenizer.setString(&line); while(!tokenizer.isComplete()) { tokens.append(tokenizer.getNextToken()); } } // Set token data Token *t = tokens.getFirst(); while(t) { tokens.findAndSetTokenDetails(t); t = t->getNext(); } // Print a list of tokens and all data tokens.print(20); TokenList *conditionalTokens = findAllConditionalExpressions(tokens); cout << "\n\nConditional tokens list: \n"; conditionalTokens->print(20); //tokens.print(20); removeTokensOfType(tokens, T_Other); cout << "No Others: \n"; tokens.print(20); /* For your testing purposes only */ /* Ensure that tokens have all type information set*/ /* Create operator,identifier,literal, etc. tokenLists from the master list of tokens */ return 0; }
TokenList* findAllConditionalExpressions(const TokenList &tokenList) { TokenList Copy = tokenList;//copies the token list TokenList *newList = new TokenList;//new list to hold the conditional expressions Token *Point;//declares a pointer to parse through all tokens Point = Copy.getFirst();//points at the beginning of the token list while(Point) { if(LowerString(Point->getStringRep()) == "if")//if the token is on the word "if" right now { if(Point->getPrev()!=nullptr && LowerString(Point->getPrev()->getStringRep())=="end")//if the token has the string end before it meaning this if is an "end if" { //do nothing } else // when not end if { Point= Point->getNext(); while( Point->getNext()!=nullptr && LowerString(Point->getStringRep()) != "then")//searches for the word then when now at end if { newList->append(Point->getStringRep());//appends all tokens before the word then Point= Point->getNext(); } newList->append("\\n"); // adds \n to the back of the conditional expression } } if(LowerString(Point->getStringRep()) == "elsif"||LowerString(Point->getStringRep()) == "when")//if the token is on the word elseif or when right now { if(LowerString(Point->getStringRep()) == "elsif")//if it is elsif do this { Point =Point-> getNext(); while(Point->getNext() && LowerString(Point->getStringRep()) != "then") { newList->append(Point->getStringRep());//appends all tokens before the word then is seen Point= Point->getNext(); } newList->append("\\n"); // adds \n to the back of the conditional expression } else if(LowerString(Point->getStringRep()) == "when")// if the token is on the string when right now { Point =Point-> getNext(); while(Point->getNext() && LowerString(Point->getStringRep()) != "else") { newList->append(Point->getStringRep());//appends all tokens before the word else is seen Point= Point->getNext(); } newList->append("\\n");// adds \n to the back of the conditional expression } } else { Point=Point->getNext(); // moves through token list } } return newList;// returns the list that holds all conditional expressions }
//Example Test code for interacting with your Token, TokenList, and Tokenizer classes //Add your own code to further test the operation of your Token, TokenList, and Tokenizer classes int main() { ifstream sourceFile; TokenList tokens; Tokenizer tokenizer; // Setting default function detectors to standard types and const, short long, signed, unsigned qualifiers tokens.tableOfFunctionDetectors[0] = ")"; for (int i = 1; i <= 11; i++) { tokens.tableOfFunctionDetectors[i] = ensc251::tableOfKeywords[i]; } tokens.tableOfFunctionDetectors[11] = "string"; int typeindex = 12; //Read in a file line-by-line and tokenize each line sourceFile.open("test.cpp"); if (!sourceFile.is_open()) { cout << "Failed to open file" << endl; return 1; } while (!sourceFile.eof()) { string lineA, lineB; getline(sourceFile, lineA); //while the current line ends with a line-continuation \ append the next line to the current line while (lineA.length() > 0 && lineA[lineA.length() - 1] == '\\') { lineA.erase(lineA.length() - 1, 1); getline(sourceFile, lineB); lineA += lineB; } tokenizer.setString(&lineA); while (!tokenizer.isComplete()) { tokens.append(tokenizer.getNextToken()); } //Re-insert newline that was removed by the getline function tokens.append("\n"); } //comment out for keeping comments in code cout << "Inline comments removed: " << removeInlineComments(tokens) << endl; cout << "Block comments removed: " << removeBlockComments(tokens) << endl; /*Test your tokenization of the file by traversing the tokens list and printing out the tokens*/ Token *t = tokens.getFirst(); cout << "************************* Tokens Printed With Types *************************" << endl << endl << endl; while (t) { cout << t->getStringRep() << " "; if (t->getStringRep() != "\n") { printType(t->getStringType()); //--------comment at beginning of line to remove token types after each token-------------- cout << " "; } t = t->getNext(); numTokensParsed++; //Counts how many tokens there are in the whole cpp file } //Test your assignment statements (prints assignment statements at the end) cout << endl << endl << "*************************** Assignment Statements ***************************" << endl << endl << endl; Token *aListPtr = getAssignmentStatements(tokens); classifyIdentifiers(tokens); while(aListPtr) { cout << aListPtr->getStringRep() << " " << aListPtr->getIdentifierType(); if (aListPtr->getStringRep() == ";") //separate assignment statements by new lines { cout << endl; } aListPtr = aListPtr->getNext(); } cout << endl << endl << "*************************** Function Declarations ***************************" << endl << endl << endl; Token *FuncListPtr = getFunctionDeclarations(tokens); while(FuncListPtr) { cout << FuncListPtr->getStringRep() << " "; if (FuncListPtr->getStringRep() == ";") //separate assignment statements by new lines { cout << endl; } FuncListPtr = FuncListPtr->getNext(); } cout << endl << endl << "********************************** Errors ***********************************" << endl << endl; cout << "There are: " << typeMismatchChecker(assigstats) << " type mismatch error(s).\n"; cout << "There are: " << bracketMismatchChecker(assigstats) << " bracket mismatch error(s).\n"; int choice = 0; cout << endl << endl << endl << endl << "********************* Welcome to Amar and Cem's Parser **********************" << endl; cout << "To begin, choose a mode: " << endl << "(1) Non-Verbose" << endl << "(2) Verbose" << endl << endl; cout << "Choice: "; while (!(cin >> choice) || (choice < 1 || choice > 2)) { //if an invalid character is read in cin.clear(); cin.ignore(numeric_limits<streamsize>::max(),'\n'); cout << "Invalid entry; please try again: " << endl << "Choice: "; } if (choice == 1) { cout << endl << "The number of assignment statements is: " << numAssignmentStatements; cout << endl << "The number of function declarations is: "; if (numFunctionDeclarations == -1) {numFunctionDeclarations = 0;} cout << numFunctionDeclarations; cout << endl << "The number of tokens parsed is: " << numTokensParsed; cout << endl << "The number of each type of token is: " << endl; for (int i = 5; i < typeindex+2; i++) { cout << tokens.tableOfFunctionDetectors[i] << endl; } for (int i = 5; i < typeindex+2; i++) { cout << tokens.tableOfFunctionDetectors[i] << ": "; int countofspecifictoken = 0; Token *t = tokens.getFirst(); while (t) { //checking if current token is of same type we're looking for bool isInteger = t->getIdentifierType() == "int" || t->getStringType() == ensc251::T_IntegerLiteral; bool isDoubleOrFloat = t->getIdentifierType() == "float" || t->getIdentifierType() == "double" || t->getStringType() == ensc251::T_FloatLiteral; bool isString = t->getIdentifierType() == "string" || t->getStringType() == ensc251::T_StringLiteral; bool integermatch = tokens.tableOfFunctionDetectors[i] == "int" && isInteger; bool doublefloatmatch = (tokens.tableOfFunctionDetectors[i] == "double" || tokens.tableOfFunctionDetectors[i] == "float") && isDoubleOrFloat; bool stringmatch = tokens.tableOfFunctionDetectors[i] == "string" && isString; if (integermatch || doublefloatmatch || stringmatch) { //if it is of the same type, increase the count of that specific type countofspecifictoken++; } t = t->getNext(); } cout << countofspecifictoken << endl; } } if (choice == 2) { cout << endl << "The number of assignment statements is: " << numAssignmentStatements; cout << endl << "The number of function declarations is: " << numFunctionDeclarations; cout << endl << "The number of tokens parsed is: " << numTokensParsed; cout << endl << "The tokens of each type are as follows: " << endl; for (int i = 5; i < typeindex+2; i++) { cout << tokens.tableOfFunctionDetectors[i] << ": \n"; int countofspecifictoken = 0; Token *t = tokens.getFirst(); while (t) { //checks if the current token is of the same type we're looking for bool isInteger = t->getIdentifierType() == "int" || t->getStringType() == ensc251::T_IntegerLiteral; bool isDoubleOrFloat = t->getIdentifierType() == "float" || t->getIdentifierType() == "double" || t->getStringType() == ensc251::T_FloatLiteral; bool isString = t->getIdentifierType() == "string" || t->getStringType() == ensc251::T_StringLiteral; bool integermatch = tokens.tableOfFunctionDetectors[i] == "int" && isInteger; bool doublefloatmatch = (tokens.tableOfFunctionDetectors[i] == "double" || tokens.tableOfFunctionDetectors[i] == "float") && isDoubleOrFloat; bool stringmatch = tokens.tableOfFunctionDetectors[i] == "string" && isString; if ((integermatch || doublefloatmatch || stringmatch) && t->getIdentifierType() != "double") { //if it is of the same type, print it out to the screen with commas as separation cout << t->getStringRep() << ", "; } t = t->getNext(); } cout << endl; } } cout << endl; return 0; }
//Example Test code for interacting with your Token, TokenList, and Tokenizer classes //Add your own code to further test the operation of your Token, TokenList, and Tokenizer classes int main() { ifstream sourceFile; TokenList tokens; //Lists for types of tokens TokenList operatorTokens; TokenList identifierTokens; TokenList literalTokens; TokenList commentBodyTokens; TokenList otherTokens; TokenList keywordTkoens; Tokenizer tokenizer; //Read in a file line-by-line and tokenize each line sourceFile.open("/Users/arlene/Desktop/251Final/251Final/test.vhd"); if (!sourceFile.is_open()) { cout << "Failed to open file" << endl; return 1; } while(!sourceFile.eof()) { string line; getline(sourceFile, line); tokenizer.setString(&line); while(!tokenizer.isComplete()) { tokens.append(tokenizer.getNextToken()); } } /*Test your tokenization of the file by traversing the tokens list and printing out the tokens*/ Token *t = tokens.getFirst(); while(t) { cout << t->getStringRep() << " "; t = t->getNext(); } //--------------------------------------test code for part 1------------------------------------------- /*Token *t = tokens.getFirst(); while(t){ tokens.findAndSetTokenDetails(t); if (t->isOperator()){ Token *temp = new Token(*t); temp->setNext(nullptr); operatorTokens.append(temp); } if (t->isLiteral()){ Token *temp = new Token(*t); temp->setNext(nullptr); literalTokens.append(temp); } if (t->isKeyword()){ Token *temp = new Token(*t); temp->setNext(nullptr); keywordTkoens.append(temp); } if (t->isComment()){ Token *temp = new Token(*t); temp->setNext(nullptr); commentBodyTokens.append(temp); } if (t->isIdentifier()){ Token *temp = new Token(*t); temp->setNext(nullptr); identifierTokens.append(temp); } if (t->isOther()){ Token *temp = new Token(*t); temp->setNext(nullptr); otherTokens.append(temp); } t = t->getNext(); } t = identifierTokens.getFirst(); while(t){ cout << t->getStringRep() << " "; if (t->getTokenDetails() != nullptr){ cout << t->getTokenDetails()->width << " "; cout << t->getTokenDetails()->type << " "<<endl;} t = t->getNext();}*/ //---------------------------------------test remove Token Type----------------------------------------- // cout << removeTokensOfType(tokens, T_CommentBody) <<endl; //---------------------------------------test removeComments-------------------------------------------- /*removeComments(tokens); t=tokens.getFirst(); while(t){ cout << t->getStringRep() << " "; t = t->getNext(); }*/ //--------------------------------------------test part 2----------------------------------------------- /* TokenList *p = findAllConditionalExpressions(tokens); Token *t = p->getFirst(); while(t!= nullptr) { cout << t->getStringRep() << " "; t = t->getNext(); }*/ return 0; }
Token* getAssignmentStatements(TokenList &tokenList) { Token *t = tokenList.getFirst(); //current position in our token of the passed in token list TokenList *assignmentList; assignmentList = new TokenList; Token * temp = new Token; //temporary position of tokens bool isIdentifier = false; bool found = false; { using namespace ensc251; while (t) { if (t->getStringRep() == "=" || t->getStringRep() == "+=" || t->getStringRep() == "-=" || t->getStringRep() == "*=" || t->getStringRep() == "/=" || t->getStringRep() == "<<=" || t->getStringRep() == ">>=" || t->getStringRep() == "|=" || t->getStringRep() == "&=" || t->getStringRep() == "^=") { //check if it is one of these equals temp = t; while (!found) { //if we find a keyword, we found the end of the expression if (temp->getStringType() == T_Keyword) { found = true; } //checks if identifier, if it is then if the last one was also an identifier then we found the end, otherwise keep searching else if (temp->getStringType() == T_Identifier) { if (isIdentifier) { found = true; isIdentifier = false; } else { isIdentifier = true; found = false; if (temp->getPrev() != NULL) { temp = temp->getPrev(); } } } //if we find a ; or a new line else if (temp->getStringRep() == ";" || temp->getStringRep() == "\n") { found = true; isIdentifier = false; } //otherwise it is all other cases which we do not have to cover else { if (temp->getPrev() != NULL) { temp = temp->getPrev(); isIdentifier = false; } else { found = true; } } } //create the string that will hold the entire expression, up till the ; isIdentifier = false; found = false; if (temp->getPrev() == NULL) { //don't change the position of temp } else { temp = temp->getNext(); } while (temp->getStringRep() != ";") { Token* temp2 = new Token; temp2->setDataType(temp->getDataType()); temp2->setStringRep(temp->getStringRep()); temp2->setStringType(temp->getStringType()); temp = temp->getNext(); assignmentList->append(temp2); } Token* temp2 = new Token; temp2->setDataType(temp->getDataType()); temp2->setStringRep(temp->getStringRep()); temp2->setStringType(temp->getStringType()); temp = temp->getNext(); assignmentList->append(temp2); //create the token and append it to the assignment list with the given string t = temp; } else { t = t->getNext(); } } } return assignmentList->getFirst(); }
//Example Test code for interacting with your Token, TokenList, and Tokenizer classes //Add your own code to further test the operation of your Token, TokenList, and Tokenizer classes int main() { ifstream sourceFile; TokenList tokens; Tokenizer tokenizer; //Read in a file line-by-line and tokenize each line sourceFile.open("test.cpp"); if (!sourceFile.is_open()) { cout << "Failed to open file" << endl; return 1; } while (!sourceFile.eof()) { string lineA, lineB; getline(sourceFile, lineA); //while the current line ends with a line-continuation \ append the next line to the current line while (lineA.length() > 0 && lineA[lineA.length() - 1] == '\\') { lineA.erase(lineA.length() - 1, 1); getline(sourceFile, lineB); lineA += lineB; } tokenizer.setString(&lineA); while (!tokenizer.isComplete()) { tokens.append(tokenizer.getNextToken()); } //Re-insert newline that was removed by the getline function tokens.append("\n"); } removeInlineComments(tokens); removeBlockComments(tokens); //asks the user what mode that they want char userResponse; cout << "Input the number corresponding to the type you want to run" << endl; cout << "Non-Verbose = 1" << endl << "Verbose = 2" << endl << "Exit = 3" << endl; cin >> userResponse; //enter non verbose mode if (userResponse == '1') { //Number of assignment Statements ensc251::setsUserDataTypes(tokens.getFirst()); ensc251::setDataTypeList(tokens.getFirst()); Token *aListPtr = getAssignmentStatements(tokens); Token* tempAList = aListPtr; int numOfAStatements = 0; while (tempAList) { if (tempAList->getStringRep() == ";") { numOfAStatements++; } tempAList = tempAList->getNext(); } //prints out the assignment statements and its following errors cout << "---------------NON VERBOSE OUTPUT----------------------" << endl; cout << "The number of assignment statements: " << numOfAStatements << endl; cout << "The number of assignment statements with unmatched types: " << dataTypes::numberOfUnmatchedTypes(aListPtr) << endl; cout << "The number of assignment statements with unmatched braces: " << dataTypes::numberOfUnmatchedBraces(aListPtr) << endl; //number of function declarations Token* fDec = dataTypes::setFunctionDeclarations(tokens.getFirst()); Token* tempFDec = fDec; int numOfFDecs = 0; while (tempFDec) { if (tempFDec->getStringRep() == ";") { numOfFDecs++; } tempFDec = tempFDec->getNext(); } cout << "The number of function declarations: " << numOfFDecs << endl; // total number of tokens in the program --------------------------------------------- Token* tcount = tokens.getFirst(); int numOftokens = 0; while (tcount) { numOftokens++; tcount = tcount->getNext(); } cout << "The number of tokens: " << numOftokens << endl; // total # of token types in the program Token* tokentype = tokens.getFirst(); int lexicaltypecount[9] = {}; while (tokentype) { using namespace ensc251; //0 if (tokentype->getStringType() == T_Identifier){ lexicaltypecount[0]++; } //1 if (tokentype->getStringType() == T_Operator){ lexicaltypecount[1]++; } //2 if (tokentype->getStringType() == T_Punctuator){ lexicaltypecount[2]++; } //3 if (tokentype->getStringType() == T_Keyword){ lexicaltypecount[3]++; } //4 if (tokentype->getStringType() == T_Boolean){ lexicaltypecount[4]++; } //5 if (tokentype->getStringType() == T_IntegerLiteral){ lexicaltypecount[5]++; } //6 if (tokentype->getStringType() == T_FloatLiteral){ lexicaltypecount[6]++; } //7 if (tokentype->getStringType() == T_StringLiteral){ lexicaltypecount[7]++; } //8 if (tokentype->getStringType() == T_Unknown){ lexicaltypecount[8]++; } tokentype = tokentype->getNext(); } //prints out the total number of each type of token cout << "-------LEXEMES TYPES-------" << endl; cout << "The number of identifiers " << lexicaltypecount[0] << endl; cout << "The number of operators " << lexicaltypecount[1] << endl; cout << "The number of keywords " << lexicaltypecount[2] << endl; cout << "The number of puntuators " << lexicaltypecount[3] << endl; cout << "The number of boolean values " << lexicaltypecount[4] << endl; cout << "The number of integer literals " << lexicaltypecount[5] << endl; cout << "The number of float literals " << lexicaltypecount[6] << endl; cout << "The number of string literals " << lexicaltypecount[7] << endl; cout << "The number of unknown tokens " << lexicaltypecount[8] << endl; cout << "------------------------------------------" << endl; //lets the user look at it, and presses a key to end the program cout << endl << endl; system("PAUSE"); } //enters verbose mode else if (userResponse == '2') { dataTypes::setsUserDataTypes(tokens.getFirst()); dataTypes::setDataTypeList(tokens.getFirst()); //Number of assignment Statements Token *aListPtr = getAssignmentStatements(tokens); Token* tempAList = aListPtr; int numOfAStatements = 1; bool printNumOfStatements = true; cout << "---------------VERBOSE OUTPUT----------------------" << endl; cout << "The assignment statements: " << endl; //prints out the assignment statements from beginning to the ";" while (tempAList) { if (printNumOfStatements) { cout << numOfAStatements << " | "; printNumOfStatements = false; } cout << tempAList->getStringRep() << " "; if (tempAList->getStringRep() == ";") { cout << endl; printNumOfStatements = true; numOfAStatements++; } tempAList = tempAList->getNext(); } dataTypes::numberOfUnmatchedTypes(aListPtr); dataTypes::numberOfUnmatchedBraces(aListPtr); cout << endl << "Assignment statements unmatched types errors: " << endl; for (int i = 0; i < dataTypes::unmatchedErrors.size(); i++) { cout << i + 1 << " | " << dataTypes::unmatchedErrors[i] << endl; } cout << endl << "Assignment statements unmatched braces errors: " << endl; for (int i = 0; i < dataTypes::bracesErrors.size(); i++) { cout << i + 1 << " | " << dataTypes::bracesErrors[i] << endl; } //number of function declarations //counts by finding the ";" which means that we have 1 function declaration, //print out everything to the ";" Token* fDec = dataTypes::setFunctionDeclarations(tokens.getFirst()); Token* tempFDec = fDec; int numOfFDecs = 1; bool printNumOfFunction = true; cout << endl << "The function declarations: " << endl; while (tempFDec) { if (printNumOfFunction) { cout << numOfFDecs << " | "; printNumOfFunction = false; } cout << tempFDec->getStringRep() << " "; if (tempFDec->getStringRep() == ";") { cout << endl; printNumOfFunction = true; numOfFDecs++; } tempFDec = tempFDec->getNext(); } //counts the number of tokens in our list Token* tcount = tokens.getFirst(); int numOftokens = 0; while (tcount) { numOftokens++; tcount = tcount->getNext(); } cout << "The number of tokens: " << numOftokens << endl; // total # of token types in the program--------------- Token* tokentype = tokens.getFirst(); int lexicaltypecount[9] = {}; while (tokentype) { using namespace ensc251; //0 if (tokentype->getStringType() == T_Identifier){ lexicaltypecount[0]++; } //1 if (tokentype->getStringType() == T_Operator){ lexicaltypecount[1]++; } //2 if (tokentype->getStringType() == T_Punctuator){ lexicaltypecount[2]++; } //3 if (tokentype->getStringType() == T_Keyword){ lexicaltypecount[3]++; } //4 if (tokentype->getStringType() == T_Boolean){ lexicaltypecount[4]++; } //5 if (tokentype->getStringType() == T_IntegerLiteral){ lexicaltypecount[5]++; } //6 if (tokentype->getStringType() == T_FloatLiteral){ lexicaltypecount[6]++; } //7 if (tokentype->getStringType() == T_StringLiteral){ lexicaltypecount[7]++; } //8 if (tokentype->getStringType() == T_Unknown){ lexicaltypecount[8]++; } tokentype = tokentype->getNext(); } //prints out the amount of each type cout << "-------LEXEMES TYPES-------" << endl; cout << "The number of identifiers " << lexicaltypecount[0] << endl; cout << "The number of operators " << lexicaltypecount[1] << endl; cout << "The number of keywords " << lexicaltypecount[2] << endl; cout << "The number of puntuators " << lexicaltypecount[3] << endl; cout << "The number of boolean values " << lexicaltypecount[4] << endl; cout << "The number of integer literals " << lexicaltypecount[5] << endl; cout << "The number of float literals " << lexicaltypecount[6] << endl; cout << "The number of string literals " << lexicaltypecount[7] << endl; cout << "The number of unknown tokens " << lexicaltypecount[8] << endl; cout << "------------------------------------------" << endl; // printing out lexemes types using namespace ensc251; { //all other examples follow the same procedure, so will only explain once //Go from the top of the list and check if it is identifier type (or other for other implementations) and if it is then print it out //go through the entire list until its done cout << "THE IDENTIFIERS:" << endl; Token* identifierparse = tokens.getFirst(); while (identifierparse){ if (identifierparse->getStringType() == T_Identifier) cout << identifierparse->getStringRep() << endl; identifierparse = identifierparse->getNext(); } cout << "THE OPERATORS:" << endl; Token* operatorparse = tokens.getFirst(); while (operatorparse){ if (operatorparse->getStringType() == T_Operator) cout << operatorparse->getStringRep() << endl; operatorparse = operatorparse->getNext(); } cout << "THE KEYWORDS:" << endl; Token* keywordparse = tokens.getFirst(); while (keywordparse){ if (keywordparse->getStringType() == T_Keyword) cout << keywordparse->getStringRep() << endl; keywordparse = keywordparse->getNext(); } cout << "THE PUNCTUATOR:" << endl; Token* puntuatorparse = tokens.getFirst(); while (puntuatorparse){ if (puntuatorparse->getStringType() == T_Punctuator) cout << puntuatorparse->getStringRep() << endl; puntuatorparse = puntuatorparse->getNext(); }/**/ cout << "THE BOOLEAN VALUES:" << endl; Token* booleanparse = tokens.getFirst(); while (booleanparse){ if (booleanparse->getStringType() == T_Boolean) { cout << booleanparse->getStringRep() << endl; } booleanparse = booleanparse->getNext(); } cout << "THE INTEGER LITERALS:" << endl; Token* integerparse = tokens.getFirst(); while (integerparse){ if (integerparse->getStringType() == T_IntegerLiteral) { cout << integerparse->getStringRep() << endl; } integerparse = integerparse->getNext(); } cout << "THE FLOAT LITERALS:" << endl; Token* floatparse = tokens.getFirst(); while (floatparse){ if (floatparse->getStringType() == T_FloatLiteral) { cout << floatparse->getStringRep() << endl; } floatparse = floatparse->getNext(); } cout << "THE STRING LITERALS:" << endl; Token* stringparse = tokens.getFirst(); while (stringparse){ if (stringparse->getStringType() == T_StringLiteral) { cout << stringparse->getStringRep() << endl; } stringparse = stringparse->getNext(); } cout << "THE UNKNOWNS:" << endl; Token* unknownparse = tokens.getFirst(); while (unknownparse){ if (unknownparse->getStringType() == T_Unknown) { cout << unknownparse->getStringRep() << endl; } unknownparse = unknownparse->getNext(); } } //waits for the user to press a button to end the program cout << endl << endl; system("PAUSE"); } //says goodbye to the user else if (userResponse == '3') { cout << "Goodbye"; } //user puts in invalid input, we end the program without a nice goodbye else { cout << "Wrong input, terminating program!"; }/**/ return 0; }
int main() { ifstream sourceFile; TokenList tokens; //Lists for types of tokens TokenList operatorTokens; TokenList identifierTokens; TokenList literalTokens; TokenList commentBodyTokens; TokenList otherTokens; Tokenizer tokenizer; bool verboseModeFlag = false; string userInput; char parsedUserInput; vector<string> errorLines; string errorMissingEndIf = "Missing \"End If\" here: "; string errorMissingThen = "Missing \"Then\" here: "; string errorMissingIf = "Extra \"End If\" Here: "; string errorExtraThen = "Extra \"Then\" here: "; string errorTypeMismatch = "Type Mismatch of types : "; string errorWidthMismatch = "Width Mismatch: "; int numberOfTokens =0; int numberOfCondExp =0; int numberOfMissingEndIfs = 0; int numberOfMissingIfs =0; int numberofMissingThens =0; int numberofMissingProcess =0; int numberofMissingEndProcess=0; int numberofMissingOpenBracket =0; int numberofMissingCloseBracket =0; int ifEndifBalance =0; //if there is a positive number there are too many ifs, negative number too many endifs. int ifthenBalance =0; //Like above except with Then. int processBalance =0; // like above except with process - end process int BracketBalance =0; // check the missing bracket //Read in a file line-by-line and tokenize each line cout << "Enter the name of the file to open: "; cin >> userInput; sourceFile.open(userInput); if (!sourceFile.is_open()) { cout << "Failed to open file" << endl; return 1; } while(!sourceFile.eof()) { string line; getline(sourceFile, line); tokenizer.setString(&line); while(!tokenizer.isComplete()) { tokens.append(tokenizer.getNextToken()); } } ///removeComments(tokens); ///test remove comment Token *t = tokens.getFirst(); while(t) { tokens.findAndSetTokenDetails(t); ///test findAndSetTokenDetails /*if(t->getTokenType() == 1 || t->getTokenType() == 2) { detailtoken = *(t->getTokenDetails()); //cout << t->getStringRep() << " Token Type: " << t->getTokenType() <<" Token Detail: " << detailtoken.type << " Token Width: " << detailtoken.width <<endl; t = t->getNext(); } else { //cout << t->getStringRep() << " Token Type: " << t->getTokenType() <<endl; t = t->getNext(); } */ t = t->getNext(); } cout << "Enter Verbose Mode? Type \"1\" for Yes, Other inputs will be a No. : "; cin >> userInput; parsedUserInput = userInput[0]; if (parsedUserInput == '1') { verboseModeFlag = true; } //This part counts the number of tokens. t = tokens.getFirst(); while(t) { numberOfTokens++; t = t->getNext(); } //This part counts the number of conditional expressions. TokenList *conditionalTokenList = findAllConditionalExpressions(tokens); Token *condtokens = conditionalTokenList->getFirst(); while(condtokens) { if(condtokens->getStringRep() == "\\n") { numberOfCondExp++; } condtokens = condtokens->getNext(); } //This part counts the number of ifs, end ifs. t = tokens.getFirst(); Token *tError = tokens.getFirst(); string errorCodeLine[7] = {" ", " ", " ", " ", " ", " ", " "}; while(t) { if(t->getStringRep() == "if") { if(t->getPrev()->getStringRep() != "end") { ifEndifBalance++; if(ifEndifBalance>0) { { // this part gets the tokens around the error. errorCodeLine[3] = t->getStringRep(); if(t->getPrev()!=nullptr) { tError = t->getPrev(); errorCodeLine[2] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[1] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[0] = tError->getStringRep(); } if(t->getNext()!=nullptr) { tError = t->getNext(); errorCodeLine[4] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[5] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[6] = tError->getStringRep(); } } errorLines.push_back(errorMissingEndIf+errorCodeLine[0]+" "+errorCodeLine[1]+" "+errorCodeLine[2]+" "+errorCodeLine[3]+" "+errorCodeLine[4]+" "+errorCodeLine[5]+" "+errorCodeLine[6]); for(int ii=0; ii <7;ii++) { errorCodeLine[ii] = " "; } } else if(ifEndifBalance<=0) { errorLines.pop_back(); } } } else if (t->getStringRep() == "end") { if(t->getNext()!= nullptr) { tError = t->getNext(); if(tError ->getStringRep() == "if") { ifEndifBalance--; if(ifEndifBalance<0) { { // This part gets the tokens around the error errorCodeLine[3] = t->getStringRep(); if(t->getPrev()!=nullptr) { tError = t->getPrev(); errorCodeLine[2] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[1] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[0] = tError->getStringRep(); } if(t->getNext()!=nullptr) { tError = t->getNext(); errorCodeLine[4] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[5] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[6] = tError->getStringRep(); } } errorLines.push_back(errorMissingIf+errorCodeLine[0]+" "+errorCodeLine[1]+" "+errorCodeLine[2]+" "+errorCodeLine[3]+" "+errorCodeLine[4]+" "+errorCodeLine[5]+" "+errorCodeLine[6]); for(int ii=0; ii <7;ii++) { errorCodeLine[ii] = " "; } } else if (ifEndifBalance>=0) { errorLines.pop_back(); } } } } t = t->getNext(); } //this part finds the missing thens. t = tokens.getFirst(); tError = tokens.getFirst(); while(t) { if(t->getStringRep() == "if" || t->getStringRep() == "elsif" ) { if(t->getPrev()->getStringRep() != "end") { ifthenBalance++; if(ifthenBalance>0) { { // this part gets the tokens around the error. errorCodeLine[3] = t->getStringRep(); if(t->getPrev()!=nullptr) { tError = t->getPrev(); errorCodeLine[2] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[1] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[0] = tError->getStringRep(); } if(t->getNext()!=nullptr) { tError = t->getNext(); errorCodeLine[4] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[5] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[6] = tError->getStringRep(); } } errorLines.push_back(errorMissingThen+errorCodeLine[0]+" "+errorCodeLine[1]+" "+errorCodeLine[2]+" "+errorCodeLine[3]+" "+errorCodeLine[4]+" "+errorCodeLine[5]+" "+errorCodeLine[6]); for(int ii=0; ii <7;ii++) { errorCodeLine[ii] = " "; } } else if(ifthenBalance<=0) { errorLines.pop_back(); } } } else if (t->getStringRep() == "then") { ifthenBalance--; if(ifthenBalance<0) { { // This part gets the tokens around the error errorCodeLine[3] = t->getStringRep(); if(t->getPrev()!=nullptr) { tError = t->getPrev(); errorCodeLine[2] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[1] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[0] = tError->getStringRep(); } if(t->getNext()!=nullptr) { tError = t->getNext(); errorCodeLine[4] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[5] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[6] = tError->getStringRep(); } } errorLines.push_back(errorExtraThen+errorCodeLine[0]+" "+errorCodeLine[1]+" "+errorCodeLine[2]+" "+errorCodeLine[3]+" "+errorCodeLine[4]+" "+errorCodeLine[5]+" "+errorCodeLine[6]); for(int ii=0; ii <7;ii++) { errorCodeLine[ii] = " "; } } else if (ifthenBalance>=0) { errorLines.pop_back(); } } t = t->getNext(); } /// ///////////////////////////////////////This part counts the number of Process and End Process//////////////////////////////////////// t = tokens.getFirst(); tError = tokens.getFirst(); while(t) { if(t->getStringRep() == "process") { if(t->getPrev()->getStringRep() != "end") { processBalance++; if(processBalance>0) { { // this part gets the tokens around the error. errorCodeLine[3] = t->getStringRep(); if(t->getPrev()!=nullptr) { tError = t->getPrev(); errorCodeLine[2] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[1] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[0] = tError->getStringRep(); } if(t->getNext()!=nullptr) { tError = t->getNext(); errorCodeLine[4] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[5] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[6] = tError->getStringRep(); } } errorLines.push_back("Missing \"End Process\" here: "+errorCodeLine[0]+" "+errorCodeLine[1]+" "+errorCodeLine[2]+" "+errorCodeLine[3]+" "+errorCodeLine[4]+" "+errorCodeLine[5]+" "+errorCodeLine[6]); for(int ii=0; ii <7;ii++) { errorCodeLine[ii] = " "; } } else if(ifEndifBalance<=0) { errorLines.pop_back(); } } } else if (t->getStringRep() == "end") { if(t->getNext()!= nullptr) { tError = t->getNext(); if(tError ->getStringRep() == "process") { processBalance--; if(processBalance<0) { { // This part gets the tokens around the error errorCodeLine[3] = t->getStringRep(); if(t->getPrev()!=nullptr) { tError = t->getPrev(); errorCodeLine[2] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[1] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[0] = tError->getStringRep(); } if(t->getNext()!=nullptr) { tError = t->getNext(); errorCodeLine[4] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[5] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[6] = tError->getStringRep(); } } errorLines.push_back("Missing \"Process\" here: "+errorCodeLine[0]+" "+errorCodeLine[1]+" "+errorCodeLine[2]+" "+errorCodeLine[3]+" "+errorCodeLine[4]+" "+errorCodeLine[5]+" "+errorCodeLine[6]); for(int ii=0; ii <7;ii++) { errorCodeLine[ii] = " "; } } else if (processBalance>=0) { errorLines.pop_back(); } } } } t = t->getNext(); } /// ///////////////////////////////////////This part counts the number of missing bracket//////////////////////////////////////// t = tokens.getFirst(); tError = tokens.getFirst(); while(t) { if(t->getStringRep() == "(") { //if(t->getPrev()->getStringRep() != "end") //{ BracketBalance++; if(BracketBalance>0) { { // this part gets the tokens around the error. errorCodeLine[3] = t->getStringRep(); if(t->getPrev()!=nullptr) { tError = t->getPrev(); errorCodeLine[2] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[1] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[0] = tError->getStringRep(); } if(t->getNext()!=nullptr) { tError = t->getNext(); errorCodeLine[4] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[5] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[6] = tError->getStringRep(); } } errorLines.push_back("Missing \"Close Bracket \")\" \" here: "+errorCodeLine[0]+" "+errorCodeLine[1]+" "+errorCodeLine[2]+" "+errorCodeLine[3]+" "+errorCodeLine[4]+" "+errorCodeLine[5]+" "+errorCodeLine[6]); for(int ii=0; ii <7;ii++) { errorCodeLine[ii] = " "; } } else if(BracketBalance<=0) { errorLines.pop_back(); } //} } else if (t->getStringRep() == ")") { //if(t->getNext()!= nullptr) //{ //tError = t->getNext(); //if(tError ->getStringRep() == "process") // { BracketBalance--; if(BracketBalance<0) { { // This part gets the tokens around the error errorCodeLine[3] = t->getStringRep(); if(t->getPrev()!=nullptr) { tError = t->getPrev(); errorCodeLine[2] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[1] = tError->getStringRep(); } if(tError->getPrev()!=nullptr) { tError = tError->getPrev(); errorCodeLine[0] = tError->getStringRep(); } if(t->getNext()!=nullptr) { tError = t->getNext(); errorCodeLine[4] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[5] = tError->getStringRep(); } if(tError->getNext()!=nullptr) { tError = tError->getNext(); errorCodeLine[6] = tError->getStringRep(); } } errorLines.push_back("Missing \"Open Bracket \"(\" \" here: "+errorCodeLine[0]+" "+errorCodeLine[1]+" "+errorCodeLine[2]+" "+errorCodeLine[3]+" "+errorCodeLine[4]+" "+errorCodeLine[5]+" "+errorCodeLine[6]); for(int ii=0; ii <7;ii++) { errorCodeLine[ii] = " "; } } else if (BracketBalance>=0) { errorLines.pop_back(); } } //} //} t = t->getNext(); } ////////////////////////////////////set the balance //////////////////////////////// if (ifEndifBalance == 0) { numberOfMissingEndIfs =0; numberOfMissingIfs = 0; } else if (ifEndifBalance > 0) { numberOfMissingEndIfs = abs(ifEndifBalance); numberOfMissingIfs = 0; } else if (ifEndifBalance <0) { numberOfMissingEndIfs =0; numberOfMissingIfs = abs(ifEndifBalance); } if (ifthenBalance == 0) { numberofMissingThens =0; } else if (ifthenBalance > 0) { numberofMissingThens = abs(ifthenBalance); } else if (ifthenBalance <0) { numberOfMissingIfs = numberOfMissingIfs + abs(ifthenBalance); numberofMissingThens =0; } if (processBalance == 0) { numberofMissingProcess=0; numberofMissingEndProcess=0; } else if (processBalance > 0) { numberofMissingEndProcess = abs(processBalance); numberofMissingProcess =0; } else if (processBalance <0) { numberofMissingProcess = numberofMissingProcess + abs(processBalance); numberofMissingEndProcess =0; } if (BracketBalance == 0) { numberofMissingOpenBracket=0; numberofMissingCloseBracket=0; } else if (BracketBalance > 0) { numberofMissingCloseBracket = abs(BracketBalance); numberofMissingOpenBracket =0; } else if (BracketBalance < 0) { numberofMissingOpenBracket = numberofMissingOpenBracket + abs(BracketBalance); numberofMissingCloseBracket=0; } t = tokens.getFirst(); Token *t_before = tokens.getFirst(); Token *t_after = tokens.getFirst(); tokenDetails *t_before_details = nullptr; tokenDetails *t_after_details = nullptr; bool operatorFlag = false; while(t) { /* operatorFlag = false; for(int ii = 0; ii < 28; ii++) { if(t->getStringRep() == operatorList[ii]) { operatorFlag = true; } } */ if(t->isOperator()) { if(t->getPrev()!= nullptr && t->getNext() != nullptr) { t_before = t->getPrev(); t_after = t->getNext(); } else if(t->getPrev()== nullptr && t->getNext() == nullptr) { errorLines.push_back("Error: Cannot find Objects for \""+t->getStringRep()+"\" Operator"); } else if (t->getPrev()!= nullptr && t->getNext() == nullptr) { t_before = t->getPrev(); errorLines.push_back("Error: Cannot find next object for \""+t_before->getStringRep()+ " " +t->getStringRep()+"\" Operator"); } else if (t->getPrev() == nullptr && t->getNext() != nullptr) { t_after = t->getNext(); errorLines.push_back("Error: Cannot find previous object for \""+t->getStringRep()+ " " + t_after->getStringRep()+ "\" Operator"); } //debug code //cout << t_before->getStringRep() << " " <<t->getStringRep() << " " << t_after->getStringRep(); t_before_details = t_before->getTokenDetails(); t_after_details = t_after->getTokenDetails(); if(t_before_details->type != t_after_details->type) // Error here. { errorLines.push_back(errorTypeMismatch+ t_before->getStringRep()+" of type "+t_before_details->type+" with operator \"" +t->getStringRep() +"\" and " + t_after->getStringRep() +" of type " + t_after_details->type); } else if (t_before_details->width != t_after_details->width) { errorLines.push_back(errorWidthMismatch+ t_before->getStringRep()+" of width "+to_string(t_before_details->width)+" with operator \"" +t->getStringRep()+"\" and " + t_after->getStringRep() +" of width " + to_string(t_after_details->width)); } } t = t->getNext(); } cout << "Number of Tokens : " << numberOfTokens <<endl; cout << "Number of Conditional Expressions : " << numberOfCondExp <<endl; cout << "Number of Missing \"end if\"s : " << numberOfMissingEndIfs << endl; cout << "Number of Missing \"if\"s : " << numberOfMissingIfs << endl; cout << "Number of Missing \"Then\"s : " << numberofMissingThens << endl; cout << "Number of Missing \"Process\" : " << numberofMissingProcess << endl; cout << "Number of Missing \"End Process\" : " << numberofMissingEndProcess << endl; cout << "Number of Missing \"Open Bracket \"(\" \" : " << numberofMissingOpenBracket << endl; cout << "Number of Missing \"Close Bracket \")\" \" : " << numberofMissingCloseBracket << endl; if(verboseModeFlag) { cout << "Error Messages: " << endl; for ( vector<string>::iterator it = errorLines.begin() ; it != errorLines.end(); ++it) { cout << *it << endl; } } /* For your testing purposes only */ ///test remove token type ///removeTokensOfType(tokens, T_Operator); /* Ensure that tokens have all type information set*/ /* Create operator,identifier,literal, etc. tokenLists from the master list of tokens */ return 0; }
//Example Test code for interacting with your Token, TokenList, and Tokenizer classes //Add your own code to further test the operation of your Token, TokenList, and Tokenizer classes int main() { ifstream sourceFile; TokenList tokens; Tokenizer tokenizer; //Read in a file line-by-line and tokenize each line sourceFile.open("test.vhd"); if (!sourceFile.is_open()) { cout << "Failed to open file" << endl; return 1; } while(!sourceFile.eof()) { string lineA, lineB; getline(sourceFile, lineA); //while the current line ends with a line-continuation \ append the next line to the current line while(lineA.length() > 0 && lineA[lineA.length()-1] == '\\') { lineA.erase(lineA.length()-1, 1); getline(sourceFile, lineB); lineA += lineB; } tokenizer.setString(&lineA); while( !tokenizer.isComplete() ) { tokens.append(tokenizer.getNextToken()); } //Re-insert newline that was removed by the getline function tokens.append("\n"); } //Test your tokenization of the file by traversing the tokens list and printing out the tokens Token *t = tokens.getFirst(); while(t) { cout << ""<<t->getStringRep(); //t->setTokenDetails("std_logic", 0); tokens.findAndSetTokenDetails(t); cout << "type Of token: " << t->getTokenType() << "\n"; if(t->isIdentifier() || t->isLiteral()) { if(t->getTokenDetails() != nullptr){ cout << "size of width: " << t->getTokenDetails()->width <<"\n"; cout << "string type: " << t->getTokenDetails()->type << "\n"; } } cout<<"\n-------------------------------\n"; //cout<< "one iteration" ; t = t->getNext(); } cout << "\n\nremove comments starts here: \n"; removeComments(tokens); //findAllConditionalExpressions(tokens); //removeTokensOfType(tokens, T_Operator); t=tokens.getFirst(); while(t) { cout << "|"<<t->getStringRep(); //cout<< "one iteration" ; t = t->getNext(); } /* cout<<"\n\n\n"; //removeComments(tokens); while(t) { //cout << t->getStringRep() << " "; //cout<< "one iteration" ; t = t->getNext(); }*/ }
//Example Test code for interacting with your Token, TokenList, and Tokenizer classes //Add your own code to further test the operation of your Token, TokenList, and Tokenizer classes int main() { ifstream sourceFile; TokenList tokens; TokenList lines; cout <<"run" << endl; //Lists for types of tokens TokenList operatorTokens; TokenList identifierTokens; TokenList literalTokens; TokenList commentBodyTokens; TokenList otherTokens; Tokenizer tokenizer; int Num = 0; string Result; //Read in a file line-by-line and tokenize each line sourceFile.open("test.vhd"); if (!sourceFile.is_open()) { cout << "Failed to open file" << endl; return 1; } while (!sourceFile.eof()) { string line; getline(sourceFile, line); Num++; ostringstream convert; convert << Num; Result = convert.str(); tokenizer.setString(&line); while (!tokenizer.isComplete()) { lines.append(Result); tokens.append(tokenizer.getNextToken()); } } /*Test your tokenization of the file by traversing the tokens list and printing out the tokens*/ cout << "[--------------------MASTER LIST----------------------]" << endl; Token *m = tokens.getFirst(); Token *m2 = lines.getFirst(); int Num_tokens = 1; while (m && m2) { cout << Num_tokens << " [" << m2->getStringRep() << "] "; cout << " [" << m->getStringRep() << "] " << endl; m = m->getNext(); m2 = m2->getNext(); Num_tokens++; } cout << endl; /* For your testing purposes only*/ cout << "[--------------------- findAndSetTokenDetails -----------------------]" << endl; string temp1; int wide = 0; Token *w = tokens.getFirst(); while (w) { tokens.findAndSetTokenDetails(w); string type; if (w->getTokenType() == 0) { type = "ID"; identifierTokens.append(w->getStringRep()); } else if (w->getTokenType() == 1) { type = "Op"; operatorTokens.append(w->getStringRep()); } else if (w->getTokenType() == 2) { type = "Lt"; literalTokens.append(w->getStringRep()); } else if (w->getTokenType() == 3) { type = "Com"; commentBodyTokens.append(w->getStringRep()); } else if (w->getTokenType() == 4) { type = "OTH"; otherTokens.append(w->getStringRep()); } if (w->getTokenDetails() == NULL) { temp1 = "NA"; wide = 0; } else { temp1 = w->getTokenDetails()->type; wide = w->getTokenDetails()->width; } cout << "[" << w->getStringRep() << "] " << endl; cout << " Type: " << type << setw(10) << left << " Details: " << temp1 << " " << wide << setw(20) << right << "Key Word: " << w->isKeyword(); cout << endl; w = w->getNext(); } cout << endl << "[-------------------- findAllConditionalExpressions ---------------------]" << endl; TokenList *Conditional = new TokenList(); Conditional = findAllConditionalExpressions(tokens); int conditional_numbers = 1; Token *C = Conditional->getFirst(); while (C) { if (C->getStringRep() == "\n") { conditional_numbers++; } cout << C->getStringRep(); C = C->getNext(); } cout << endl; cout << endl << "[-------------------------- ERROR CHECK ----------------------------]" << endl << endl; cout << "verbose:" << endl << endl; int end_if = error_end_if(tokens, lines); int then = error_then(tokens, lines); int else_ = error_else(tokens, lines); int type = error_type(tokens, lines); int wides = error_width(tokens, lines); int headL = error_head_LIB(tokens, lines); int colon = error_colon(tokens, lines); int semi_col = error_semi_col(tokens, lines); int brac = error_bracket(tokens, lines); cout << endl << "non-verbose:" << endl << endl; cout << "Number of Tokens: " << Num_tokens << endl; cout << "Number of Conditional Expressions: " << conditional_numbers-1 << endl; cout << "Number of Missing \"end if\": " << end_if << endl; cout << "Number of Missing \"then\": " << then << endl; cout << "Number of Missing \"else\": " << else_ << endl; cout << "Number of Type mismatch: " << type << endl; cout << "Number of Width mismatch: " << wides << endl; cout << "Missing head Library: " << headL << endl; cout << "Number of Missing colon: " << colon << endl; cout << "Number of Missing semi-colon: " << semi_col << endl; cout << "Number of Missing bracket: " << brac << endl; return 0; }