int main() { Helper::Title("Testing Tokenizer"); //std::string fileSpec = "../Tokenizer/Tokenizer.cpp"; //std::string fileSpec = "../Tokenizer/Tokenizer.h"; std::string fileSpec = "../Tokenizer/Test.txt";//from here try { std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } { Toker toker; toker.returnComments(); toker.attach(&in); std::cout << "\n current line count = " << toker.currentLineCount(); do { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in.good()); std::cout << "\n current line count = " << toker.currentLineCount(); } putline(); Helper::title("Testing change of special characters"); std::string newSpecialChars = "., :, +, +=, \n { }"; Toker toker; toker.returnComments(); toker.setSpecialTokens(newSpecialChars); in.clear(); in.seekg(std::ios::beg); toker.attach(&in); std::cout << "\n new special tokens: " << newSpecialChars; do { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok;//check if tok is in type table } while (in.good());// if there , get filename and say this file is dependent on this file std::cout << "\n current line count = " << toker.currentLineCount(); } catch (std::logic_error& ex) { std::cout << "\n " << ex.what(); } std::cout << "\n\n";//here return 0; }
int dealSemiExpressions() { Toker tok; std::string fileSpec1 = "../Tokenizer/test_tokenizer.txt"; std::fstream inn(fileSpec1); if (!inn.good()) { std::cout << "\n can't open file " << fileSpec1 << "\n\n"; return 1; } tok.attach(&inn); tok.setComments(false); tok.set("<,>,[,],(,),{,},:,=,+,-,*,\n", "<,<,>,>,:,:,+,+,-,-,=,=,+,=,-,=,*,=,/,=,-,>"); SemiExp semi(&tok); semi.get(); while (semi.get()) { std::cout << "\n -- semiExpression --"; std::string str = semi.show(); std::cout << str; } if (semi.length() > 0) { std::cout << "\n -- semiExpression --"; semi.show(); std::cout << "\n\n"; } return 0; }
int testSemiExpOnCpp(std::string fileSpecSemi) { Toker tokercpp; std::fstream insemicpp(fileSpecSemi); if (!insemicpp.good()) { std::cout << "\n can't open file " << fileSpecSemi << "\n\n"; return 1; } tokercpp.attach(&insemicpp); SemiExp semi(&tokercpp); while (semi.get()) { std::cout << "\n -- semiExpression --"; semi.show(); } if (semi.length() > 0) { std::cout << "\n -- semiExpression --"; semi.show(); std::cout << "\n\n"; } Toker::reset(); return 0; }
void CodeAnalyzer::AnalyzeFileDepedency(std::string file) { Toker toker; std::fstream in(file); // Check if the filename is valid if (!in.good()) { std::cout << "\n can't open file " << file << "\n\n"; return; } toker.attach(&in); do { std::string tok = toker.getTok(); ProcessTok(file, tok); // Check for match in Final Type Table and perform corresponding action } while (in.good()); }
/////////////////////////// // Main Test Stub function int main() { Timer time; time.StartClock(); StringHelper::Title("\n Testing SemiExp class", '='); Toker toker; toker.setVerbose(true); std::string fileSpec = "SemiExp.cpp"; std::fstream in(fileSpec); if (!in.good()) { std::cout << "\n Unable to read file \"" << fileSpec << "\"\n\n"; return 1; } else { std::cout << "\n Processing file \"" << fileSpec << "\"\n"; } toker.attach(&in); SemiExp semi(&toker); semi.setVerbose(true); while (semi.get()) { std::cout << "\n -- semiExpression -- at line number " << semi.currentLineCount(); std::cout << semi.show(); } /* May have collected tokens, but reached end of stream before finding SemiExp terminator. */ if (semi.length() > 0) { std::cout << "\n -- semiExpression --"; semi.show(true); } std::cout << "\n\n\n [Execution Time] : " << time.StopClock() << " ms"; std::cout << "\n\n "; system("pause"); return 0; }
int main1() { showInit(); std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; } Toker toker; toker.attach(&in); std::ofstream myfile; tokenizerSetUp(toker); while (in.good()) { std::string* toks = toker.getToks(); std::cout << "\n--" << toks[1]; if (toks[1] != "NewLine") { std::cout << "\n" << toks[0]; } } std::cout << "\n\n"; /////////// cout << "\n-------------------------Semi Exp------------------------"; Toker toker1; std::string fileSpec1 = "/home/malz/workspace/new_workspace/testtok.cpp"; std::fstream in1(fileSpec1); if (!in1.good()) { std::cout << "\n can't open file " << fileSpec1 << "\n\n"; return 1; } toker1.attach(&in1); SemiExp semi(&toker1); tokenizerSetUp(toker1); while (semi.get1(true)) { std::cout << "\n -- semiExpression --"; cout<<semi.show1(true); } if (semi.length() > 0) { std::cout << "\n -- semiExpression --"; semi.show(); std::cout << "\n\n"; } return 0; }
int dealTokenizer(Toker& toker) { std::cout << "Input for Tokenizer" << std::endl; std::string fileSpec = "../Tokenizer/test_tokenizer.txt"; std::ifstream file("../Tokenizer/test_tokenizer.txt"); std::string str; while (std::getline(file, str)) { std::cout << str << std::endl; } std::cout << "-----------------------------------------------------------------------------------------\n"; std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } toker.attach(&in); toker.setComments(true); toker.set("<,>,[,],(,),{,},:,=,+,-,*,\n", "<,<,>,>,:,:,+,+,-,-,=,=,+,=,-,=,*,=,/,=,-,>"); std::cout << "OUPUT \n"; do { std::string tok = toker.getTok(); if (tok == "\n") { tok = "newline"; } std::cout << "\n -- " << tok; } while (in.good()); std::cout << "\n"; return 0; }
int main() { std::cout << "\n Testing SemiExp class"; std::cout << "\n =======================\n"; Toker toker; //std::string fileSpec = "../Tokenizer/Tokenizer.cpp"; std::string fileSpec = "SemiExp.cpp"; std::fstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open file " << fileSpec << "\n\n"; return 1; } else { std::cout << "\n processing file \"" << fileSpec << "\"\n"; } toker.attach(&in); SemiExp semi(&toker); while(semi.get()) { std::cout << "\n -- semiExpression -- at line number " << semi.currentLineCount(); std::cout << semi.show(); } /* May have collected tokens, but reached end of stream before finding SemiExp terminator. */ if (semi.length() > 0) { std::cout << "\n -- semiExpression --"; semi.show(true); } std::cout << "\n\n"; return 0; }
int main() { //std::string fileSpec = "../Tokenizer/Tokenizer.cpp"; //std::string fileSpec = "../Tokenizer/Tokenizer.h"; std::string fileSpec = "../Tokenizer/Test.txt"; std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } Toker toker; toker.attach(&in); while (in.good()) { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } std::cout << "\n\n"; return 0; }
int main() { /*std::fstream fIn; fIn.open("C:\\Users\thzha_000\Desktop\finaltest.txt", std::ios::in);*/ //Tokenizer testing std::string fileSpec = "../Tokenizer/test_R4.txt"; std::cout << "---------------------------------------" << "\n"; std::cout << "--------- Testing Tokenizer ----------" << "\n"; std::cout << "---------------------------------------" << "\n"; std::cout << "--------Operating authentic code-------" << "\n"; std::cout << "Requirements 5 is well shown in this process" << "\n\n"; std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } else std::cout << "\n Read file successfull!" << "\n"; Toker toker; toker.attach(&in); do { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in.good()); std::cout << "\n\n"; toker.reset(); // SemiExp testing Toker toker2; std::cout << "---------------------------------------" << "\n"; std::cout << "------- Testing SemiExpression --------" << "\n"; std::cout << "---------------------------------------" << "\n"; std::cout << "--------Operating authentic code-------" << "\n\n"; std::cout << "Requirements 6-8 is well shown in this process" << "\n\n"; std::fstream in2(fileSpec); if (!in2.good()) { std::cout << "\n can't open file " << fileSpec << "\n\n"; return 1; } else std::cout << "\n Read file successfull!" << "\n"; toker2.attach(&in2); SemiExp semi(&toker2); while (semi.get()) { std::cout << "\n -------------- semiExpression --------------"; semi.show(); } /* May have collected tokens, but reached end of stream before finding SemiExp terminator. */ if (semi.length() > 0) { std::cout << "\n -------------- semiExpression --------------"; semi.show(); std::cout << "\n\n"; } toker2.reset(); // Test Requirement 4 std::string fileSpec_R4 = "../Tokenizer/test_R4.txt"; std::vector<std::string> alter_setSpecialSingleChar = { "=", "+" }; std::vector<std::string> alter_setSpecialDoubleChar = { "++" , "--" }; std::cout << "------------------- Requiement 4 ------------------" << "\n"; std::cout << "---------------------------------------------------" << "\n"; std::cout << "------Using a small piece of text for testing------" << "\n\n"; Toker toker1; std::ifstream in1(fileSpec_R4); if (!in1.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } else std::cout << "\n Read file successfull!" << "\n"; toker.attach(&in1); do { std::string tok = toker1.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in1.good()); std::cout << "\n\n"; toker1.reset(); // reset static member in consumestate class std::cout << "-----setSpecialSingleChars setSpecialCharPairs-----" << "\n"; std::cout << "---------------------------------------------------" << "\n\n"; Toker toker3; std::cout << " change the default set of single special chars to: { \" = \", \"+ \" } " << "\n\n"; std::cout << " change the default set of single special chars to: { \"++ \" , \"--\" } " << "\n\n"; std::cout << "**After the change, Tokernizer is not able to detect += as a combined punctuator.**" << "\n\n"; std::cout << "This change can be seen at last line 6 and 7." << "\n\n"; toker3.setSingleSpecial("replace", alter_setSpecialSingleChar); toker3.setDoubleSpecial("replace", alter_setSpecialDoubleChar); std::ifstream in3(fileSpec_R4); if (!in3.good()) { std::cout << "\n can't open " << fileSpec_R4 << "\n\n"; return 1; } else std::cout << "\n Read file successfull!" << "\n"; toker.attach(&in3); do { std::string tok = toker3.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in3.good()); std::cout << "\n\n"; toker3.reset(); return 0; }
void main() { try { std::stack<PNode<XMLValue>*> *nStack = new std::stack<PNode<XMLValue>*>; PNode<XMLValue>* rootnode = new PNode<XMLValue>(XMLValue(XMLValue::TAG,"Document")); XMLTree* xtree = new XMLTree(rootnode); ParserState* ps = new ParserState; Toker* pToker = new Toker; pToker->setMode(Toker::xml); //XML parser mode XmlParts* pSemi = new XmlParts(pToker); Parser* pParser = new Parser(pSemi); ElementEnd* pElementendrule = new ElementEnd(nStack,ps); ElementStart* pElementstartrule = new ElementStart(nStack,ps); ElementSelfClosed* pElementselfcrule = new ElementSelfClosed(nStack,ps); Text* ptextrule = new Text(nStack,ps); Comment* pCommentrule = new Comment(nStack,ps); ProcInstr* pIinstrrule = new ProcInstr(nStack,ps); EndFound* pEndact = new EndFound(nStack,xtree); StartFound* pStartact = new StartFound(nStack,xtree); SelfClosedFound* pSelfClosedact = new SelfClosedFound(nStack,xtree); TextFound* pTextact = new TextFound(nStack,xtree); CommentFound* pCommentact = new CommentFound(nStack,xtree); ProcInstrFound* pInstract = new ProcInstrFound(nStack,xtree); pElementendrule->addAction(pEndact); pElementstartrule->addAction(pStartact); pElementselfcrule->addAction(pSelfClosedact); ptextrule->addAction(pTextact); pCommentrule->addAction(pCommentact); pIinstrrule->addAction(pInstract); pParser->addRule(pElementendrule); pParser->addRule(pElementstartrule); pParser->addRule(pElementselfcrule); pParser->addRule(ptextrule); pParser->addRule(pCommentrule); pParser->addRule(pIinstrrule); nStack->push(rootnode); if(pParser) { if(! pToker->attach("..\\LectureNote.xml")) { std::cout << "\n could not open file " << std::endl; return; } } while(pParser->next()) pParser->parse(); //display the tree and the rebuilt xml XMLDisplay disply(xtree); //generate the string of tree and rebuilt xml disply.buildTreeDisplay(); disply.rebuildXML(); //attach the output file to the display object disply.AttachFile("..\\output.txt"); //display them std::cout<<"=======================Parser Tree====================="<<std::endl; std::cout<<disply.OutputTree(); std::cout << "\n\n"; std::cout<<"=====================Rebuild XML====================="<<std::endl; std::cout<<disply.OutputXML(); delete pEndact; delete pStartact; delete pSelfClosedact; delete pTextact; delete pCommentact; delete pInstract; delete pElementendrule; delete pElementstartrule; delete pElementselfcrule; delete ptextrule; delete pCommentrule; delete pIinstrrule; delete pToker; delete pSemi; delete pParser; delete ps; delete nStack; xtree->remove(xtree->getroot()); std::vector< PNode<XMLValue>* > deletednodes = xtree->getdeletednodes(); for(size_t i = 0; i < deletednodes.size(); ++i) delete deletednodes[i]; delete xtree; } catch(std::exception& ex) { std::cout << "\n\n " << ex.what() << "\n\n"; } ::system("pause"); }
int main(int argc, char* argv[]) { std::cout << "\n Testing Tokenizer class\n " << std::string(25, '=') << std::endl; std::cout << "\n Note that comments and quotes are returned as single tokens\n\n"; // collecting tokens from a string Toker t_fromStr("tokens from a string: -> int x; /* a comment */", false); std::string tok; do { tok = t_fromStr.getTok(); std::cout << " " << tok; } while (tok != ""); std::cout << "\n\n"; // collecting tokens from files, named on the command line if (argc < 2) { std::cout << "\n please enter name of file to process on command line\n\n"; return 1; } for (int i = 1; i<argc; ++i) { std::cout << "\n Processing file " << argv[i]; std::cout << "\n " << std::string(16 + strlen(argv[i]), '-') << "\n"; try { Toker t; t.setMode(Toker::xml); // comment out to show tokenizing for code // t.setSingleCharTokens("<>"); // will return same results as above for XML if (!t.attach(argv[i])) { std::cout << "\n can't open file " << argv[i] << "\n\n"; continue; } t.returnComments(); // remove this statement to discard comment tokens std::string temp; do { temp = t.getTok(); std::cout << " ln: " << t.lines() << ", br lev: " << t.braceLevel() << ", tok size: " << std::setw(3) << temp.length() << " -- "; if (temp != "\n") std::cout << temp << std::endl; else std::cout << "newline\n"; } while (temp != ""); } catch (std::exception& ex) { std::cout << " " << ex.what() << "\n\n"; } } }
int main() { { Toker toker; //std::string fileSpec = "../Tokenizer/Tokenizer.cpp"; //std::string fileSpec = "../../Tokenizer/test.txt"; //std::string fileSpec = "../../SemiExp/SemiExp.h"; std::string fileSpec = "../../Tokenizer/test_cases.txt"; std::fstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open file " << fileSpec << "\n\n"; return 1; } std::cout << "\n File: " << fileSpec << "\n\n"; toker.attach(&in); SemiExp semi(&toker); while (semi.get()) { std::cout << "\n -- semiExpression --"; semi.show(true); } if (semi.length() > 0) { std::cout << "\n -- semiExpression --"; semi.show(); std::cout << "\n\n"; } } std::cout << "\n -----Testing ITokcollection functions -----------\n"; try { Test::test0(); Test::test1(); Test::test2(); Test::test3(); Test::test4(); Test::test5(); Test::test6(); Test::test7(); Test::test8(); } catch (std::exception& ex) { std::cout << "\n\n " << ex.what() << "\n\n"; return 1; } return 0; }