int main() { Helper::Title("Testing Tokenizer"); //std::string fileSpec = "../Tokenizer/Tokenizer.cpp"; //std::string fileSpec = "../Tokenizer/Tokenizer.h"; std::string fileSpec = "../Tokenizer/Test.txt";//from here try { std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } { Toker toker; toker.returnComments(); toker.attach(&in); std::cout << "\n current line count = " << toker.currentLineCount(); do { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in.good()); std::cout << "\n current line count = " << toker.currentLineCount(); } putline(); Helper::title("Testing change of special characters"); std::string newSpecialChars = "., :, +, +=, \n { }"; Toker toker; toker.returnComments(); toker.setSpecialTokens(newSpecialChars); in.clear(); in.seekg(std::ios::beg); toker.attach(&in); std::cout << "\n new special tokens: " << newSpecialChars; do { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok;//check if tok is in type table } while (in.good());// if there , get filename and say this file is dependent on this file std::cout << "\n current line count = " << toker.currentLineCount(); } catch (std::logic_error& ex) { std::cout << "\n " << ex.what(); } std::cout << "\n\n";//here return 0; }
void CodeAnalyzer::AnalyzeFileDepedency(std::string file) { Toker toker; std::fstream in(file); // Check if the filename is valid if (!in.good()) { std::cout << "\n can't open file " << file << "\n\n"; return; } toker.attach(&in); do { std::string tok = toker.getTok(); ProcessTok(file, tok); // Check for match in Final Type Table and perform corresponding action } while (in.good()); }
int dealTokenizer(Toker& toker) { std::cout << "Input for Tokenizer" << std::endl; std::string fileSpec = "../Tokenizer/test_tokenizer.txt"; std::ifstream file("../Tokenizer/test_tokenizer.txt"); std::string str; while (std::getline(file, str)) { std::cout << str << std::endl; } std::cout << "-----------------------------------------------------------------------------------------\n"; std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } toker.attach(&in); toker.setComments(true); toker.set("<,>,[,],(,),{,},:,=,+,-,*,\n", "<,<,>,>,:,:,+,+,-,-,=,=,+,=,-,=,*,=,/,=,-,>"); std::cout << "OUPUT \n"; do { std::string tok = toker.getTok(); if (tok == "\n") { tok = "newline"; } std::cout << "\n -- " << tok; } while (in.good()); std::cout << "\n"; return 0; }
int main() { //std::string fileSpec = "../Tokenizer/Tokenizer.cpp"; //std::string fileSpec = "../Tokenizer/Tokenizer.h"; std::string fileSpec = "../Tokenizer/Test.txt"; std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } Toker toker; toker.attach(&in); while (in.good()) { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } std::cout << "\n\n"; return 0; }
int main() { /*std::fstream fIn; fIn.open("C:\\Users\thzha_000\Desktop\finaltest.txt", std::ios::in);*/ //Tokenizer testing std::string fileSpec = "../Tokenizer/test_R4.txt"; std::cout << "---------------------------------------" << "\n"; std::cout << "--------- Testing Tokenizer ----------" << "\n"; std::cout << "---------------------------------------" << "\n"; std::cout << "--------Operating authentic code-------" << "\n"; std::cout << "Requirements 5 is well shown in this process" << "\n\n"; std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } else std::cout << "\n Read file successfull!" << "\n"; Toker toker; toker.attach(&in); do { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in.good()); std::cout << "\n\n"; toker.reset(); // SemiExp testing Toker toker2; std::cout << "---------------------------------------" << "\n"; std::cout << "------- Testing SemiExpression --------" << "\n"; std::cout << "---------------------------------------" << "\n"; std::cout << "--------Operating authentic code-------" << "\n\n"; std::cout << "Requirements 6-8 is well shown in this process" << "\n\n"; std::fstream in2(fileSpec); if (!in2.good()) { std::cout << "\n can't open file " << fileSpec << "\n\n"; return 1; } else std::cout << "\n Read file successfull!" << "\n"; toker2.attach(&in2); SemiExp semi(&toker2); while (semi.get()) { std::cout << "\n -------------- semiExpression --------------"; semi.show(); } /* May have collected tokens, but reached end of stream before finding SemiExp terminator. */ if (semi.length() > 0) { std::cout << "\n -------------- semiExpression --------------"; semi.show(); std::cout << "\n\n"; } toker2.reset(); // Test Requirement 4 std::string fileSpec_R4 = "../Tokenizer/test_R4.txt"; std::vector<std::string> alter_setSpecialSingleChar = { "=", "+" }; std::vector<std::string> alter_setSpecialDoubleChar = { "++" , "--" }; std::cout << "------------------- Requiement 4 ------------------" << "\n"; std::cout << "---------------------------------------------------" << "\n"; std::cout << "------Using a small piece of text for testing------" << "\n\n"; Toker toker1; std::ifstream in1(fileSpec_R4); if (!in1.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } else std::cout << "\n Read file successfull!" << "\n"; toker.attach(&in1); do { std::string tok = toker1.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in1.good()); std::cout << "\n\n"; toker1.reset(); // reset static member in consumestate class std::cout << "-----setSpecialSingleChars setSpecialCharPairs-----" << "\n"; std::cout << "---------------------------------------------------" << "\n\n"; Toker toker3; std::cout << " change the default set of single special chars to: { \" = \", \"+ \" } " << "\n\n"; std::cout << " change the default set of single special chars to: { \"++ \" , \"--\" } " << "\n\n"; std::cout << "**After the change, Tokernizer is not able to detect += as a combined punctuator.**" << "\n\n"; std::cout << "This change can be seen at last line 6 and 7." << "\n\n"; toker3.setSingleSpecial("replace", alter_setSpecialSingleChar); toker3.setDoubleSpecial("replace", alter_setSpecialDoubleChar); std::ifstream in3(fileSpec_R4); if (!in3.good()) { std::cout << "\n can't open " << fileSpec_R4 << "\n\n"; return 1; } else std::cout << "\n Read file successfull!" << "\n"; toker.attach(&in3); do { std::string tok = toker3.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in3.good()); std::cout << "\n\n"; toker3.reset(); return 0; }
int main(int argc, char* argv[]) { std::cout << "\n Testing Tokenizer class\n " << std::string(25, '=') << std::endl; std::cout << "\n Note that comments and quotes are returned as single tokens\n\n"; // collecting tokens from a string Toker t_fromStr("tokens from a string: -> int x; /* a comment */", false); std::string tok; do { tok = t_fromStr.getTok(); std::cout << " " << tok; } while (tok != ""); std::cout << "\n\n"; // collecting tokens from files, named on the command line if (argc < 2) { std::cout << "\n please enter name of file to process on command line\n\n"; return 1; } for (int i = 1; i<argc; ++i) { std::cout << "\n Processing file " << argv[i]; std::cout << "\n " << std::string(16 + strlen(argv[i]), '-') << "\n"; try { Toker t; t.setMode(Toker::xml); // comment out to show tokenizing for code // t.setSingleCharTokens("<>"); // will return same results as above for XML if (!t.attach(argv[i])) { std::cout << "\n can't open file " << argv[i] << "\n\n"; continue; } t.returnComments(); // remove this statement to discard comment tokens std::string temp; do { temp = t.getTok(); std::cout << " ln: " << t.lines() << ", br lev: " << t.braceLevel() << ", tok size: " << std::setw(3) << temp.length() << " -- "; if (temp != "\n") std::cout << temp << std::endl; else std::cout << "newline\n"; } while (temp != ""); } catch (std::exception& ex) { std::cout << " " << ex.what() << "\n\n"; } } }