int main() { Helper::Title("Testing Tokenizer"); //std::string fileSpec = "../Tokenizer/Tokenizer.cpp"; //std::string fileSpec = "../Tokenizer/Tokenizer.h"; std::string fileSpec = "../Tokenizer/Test.txt";//from here try { std::ifstream in(fileSpec); if (!in.good()) { std::cout << "\n can't open " << fileSpec << "\n\n"; return 1; } { Toker toker; toker.returnComments(); toker.attach(&in); std::cout << "\n current line count = " << toker.currentLineCount(); do { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok; } while (in.good()); std::cout << "\n current line count = " << toker.currentLineCount(); } putline(); Helper::title("Testing change of special characters"); std::string newSpecialChars = "., :, +, +=, \n { }"; Toker toker; toker.returnComments(); toker.setSpecialTokens(newSpecialChars); in.clear(); in.seekg(std::ios::beg); toker.attach(&in); std::cout << "\n new special tokens: " << newSpecialChars; do { std::string tok = toker.getTok(); if (tok == "\n") tok = "newline"; std::cout << "\n -- " << tok;//check if tok is in type table } while (in.good());// if there , get filename and say this file is dependent on this file std::cout << "\n current line count = " << toker.currentLineCount(); } catch (std::logic_error& ex) { std::cout << "\n " << ex.what(); } std::cout << "\n\n";//here return 0; }
int main(int argc, char* argv[]) { std::cout << "\n Testing Tokenizer class\n " << std::string(25, '=') << std::endl; std::cout << "\n Note that comments and quotes are returned as single tokens\n\n"; // collecting tokens from a string Toker t_fromStr("tokens from a string: -> int x; /* a comment */", false); std::string tok; do { tok = t_fromStr.getTok(); std::cout << " " << tok; } while (tok != ""); std::cout << "\n\n"; // collecting tokens from files, named on the command line if (argc < 2) { std::cout << "\n please enter name of file to process on command line\n\n"; return 1; } for (int i = 1; i<argc; ++i) { std::cout << "\n Processing file " << argv[i]; std::cout << "\n " << std::string(16 + strlen(argv[i]), '-') << "\n"; try { Toker t; t.setMode(Toker::xml); // comment out to show tokenizing for code // t.setSingleCharTokens("<>"); // will return same results as above for XML if (!t.attach(argv[i])) { std::cout << "\n can't open file " << argv[i] << "\n\n"; continue; } t.returnComments(); // remove this statement to discard comment tokens std::string temp; do { temp = t.getTok(); std::cout << " ln: " << t.lines() << ", br lev: " << t.braceLevel() << ", tok size: " << std::setw(3) << temp.length() << " -- "; if (temp != "\n") std::cout << temp << std::endl; else std::cout << "newline\n"; } while (temp != ""); } catch (std::exception& ex) { std::cout << " " << ex.what() << "\n\n"; } } }