int main()
{
Helper::Title("Testing Tokenizer");
//std::string fileSpec = "../Tokenizer/Tokenizer.cpp";
//std::string fileSpec = "../Tokenizer/Tokenizer.h";
std::string fileSpec = "../Tokenizer/Test.txt";//from here
try
{
std::ifstream in(fileSpec);
if (!in.good())
{
std::cout << "\n can't open " << fileSpec << "\n\n";
return 1;
}
{
Toker toker;
toker.returnComments();
toker.attach(&in);
std::cout << "\n current line count = " << toker.currentLineCount();
do
{
std::string tok = toker.getTok();
if (tok == "\n")
tok = "newline";
std::cout << "\n -- " << tok;
} while (in.good());
std::cout << "\n current line count = " << toker.currentLineCount();
}
putline();
Helper::title("Testing change of special characters");
std::string newSpecialChars = "., :, +, +=, \n { }";
Toker toker;
toker.returnComments();
toker.setSpecialTokens(newSpecialChars);
in.clear();
in.seekg(std::ios::beg);
toker.attach(&in);
std::cout << "\n new special tokens: " << newSpecialChars;
do
{
std::string tok = toker.getTok();
if (tok == "\n")
tok = "newline";
std::cout << "\n -- " << tok;//check if tok is in type table
} while (in.good());// if there , get filename and say this file is dependent on this file
std::cout << "\n current line count = " << toker.currentLineCount();
}
catch (std::logic_error& ex)
{
std::cout << "\n " << ex.what();
}
std::cout << "\n\n";//here
return 0;
}
int main(int argc, char* argv[])
{
std::cout << "\n Testing Tokenizer class\n "
<< std::string(25, '=') << std::endl;
std::cout
<< "\n Note that comments and quotes are returned as single tokens\n\n";
// collecting tokens from a string
Toker t_fromStr("tokens from a string: -> int x; /* a comment */", false);
std::string tok;
do
{
tok = t_fromStr.getTok();
std::cout << " " << tok;
} while (tok != "");
std::cout << "\n\n";
// collecting tokens from files, named on the command line
if (argc < 2)
{
std::cout
<< "\n please enter name of file to process on command line\n\n";
return 1;
}
for (int i = 1; i<argc; ++i)
{
std::cout << "\n Processing file " << argv[i];
std::cout << "\n " << std::string(16 + strlen(argv[i]), '-') << "\n";
try
{
Toker t;
t.setMode(Toker::xml); // comment out to show tokenizing for code
// t.setSingleCharTokens("<>"); // will return same results as above for XML
if (!t.attach(argv[i]))
{
std::cout << "\n can't open file " << argv[i] << "\n\n";
continue;
}
t.returnComments(); // remove this statement to discard comment tokens
std::string temp;
do
{
temp = t.getTok();
std::cout << " ln: " << t.lines() << ", br lev: "
<< t.braceLevel() << ", tok size: " << std::setw(3) << temp.length() << " -- ";
if (temp != "\n")
std::cout << temp << std::endl;
else
std::cout << "newline\n";
} while (temp != "");
}
catch (std::exception& ex)
{
std::cout << " " << ex.what() << "\n\n";
}
}
}
