int main()
{
std::cout << "-----------------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 1 -" << " " << " I have used Microsoft Visual Studio C++ Console Projects" << std::endl;
std::cout << "-------------------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 2 -" << " " << " I have used <iostream> for all I/O and new and delete for heap based management " << std::endl;
std::cout << "----------------------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 3 -" << " " << " I have provided Tokenzier Package, SemiExp package whose .ccp implements an interface named iTokCollection.h" << std::endl;
std::cout << "------------------------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 4 -" << " " << "Tokenizing Test File into Alphanumeric, Punctuator, Special One Character, Special Two Character, C Style Comments, C++ Comments and Quoted Strings" << std::endl;
std::cout << "-------------------------------------------------------------------------------------------------------------\n";
Toker toker;
int f = dealTokenizer(toker);
toker.des();
std::cout << "------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 5 - " << " " << "The getTok() method prodduces one token for each call to it" << std::endl;
std::cout << "-------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 6 -" << " " << "SemiExp package produces set of tokens by repeatdely calling getTok() until we run into a termination condition " << std::endl;
std::cout << "--------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 7 - " << " " << "SemiExpressions are terminated on },{,; and : public, :private and :protected " << std::endl;
std::cout << "---------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 8 - " << " " << "Folding for" << std::endl;
std::cout << "----------------------------------------------------------------------------------------------\n";
std::cout << "Output for 5,6,7,8 with same input file passed to Tokennizer with the Comment toggle turned off" << std::endl;
int g = dealSemiExpressions();
std::cout << "\n------------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 9 -" << " " << " All the virtual methods in the iTokCollection interface have been implemented by the SemiExp Class " << std::endl;
std::cout << "------------------------------------------------------------------------------------------------\n";
std::cout << "Requirement 10 -" << " " << " TestExecutive which illustrates all possible test cases" << std::endl;
std::cout << "---------------------------------------------------------------------------------------------------\n";
std::cout << "Please enter any character to continue " << std::endl;
std::getchar();
}
int dealSemiExpressions()
{
Toker tok;
std::string fileSpec1 = "../Tokenizer/test_tokenizer.txt";
std::fstream inn(fileSpec1);
if (!inn.good())
{
std::cout << "\n can't open file " << fileSpec1 << "\n\n";
return 1;
}
tok.attach(&inn);
tok.setComments(false);
tok.set("<,>,[,],(,),{,},:,=,+,-,*,\n", "<,<,>,>,:,:,+,+,-,-,=,=,+,=,-,=,*,=,/,=,-,>");
SemiExp semi(&tok);
semi.get();
while (semi.get())
{
std::cout << "\n -- semiExpression --";
std::string str = semi.show();
std::cout << str;
}
if (semi.length() > 0)
{
std::cout << "\n -- semiExpression --";
semi.show();
std::cout << "\n\n";
}
return 0;
}
int testSemiExpOnCpp(std::string fileSpecSemi) {
Toker tokercpp;
std::fstream insemicpp(fileSpecSemi);
if (!insemicpp.good())
{
std::cout << "\n can't open file " << fileSpecSemi << "\n\n";
return 1;
}
tokercpp.attach(&insemicpp);
SemiExp semi(&tokercpp);
while (semi.get())
{
std::cout << "\n -- semiExpression --";
semi.show();
}
if (semi.length() > 0)
{
std::cout << "\n -- semiExpression --";
semi.show();
std::cout << "\n\n";
}
Toker::reset();
return 0;
}
void CodeAnalyzer::AnalyzeFileDepedency(std::string file) {
Toker toker;
std::fstream in(file);
// Check if the filename is valid
if (!in.good())
{
std::cout << "\n can't open file " << file << "\n\n";
return;
}
toker.attach(&in);
do
{
std::string tok = toker.getTok();
ProcessTok(file, tok); // Check for match in Final Type Table and perform corresponding action
} while (in.good());
}
///////////////////////////
// Main Test Stub function
int main()
{
Timer time;
time.StartClock();
StringHelper::Title("\n Testing SemiExp class", '=');
Toker toker;
toker.setVerbose(true);
std::string fileSpec = "SemiExp.cpp";
std::fstream in(fileSpec);
if (!in.good())
{
std::cout << "\n Unable to read file \"" << fileSpec << "\"\n\n";
return 1;
}
else
{
std::cout << "\n Processing file \"" << fileSpec << "\"\n";
}
toker.attach(&in);
SemiExp semi(&toker);
semi.setVerbose(true);
while (semi.get())
{
std::cout << "\n -- semiExpression -- at line number " << semi.currentLineCount();
std::cout << semi.show();
}
/*
May have collected tokens, but reached end of stream
before finding SemiExp terminator.
*/
if (semi.length() > 0)
{
std::cout << "\n -- semiExpression --";
semi.show(true);
}
std::cout << "\n\n\n [Execution Time] : " << time.StopClock() << " ms";
std::cout << "\n\n ";
system("pause");
return 0;
}
int main1() {
showInit();
std::ifstream in(fileSpec);
if (!in.good()) {
std::cout << "\n can't open " << fileSpec << "\n\n";
}
Toker toker;
toker.attach(&in);
std::ofstream myfile;
tokenizerSetUp(toker);
while (in.good()) {
std::string* toks = toker.getToks();
std::cout << "\n--" << toks[1];
if (toks[1] != "NewLine") {
std::cout << "\n" << toks[0];
}
}
std::cout << "\n\n";
///////////
cout << "\n-------------------------Semi Exp------------------------";
Toker toker1;
std::string fileSpec1 = "/home/malz/workspace/new_workspace/testtok.cpp";
std::fstream in1(fileSpec1);
if (!in1.good()) {
std::cout << "\n can't open file " << fileSpec1 << "\n\n";
return 1;
}
toker1.attach(&in1);
SemiExp semi(&toker1);
tokenizerSetUp(toker1);
while (semi.get1(true)) {
std::cout << "\n -- semiExpression --";
cout<<semi.show1(true);
}
if (semi.length() > 0) {
std::cout << "\n -- semiExpression --";
semi.show();
std::cout << "\n\n";
}
return 0;
}
int dealTokenizer(Toker& toker)
{
std::cout << "Input for Tokenizer" << std::endl;
std::string fileSpec = "../Tokenizer/test_tokenizer.txt";
std::ifstream file("../Tokenizer/test_tokenizer.txt");
std::string str;
while (std::getline(file, str))
{
std::cout << str << std::endl;
}
std::cout << "-----------------------------------------------------------------------------------------\n";
std::ifstream in(fileSpec);
if (!in.good())
{
std::cout << "\n can't open " << fileSpec << "\n\n";
return 1;
}
toker.attach(&in);
toker.setComments(true);
toker.set("<,>,[,],(,),{,},:,=,+,-,*,\n", "<,<,>,>,:,:,+,+,-,-,=,=,+,=,-,=,*,=,/,=,-,>");
std::cout << "OUPUT \n";
do
{
std::string tok = toker.getTok();
if (tok == "\n")
{
tok = "newline";
}
std::cout << "\n -- " << tok;
} while (in.good());
std::cout << "\n";
return 0;
}
int main()
{
std::cout << "\n Testing SemiExp class";
std::cout << "\n =======================\n";
Toker toker;
//std::string fileSpec = "../Tokenizer/Tokenizer.cpp";
std::string fileSpec = "SemiExp.cpp";
std::fstream in(fileSpec);
if (!in.good())
{
std::cout << "\n can't open file " << fileSpec << "\n\n";
return 1;
}
else
{
std::cout << "\n processing file \"" << fileSpec << "\"\n";
}
toker.attach(&in);
SemiExp semi(&toker);
while(semi.get())
{
std::cout << "\n -- semiExpression -- at line number " << semi.currentLineCount();
std::cout << semi.show();
}
/*
May have collected tokens, but reached end of stream
before finding SemiExp terminator.
*/
if (semi.length() > 0)
{
std::cout << "\n -- semiExpression --";
semi.show(true);
}
std::cout << "\n\n";
return 0;
}
int main()
{
//std::string fileSpec = "../Tokenizer/Tokenizer.cpp";
//std::string fileSpec = "../Tokenizer/Tokenizer.h";
std::string fileSpec = "../Tokenizer/Test.txt";
std::ifstream in(fileSpec);
if (!in.good())
{
std::cout << "\n can't open " << fileSpec << "\n\n";
return 1;
}
Toker toker;
toker.attach(&in);
while (in.good())
{
std::string tok = toker.getTok();
if (tok == "\n")
tok = "newline";
std::cout << "\n -- " << tok;
}
std::cout << "\n\n";
return 0;
}
int main()
{
/*std::fstream fIn;
fIn.open("C:\\Users\thzha_000\Desktop\finaltest.txt", std::ios::in);*/
//Tokenizer testing
std::string fileSpec = "../Tokenizer/test_R4.txt";
std::cout << "---------------------------------------" << "\n";
std::cout << "--------- Testing Tokenizer ----------" << "\n";
std::cout << "---------------------------------------" << "\n";
std::cout << "--------Operating authentic code-------" << "\n";
std::cout << "Requirements 5 is well shown in this process" << "\n\n";
std::ifstream in(fileSpec);
if (!in.good())
{
std::cout << "\n can't open " << fileSpec << "\n\n";
return 1;
}
else
std::cout << "\n Read file successfull!" << "\n";
Toker toker;
toker.attach(&in);
do
{
std::string tok = toker.getTok();
if (tok == "\n")
tok = "newline";
std::cout << "\n -- " << tok;
} while (in.good());
std::cout << "\n\n";
toker.reset();
// SemiExp testing
Toker toker2;
std::cout << "---------------------------------------" << "\n";
std::cout << "------- Testing SemiExpression --------" << "\n";
std::cout << "---------------------------------------" << "\n";
std::cout << "--------Operating authentic code-------" << "\n\n";
std::cout << "Requirements 6-8 is well shown in this process" << "\n\n";
std::fstream in2(fileSpec);
if (!in2.good())
{
std::cout << "\n can't open file " << fileSpec << "\n\n";
return 1;
}
else
std::cout << "\n Read file successfull!" << "\n";
toker2.attach(&in2);
SemiExp semi(&toker2);
while (semi.get())
{
std::cout << "\n -------------- semiExpression --------------";
semi.show();
}
/*
May have collected tokens, but reached end of stream
before finding SemiExp terminator.
*/
if (semi.length() > 0)
{
std::cout << "\n -------------- semiExpression --------------";
semi.show();
std::cout << "\n\n";
}
toker2.reset();
// Test Requirement 4
std::string fileSpec_R4 = "../Tokenizer/test_R4.txt";
std::vector<std::string> alter_setSpecialSingleChar = { "=", "+" };
std::vector<std::string> alter_setSpecialDoubleChar = { "++" , "--" };
std::cout << "------------------- Requiement 4 ------------------" << "\n";
std::cout << "---------------------------------------------------" << "\n";
std::cout << "------Using a small piece of text for testing------" << "\n\n";
Toker toker1;
std::ifstream in1(fileSpec_R4);
if (!in1.good())
{
std::cout << "\n can't open " << fileSpec << "\n\n";
return 1;
}
else
std::cout << "\n Read file successfull!" << "\n";
toker.attach(&in1);
do
{
std::string tok = toker1.getTok();
if (tok == "\n")
tok = "newline";
std::cout << "\n -- " << tok;
} while (in1.good());
std::cout << "\n\n";
toker1.reset(); // reset static member in consumestate class
std::cout << "-----setSpecialSingleChars setSpecialCharPairs-----" << "\n";
std::cout << "---------------------------------------------------" << "\n\n";
Toker toker3;
std::cout << " change the default set of single special chars to: { \" = \", \"+ \" } " << "\n\n";
std::cout << " change the default set of single special chars to: { \"++ \" , \"--\" } " << "\n\n";
std::cout << "**After the change, Tokernizer is not able to detect += as a combined punctuator.**" << "\n\n";
std::cout << "This change can be seen at last line 6 and 7." << "\n\n";
toker3.setSingleSpecial("replace", alter_setSpecialSingleChar);
toker3.setDoubleSpecial("replace", alter_setSpecialDoubleChar);
std::ifstream in3(fileSpec_R4);
if (!in3.good())
{
std::cout << "\n can't open " << fileSpec_R4 << "\n\n";
return 1;
}
else
std::cout << "\n Read file successfull!" << "\n";
toker.attach(&in3);
do
{
std::string tok = toker3.getTok();
if (tok == "\n")
tok = "newline";
std::cout << "\n -- " << tok;
} while (in3.good());
std::cout << "\n\n";
toker3.reset();
return 0;
}
void main()
{
try
{
std::stack<PNode<XMLValue>*> *nStack = new std::stack<PNode<XMLValue>*>;
PNode<XMLValue>* rootnode = new PNode<XMLValue>(XMLValue(XMLValue::TAG,"Document"));
XMLTree* xtree = new XMLTree(rootnode);
ParserState* ps = new ParserState;
Toker* pToker = new Toker;
pToker->setMode(Toker::xml); //XML parser mode
XmlParts* pSemi = new XmlParts(pToker);
Parser* pParser = new Parser(pSemi);
ElementEnd* pElementendrule = new ElementEnd(nStack,ps);
ElementStart* pElementstartrule = new ElementStart(nStack,ps);
ElementSelfClosed* pElementselfcrule = new ElementSelfClosed(nStack,ps);
Text* ptextrule = new Text(nStack,ps);
Comment* pCommentrule = new Comment(nStack,ps);
ProcInstr* pIinstrrule = new ProcInstr(nStack,ps);
EndFound* pEndact = new EndFound(nStack,xtree);
StartFound* pStartact = new StartFound(nStack,xtree);
SelfClosedFound* pSelfClosedact = new SelfClosedFound(nStack,xtree);
TextFound* pTextact = new TextFound(nStack,xtree);
CommentFound* pCommentact = new CommentFound(nStack,xtree);
ProcInstrFound* pInstract = new ProcInstrFound(nStack,xtree);
pElementendrule->addAction(pEndact);
pElementstartrule->addAction(pStartact);
pElementselfcrule->addAction(pSelfClosedact);
ptextrule->addAction(pTextact);
pCommentrule->addAction(pCommentact);
pIinstrrule->addAction(pInstract);
pParser->addRule(pElementendrule);
pParser->addRule(pElementstartrule);
pParser->addRule(pElementselfcrule);
pParser->addRule(ptextrule);
pParser->addRule(pCommentrule);
pParser->addRule(pIinstrrule);
nStack->push(rootnode);
if(pParser)
{
if(! pToker->attach("..\\LectureNote.xml"))
{
std::cout << "\n could not open file " << std::endl;
return;
}
}
while(pParser->next())
pParser->parse();
//display the tree and the rebuilt xml
XMLDisplay disply(xtree);
//generate the string of tree and rebuilt xml
disply.buildTreeDisplay();
disply.rebuildXML();
//attach the output file to the display object
disply.AttachFile("..\\output.txt");
//display them
std::cout<<"=======================Parser Tree====================="<<std::endl;
std::cout<<disply.OutputTree();
std::cout << "\n\n";
std::cout<<"=====================Rebuild XML====================="<<std::endl;
std::cout<<disply.OutputXML();
delete pEndact;
delete pStartact;
delete pSelfClosedact;
delete pTextact;
delete pCommentact;
delete pInstract;
delete pElementendrule;
delete pElementstartrule;
delete pElementselfcrule;
delete ptextrule;
delete pCommentrule;
delete pIinstrrule;
delete pToker;
delete pSemi;
delete pParser;
delete ps;
delete nStack;
xtree->remove(xtree->getroot());
std::vector< PNode<XMLValue>* > deletednodes = xtree->getdeletednodes();
for(size_t i = 0; i < deletednodes.size(); ++i)
delete deletednodes[i];
delete xtree;
}
catch(std::exception& ex)
{
std::cout << "\n\n " << ex.what() << "\n\n";
}
::system("pause");
}
int main(int argc, char* argv[])
{
std::cout << "\n Testing Tokenizer class\n "
<< std::string(25, '=') << std::endl;
std::cout
<< "\n Note that comments and quotes are returned as single tokens\n\n";
// collecting tokens from a string
Toker t_fromStr("tokens from a string: -> int x; /* a comment */", false);
std::string tok;
do
{
tok = t_fromStr.getTok();
std::cout << " " << tok;
} while (tok != "");
std::cout << "\n\n";
// collecting tokens from files, named on the command line
if (argc < 2)
{
std::cout
<< "\n please enter name of file to process on command line\n\n";
return 1;
}
for (int i = 1; i<argc; ++i)
{
std::cout << "\n Processing file " << argv[i];
std::cout << "\n " << std::string(16 + strlen(argv[i]), '-') << "\n";
try
{
Toker t;
t.setMode(Toker::xml); // comment out to show tokenizing for code
// t.setSingleCharTokens("<>"); // will return same results as above for XML
if (!t.attach(argv[i]))
{
std::cout << "\n can't open file " << argv[i] << "\n\n";
continue;
}
t.returnComments(); // remove this statement to discard comment tokens
std::string temp;
do
{
temp = t.getTok();
std::cout << " ln: " << t.lines() << ", br lev: "
<< t.braceLevel() << ", tok size: " << std::setw(3) << temp.length() << " -- ";
if (temp != "\n")
std::cout << temp << std::endl;
else
std::cout << "newline\n";
} while (temp != "");
}
catch (std::exception& ex)
{
std::cout << " " << ex.what() << "\n\n";
}
}
}
int main()
{
{
Toker toker;
//std::string fileSpec = "../Tokenizer/Tokenizer.cpp";
//std::string fileSpec = "../../Tokenizer/test.txt";
//std::string fileSpec = "../../SemiExp/SemiExp.h";
std::string fileSpec = "../../Tokenizer/test_cases.txt";
std::fstream in(fileSpec);
if (!in.good())
{
std::cout << "\n can't open file " << fileSpec << "\n\n";
return 1;
}
std::cout << "\n File: " << fileSpec << "\n\n";
toker.attach(&in);
SemiExp semi(&toker);
while (semi.get())
{
std::cout << "\n -- semiExpression --";
semi.show(true);
}
if (semi.length() > 0)
{
std::cout << "\n -- semiExpression --";
semi.show();
std::cout << "\n\n";
}
}
std::cout << "\n -----Testing ITokcollection functions -----------\n";
try {
Test::test0();
Test::test1();
Test::test2();
Test::test3();
Test::test4();
Test::test5();
Test::test6();
Test::test7();
Test::test8();
}
catch (std::exception& ex)
{
std::cout << "\n\n " << ex.what() << "\n\n";
return 1;
}
return 0;
}
int main()
{
Helper::Title("Testing Tokenizer");
//std::string fileSpec = "../Tokenizer/Tokenizer.cpp";
//std::string fileSpec = "../Tokenizer/Tokenizer.h";
std::string fileSpec = "../Tokenizer/Test.txt";//from here
try
{
std::ifstream in(fileSpec);
if (!in.good())
{
std::cout << "\n can't open " << fileSpec << "\n\n";
return 1;
}
{
Toker toker;
toker.returnComments();
toker.attach(&in);
std::cout << "\n current line count = " << toker.currentLineCount();
do
{
std::string tok = toker.getTok();
if (tok == "\n")
tok = "newline";
std::cout << "\n -- " << tok;
} while (in.good());
std::cout << "\n current line count = " << toker.currentLineCount();
}
putline();
Helper::title("Testing change of special characters");
std::string newSpecialChars = "., :, +, +=, \n { }";
Toker toker;
toker.returnComments();
toker.setSpecialTokens(newSpecialChars);
in.clear();
in.seekg(std::ios::beg);
toker.attach(&in);
std::cout << "\n new special tokens: " << newSpecialChars;
do
{
std::string tok = toker.getTok();
if (tok == "\n")
tok = "newline";
std::cout << "\n -- " << tok;//check if tok is in type table
} while (in.good());// if there , get filename and say this file is dependent on this file
std::cout << "\n current line count = " << toker.currentLineCount();
}
catch (std::logic_error& ex)
{
std::cout << "\n " << ex.what();
}
std::cout << "\n\n";//here
return 0;
}
void tokenizerSetUp(Toker& toker) {
toker.setSpecialCharPairs("<<");
toker.setSpecialCharPairs("<=");
toker.setSpecialCharPairs(">=");
toker.setSpecialCharPairs(">>");
toker.setSpecialCharPairs("::");
toker.setSpecialCharPairs("++");
toker.setSpecialCharPairs("--");
toker.setSpecialCharPairs("==");
toker.setSpecialCharPairs("+=");
toker.setSpecialCharPairs("-=");
toker.setSpecialCharPairs("*=");
toker.setSpecialCharPairs("/=");
toker.setSpecialCharPairs("&&");
toker.setSpecialSingleChars(">");
toker.setSpecialSingleChars("<");
toker.setSpecialSingleChars("[");
toker.setSpecialSingleChars("]");
toker.setSpecialSingleChars("(");
toker.setSpecialSingleChars(")");
toker.setSpecialSingleChars("{");
toker.setSpecialSingleChars("}");
toker.setSpecialSingleChars(":");
toker.setSpecialSingleChars(";");
toker.setSpecialSingleChars("=");
toker.setSpecialSingleChars("+");
toker.setSpecialSingleChars("-");
toker.setSpecialSingleChars("*");
toker.setSpecialSingleChars("\n");
toker.setTokenTypesFlag("white_spaces");
toker.setTokenTypesFlag("CppComment-Req4.5");
toker.setTokenTypesFlag("C Comment-Req4.4");
toker.setTokenTypesFlag("Punctuator-Req4");
toker.setTokenTypesFlag("Punctuator-Req-4.2");
toker.setTokenTypesFlag("character_single_quote_string_Req4.6");
toker.setTokenTypesFlag("character string-Req4.6");
toker.setTokenTypesFlag("character_double_quote_string_Req4.6");
toker.setTokenTypesFlag("Special_two_characters_Req4.3");
toker.setTokenTypesFlag("Special_Single_character_Req4.3");
toker.setTokenTypesFlag("Alphanum_Req4");
toker.setTokenTypesFlag(" ");
}
