SymVarConst* Parser::ParseConst()
{
SymType* type = NULL;
string Value = "";
while (true)
{
if(t.GetValue() == "+" || t.GetValue() == "-")
{
Value = t.GetValue() + Value;
t = sc.GetNextToken();
}
else
if(t.GetType() == int_const_dec)
return new SymVarConst(Value + t.GetValue(), Value + t.GetValue(), (SymType*)mainTable->find("integer")->second);
else
if (t.GetType() == float_const)
return new SymVarConst(Value + t.GetValue(), Value + t.GetValue(), (SymType*)mainTable->find("real")->second);
else
{
bool found = false;
_Table::iterator it = TableStack.Find(t.GetValue(), found);
if (t.GetType() != identifier)
throw Error("wrongConst", t);
if (!found)
throw Error("NotConst", t);
if (!it->second->IsConst())
throw Error("wrongConstType",t);
return new SymVarConst(t.GetValue(), Value + ((SymVarConst*)it->second)->GetValue(), ((SymVarConst*)it->second)->GetType());
}
}
}
int main()
{
ifstream in;
in.open("C:/Users/taoyanqi/Desktop/Êý¾Ý½á¹¹/ImproveLexer/ImproveLexer/aa.txt");
Scanner scan(in);
for (Token t; t.GetType() != END_OF_FILE;)
{
// cout << ( t = scan.NextToken() );
t = scan.GetNextToken();
cout << t.GetRow() << ':' << t.GetColumn() << '\t' << TOKEN_DESCRIPTION[t.GetType()]
<< '\t' << TOKEN_VALUE_DESCRIPTION[t.GetValue()] << '\t' << t.GetName() << endl;
}
}
NodeWrite* Parser::ParseWrite(bool isFirst, bool isWriteln)
{
NodeWrite* ans = NULL;
t = sc.GetNextToken();
if (isFirst)
t = RequireToken("(", "\"(\" expected");
if (t.GetType() == string_const)
{
ans = new NodeWriteStr(STR, "\""+t.GetValue().substr(1, t.GetValue().size()-2)+"\"");
t = sc.GetNextToken();
}
else
{
NodeExpression* exp = ParseComparision();
OutputType T;
if (exp->GetType()->IsInt())
T = INT;
else
T = FLOAT;
ans = new NodeWriteExp(T, exp);
}
if (t.GetValue() == ",")
ans->SetNext(ParseWrite(false, isWriteln));
else
{
t = RequireToken(")", "\")\" expected");
if (isWriteln)
ans->SetNext(new NodeWriteln());
}
return ans;
}
int FAO::GetPrecedence()
{
if (cur_token_.GetType() == TokenType::TOKEN_OPERATION){
return this->binary_op_precedence_[cur_token_.GetOperation()];
}
return -1;
}
// ****************************************************************************
// Method: ExprParser::Parse
//
// Purpose:
// Main external routine for clients.
// Re-initialize the parser and scanner, and process one at a time until
// we get an error or acceptance. Return the parse tree when finished.
//
// Programmer: Jeremy Meredith
// Creation: April 5, 2002
//
// Modifications:
// Jeremy Meredith, Mon Jul 28 16:13:35 PDT 2003
// Made it report normal parse errors through the viewer window
// mechanism if needed. Unhandled reductions will only occur during
// debugging and can still go to cerr.
//
// Hank Childs, Fri Aug 8 08:13:21 PDT 2003
// Have error messages be issued in a way that it is independent of
// component.
//
// Jeremy Meredith, Fri Aug 15 12:49:01 PDT 2003
// Added the EMT_EXCEPTION type, and renamed EMT_VIEWER to EMT_COMPONENT.
//
// Jeremy Meredith, Wed Nov 24 11:51:59 PST 2004
// Refactored. There's a new base class for the ExprParser and the
// return types became more general.
//
// Jeremy Meredith, Tue Dec 28 11:18:37 PST 2004
// Made the current text a data member to help with macro support.
//
// Hank Childs, Tue Dec 28 16:14:47 PST 2004
// Rename GetText and PrintText to GetErrorText and PrintErrorText.
//
// Hank Childs, Fri Jan 28 14:07:18 PST 2005
// Use exception macros.
//
// Jeremy Meredith, Mon Jun 13 16:17:14 PDT 2005
// Delete the tokens that have not taken part in a rule reduction -- in
// this case that means Space tokens and the final EOF token.
//
// ****************************************************************************
ParseTreeNode*
ExprParser::Parse(const std::string &s)
{
text = s;
// Change weird spacing (tabs, newlines) to normal ones.
// This will make printing error messages later much easier.
for (size_t i=0; i<text.length(); i++)
if (text[i] == '\t' || text[i] == '\n') text[i] = ' ';
TRY
{
Init();
scanner.SetInput(text);
Token *token = NULL;
while (!Accept())
{
token = scanner.ScanOneToken();
if (token->GetType() != TT_Space)
ParseOneToken(token);
else
delete token;
}
delete token;
}
CATCH2(UnhandledReductionException, e)
{
// This should only occur during debugging; print to cerr anyway
cerr << e.Message() << endl;
cerr << "Rule = " << *(e.GetRule()) << endl;
e.GetPos().PrintErrorText(cerr, text);
CATCH_RETURN2(1, NULL);
}
int CompilerMain::GetNextToken()
{
Token* token = tokenList->at(nextTokenIndex);
yylval = token->GetValue().toStdString();
//qDebug()<<("Next Token is " + token->GetValue());
nextTokenIndex++;
return token->GetType();
}
ExprAST * FAO::ParsePrimary()
{
switch (cur_token_.GetType()){
case TokenType::TOKEN_INTEGER:
return ParseNumberExpr();
case TokenType::TOKEN_LEFT_PARAM:
return ParseParentExpr();
default:
return Error("unknow token when expecting an expression");
}
}
ExprAST * FAO::ParseParentExpr()
{
Advance(); // eat "("
ExprAST * ret = ParseExpression();
if (ret == nullptr)return 0;
if (cur_token_.GetType() != TokenType::TOKEN_RIGHT_PARAM){
return Error("expected ')'");
}
Advance(); // eat ")"
return ret;
}
SymTable* Parser::ParseArguments(vector<string>* &arguments)
{
SymTable* argTable = new SymTable;
if (t.GetValue() != "(")
return argTable;
vector<string> argNames;
bool isVar = false;
do
{
t = sc.GetNextToken();
if (t.GetValue() == "var")
{
isVar = true;
t = sc.GetNextToken();
}
if (t.GetType() != identifier)
throw Error("incorrect name of argument", t);
argNames.push_back(t.GetValue());
arguments->push_back(t.GetValue());
t = sc.GetNextToken();
if (t.GetValue() == ",")
continue;
if (t.GetValue() == ":")
{
SymType* Type = ParseType(false);
if (IsExplType(Type->GetName()))
throw Error("arguments type mismatch", t);
t = sc.GetNextToken();
for(vector<string>::iterator it= argNames.begin(); it != argNames.end(); ++it)
{
if (argTable->find(*it) != argTable->end())
throw Error("identifier already declared",t);
if (isVar)
argTable->insert(pair<string, SymVar*> (*it, new SymVarParam_var(*it, Type)));
else
argTable->insert(pair<string, SymVar*> (*it, new SymVarParam(*it, Type)));
}
isVar = false;
argNames.clear();
if (t.GetValue() != ";" && t.GetValue() != ")")
throw Error("incorrect function defenition", t);
}
}
while(t.GetValue() != ")");
t = sc.GetNextToken();
return argTable;
}
void MainWindow::on_selectFile_clicked()
{
QFileDialog dialog;
if (dialog.exec())
{
if (!dialog.selectedFiles().size()) return;
int counter = 0;
QFile file(dialog.selectedFiles()[0]);
file.open(QFile::ReadOnly);
Tokenizer tokenizer(file);
Token tok;
QString sb;
QString sb1k;
QString sb32k;
do
{
tok = tokenizer.Next();
if (tok.GetType() == Token::eof) break;
if (!(counter++ % 256))
{
std::cout << "counter: " << counter << std::endl;
}
sb1k = sb1k % tok.GetText();
if (sb1k.size() == 256)
{
sb32k = sb32k % sb1k;
sb1k.clear();
if (sb32k.size() == 32768)
{
sb = sb % sb32k;
sb32k.clear();
}
}
} while (true);
if (sb1k.size())
{
sb32k = sb32k % sb1k;
}
if (sb32k.size())
{
sb = sb % sb32k;
}
ui->textBrowser->setText(sb);
file.close();
}
}
Token &
Compiler::
Parse()
{
Token *
ret;
bool
real = false;
for ( ; ; )
{
ret = &m_parser.Statement();
// We need to make sure this
// is really the end of the
// stream. When a new file
// is added after all the
// others have been done, then
// end of stream token is
// still in the system, even
// though it's no longer the
// end of the stream.
TokenType::E_TYPE
type = ret->GetType();
if (type == TokenType::TT__END)
{
if (real)
{
// Really is the end
break;
}
else
{
real = true;
}
}
else if (type != TokenType::TT__EOF)
{
break;
}
}
return *ret;
}
NodeStatement* Parser::ParseElement()
{
if (t.GetValue() == "begin")
return ParseBlock();
if (t.GetValue() == "if")
return ParseIf();
if (t.GetValue() == "while")
return ParseWhile();
if (t.GetValue() == "repeat")
return ParseRepeat();
if (t.GetValue() == "for")
return ParseFor();
if (t.GetValue() == "write")
return ParseWrite(true, false);
if (t.GetValue() == "writeln")
return ParseWrite(true, true);
if (t.GetValue() == "exit")
{
t = sc.GetNextToken();
return new StatementExit(true);
}
if (t.GetType() == identifier)
{
bool found = false;
_Table::iterator it = TableStack.Find(t.GetValue(), found);
if (found)
{
if (it->second->IsProc() || it->second->IsFunc())
{
t = sc.GetNextToken();
return ParseSub((SymProc*)it->second);
}
}
return ParseAssignment();
}
if (t.GetValue() == "(")
{
return ParseAssignment();
}
throw Error("Statement error", t);
}
Statement *Statement::GetNext(Tokenizer &program) {
Token token = program.Look();
if (token.GetType() == "UNKNOWN")
MalformedExpressionError::Raise(token, __FILE__, __LINE__);
Statement *statement;
if (token.Match("INTEGER_TYPE") || token.Match("STRING_TYPE"))
statement = new AssignStatement();
else if (token.Match("OUTPUT"))
statement = new OutputStatement();
else if (token.Match("IF"))
statement = new IfStatement();
else if (token.Match("SPAWN"))
statement = new SpawnStatement();
else if (token.Match("READ_LINE"))
statement = new ReadLineStatement();
statement->Read(program);
LOG(INFO) << (std::string) typeid(*statement).name() << " load";
return statement;
}
sint32 FilenameDB::Parse(char *filename)
{
Token *token = new Token(filename, C3DIR_GAMEDATA);
Assert(token);
if (token->GetType() != TOKEN_NUMBER) {
c3errors_ErrorDialog (token->ErrStr(), "Missing number of filenames");
g_abort_parse = TRUE;
delete token;
return FALSE;
} else {
sint32 n;
token->GetNumber(n);
token->Next();
if (n <0) {
c3errors_ErrorDialog(token->ErrStr(), "Number of filename is negative");
g_abort_parse = TRUE;
delete token;
return FALSE;
}
SetSize(n);
}
int count = 0;
while (ParseAFilename(token, count)) {
count++;
}
if (g_abort_parse) {
delete token;
return FALSE;
}
delete token;
return TRUE;
}
SymProc* Parser::ProcFunc(bool isFunc)
{
t = sc.GetNextToken();
SymType* Type = NULL;
SymProc* sub = NULL;
if (t.GetType() != identifier)
throw Error("incorrect procedure name", t);
SymTable* argTable = NULL;
vector<string>* argNames = new vector<string>;
_Table::iterator it = table->find(t.GetValue());
string name = t.GetValue();
t = sc.GetNextToken();
argTable = ParseArguments(argNames);
TableStack.PushTable(argTable);
if (isFunc)
{
if (t.GetValue() != ":")
throw Error("type expected", t);
Type = ParseType(false);
t = sc.GetNextToken();
}
t = RequireToken(";", "\";\" expected");
if (it != table->end())
{
if (!(it->second->IsProc()) || !(((SymProc*)it->second)->IsForward()))
throw Error("incorrect procedure definition", t);
SymTable* argTableF = ((SymProc*)it->second)->GetArgTable();
vector<string>* argNamesF = ((SymProc*)it->second)->GetArgNames();
for (size_t i=0; i<argNames->size(); ++i)
{
if ((*argNames)[i] != (*argNamesF)[i] ||
!SymComp(argTable->find((*argNames)[i])->second , argTableF->find((*argNamesF)[i])->second))
throw Error("Header does not match previouse definition", t);
}
if (isFunc)
sub = ParseProcDecl(name, argNames, argTable, true, Type);
else
sub = ParseProcDecl(name, argNames, argTable, false, Type);
t = RequireToken(";", "\";\" expected");
}
else
{
if (t.GetValue() == "forward")
{
t = sc.GetNextToken();
if (t.GetValue() != ";")
throw Error("\";\" expected", t);
if (isFunc)
{
sub = new SymFunc(name, argNames, argTable, NULL, true, NULL);
((SymFunc*)sub)->SetType(Type);
}
else
sub = new SymProc(name, argNames, argTable, NULL, true);
sub->print(os, false);
table->insert(pair<string, SymProc*> (name, sub));
TableStack.PopTable();
}
else
{
sub = ParseProcDecl(name, argNames, argTable, isFunc, Type);
if (isFunc)
((SymFunc*)sub)->SetType(Type);
sub->GetBody()->print(os, 0);
}
t = sc.GetNextToken();
}
return sub;
}
int main(int argc, char* argv[])
{
if (argc == 1)
{
PrintHelp();
return 0;
}
try
{
if (argc > 3) throw CompilerException("too many parametrs");
if (argc == 2)
{
if (argv[1][1] == 'h')
{
PrintHelp();
return 0;
}
if (argv[1][1] == 'l' || argv[1][1] == 's')
throw CompilerException("no files specified");
else
throw CompilerException("uncknown option");
}
ifstream in;
in.open(argv[2], ios::in);
if (!in.good()) throw CompilerException("can't open file");
if (argv[1][0] != '-')
throw CompilerException("invalid option");
else
{
if (!argv[1][1] || argv[1][2]) throw CompilerException("invalid option");
bool optimize = isupper(argv[1][1]);
switch (tolower(argv[1][1]))
{
case 'b':
{
Scanner scan(in);
Parser parser(scan, optimize);
parser.PrintSymTable(std::cout);
parser.PrintSyntaxTree(std::cout);
}
break;
case 's':
{
Scanner scan(in);
Parser parser(scan, optimize);
parser.PrintSyntaxTree(std::cout);
}
break;
case 't':
{
Scanner scan(in);
Parser parser(scan, optimize);
parser.PrintSymTable(std::cout);
}
break;
case 'g':
{
Scanner scan(in);
Parser parser(scan, optimize);
parser.Generate(std::cout);
}
break;
case 'l':
{
Scanner scan(in);
for (Token t; t.GetType() != END_OF_FILE;)
{
cout << ( t = scan.NextToken() );
}
}
break;
}
}
}
catch (CompilerException& e)
{
cout << e.what() << endl;
return 1;
}
catch (exception& e)
{
cout << e.what() << endl;
return 1;
}
return 0;
}
std::vector<FunctionModel> ConvertToFunctionModel(const std::vector<Token>& input, std::string& LibraryName)
{
std::vector<FunctionModel> output;
bool timeDefault=false;
std::vector<Token>::const_iterator it = input.begin();
while (it != input.end())
{
Token ThisToken = *it;
switch (ThisToken.GetType())
{
case Token::preprocessor :
++it;
break; // ignore preprocessor directives
case Token::curlyleft :
throw("curly bracket found, only functions can be coped with");
break;
case Token::curlyright :
throw("curly bracket found, only functions can be coped with");
break;
case Token::ampersand :
throw("unexpected ampersand found, return type expected");
break;
case Token::comma :
throw("unexpected comma found, return type expected");
break;
case Token::right :
throw("unexpected ) found, return type expected");
break;
case Token::left :
throw("unexpected ( found, return type expected");
break;
case Token::semicolon :
++it;
break;
case Token::comment :
{
std::string val = it->GetValue();
if (LeftString(val,5) == "<xlw:")
{
bool found = false;
if (val.size()>= 19 && val.substr(0,17) == "<xlw:libraryname=")
{
LibraryName = val.substr(17,val.size());
found =true;
}
if (LeftString(val,12) == "<xlw:timeall")
{
timeDefault = true;
found =true;
}
if (LeftString(val,13) == "<xlw:timenone")
{
timeDefault = false;
found =true;
}
if (!found)
std::cout << "Unknown xlw command "+val+"\n";
}
++it; //ignore comment unless in function definition
}
break;
case Token::identifier :
if (it->GetValue() == "using") {
// Ignore 'using namespace xxx;'
++it;
if (it == input.end() || it->GetType() != Token::identifier || it->GetValue() != "namespace")
throw("invalid syntax for declaration : 'using namespace xxx;'");
++it;
if (it == input.end() || it->GetType() != Token::identifier)
throw("invalid syntax for declaration : 'using namespace xxx;'");
++it;
} else {
// Process a function declaration
output.push_back(FunctionFind(it,input.end(),timeDefault));
}
break;
default:
throw("unknown token type found");
}
}
return output;
}
bool Lexer::GetNextToken(Token& outToken) const
{
bool isOk = true;
outToken = Token(Token::NO_Type);
while(isOk && outToken.GetType() == Token::NO_Type)
{
char c = GetNextChar();
if(m_stream.eof())
{
outToken = Token(Token::EOF_Type);
}
else if(c == '\n')
{
++m_line;
continue;
}
else if(c == ' ')
{
continue;
}
else if(c == '\t')
{
continue;
}
else if(c == '=')
{
outToken = Token(Token::Assign_Type);
}
else if(c == '{')
{
outToken = Token(Token::LeftBrace_Type);
}
else if(c == '}')
{
outToken = Token(Token::RightBrace_Type);
}
else if(c == ';')
{
outToken = Token(Token::SemiColon_Type);
}
else if(c == '$')
{
std::string value;
char cc = GetNextChar();
if(IsValidVariableStartChar(cc))
{
while(IsValidVariableChar(cc) && !m_stream.eof())
{
value+=cc;
cc = GetNextChar();
}
}
else
{
//throw SyntaxErrorException("Syntax error when reading variable", "Lexer::GetNextToken", __LINE__, __FILE__);
isOk = false;
}
//Jump back one char
m_stream.unget();
outToken = Token(Token::Variable_Type, value);
}
else if (c == '\"')
{
char cc = GetNextChar();
std::string value;
while(cc != '\"' && !m_stream.eof())
{
value += cc;
cc = GetNextChar();
}
if(m_stream.eof())
{
//throw SyntaxErrorException("Syntax error when reading variable content", "Lexer::GetNextToken", __LINE__, __FILE__);
isOk = false;
}
outToken = Token(Token::String_Type, value);
}
else //Check for reserved words
{
std::string value;
char cc = c;
if(IsValidReservedWordStartChar(cc))
{
while(IsValidReservedWordChar(cc) && !m_stream.eof())
{
value += cc;
cc = GetNextChar();
}
m_stream.unget();
if(value == "OVERRIDE")
{
outToken = Token(Token::Override_Type, value);
}
else
{
//Error
isOk = false;
}
}
else
{
//Error
isOk = false;
}
}
//if(outToken.GetType() != Token::NO_Type)
//return isOk;
}
return isOk;
}
Symbol::Symbol(Token token_)
{
if (token_.GetType() != STR_CONST) token_.NameToLowerCase();
token = token_;
}
NodeExpression* Parser::ParseFactor()
{
NodeExpression* r = NULL;
if (t.GetValue() == "-" || t.GetValue() == "+" || t.GetValue() == "not")
{
string op = t.GetValue();
t = sc.GetNextToken();
NodeExpression* _r = ParseFactor();
r = new NodeUnaryOp(new Symbol(op), _r);
r->SetType(_r->GetType());
}
else
{
if (t.GetValue() != "(" && t.GetType() != identifier && t.GetType() != int_const_dec && t.GetType() != float_const)
throw Error("Syntax error : unexpected \""+t.GetValue()+"\";", t);
if(t.GetValue() == "(")
{
t = sc.GetNextToken();
r = ParseComparision();
t = RequireToken(")" , "\")\" expected");
}
switch(t.GetType()){
case int_const_dec : case float_const:
{
SymVarConst* _const = new SymVarConst(t.GetValue(), t.GetValue(), NULL);
if (t.GetType() == int_const_dec)
_const->SetType((SymType*)mainTable->find("integer")->second);
else
_const->SetType((SymType*)mainTable->find("real")->second);
t = sc.GetNextToken();
r = new NodeConst(_const);
r->SetType(_const->GetType());
return r;
break;
}
case identifier:
{
Token t1 = t;
bool found = false;
_Table::iterator it = TableStack.Find(t1.GetValue(), found);
if (!found)
throw Error("unknown identifier", t);
t = sc.GetNextToken();
SymVar* name = (SymVar*)it->second;
SymType* Type = name->GetType();
if (t.GetValue() == "(")
{
if (!it->second->IsFunc())
throw Error("unexpected identifier", t);
if (((SymFunc*)it->second)->IsForward())
throw Error("undefined forward ("+it->second->GetName()+")", t);
r = ParseSub((SymFunc*)it->second);
}
else
if(name->GetType()->IsRec() || name->GetType()->IsArr())
{
r = new NodeVar(name);
r->SetType(Type);
while(name->GetType()->IsRec() || name->GetType()->IsArr())
{
SymVar* name2 = name;
if (name->GetType()->IsRec())
r = ParseRec(r, name);
if (name->GetType()->IsArr())
r = ParseArr(r,name);
if (name2 == name)
break;
}
}
else
{
if (name->IsConst())
r = new NodeConst(it->second);
else
r = new NodeVar(it->second);
r->SetType(Type);
}
break;
}
}
}
return r;
}