void TestScanner()
{
TestBegin("Scanner");
Scanner s;
Token tok;
s.From(
" \t\n"
" \ta.b = 1;\n"
" > < == <= >= != =~ && || . , : ; [ ] ( ) { } = + += -= *\r\n"
"123 0123 0xafbDDdd00\r"
"01233999 00. 1E9 1E-9 1e+9 1.1e900\n"
"\'fasdfa/#@$@@@\\n\'\n"
"\"fjalksdfjal#@$@#$@#\\n\"\n"
"_abcd\r"
);
while( s.NextToken(&tok) && tok.Rep() != TOK_eof ){
tok.Print();
std::putchar('\n');
}
TestEnd();
}
void Syntax::ReadSyntaxFile(Scanner &scanner, uint indents)
// ----------------------------------------------------------------------------
// Parse the syntax description table
// ----------------------------------------------------------------------------
{
enum State
{
inUnknown, inPrefix, inInfix, inPostfix,
inComment, inCommentDef,
inText, inTextDef,
inBlock, inBlockDef,
inSyntaxName, inSyntax, inSyntaxDef
};
State state = inUnknown;
text txt, entry;
token_t tok = tokNAME;
int priority = 0;
bool done = false;
ChildSyntax *childSyntax = NULL;
while(tok != tokEOF && !done)
{
tok = scanner.NextToken(true);
if (tok == tokSYMBOL || state >= inComment)
{
text t = scanner.TextValue();
uint i, len = t.length();
for (i = 1; i < len; i++)
{
text sub = t.substr(0, i);
known_prefixes.insert(sub);
}
known_tokens.insert(t);
}
switch(tok)
{
case tokEOF:
break;
case tokINTEGER:
priority = scanner.IntegerValue();
break;
case tokINDENT:
case tokPAROPEN:
indents++;
break;
case tokUNINDENT:
case tokPARCLOSE:
indents--;
if (!indents)
done = true;
break;
case tokNAME:
case tokSYMBOL:
case tokSTRING:
case tokQUOTE:
txt = scanner.TextValue();
if (txt == "NEWLINE")
txt = "\n";
else if (txt == "INDENT")
txt = Block::indent;
else if (txt == "UNINDENT")
txt = Block::unindent;
if (txt == "INFIX")
state = inInfix;
else if (txt == "PREFIX")
state = inPrefix;
else if (txt == "POSTFIX")
state = inPostfix;
else if (txt == "BLOCK")
state = inBlock;
else if (txt == "COMMENT")
state = inComment;
else if (txt == "TEXT")
state = inText;
else if (txt == "SYNTAX")
state = inSyntaxName;
else if (txt == "STATEMENT")
statement_priority = priority;
else if (txt == "FUNCTION")
function_priority = priority;
else if (txt == "DEFAULT")
default_priority = priority;
else switch(state)
{
case inUnknown:
break;
case inPrefix:
prefix_priority[txt] = priority;
break;
case inPostfix:
postfix_priority[txt] = priority;
break;
case inInfix:
infix_priority[txt] = priority;
break;
case inComment:
entry = txt;
state = inCommentDef;
break;
case inCommentDef:
comment_delimiters[entry] = txt;
state = inComment;
break;
case inText:
entry = txt;
state = inTextDef;
break;
case inTextDef:
text_delimiters[entry] = txt;
state = inText;
break;
case inBlock:
entry = txt;
state = inBlockDef;
infix_priority[entry] = priority;
break;
case inBlockDef:
block_delimiters[entry] = txt;
block_delimiters[txt] = "";
infix_priority[txt] = priority;
state = inBlock;
break;
case inSyntaxName:
if (txt.find(".syntax") == txt.npos)
txt += ".syntax";
if (txt.find("/") == txt.npos)
txt = ELFE_LIB + txt;
childSyntax = &subsyntax[txt];
if (childSyntax->filename == "")
{
childSyntax->filename = txt;
childSyntax->ReadSyntaxFile(txt);
}
state = inSyntax;
break;
case inSyntax:
entry = txt;
state = inSyntaxDef;
break;
case inSyntaxDef:
childSyntax->delimiters[entry] = txt;
subsyntax_file[entry] = childSyntax->filename;
state = inSyntax;
break;
}
break;
default:
break;
}
}
}