void TestScanner() { TestBegin("Scanner"); Scanner s; Token tok; s.From( " \t\n" " \ta.b = 1;\n" " > < == <= >= != =~ && || . , : ; [ ] ( ) { } = + += -= *\r\n" "123 0123 0xafbDDdd00\r" "01233999 00. 1E9 1E-9 1e+9 1.1e900\n" "\'fasdfa/#@$@@@\\n\'\n" "\"fjalksdfjal#@$@#$@#\\n\"\n" "_abcd\r" ); while( s.NextToken(&tok) && tok.Rep() != TOK_eof ){ tok.Print(); std::putchar('\n'); } TestEnd(); }
void Syntax::ReadSyntaxFile(Scanner &scanner, uint indents) // ---------------------------------------------------------------------------- // Parse the syntax description table // ---------------------------------------------------------------------------- { enum State { inUnknown, inPrefix, inInfix, inPostfix, inComment, inCommentDef, inText, inTextDef, inBlock, inBlockDef, inSyntaxName, inSyntax, inSyntaxDef }; State state = inUnknown; text txt, entry; token_t tok = tokNAME; int priority = 0; bool done = false; ChildSyntax *childSyntax = NULL; while(tok != tokEOF && !done) { tok = scanner.NextToken(true); if (tok == tokSYMBOL || state >= inComment) { text t = scanner.TextValue(); uint i, len = t.length(); for (i = 1; i < len; i++) { text sub = t.substr(0, i); known_prefixes.insert(sub); } known_tokens.insert(t); } switch(tok) { case tokEOF: break; case tokINTEGER: priority = scanner.IntegerValue(); break; case tokINDENT: case tokPAROPEN: indents++; break; case tokUNINDENT: case tokPARCLOSE: indents--; if (!indents) done = true; break; case tokNAME: case tokSYMBOL: case tokSTRING: case tokQUOTE: txt = scanner.TextValue(); if (txt == "NEWLINE") txt = "\n"; else if (txt == "INDENT") txt = Block::indent; else if (txt == "UNINDENT") txt = Block::unindent; if (txt == "INFIX") state = inInfix; else if (txt == "PREFIX") state = inPrefix; else if (txt == "POSTFIX") state = inPostfix; else if (txt == "BLOCK") state = inBlock; else if (txt == "COMMENT") state = inComment; else if (txt == "TEXT") state = inText; else if (txt == "SYNTAX") state = inSyntaxName; else if (txt == "STATEMENT") statement_priority = priority; else if (txt == "FUNCTION") function_priority = priority; else if (txt == "DEFAULT") default_priority = priority; else switch(state) { case inUnknown: break; case inPrefix: prefix_priority[txt] = priority; break; case inPostfix: postfix_priority[txt] = priority; break; case inInfix: infix_priority[txt] = priority; break; case inComment: entry = txt; state = inCommentDef; break; case inCommentDef: comment_delimiters[entry] = txt; state = inComment; break; case inText: entry = txt; state = inTextDef; break; case inTextDef: text_delimiters[entry] = txt; state = inText; break; case inBlock: entry = txt; state = inBlockDef; infix_priority[entry] = priority; break; case inBlockDef: block_delimiters[entry] = txt; block_delimiters[txt] = ""; infix_priority[txt] = priority; state = inBlock; break; case inSyntaxName: if (txt.find(".syntax") == txt.npos) txt += ".syntax"; if (txt.find("/") == txt.npos) txt = ELFE_LIB + txt; childSyntax = &subsyntax[txt]; if (childSyntax->filename == "") { childSyntax->filename = txt; childSyntax->ReadSyntaxFile(txt); } state = inSyntax; break; case inSyntax: entry = txt; state = inSyntaxDef; break; case inSyntaxDef: childSyntax->delimiters[entry] = txt; subsyntax_file[entry] = childSyntax->filename; state = inSyntax; break; } break; default: break; } } }