예제 #1
0
void TestScanner()
{
  TestBegin("Scanner");

  Scanner s;
  Token tok;

  s.From(
    "    \t\n"
    "  \ta.b = 1;\n"
    "  > < == <= >=  != =~ && || . , : ; [ ] ( ) { } = + += -= *\r\n"
    "123 0123 0xafbDDdd00\r"
    "01233999 00. 1E9 1E-9 1e+9 1.1e900\n"
    "\'fasdfa/#@$@@@\\n\'\n"
    "\"fjalksdfjal#@$@#$@#\\n\"\n"
    "_abcd\r"
  );

  while( s.NextToken(&tok) && tok.Rep() != TOK_eof ){
    tok.Print();
    std::putchar('\n');
  }

  TestEnd();
}
예제 #2
0
파일: syntax.cpp 프로젝트: c3d/elfe
void Syntax::ReadSyntaxFile(Scanner &scanner, uint indents)
// ----------------------------------------------------------------------------
//   Parse the syntax description table
// ----------------------------------------------------------------------------
{
    enum State
    {
        inUnknown, inPrefix, inInfix, inPostfix,
        inComment, inCommentDef,
        inText, inTextDef,
        inBlock, inBlockDef,
        inSyntaxName, inSyntax, inSyntaxDef
    };

    State       state = inUnknown;
    text        txt, entry;
    token_t     tok = tokNAME;
    int         priority = 0;
    bool        done = false;
    ChildSyntax *childSyntax = NULL;

    while(tok != tokEOF && !done)
    {
        tok = scanner.NextToken(true);

        if (tok == tokSYMBOL || state >= inComment)
        {
            text t = scanner.TextValue();
            uint i, len = t.length();
            for (i = 1; i < len; i++)
            {
                text sub = t.substr(0, i);
                known_prefixes.insert(sub);
            }
            known_tokens.insert(t);
        }

        switch(tok)
        {
        case tokEOF:
            break;
        case tokINTEGER:
            priority = scanner.IntegerValue();
            break;
        case tokINDENT:
        case tokPAROPEN:
            indents++;
            break;
        case tokUNINDENT:
        case tokPARCLOSE:
            indents--;
            if (!indents)
                done = true;
            break;
        case tokNAME:
        case tokSYMBOL:
        case tokSTRING:
        case tokQUOTE:
            txt = scanner.TextValue();

            if (txt == "NEWLINE")
                txt = "\n";
            else if (txt == "INDENT")
                txt = Block::indent;
            else if (txt == "UNINDENT")
                txt = Block::unindent;

            if (txt == "INFIX")
                state = inInfix;
            else if (txt == "PREFIX")
                state = inPrefix;
            else if (txt == "POSTFIX")
                state = inPostfix;
            else if (txt == "BLOCK")
                state = inBlock;
            else if (txt == "COMMENT")
                state = inComment;
            else if (txt == "TEXT")
                state = inText;
            else if (txt == "SYNTAX")
                state = inSyntaxName;

            else if (txt == "STATEMENT")
                statement_priority = priority;
            else if (txt == "FUNCTION")
                function_priority = priority;
            else if (txt == "DEFAULT")
                default_priority = priority;

            else switch(state)
            {
            case inUnknown:
                break;
            case inPrefix:
                prefix_priority[txt] = priority;
                break;
            case inPostfix:
                postfix_priority[txt] = priority;
                break;
            case inInfix:
                infix_priority[txt] = priority;
                break;
            case inComment:
                entry = txt;
                state = inCommentDef;
                break;
            case inCommentDef:
                comment_delimiters[entry] = txt;
                state = inComment;
                break;
            case inText:
                entry = txt;
                state = inTextDef;
                break;
            case inTextDef:
                text_delimiters[entry] = txt;
                state = inText;
                break;
            case inBlock:
                entry = txt;
                state = inBlockDef;
                infix_priority[entry] = priority;
                break;
            case inBlockDef:
                block_delimiters[entry] = txt;
                block_delimiters[txt] = "";
                infix_priority[txt] = priority;
                state = inBlock;
                break;
            case inSyntaxName:
                if (txt.find(".syntax") == txt.npos)
                    txt += ".syntax";
                if (txt.find("/") == txt.npos)
                    txt = ELFE_LIB + txt;
                childSyntax = &subsyntax[txt];
                if (childSyntax->filename == "")
                {
                    childSyntax->filename = txt;
                    childSyntax->ReadSyntaxFile(txt);
                }
                state = inSyntax;
                break;
            case inSyntax:
                entry = txt;
                state = inSyntaxDef;
                break;
            case inSyntaxDef:
                childSyntax->delimiters[entry] = txt;
                subsyntax_file[entry] = childSyntax->filename;
                state = inSyntax;
                break;
            }
            break;
        default:
            break;
        }
    }
}