Ejemplo n.º 1
0
Archivo: lexer.c Proyecto: xxyzzzq/quex
int 
main(int argc, char** argv) 
{        
#   ifdef PRINT_TOKEN
    const size_t BufferSize = 1024;
    char         buffer[1024];
#   endif
    quex_Token*   token_p = 0x0;
    long          token_n = 0;
    quex_Simple   qlex;
#   ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
    QUEX_TYPE_TOKEN_ID token_id = (QUEX_TYPE_TOKEN_ID)0x0;
#   endif
    const char*        file_name = argc > 1 ? argv[1] : "example.txt";
    QUEX_NAME(from_file_name)(&qlex, file_name, CHARACTER_ENCODING_NAME);

    printf(",------------------------------------------------------------------------------------\n");
    printf("| [START]\n");
    fflush(stdout);
    fflush(stderr);

    /* Loop until the 'termination' token arrives */
    token_n = 0;

#   ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
    token_p = QUEX_NAME(token_p)(&qlex);
#   endif

    do {
        /* Get next token from the token stream   */
#       ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
        token_id = QUEX_NAME(receive)(&qlex);
#       else
        QUEX_NAME(receive)(&qlex, &token_p);
#       endif

#       ifdef PRINT_LINE_COLUMN_NUMBER
        printf("(%i, %i)  \t", (int)token_p->_line_n, (int)token_p->_column_n);
#       endif
        /* Print out token information            */
        fflush(stderr);
#       ifdef PRINT_TOKEN
        if( token_n >= PRINT_TOKEN_FIRST_NUMBER ) {
            printf("%s", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
            printf("\n");
        }
#       else
        printf("%s", QUEX_NAME_TOKEN(map_id_to_name)(token_p->_id));
        printf("\n");
#       endif
        fflush(stdout);

        ++token_n;
        /* Check against 'termination'            */
#   ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
    } while( token_id != QUEX_TKN_TERMINATION );
#   else
    } while( token_p->_id != QUEX_TKN_TERMINATION );
Ejemplo n.º 2
0
int 
main(int argc, char** argv) 
{        
    QUEX_TYPE_TOKEN       token_bank[2];
    QUEX_TYPE_TOKEN*      prev_token;
    quex_tiny_lexer       qlex;
    size_t                BufferSize = 1024;
    char                  buffer[1024];
    QUEX_TYPE_CHARACTER*  prev_lexeme_start_p = 0x0;
    size_t                receive_n = (size_t)-1;
    QUEX_TYPE_TOKEN_ID    token_id  = 0;
    QUEX_NAME(construct_memory)(&qlex, 0x0, 0x0, 0, 0x0, false);

    /* -- initialize the token pointers */
    QUEX_NAME_TOKEN(construct)(&token_bank[0]);
    QUEX_NAME_TOKEN(construct)(&token_bank[1]);
    token_bank[0]._id = QUEX_TKN_TERMINATION;

    prev_token = &(token_bank[1]);

    QUEX_NAME(token_p_switch)(&qlex, &token_bank[0]);

    while( 1 + 1 == 2 ) {
        /* -- Initialize the filling of the fill region                                    */
        QUEX_NAME(buffer_fill_region_prepare)(&qlex);

        /* -- Call the low lever driver to fill the fill region                            */
        receive_n = messaging_framework_receive_into_buffer(QUEX_NAME(buffer_fill_region_begin)(&qlex), 
                                                            QUEX_NAME(buffer_fill_region_size)(&qlex));

        /* -- Inform the buffer about the number of loaded characters NOT NUMBER OF BYTES! */
        QUEX_NAME(buffer_fill_region_finish)(&qlex, receive_n);

        /* -- Loop until the 'termination' token arrives                                   */
        token_id = 0;
        while( 1 + 1 == 2 ) {
            prev_lexeme_start_p = QUEX_NAME(buffer_lexeme_start_pointer_get)(&qlex);
            
            /* Let the previous token be the current token of the previous run.            */
            prev_token = QUEX_NAME(token_p_switch)(&qlex, prev_token);

            token_id = QUEX_NAME(receive)(&qlex);
            if( token_id == QUEX_TKN_TERMINATION || token_id == QUEX_TKN_BYE )
                break;
            if( prev_token->_id != QUEX_TKN_TERMINATION ) 
                printf("Consider: %s\n", QUEX_NAME_TOKEN(get_string)(prev_token, buffer, BufferSize));
        }

        if( token_id == QUEX_TKN_BYE ) break;

        QUEX_NAME(buffer_input_pointer_set)(&qlex, prev_lexeme_start_p);
    }

    QUEX_NAME(destruct)(&qlex);
    QUEX_NAME_TOKEN(destruct)(&token_bank[0]);
    QUEX_NAME_TOKEN(destruct)(&token_bank[1]);
    return 0;
}
Ejemplo n.º 3
0
int 
main(int argc, char** argv) 
{        
    quex_Token*  token_p = 0x0;
    size_t       number_of_tokens = 0;
    EasyLexer    qlex;
#   ifdef PRINT_TOKEN
    const size_t BufferSize = 1024;
    char         buffer[1024];
#   endif
    const char*  FileName = (argc == 1) ? "example.txt" : argv[1];

    QUEX_NAME(construct_file_name)(&qlex, FileName, ENCODING_NAME, false);
    /* Alternatives:
     * QUEX_NAME(construct_memory)(&qlex, MemoryBegin, MemorySize,
     *                             CharacterEncodingName (default 0x0),
     *                             ByteOrderReversionF   (default false));
     * QUEX_NAME(construct_FILE)(&qlex, FILE_handle, 
     *                           CharacterEncodingName (default 0x0),
     *                           ByteOrderReversionF   (default false)); */
    printf(",-----------------------------------------------------------------\n");
    printf("| [START]\n");

    /* Loop until the 'termination' token arrives */
    do {
        /* Get next token from the token stream   */
        QUEX_NAME(receive)(&qlex, &token_p);

        /* Print out token information            */
#       ifdef PRINT_LINE_COLUMN_NUMBER
        printf("(%i, %i)  \t", (int)token_p->_line_n, (int)token_p->_column_n);
#       endif
#       ifdef PRINT_TOKEN
        printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
#       else
        printf("%s\n", QUEX_NAME_TOKEN(map_id_to_name)(token_p->_id));
#       endif

#       ifdef SPECIAL_ACTION
        SPECIAL_ACTION(&qlex, &my_token);
#       endif
        ++number_of_tokens;

        /* Check against 'termination'            */
    } while( token_p->_id != QUEX_TKN_TERMINATION );

    printf("| [END] number of token = %i\n", number_of_tokens);
    printf("`-----------------------------------------------------------------\n");

    QUEX_NAME(destruct)(&qlex);

    return 0;
}
Ejemplo n.º 4
0
int 
main(int argc, char** argv) 
{        
    QUEX_TYPE_TOKEN*   token_p = 0x0;
    size_t             BufferSize = 1024;
    char               buffer[1024];
    UTF8Lex            qlex;
    
    QUEX_NAME(construct_file_name)(&qlex, "example-utf8.txt", 0x0, false);

    // (*) loop until the 'termination' token arrives
    do {
        // (*) get next token from the token stream
        QUEX_NAME(receive)(&qlex, &token_p);

        /* (*) print out token information
         *     'get_string' automagically converts codec bytes into utf8 */
        printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));

        // (*) check against 'termination'
    } while( token_p->_id != TKN_TERMINATION );

    QUEX_NAME(destruct)(&qlex);
    return 0;
}
Ejemplo n.º 5
0
int 
main(int argc, char** argv) 
{        
    quex_Token*        token_p;
    quex_ISO8859_7_Lex qlex;
    size_t             BufferSize = 1024;
    char               buffer[1024];
    
    QUEX_NAME(construct_file_name)(&qlex, "example-iso8859-7.txt", 0x0, false);

    // (*) loop until the 'termination' token arrives
    do {
        // (*) get next token from the token stream
        QUEX_NAME(receive)(&qlex, &token_p);

        /* (*) print out token information
         *     'get_string' automagically converts codec bytes into utf8 */
        printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
#       if 0
        cout << "\t\t plain bytes: ";
        for(QUEX_TYPE_CHARACTER* iterator = (uint8_t*)tmp.c_str(); *iterator ; ++iterator) {
            printf("%02X.", (int)*iterator);
        }
#       endif

        // (*) check against 'termination'
    } while( token_p->_id != TKN_TERMINATION );

    QUEX_NAME(destruct)(&qlex);
    return 0;
}
Ejemplo n.º 6
0
static rsexp read_simple_datum (Reader* reader)
{
    RToken* token;
    rsexp datum;
    rcstring text;
    rsize size;

    token = lookahead (reader);
    size = QUEX_NAME (strlen) (token->text) + 5;
    text = alloca (size);

    QUEX_NAME_TOKEN (pretty_char_text) (token, text, size);

    switch (token->_id) {
        case TKN_TRUE:
            datum = R_TRUE;
            break;

        case TKN_FALSE:
            datum = R_FALSE;
            break;

        case TKN_NUMBER:
            datum = r_cstr_to_number (reader->r, text);
            break;

        case TKN_CHARACTER:
            datum = lexeme_to_char (text);
            break;

        case TKN_STRING:
            datum = r_string_new (reader->r, text);
            break;

        case TKN_IDENTIFIER:
            datum = r_intern (reader->r, text);
            break;

        case TKN_HASH_U8_LP:
            datum = read_bytevector (reader);
            break;

        case TKN_TERMINATION:
            datum = R_EOF;
            break;

        default:
            datum = R_FAILURE;
            goto exit;
    }

    consume (reader);

exit:
    return datum;
}
Ejemplo n.º 7
0
int 
main(int argc, char** argv) 
{        
    QUEX_TYPE_TOKEN  token;
    tiny_lexer       qlex;
    size_t           receive_n = (size_t)-1;
    size_t           BufferSize = 1024;
    char             buffer[1024];
    bool             out_f = false;

    QUEX_NAME(construct_memory)(&qlex, 0x0, 0, 0x0, 0x0, false);
    QUEX_NAME_TOKEN(construct)(&token);

    (void)QUEX_NAME(token_p_switch)(&qlex, &token);
    while( 1 + 1 == 2 ) {
        /* -- Initialize the filling of the fill region         */
        QUEX_NAME(buffer_fill_region_prepare)(&qlex);

        /* -- Call the low lever driver to fill the fill region */
        receive_n = messaging_framework_receive_into_buffer_syntax_chunk(QUEX_NAME(buffer_fill_region_begin)(&qlex), 
                                                                         QUEX_NAME(buffer_fill_region_size)(&qlex));

        /* -- Inform the buffer about the number of loaded characters NOT NUMBER OF BYTES! */
        QUEX_NAME(buffer_fill_region_finish)(&qlex, receive_n);

        /* -- Loop until the 'termination' token arrives */
        while( 1 + 1 == 2 ) {
            const QUEX_TYPE_TOKEN_ID TokenID = QUEX_NAME(receive)(&qlex);

            if( TokenID == QUEX_TKN_TERMINATION ) break;
            if( TokenID == QUEX_TKN_BYE )         { out_f = true; break; }

            printf("Consider: %s \n", QUEX_NAME_TOKEN(get_string)(&token, buffer, BufferSize));
        }
        if( out_f ) break;
    }

    QUEX_NAME(destruct)(&qlex);
    QUEX_NAME_TOKEN(destruct)(&token);
    return 0;
}
Ejemplo n.º 8
0
int 
main(int argc, char** argv) 
{        
    quex_Token*    token_p = 0x0;
    quex_EasyLexer qlex;
    const size_t   BufferSize = 1024;
    char           buffer[1024];
    const char*    FileName = "example.txt";

    QUEX_NAME(construct_file_name)(&qlex, FileName, (void*)0, false);

    do {
        QUEX_NAME(receive)(&qlex, &token_p);

        printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));

    } while( token_p->_id != QUEX_TKN_TERMINATION );

    QUEX_NAME(destruct)(&qlex);

    return 0;
}