int
main(int argc, char** argv)
{
# ifdef PRINT_TOKEN
const size_t BufferSize = 1024;
char buffer[1024];
# endif
quex_Token* token_p = 0x0;
long token_n = 0;
quex_Simple qlex;
# ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
QUEX_TYPE_TOKEN_ID token_id = (QUEX_TYPE_TOKEN_ID)0x0;
# endif
const char* file_name = argc > 1 ? argv[1] : "example.txt";
QUEX_NAME(from_file_name)(&qlex, file_name, CHARACTER_ENCODING_NAME);
printf(",------------------------------------------------------------------------------------\n");
printf("| [START]\n");
fflush(stdout);
fflush(stderr);
/* Loop until the 'termination' token arrives */
token_n = 0;
# ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
token_p = QUEX_NAME(token_p)(&qlex);
# endif
do {
/* Get next token from the token stream */
# ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
token_id = QUEX_NAME(receive)(&qlex);
# else
QUEX_NAME(receive)(&qlex, &token_p);
# endif
# ifdef PRINT_LINE_COLUMN_NUMBER
printf("(%i, %i) \t", (int)token_p->_line_n, (int)token_p->_column_n);
# endif
/* Print out token information */
fflush(stderr);
# ifdef PRINT_TOKEN
if( token_n >= PRINT_TOKEN_FIRST_NUMBER ) {
printf("%s", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
printf("\n");
}
# else
printf("%s", QUEX_NAME_TOKEN(map_id_to_name)(token_p->_id));
printf("\n");
# endif
fflush(stdout);
++token_n;
/* Check against 'termination' */
# ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
} while( token_id != QUEX_TKN_TERMINATION );
# else
} while( token_p->_id != QUEX_TKN_TERMINATION );
int
main(int argc, char** argv)
{
QUEX_TYPE_TOKEN token_bank[2];
QUEX_TYPE_TOKEN* prev_token;
quex_tiny_lexer qlex;
size_t BufferSize = 1024;
char buffer[1024];
QUEX_TYPE_CHARACTER* prev_lexeme_start_p = 0x0;
size_t receive_n = (size_t)-1;
QUEX_TYPE_TOKEN_ID token_id = 0;
QUEX_NAME(construct_memory)(&qlex, 0x0, 0x0, 0, 0x0, false);
/* -- initialize the token pointers */
QUEX_NAME_TOKEN(construct)(&token_bank[0]);
QUEX_NAME_TOKEN(construct)(&token_bank[1]);
token_bank[0]._id = QUEX_TKN_TERMINATION;
prev_token = &(token_bank[1]);
QUEX_NAME(token_p_switch)(&qlex, &token_bank[0]);
while( 1 + 1 == 2 ) {
/* -- Initialize the filling of the fill region */
QUEX_NAME(buffer_fill_region_prepare)(&qlex);
/* -- Call the low lever driver to fill the fill region */
receive_n = messaging_framework_receive_into_buffer(QUEX_NAME(buffer_fill_region_begin)(&qlex),
QUEX_NAME(buffer_fill_region_size)(&qlex));
/* -- Inform the buffer about the number of loaded characters NOT NUMBER OF BYTES! */
QUEX_NAME(buffer_fill_region_finish)(&qlex, receive_n);
/* -- Loop until the 'termination' token arrives */
token_id = 0;
while( 1 + 1 == 2 ) {
prev_lexeme_start_p = QUEX_NAME(buffer_lexeme_start_pointer_get)(&qlex);
/* Let the previous token be the current token of the previous run. */
prev_token = QUEX_NAME(token_p_switch)(&qlex, prev_token);
token_id = QUEX_NAME(receive)(&qlex);
if( token_id == QUEX_TKN_TERMINATION || token_id == QUEX_TKN_BYE )
break;
if( prev_token->_id != QUEX_TKN_TERMINATION )
printf("Consider: %s\n", QUEX_NAME_TOKEN(get_string)(prev_token, buffer, BufferSize));
}
if( token_id == QUEX_TKN_BYE ) break;
QUEX_NAME(buffer_input_pointer_set)(&qlex, prev_lexeme_start_p);
}
QUEX_NAME(destruct)(&qlex);
QUEX_NAME_TOKEN(destruct)(&token_bank[0]);
QUEX_NAME_TOKEN(destruct)(&token_bank[1]);
return 0;
}
int
main(int argc, char** argv)
{
quex_Token* token_p = 0x0;
size_t number_of_tokens = 0;
EasyLexer qlex;
# ifdef PRINT_TOKEN
const size_t BufferSize = 1024;
char buffer[1024];
# endif
const char* FileName = (argc == 1) ? "example.txt" : argv[1];
QUEX_NAME(construct_file_name)(&qlex, FileName, ENCODING_NAME, false);
/* Alternatives:
* QUEX_NAME(construct_memory)(&qlex, MemoryBegin, MemorySize,
* CharacterEncodingName (default 0x0),
* ByteOrderReversionF (default false));
* QUEX_NAME(construct_FILE)(&qlex, FILE_handle,
* CharacterEncodingName (default 0x0),
* ByteOrderReversionF (default false)); */
printf(",-----------------------------------------------------------------\n");
printf("| [START]\n");
/* Loop until the 'termination' token arrives */
do {
/* Get next token from the token stream */
QUEX_NAME(receive)(&qlex, &token_p);
/* Print out token information */
# ifdef PRINT_LINE_COLUMN_NUMBER
printf("(%i, %i) \t", (int)token_p->_line_n, (int)token_p->_column_n);
# endif
# ifdef PRINT_TOKEN
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
# else
printf("%s\n", QUEX_NAME_TOKEN(map_id_to_name)(token_p->_id));
# endif
# ifdef SPECIAL_ACTION
SPECIAL_ACTION(&qlex, &my_token);
# endif
++number_of_tokens;
/* Check against 'termination' */
} while( token_p->_id != QUEX_TKN_TERMINATION );
printf("| [END] number of token = %i\n", number_of_tokens);
printf("`-----------------------------------------------------------------\n");
QUEX_NAME(destruct)(&qlex);
return 0;
}
int
main(int argc, char** argv)
{
QUEX_TYPE_TOKEN* token_p = 0x0;
size_t BufferSize = 1024;
char buffer[1024];
UTF8Lex qlex;
QUEX_NAME(construct_file_name)(&qlex, "example-utf8.txt", 0x0, false);
// (*) loop until the 'termination' token arrives
do {
// (*) get next token from the token stream
QUEX_NAME(receive)(&qlex, &token_p);
/* (*) print out token information
* 'get_string' automagically converts codec bytes into utf8 */
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
// (*) check against 'termination'
} while( token_p->_id != TKN_TERMINATION );
QUEX_NAME(destruct)(&qlex);
return 0;
}
int
main(int argc, char** argv)
{
quex_Token* token_p;
quex_ISO8859_7_Lex qlex;
size_t BufferSize = 1024;
char buffer[1024];
QUEX_NAME(construct_file_name)(&qlex, "example-iso8859-7.txt", 0x0, false);
// (*) loop until the 'termination' token arrives
do {
// (*) get next token from the token stream
QUEX_NAME(receive)(&qlex, &token_p);
/* (*) print out token information
* 'get_string' automagically converts codec bytes into utf8 */
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
# if 0
cout << "\t\t plain bytes: ";
for(QUEX_TYPE_CHARACTER* iterator = (uint8_t*)tmp.c_str(); *iterator ; ++iterator) {
printf("%02X.", (int)*iterator);
}
# endif
// (*) check against 'termination'
} while( token_p->_id != TKN_TERMINATION );
QUEX_NAME(destruct)(&qlex);
return 0;
}
static rsexp read_simple_datum (Reader* reader)
{
RToken* token;
rsexp datum;
rcstring text;
rsize size;
token = lookahead (reader);
size = QUEX_NAME (strlen) (token->text) + 5;
text = alloca (size);
QUEX_NAME_TOKEN (pretty_char_text) (token, text, size);
switch (token->_id) {
case TKN_TRUE:
datum = R_TRUE;
break;
case TKN_FALSE:
datum = R_FALSE;
break;
case TKN_NUMBER:
datum = r_cstr_to_number (reader->r, text);
break;
case TKN_CHARACTER:
datum = lexeme_to_char (text);
break;
case TKN_STRING:
datum = r_string_new (reader->r, text);
break;
case TKN_IDENTIFIER:
datum = r_intern (reader->r, text);
break;
case TKN_HASH_U8_LP:
datum = read_bytevector (reader);
break;
case TKN_TERMINATION:
datum = R_EOF;
break;
default:
datum = R_FAILURE;
goto exit;
}
consume (reader);
exit:
return datum;
}
int
main(int argc, char** argv)
{
QUEX_TYPE_TOKEN token;
tiny_lexer qlex;
size_t receive_n = (size_t)-1;
size_t BufferSize = 1024;
char buffer[1024];
bool out_f = false;
QUEX_NAME(construct_memory)(&qlex, 0x0, 0, 0x0, 0x0, false);
QUEX_NAME_TOKEN(construct)(&token);
(void)QUEX_NAME(token_p_switch)(&qlex, &token);
while( 1 + 1 == 2 ) {
/* -- Initialize the filling of the fill region */
QUEX_NAME(buffer_fill_region_prepare)(&qlex);
/* -- Call the low lever driver to fill the fill region */
receive_n = messaging_framework_receive_into_buffer_syntax_chunk(QUEX_NAME(buffer_fill_region_begin)(&qlex),
QUEX_NAME(buffer_fill_region_size)(&qlex));
/* -- Inform the buffer about the number of loaded characters NOT NUMBER OF BYTES! */
QUEX_NAME(buffer_fill_region_finish)(&qlex, receive_n);
/* -- Loop until the 'termination' token arrives */
while( 1 + 1 == 2 ) {
const QUEX_TYPE_TOKEN_ID TokenID = QUEX_NAME(receive)(&qlex);
if( TokenID == QUEX_TKN_TERMINATION ) break;
if( TokenID == QUEX_TKN_BYE ) { out_f = true; break; }
printf("Consider: %s \n", QUEX_NAME_TOKEN(get_string)(&token, buffer, BufferSize));
}
if( out_f ) break;
}
QUEX_NAME(destruct)(&qlex);
QUEX_NAME_TOKEN(destruct)(&token);
return 0;
}
int
main(int argc, char** argv)
{
quex_Token* token_p = 0x0;
quex_EasyLexer qlex;
const size_t BufferSize = 1024;
char buffer[1024];
const char* FileName = "example.txt";
QUEX_NAME(construct_file_name)(&qlex, FileName, (void*)0, false);
do {
QUEX_NAME(receive)(&qlex, &token_p);
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
} while( token_p->_id != QUEX_TKN_TERMINATION );
QUEX_NAME(destruct)(&qlex);
return 0;
}
