int
main(int argc, char** argv)
{
quex_Token* token_p;
quex_ISO8859_7_Lex qlex;
size_t BufferSize = 1024;
char buffer[1024];
QUEX_NAME(construct_file_name)(&qlex, "example-iso8859-7.txt", 0x0, false);
// (*) loop until the 'termination' token arrives
do {
// (*) get next token from the token stream
QUEX_NAME(receive)(&qlex, &token_p);
/* (*) print out token information
* 'get_string' automagically converts codec bytes into utf8 */
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
# if 0
cout << "\t\t plain bytes: ";
for(QUEX_TYPE_CHARACTER* iterator = (uint8_t*)tmp.c_str(); *iterator ; ++iterator) {
printf("%02X.", (int)*iterator);
}
# endif
// (*) check against 'termination'
} while( token_p->_id != TKN_TERMINATION );
QUEX_NAME(destruct)(&qlex);
return 0;
}
int
main(int argc, char** argv)
{
QUEX_TYPE_TOKEN* token_p = 0x0;
size_t BufferSize = 1024;
char buffer[1024];
UTF8Lex qlex;
QUEX_NAME(construct_file_name)(&qlex, "example-utf8.txt", 0x0, false);
// (*) loop until the 'termination' token arrives
do {
// (*) get next token from the token stream
QUEX_NAME(receive)(&qlex, &token_p);
/* (*) print out token information
* 'get_string' automagically converts codec bytes into utf8 */
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
// (*) check against 'termination'
} while( token_p->_id != TKN_TERMINATION );
QUEX_NAME(destruct)(&qlex);
return 0;
}
int
main(int argc, char** argv)
{
# ifdef PRINT_TOKEN
const size_t BufferSize = 1024;
char buffer[1024];
# endif
quex_Token* token_p = 0x0;
long token_n = 0;
quex_Simple qlex;
# ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
QUEX_TYPE_TOKEN_ID token_id = (QUEX_TYPE_TOKEN_ID)0x0;
# endif
const char* file_name = argc > 1 ? argv[1] : "example.txt";
QUEX_NAME(from_file_name)(&qlex, file_name, CHARACTER_ENCODING_NAME);
printf(",------------------------------------------------------------------------------------\n");
printf("| [START]\n");
fflush(stdout);
fflush(stderr);
/* Loop until the 'termination' token arrives */
token_n = 0;
# ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
token_p = QUEX_NAME(token_p)(&qlex);
# endif
do {
/* Get next token from the token stream */
# ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
token_id = QUEX_NAME(receive)(&qlex);
# else
QUEX_NAME(receive)(&qlex, &token_p);
# endif
# ifdef PRINT_LINE_COLUMN_NUMBER
printf("(%i, %i) \t", (int)token_p->_line_n, (int)token_p->_column_n);
# endif
/* Print out token information */
fflush(stderr);
# ifdef PRINT_TOKEN
if( token_n >= PRINT_TOKEN_FIRST_NUMBER ) {
printf("%s", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
printf("\n");
}
# else
printf("%s", QUEX_NAME_TOKEN(map_id_to_name)(token_p->_id));
printf("\n");
# endif
fflush(stdout);
++token_n;
/* Check against 'termination' */
# ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE
} while( token_id != QUEX_TKN_TERMINATION );
# else
} while( token_p->_id != QUEX_TKN_TERMINATION );
int
main(int argc, char** argv)
{
quex_Token* token_p = 0x0;
size_t number_of_tokens = 0;
EasyLexer qlex;
# ifdef PRINT_TOKEN
const size_t BufferSize = 1024;
char buffer[1024];
# endif
const char* FileName = (argc == 1) ? "example.txt" : argv[1];
QUEX_NAME(construct_file_name)(&qlex, FileName, ENCODING_NAME, false);
/* Alternatives:
* QUEX_NAME(construct_memory)(&qlex, MemoryBegin, MemorySize,
* CharacterEncodingName (default 0x0),
* ByteOrderReversionF (default false));
* QUEX_NAME(construct_FILE)(&qlex, FILE_handle,
* CharacterEncodingName (default 0x0),
* ByteOrderReversionF (default false)); */
printf(",-----------------------------------------------------------------\n");
printf("| [START]\n");
/* Loop until the 'termination' token arrives */
do {
/* Get next token from the token stream */
QUEX_NAME(receive)(&qlex, &token_p);
/* Print out token information */
# ifdef PRINT_LINE_COLUMN_NUMBER
printf("(%i, %i) \t", (int)token_p->_line_n, (int)token_p->_column_n);
# endif
# ifdef PRINT_TOKEN
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
# else
printf("%s\n", QUEX_NAME_TOKEN(map_id_to_name)(token_p->_id));
# endif
# ifdef SPECIAL_ACTION
SPECIAL_ACTION(&qlex, &my_token);
# endif
++number_of_tokens;
/* Check against 'termination' */
} while( token_p->_id != QUEX_TKN_TERMINATION );
printf("| [END] number of token = %i\n", number_of_tokens);
printf("`-----------------------------------------------------------------\n");
QUEX_NAME(destruct)(&qlex);
return 0;
}
static RToken* next_token (Reader* reader)
{
rtokenid id;
RToken* token;
do
id = QUEX_NAME (receive) (&reader->lexer);
while (TKN_TERMINATION == id && feed_lexer (reader));
token = QUEX_NAME (token_p) (&reader->lexer);
reader->current_line = token->_line_n;
reader->current_column = token->_column_n;
return token;
}
int main (int argc, char* argv[])
{
my_scanner lexer;
QUEX_NAME (construct_file_name) (&lexer, "test.in", NULL, 0);
while (1) {
QUEX_TYPE_TOKEN* token = NULL;
QUEX_NAME (receive) (&lexer, &token);
if (QUEX_TKN_TERMINATION == token->_id)
break;
printf ("id=%d\n", token->_id);
}
return 0;
}
static rsexp read_simple_datum (Reader* reader)
{
RToken* token;
rsexp datum;
rcstring text;
rsize size;
token = lookahead (reader);
size = QUEX_NAME (strlen) (token->text) + 5;
text = alloca (size);
QUEX_NAME_TOKEN (pretty_char_text) (token, text, size);
switch (token->_id) {
case TKN_TRUE:
datum = R_TRUE;
break;
case TKN_FALSE:
datum = R_FALSE;
break;
case TKN_NUMBER:
datum = r_cstr_to_number (reader->r, text);
break;
case TKN_CHARACTER:
datum = lexeme_to_char (text);
break;
case TKN_STRING:
datum = r_string_new (reader->r, text);
break;
case TKN_IDENTIFIER:
datum = r_intern (reader->r, text);
break;
case TKN_HASH_U8_LP:
datum = read_bytevector (reader);
break;
case TKN_TERMINATION:
datum = R_EOF;
break;
default:
datum = R_FAILURE;
goto exit;
}
consume (reader);
exit:
return datum;
}
int main(int argc, char** argv)
{
quex_Calc_lexer qlex;
QUEX_NAME(construct_file_name)(&qlex, argc == 1 ? "example.txt" : argv[1], 0x0, false);
printf("Calculator Example Application\n");
printf("Contributed by: Marco Antonelli (date: 09y11m7d)\n\n");
int ret = Calc_yyparse(&qlex);
if (ret!=0)
{
printf("Some error in yyparse\n");
return ret;
}
QUEX_NAME(destruct)(&qlex);
return 0;
}
static void
test_file(bool BinaryF, const char* FileStem)
{
QUEX_NAME(Buffer) buffer;
/* With 'LexatomLoader_Plain()' no conversion takes place. Thus, the file
* containing the REFERENCE data and the INPUT file are the SAME. */
const char* file_name = find_reference(FileStem);
FILE* fh = fopen(file_name, "rb");
QUEX_NAME(ByteLoader)* byte_loader = QUEX_NAME(ByteLoader_FILE_new)(fh, true);
QUEX_NAME(LexatomLoader)* filler;
const size_t MemorySize = true ? 5 : 16;
QUEX_TYPE_LEXATOM memory[MemorySize];
if( ! fh ) {
printf("Failed to open '%s'.", file_name);
hwut_verify(false);
}
byte_loader->binary_mode_f = BinaryF;
filler = QUEX_NAME(LexatomLoader_Plain_new)(byte_loader);
QUEX_NAME(Buffer_construct)(&buffer, filler, &memory[0], MemorySize, 0, E_Ownership_EXTERNAL);
/* REFERENCE file and INPUT file are the SAME. */
hwut_verify(basic_functionality(&buffer, file_name));
filler->delete_self(filler);
}
int
main(int argc, char** argv)
{
quex_Token* token_p = 0x0;
quex_EasyLexer qlex;
const size_t BufferSize = 1024;
char buffer[1024];
const char* FileName = "example.txt";
QUEX_NAME(construct_file_name)(&qlex, FileName, (void*)0, false);
do {
QUEX_NAME(receive)(&qlex, &token_p);
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
} while( token_p->_id != QUEX_TKN_TERMINATION );
QUEX_NAME(destruct)(&qlex);
return 0;
}
static ptrdiff_t
walk_backward(ptrdiff_t ReadPDelta, ptrdiff_t LexemeStartPDelta)
/* Walk through file by incrementing the 'read_p' by 'ReadPDelta' until the
* end of file is reached. The 'lexeme_start_p' remains in a constant distance
* to 'read_p' given by 'LexemeStartPDelta'. */
{
QUEX_NAME(Buffer) buffer;
QUEX_NAME(ByteLoader_Memory) loader;
QUEX_NAME(LexatomLoader)* filler;
int count = 0;
QUEX_TYPE_LEXATOM memory[5];
const int MemorySize = 5;
QUEX_NAME(ByteLoader_Memory_construct)(&loader,
(uint8_t*)&PseudoFile[0],
(const uint8_t*)&PseudoFile[sizeof(PseudoFile)/sizeof(PseudoFile[0])]);
filler = QUEX_NAME(LexatomLoader_new)(&loader.base,
(QUEX_NAME(Converter)*)0, 0);
QUEX_NAME(Buffer_construct)(&buffer, filler,
&memory[0], MemorySize,
(QUEX_TYPE_LEXATOM*)0, E_Ownership_EXTERNAL);
load_forward_until_eos(&buffer);
for(buffer._read_p = &buffer.input.end_p[-1];
buffer._read_p >= &buffer._memory._front[1];
buffer._read_p -= ReadPDelta) {
buffer._lexeme_start_p = buffer._read_p + LexemeStartPDelta;
if( buffer._lexeme_start_p >= &buffer._memory._back[-1] ) {
buffer._lexeme_start_p = &buffer._memory._back[-2];
}
if( buffer._lexeme_start_p <= buffer._memory._front ) {
buffer._lexeme_start_p = &buffer._memory._front[1];
}
count += test_load_backward(&buffer);
}
return count;
}
static rintw feed_lexer (Reader* reader)
{
RState* r;
rcstring begin;
rcstring line;
rintw size;
rintw len;
QUEX_NAME (buffer_fill_region_prepare) (&reader->lexer);
begin = r_cast (rcstring,
QUEX_NAME (buffer_fill_region_begin)
(&reader->lexer));
r = reader->r;
size = QUEX_NAME (buffer_fill_region_size) (&reader->lexer);
line = r_port_gets (r, reader->input_port, begin, size);
len = line ? strlen (line) : 0;
QUEX_NAME (buffer_fill_region_finish) (&reader->lexer, len);
return len;
}
int
main(int argc, char** argv)
{
QUEX_TYPE_TOKEN token;
tiny_lexer qlex;
size_t receive_n = (size_t)-1;
size_t BufferSize = 1024;
char buffer[1024];
bool out_f = false;
QUEX_NAME(construct_memory)(&qlex, 0x0, 0, 0x0, 0x0, false);
QUEX_NAME_TOKEN(construct)(&token);
(void)QUEX_NAME(token_p_switch)(&qlex, &token);
while( 1 + 1 == 2 ) {
/* -- Initialize the filling of the fill region */
QUEX_NAME(buffer_fill_region_prepare)(&qlex);
/* -- Call the low lever driver to fill the fill region */
receive_n = messaging_framework_receive_into_buffer_syntax_chunk(QUEX_NAME(buffer_fill_region_begin)(&qlex),
QUEX_NAME(buffer_fill_region_size)(&qlex));
/* -- Inform the buffer about the number of loaded characters NOT NUMBER OF BYTES! */
QUEX_NAME(buffer_fill_region_finish)(&qlex, receive_n);
/* -- Loop until the 'termination' token arrives */
while( 1 + 1 == 2 ) {
const QUEX_TYPE_TOKEN_ID TokenID = QUEX_NAME(receive)(&qlex);
if( TokenID == QUEX_TKN_TERMINATION ) break;
if( TokenID == QUEX_TKN_BYE ) { out_f = true; break; }
printf("Consider: %s \n", QUEX_NAME_TOKEN(get_string)(&token, buffer, BufferSize));
}
if( out_f ) break;
}
QUEX_NAME(destruct)(&qlex);
QUEX_NAME_TOKEN(destruct)(&token);
return 0;
}
* before the load procedure. That is, they need. */
/*
<<hwut-iterator: G>>
------------------------------------------------------------------------
#include <stdint.h>
------------------------------------------------------------------------
ptrdiff_t read_p_delta; ptrdiff_t lexeme_start_p_delta;
[ 1, 2, 3 ]; |0:6|;
------------------------------------------------------------------------
*/
#include "commonly_pasted.c"
#include <backward-gen.h>
static ptrdiff_t test_load_backward(QUEX_NAME(Buffer)* buffer);
static ptrdiff_t walk_backward(ptrdiff_t ReadPDelta, ptrdiff_t LexemeStartPDelta);
static void load_forward_until_eos(QUEX_NAME(Buffer)* me);
int
main(int argc, char**argv)
{
G_t it;
int count = 0;
if( argc > 1 && strcmp(argv[1], "--hwut-info") == 0 ) {
printf("Buffer_load_backward: (BPC=%i, FB=%i);\n",
sizeof(QUEX_TYPE_LEXATOM),
(int)QUEX_SETTING_BUFFER_MIN_FALLBACK_N);
return 0;
}
static void
test_file(E_ConverterTestType CTT, const char* Codec, bool LinearF, bool ClueLessStomachF, const char* FileName, const char* FileStem)
{
QUEX_NAME(Buffer) buffer;
QUEX_NAME(Converter)* converter;
/* With 'LexatomLoader_Plain()' no conversion takes place. Thus, the file
* containing the REFERENCE data and the INPUT file are the SAME. */
const char* ref_file_name = find_reference(FileStem);
# if 0
QUEX_NAME(ByteLoader)* byte_loader = QUEX_NAME(ByteLoader_FILE_new_from_file_name)(FileName);
# else
QUEX_NAME(ByteLoader)* byte_loader = QUEX_NAME(ByteLoader_Memory_new_from_file_name)(FileName);
# endif
const size_t MemorySize = true ? 5 : 16;
QUEX_TYPE_LEXATOM memory[MemorySize];
QUEX_NAME(LexatomLoader)* filler;
switch( CTT ) {
case TEST_ICU: converter = QUEX_NAME(Converter_ICU_new)(Codec, 0); break;
case TEST_ICONV: converter = QUEX_NAME(Converter_IConv_new)(Codec, 0); break;
default: __quex_assert(false);
}
__quex_assert(converter);
if( ClueLessStomachF ) {
converter->stomach_byte_n = clueless_stomach_byte_n;
}
if( ! byte_loader ) {
printf("Failed to open '%s'.", FileName);
hwut_verify(false);
}
filler = QUEX_NAME(LexatomLoader_Converter_new)(byte_loader, converter, 7);
/* If file was not opened in binary mode no converter filler is created! */
__quex_assert(filler);
QUEX_NAME(Buffer_construct)(&buffer, filler, &memory[0], MemorySize, 0,
E_Ownership_EXTERNAL);
if( LinearF ) {
__quex_assert(filler->byte_n_per_lexatom != -1);
__quex_assert(byte_loader->binary_mode_f);
}
else {
__quex_assert(filler->byte_n_per_lexatom == -1);
}
/* REFERENCE file and INPUT file are the SAME. */
hwut_verify(basic_functionality(&buffer, ref_file_name));
filler->delete_self(filler);
byte_loader->delete_self(byte_loader);
converter->delete_self(converter);
}
* (C) Frank-Rene Schaefer */
#include <hwut_unit.h>
#include <basic_functionality.h>
#include <quex/code_base/buffer/Buffer.i>
#include <quex/code_base/buffer/lexatoms/converter/iconv/Converter_IConv>
#include <quex/code_base/buffer/lexatoms/converter/iconv/Converter_IConv.i>
#include <quex/code_base/buffer/bytes/ByteLoader_Memory>
#include <quex/code_base/buffer/bytes/ByteLoader_Memory.i>
// #include <quex/code_base/MemoryManager.i>
QUEX_NAMESPACE_MAIN_OPEN
typedef enum { TEST_ICU, TEST_ICONV } E_ConverterTestType;
static void test(E_ConverterTestType CTT, bool LinearF, bool ClueLessStomachF, size_t BPC);
static void test_file(E_ConverterTestType CTT, const char* Codec, bool LinearF, bool ClueLessStomachF, const char* FileName, const char* FileStem);
static ptrdiff_t clueless_stomach_byte_n(QUEX_NAME(Converter)* me);
QUEX_NAMESPACE_MAIN_CLOSE
int
main(int argc, char** argv)
{
const size_t BPC = sizeof(QUEX_TYPE_LEXATOM);
if( argc > 1 && strcmp(argv[1], "--hwut-info") == 0 ) {
printf("Buffer Tell&Seek: LexatomLoader_Converter_IConv (BPC=%i, FALLBACK=%i);\n",
BPC, QUEX_SETTING_BUFFER_MIN_FALLBACK_N);
printf("CHOICES: ICU-linear, ICU-stepping,\n"
" IConv-linear, IConv-stepping,\n"
" IConv-stepping-cls, ICU-stepping-cls;\n"
"SAME;\n");
int
main(int argc, char** argv)
{
QUEX_TYPE_TOKEN token_bank[2];
QUEX_TYPE_TOKEN* prev_token;
quex_tiny_lexer qlex;
size_t BufferSize = 1024;
char buffer[1024];
QUEX_TYPE_CHARACTER* prev_lexeme_start_p = 0x0;
size_t receive_n = (size_t)-1;
QUEX_TYPE_TOKEN_ID token_id = 0;
QUEX_NAME(construct_memory)(&qlex, 0x0, 0x0, 0, 0x0, false);
/* -- initialize the token pointers */
QUEX_NAME_TOKEN(construct)(&token_bank[0]);
QUEX_NAME_TOKEN(construct)(&token_bank[1]);
token_bank[0]._id = QUEX_TKN_TERMINATION;
prev_token = &(token_bank[1]);
QUEX_NAME(token_p_switch)(&qlex, &token_bank[0]);
while( 1 + 1 == 2 ) {
/* -- Initialize the filling of the fill region */
QUEX_NAME(buffer_fill_region_prepare)(&qlex);
/* -- Call the low lever driver to fill the fill region */
receive_n = messaging_framework_receive_into_buffer(QUEX_NAME(buffer_fill_region_begin)(&qlex),
QUEX_NAME(buffer_fill_region_size)(&qlex));
/* -- Inform the buffer about the number of loaded characters NOT NUMBER OF BYTES! */
QUEX_NAME(buffer_fill_region_finish)(&qlex, receive_n);
/* -- Loop until the 'termination' token arrives */
token_id = 0;
while( 1 + 1 == 2 ) {
prev_lexeme_start_p = QUEX_NAME(buffer_lexeme_start_pointer_get)(&qlex);
/* Let the previous token be the current token of the previous run. */
prev_token = QUEX_NAME(token_p_switch)(&qlex, prev_token);
token_id = QUEX_NAME(receive)(&qlex);
if( token_id == QUEX_TKN_TERMINATION || token_id == QUEX_TKN_BYE )
break;
if( prev_token->_id != QUEX_TKN_TERMINATION )
printf("Consider: %s\n", QUEX_NAME_TOKEN(get_string)(prev_token, buffer, BufferSize));
}
if( token_id == QUEX_TKN_BYE ) break;
QUEX_NAME(buffer_input_pointer_set)(&qlex, prev_lexeme_start_p);
}
QUEX_NAME(destruct)(&qlex);
QUEX_NAME_TOKEN(destruct)(&token_bank[0]);
QUEX_NAME_TOKEN(destruct)(&token_bank[1]);
return 0;
}
