int main(int argc, char** argv) { quex_Token* token_p; quex_ISO8859_7_Lex qlex; size_t BufferSize = 1024; char buffer[1024]; QUEX_NAME(construct_file_name)(&qlex, "example-iso8859-7.txt", 0x0, false); // (*) loop until the 'termination' token arrives do { // (*) get next token from the token stream QUEX_NAME(receive)(&qlex, &token_p); /* (*) print out token information * 'get_string' automagically converts codec bytes into utf8 */ printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize)); # if 0 cout << "\t\t plain bytes: "; for(QUEX_TYPE_CHARACTER* iterator = (uint8_t*)tmp.c_str(); *iterator ; ++iterator) { printf("%02X.", (int)*iterator); } # endif // (*) check against 'termination' } while( token_p->_id != TKN_TERMINATION ); QUEX_NAME(destruct)(&qlex); return 0; }
int main(int argc, char** argv) { QUEX_TYPE_TOKEN* token_p = 0x0; size_t BufferSize = 1024; char buffer[1024]; UTF8Lex qlex; QUEX_NAME(construct_file_name)(&qlex, "example-utf8.txt", 0x0, false); // (*) loop until the 'termination' token arrives do { // (*) get next token from the token stream QUEX_NAME(receive)(&qlex, &token_p); /* (*) print out token information * 'get_string' automagically converts codec bytes into utf8 */ printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize)); // (*) check against 'termination' } while( token_p->_id != TKN_TERMINATION ); QUEX_NAME(destruct)(&qlex); return 0; }
int main(int argc, char** argv) { # ifdef PRINT_TOKEN const size_t BufferSize = 1024; char buffer[1024]; # endif quex_Token* token_p = 0x0; long token_n = 0; quex_Simple qlex; # ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE QUEX_TYPE_TOKEN_ID token_id = (QUEX_TYPE_TOKEN_ID)0x0; # endif const char* file_name = argc > 1 ? argv[1] : "example.txt"; QUEX_NAME(from_file_name)(&qlex, file_name, CHARACTER_ENCODING_NAME); printf(",------------------------------------------------------------------------------------\n"); printf("| [START]\n"); fflush(stdout); fflush(stderr); /* Loop until the 'termination' token arrives */ token_n = 0; # ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE token_p = QUEX_NAME(token_p)(&qlex); # endif do { /* Get next token from the token stream */ # ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE token_id = QUEX_NAME(receive)(&qlex); # else QUEX_NAME(receive)(&qlex, &token_p); # endif # ifdef PRINT_LINE_COLUMN_NUMBER printf("(%i, %i) \t", (int)token_p->_line_n, (int)token_p->_column_n); # endif /* Print out token information */ fflush(stderr); # ifdef PRINT_TOKEN if( token_n >= PRINT_TOKEN_FIRST_NUMBER ) { printf("%s", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize)); printf("\n"); } # else printf("%s", QUEX_NAME_TOKEN(map_id_to_name)(token_p->_id)); printf("\n"); # endif fflush(stdout); ++token_n; /* Check against 'termination' */ # ifdef QUEX_OPTION_TOKEN_POLICY_SINGLE } while( token_id != QUEX_TKN_TERMINATION ); # else } while( token_p->_id != QUEX_TKN_TERMINATION );
int main(int argc, char** argv) { quex_Token* token_p = 0x0; size_t number_of_tokens = 0; EasyLexer qlex; # ifdef PRINT_TOKEN const size_t BufferSize = 1024; char buffer[1024]; # endif const char* FileName = (argc == 1) ? "example.txt" : argv[1]; QUEX_NAME(construct_file_name)(&qlex, FileName, ENCODING_NAME, false); /* Alternatives: * QUEX_NAME(construct_memory)(&qlex, MemoryBegin, MemorySize, * CharacterEncodingName (default 0x0), * ByteOrderReversionF (default false)); * QUEX_NAME(construct_FILE)(&qlex, FILE_handle, * CharacterEncodingName (default 0x0), * ByteOrderReversionF (default false)); */ printf(",-----------------------------------------------------------------\n"); printf("| [START]\n"); /* Loop until the 'termination' token arrives */ do { /* Get next token from the token stream */ QUEX_NAME(receive)(&qlex, &token_p); /* Print out token information */ # ifdef PRINT_LINE_COLUMN_NUMBER printf("(%i, %i) \t", (int)token_p->_line_n, (int)token_p->_column_n); # endif # ifdef PRINT_TOKEN printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize)); # else printf("%s\n", QUEX_NAME_TOKEN(map_id_to_name)(token_p->_id)); # endif # ifdef SPECIAL_ACTION SPECIAL_ACTION(&qlex, &my_token); # endif ++number_of_tokens; /* Check against 'termination' */ } while( token_p->_id != QUEX_TKN_TERMINATION ); printf("| [END] number of token = %i\n", number_of_tokens); printf("`-----------------------------------------------------------------\n"); QUEX_NAME(destruct)(&qlex); return 0; }
static RToken* next_token (Reader* reader) { rtokenid id; RToken* token; do id = QUEX_NAME (receive) (&reader->lexer); while (TKN_TERMINATION == id && feed_lexer (reader)); token = QUEX_NAME (token_p) (&reader->lexer); reader->current_line = token->_line_n; reader->current_column = token->_column_n; return token; }
int main (int argc, char* argv[]) { my_scanner lexer; QUEX_NAME (construct_file_name) (&lexer, "test.in", NULL, 0); while (1) { QUEX_TYPE_TOKEN* token = NULL; QUEX_NAME (receive) (&lexer, &token); if (QUEX_TKN_TERMINATION == token->_id) break; printf ("id=%d\n", token->_id); } return 0; }
static rsexp read_simple_datum (Reader* reader) { RToken* token; rsexp datum; rcstring text; rsize size; token = lookahead (reader); size = QUEX_NAME (strlen) (token->text) + 5; text = alloca (size); QUEX_NAME_TOKEN (pretty_char_text) (token, text, size); switch (token->_id) { case TKN_TRUE: datum = R_TRUE; break; case TKN_FALSE: datum = R_FALSE; break; case TKN_NUMBER: datum = r_cstr_to_number (reader->r, text); break; case TKN_CHARACTER: datum = lexeme_to_char (text); break; case TKN_STRING: datum = r_string_new (reader->r, text); break; case TKN_IDENTIFIER: datum = r_intern (reader->r, text); break; case TKN_HASH_U8_LP: datum = read_bytevector (reader); break; case TKN_TERMINATION: datum = R_EOF; break; default: datum = R_FAILURE; goto exit; } consume (reader); exit: return datum; }
int main(int argc, char** argv) { quex_Calc_lexer qlex; QUEX_NAME(construct_file_name)(&qlex, argc == 1 ? "example.txt" : argv[1], 0x0, false); printf("Calculator Example Application\n"); printf("Contributed by: Marco Antonelli (date: 09y11m7d)\n\n"); int ret = Calc_yyparse(&qlex); if (ret!=0) { printf("Some error in yyparse\n"); return ret; } QUEX_NAME(destruct)(&qlex); return 0; }
static void test_file(bool BinaryF, const char* FileStem) { QUEX_NAME(Buffer) buffer; /* With 'LexatomLoader_Plain()' no conversion takes place. Thus, the file * containing the REFERENCE data and the INPUT file are the SAME. */ const char* file_name = find_reference(FileStem); FILE* fh = fopen(file_name, "rb"); QUEX_NAME(ByteLoader)* byte_loader = QUEX_NAME(ByteLoader_FILE_new)(fh, true); QUEX_NAME(LexatomLoader)* filler; const size_t MemorySize = true ? 5 : 16; QUEX_TYPE_LEXATOM memory[MemorySize]; if( ! fh ) { printf("Failed to open '%s'.", file_name); hwut_verify(false); } byte_loader->binary_mode_f = BinaryF; filler = QUEX_NAME(LexatomLoader_Plain_new)(byte_loader); QUEX_NAME(Buffer_construct)(&buffer, filler, &memory[0], MemorySize, 0, E_Ownership_EXTERNAL); /* REFERENCE file and INPUT file are the SAME. */ hwut_verify(basic_functionality(&buffer, file_name)); filler->delete_self(filler); }
int main(int argc, char** argv) { quex_Token* token_p = 0x0; quex_EasyLexer qlex; const size_t BufferSize = 1024; char buffer[1024]; const char* FileName = "example.txt"; QUEX_NAME(construct_file_name)(&qlex, FileName, (void*)0, false); do { QUEX_NAME(receive)(&qlex, &token_p); printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize)); } while( token_p->_id != QUEX_TKN_TERMINATION ); QUEX_NAME(destruct)(&qlex); return 0; }
static ptrdiff_t walk_backward(ptrdiff_t ReadPDelta, ptrdiff_t LexemeStartPDelta) /* Walk through file by incrementing the 'read_p' by 'ReadPDelta' until the * end of file is reached. The 'lexeme_start_p' remains in a constant distance * to 'read_p' given by 'LexemeStartPDelta'. */ { QUEX_NAME(Buffer) buffer; QUEX_NAME(ByteLoader_Memory) loader; QUEX_NAME(LexatomLoader)* filler; int count = 0; QUEX_TYPE_LEXATOM memory[5]; const int MemorySize = 5; QUEX_NAME(ByteLoader_Memory_construct)(&loader, (uint8_t*)&PseudoFile[0], (const uint8_t*)&PseudoFile[sizeof(PseudoFile)/sizeof(PseudoFile[0])]); filler = QUEX_NAME(LexatomLoader_new)(&loader.base, (QUEX_NAME(Converter)*)0, 0); QUEX_NAME(Buffer_construct)(&buffer, filler, &memory[0], MemorySize, (QUEX_TYPE_LEXATOM*)0, E_Ownership_EXTERNAL); load_forward_until_eos(&buffer); for(buffer._read_p = &buffer.input.end_p[-1]; buffer._read_p >= &buffer._memory._front[1]; buffer._read_p -= ReadPDelta) { buffer._lexeme_start_p = buffer._read_p + LexemeStartPDelta; if( buffer._lexeme_start_p >= &buffer._memory._back[-1] ) { buffer._lexeme_start_p = &buffer._memory._back[-2]; } if( buffer._lexeme_start_p <= buffer._memory._front ) { buffer._lexeme_start_p = &buffer._memory._front[1]; } count += test_load_backward(&buffer); } return count; }
static rintw feed_lexer (Reader* reader) { RState* r; rcstring begin; rcstring line; rintw size; rintw len; QUEX_NAME (buffer_fill_region_prepare) (&reader->lexer); begin = r_cast (rcstring, QUEX_NAME (buffer_fill_region_begin) (&reader->lexer)); r = reader->r; size = QUEX_NAME (buffer_fill_region_size) (&reader->lexer); line = r_port_gets (r, reader->input_port, begin, size); len = line ? strlen (line) : 0; QUEX_NAME (buffer_fill_region_finish) (&reader->lexer, len); return len; }
int main(int argc, char** argv) { QUEX_TYPE_TOKEN token; tiny_lexer qlex; size_t receive_n = (size_t)-1; size_t BufferSize = 1024; char buffer[1024]; bool out_f = false; QUEX_NAME(construct_memory)(&qlex, 0x0, 0, 0x0, 0x0, false); QUEX_NAME_TOKEN(construct)(&token); (void)QUEX_NAME(token_p_switch)(&qlex, &token); while( 1 + 1 == 2 ) { /* -- Initialize the filling of the fill region */ QUEX_NAME(buffer_fill_region_prepare)(&qlex); /* -- Call the low lever driver to fill the fill region */ receive_n = messaging_framework_receive_into_buffer_syntax_chunk(QUEX_NAME(buffer_fill_region_begin)(&qlex), QUEX_NAME(buffer_fill_region_size)(&qlex)); /* -- Inform the buffer about the number of loaded characters NOT NUMBER OF BYTES! */ QUEX_NAME(buffer_fill_region_finish)(&qlex, receive_n); /* -- Loop until the 'termination' token arrives */ while( 1 + 1 == 2 ) { const QUEX_TYPE_TOKEN_ID TokenID = QUEX_NAME(receive)(&qlex); if( TokenID == QUEX_TKN_TERMINATION ) break; if( TokenID == QUEX_TKN_BYE ) { out_f = true; break; } printf("Consider: %s \n", QUEX_NAME_TOKEN(get_string)(&token, buffer, BufferSize)); } if( out_f ) break; } QUEX_NAME(destruct)(&qlex); QUEX_NAME_TOKEN(destruct)(&token); return 0; }
* before the load procedure. That is, they need. */ /* <<hwut-iterator: G>> ------------------------------------------------------------------------ #include <stdint.h> ------------------------------------------------------------------------ ptrdiff_t read_p_delta; ptrdiff_t lexeme_start_p_delta; [ 1, 2, 3 ]; |0:6|; ------------------------------------------------------------------------ */ #include "commonly_pasted.c" #include <backward-gen.h> static ptrdiff_t test_load_backward(QUEX_NAME(Buffer)* buffer); static ptrdiff_t walk_backward(ptrdiff_t ReadPDelta, ptrdiff_t LexemeStartPDelta); static void load_forward_until_eos(QUEX_NAME(Buffer)* me); int main(int argc, char**argv) { G_t it; int count = 0; if( argc > 1 && strcmp(argv[1], "--hwut-info") == 0 ) { printf("Buffer_load_backward: (BPC=%i, FB=%i);\n", sizeof(QUEX_TYPE_LEXATOM), (int)QUEX_SETTING_BUFFER_MIN_FALLBACK_N); return 0; }
static void test_file(E_ConverterTestType CTT, const char* Codec, bool LinearF, bool ClueLessStomachF, const char* FileName, const char* FileStem) { QUEX_NAME(Buffer) buffer; QUEX_NAME(Converter)* converter; /* With 'LexatomLoader_Plain()' no conversion takes place. Thus, the file * containing the REFERENCE data and the INPUT file are the SAME. */ const char* ref_file_name = find_reference(FileStem); # if 0 QUEX_NAME(ByteLoader)* byte_loader = QUEX_NAME(ByteLoader_FILE_new_from_file_name)(FileName); # else QUEX_NAME(ByteLoader)* byte_loader = QUEX_NAME(ByteLoader_Memory_new_from_file_name)(FileName); # endif const size_t MemorySize = true ? 5 : 16; QUEX_TYPE_LEXATOM memory[MemorySize]; QUEX_NAME(LexatomLoader)* filler; switch( CTT ) { case TEST_ICU: converter = QUEX_NAME(Converter_ICU_new)(Codec, 0); break; case TEST_ICONV: converter = QUEX_NAME(Converter_IConv_new)(Codec, 0); break; default: __quex_assert(false); } __quex_assert(converter); if( ClueLessStomachF ) { converter->stomach_byte_n = clueless_stomach_byte_n; } if( ! byte_loader ) { printf("Failed to open '%s'.", FileName); hwut_verify(false); } filler = QUEX_NAME(LexatomLoader_Converter_new)(byte_loader, converter, 7); /* If file was not opened in binary mode no converter filler is created! */ __quex_assert(filler); QUEX_NAME(Buffer_construct)(&buffer, filler, &memory[0], MemorySize, 0, E_Ownership_EXTERNAL); if( LinearF ) { __quex_assert(filler->byte_n_per_lexatom != -1); __quex_assert(byte_loader->binary_mode_f); } else { __quex_assert(filler->byte_n_per_lexatom == -1); } /* REFERENCE file and INPUT file are the SAME. */ hwut_verify(basic_functionality(&buffer, ref_file_name)); filler->delete_self(filler); byte_loader->delete_self(byte_loader); converter->delete_self(converter); }
* (C) Frank-Rene Schaefer */ #include <hwut_unit.h> #include <basic_functionality.h> #include <quex/code_base/buffer/Buffer.i> #include <quex/code_base/buffer/lexatoms/converter/iconv/Converter_IConv> #include <quex/code_base/buffer/lexatoms/converter/iconv/Converter_IConv.i> #include <quex/code_base/buffer/bytes/ByteLoader_Memory> #include <quex/code_base/buffer/bytes/ByteLoader_Memory.i> // #include <quex/code_base/MemoryManager.i> QUEX_NAMESPACE_MAIN_OPEN typedef enum { TEST_ICU, TEST_ICONV } E_ConverterTestType; static void test(E_ConverterTestType CTT, bool LinearF, bool ClueLessStomachF, size_t BPC); static void test_file(E_ConverterTestType CTT, const char* Codec, bool LinearF, bool ClueLessStomachF, const char* FileName, const char* FileStem); static ptrdiff_t clueless_stomach_byte_n(QUEX_NAME(Converter)* me); QUEX_NAMESPACE_MAIN_CLOSE int main(int argc, char** argv) { const size_t BPC = sizeof(QUEX_TYPE_LEXATOM); if( argc > 1 && strcmp(argv[1], "--hwut-info") == 0 ) { printf("Buffer Tell&Seek: LexatomLoader_Converter_IConv (BPC=%i, FALLBACK=%i);\n", BPC, QUEX_SETTING_BUFFER_MIN_FALLBACK_N); printf("CHOICES: ICU-linear, ICU-stepping,\n" " IConv-linear, IConv-stepping,\n" " IConv-stepping-cls, ICU-stepping-cls;\n" "SAME;\n");
int main(int argc, char** argv) { QUEX_TYPE_TOKEN token_bank[2]; QUEX_TYPE_TOKEN* prev_token; quex_tiny_lexer qlex; size_t BufferSize = 1024; char buffer[1024]; QUEX_TYPE_CHARACTER* prev_lexeme_start_p = 0x0; size_t receive_n = (size_t)-1; QUEX_TYPE_TOKEN_ID token_id = 0; QUEX_NAME(construct_memory)(&qlex, 0x0, 0x0, 0, 0x0, false); /* -- initialize the token pointers */ QUEX_NAME_TOKEN(construct)(&token_bank[0]); QUEX_NAME_TOKEN(construct)(&token_bank[1]); token_bank[0]._id = QUEX_TKN_TERMINATION; prev_token = &(token_bank[1]); QUEX_NAME(token_p_switch)(&qlex, &token_bank[0]); while( 1 + 1 == 2 ) { /* -- Initialize the filling of the fill region */ QUEX_NAME(buffer_fill_region_prepare)(&qlex); /* -- Call the low lever driver to fill the fill region */ receive_n = messaging_framework_receive_into_buffer(QUEX_NAME(buffer_fill_region_begin)(&qlex), QUEX_NAME(buffer_fill_region_size)(&qlex)); /* -- Inform the buffer about the number of loaded characters NOT NUMBER OF BYTES! */ QUEX_NAME(buffer_fill_region_finish)(&qlex, receive_n); /* -- Loop until the 'termination' token arrives */ token_id = 0; while( 1 + 1 == 2 ) { prev_lexeme_start_p = QUEX_NAME(buffer_lexeme_start_pointer_get)(&qlex); /* Let the previous token be the current token of the previous run. */ prev_token = QUEX_NAME(token_p_switch)(&qlex, prev_token); token_id = QUEX_NAME(receive)(&qlex); if( token_id == QUEX_TKN_TERMINATION || token_id == QUEX_TKN_BYE ) break; if( prev_token->_id != QUEX_TKN_TERMINATION ) printf("Consider: %s\n", QUEX_NAME_TOKEN(get_string)(prev_token, buffer, BufferSize)); } if( token_id == QUEX_TKN_BYE ) break; QUEX_NAME(buffer_input_pointer_set)(&qlex, prev_lexeme_start_p); } QUEX_NAME(destruct)(&qlex); QUEX_NAME_TOKEN(destruct)(&token_bank[0]); QUEX_NAME_TOKEN(destruct)(&token_bank[1]); return 0; }