int tokenizer_next(struct tokenizer* tokenizer) //@ requires Tokenizer(tokenizer); //@ ensures Tokenizer(tokenizer); { int c; int token; string_buffer_clear(tokenizer->buffer); tokenizer_skip_whitespace(tokenizer); c = tokenizer_peek(tokenizer); if ( c == '(' || c == ')' || c == -1 ) { tokenizer_drop(tokenizer); token = c; } else if ( is_digit(c) ) { token = tokenizer_eat_number(tokenizer); } else if ( is_symbol_char(c) ) { token = tokenizer_eat_symbol(tokenizer); } else { tokenizer_drop(tokenizer); token = 'B'; // bad character } tokenizer->lasttoken = token; return token; }
void string_buffer_drop_front(struct string_buffer *buffer, int length) //@ requires string_buffer(buffer, ?bcs) &*& length >= 0; //@ ensures string_buffer(buffer, _); { int length_buffer = string_buffer_get_length(buffer); if (length >= length_buffer){ string_buffer_clear(buffer); }else{ char *chars = string_buffer_get_chars(buffer); struct string_buffer *temp = create_string_buffer(); //@ chars_split(chars, length); //@ chars_limits(chars); string_buffer_append_chars(temp, chars+length, length_buffer - length); //@ string_buffer_merge_chars(buffer); string_buffer_clear(buffer); string_buffer_append_string_buffer(buffer, temp); string_buffer_dispose(temp); } }
bool string_buffer_split(struct string_buffer *buffer, char *separator, struct string_buffer *before, struct string_buffer *after) //@ requires [?f1]string_buffer(buffer, ?bcs) &*& [?f2]string(separator, ?cs) &*& string_buffer(before, _) &*& string_buffer(after, _); //@ ensures [f1]string_buffer(buffer, bcs) &*& [f2]string(separator, cs) &*& string_buffer(before, _) &*& string_buffer(after, _); { int n = strlen(separator); char *chars = buffer->chars; int length = buffer->length; int index = chars_index_of_string(chars, length, separator); if (index == -1) { return false; } string_buffer_clear(before); string_buffer_append_chars(before, chars, index); //@ chars_join(chars); string_buffer_clear(after); //@ chars_limits(chars); //@ chars_split(chars, index + n); //@ chars_split(chars + index + n, length - index - n); string_buffer_append_chars(after, chars + index + n, length - index - n); return true; }
/* * load the lexicon configuration file. * and load all the valid lexicon from the configuration file. * * @param friso friso instance * @param config friso_config instance * @param _path dictionary directory * @param _limitts words length limit */ FRISO_API void friso_dic_load_from_ifile( friso_t friso, friso_config_t config, fstring _path, uint_t _limits ) { //1.parse the configuration file. FILE *__stream; char __chars__[1024], __key__[30], *__line__; uint_t __length__, i, t; friso_lex_t lex_t; string_buffer_t sb; //get the lexicon configruation file path sb = new_string_buffer(); string_buffer_append( sb, _path ); string_buffer_append( sb, __FRISO_LEX_IFILE__ ); //printf("%s\n", sb->buffer); if ( ( __stream = fopen( sb->buffer, "rb" ) ) != NULL ) { while ( ( __line__ = file_get_line( __chars__, __stream ) ) != NULL ) { //comment filter. if ( __line__[0] == '#' ) continue; if ( __line__[0] == '\0' ) continue; __length__ = strlen( __line__ ); //item start if ( __line__[ __length__ - 1 ] == '[' ) { //get the type key for ( i = 0; i < __length__ && ( __line__[i] == ' ' || __line__[i] == '\t' ); i++ ); for ( t = 0; i < __length__; i++,t++ ) { if ( __line__[i] == ' ' || __line__[i] == '\t' || __line__[i] == ':' ) break; __key__[t] = __line__[i]; } __key__[t] = '\0'; //get the lexicon type lex_t = get_lexicon_type_with_constant(__key__); if ( lex_t == -1 ) continue; //printf("key=%s, type=%d\n", __key__, lex_t ); while ( ( __line__ = file_get_line( __chars__, __stream ) ) != NULL ) { //comments filter. if ( __line__[0] == '#' ) continue; if ( __line__[0] == '\0' ) continue; __length__ = strlen( __line__ ); if ( __line__[ __length__ - 1 ] == ']' ) break; for ( i = 0; i < __length__ && ( __line__[i] == ' ' || __line__[i] == '\t' ); i++ ); for ( t = 0; i < __length__; i++,t++ ) { if ( __line__[i] == ' ' || __line__[i] == '\t' || __line__[i] == ';' ) break; __key__[t] = __line__[i]; } __key__[t] = '\0'; //load the lexicon item from the lexicon file. string_buffer_clear( sb ); string_buffer_append( sb, _path ); string_buffer_append( sb, __key__ ); //printf("key=%s, type=%d\n", __key__, lex_t); friso_dic_load( friso, config, lex_t, sb->buffer, _limits ); } } } //end while fclose( __stream ); } else { printf("Warning: Fail to open the lexicon configuration file %s\n", sb->buffer); } free_string_buffer(sb); }