int tokenizer_next(struct tokenizer* tokenizer)
//@ requires Tokenizer(tokenizer);
//@ ensures Tokenizer(tokenizer);
{
int c;
int token;
string_buffer_clear(tokenizer->buffer);
tokenizer_skip_whitespace(tokenizer);
c = tokenizer_peek(tokenizer);
if ( c == '(' || c == ')' || c == -1 )
{
tokenizer_drop(tokenizer);
token = c;
}
else if ( is_digit(c) )
{
token = tokenizer_eat_number(tokenizer);
}
else if ( is_symbol_char(c) )
{
token = tokenizer_eat_symbol(tokenizer);
}
else
{
tokenizer_drop(tokenizer);
token = 'B'; // bad character
}
tokenizer->lasttoken = token;
return token;
}
void string_buffer_drop_front(struct string_buffer *buffer, int length)
//@ requires string_buffer(buffer, ?bcs) &*& length >= 0;
//@ ensures string_buffer(buffer, _);
{
int length_buffer = string_buffer_get_length(buffer);
if (length >= length_buffer){
string_buffer_clear(buffer);
}else{
char *chars = string_buffer_get_chars(buffer);
struct string_buffer *temp = create_string_buffer();
//@ chars_split(chars, length);
//@ chars_limits(chars);
string_buffer_append_chars(temp, chars+length, length_buffer - length);
//@ string_buffer_merge_chars(buffer);
string_buffer_clear(buffer);
string_buffer_append_string_buffer(buffer, temp);
string_buffer_dispose(temp);
}
}
bool string_buffer_split(struct string_buffer *buffer, char *separator, struct string_buffer *before, struct string_buffer *after)
//@ requires [?f1]string_buffer(buffer, ?bcs) &*& [?f2]string(separator, ?cs) &*& string_buffer(before, _) &*& string_buffer(after, _);
//@ ensures [f1]string_buffer(buffer, bcs) &*& [f2]string(separator, cs) &*& string_buffer(before, _) &*& string_buffer(after, _);
{
int n = strlen(separator);
char *chars = buffer->chars;
int length = buffer->length;
int index = chars_index_of_string(chars, length, separator);
if (index == -1) { return false; }
string_buffer_clear(before);
string_buffer_append_chars(before, chars, index);
//@ chars_join(chars);
string_buffer_clear(after);
//@ chars_limits(chars);
//@ chars_split(chars, index + n);
//@ chars_split(chars + index + n, length - index - n);
string_buffer_append_chars(after, chars + index + n, length - index - n);
return true;
}
/*
* load the lexicon configuration file.
* and load all the valid lexicon from the configuration file.
*
* @param friso friso instance
* @param config friso_config instance
* @param _path dictionary directory
* @param _limitts words length limit
*/
FRISO_API void friso_dic_load_from_ifile(
friso_t friso,
friso_config_t config,
fstring _path,
uint_t _limits )
{
//1.parse the configuration file.
FILE *__stream;
char __chars__[1024], __key__[30], *__line__;
uint_t __length__, i, t;
friso_lex_t lex_t;
string_buffer_t sb;
//get the lexicon configruation file path
sb = new_string_buffer();
string_buffer_append( sb, _path );
string_buffer_append( sb, __FRISO_LEX_IFILE__ );
//printf("%s\n", sb->buffer);
if ( ( __stream = fopen( sb->buffer, "rb" ) ) != NULL )
{
while ( ( __line__ =
file_get_line( __chars__, __stream ) ) != NULL )
{
//comment filter.
if ( __line__[0] == '#' ) continue;
if ( __line__[0] == '\0' ) continue;
__length__ = strlen( __line__ );
//item start
if ( __line__[ __length__ - 1 ] == '[' )
{
//get the type key
for ( i = 0; i < __length__
&& ( __line__[i] == ' ' || __line__[i] == '\t' ); i++ );
for ( t = 0; i < __length__; i++,t++ ) {
if ( __line__[i] == ' '
|| __line__[i] == '\t' || __line__[i] == ':' ) break;
__key__[t] = __line__[i];
}
__key__[t] = '\0';
//get the lexicon type
lex_t = get_lexicon_type_with_constant(__key__);
if ( lex_t == -1 ) continue;
//printf("key=%s, type=%d\n", __key__, lex_t );
while ( ( __line__ = file_get_line( __chars__, __stream ) ) != NULL )
{
//comments filter.
if ( __line__[0] == '#' ) continue;
if ( __line__[0] == '\0' ) continue;
__length__ = strlen( __line__ );
if ( __line__[ __length__ - 1 ] == ']' ) break;
for ( i = 0; i < __length__
&& ( __line__[i] == ' ' || __line__[i] == '\t' ); i++ );
for ( t = 0; i < __length__; i++,t++ ) {
if ( __line__[i] == ' '
|| __line__[i] == '\t' || __line__[i] == ';' ) break;
__key__[t] = __line__[i];
}
__key__[t] = '\0';
//load the lexicon item from the lexicon file.
string_buffer_clear( sb );
string_buffer_append( sb, _path );
string_buffer_append( sb, __key__ );
//printf("key=%s, type=%d\n", __key__, lex_t);
friso_dic_load( friso, config, lex_t, sb->buffer, _limits );
}
}
} //end while
fclose( __stream );
} else {
printf("Warning: Fail to open the lexicon configuration file %s\n", sb->buffer);
}
free_string_buffer(sb);
}
