int tokenizer_next(struct tokenizer* tokenizer)
//@ requires Tokenizer(tokenizer);
//@ ensures Tokenizer(tokenizer);
{
int c;
int token;
string_buffer_clear(tokenizer->buffer);
tokenizer_skip_whitespace(tokenizer);
c = tokenizer_peek(tokenizer);
if ( c == '(' || c == ')' || c == -1 )
{
tokenizer_drop(tokenizer);
token = c;
}
else if ( is_digit(c) )
{
token = tokenizer_eat_number(tokenizer);
}
else if ( is_symbol_char(c) )
{
token = tokenizer_eat_symbol(tokenizer);
}
else
{
tokenizer_drop(tokenizer);
token = 'B'; // bad character
}
tokenizer->lasttoken = token;
return token;
}
void tokenizer_skip_whitespace(struct tokenizer* tokenizer)
//@ requires Tokenizer(tokenizer);
//@ ensures Tokenizer(tokenizer);
{
while ( is_whitespace( tokenizer_peek(tokenizer) ) )
//@ invariant Tokenizer(tokenizer);
{
tokenizer_drop(tokenizer);
}
}
static ChessBoolean read_string_token(ChessPgnTokenizer* tokenizer)
{
/* Eat everything, but check for escape chars */
int c;
while ((c = tokenizer_getc(tokenizer)) != EOF)
{
if (c == '"')
return CHESS_TRUE;
if (c == '\\' && tokenizer_peek(tokenizer) == '"')
c = tokenizer_getc(tokenizer);
chess_buffer_append_char(&tokenizer->buffer, c);
}
return CHESS_FALSE; /* Not terminated */
}
int tokenizer_eat_number(struct tokenizer* tokenizer)
//@ requires Tokenizer(tokenizer);
//@ ensures Tokenizer(tokenizer);
{
for (;;)
//@ invariant Tokenizer(tokenizer);
{
int result;
bool isDigit;
result = tokenizer_peek(tokenizer);
isDigit = is_digit(result);
if ( !isDigit ) break;
result = tokenizer_next_char(tokenizer);
string_buffer_append_char(tokenizer->buffer, (char)result);
}
return '0';
}
int tokenizer_eat_symbol(struct tokenizer* tokenizer)
//@ requires Tokenizer(tokenizer);
//@ ensures Tokenizer(tokenizer);
{
for (;;)
//@ invariant Tokenizer(tokenizer);
{
int result;
bool isSymbolChar;
result = tokenizer_peek(tokenizer);
isSymbolChar = is_symbol_char(result);
if (!isSymbolChar) break;
result = tokenizer_next_char(tokenizer);
string_buffer_append_char(tokenizer->buffer, (char)result);
}
return 'S';
}
