static int next_token(JSONRD_T *jsonrd, const char *chunk, size_t size) {
int i;
// reset state
if (jsonrd->token.state == TS_FOUND_TOKEN) {
str_clear(jsonrd->buf);
jsonrd->token.state = TS_FIND_TOKEN;
jsonrd->token.type = TOK_ILLEGAL;
jsonrd->token.value_bool = false;
jsonrd->token.num_state = NS_BEGIN;
jsonrd->token.value_number = 0;
jsonrd->token.str_state = SS_NORMAL;
str_clear(jsonrd->token.value_string);
}
i = 0;
if (jsonrd->token.state == TS_FIND_TOKEN) {
// skip whitespace
for (; i < size; i++) {
if (!isspace(chunk[i])) break;
}
if (i == size) { // only whitespace
jsonrd->token.state = TS_FIND_TOKEN;
return size;
}
// try to determine the type of token by the first letter
jsonrd->token.type = guess_token(chunk[i]);
jsonrd->token.state = TS_IN_PROGRESS;
switch (jsonrd->token.type) {
case TOK_TRUE:
jsonrd->token.value_bool = true;
case TOK_FALSE:
case TOK_NULL:
break;
case TOK_STRING:
jsonrd->token.str_state = SS_NORMAL;
i++;
break;
case TOK_NUMBER:
jsonrd->token.num_state = NS_BEGIN;
break;
default:
jsonrd->token.state = TS_FOUND_TOKEN;
return i+1;
}
}
// in progress
assert(jsonrd->token.state = TS_IN_PROGRESS);
switch (jsonrd->token.type) {
case TOK_TRUE:
return process_keyword_token(jsonrd, chunk, size, i, "true", 4);
case TOK_FALSE:
return process_keyword_token(jsonrd, chunk, size, i, "false", 5);
case TOK_NULL:
return process_keyword_token(jsonrd, chunk, size, i, "null", 4);
case TOK_NUMBER:
return process_number_token(jsonrd, chunk, size, i);
case TOK_STRING:
return process_string_token(jsonrd, chunk, size, i);
default:
die("Bad state");
return size; // should not get here
}
}
Token InputReader::get_next_token() {
skip_white_spaces();
Token token, raw_token;
std::string qstr;
std::string value;
get_raw_token(raw_token);
switch(raw_token.type_) {
case null_token:
token.type_ = null_token; // this means no more tokens left
break;
case digit_token:
case negative_token:
float_t n_value;
input_stream_.unget();
if(!read_number(n_value)) {
std::cerr << "fatal error: failed while reading a number" << std::endl;
token.type_ = error_token;
} else {
token.type_ = number_token;
token.dvalue_ = n_value;
} // if-else
break;
case object_begin_token:
token.type_ = object_begin_token;
structure_stack_.push(object_begin_token);
break;
case object_end_token:
token.type_ = object_end_token;
if(structure_stack_.top() != object_begin_token) {
std::cerr << "fatal error: mismatched object encapsulators" << std::endl;
token.type_ = error_token;
} else {
structure_stack_.pop();
} // if-else
break;
case array_begin_token:
token.type_ = array_begin_token;
structure_stack_.push(array_begin_token);
break;
case array_end_token:
token.type_ = array_end_token;
if(structure_stack_.top() != array_begin_token) {
std::cerr << "fatal error: mismatched array encapsulators" << std::endl;
token.type_ = error_token;
} else {
structure_stack_.pop();
} // if-else
break;
case string_begin_end_token: // will always be begin since
// end will be removed while reading
//the whole string earlier
if(!read_quoted_string(qstr)) {
std::cerr << "fatal error: premature EOF reached while reading string" << std::endl;
token.type_ = error_token;
} else {
token.type_ = string_token;
token.svalue_ = qstr;
} // if-else
break;
case character_token:
input_stream_.unget();
read_keyword(value);
token.type_ = process_keyword_token(value);
if(token.type_ == error_token) {
std::cerr << "fatal error: unknown keyword '" << value << "'" << std::endl;
} // if
token.svalue_ = value;
break;
case assignment_token:
token.type_ = assignment_token;
break;
case separator_token:
token.type_ = separator_token;
break;
case comment_token:
skip_comments();
token.type_ = comment_token;
break;
default:
std::cerr << "fatal error: unknown token" << std::endl;
token.type_ = error_token;
} // switch
return token;
} // InputReader::get_next_token()
