static int next_token(JSONRD_T *jsonrd, const char *chunk, size_t size) { int i; // reset state if (jsonrd->token.state == TS_FOUND_TOKEN) { str_clear(jsonrd->buf); jsonrd->token.state = TS_FIND_TOKEN; jsonrd->token.type = TOK_ILLEGAL; jsonrd->token.value_bool = false; jsonrd->token.num_state = NS_BEGIN; jsonrd->token.value_number = 0; jsonrd->token.str_state = SS_NORMAL; str_clear(jsonrd->token.value_string); } i = 0; if (jsonrd->token.state == TS_FIND_TOKEN) { // skip whitespace for (; i < size; i++) { if (!isspace(chunk[i])) break; } if (i == size) { // only whitespace jsonrd->token.state = TS_FIND_TOKEN; return size; } // try to determine the type of token by the first letter jsonrd->token.type = guess_token(chunk[i]); jsonrd->token.state = TS_IN_PROGRESS; switch (jsonrd->token.type) { case TOK_TRUE: jsonrd->token.value_bool = true; case TOK_FALSE: case TOK_NULL: break; case TOK_STRING: jsonrd->token.str_state = SS_NORMAL; i++; break; case TOK_NUMBER: jsonrd->token.num_state = NS_BEGIN; break; default: jsonrd->token.state = TS_FOUND_TOKEN; return i+1; } } // in progress assert(jsonrd->token.state = TS_IN_PROGRESS); switch (jsonrd->token.type) { case TOK_TRUE: return process_keyword_token(jsonrd, chunk, size, i, "true", 4); case TOK_FALSE: return process_keyword_token(jsonrd, chunk, size, i, "false", 5); case TOK_NULL: return process_keyword_token(jsonrd, chunk, size, i, "null", 4); case TOK_NUMBER: return process_number_token(jsonrd, chunk, size, i); case TOK_STRING: return process_string_token(jsonrd, chunk, size, i); default: die("Bad state"); return size; // should not get here } }
Token InputReader::get_next_token() { skip_white_spaces(); Token token, raw_token; std::string qstr; std::string value; get_raw_token(raw_token); switch(raw_token.type_) { case null_token: token.type_ = null_token; // this means no more tokens left break; case digit_token: case negative_token: float_t n_value; input_stream_.unget(); if(!read_number(n_value)) { std::cerr << "fatal error: failed while reading a number" << std::endl; token.type_ = error_token; } else { token.type_ = number_token; token.dvalue_ = n_value; } // if-else break; case object_begin_token: token.type_ = object_begin_token; structure_stack_.push(object_begin_token); break; case object_end_token: token.type_ = object_end_token; if(structure_stack_.top() != object_begin_token) { std::cerr << "fatal error: mismatched object encapsulators" << std::endl; token.type_ = error_token; } else { structure_stack_.pop(); } // if-else break; case array_begin_token: token.type_ = array_begin_token; structure_stack_.push(array_begin_token); break; case array_end_token: token.type_ = array_end_token; if(structure_stack_.top() != array_begin_token) { std::cerr << "fatal error: mismatched array encapsulators" << std::endl; token.type_ = error_token; } else { structure_stack_.pop(); } // if-else break; case string_begin_end_token: // will always be begin since // end will be removed while reading //the whole string earlier if(!read_quoted_string(qstr)) { std::cerr << "fatal error: premature EOF reached while reading string" << std::endl; token.type_ = error_token; } else { token.type_ = string_token; token.svalue_ = qstr; } // if-else break; case character_token: input_stream_.unget(); read_keyword(value); token.type_ = process_keyword_token(value); if(token.type_ == error_token) { std::cerr << "fatal error: unknown keyword '" << value << "'" << std::endl; } // if token.svalue_ = value; break; case assignment_token: token.type_ = assignment_token; break; case separator_token: token.type_ = separator_token; break; case comment_token: skip_comments(); token.type_ = comment_token; break; default: std::cerr << "fatal error: unknown token" << std::endl; token.type_ = error_token; } // switch return token; } // InputReader::get_next_token()