JsonToken JsonScanner::getNextToken() { char c; std::string token; JsonTypes type; ScannerState state = ScannerState::START; c = m_reader->getNextChar(); while ( isBlankOrNewline( c ) ) { c = m_reader->getNextChar(); } if( c == '\0' ) { return JsonToken( JsonTypes::END_OF_STREAM, token ); } if( c == '{' ) { token.push_back( c ); return JsonToken( JsonTypes::OPEN_BRACE, token ); } else if( c == '}' ) { token.push_back( c ); return JsonToken( JsonTypes::CLOSE_BRACE, token ); } else if( c == '[' ) { token.push_back( c ); return JsonToken( JsonTypes::OPEN_BRACKET, token ); } else if( c == ']' ) { token.push_back( c ); return JsonToken( JsonTypes::CLOSE_BRACKET, token ); } else if( c == ',' ) { token.push_back( c ); return JsonToken( JsonTypes::COMMA, token ); } else if( c == ':' ) { token.push_back( c ); return JsonToken( JsonTypes::COLON, token ); } else if( isdigit( c ) || c == '-' || c == '.' ) { if( c == '.' ) { state = ScannerState::REAL; type = JsonTypes::REAL; } else { state = ScannerState::INTEGER; type = JsonTypes::INTEGER; } token.push_back( c ); return readNumberToken( state, type, token ); } else if( c == 't' || c == 'f' ) { token.push_back( c ); return readBooleanToken( token ); } else if( c == '\"' ) { return readStringToken( token ); } else if( c == 'n' ) { token.push_back( c ); return readNullToken( token ); } else { throw SyntaxException( "error in JsonScanner" ); } }
void CSVParser::readDoubleToken() { while (_curchar!=CHAR_EOF) { if (_curchar==CHAR_QUOTE) { throw CSVParserException(PARSERERROR_PARSER, "CSVParser::readDoubleToken", &_token, this, "Unexpected quote"); } else if (_curchar==CHAR_NEWLINE || _curchar==_sepchar) { break; } else if (_numeralset.contains(_curchar)) { _token.addChar(_curchar); } else { // (_curchar==CHAR_DECIMALPOINT) // the token should've already contained a decimal point, so it must be a string. _token.setType(StringToken); readStringToken(); return; } nextChar(); } }
void CSVParser::readToken() { switch (_token.getType()) { case QuotedStringToken: readQuotedStringToken(); nextChar(); break; case StringToken: readStringToken(); break; case NumberToken: nextChar(); readNumberToken(); break; case DoubleToken: nextChar(); readDoubleToken(); break; case SeparatorToken: nextChar(); break; case EOLToken: nextChar(); break; case EOFToken: // Do nothing break; default: throw CSVParserException(PARSERERROR_PARSER, "CSVParser::readToken", NULL, this, "Unhandled token: %s", __csv_tokenName(_token)); break; } // Special resolve when Number and Double tokens only contain signs or periods switch (_token.getType()) { case NumberToken: if (_token.compare(_signset)) _token.setType(StringToken); break; case DoubleToken: if (_token.compare(_signset) || _token.compare(CHAR_DECIMALPOINT)) _token.setType(StringToken); break; default: break; } _handler->handleToken(_token); }
void IniParser::readDoubleToken() { while (_curchar!=CHAR_EOF) { if (_curchar==CHAR_QUOTE) { throw IniParserException(PARSERERROR_PARSER, "IniParser::readDoubleToken", &_token, this, "Unexpected quote"); } else if (_curchar==CHAR_NEWLINE || _whitespaceset.contains(_curchar) || _curchar==CHAR_SEMICOLON /*^^^|| _curchar==CHAR_EQUALSIGN*/) { break; } else { if (_numberset.contains(_curchar)) { _token.addChar(_curchar); } else { // (_curchar==CHAR_DECIMALPOINT) // The token should've already contained a decimal point, so it must be a string. _token.setType(StringToken); readStringToken(); return; } } nextChar(); } }
void IniParser::readToken() { //printf("(IniParser::readToken) token-type:%s line:%d, col:%d\n", __ini_tokenName(_token), _token.getLine(), _token.getColumn()); switch (_token.getType()) { case QuotedStringToken: readQuotedStringToken(); nextChar(); break; case StringToken: readStringToken(); break; case NumberToken: readNumberToken(); break; case DoubleToken: nextChar(); readDoubleToken(); break; case EqualsToken: nextChar(); break; case CommentToken: skipToEOL(); //nextChar(); // Bad to get the next char, as it could be the EOL needed to terminate the current identifier break; case NodeToken: readNodeToken(); nextChar(); break; case EOLToken: nextChar(); break; case EOFToken: // Do nothing break; default: throw IniParserException(PARSERERROR_PARSER, "IniParser::readToken", NULL, this, "Unhandled token: %s", __ini_tokenName(_token)); break; } _handler->handleToken(_token); }
void CSVParser::readNumberToken() { while (_curchar!=CHAR_EOF) { if (_curchar==CHAR_QUOTE) { throw CSVParserException(PARSERERROR_PARSER, "CSVParser::readNumberToken", &_token, this, "Unexpected quote"); } else if (_curchar==CHAR_NEWLINE || _curchar==_sepchar) { break; } else if (_numeralset.contains(_curchar)) { _token.addChar(_curchar); } else if (_curchar==CHAR_DECIMALPOINT) { _token.addChar(_curchar); nextChar(); _token.setType(DoubleToken); readDoubleToken(); return; } else { _token.setType(StringToken); readStringToken(); return; } nextChar(); } }
void IniParser::readNumberToken() { while (_curchar!=CHAR_EOF) { if (_curchar==CHAR_QUOTE) { throw IniParserException(PARSERERROR_PARSER, "IniParser::readNumberToken", &_token, this, "Unexpected quote"); } else if (_curchar==CHAR_NEWLINE || _whitespaceset.contains(_curchar) || _curchar==CHAR_SEMICOLON /*|| _curchar==CHAR_EQUALSIGN*/) { break; } else { if (_numberset.contains(_curchar)) { _token.addChar(_curchar); } else if (_curchar==CHAR_DECIMALPOINT) { _token.addChar(_curchar); nextChar(); _token.setType(DoubleToken); readDoubleToken(); return; } else { _token.setType(StringToken); readStringToken(); return; } } nextChar(); } }