JsonToken JsonScanner::getNextToken() {
char c;
std::string token;
JsonTypes type;
ScannerState state = ScannerState::START;
c = m_reader->getNextChar();
while ( isBlankOrNewline( c ) ) {
c = m_reader->getNextChar();
}
if( c == '\0' ) {
return JsonToken( JsonTypes::END_OF_STREAM, token );
}
if( c == '{' ) {
token.push_back( c );
return JsonToken( JsonTypes::OPEN_BRACE, token );
} else if( c == '}' ) {
token.push_back( c );
return JsonToken( JsonTypes::CLOSE_BRACE, token );
} else if( c == '[' ) {
token.push_back( c );
return JsonToken( JsonTypes::OPEN_BRACKET, token );
} else if( c == ']' ) {
token.push_back( c );
return JsonToken( JsonTypes::CLOSE_BRACKET, token );
} else if( c == ',' ) {
token.push_back( c );
return JsonToken( JsonTypes::COMMA, token );
} else if( c == ':' ) {
token.push_back( c );
return JsonToken( JsonTypes::COLON, token );
} else if( isdigit( c ) || c == '-' || c == '.' ) {
if( c == '.' ) {
state = ScannerState::REAL;
type = JsonTypes::REAL;
} else {
state = ScannerState::INTEGER;
type = JsonTypes::INTEGER;
}
token.push_back( c );
return readNumberToken( state, type, token );
} else if( c == 't' || c == 'f' ) {
token.push_back( c );
return readBooleanToken( token );
} else if( c == '\"' ) {
return readStringToken( token );
} else if( c == 'n' ) {
token.push_back( c );
return readNullToken( token );
} else {
throw SyntaxException( "error in JsonScanner" );
}
}
void CSVParser::readDoubleToken() {
while (_curchar!=CHAR_EOF) {
if (_curchar==CHAR_QUOTE) {
throw CSVParserException(PARSERERROR_PARSER, "CSVParser::readDoubleToken", &_token, this, "Unexpected quote");
} else if (_curchar==CHAR_NEWLINE || _curchar==_sepchar) {
break;
} else if (_numeralset.contains(_curchar)) {
_token.addChar(_curchar);
} else { // (_curchar==CHAR_DECIMALPOINT)
// the token should've already contained a decimal point, so it must be a string.
_token.setType(StringToken);
readStringToken();
return;
}
nextChar();
}
}
void CSVParser::readToken() {
switch (_token.getType()) {
case QuotedStringToken:
readQuotedStringToken();
nextChar();
break;
case StringToken:
readStringToken();
break;
case NumberToken:
nextChar();
readNumberToken();
break;
case DoubleToken:
nextChar();
readDoubleToken();
break;
case SeparatorToken:
nextChar();
break;
case EOLToken:
nextChar();
break;
case EOFToken:
// Do nothing
break;
default:
throw CSVParserException(PARSERERROR_PARSER, "CSVParser::readToken", NULL, this, "Unhandled token: %s", __csv_tokenName(_token));
break;
}
// Special resolve when Number and Double tokens only contain signs or periods
switch (_token.getType()) {
case NumberToken:
if (_token.compare(_signset))
_token.setType(StringToken);
break;
case DoubleToken:
if (_token.compare(_signset) || _token.compare(CHAR_DECIMALPOINT))
_token.setType(StringToken);
break;
default:
break;
}
_handler->handleToken(_token);
}
void IniParser::readDoubleToken() {
while (_curchar!=CHAR_EOF) {
if (_curchar==CHAR_QUOTE) {
throw IniParserException(PARSERERROR_PARSER, "IniParser::readDoubleToken", &_token, this, "Unexpected quote");
} else if (_curchar==CHAR_NEWLINE || _whitespaceset.contains(_curchar) || _curchar==CHAR_SEMICOLON /*^^^|| _curchar==CHAR_EQUALSIGN*/) {
break;
} else {
if (_numberset.contains(_curchar)) {
_token.addChar(_curchar);
} else { // (_curchar==CHAR_DECIMALPOINT)
// The token should've already contained a decimal point, so it must be a string.
_token.setType(StringToken);
readStringToken();
return;
}
}
nextChar();
}
}
void IniParser::readToken() {
//printf("(IniParser::readToken) token-type:%s line:%d, col:%d\n", __ini_tokenName(_token), _token.getLine(), _token.getColumn());
switch (_token.getType()) {
case QuotedStringToken:
readQuotedStringToken();
nextChar();
break;
case StringToken:
readStringToken();
break;
case NumberToken:
readNumberToken();
break;
case DoubleToken:
nextChar();
readDoubleToken();
break;
case EqualsToken:
nextChar();
break;
case CommentToken:
skipToEOL();
//nextChar(); // Bad to get the next char, as it could be the EOL needed to terminate the current identifier
break;
case NodeToken:
readNodeToken();
nextChar();
break;
case EOLToken:
nextChar();
break;
case EOFToken:
// Do nothing
break;
default:
throw IniParserException(PARSERERROR_PARSER, "IniParser::readToken", NULL, this, "Unhandled token: %s", __ini_tokenName(_token));
break;
}
_handler->handleToken(_token);
}
void CSVParser::readNumberToken() {
while (_curchar!=CHAR_EOF) {
if (_curchar==CHAR_QUOTE) {
throw CSVParserException(PARSERERROR_PARSER, "CSVParser::readNumberToken", &_token, this, "Unexpected quote");
} else if (_curchar==CHAR_NEWLINE || _curchar==_sepchar) {
break;
} else if (_numeralset.contains(_curchar)) {
_token.addChar(_curchar);
} else if (_curchar==CHAR_DECIMALPOINT) {
_token.addChar(_curchar);
nextChar();
_token.setType(DoubleToken);
readDoubleToken();
return;
} else {
_token.setType(StringToken);
readStringToken();
return;
}
nextChar();
}
}
void IniParser::readNumberToken() {
while (_curchar!=CHAR_EOF) {
if (_curchar==CHAR_QUOTE) {
throw IniParserException(PARSERERROR_PARSER, "IniParser::readNumberToken", &_token, this, "Unexpected quote");
} else if (_curchar==CHAR_NEWLINE || _whitespaceset.contains(_curchar) || _curchar==CHAR_SEMICOLON /*|| _curchar==CHAR_EQUALSIGN*/) {
break;
} else {
if (_numberset.contains(_curchar)) {
_token.addChar(_curchar);
} else if (_curchar==CHAR_DECIMALPOINT) {
_token.addChar(_curchar);
nextChar();
_token.setType(DoubleToken);
readDoubleToken();
return;
} else {
_token.setType(StringToken);
readStringToken();
return;
}
}
nextChar();
}
}