Token *Lexer::nextToken() {
while(true) {
switch(_currentChar.toAscii()) {
case '\'': return scanCharacter();
case '"': return scanText();
case '(': return scan(Token::LeftParenthesis);
case ')': return scan(Token::RightParenthesis);
case '[': return scan(Token::LeftBracket);
case ']': return scanRightBracket();
case '{': return scan(Token::LeftBrace);
case '}': return scan(Token::RightBrace);
case ';': return scan(Token::Semicolon);
default:
if(isEof()) return scan(Token::Eof);
else if(isLineComment()) consumeLineComment();
else if(isBlockComment()) consumeBlockComment();
else if(isNewline()) return scanNewline();
else if(isSpace()) consumeSpaces();
else if(isName()) return scanName();
else if(isBackquotedName()) return scanBackquotedName();
else if(isNumber()) return scanNumber();
else if(isOperator()) return scanOperator();
else throw lexerException(QString("invalid character: '%1'").arg(_currentChar));
}
}
}
void RppLexer::scanComment(int *kind)
{
switch(m_buffer[m_ptr + 1]) {
case '/':
*kind = Token_line_comment;
break;
case '*':
*kind = Token_multiline_comment;
break;
default:
scanOperator(kind);
}
}
void EBNFScanner::nextToken()
{
if(currentToken > -1 && currentToken < tokens.size() - 1)
{
currentToken++;
return;
}
while (hasNextCharacter())
{
nextCharacter();
// Scan the Identifier type. [a-zA-Z\-_]
if(TokenScannerHelper::isAlpha(currentCharacter))
{
Token tok = scanIdentifier();
tokens.append(std::move(tok));
currentToken++;
return;
}
// Scan a literal
else if (TokenScannerHelper::isQuote(currentCharacter))
{
Token tok = scanLiteral();
tokens.append(std::move(tok));
currentToken++;
return;
}
// Scan an operator Token.
else if (TokenScannerHelper::isOperator(currentCharacter))
{
if (currentCharacter == '(' && peekCharacter() == '*') {
Token tok = scanComment();
tokens.append(std::move(tok));
currentToken++;
return;
}
else {
Token tok = scanOperator();
tokens.append(std::move(tok));
currentToken++;
return;
}
}
else if (TokenScannerHelper::isWhiteSpace(currentCharacter))
{
// Ignore white spaces.
}
else
{
raiseError (String("Invalid Character."));
}
checkNewLine();
}
tokens.append(Token(Token::TypeEof, String(""), createPosition()));
currentToken++;
}
