/* Advances the parser one token, optionally skipping whitespace
* (otherwise it is concatenated and returned as a single whitespace token).
* Whitespace is needed to properly render function signatures. Unrecognized
* token starts are stored literally, e.g. token may equal to a character '#'. */
static int advanceToken (lexerState *lexer, boolean skip_whitspace)
{
boolean have_whitespace = FALSE;
lexer->line = getSourceLineNumber();
lexer->pos = getInputFilePosition();
while (lexer->cur_c != EOF)
{
if (isWhitespace(lexer->cur_c))
{
scanWhitespace(lexer);
have_whitespace = TRUE;
}
else if (lexer->cur_c == '/' && (lexer->next_c == '/' || lexer->next_c == '*'))
{
scanComments(lexer);
have_whitespace = TRUE;
}
else
{
if (have_whitespace && !skip_whitspace)
return lexer->cur_token = TOKEN_WHITESPACE;
break;
}
}
lexer->line = getSourceLineNumber();
lexer->pos = getInputFilePosition();
while (lexer->cur_c != EOF)
{
if (lexer->cur_c == '"')
{
scanString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (lexer->cur_c == 'r' && (lexer->next_c == '#' || lexer->next_c == '"'))
{
scanRawString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (lexer->cur_c == '\'')
{
scanCharacterOrLifetime(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (isIdentifierStart(lexer->cur_c))
{
scanIdentifier(lexer);
return lexer->cur_token = TOKEN_IDENT;
}
/* These shift tokens aren't too important for tag-generation per se,
* but they confuse the skipUntil code which tracks the <> pairs. */
else if (lexer->cur_c == '>' && lexer->next_c == '>')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_RSHIFT;
}
else if (lexer->cur_c == '<' && lexer->next_c == '<')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_LSHIFT;
}
else if (lexer->cur_c == '-' && lexer->next_c == '>')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_RARROW;
}
else
{
int c = lexer->cur_c;
advanceChar(lexer);
return lexer->cur_token = c;
}
}
return lexer->cur_token = TOKEN_EOF;
}
Token* AnnotationLexer::nextToken() {
Token* token = 0;
int tokenLine = getCurrentLineNumber();
int tokenStart = getCurrentCharNumberInLine();
try {
do {
if (readingString_) {
readChar();
if (peek_ == '"') {
readingString_ = false;
return new Token(AnnotationTerminals::ID_QUOTE);
} else {
std::string str;
do {
str += peek_;
if (nextChar() != '"')
readChar();
else
break;
} while (true);
return new StringToken(AnnotationTerminals::ID_STRING, str);
}
}
// Skip whitespace including newlines
//
do {
readChar();
} while (isWhitespace(peek_));
switch (peek_) {
case '@':
token = new Token(AnnotationTerminals::ID_AT);
break;
case '=':
token = new Token(AnnotationTerminals::ID_EQUAL);
break;
case '[':
token = new Token(AnnotationTerminals::ID_LBRACKET);
break;
case ']':
token = new Token(AnnotationTerminals::ID_RBRACKET);
break;
case ',':
token = new Token(AnnotationTerminals::ID_COMMA);
break;
case '\"':
token = new Token(AnnotationTerminals::ID_QUOTE);
readingString_ = true;
break;
case '\'':
token = new Token(AnnotationTerminals::ID_APOSTROPHE);
break;
} // switch (peek_)
if (token == 0) {
if (isAlpha(peek_))
token = scanIdentifier(AnnotationTerminals::ID_IDENTIFIER);
else if (isDigit(peek_) || (peek_ == '.'))
token = scanNumber();
}
} while (token == 0);
} catch (std::runtime_error&) {
if (token != 0) {
token->setLineNumber(tokenLine);
token->setCharNumber(tokenStart);
}
return token;
}
if (token != 0) {
token->setLineNumber(tokenLine);
token->setCharNumber(tokenStart);
}
return token;
}
void EBNFScanner::nextToken()
{
if(currentToken > -1 && currentToken < tokens.size() - 1)
{
currentToken++;
return;
}
while (hasNextCharacter())
{
nextCharacter();
// Scan the Identifier type. [a-zA-Z\-_]
if(TokenScannerHelper::isAlpha(currentCharacter))
{
Token tok = scanIdentifier();
tokens.append(std::move(tok));
currentToken++;
return;
}
// Scan a literal
else if (TokenScannerHelper::isQuote(currentCharacter))
{
Token tok = scanLiteral();
tokens.append(std::move(tok));
currentToken++;
return;
}
// Scan an operator Token.
else if (TokenScannerHelper::isOperator(currentCharacter))
{
if (currentCharacter == '(' && peekCharacter() == '*') {
Token tok = scanComment();
tokens.append(std::move(tok));
currentToken++;
return;
}
else {
Token tok = scanOperator();
tokens.append(std::move(tok));
currentToken++;
return;
}
}
else if (TokenScannerHelper::isWhiteSpace(currentCharacter))
{
// Ignore white spaces.
}
else
{
raiseError (String("Invalid Character."));
}
checkNewLine();
}
tokens.append(Token(Token::TypeEof, String(""), createPosition()));
currentToken++;
}
// private methods
//
Token* PlotFunctionLexer::nextToken() {
Token* token = 0;
int tokenLine = getCurrentLineNumber();
int tokenStart = getCurrentCharNumberInLine();
try {
do {
// Skip whitespace including newlines
//
do {
readChar();
} while (isWhitespace(peek_));
switch (peek_) {
case '.':
if (isDigit(nextChar()))
token = scanNumber();
else
token = new Token(PlotFunctionTerminals::ID_DOT);
break; // case '.'
case '+':
token = new OperatorToken(PlotFunctionTerminals::ID_PLUS,peek_);
break; // case '+'
case '-':
token = new OperatorToken(PlotFunctionTerminals::ID_DASH,peek_);
break; // case '-'
case '/':
token = new OperatorToken(PlotFunctionTerminals::ID_SLASH,peek_);
break; // case '/'
case '*':
token = new OperatorToken(PlotFunctionTerminals::ID_STAR,peek_);
break; // case '*'
case '[':
token = new BracketToken(PlotFunctionTerminals::ID_LBRACKET,peek_);
break; // case '['
case ']':
token = new BracketToken(PlotFunctionTerminals::ID_RBRACKET,peek_);
break; // case ']'
case '(':
token = new BracketToken(PlotFunctionTerminals::ID_LPAREN,peek_);
break; // case '('
case ')':
token = new BracketToken(PlotFunctionTerminals::ID_RPAREN,peek_);
break; // case ')'
case '^':
token = new OperatorToken(PlotFunctionTerminals::ID_CARET,peek_);
break; // case '^'
case ':':
token = new IsolatorToken(PlotFunctionTerminals::ID_COLON,peek_);
break; // case ':'
case ';':
token = new IsolatorToken(PlotFunctionTerminals::ID_SEMICOLON,peek_);
break; // case ';'
case ',':
token = new IsolatorToken(PlotFunctionTerminals::ID_COMMA,peek_);
break; // case ','
case '|':
token = new IsolatorToken(PlotFunctionTerminals::ID_VERTICAL_BAR,peek_);
break; // case '|'
} // switch (peek_)
if (token == 0) {
if (isAlpha(peek_))
token = scanIdentifier();
else if (isDigit(peek_) || (peek_ == '.'))
token = scanNumber();
}
} while (token == 0);
} catch (std::runtime_error&) {
if (token != 0) {
token->setLineNumber(tokenLine);
token->setCharNumber(tokenStart);
}
else if (tokenStart == 0)
token = new IdentifierToken(PlotFunctionTerminals::ID_EMPTY,""); // EMPTY Expression
return token;
}
if (token != 0) {
token->setLineNumber(tokenLine);
token->setCharNumber(tokenStart);
}
return token;
}
