/* Advances the parser one token, optionally skipping whitespace * (otherwise it is concatenated and returned as a single whitespace token). * Whitespace is needed to properly render function signatures. Unrecognized * token starts are stored literally, e.g. token may equal to a character '#'. */ static int advanceToken (lexerState *lexer, boolean skip_whitspace) { boolean have_whitespace = FALSE; lexer->line = getSourceLineNumber(); lexer->pos = getInputFilePosition(); while (lexer->cur_c != EOF) { if (isWhitespace(lexer->cur_c)) { scanWhitespace(lexer); have_whitespace = TRUE; } else if (lexer->cur_c == '/' && (lexer->next_c == '/' || lexer->next_c == '*')) { scanComments(lexer); have_whitespace = TRUE; } else { if (have_whitespace && !skip_whitspace) return lexer->cur_token = TOKEN_WHITESPACE; break; } } lexer->line = getSourceLineNumber(); lexer->pos = getInputFilePosition(); while (lexer->cur_c != EOF) { if (lexer->cur_c == '"') { scanString(lexer); return lexer->cur_token = TOKEN_STRING; } else if (lexer->cur_c == 'r' && (lexer->next_c == '#' || lexer->next_c == '"')) { scanRawString(lexer); return lexer->cur_token = TOKEN_STRING; } else if (lexer->cur_c == '\'') { scanCharacterOrLifetime(lexer); return lexer->cur_token = TOKEN_STRING; } else if (isIdentifierStart(lexer->cur_c)) { scanIdentifier(lexer); return lexer->cur_token = TOKEN_IDENT; } /* These shift tokens aren't too important for tag-generation per se, * but they confuse the skipUntil code which tracks the <> pairs. */ else if (lexer->cur_c == '>' && lexer->next_c == '>') { advanceNChar(lexer, 2); return lexer->cur_token = TOKEN_RSHIFT; } else if (lexer->cur_c == '<' && lexer->next_c == '<') { advanceNChar(lexer, 2); return lexer->cur_token = TOKEN_LSHIFT; } else if (lexer->cur_c == '-' && lexer->next_c == '>') { advanceNChar(lexer, 2); return lexer->cur_token = TOKEN_RARROW; } else { int c = lexer->cur_c; advanceChar(lexer); return lexer->cur_token = c; } } return lexer->cur_token = TOKEN_EOF; }
Token* AnnotationLexer::nextToken() { Token* token = 0; int tokenLine = getCurrentLineNumber(); int tokenStart = getCurrentCharNumberInLine(); try { do { if (readingString_) { readChar(); if (peek_ == '"') { readingString_ = false; return new Token(AnnotationTerminals::ID_QUOTE); } else { std::string str; do { str += peek_; if (nextChar() != '"') readChar(); else break; } while (true); return new StringToken(AnnotationTerminals::ID_STRING, str); } } // Skip whitespace including newlines // do { readChar(); } while (isWhitespace(peek_)); switch (peek_) { case '@': token = new Token(AnnotationTerminals::ID_AT); break; case '=': token = new Token(AnnotationTerminals::ID_EQUAL); break; case '[': token = new Token(AnnotationTerminals::ID_LBRACKET); break; case ']': token = new Token(AnnotationTerminals::ID_RBRACKET); break; case ',': token = new Token(AnnotationTerminals::ID_COMMA); break; case '\"': token = new Token(AnnotationTerminals::ID_QUOTE); readingString_ = true; break; case '\'': token = new Token(AnnotationTerminals::ID_APOSTROPHE); break; } // switch (peek_) if (token == 0) { if (isAlpha(peek_)) token = scanIdentifier(AnnotationTerminals::ID_IDENTIFIER); else if (isDigit(peek_) || (peek_ == '.')) token = scanNumber(); } } while (token == 0); } catch (std::runtime_error&) { if (token != 0) { token->setLineNumber(tokenLine); token->setCharNumber(tokenStart); } return token; } if (token != 0) { token->setLineNumber(tokenLine); token->setCharNumber(tokenStart); } return token; }
void EBNFScanner::nextToken() { if(currentToken > -1 && currentToken < tokens.size() - 1) { currentToken++; return; } while (hasNextCharacter()) { nextCharacter(); // Scan the Identifier type. [a-zA-Z\-_] if(TokenScannerHelper::isAlpha(currentCharacter)) { Token tok = scanIdentifier(); tokens.append(std::move(tok)); currentToken++; return; } // Scan a literal else if (TokenScannerHelper::isQuote(currentCharacter)) { Token tok = scanLiteral(); tokens.append(std::move(tok)); currentToken++; return; } // Scan an operator Token. else if (TokenScannerHelper::isOperator(currentCharacter)) { if (currentCharacter == '(' && peekCharacter() == '*') { Token tok = scanComment(); tokens.append(std::move(tok)); currentToken++; return; } else { Token tok = scanOperator(); tokens.append(std::move(tok)); currentToken++; return; } } else if (TokenScannerHelper::isWhiteSpace(currentCharacter)) { // Ignore white spaces. } else { raiseError (String("Invalid Character.")); } checkNewLine(); } tokens.append(Token(Token::TypeEof, String(""), createPosition())); currentToken++; }
// private methods // Token* PlotFunctionLexer::nextToken() { Token* token = 0; int tokenLine = getCurrentLineNumber(); int tokenStart = getCurrentCharNumberInLine(); try { do { // Skip whitespace including newlines // do { readChar(); } while (isWhitespace(peek_)); switch (peek_) { case '.': if (isDigit(nextChar())) token = scanNumber(); else token = new Token(PlotFunctionTerminals::ID_DOT); break; // case '.' case '+': token = new OperatorToken(PlotFunctionTerminals::ID_PLUS,peek_); break; // case '+' case '-': token = new OperatorToken(PlotFunctionTerminals::ID_DASH,peek_); break; // case '-' case '/': token = new OperatorToken(PlotFunctionTerminals::ID_SLASH,peek_); break; // case '/' case '*': token = new OperatorToken(PlotFunctionTerminals::ID_STAR,peek_); break; // case '*' case '[': token = new BracketToken(PlotFunctionTerminals::ID_LBRACKET,peek_); break; // case '[' case ']': token = new BracketToken(PlotFunctionTerminals::ID_RBRACKET,peek_); break; // case ']' case '(': token = new BracketToken(PlotFunctionTerminals::ID_LPAREN,peek_); break; // case '(' case ')': token = new BracketToken(PlotFunctionTerminals::ID_RPAREN,peek_); break; // case ')' case '^': token = new OperatorToken(PlotFunctionTerminals::ID_CARET,peek_); break; // case '^' case ':': token = new IsolatorToken(PlotFunctionTerminals::ID_COLON,peek_); break; // case ':' case ';': token = new IsolatorToken(PlotFunctionTerminals::ID_SEMICOLON,peek_); break; // case ';' case ',': token = new IsolatorToken(PlotFunctionTerminals::ID_COMMA,peek_); break; // case ',' case '|': token = new IsolatorToken(PlotFunctionTerminals::ID_VERTICAL_BAR,peek_); break; // case '|' } // switch (peek_) if (token == 0) { if (isAlpha(peek_)) token = scanIdentifier(); else if (isDigit(peek_) || (peek_ == '.')) token = scanNumber(); } } while (token == 0); } catch (std::runtime_error&) { if (token != 0) { token->setLineNumber(tokenLine); token->setCharNumber(tokenStart); } else if (tokenStart == 0) token = new IdentifierToken(PlotFunctionTerminals::ID_EMPTY,""); // EMPTY Expression return token; } if (token != 0) { token->setLineNumber(tokenLine); token->setCharNumber(tokenStart); } return token; }