xml::token Lexer::nextToken() { while (skipWhitespace() || skipComments()) { } if (buffer_empty()) { buffer_put(m_input->get()); } switch (buffer_peek()) { case '<': m_token = token::chevron_left; buffer_pop(); break; case '>': m_token = token::chevron_right; buffer_pop(); break; case '?': m_token = token::questionMark; buffer_pop(); break; case '=': m_token = token::assignment; buffer_pop(); break; case '/': m_token = token::slash; buffer_pop(); break; case '"': case '\'': consumeString(); break; case EOF: /* * TODO: Replace EOF with std::char_traits<char>::eof() once VS * will learn what "constexpr" is. */ m_token = token::eof; buffer_pop(); break; default: consumeIdentifier(); break; } return m_token; }
AnimExpression::Token AnimExpression::consumeToken(const QString& str, QString::const_iterator& iter) const { if (!_tokenStack.empty()) { Token top = _tokenStack.top(); _tokenStack.pop(); return top; } else { while (iter != str.end()) { if (iter->isSpace()) { ++iter; } else if (iter->isLetter()) { return consumeIdentifier(str, iter); } else if (iter->isDigit()) { return consumeNumber(str, iter); } else { switch (iter->unicode()) { case '&': return consumeAnd(str, iter); case '|': return consumeOr(str, iter); case '>': return consumeGreaterThan(str, iter); case '<': return consumeLessThan(str, iter); case '(': ++iter; return Token(Token::LeftParen); case ')': ++iter; return Token(Token::RightParen); case '!': return consumeNot(str, iter); case '-': ++iter; return Token(Token::Minus); case '+': ++iter; return Token(Token::Plus); case '*': ++iter; return Token(Token::Multiply); case '/': ++iter; return Token(Token::Divide); case '%': ++iter; return Token(Token::Modulus); case ',': ++iter; return Token(Token::Comma); default: qCCritical(animation) << "AnimExpression: unexpected char" << *iter << "at index " << (int)(iter - str.begin()); return Token(Token::Error); } } } return Token(Token::End); } }
void Tokenizer::shift() { mpCurrent.reset(); if (mBlockComment) { skipEOL(); } else { skipWhiteSpaces(); if (mpInput->eof()) return; } mpCurrent.reset(new Token); mpCurrent->setLine(line()); mpCurrent->setBeginColumn(column()); if (mBlockComment) { consumeCStyleBlockComment(); return; } auto ch = mpInput->get(); if (isLetter(ch) || isUnderscore(ch)) { consumeIdentifier(ch); } else if (isBiwiseOperatorSymbol(ch)) { mpCurrent->setType(Token::TYPE_BITWISE_OPERATOR); absorbed(ch); mpCurrent->addChar(ch); mpCurrent->setEndColumn(column()); } else if (isDot(ch) && consumeDot(ch)) { // nothing } else if ((isDecimalDigit(ch) || isDot(ch) || isSign(ch)) && consumeNumber(ch)) { // nothing } else if (isQuotationMark(ch)) { if (!consumeString(ch)) shift(); } else if (isCStyleInitialCommentChar(ch)) { consumeComment(ch); } else if (isArrowSymbol(ch) && consumeArrow(ch)) { // nothing } else if (isBracket(ch)) { mpCurrent->setType(Token::TYPE_BRACKET); absorbed(ch); mpCurrent->addChar(ch); mpCurrent->setEndColumn(column()); } else if (isAngleBracket(ch)) { mpCurrent->setType(Token::TYPE_ANGLE_BRACKET); absorbed(ch); mpCurrent->addChar(ch); mpCurrent->setEndColumn(column()); } else if (isDelimiter(ch)) { mpCurrent->setType(Token::TYPE_DELIMITER); absorbed(ch); mpCurrent->addChar(ch); mpCurrent->setEndColumn(column()); } else if (isOperator(ch) && consumeEqualOperator(ch)) { // nothing } else if (isOperator(ch)) { mpCurrent->setType(Token::TYPE_OPERATOR); absorbed(ch); mpCurrent->addChar(ch); mpCurrent->setEndColumn(column()); } else if (isAsterisk(ch)) { mpCurrent->setType(Token::TYPE_ASTERISK); absorbed(ch); mpCurrent->addChar(ch); mpCurrent->setEndColumn(column()); } else { mpCurrent.reset(); } }