xml::token Lexer::nextToken()
{
while (skipWhitespace() || skipComments()) {
}
if (buffer_empty()) {
buffer_put(m_input->get());
}
switch (buffer_peek()) {
case '<':
m_token = token::chevron_left;
buffer_pop();
break;
case '>':
m_token = token::chevron_right;
buffer_pop();
break;
case '?':
m_token = token::questionMark;
buffer_pop();
break;
case '=':
m_token = token::assignment;
buffer_pop();
break;
case '/':
m_token = token::slash;
buffer_pop();
break;
case '"': case '\'':
consumeString();
break;
case EOF:
/*
* TODO: Replace EOF with std::char_traits<char>::eof() once VS
* will learn what "constexpr" is.
*/
m_token = token::eof;
buffer_pop();
break;
default:
consumeIdentifier();
break;
}
return m_token;
}
AnimExpression::Token AnimExpression::consumeToken(const QString& str, QString::const_iterator& iter) const {
if (!_tokenStack.empty()) {
Token top = _tokenStack.top();
_tokenStack.pop();
return top;
} else {
while (iter != str.end()) {
if (iter->isSpace()) {
++iter;
} else if (iter->isLetter()) {
return consumeIdentifier(str, iter);
} else if (iter->isDigit()) {
return consumeNumber(str, iter);
} else {
switch (iter->unicode()) {
case '&': return consumeAnd(str, iter);
case '|': return consumeOr(str, iter);
case '>': return consumeGreaterThan(str, iter);
case '<': return consumeLessThan(str, iter);
case '(': ++iter; return Token(Token::LeftParen);
case ')': ++iter; return Token(Token::RightParen);
case '!': return consumeNot(str, iter);
case '-': ++iter; return Token(Token::Minus);
case '+': ++iter; return Token(Token::Plus);
case '*': ++iter; return Token(Token::Multiply);
case '/': ++iter; return Token(Token::Divide);
case '%': ++iter; return Token(Token::Modulus);
case ',': ++iter; return Token(Token::Comma);
default:
qCCritical(animation) << "AnimExpression: unexpected char" << *iter << "at index " << (int)(iter - str.begin());
return Token(Token::Error);
}
}
}
return Token(Token::End);
}
}
void Tokenizer::shift()
{
mpCurrent.reset();
if (mBlockComment)
{
skipEOL();
}
else
{
skipWhiteSpaces();
if (mpInput->eof())
return;
}
mpCurrent.reset(new Token);
mpCurrent->setLine(line());
mpCurrent->setBeginColumn(column());
if (mBlockComment)
{
consumeCStyleBlockComment();
return;
}
auto ch = mpInput->get();
if (isLetter(ch) || isUnderscore(ch))
{
consumeIdentifier(ch);
}
else if (isBiwiseOperatorSymbol(ch))
{
mpCurrent->setType(Token::TYPE_BITWISE_OPERATOR);
absorbed(ch);
mpCurrent->addChar(ch);
mpCurrent->setEndColumn(column());
}
else if (isDot(ch) && consumeDot(ch))
{
// nothing
}
else if ((isDecimalDigit(ch) || isDot(ch) || isSign(ch)) && consumeNumber(ch))
{
// nothing
}
else if (isQuotationMark(ch))
{
if (!consumeString(ch))
shift();
}
else if (isCStyleInitialCommentChar(ch))
{
consumeComment(ch);
}
else if (isArrowSymbol(ch) && consumeArrow(ch))
{
// nothing
}
else if (isBracket(ch))
{
mpCurrent->setType(Token::TYPE_BRACKET);
absorbed(ch);
mpCurrent->addChar(ch);
mpCurrent->setEndColumn(column());
}
else if (isAngleBracket(ch))
{
mpCurrent->setType(Token::TYPE_ANGLE_BRACKET);
absorbed(ch);
mpCurrent->addChar(ch);
mpCurrent->setEndColumn(column());
}
else if (isDelimiter(ch))
{
mpCurrent->setType(Token::TYPE_DELIMITER);
absorbed(ch);
mpCurrent->addChar(ch);
mpCurrent->setEndColumn(column());
}
else if (isOperator(ch) && consumeEqualOperator(ch))
{
// nothing
}
else if (isOperator(ch))
{
mpCurrent->setType(Token::TYPE_OPERATOR);
absorbed(ch);
mpCurrent->addChar(ch);
mpCurrent->setEndColumn(column());
}
else if (isAsterisk(ch))
{
mpCurrent->setType(Token::TYPE_ASTERISK);
absorbed(ch);
mpCurrent->addChar(ch);
mpCurrent->setEndColumn(column());
}
else
{
mpCurrent.reset();
}
}
