NimLexer::Token NimLexer::onDefaultState()
{
while (!m_stream->isEnd())
{
if (isSkipChar(m_stream)) {
m_stream->move();
continue;
}
if (isOperator(m_stream))
return readOperator(m_stream);
if (matchDocumentationStart(m_stream))
return readDocumentation(m_stream);
if (matchCommentStart(m_stream))
return readComment(m_stream);
if (matchMultiLineStringLiteralStart(m_stream))
return readMultiLineStringLiteral(m_stream, true);
if (matchStringLiteralStart(m_stream))
return readStringLiteral(m_stream);
if (matchIdentifierOrKeywordStart(m_stream))
return readIdentifierOrKeyword(m_stream);
m_stream->move();
}
return Token {0,0, TokenType::EndOfText };
}
Token Scanner::readPercentageNotation()
{
QChar ch = m_src.peek();
if (ch.isSpace() || ch.isDigit())
return Token(Token::Operator, m_src.anchor(), m_src.length());
State state = State_String;
if (ch.isLetter()) {
if (ch == 'r')
state = State_Regexp;
if (ch == 'i')
state = State_Symbols;
m_src.move();
}
QChar delimiter = translateDelimiter(m_src.peek());
m_src.move();
return readStringLiteral(delimiter, state);
}
Token Scanner::onDefaultState()
{
QChar first = m_src.peek();
m_src.move();
// Ignore new lines
bool hasNewLine = false;
while (isLineFeed(first)) {
hasNewLine = true;
m_line++;
m_lineStartOffset = m_src.position();
first = m_src.peek();
m_src.setAnchor();
m_src.move();
}
if (hasNewLine)
m_tokenSequence.clear();
Token token;
if (first.isDigit()) {
token = readFloatNumber();
} else if (first == '\'' || first == '\"' || first == '`') {
token = readStringLiteral(first, State_String);
} else if (m_methodPattern.match(m_tokenSequence).hasMatch()) {
token = readMethodDefinition();
} else if (first.isLetter() || first == '_' || first == '@'
|| first == '$' || (first == ':' && m_src.peek() != ':')) {
token = readIdentifier();
} else if (first.isDigit()) {
token = readNumber();
} else if (first == '#') {
token = readComment();
} else if (first == '/') {
token = readRegexp();
} else if (first.isSpace()) {
token = readWhiteSpace();
} else if (first == ',') {
token = Token(Token::OperatorComma, m_src.anchor(), m_src.length());
} else if (first == '.') {
token = Token(Token::OperatorDot, m_src.anchor(), m_src.length());
} else if (first == '=' && m_src.peek() != '=') {
token = Token(Token::OperatorAssign, m_src.anchor(), m_src.length());
} else if (first == ';') {
token = Token(Token::OperatorSemiColon, m_src.anchor(), m_src.length());
} else if (first == '%') {
token = readPercentageNotation();
} else if (first == '{') {
token = Token(Token::OpenBraces, m_src.anchor(), m_src.length());
} else if (first == '}') {
token = Token(Token::CloseBraces, m_src.anchor(), m_src.length());
} else if (first == '[') {
token = Token(Token::OpenBrackets, m_src.anchor(), m_src.length());
} else if (first == ']') {
token = Token(Token::CloseBrackets, m_src.anchor(), m_src.length());
// For historic reasons, ( and ) are the Operator token, this will
// be changed soon.
} else if (first == '(' || first == ')') {
token = Token(Token::Operator, m_src.anchor(), m_src.length());
} else {
token = readOperator(first);
}
m_tokenSequence += QString::number(token.kind);
m_tokenSequence += '_';
return token;
}
