Token *Tokenizer::next()
{
skipSpaces();
Token *token = 0;
if( (token = makeTokenInteger()) != 0 ){
return token;
}
if( (token = makeTokenName()) != 0 ){
return token;
}
if( (token = makeOperatorToken()) != 0 ){
return token;
}
m_remaindString = m_nextc;
return new Token( "", Token::TYPE_END);
}
void Lexer::storePreviousToken() {
switch (state) {
case GettingIdentifier:
if (location.ptr - start.ptr == 1 && *(start.ptr) == '_') {
makeOperatorToken(start);
} else {
makeIdentifierOrKeywordToken(start.ptr, location.ptr);
}
break;
case GettingIntegerNumber: {
Token token(Token::Integer, start.ptr, location.ptr);
storeToken(token, start);
break;
}
case GettingFloatingPointNumber: {
Token token(Token::Float, start.ptr, location.ptr);
storeToken(token, start);
break;
}
default:
break;
}
}
void Lexer::tokenize() {
while (location.ptr < eof) {
bool newLine = false;
char c = *location.ptr;
switch (c) {
case ' ':
case '\t':
case '\r':
storePreviousToken();
break;
case '\n':
newLine = true;
storePreviousToken();
storeToken(Token::Newline);
break;
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
case '_':
switch (state) {
case Idle:
state = GettingIdentifier;
start = location;
break;
case GettingIntegerNumber:
case GettingFloatingPointNumber:
storeToken(Token::Invalid);
break;
default:
break;
}
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
switch (state) {
case Idle:
state = GettingIntegerNumber;
start = location;
break;
default:
break;
}
break;
case '/':
storePreviousToken();
if (isNextChar('/', location.ptr)) {
skipUntilNewline();
continue;
} else if (isNextChar('*', location.ptr)) {
skipMultilineComment();
continue;
} else {
makeOperatorToken(location);
}
break;
case '=':
case '!':
case '+':
case '-':
case '*':
case '%':
case '.':
case ',':
case '>':
case '<':
case ':':
case ';':
case '?':
case '|':
case '&':
case '^':
case '~':
case '(':
case ')':
case '{':
case '}':
case '[':
case ']':
if (c == '.' && !isNextChar('.') &&
state == GettingIntegerNumber) {
state = GettingFloatingPointNumber;
} else {
storePreviousToken();
makeOperatorToken(location);
}
break;
case '"':
storePreviousToken();
makeStringLiteral(location.ptr);
continue;
case '\'':
storePreviousToken();
makeCharLiteral();
continue;
default:
storeToken(Token::Invalid);
break;
}
if (newLine) {
location.stepLine();
} else {
location.stepColumn();
}
}
storeToken(Token::Eof);
}
