Token *Tokenizer::next() { skipSpaces(); Token *token = 0; if( (token = makeTokenInteger()) != 0 ){ return token; } if( (token = makeTokenName()) != 0 ){ return token; } if( (token = makeOperatorToken()) != 0 ){ return token; } m_remaindString = m_nextc; return new Token( "", Token::TYPE_END); }
void Lexer::storePreviousToken() { switch (state) { case GettingIdentifier: if (location.ptr - start.ptr == 1 && *(start.ptr) == '_') { makeOperatorToken(start); } else { makeIdentifierOrKeywordToken(start.ptr, location.ptr); } break; case GettingIntegerNumber: { Token token(Token::Integer, start.ptr, location.ptr); storeToken(token, start); break; } case GettingFloatingPointNumber: { Token token(Token::Float, start.ptr, location.ptr); storeToken(token, start); break; } default: break; } }
void Lexer::tokenize() { while (location.ptr < eof) { bool newLine = false; char c = *location.ptr; switch (c) { case ' ': case '\t': case '\r': storePreviousToken(); break; case '\n': newLine = true; storePreviousToken(); storeToken(Token::Newline); break; case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '_': switch (state) { case Idle: state = GettingIdentifier; start = location; break; case GettingIntegerNumber: case GettingFloatingPointNumber: storeToken(Token::Invalid); break; default: break; } break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': switch (state) { case Idle: state = GettingIntegerNumber; start = location; break; default: break; } break; case '/': storePreviousToken(); if (isNextChar('/', location.ptr)) { skipUntilNewline(); continue; } else if (isNextChar('*', location.ptr)) { skipMultilineComment(); continue; } else { makeOperatorToken(location); } break; case '=': case '!': case '+': case '-': case '*': case '%': case '.': case ',': case '>': case '<': case ':': case ';': case '?': case '|': case '&': case '^': case '~': case '(': case ')': case '{': case '}': case '[': case ']': if (c == '.' && !isNextChar('.') && state == GettingIntegerNumber) { state = GettingFloatingPointNumber; } else { storePreviousToken(); makeOperatorToken(location); } break; case '"': storePreviousToken(); makeStringLiteral(location.ptr); continue; case '\'': storePreviousToken(); makeCharLiteral(); continue; default: storeToken(Token::Invalid); break; } if (newLine) { location.stepLine(); } else { location.stepColumn(); } } storeToken(Token::Eof); }