QueryToken* TokenList::peek() {
/* DSR:2004.11.01: Reverted my previous (circa April 2004) fix (which
** raised an exception if Peek was called when there were no tokens) in
** favor of returning the EOF token. This solution is much better
** integrated with the rest of the code in the queryParser subsystem. */
size_t nTokens = tokens.size();
if (nTokens == 0) {
push(_CLNEW QueryToken(QueryToken::EOF_));
nTokens++;
}
return tokens[nTokens-1];
}
QueryToken FTSQueryParser::next() {
if (_pos >= _raw.size())
return QueryToken(QueryToken::INVALID, "", 0, false);
unsigned start = _pos++;
QueryToken::Type type = getType(_raw[start]);
// Query Parser should never land on whitespace
if (type == QueryToken::WHITESPACE) {
invariant(false);
}
if (type == QueryToken::TEXT) {
while (_pos < _raw.size() && getType(_raw[_pos]) == type) {
_pos++;
}
}
StringData ret = _raw.substr(start, _pos - start);
bool old = _previousWhiteSpace;
_previousWhiteSpace = skipWhitespace();
return QueryToken(type, ret, start, old);
}
void Lexer::Lex(TokenList *tokenList) {
//Func - Breaks the input stream onto the tokens list tokens
//Pre - tokens != NULL and contains a TokenList in which the tokens can be stored
//Post - The tokens have been added to the TokenList tokens
CND_PRECONDITION(tokenList != NULL, "tokens is NULL");
//Get all the tokens
while(true) {
//Add the token to the tokens list
//Get the next token
QueryToken* token = _CLNEW QueryToken;
if ( !GetNextToken(token) ){
_CLDELETE(token);
break;
}
tokenList->add(token);
}
//The end has been reached so create an EOF_ token
//Add the final token to the TokenList _tokens
tokenList->add(_CLNEW QueryToken( QueryToken::EOF_));
}
