void parseNextToken(LexerCarriage* carriage)
{
skipDelimiters(carriage);
if (carriage->posInText < carriage->lexingText.length)
{
bool(*functions[])(LexerCarriage*, Token*) = {
parseNumber,
parseOperatorToken,
parseKeywordOrIdentifier,
parseUnknown // if we coudn't parse this, we'll just return it as a 'unknown' token
};
for (int i = 0; i < ArrayCount(functions); ++i)
{
if (functions[i](carriage, &carriage->topToken))
{
break;
}
}
}
else
{
carriage->topToken = Token{ TOK_EOF };
}
skipComment(carriage);
}
string StringTokenizer::nextToken()
{
skipDelimiters();
if (currentPosition >= maxPosition)
return "";
int start = currentPosition;
while ((currentPosition < maxPosition) &&
(delim.find(str[currentPosition]) == string::npos))
{
currentPosition++;
}
if (retTok && (start == currentPosition) &&
(delim.find(str[currentPosition]) != string::npos))
currentPosition++;
return str.substr(start, currentPosition - start);
}
bool StringTokenizer::hasMoreTokens()
{
skipDelimiters();
return (currentPosition < maxPosition);
}