Ejemplo n.º 1
0
void parseNextToken(LexerCarriage* carriage)
{
    skipDelimiters(carriage);
    if (carriage->posInText < carriage->lexingText.length)
    {
        bool(*functions[])(LexerCarriage*, Token*) = {
            parseNumber,
            parseOperatorToken,
            parseKeywordOrIdentifier,
            parseUnknown // if we coudn't parse this, we'll just return it as a 'unknown' token
        };

        for (int i = 0; i < ArrayCount(functions); ++i)
        {
            if (functions[i](carriage, &carriage->topToken))
            {
                break;
            }
        }
    }
    else
    {
        carriage->topToken = Token{ TOK_EOF };
    }

    skipComment(carriage);
}
Ejemplo n.º 2
0
string StringTokenizer::nextToken()
{
	skipDelimiters();

	if (currentPosition >= maxPosition)
		return "";

	int start = currentPosition;
	while ((currentPosition < maxPosition) &&
	       (delim.find(str[currentPosition]) == string::npos))
	{
		currentPosition++;
	}
	if (retTok && (start == currentPosition) &&
	    (delim.find(str[currentPosition]) != string::npos))
		currentPosition++;
	return str.substr(start, currentPosition - start);
}
Ejemplo n.º 3
0
bool StringTokenizer::hasMoreTokens()
{
	skipDelimiters();
	return (currentPosition < maxPosition);
}