void consume_multiline_comment(TokenizeContext& context)
{
int lookahead = 0;
// Keep track of the current depth, for nested blocks.
int depth = 0;
while (context.withinRange(lookahead)) {
if (context.next(lookahead) == '{' && context.next(lookahead + 1) == '-') {
// Found a comment opener, increase depth. Also advance lookahead so that
// we don't get confused by this: {-}
lookahead += 2;
depth++;
continue;
}
if (context.next(lookahead) == '-' && context.next(lookahead + 1) == '}') {
// Found a comment ender.
depth--;
lookahead += 2;
if (depth == 0)
break;
continue;
}
lookahead++;
}
context.consume(tok_Comment, lookahead);
}
void consume_string_literal(TokenizeContext &context)
{
int lookahead = 0;
// Consume starting quote, this can be ' or "
char quote_type = context.next();
lookahead++;
bool escapedNext = false;
while (context.withinRange(lookahead)) {
char c = context.next(lookahead);
if (c == quote_type && !escapedNext)
break;
if (c == '\\' && !escapedNext)
escapedNext = true;
else
escapedNext = false;
lookahead++;
}
// consume ending quote
lookahead++;
context.consume(tok_String, lookahead);
}
void consume_comment(TokenizeContext& context)
{
int lookahead = 0;
while (context.withinRange(lookahead) && !is_newline(context.next(lookahead)))
lookahead++;
context.consume(tok_Comment, lookahead);
}
void consume_triple_quoted_string_literal(TokenizeContext &context)
{
int lookahead = 0;
// Consume initial <<<
lookahead += 3;
while (context.withinRange(lookahead) &&
!(context.next(lookahead) == '>'
&& context.next(lookahead + 1) == '>'
&& context.next(lookahead + 2) == '>'))
lookahead++;
// Consume closing >>>
lookahead += 3;
context.consume(tok_String, lookahead);
}