char* SExpression::scan(char* head, char* tail, SExpression* expression) {
if (expression) expression->head_ = expression->tail_ = 0;
char* s = skipWhitespace(head, tail);
if (s < tail) {
switch(*s) {
case '(':
s = scanArray(s, tail, expression);
break;
default:
s = scanRawString(s, tail, expression);
break;
}
s = skipWhitespace(s, tail);
}
return s;
}
/* Advances the parser one token, optionally skipping whitespace
* (otherwise it is concatenated and returned as a single whitespace token).
* Whitespace is needed to properly render function signatures. Unrecognized
* token starts are stored literally, e.g. token may equal to a character '#'. */
static int advanceToken (lexerState *lexer, boolean skip_whitspace)
{
boolean have_whitespace = FALSE;
lexer->line = getSourceLineNumber();
lexer->pos = getInputFilePosition();
while (lexer->cur_c != EOF)
{
if (isWhitespace(lexer->cur_c))
{
scanWhitespace(lexer);
have_whitespace = TRUE;
}
else if (lexer->cur_c == '/' && (lexer->next_c == '/' || lexer->next_c == '*'))
{
scanComments(lexer);
have_whitespace = TRUE;
}
else
{
if (have_whitespace && !skip_whitspace)
return lexer->cur_token = TOKEN_WHITESPACE;
break;
}
}
lexer->line = getSourceLineNumber();
lexer->pos = getInputFilePosition();
while (lexer->cur_c != EOF)
{
if (lexer->cur_c == '"')
{
scanString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (lexer->cur_c == 'r' && (lexer->next_c == '#' || lexer->next_c == '"'))
{
scanRawString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (lexer->cur_c == '\'')
{
scanCharacterOrLifetime(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (isIdentifierStart(lexer->cur_c))
{
scanIdentifier(lexer);
return lexer->cur_token = TOKEN_IDENT;
}
/* These shift tokens aren't too important for tag-generation per se,
* but they confuse the skipUntil code which tracks the <> pairs. */
else if (lexer->cur_c == '>' && lexer->next_c == '>')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_RSHIFT;
}
else if (lexer->cur_c == '<' && lexer->next_c == '<')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_LSHIFT;
}
else if (lexer->cur_c == '-' && lexer->next_c == '>')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_RARROW;
}
else
{
int c = lexer->cur_c;
advanceChar(lexer);
return lexer->cur_token = c;
}
}
return lexer->cur_token = TOKEN_EOF;
}
