/* Normal line comments start with two /'s and continue until the next \n
* (NOT any other newline character!). Additionally, a shebang in the beginning
* of the file also counts as a line comment.
* Block comments start with / followed by a * and end with a * followed by a /.
* Unlike in C/C++ they nest. */
static void scanComments (lexerState *lexer)
{
/* // or #! */
if (lexer->next_c == '/' || lexer->next_c == '!')
{
advanceNChar(lexer, 2);
while (lexer->cur_c != EOF && lexer->cur_c != '\n')
advanceChar(lexer);
}
else if (lexer->next_c == '*')
{
int level = 1;
advanceNChar(lexer, 2);
while (lexer->cur_c != EOF && level > 0)
{
if (lexer->cur_c == '*' && lexer->next_c == '/')
{
level--;
advanceNChar(lexer, 2);
}
else if (lexer->cur_c == '/' && lexer->next_c == '*')
{
level++;
advanceNChar(lexer, 2);
}
else
{
advanceChar(lexer);
}
}
}
}
static void initLexer (lexerState *lexer)
{
advanceNChar(lexer, 2);
lexer->token_str = vStringNew();
if (lexer->cur_c == '#' && lexer->next_c == '!')
scanComments(lexer);
advanceToken(lexer, TRUE);
}
/* Normal line comments start with two /'s and continue until the next \n
* (potentially after a \r). Additionally, a shebang in the beginning of the
* file also counts as a line comment as long as it is not this sequence: #![ .
* Block comments start with / followed by a * and end with a * followed by a /.
* Unlike in C/C++ they nest. */
static void scanComments (lexerState *lexer)
{
/* // */
if (lexer->next_c == '/')
{
advanceNChar(lexer, 2);
while (lexer->cur_c != EOF && lexer->cur_c != '\n')
advanceChar(lexer);
}
/* #! */
else if (lexer->next_c == '!')
{
advanceNChar(lexer, 2);
/* If it is exactly #![ then it is not a comment, but an attribute */
if (lexer->cur_c == '[')
return;
while (lexer->cur_c != EOF && lexer->cur_c != '\n')
advanceChar(lexer);
}
/* block comment */
else if (lexer->next_c == '*')
{
int level = 1;
advanceNChar(lexer, 2);
while (lexer->cur_c != EOF && level > 0)
{
if (lexer->cur_c == '*' && lexer->next_c == '/')
{
level--;
advanceNChar(lexer, 2);
}
else if (lexer->cur_c == '/' && lexer->next_c == '*')
{
level++;
advanceNChar(lexer, 2);
}
else
{
advanceChar(lexer);
}
}
}
}
/* Advances the parser one token, optionally skipping whitespace
* (otherwise it is concatenated and returned as a single whitespace token).
* Whitespace is needed to properly render function signatures. Unrecognized
* token starts are stored literally, e.g. token may equal to a character '#'. */
static int advanceToken (lexerState *lexer, boolean skip_whitspace)
{
boolean have_whitespace = FALSE;
lexer->line = getSourceLineNumber();
lexer->pos = getInputFilePosition();
while (lexer->cur_c != EOF)
{
if (isWhitespace(lexer->cur_c))
{
scanWhitespace(lexer);
have_whitespace = TRUE;
}
else if (lexer->cur_c == '/' && (lexer->next_c == '/' || lexer->next_c == '*'))
{
scanComments(lexer);
have_whitespace = TRUE;
}
else
{
if (have_whitespace && !skip_whitspace)
return lexer->cur_token = TOKEN_WHITESPACE;
break;
}
}
lexer->line = getSourceLineNumber();
lexer->pos = getInputFilePosition();
while (lexer->cur_c != EOF)
{
if (lexer->cur_c == '"')
{
scanString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (lexer->cur_c == 'r' && (lexer->next_c == '#' || lexer->next_c == '"'))
{
scanRawString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (lexer->cur_c == '\'')
{
scanCharacterOrLifetime(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (isIdentifierStart(lexer->cur_c))
{
scanIdentifier(lexer);
return lexer->cur_token = TOKEN_IDENT;
}
/* These shift tokens aren't too important for tag-generation per se,
* but they confuse the skipUntil code which tracks the <> pairs. */
else if (lexer->cur_c == '>' && lexer->next_c == '>')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_RSHIFT;
}
else if (lexer->cur_c == '<' && lexer->next_c == '<')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_LSHIFT;
}
else if (lexer->cur_c == '-' && lexer->next_c == '>')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_RARROW;
}
else
{
int c = lexer->cur_c;
advanceChar(lexer);
return lexer->cur_token = c;
}
}
return lexer->cur_token = TOKEN_EOF;
}