token
lexer_next_token (void)
{
lit_utf8_iterator_pos_t src_pos = lit_utf8_iterator_get_pos (&src_iter);
if (src_pos.offset == 0 && !src_pos.is_non_bmp_middle)
{
dump_current_line ();
}
if (!is_empty (saved_token))
{
sent_token = saved_token;
saved_token = empty_token;
goto end;
}
/**
* FIXME:
* The way to raise syntax errors for unexpected EOF
* should be reworked so that EOF would be checked by
* caller of the routine, and the following condition
* would be checked as assertion in the routine.
*/
if (prev_token.type == TOK_EOF
&& sent_token.type == TOK_EOF)
{
PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Unexpected EOF", lit_utf8_iterator_get_pos (&src_iter));
}
prev_token = sent_token;
sent_token = lexer_parse_token ();
if (sent_token.type == TOK_NEWLINE)
{
dump_current_line ();
}
else
{
prev_non_lf_token = sent_token;
}
end:
return sent_token;
}
/**
* Skip any whitespace and comment tokens
*
* @return true - if a newline token was skipped,
* false - otherwise
*/
static bool
lexer_skip_whitespace_and_comments (void)
{
bool new_lines_occurred = false;
while (true)
{
ecma_char_t c = LA (0);
if (lit_char_is_white_space (c))
{
do
{
consume_char ();
c = LA (0);
}
while (lit_char_is_white_space (c));
}
else if (lit_char_is_line_terminator (c))
{
dump_current_line ();
new_lines_occurred = true;
do
{
consume_char ();
c = LA (0);
}
while (lit_char_is_line_terminator (c));
}
else if (c == LIT_CHAR_SLASH
&& (LA (1) == LIT_CHAR_SLASH
|| LA (1) == LIT_CHAR_ASTERISK))
{
/* ECMA-262 v5, 7.4, SingleLineComment or MultiLineComment */
if (lexer_parse_comment ())
{
new_lines_occurred = true;
}
}
else
{
break;
}
}
return new_lines_occurred;
} /* lexer_skip_whitespace_and_comments */
token
lexer_next_token (void)
{
if (buffer == buffer_start)
{
dump_current_line ();
}
if (!is_empty (saved_token))
{
sent_token = saved_token;
saved_token = empty_token;
goto end;
}
/**
* FIXME:
* The way to raise syntax errors for unexpected EOF
* should be reworked so that EOF would be checked by
* caller of the routine, and the following condition
* would be checked as assertion in the routine.
*/
if (prev_token.type == TOK_EOF
&& sent_token.type == TOK_EOF)
{
PARSE_ERROR ("Unexpected EOF", buffer - buffer_start);
}
prev_token = sent_token;
sent_token = lexer_next_token_private ();
if (sent_token.type == TOK_NEWLINE)
{
dump_current_line ();
}
end:
return sent_token;
}
/**
* Construct next token from current source code position and increment the position
*
* @return the constructed token
*/
token
lexer_next_token (bool maybe_regexp, /**< read '/' as regexp? */
bool is_strict) /**< strict mode is on (true) / off (false) */
{
lit_utf8_iterator_pos_t src_pos = lit_utf8_iterator_get_pos (&src_iter);
if (src_pos.offset == 0 && !src_pos.is_non_bmp_middle)
{
dump_current_line ();
}
if (!is_empty (saved_token))
{
sent_token = saved_token;
saved_token = empty_token;
}
else
{
/**
* FIXME:
* The way to raise syntax errors for unexpected EOF
* should be reworked so that EOF would be checked by
* caller of the routine, and the following condition
* would be checked as assertion in the routine.
*/
if (lexer_get_token_type (prev_token) == TOK_EOF
&& lexer_get_token_type (sent_token) == TOK_EOF)
{
PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Unexpected EOF", lit_utf8_iterator_get_pos (&src_iter));
}
prev_token = sent_token;
jsp_token_flag_t flags = JSP_TOKEN_FLAG__NO_FLAGS;
bool is_preceded_by_new_lines;
sent_token = lexer_parse_token (maybe_regexp, &is_preceded_by_new_lines, is_strict);
if (is_preceded_by_new_lines)
{
flags = (jsp_token_flag_t) (flags | JSP_TOKEN_FLAG_PRECEDED_BY_NEWLINES);
}
sent_token.flags = flags;
}
return sent_token;
} /* lexer_next_token */
