/**
* Parse and construct lexer token
*
* Note:
* Currently, lexer token doesn't fully correspond to Token, defined in ECMA-262, v5, 7.5.
* For example, there is no new-line token type in the token definition of ECMA-262 v5.
*
* @return constructed token
*/
static token
lexer_parse_token (void)
{
ecma_char_t c = LA (0);
if (lit_char_is_white_space (c))
{
while (lit_char_is_white_space (c))
{
consume_char ();
c = LA (0);
}
}
if (lit_char_is_line_terminator (c))
{
while (lit_char_is_line_terminator (c))
{
consume_char ();
c = LA (0);
}
return create_token (TOK_NEWLINE, 0);
}
JERRY_ASSERT (is_token_parse_in_progress == false);
/* ECMA-262 v5, 7.6, Identifier */
if (lexer_is_char_can_be_identifier_start (c))
{
return lexer_parse_identifier_or_keyword ();
}
/* ECMA-262 v5, 7.8.3, Numeric literal */
if (lit_char_is_decimal_digit (c)
|| (c == LIT_CHAR_DOT
&& lit_char_is_decimal_digit (LA (1))))
{
return lexer_parse_number ();
}
if (c == LIT_CHAR_LF)
{
consume_char ();
return create_token (TOK_NEWLINE, 0);
}
if (c == LIT_CHAR_NULL)
{
return create_token (TOK_EOF, 0);
}
if (c == LIT_CHAR_SINGLE_QUOTE
|| c == LIT_CHAR_DOUBLE_QUOTE)
{
return lexer_parse_string ();
}
/* ECMA-262 v5, 7.4, SingleLineComment or MultiLineComment */
if (c == LIT_CHAR_SLASH
&& (LA (1) == LIT_CHAR_SLASH
|| LA (1) == LIT_CHAR_ASTERISK))
{
if (lexer_parse_comment ())
{
return create_token (TOK_NEWLINE, 0);
}
else
{
return lexer_parse_token ();
}
}
if (c == LIT_CHAR_SLASH
&& !(prev_non_lf_token.type == TOK_NAME
|| prev_non_lf_token.type == TOK_NULL
|| prev_non_lf_token.type == TOK_BOOL
|| prev_non_lf_token.type == TOK_CLOSE_BRACE
|| prev_non_lf_token.type == TOK_CLOSE_SQUARE
|| prev_non_lf_token.type == TOK_CLOSE_PAREN
|| prev_non_lf_token.type == TOK_SMALL_INT
|| prev_non_lf_token.type == TOK_NUMBER
|| prev_non_lf_token.type == TOK_STRING
|| prev_non_lf_token.type == TOK_REGEXP))
{
return lexer_parse_regexp ();
}
/* ECMA-262 v5, 7.7, Punctuator */
switch (c)
{
case LIT_CHAR_LEFT_BRACE:
{
RETURN_PUNC (TOK_OPEN_BRACE);
break;
}
case LIT_CHAR_RIGHT_BRACE:
{
RETURN_PUNC (TOK_CLOSE_BRACE);
break;
}
case LIT_CHAR_LEFT_PAREN:
{
RETURN_PUNC (TOK_OPEN_PAREN);
break;
}
case LIT_CHAR_RIGHT_PAREN:
{
RETURN_PUNC (TOK_CLOSE_PAREN);
break;
}
case LIT_CHAR_LEFT_SQUARE:
{
RETURN_PUNC (TOK_OPEN_SQUARE);
break;
}
case LIT_CHAR_RIGHT_SQUARE:
{
RETURN_PUNC (TOK_CLOSE_SQUARE);
break;
}
case LIT_CHAR_DOT:
{
RETURN_PUNC (TOK_DOT);
break;
}
case LIT_CHAR_SEMICOLON:
{
RETURN_PUNC (TOK_SEMICOLON);
break;
}
case LIT_CHAR_COMMA:
{
RETURN_PUNC (TOK_COMMA);
break;
}
case LIT_CHAR_TILDE:
{
RETURN_PUNC (TOK_COMPL);
break;
}
case LIT_CHAR_COLON:
{
RETURN_PUNC (TOK_COLON);
break;
}
case LIT_CHAR_QUESTION:
{
RETURN_PUNC (TOK_QUERY);
break;
}
case LIT_CHAR_ASTERISK:
{
IF_LA_IS (LIT_CHAR_EQUALS, TOK_MULT_EQ, TOK_MULT);
break;
}
case LIT_CHAR_SLASH:
{
IF_LA_IS (LIT_CHAR_EQUALS, TOK_DIV_EQ, TOK_DIV);
break;
}
case LIT_CHAR_CIRCUMFLEX:
{
IF_LA_IS (LIT_CHAR_EQUALS, TOK_XOR_EQ, TOK_XOR);
break;
}
case LIT_CHAR_PERCENT:
{
IF_LA_IS (LIT_CHAR_EQUALS, TOK_MOD_EQ, TOK_MOD);
break;
}
case LIT_CHAR_PLUS:
{
IF_LA_IS_OR (LIT_CHAR_PLUS, TOK_DOUBLE_PLUS, LIT_CHAR_EQUALS, TOK_PLUS_EQ, TOK_PLUS);
break;
}
case LIT_CHAR_MINUS:
{
IF_LA_IS_OR (LIT_CHAR_MINUS, TOK_DOUBLE_MINUS, LIT_CHAR_EQUALS, TOK_MINUS_EQ, TOK_MINUS);
break;
}
case LIT_CHAR_AMPERSAND:
{
IF_LA_IS_OR (LIT_CHAR_AMPERSAND, TOK_DOUBLE_AND, LIT_CHAR_EQUALS, TOK_AND_EQ, TOK_AND);
break;
}
case LIT_CHAR_VLINE:
{
IF_LA_IS_OR (LIT_CHAR_VLINE, TOK_DOUBLE_OR, LIT_CHAR_EQUALS, TOK_OR_EQ, TOK_OR);
break;
}
case LIT_CHAR_LESS_THAN:
{
switch (LA (1))
{
case LIT_CHAR_LESS_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_LSHIFT_EQ, TOK_LSHIFT, 2); break;
case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_LESS_EQ, 2); break;
default: RETURN_PUNC (TOK_LESS);
}
break;
}
case LIT_CHAR_GREATER_THAN:
{
switch (LA (1))
{
case LIT_CHAR_GREATER_THAN:
{
switch (LA (2))
{
case LIT_CHAR_GREATER_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_RSHIFT_EX_EQ, TOK_RSHIFT_EX, 3); break;
case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_RSHIFT_EQ, 3); break;
default: RETURN_PUNC_EX (TOK_RSHIFT, 2);
}
break;
}
case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_GREATER_EQ, 2); break;
default: RETURN_PUNC (TOK_GREATER);
}
break;
}
case LIT_CHAR_EQUALS:
{
if (LA (1) == LIT_CHAR_EQUALS)
{
IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_TRIPLE_EQ, TOK_DOUBLE_EQ, 2);
}
else
{
RETURN_PUNC (TOK_EQ);
}
break;
}
case LIT_CHAR_EXCLAMATION:
{
if (LA (1) == LIT_CHAR_EQUALS)
{
IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_NOT_DOUBLE_EQ, TOK_NOT_EQ, 2);
}
else
{
RETURN_PUNC (TOK_NOT);
}
break;
}
}
PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Illegal character", lit_utf8_iterator_get_pos (&src_iter));
} /* lexer_parse_token */
static token
lexer_next_token_private (void)
{
ecma_char_t c = LA (0);
JERRY_ASSERT (token_start == NULL);
if (isalpha (c) || c == '$' || c == '_')
{
return parse_name ();
}
if (isdigit (c) || (c == '.' && isdigit (LA (1))))
{
return parse_number ();
}
if (c == '\n')
{
consume_char ();
return create_token (TOK_NEWLINE, 0);
}
if (c == '\0')
{
return create_token (TOK_EOF, 0);
}
if (c == '\'' || c == '"')
{
return parse_string ();
}
if (isspace (c))
{
grobble_whitespaces ();
return lexer_next_token_private ();
}
if (c == '/' && LA (1) == '*')
{
if (replace_comment_by_newline ())
{
token ret;
ret.type = TOK_NEWLINE;
ret.uid = 0;
return ret;
}
else
{
return lexer_next_token_private ();
}
}
if (c == '/')
{
if (LA (1) == '/')
{
replace_comment_by_newline ();
return lexer_next_token_private ();
}
else if (!(sent_token.type == TOK_NAME
|| sent_token.type == TOK_NULL
|| sent_token.type == TOK_BOOL
|| sent_token.type == TOK_CLOSE_BRACE
|| sent_token.type == TOK_CLOSE_SQUARE
|| sent_token.type == TOK_CLOSE_PAREN
|| sent_token.type == TOK_SMALL_INT
|| sent_token.type == TOK_NUMBER
|| sent_token.type == TOK_STRING
|| sent_token.type == TOK_REGEXP))
{
return parse_regexp ();
}
}
switch (c)
{
case '{': RETURN_PUNC (TOK_OPEN_BRACE); break;
case '}': RETURN_PUNC (TOK_CLOSE_BRACE); break;
case '(': RETURN_PUNC (TOK_OPEN_PAREN); break;
case ')': RETURN_PUNC (TOK_CLOSE_PAREN); break;
case '[': RETURN_PUNC (TOK_OPEN_SQUARE); break;
case ']': RETURN_PUNC (TOK_CLOSE_SQUARE); break;
case '.': RETURN_PUNC (TOK_DOT); break;
case ';': RETURN_PUNC (TOK_SEMICOLON); break;
case ',': RETURN_PUNC (TOK_COMMA); break;
case '~': RETURN_PUNC (TOK_COMPL); break;
case ':': RETURN_PUNC (TOK_COLON); break;
case '?': RETURN_PUNC (TOK_QUERY); break;
case '*': IF_LA_IS ('=', TOK_MULT_EQ, TOK_MULT); break;
case '/': IF_LA_IS ('=', TOK_DIV_EQ, TOK_DIV); break;
case '^': IF_LA_IS ('=', TOK_XOR_EQ, TOK_XOR); break;
case '%': IF_LA_IS ('=', TOK_MOD_EQ, TOK_MOD); break;
case '+': IF_LA_IS_OR ('+', TOK_DOUBLE_PLUS, '=', TOK_PLUS_EQ, TOK_PLUS); break;
case '-': IF_LA_IS_OR ('-', TOK_DOUBLE_MINUS, '=', TOK_MINUS_EQ, TOK_MINUS); break;
case '&': IF_LA_IS_OR ('&', TOK_DOUBLE_AND, '=', TOK_AND_EQ, TOK_AND); break;
case '|': IF_LA_IS_OR ('|', TOK_DOUBLE_OR, '=', TOK_OR_EQ, TOK_OR); break;
case '<':
{
switch (LA (1))
{
case '<': IF_LA_N_IS ('=', TOK_LSHIFT_EQ, TOK_LSHIFT, 2); break;
case '=': RETURN_PUNC_EX (TOK_LESS_EQ, 2); break;
default: RETURN_PUNC (TOK_LESS);
}
break;
}
case '>':
{
switch (LA (1))
{
case '>':
{
switch (LA (2))
{
case '>': IF_LA_N_IS ('=', TOK_RSHIFT_EX_EQ, TOK_RSHIFT_EX, 3); break;
case '=': RETURN_PUNC_EX (TOK_RSHIFT_EQ, 3); break;
default: RETURN_PUNC_EX (TOK_RSHIFT, 2);
}
break;
}
case '=': RETURN_PUNC_EX (TOK_GREATER_EQ, 2); break;
default: RETURN_PUNC (TOK_GREATER);
}
break;
}
case '=':
{
if (LA (1) == '=')
{
IF_LA_N_IS ('=', TOK_TRIPLE_EQ, TOK_DOUBLE_EQ, 2);
}
else
{
RETURN_PUNC (TOK_EQ);
}
break;
}
case '!':
{
if (LA (1) == '=')
{
IF_LA_N_IS ('=', TOK_NOT_DOUBLE_EQ, TOK_NOT_EQ, 2);
}
else
{
RETURN_PUNC (TOK_NOT);
}
break;
}
default: PARSE_SORRY ("Unknown character", buffer - buffer_start);
}
PARSE_SORRY ("Unknown character", buffer - buffer_start);
}
/**
* Parse and construct lexer token
*
* Note:
* Currently, lexer token doesn't fully correspond to Token, defined in ECMA-262, v5, 7.5.
* For example, there is no new-line token type in the token definition of ECMA-262 v5.
*
* Note:
* For Lexer alone, it is hard to find out a token is whether a regexp or a division.
* Parser must set maybe_regexp to true if a regexp is expected.
* Otherwise, a division is expected.
*
* @return constructed token
*/
static token
lexer_parse_token (bool maybe_regexp, /**< read '/' as regexp? */
bool *out_is_preceed_by_new_lines_p, /**< out: is constructed token preceded by newlines? */
bool is_strict) /**< flag, indicating whether current code is in strict mode code */
{
JERRY_ASSERT (is_token_parse_in_progress == false);
*out_is_preceed_by_new_lines_p = lexer_skip_whitespace_and_comments ();
ecma_char_t c = LA (0);
/* ECMA-262 v5, 7.6, Identifier */
if (lexer_is_char_can_be_identifier_start (c))
{
return lexer_parse_identifier_or_keyword (is_strict);
}
/* ECMA-262 v5, 7.8.3, Numeric literal */
if (lit_char_is_decimal_digit (c)
|| (c == LIT_CHAR_DOT
&& lit_char_is_decimal_digit (LA (1))))
{
return lexer_parse_number (is_strict);
}
if (c == LIT_CHAR_NULL)
{
return create_token (TOK_EOF, 0);
}
if (c == LIT_CHAR_SINGLE_QUOTE
|| c == LIT_CHAR_DOUBLE_QUOTE)
{
return lexer_parse_string ();
}
if (c == LIT_CHAR_SLASH && maybe_regexp)
{
return lexer_parse_regexp ();
}
/* ECMA-262 v5, 7.7, Punctuator */
switch (c)
{
case LIT_CHAR_LEFT_BRACE:
{
RETURN_PUNC (TOK_OPEN_BRACE);
break;
}
case LIT_CHAR_RIGHT_BRACE:
{
RETURN_PUNC (TOK_CLOSE_BRACE);
break;
}
case LIT_CHAR_LEFT_PAREN:
{
RETURN_PUNC (TOK_OPEN_PAREN);
break;
}
case LIT_CHAR_RIGHT_PAREN:
{
RETURN_PUNC (TOK_CLOSE_PAREN);
break;
}
case LIT_CHAR_LEFT_SQUARE:
{
RETURN_PUNC (TOK_OPEN_SQUARE);
break;
}
case LIT_CHAR_RIGHT_SQUARE:
{
RETURN_PUNC (TOK_CLOSE_SQUARE);
break;
}
case LIT_CHAR_DOT:
{
RETURN_PUNC (TOK_DOT);
break;
}
case LIT_CHAR_SEMICOLON:
{
RETURN_PUNC (TOK_SEMICOLON);
break;
}
case LIT_CHAR_COMMA:
{
RETURN_PUNC (TOK_COMMA);
break;
}
case LIT_CHAR_TILDE:
{
RETURN_PUNC (TOK_COMPL);
break;
}
case LIT_CHAR_COLON:
{
RETURN_PUNC (TOK_COLON);
break;
}
case LIT_CHAR_QUESTION:
{
RETURN_PUNC (TOK_QUERY);
break;
}
case LIT_CHAR_ASTERISK:
{
IF_LA_IS (LIT_CHAR_EQUALS, TOK_MULT_EQ, TOK_MULT);
break;
}
case LIT_CHAR_SLASH:
{
IF_LA_IS (LIT_CHAR_EQUALS, TOK_DIV_EQ, TOK_DIV);
break;
}
case LIT_CHAR_CIRCUMFLEX:
{
IF_LA_IS (LIT_CHAR_EQUALS, TOK_XOR_EQ, TOK_XOR);
break;
}
case LIT_CHAR_PERCENT:
{
IF_LA_IS (LIT_CHAR_EQUALS, TOK_MOD_EQ, TOK_MOD);
break;
}
case LIT_CHAR_PLUS:
{
IF_LA_IS_OR (LIT_CHAR_PLUS, TOK_DOUBLE_PLUS, LIT_CHAR_EQUALS, TOK_PLUS_EQ, TOK_PLUS);
break;
}
case LIT_CHAR_MINUS:
{
IF_LA_IS_OR (LIT_CHAR_MINUS, TOK_DOUBLE_MINUS, LIT_CHAR_EQUALS, TOK_MINUS_EQ, TOK_MINUS);
break;
}
case LIT_CHAR_AMPERSAND:
{
IF_LA_IS_OR (LIT_CHAR_AMPERSAND, TOK_DOUBLE_AND, LIT_CHAR_EQUALS, TOK_AND_EQ, TOK_AND);
break;
}
case LIT_CHAR_VLINE:
{
IF_LA_IS_OR (LIT_CHAR_VLINE, TOK_DOUBLE_OR, LIT_CHAR_EQUALS, TOK_OR_EQ, TOK_OR);
break;
}
case LIT_CHAR_LESS_THAN:
{
switch (LA (1))
{
case LIT_CHAR_LESS_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_LSHIFT_EQ, TOK_LSHIFT, 2); break;
case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_LESS_EQ, 2); break;
default: RETURN_PUNC (TOK_LESS);
}
break;
}
case LIT_CHAR_GREATER_THAN:
{
switch (LA (1))
{
case LIT_CHAR_GREATER_THAN:
{
switch (LA (2))
{
case LIT_CHAR_GREATER_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_RSHIFT_EX_EQ, TOK_RSHIFT_EX, 3); break;
case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_RSHIFT_EQ, 3); break;
default: RETURN_PUNC_EX (TOK_RSHIFT, 2);
}
break;
}
case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_GREATER_EQ, 2); break;
default: RETURN_PUNC (TOK_GREATER);
}
break;
}
case LIT_CHAR_EQUALS:
{
if (LA (1) == LIT_CHAR_EQUALS)
{
IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_TRIPLE_EQ, TOK_DOUBLE_EQ, 2);
}
else
{
RETURN_PUNC (TOK_EQ);
}
break;
}
case LIT_CHAR_EXCLAMATION:
{
if (LA (1) == LIT_CHAR_EQUALS)
{
IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_NOT_DOUBLE_EQ, TOK_NOT_EQ, 2);
}
else
{
RETURN_PUNC (TOK_NOT);
}
break;
}
}
PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Illegal character", lit_utf8_iterator_get_pos (&src_iter));
} /* lexer_parse_token */
