/**
 * Helper function to try to parse a part of a date string
 *
 * @return NaN if cannot read from string, ToNumber() otherwise
 */
static ecma_number_t
ecma_date_parse_date_chars (lit_utf8_byte_t **str_p, /**< pointer to the cesu8 string */
                            const lit_utf8_byte_t *str_end_p, /**< pointer to the end of the string */
                            uint32_t num_of_chars) /**< number of characters to read and convert */
{
  JERRY_ASSERT (num_of_chars > 0);
  const lit_utf8_byte_t *str_start_p = *str_p;

  while (num_of_chars--)
  {
    if (*str_p >= str_end_p || !lit_char_is_decimal_digit (lit_utf8_read_next (str_p)))
    {
      return ecma_number_make_nan ();
    }
  }

  return ecma_utf8_string_to_number (str_start_p, (lit_utf8_size_t) (*str_p - str_start_p));
} /* ecma_date_parse_date_chars */
/**
 * Helper function to try to parse a part of a date string
 *
 * @return NaN if cannot read from string, ToNumber() otherwise
 */
static ecma_number_t
ecma_date_parse_date_chars (lit_utf8_iterator_t *iter, /**< iterator of the utf8 string */
                            uint32_t num_of_chars) /**< number of characters to read and convert */
{
  JERRY_ASSERT (num_of_chars > 0);

  lit_utf8_size_t copy_size = 0;
  const lit_utf8_byte_t *str_start_p = iter->buf_p + iter->buf_pos.offset;

  while (num_of_chars--)
  {
    if (lit_utf8_iterator_is_eos (iter)
        || !lit_char_is_decimal_digit (lit_utf8_iterator_peek_next (iter)))
    {
      return ecma_number_make_nan ();
    }

    copy_size += lit_get_unicode_char_size_by_utf8_first_byte (*(iter->buf_p + iter->buf_pos.offset));
    lit_utf8_iterator_incr (iter);
  }

  return ecma_utf8_string_to_number (str_start_p, copy_size);
} /* ecma_date_parse_date_chars */
Beispiel #3
0
/**
 * Read the input pattern and parse the next token for the RegExp compiler
 *
 * @return completion value
 *         Returned value must be freed with ecma_free_completion_value
 */
ecma_completion_value_t
re_parse_next_token (re_parser_ctx_t *parser_ctx_p, /**< RegExp parser context */
                     re_token_t *out_token_p) /**< out: output token */
{
  ecma_completion_value_t ret_value = ecma_make_empty_completion_value ();

  if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
  {
    out_token_p->type = RE_TOK_EOF;
    return ret_value;
  }

  ecma_char_t ch = lit_utf8_read_next (&parser_ctx_p->input_curr_p);

  switch (ch)
  {
    case LIT_CHAR_VLINE:
    {
      out_token_p->type = RE_TOK_ALTERNATIVE;
      break;
    }
    case LIT_CHAR_CIRCUMFLEX:
    {
      out_token_p->type = RE_TOK_ASSERT_START;
      break;
    }
    case LIT_CHAR_DOLLAR_SIGN:
    {
      out_token_p->type = RE_TOK_ASSERT_END;
      break;
    }
    case LIT_CHAR_DOT:
    {
      out_token_p->type = RE_TOK_PERIOD;
      ret_value = re_parse_iterator (parser_ctx_p, out_token_p);
      break;
    }
    case LIT_CHAR_BACKSLASH:
    {
      if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
      {
        return ecma_raise_syntax_error ("invalid regular experssion");
      }

      out_token_p->type = RE_TOK_CHAR;
      ch = lit_utf8_read_next (&parser_ctx_p->input_curr_p);

      if (ch == LIT_CHAR_LOWERCASE_B)
      {
        out_token_p->type = RE_TOK_ASSERT_WORD_BOUNDARY;
      }
      else if (ch == LIT_CHAR_UPPERCASE_B)
      {
        out_token_p->type = RE_TOK_ASSERT_NOT_WORD_BOUNDARY;
      }
      else if (ch == LIT_CHAR_LOWERCASE_F)
      {
        out_token_p->value = LIT_CHAR_FF;
      }
      else if (ch == LIT_CHAR_LOWERCASE_N)
      {
        out_token_p->value = LIT_CHAR_LF;
      }
      else if (ch == LIT_CHAR_LOWERCASE_T)
      {
        out_token_p->value = LIT_CHAR_TAB;
      }
      else if (ch == LIT_CHAR_LOWERCASE_R)
      {
        out_token_p->value = LIT_CHAR_CR;
      }
      else if (ch == LIT_CHAR_LOWERCASE_V)
      {
        out_token_p->value = LIT_CHAR_VTAB;
      }
      else if (ch == LIT_CHAR_LOWERCASE_C)
      {
        if (parser_ctx_p->input_curr_p < parser_ctx_p->input_end_p)
        {
          ch = *parser_ctx_p->input_curr_p;

          if ((ch >= LIT_CHAR_ASCII_UPPERCASE_LETTERS_BEGIN && ch <= LIT_CHAR_ASCII_UPPERCASE_LETTERS_END)
              || (ch >= LIT_CHAR_ASCII_LOWERCASE_LETTERS_BEGIN && ch <= LIT_CHAR_ASCII_LOWERCASE_LETTERS_END))
          {
            out_token_p->value = (ch % 32);
            parser_ctx_p->input_curr_p++;
          }
          else
          {
            out_token_p->value = LIT_CHAR_BACKSLASH;
            parser_ctx_p->input_curr_p--;
          }
        }
        else
        {
          out_token_p->value = LIT_CHAR_BACKSLASH;
          parser_ctx_p->input_curr_p--;
        }
      }
      else if (ch == LIT_CHAR_LOWERCASE_X
               && re_hex_lookup (parser_ctx_p, 2))
      {
        lit_code_point_t code_point;

        if (!lit_read_code_point_from_hex (parser_ctx_p->input_curr_p, 2, &code_point))
        {
          return ecma_raise_syntax_error ("decode error");
        }

        parser_ctx_p->input_curr_p += 2;
        out_token_p->value = code_point;
      }
      else if (ch == LIT_CHAR_LOWERCASE_U
               && re_hex_lookup (parser_ctx_p, 4))
      {
        lit_code_point_t code_point;

        if (!lit_read_code_point_from_hex (parser_ctx_p->input_curr_p, 4, &code_point))
        {
          return ecma_raise_syntax_error ("decode error");
        }

        parser_ctx_p->input_curr_p += 4;
        out_token_p->value = code_point;
      }
      else if (ch == LIT_CHAR_LOWERCASE_D)
      {
        out_token_p->type = RE_TOK_DIGIT;
        break;
      }
      else if (ch == LIT_CHAR_UPPERCASE_D)
      {
        out_token_p->type = RE_TOK_NOT_DIGIT;
        break;
      }
      else if (ch == LIT_CHAR_LOWERCASE_S)
      {
        out_token_p->type = RE_TOK_WHITE;
        break;
      }
      else if (ch == LIT_CHAR_UPPERCASE_S)
      {
        out_token_p->type = RE_TOK_NOT_WHITE;
        break;
      }
      else if (ch == LIT_CHAR_LOWERCASE_W)
      {
        out_token_p->type = RE_TOK_WORD_CHAR;
        break;
      }
      else if (ch == LIT_CHAR_UPPERCASE_W)
      {
        out_token_p->type = RE_TOK_NOT_WORD_CHAR;
        break;
      }
      else if (lit_char_is_decimal_digit (ch))
      {
        if (ch == LIT_CHAR_0)
        {
          if (parser_ctx_p->input_curr_p < parser_ctx_p->input_end_p
              && lit_char_is_decimal_digit (*parser_ctx_p->input_curr_p))
          {
            return ecma_raise_syntax_error ("RegExp escape pattern error.");
          }

          out_token_p->value = LIT_UNICODE_CODE_POINT_NULL;
        }
        else
        {
          if (parser_ctx_p->num_of_groups == -1)
          {
            re_count_num_of_groups (parser_ctx_p);
          }

          if (parser_ctx_p->num_of_groups)
          {
            parser_ctx_p->input_curr_p--;
            uint32_t number = 0;
            int index = 0;

            do
            {
              if (index >= RE_MAX_RE_DECESC_DIGITS)
              {
                ret_value = ecma_raise_syntax_error ("RegExp escape pattern error: decimal escape too long.");
                return ret_value;
              }
              if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
              {
                break;
              }

              ecma_char_t digit = *parser_ctx_p->input_curr_p++;

              if (!lit_char_is_decimal_digit (digit))
              {
                parser_ctx_p->input_curr_p--;
                break;
              }
              number = number * 10 + lit_char_hex_to_int (digit);
              index++;
            }
            while (true);

            if ((int) number <= parser_ctx_p->num_of_groups)
            {
              out_token_p->type = RE_TOK_BACKREFERENCE;
            }
            else
            /* Invalid backreference, fallback to octal */
            {
              /* Rewind to start of number. */
              parser_ctx_p->input_curr_p -= index;

              /* Try to reparse as octal. */
              ecma_char_t digit = *parser_ctx_p->input_curr_p;

              if (!lit_char_is_octal_digit (digit))
              {
                /* Not octal, keep digit character value. */
                number = digit;
                parser_ctx_p->input_curr_p++;
              }
              else
              {
                number = re_parse_octal (parser_ctx_p);
              }
            }
            out_token_p->value = number;
          }
          else
          /* Invalid backreference, fallback to octal if possible */
          {
            if (!lit_char_is_octal_digit (ch))
            {
              /* Not octal, keep character value. */
              out_token_p->value = ch;
            }
            else
            {
              parser_ctx_p->input_curr_p--;
              out_token_p->value = re_parse_octal (parser_ctx_p);
            }
          }
        }
      }
      else
      {
        out_token_p->value = ch;
      }

      ret_value = re_parse_iterator (parser_ctx_p, out_token_p);
      break;
    }
    case LIT_CHAR_LEFT_PAREN:
    {
      if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
      {
        return ecma_raise_syntax_error ("Unterminated group");
      }

      if (*parser_ctx_p->input_curr_p == LIT_CHAR_QUESTION)
      {
        parser_ctx_p->input_curr_p++;
        if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
        {
          return ecma_raise_syntax_error ("Invalid group");
        }

        ch = *parser_ctx_p->input_curr_p++;

        if (ch == LIT_CHAR_EQUALS)
        {
          /* (?= */
          out_token_p->type = RE_TOK_ASSERT_START_POS_LOOKAHEAD;
        }
        else if (ch == LIT_CHAR_EXCLAMATION)
        {
          /* (?! */
          out_token_p->type = RE_TOK_ASSERT_START_NEG_LOOKAHEAD;
        }
        else if (ch == LIT_CHAR_COLON)
        {
          /* (?: */
          out_token_p->type = RE_TOK_START_NON_CAPTURE_GROUP;
        }
        else
        {
          return ecma_raise_syntax_error ("Invalid group");
        }
      }
      else
      {
        /* ( */
        out_token_p->type = RE_TOK_START_CAPTURE_GROUP;
      }
      break;
    }
    case LIT_CHAR_RIGHT_PAREN:
    {
      out_token_p->type = RE_TOK_END_GROUP;
      ret_value = re_parse_iterator (parser_ctx_p, out_token_p);
      break;
    }
    case LIT_CHAR_LEFT_SQUARE:
    {
      out_token_p->type = RE_TOK_START_CHAR_CLASS;

      if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
      {
        return ecma_raise_syntax_error ("invalid character class");
      }

      if (*parser_ctx_p->input_curr_p == LIT_CHAR_CIRCUMFLEX)
      {
        out_token_p->type = RE_TOK_START_INV_CHAR_CLASS;
        parser_ctx_p->input_curr_p++;
      }

      break;
    }
    case LIT_CHAR_QUESTION:
    case LIT_CHAR_ASTERISK:
    case LIT_CHAR_PLUS:
    case LIT_CHAR_LEFT_BRACE:
    {
      return ecma_raise_syntax_error ("Invalid RegExp token.");
    }
    case LIT_CHAR_NULL:
    {
      out_token_p->type = RE_TOK_EOF;
      break;
    }
    default:
    {
      out_token_p->type = RE_TOK_CHAR;
      out_token_p->value = ch;
      ret_value = re_parse_iterator (parser_ctx_p, out_token_p);
      break;
    }
  }

  return ret_value;
} /* re_parse_next_token */
Beispiel #4
0
/**
 * Parse numeric literal (ECMA-262, v5, 7.8.3)
 *
 * @return token of TOK_SMALL_INT or TOK_NUMBER types
 */
static token
lexer_parse_number (void)
{
  ecma_char_t c = LA (0);
  bool is_hex = false;
  bool is_fp = false;
  ecma_number_t fp_res = .0;
  size_t tok_length = 0, i;
  token known_token;

  JERRY_ASSERT (lit_char_is_decimal_digit (c)
                || c == LIT_CHAR_DOT);

  if (c == LIT_CHAR_0)
  {
    if (LA (1) == LIT_CHAR_LOWERCASE_X
        || LA (1) == LIT_CHAR_UPPERCASE_X)
    {
      is_hex = true;
    }
  }
  else if (c == LIT_CHAR_DOT)
  {
    JERRY_ASSERT (lit_char_is_decimal_digit (LA (1)));
    is_fp = true;
  }

  if (is_hex)
  {
    // Eat up '0x'
    consume_char ();
    consume_char ();
    new_token ();

    c = LA (0);
    if (!lit_char_is_hex_digit (c))
    {
      PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Invalid HexIntegerLiteral", lit_utf8_iterator_get_pos (&src_iter));
    }

    do
    {
      consume_char ();
      c = LA (0);
    }
    while (lit_char_is_hex_digit (c));

    if (lexer_is_char_can_be_identifier_start (c))
    {
      PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX,
                   "Identifier just after integer literal",
                   lit_utf8_iterator_get_pos (&src_iter));
    }

    tok_length = (size_t) (TOK_SIZE ());

    const lit_utf8_byte_t *fp_buf_p = TOK_START ();
    /* token is constructed at end of function */
    for (i = 0; i < tok_length; i++)
    {
      fp_res = fp_res * 16 + (ecma_number_t) lit_char_hex_to_int (fp_buf_p[i]);
    }
  }
  else
  {
    bool is_exp = false;

    new_token ();

    // Eat up '.'
    if (is_fp)
    {
      consume_char ();
    }

    while (true)
    {
      c = LA (0);

      if (c == LIT_CHAR_DOT)
      {
        if (is_fp)
        {
          /* token is constructed at end of function */
          break;
        }
        else
        {
          is_fp = true;
          consume_char ();

          continue;
        }
      }
      else if (c == LIT_CHAR_LOWERCASE_E
               || c == LIT_CHAR_UPPERCASE_E)
      {
        if (is_exp)
        {
          PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX,
                       "Numeric literal shall not contain more than exponential marker ('e' or 'E')",
                       lit_utf8_iterator_get_pos (&src_iter));
        }
        else
        {
          is_exp = true;
          consume_char ();

          if (LA (0) == LIT_CHAR_MINUS
              || LA (0) == LIT_CHAR_PLUS)
          {
            consume_char ();
          }

          continue;
        }
      }
      else if (!lit_char_is_decimal_digit (c))
      {
        if (lexer_is_char_can_be_identifier_start (c))
        {
          PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX,
                       "Numeric literal shall not contain non-numeric characters",
                       lit_utf8_iterator_get_pos (&src_iter));
        }

        /* token is constructed at end of function */
        break;
      }

      consume_char ();
    }

    tok_length = (size_t) (TOK_SIZE ());

    if (is_fp || is_exp)
    {
      ecma_number_t res = ecma_utf8_string_to_number (TOK_START (), (jerry_api_size_t) tok_length);
      JERRY_ASSERT (!ecma_number_is_nan (res));

      known_token = convert_seen_num_to_token (res);
      is_token_parse_in_progress = NULL;

      return known_token;
    }
    else if (*TOK_START () == LIT_CHAR_0
             && tok_length != 1)
    {
      /* Octal integer literals */
      if (strict_mode)
      {
        PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Octal integer literals are not allowed in strict mode", token_start_pos);
      }
      else
      {
        /* token is constructed at end of function */
        const lit_utf8_byte_t *fp_buf_p = TOK_START ();

        for (i = 0; i < tok_length; i++)
        {
          fp_res = fp_res * 8 + (ecma_number_t) lit_char_hex_to_int (fp_buf_p[i]);
        }
      }
    }
    else
    {
      const lit_utf8_byte_t *fp_buf_p = TOK_START ();
      /* token is constructed at end of function */

      ecma_number_t mult = 1.0f;
      for (i = tok_length; i > 0; i--, mult *= 10)
      {
        fp_res += (ecma_number_t) lit_char_hex_to_int (fp_buf_p[i - 1]) * mult;
      }
    }
  }

  if (fp_res >= 0 && fp_res <= 255 && (uint8_t) fp_res == fp_res)
  {
    known_token = create_token (TOK_SMALL_INT, (uint8_t) fp_res);
    is_token_parse_in_progress = NULL;
    return known_token;
  }
  else
  {
    known_token = convert_seen_num_to_token (fp_res);
    is_token_parse_in_progress = NULL;
    return known_token;
  }
} /* lexer_parse_number */
Beispiel #5
0
/**
 * Parse RegExp iterators
 *
 * @return completion value
 *         Returned value must be freed with ecma_free_completion_value
 */
static ecma_completion_value_t
re_parse_iterator (re_parser_ctx_t *parser_ctx_p, /**< RegExp parser context */
                   re_token_t *re_token_p) /**< out: output token */
{
  ecma_completion_value_t ret_value = ecma_make_empty_completion_value ();

  re_token_p->qmin = 1;
  re_token_p->qmax = 1;
  re_token_p->greedy = true;

  if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
  {
    return ret_value;
  }

  ecma_char_t ch = *parser_ctx_p->input_curr_p;

  switch (ch)
  {
    case LIT_CHAR_QUESTION:
    {
      parser_ctx_p->input_curr_p++;
      re_token_p->qmin = 0;
      re_token_p->qmax = 1;
      re_token_p->greedy = !re_parse_non_greedy_char (parser_ctx_p);
      break;
    }
    case LIT_CHAR_ASTERISK:
    {
      parser_ctx_p->input_curr_p++;
      re_token_p->qmin = 0;
      re_token_p->qmax = RE_ITERATOR_INFINITE;
      re_token_p->greedy = !re_parse_non_greedy_char (parser_ctx_p);
      break;
    }
    case LIT_CHAR_PLUS:
    {
      parser_ctx_p->input_curr_p++;
      re_token_p->qmin = 1;
      re_token_p->qmax = RE_ITERATOR_INFINITE;
      re_token_p->greedy = !re_parse_non_greedy_char (parser_ctx_p);
      break;
    }
    case LIT_CHAR_LEFT_BRACE:
    {
      parser_ctx_p->input_curr_p++;
      uint32_t qmin = 0;
      uint32_t qmax = RE_ITERATOR_INFINITE;
      uint32_t digits = 0;

      while (true)
      {
        if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
        {
          return ecma_raise_syntax_error ("invalid quantifier");
        }

        ch = *parser_ctx_p->input_curr_p++;

        if (lit_char_is_decimal_digit (ch))
        {
          if (digits >= ECMA_NUMBER_MAX_DIGITS)
          {
            return ecma_raise_syntax_error ("RegExp quantifier error: too many digits.");
          }
          digits++;
          qmin = qmin * 10 + lit_char_hex_to_int (ch);
        }
        else if (ch == LIT_CHAR_COMMA)
        {
          if (qmax != RE_ITERATOR_INFINITE)
          {
            return ecma_raise_syntax_error ("RegExp quantifier error: double comma.");
          }

          if (parser_ctx_p->input_curr_p >= parser_ctx_p->input_end_p)
          {
            return ecma_raise_syntax_error ("invalid quantifier");
          }

          if (*parser_ctx_p->input_curr_p == LIT_CHAR_RIGHT_BRACE)
          {
            if (digits == 0)
            {
              return ecma_raise_syntax_error ("RegExp quantifier error: missing digits.");
            }

            parser_ctx_p->input_curr_p++;
            re_token_p->qmin = qmin;
            re_token_p->qmax = RE_ITERATOR_INFINITE;
            break;
          }
          qmax = qmin;
          qmin = 0;
          digits = 0;
        }
        else if (ch == LIT_CHAR_RIGHT_BRACE)
        {
          if (digits == 0)
          {
            return ecma_raise_syntax_error ("RegExp quantifier error: missing digits.");
          }

          if (qmax != RE_ITERATOR_INFINITE)
          {
            re_token_p->qmin = qmax;
            re_token_p->qmax = qmin;
          }
          else
          {
            re_token_p->qmin = qmin;
            re_token_p->qmax = qmin;
          }

          break;
        }
        else
        {
          return ecma_raise_syntax_error ("RegExp quantifier error: unknown char.");
        }
      }

      re_token_p->greedy = !re_parse_non_greedy_char (parser_ctx_p);
      break;
    }
    default:
    {
      break;
    }
  }

  JERRY_ASSERT (ecma_is_completion_value_empty (ret_value));

  if (re_token_p->qmin > re_token_p->qmax)
  {
    ret_value = ecma_raise_syntax_error ("RegExp quantifier error: qmin > qmax.");
  }

  return ret_value;
} /* re_parse_iterator */
Beispiel #6
0
/**
 * Transforming escape sequences in the charset, outputting converted string to specified buffer
 *
 * Note:
 *      Size of string with transformed escape sequences is always
 *      less or equal to size of corresponding source string.
 *
 * @return size of converted string
 */
static lit_utf8_size_t
lexer_transform_escape_sequences (const jerry_api_char_t *source_str_p, /**< string to convert,
                                                                         *   located in source buffer */
                                  lit_utf8_size_t source_str_size, /**< size of the string and of the output buffer */
                                  jerry_api_char_t *output_str_buf_p) /**< output buffer for converted string */
{
  if (source_str_size == 0)
  {
    return 0;
  }
  else
  {
    JERRY_ASSERT (source_str_p != NULL);
  }

  lit_utf8_byte_t *output_str_buf_iter_p = output_str_buf_p;
  const size_t output_str_buf_size = source_str_size;
  bool is_correct_sequence = true;

  lit_utf8_iterator_t source_str_iter = lit_utf8_iterator_create (source_str_p,
                                                                  source_str_size);

  ecma_char_t prev_converted_char = LIT_CHAR_NULL;

  while (!lit_utf8_iterator_is_eos (&source_str_iter))
  {
    ecma_char_t converted_char;

    const ecma_char_t next_char = lit_utf8_iterator_read_next (&source_str_iter);

    if (next_char == LIT_CHAR_BACKSLASH)
    {
      if (lit_utf8_iterator_is_eos (&source_str_iter))
      {
        is_correct_sequence = false;
        break;
      }

      const ecma_char_t char_after_next = lit_utf8_iterator_read_next (&source_str_iter);

      if (lit_char_is_decimal_digit (char_after_next))
      {
        if (lit_char_is_octal_digit (char_after_next))
        {
          if (char_after_next == LIT_CHAR_0
              && (lit_utf8_iterator_is_eos (&source_str_iter)
                  || !lit_char_is_octal_digit (lit_utf8_iterator_peek_next (&source_str_iter))))
          {
            converted_char = LIT_CHAR_NULL;
          }
          else
          {
            /* Implementation-defined (ECMA-262 v5, B.1.2): octal escape sequences are not implemented */
            is_correct_sequence = false;
            break;
          }
        }
        else
        {
          converted_char = char_after_next;
        }
      }
      else if (char_after_next == LIT_CHAR_LOWERCASE_U
               || char_after_next == LIT_CHAR_LOWERCASE_X)
      {
        if (!lexer_convert_escape_sequence_digits_to_char (&source_str_iter,
                                                           char_after_next == LIT_CHAR_LOWERCASE_U,
                                                           &converted_char))
        {
          is_correct_sequence = false;
          break;
        }
      }
      else if (lit_char_is_line_terminator (char_after_next))
      {
        /* Skip \, followed by a LineTerminatorSequence (ECMA-262, v5, 7.3) */
        if (char_after_next == LIT_CHAR_CR
            && !lit_utf8_iterator_is_eos (&source_str_iter)
            && lit_utf8_iterator_peek_next (&source_str_iter) == LIT_CHAR_LF)
        {
          lit_utf8_iterator_incr (&source_str_iter);
        }

        continue;
      }
      else
      {
        lexer_convert_single_escape_character (char_after_next, &converted_char);
      }
    }
    else
    {
      converted_char = next_char;
    }

    if (lit_is_code_unit_high_surrogate (prev_converted_char)
        && lit_is_code_unit_low_surrogate (converted_char))
    {
      output_str_buf_iter_p -= LIT_UTF8_MAX_BYTES_IN_CODE_UNIT;

      lit_code_point_t code_point = lit_convert_surrogate_pair_to_code_point (prev_converted_char,
                                                                              converted_char);
      output_str_buf_iter_p += lit_code_point_to_utf8 (code_point, output_str_buf_iter_p);
    }
    else
    {
      output_str_buf_iter_p += lit_code_unit_to_utf8 (converted_char, output_str_buf_iter_p);
      JERRY_ASSERT (output_str_buf_iter_p <= output_str_buf_p + output_str_buf_size);
    }

    prev_converted_char = converted_char;
  }

  if (is_correct_sequence)
  {
    return (lit_utf8_size_t) (output_str_buf_iter_p - output_str_buf_p);
  }
  else
  {
    PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Illegal escape sequence", token_start_pos);
  }
} /* lexer_transform_escape_sequences */
Beispiel #7
0
/**
 * Parse and construct lexer token
 *
 * Note:
 *      Currently, lexer token doesn't fully correspond to Token, defined in ECMA-262, v5, 7.5.
 *      For example, there is no new-line token type in the token definition of ECMA-262 v5.
 *
 * @return constructed token
 */
static token
lexer_parse_token (void)
{
  ecma_char_t c = LA (0);

  if (lit_char_is_white_space (c))
  {
    while (lit_char_is_white_space (c))
    {
      consume_char ();

      c = LA (0);
    }
  }

  if (lit_char_is_line_terminator (c))
  {
    while (lit_char_is_line_terminator (c))
    {
      consume_char ();

      c = LA (0);
    }

    return create_token (TOK_NEWLINE, 0);
  }

  JERRY_ASSERT (is_token_parse_in_progress == false);

  /* ECMA-262 v5, 7.6, Identifier */
  if (lexer_is_char_can_be_identifier_start (c))
  {
    return lexer_parse_identifier_or_keyword ();
  }

  /* ECMA-262 v5, 7.8.3, Numeric literal */
  if (lit_char_is_decimal_digit (c)
      || (c == LIT_CHAR_DOT
          && lit_char_is_decimal_digit (LA (1))))
  {
    return lexer_parse_number ();
  }

  if (c == LIT_CHAR_LF)
  {
    consume_char ();
    return create_token (TOK_NEWLINE, 0);
  }

  if (c == LIT_CHAR_NULL)
  {
    return create_token (TOK_EOF, 0);
  }

  if (c == LIT_CHAR_SINGLE_QUOTE
      || c == LIT_CHAR_DOUBLE_QUOTE)
  {
    return lexer_parse_string ();
  }

  /* ECMA-262 v5, 7.4, SingleLineComment or MultiLineComment */
  if (c == LIT_CHAR_SLASH
      && (LA (1) == LIT_CHAR_SLASH
          || LA (1) == LIT_CHAR_ASTERISK))
  {
    if (lexer_parse_comment ())
    {
      return create_token (TOK_NEWLINE, 0);
    }
    else
    {
      return lexer_parse_token ();
    }
  }

  if (c == LIT_CHAR_SLASH
      && !(prev_non_lf_token.type == TOK_NAME
           || prev_non_lf_token.type == TOK_NULL
           || prev_non_lf_token.type == TOK_BOOL
           || prev_non_lf_token.type == TOK_CLOSE_BRACE
           || prev_non_lf_token.type == TOK_CLOSE_SQUARE
           || prev_non_lf_token.type == TOK_CLOSE_PAREN
           || prev_non_lf_token.type == TOK_SMALL_INT
           || prev_non_lf_token.type == TOK_NUMBER
           || prev_non_lf_token.type == TOK_STRING
           || prev_non_lf_token.type == TOK_REGEXP))
  {
    return lexer_parse_regexp ();
  }

  /* ECMA-262 v5, 7.7, Punctuator */
  switch (c)
  {
    case LIT_CHAR_LEFT_BRACE:
    {
      RETURN_PUNC (TOK_OPEN_BRACE);
      break;
    }
    case LIT_CHAR_RIGHT_BRACE:
    {
      RETURN_PUNC (TOK_CLOSE_BRACE);
      break;
    }
    case LIT_CHAR_LEFT_PAREN:
    {
      RETURN_PUNC (TOK_OPEN_PAREN);
      break;
    }
    case LIT_CHAR_RIGHT_PAREN:
    {
      RETURN_PUNC (TOK_CLOSE_PAREN);
      break;
    }
    case LIT_CHAR_LEFT_SQUARE:
    {
      RETURN_PUNC (TOK_OPEN_SQUARE);
      break;
    }
    case LIT_CHAR_RIGHT_SQUARE:
    {
      RETURN_PUNC (TOK_CLOSE_SQUARE);
      break;
    }
    case LIT_CHAR_DOT:
    {
      RETURN_PUNC (TOK_DOT);
      break;
    }
    case LIT_CHAR_SEMICOLON:
    {
      RETURN_PUNC (TOK_SEMICOLON);
      break;
    }
    case LIT_CHAR_COMMA:
    {
      RETURN_PUNC (TOK_COMMA);
      break;
    }
    case LIT_CHAR_TILDE:
    {
      RETURN_PUNC (TOK_COMPL);
      break;
    }
    case LIT_CHAR_COLON:
    {
      RETURN_PUNC (TOK_COLON);
      break;
    }
    case LIT_CHAR_QUESTION:
    {
      RETURN_PUNC (TOK_QUERY);
      break;
    }

    case LIT_CHAR_ASTERISK:
    {
      IF_LA_IS (LIT_CHAR_EQUALS, TOK_MULT_EQ, TOK_MULT);
      break;
    }
    case LIT_CHAR_SLASH:
    {
      IF_LA_IS (LIT_CHAR_EQUALS, TOK_DIV_EQ, TOK_DIV);
      break;
    }
    case LIT_CHAR_CIRCUMFLEX:
    {
      IF_LA_IS (LIT_CHAR_EQUALS, TOK_XOR_EQ, TOK_XOR);
      break;
    }
    case LIT_CHAR_PERCENT:
    {
      IF_LA_IS (LIT_CHAR_EQUALS, TOK_MOD_EQ, TOK_MOD);
      break;
    }
    case LIT_CHAR_PLUS:
    {
      IF_LA_IS_OR (LIT_CHAR_PLUS, TOK_DOUBLE_PLUS, LIT_CHAR_EQUALS, TOK_PLUS_EQ, TOK_PLUS);
      break;
    }
    case LIT_CHAR_MINUS:
    {
      IF_LA_IS_OR (LIT_CHAR_MINUS, TOK_DOUBLE_MINUS, LIT_CHAR_EQUALS, TOK_MINUS_EQ, TOK_MINUS);
      break;
    }
    case LIT_CHAR_AMPERSAND:
    {
      IF_LA_IS_OR (LIT_CHAR_AMPERSAND, TOK_DOUBLE_AND, LIT_CHAR_EQUALS, TOK_AND_EQ, TOK_AND);
      break;
    }
    case LIT_CHAR_VLINE:
    {
      IF_LA_IS_OR (LIT_CHAR_VLINE, TOK_DOUBLE_OR, LIT_CHAR_EQUALS, TOK_OR_EQ, TOK_OR);
      break;
    }
    case LIT_CHAR_LESS_THAN:
    {
      switch (LA (1))
      {
        case LIT_CHAR_LESS_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_LSHIFT_EQ, TOK_LSHIFT, 2); break;
        case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_LESS_EQ, 2); break;
        default: RETURN_PUNC (TOK_LESS);
      }
      break;
    }
    case LIT_CHAR_GREATER_THAN:
    {
      switch (LA (1))
      {
        case LIT_CHAR_GREATER_THAN:
        {
          switch (LA (2))
          {
            case LIT_CHAR_GREATER_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_RSHIFT_EX_EQ, TOK_RSHIFT_EX, 3); break;
            case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_RSHIFT_EQ, 3); break;
            default: RETURN_PUNC_EX (TOK_RSHIFT, 2);
          }
          break;
        }
        case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_GREATER_EQ, 2); break;
        default: RETURN_PUNC (TOK_GREATER);
      }
      break;
    }
    case LIT_CHAR_EQUALS:
    {
      if (LA (1) == LIT_CHAR_EQUALS)
      {
        IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_TRIPLE_EQ, TOK_DOUBLE_EQ, 2);
      }
      else
      {
        RETURN_PUNC (TOK_EQ);
      }
      break;
    }
    case LIT_CHAR_EXCLAMATION:
    {
      if (LA (1) == LIT_CHAR_EQUALS)
      {
        IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_NOT_DOUBLE_EQ, TOK_NOT_EQ, 2);
      }
      else
      {
        RETURN_PUNC (TOK_NOT);
      }
      break;
    }
  }

  PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Illegal character", lit_utf8_iterator_get_pos (&src_iter));
} /* lexer_parse_token */
Beispiel #8
0
/**
 * Parse and construct lexer token
 *
 * Note:
 *      Currently, lexer token doesn't fully correspond to Token, defined in ECMA-262, v5, 7.5.
 *      For example, there is no new-line token type in the token definition of ECMA-262 v5.
 *
 * Note:
 *      For Lexer alone, it is hard to find out a token is whether a regexp or a division.
 *      Parser must set maybe_regexp to true if a regexp is expected.
 *      Otherwise, a division is expected.
 *
 * @return constructed token
 */
static token
lexer_parse_token (bool maybe_regexp, /**< read '/' as regexp? */
                   bool *out_is_preceed_by_new_lines_p, /**< out: is constructed token preceded by newlines? */
                   bool is_strict) /**< flag, indicating whether current code is in strict mode code */

{
  JERRY_ASSERT (is_token_parse_in_progress == false);

  *out_is_preceed_by_new_lines_p = lexer_skip_whitespace_and_comments ();

  ecma_char_t c = LA (0);

  /* ECMA-262 v5, 7.6, Identifier */
  if (lexer_is_char_can_be_identifier_start (c))
  {
    return lexer_parse_identifier_or_keyword (is_strict);
  }

  /* ECMA-262 v5, 7.8.3, Numeric literal */
  if (lit_char_is_decimal_digit (c)
      || (c == LIT_CHAR_DOT
          && lit_char_is_decimal_digit (LA (1))))
  {
    return lexer_parse_number (is_strict);
  }

  if (c == LIT_CHAR_NULL)
  {
    return create_token (TOK_EOF, 0);
  }

  if (c == LIT_CHAR_SINGLE_QUOTE
      || c == LIT_CHAR_DOUBLE_QUOTE)
  {
    return lexer_parse_string ();
  }

  if (c == LIT_CHAR_SLASH && maybe_regexp)
  {
    return lexer_parse_regexp ();
  }

  /* ECMA-262 v5, 7.7, Punctuator */
  switch (c)
  {
    case LIT_CHAR_LEFT_BRACE:
    {
      RETURN_PUNC (TOK_OPEN_BRACE);
      break;
    }
    case LIT_CHAR_RIGHT_BRACE:
    {
      RETURN_PUNC (TOK_CLOSE_BRACE);
      break;
    }
    case LIT_CHAR_LEFT_PAREN:
    {
      RETURN_PUNC (TOK_OPEN_PAREN);
      break;
    }
    case LIT_CHAR_RIGHT_PAREN:
    {
      RETURN_PUNC (TOK_CLOSE_PAREN);
      break;
    }
    case LIT_CHAR_LEFT_SQUARE:
    {
      RETURN_PUNC (TOK_OPEN_SQUARE);
      break;
    }
    case LIT_CHAR_RIGHT_SQUARE:
    {
      RETURN_PUNC (TOK_CLOSE_SQUARE);
      break;
    }
    case LIT_CHAR_DOT:
    {
      RETURN_PUNC (TOK_DOT);
      break;
    }
    case LIT_CHAR_SEMICOLON:
    {
      RETURN_PUNC (TOK_SEMICOLON);
      break;
    }
    case LIT_CHAR_COMMA:
    {
      RETURN_PUNC (TOK_COMMA);
      break;
    }
    case LIT_CHAR_TILDE:
    {
      RETURN_PUNC (TOK_COMPL);
      break;
    }
    case LIT_CHAR_COLON:
    {
      RETURN_PUNC (TOK_COLON);
      break;
    }
    case LIT_CHAR_QUESTION:
    {
      RETURN_PUNC (TOK_QUERY);
      break;
    }

    case LIT_CHAR_ASTERISK:
    {
      IF_LA_IS (LIT_CHAR_EQUALS, TOK_MULT_EQ, TOK_MULT);
      break;
    }
    case LIT_CHAR_SLASH:
    {
      IF_LA_IS (LIT_CHAR_EQUALS, TOK_DIV_EQ, TOK_DIV);
      break;
    }
    case LIT_CHAR_CIRCUMFLEX:
    {
      IF_LA_IS (LIT_CHAR_EQUALS, TOK_XOR_EQ, TOK_XOR);
      break;
    }
    case LIT_CHAR_PERCENT:
    {
      IF_LA_IS (LIT_CHAR_EQUALS, TOK_MOD_EQ, TOK_MOD);
      break;
    }
    case LIT_CHAR_PLUS:
    {
      IF_LA_IS_OR (LIT_CHAR_PLUS, TOK_DOUBLE_PLUS, LIT_CHAR_EQUALS, TOK_PLUS_EQ, TOK_PLUS);
      break;
    }
    case LIT_CHAR_MINUS:
    {
      IF_LA_IS_OR (LIT_CHAR_MINUS, TOK_DOUBLE_MINUS, LIT_CHAR_EQUALS, TOK_MINUS_EQ, TOK_MINUS);
      break;
    }
    case LIT_CHAR_AMPERSAND:
    {
      IF_LA_IS_OR (LIT_CHAR_AMPERSAND, TOK_DOUBLE_AND, LIT_CHAR_EQUALS, TOK_AND_EQ, TOK_AND);
      break;
    }
    case LIT_CHAR_VLINE:
    {
      IF_LA_IS_OR (LIT_CHAR_VLINE, TOK_DOUBLE_OR, LIT_CHAR_EQUALS, TOK_OR_EQ, TOK_OR);
      break;
    }
    case LIT_CHAR_LESS_THAN:
    {
      switch (LA (1))
      {
        case LIT_CHAR_LESS_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_LSHIFT_EQ, TOK_LSHIFT, 2); break;
        case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_LESS_EQ, 2); break;
        default: RETURN_PUNC (TOK_LESS);
      }
      break;
    }
    case LIT_CHAR_GREATER_THAN:
    {
      switch (LA (1))
      {
        case LIT_CHAR_GREATER_THAN:
        {
          switch (LA (2))
          {
            case LIT_CHAR_GREATER_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_RSHIFT_EX_EQ, TOK_RSHIFT_EX, 3); break;
            case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_RSHIFT_EQ, 3); break;
            default: RETURN_PUNC_EX (TOK_RSHIFT, 2);
          }
          break;
        }
        case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_GREATER_EQ, 2); break;
        default: RETURN_PUNC (TOK_GREATER);
      }
      break;
    }
    case LIT_CHAR_EQUALS:
    {
      if (LA (1) == LIT_CHAR_EQUALS)
      {
        IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_TRIPLE_EQ, TOK_DOUBLE_EQ, 2);
      }
      else
      {
        RETURN_PUNC (TOK_EQ);
      }
      break;
    }
    case LIT_CHAR_EXCLAMATION:
    {
      if (LA (1) == LIT_CHAR_EQUALS)
      {
        IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_NOT_DOUBLE_EQ, TOK_NOT_EQ, 2);
      }
      else
      {
        RETURN_PUNC (TOK_NOT);
      }
      break;
    }
  }

  PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Illegal character", lit_utf8_iterator_get_pos (&src_iter));
} /* lexer_parse_token */
/**
 * Parse next token.
 *
 * The function fills the fields of the ecma_json_token_t
 * argument and advances the string pointer.
 */
static void
ecma_builtin_json_parse_next_token (ecma_json_token_t *token_p) /**< token argument */
{
  lit_utf8_byte_t *current_p = token_p->current_p;
  token_p->type = invalid_token;

  /*
   * No need for end check since the string is zero terminated.
   */
  while (*current_p == LIT_CHAR_SP || *current_p == LIT_CHAR_CR
         || *current_p == LIT_CHAR_LF || *current_p == LIT_CHAR_TAB)
  {
    current_p++;
  }

  if (current_p == token_p->end_p)
  {
    token_p->type = end_token;
    return;
  }

  switch (*current_p)
  {
    case LIT_CHAR_LEFT_BRACE:
    {
      token_p->type = left_brace_token;
      token_p->current_p = current_p + 1;
      return;
    }
    case LIT_CHAR_RIGHT_BRACE:
    {
      token_p->type = right_brace_token;
      token_p->current_p = current_p + 1;
      return;
    }
    case LIT_CHAR_LEFT_SQUARE:
    {
      token_p->type = left_square_token;
      token_p->current_p = current_p + 1;
      return;
    }
    case LIT_CHAR_COMMA:
    {
      token_p->type = comma_token;
      token_p->current_p = current_p + 1;
      return;
    }
    case LIT_CHAR_COLON:
    {
      token_p->type = colon_token;
      token_p->current_p = current_p + 1;
      return;
    }
    case LIT_CHAR_DOUBLE_QUOTE:
    {
      token_p->current_p = current_p + 1;
      ecma_builtin_json_parse_string (token_p);
      return;
    }
    case LIT_CHAR_LOWERCASE_N:
    {
      if (ecma_builtin_json_check_id (current_p, "null"))
      {
        token_p->type = null_token;
        token_p->current_p = current_p + 4;
        return;
      }
      break;
    }
    case LIT_CHAR_LOWERCASE_T:
    {
      if (ecma_builtin_json_check_id (current_p, "true"))
      {
        token_p->type = true_token;
        token_p->current_p = current_p + 4;
        return;
      }
      break;
    }
    case LIT_CHAR_LOWERCASE_F:
    {
      if (ecma_builtin_json_check_id (current_p, "false"))
      {
        token_p->type = false_token;
        token_p->current_p = current_p + 5;
        return;
      }
      break;
    }
    default:
    {
      if (*current_p == LIT_CHAR_MINUS || lit_char_is_decimal_digit (*current_p))
      {
        token_p->current_p = current_p;
        ecma_builtin_json_parse_number (token_p);
        return;
      }
      break;
    }
  }
} /* ecma_builtin_json_parse_next_token */
/**
 * Parse and extract string token.
 */
static void
ecma_builtin_json_parse_number (ecma_json_token_t *token_p) /**< token argument */
{
  lit_utf8_byte_t *current_p = token_p->current_p;
  lit_utf8_byte_t *start_p = current_p;

  if (*current_p == LIT_CHAR_MINUS)
  {
    current_p++;
  }

  if (*current_p == LIT_CHAR_0)
  {
    current_p++;
    if (lit_char_is_decimal_digit (*current_p))
    {
      return;
    }
  }
  else if (lit_char_is_decimal_digit (*current_p))
  {
    do
    {
      current_p++;
    }
    while (lit_char_is_decimal_digit (*current_p));
  }

  if (*current_p == LIT_CHAR_DOT)
  {
    current_p++;
    if (!lit_char_is_decimal_digit (*current_p))
    {
      return;
    }

    do
    {
      current_p++;
    }
    while (lit_char_is_decimal_digit (*current_p));
  }

  if (*current_p == LIT_CHAR_LOWERCASE_E || *current_p == LIT_CHAR_UPPERCASE_E)
  {
    current_p++;
    if (*current_p == LIT_CHAR_PLUS || *current_p == LIT_CHAR_MINUS)
    {
      current_p++;
    }

    if (!lit_char_is_decimal_digit (*current_p))
    {
      return;
    }

    do
    {
      current_p++;
    }
    while (lit_char_is_decimal_digit (*current_p));
  }
  token_p->type = number_token;
  token_p->u.number = ecma_utf8_string_to_number (start_p, (lit_utf8_size_t) (current_p - start_p));

  token_p->current_p = current_p;
} /* ecma_builtin_json_parse_number */