static token convert_seen_num_to_token (ecma_number_t num) { literal_t lit = lit_find_literal_by_num (num); if (lit != NULL) { return create_token_from_lit (TOK_NUMBER, lit); } return create_token_from_lit (TOK_NUMBER, lit_create_literal_from_num (num)); }
/** * Create token of specified type from charset * * @return token descriptor */ static token lexer_create_token_for_charset (token_type tt, /**< token type */ const lit_utf8_byte_t *charset_p, /**< charset buffer */ lit_utf8_size_t size) /**< size of the charset */ { JERRY_ASSERT (charset_p != NULL); literal_t lit = lit_find_literal_by_utf8_string (charset_p, size); if (lit != NULL) { return create_token_from_lit (tt, lit); } lit = lit_create_literal_from_utf8_string (charset_p, size); JERRY_ASSERT (lit->get_type () == LIT_STR_T || lit->get_type () == LIT_MAGIC_STR_T || lit->get_type () == LIT_MAGIC_STR_EX_T); return create_token_from_lit (tt, lit); } /* lexer_create_token_for_charset */
/** * Convert string to token of specified type * * @return token descriptor */ static token convert_string_to_token (token_type tt, /**< token type */ const lit_utf8_byte_t *str_p, /**< characters buffer */ lit_utf8_size_t length) /**< string's length */ { JERRY_ASSERT (str_p != NULL); literal_t lit = lit_find_literal_by_utf8_string (str_p, length); if (lit != NULL) { return create_token_from_lit (tt, lit); } lit = lit_create_literal_from_utf8_string (str_p, length); JERRY_ASSERT (lit->get_type () == LIT_STR_T || lit->get_type () == LIT_MAGIC_STR_T || lit->get_type () == LIT_MAGIC_STR_EX_T); return create_token_from_lit (tt, lit); }
/** * Create token of specified type from charset * * @return token descriptor */ static token lexer_create_token_for_charset (jsp_token_type_t tt, /**< token type */ const lit_utf8_byte_t *charset_p, /**< charset buffer */ lit_utf8_size_t size) /**< size of the charset */ { JERRY_ASSERT (charset_p != NULL); lit_utf8_iterator_t iter = lit_utf8_iterator_create (charset_p, (lit_utf8_size_t) size); lit_utf8_size_t new_size = 0; lit_utf8_size_t new_length = 0; bool should_convert = false; while (!lit_utf8_iterator_is_eos (&iter)) { if (iter.buf_pos.is_non_bmp_middle) { should_convert = true; } lit_utf8_iterator_incr (&iter); new_size += LIT_CESU8_MAX_BYTES_IN_CODE_UNIT; } lit_utf8_byte_t *converted_str_p; if (unlikely (should_convert)) { lit_utf8_iterator_seek_bos (&iter); converted_str_p = (lit_utf8_byte_t *) jsp_mm_alloc (new_size); while (!lit_utf8_iterator_is_eos (&iter)) { ecma_char_t ch = lit_utf8_iterator_read_next (&iter); new_length += lit_code_unit_to_utf8 (ch, converted_str_p + new_length); } } else { converted_str_p = (lit_utf8_byte_t *) charset_p; new_length = size; JERRY_ASSERT (lit_is_cesu8_string_valid (converted_str_p, new_length)); } lit_literal_t lit = lit_find_literal_by_utf8_string (converted_str_p, new_length); if (lit != NULL) { if (unlikely (should_convert)) { jsp_mm_free (converted_str_p); } return create_token_from_lit (tt, lit); } lit = lit_create_literal_from_utf8_string (converted_str_p, new_length); rcs_record_type_t type = rcs_record_get_type (lit); JERRY_ASSERT (RCS_RECORD_TYPE_IS_CHARSET (type) || RCS_RECORD_TYPE_IS_MAGIC_STR (type) || RCS_RECORD_TYPE_IS_MAGIC_STR_EX (type)); if (unlikely (should_convert)) { jsp_mm_free (converted_str_p); } return create_token_from_lit (tt, lit); } /* lexer_create_token_for_charset */