static token
convert_seen_num_to_token (ecma_number_t num)
{
literal_t lit = lit_find_literal_by_num (num);
if (lit != NULL)
{
return create_token_from_lit (TOK_NUMBER, lit);
}
return create_token_from_lit (TOK_NUMBER, lit_create_literal_from_num (num));
}
/**
* Create token of specified type from charset
*
* @return token descriptor
*/
static token
lexer_create_token_for_charset (token_type tt, /**< token type */
const lit_utf8_byte_t *charset_p, /**< charset buffer */
lit_utf8_size_t size) /**< size of the charset */
{
JERRY_ASSERT (charset_p != NULL);
literal_t lit = lit_find_literal_by_utf8_string (charset_p, size);
if (lit != NULL)
{
return create_token_from_lit (tt, lit);
}
lit = lit_create_literal_from_utf8_string (charset_p, size);
JERRY_ASSERT (lit->get_type () == LIT_STR_T
|| lit->get_type () == LIT_MAGIC_STR_T
|| lit->get_type () == LIT_MAGIC_STR_EX_T);
return create_token_from_lit (tt, lit);
} /* lexer_create_token_for_charset */
/**
* Convert string to token of specified type
*
* @return token descriptor
*/
static token
convert_string_to_token (token_type tt, /**< token type */
const lit_utf8_byte_t *str_p, /**< characters buffer */
lit_utf8_size_t length) /**< string's length */
{
JERRY_ASSERT (str_p != NULL);
literal_t lit = lit_find_literal_by_utf8_string (str_p, length);
if (lit != NULL)
{
return create_token_from_lit (tt, lit);
}
lit = lit_create_literal_from_utf8_string (str_p, length);
JERRY_ASSERT (lit->get_type () == LIT_STR_T
|| lit->get_type () == LIT_MAGIC_STR_T
|| lit->get_type () == LIT_MAGIC_STR_EX_T);
return create_token_from_lit (tt, lit);
}
/**
* Create token of specified type from charset
*
* @return token descriptor
*/
static token
lexer_create_token_for_charset (jsp_token_type_t tt, /**< token type */
const lit_utf8_byte_t *charset_p, /**< charset buffer */
lit_utf8_size_t size) /**< size of the charset */
{
JERRY_ASSERT (charset_p != NULL);
lit_utf8_iterator_t iter = lit_utf8_iterator_create (charset_p, (lit_utf8_size_t) size);
lit_utf8_size_t new_size = 0;
lit_utf8_size_t new_length = 0;
bool should_convert = false;
while (!lit_utf8_iterator_is_eos (&iter))
{
if (iter.buf_pos.is_non_bmp_middle)
{
should_convert = true;
}
lit_utf8_iterator_incr (&iter);
new_size += LIT_CESU8_MAX_BYTES_IN_CODE_UNIT;
}
lit_utf8_byte_t *converted_str_p;
if (unlikely (should_convert))
{
lit_utf8_iterator_seek_bos (&iter);
converted_str_p = (lit_utf8_byte_t *) jsp_mm_alloc (new_size);
while (!lit_utf8_iterator_is_eos (&iter))
{
ecma_char_t ch = lit_utf8_iterator_read_next (&iter);
new_length += lit_code_unit_to_utf8 (ch, converted_str_p + new_length);
}
}
else
{
converted_str_p = (lit_utf8_byte_t *) charset_p;
new_length = size;
JERRY_ASSERT (lit_is_cesu8_string_valid (converted_str_p, new_length));
}
lit_literal_t lit = lit_find_literal_by_utf8_string (converted_str_p, new_length);
if (lit != NULL)
{
if (unlikely (should_convert))
{
jsp_mm_free (converted_str_p);
}
return create_token_from_lit (tt, lit);
}
lit = lit_create_literal_from_utf8_string (converted_str_p, new_length);
rcs_record_type_t type = rcs_record_get_type (lit);
JERRY_ASSERT (RCS_RECORD_TYPE_IS_CHARSET (type)
|| RCS_RECORD_TYPE_IS_MAGIC_STR (type)
|| RCS_RECORD_TYPE_IS_MAGIC_STR_EX (type));
if (unlikely (should_convert))
{
jsp_mm_free (converted_str_p);
}
return create_token_from_lit (tt, lit);
} /* lexer_create_token_for_charset */
