/** * Check if a literal which holds the passed string exists. * If it doesn't exist, create a new one. * * @return pointer to existing or newly created record */ literal_t lit_find_or_create_literal_from_utf8_string (const lit_utf8_byte_t *str_p, /**< string, could be non-zero-terminated */ lit_utf8_size_t str_size) /**< length of the string */ { literal_t lit = lit_find_literal_by_utf8_string (str_p, str_size); if (lit == NULL) { lit = lit_create_literal_from_utf8_string (str_p, str_size); } return lit; } /* lit_find_or_create_literal_from_utf8_string */
/** * Convert string to token of specified type * * @return token descriptor */ static token convert_string_to_token (token_type tt, /**< token type */ const lit_utf8_byte_t *str_p, /**< characters buffer */ lit_utf8_size_t length) /**< string's length */ { JERRY_ASSERT (str_p != NULL); literal_t lit = lit_find_literal_by_utf8_string (str_p, length); if (lit != NULL) { return create_token_from_lit (tt, lit); } lit = lit_create_literal_from_utf8_string (str_p, length); JERRY_ASSERT (lit->get_type () == LIT_STR_T || lit->get_type () == LIT_MAGIC_STR_T || lit->get_type () == LIT_MAGIC_STR_EX_T); return create_token_from_lit (tt, lit); }
/** * Create token of specified type from charset * * @return token descriptor */ static token lexer_create_token_for_charset (token_type tt, /**< token type */ const lit_utf8_byte_t *charset_p, /**< charset buffer */ lit_utf8_size_t size) /**< size of the charset */ { JERRY_ASSERT (charset_p != NULL); literal_t lit = lit_find_literal_by_utf8_string (charset_p, size); if (lit != NULL) { return create_token_from_lit (tt, lit); } lit = lit_create_literal_from_utf8_string (charset_p, size); JERRY_ASSERT (lit->get_type () == LIT_STR_T || lit->get_type () == LIT_MAGIC_STR_T || lit->get_type () == LIT_MAGIC_STR_EX_T); return create_token_from_lit (tt, lit); } /* lexer_create_token_for_charset */
int main (int __attr_unused___ argc, char __attr_unused___ **argv) { TEST_INIT (); const lit_utf8_byte_t *ptrs[test_sub_iters]; ecma_number_t numbers[test_sub_iters]; lit_utf8_byte_t strings[test_sub_iters][max_characters_in_string + 1]; lit_utf8_size_t lengths[test_sub_iters]; jmem_init (); lit_init (); for (uint32_t i = 0; i < test_iters; i++) { memset (numbers, 0, sizeof (ecma_number_t) * test_sub_iters); memset (lengths, 0, sizeof (lit_utf8_size_t) * test_sub_iters); memset (ptrs, 0, sizeof (lit_utf8_byte_t *) * test_sub_iters); for (uint32_t j = 0; j < test_sub_iters; j++) { int type = rand () % 3; if (type == 0) { lengths[j] = (lit_utf8_size_t) (rand () % max_characters_in_string + 1); generate_string (strings[j], lengths[j]); lit_create_literal_from_utf8_string (strings[j], lengths[j]); strings[j][lengths[j]] = '\0'; ptrs[j] = strings[j]; JERRY_ASSERT (ptrs[j]); } else if (type == 1) { lit_magic_string_id_t msi = (lit_magic_string_id_t) (rand () % LIT_MAGIC_STRING__COUNT); ptrs[j] = lit_get_magic_string_utf8 (msi); JERRY_ASSERT (ptrs[j]); lengths[j] = (lit_utf8_size_t) lit_zt_utf8_string_size (ptrs[j]); lit_create_literal_from_utf8_string (ptrs[j], lengths[j]); } else { ecma_number_t num = generate_number (); lengths[j] = ecma_number_to_utf8_string (num, strings[j], max_characters_in_string); lit_create_literal_from_num (num); } } // Add empty string lit_create_literal_from_utf8_string (NULL, 0); for (uint32_t j = 0; j < test_sub_iters; j++) { lit_literal_t lit1; lit_literal_t lit2; if (ptrs[j]) { lit1 = lit_find_or_create_literal_from_utf8_string (ptrs[j], lengths[j]); lit2 = lit_find_literal_by_utf8_string (ptrs[j], lengths[j]); JERRY_ASSERT (compare_utf8_string_and_string_literal (ptrs[j], lengths[j], lit1)); JERRY_ASSERT (compare_utf8_string_and_string_literal (ptrs[j], lengths[j], lit2)); } else { lit1 = lit_find_or_create_literal_from_num (numbers[j]); lit2 = lit_find_literal_by_num (numbers[j]); JERRY_ASSERT (numbers[j] == lit_number_literal_get_number (lit1)); JERRY_ASSERT (numbers[j] == lit_number_literal_get_number (lit2)); } JERRY_ASSERT (lit1); JERRY_ASSERT (lit2); JERRY_ASSERT (lit1 == lit2); } // Check empty string exists JERRY_ASSERT (lit_find_literal_by_utf8_string (NULL, 0)); } lit_finalize (); jmem_finalize (true); return 0; } /* main */
/** * Create token of specified type from charset * * @return token descriptor */ static token lexer_create_token_for_charset (jsp_token_type_t tt, /**< token type */ const lit_utf8_byte_t *charset_p, /**< charset buffer */ lit_utf8_size_t size) /**< size of the charset */ { JERRY_ASSERT (charset_p != NULL); lit_utf8_iterator_t iter = lit_utf8_iterator_create (charset_p, (lit_utf8_size_t) size); lit_utf8_size_t new_size = 0; lit_utf8_size_t new_length = 0; bool should_convert = false; while (!lit_utf8_iterator_is_eos (&iter)) { if (iter.buf_pos.is_non_bmp_middle) { should_convert = true; } lit_utf8_iterator_incr (&iter); new_size += LIT_CESU8_MAX_BYTES_IN_CODE_UNIT; } lit_utf8_byte_t *converted_str_p; if (unlikely (should_convert)) { lit_utf8_iterator_seek_bos (&iter); converted_str_p = (lit_utf8_byte_t *) jsp_mm_alloc (new_size); while (!lit_utf8_iterator_is_eos (&iter)) { ecma_char_t ch = lit_utf8_iterator_read_next (&iter); new_length += lit_code_unit_to_utf8 (ch, converted_str_p + new_length); } } else { converted_str_p = (lit_utf8_byte_t *) charset_p; new_length = size; JERRY_ASSERT (lit_is_cesu8_string_valid (converted_str_p, new_length)); } lit_literal_t lit = lit_find_literal_by_utf8_string (converted_str_p, new_length); if (lit != NULL) { if (unlikely (should_convert)) { jsp_mm_free (converted_str_p); } return create_token_from_lit (tt, lit); } lit = lit_create_literal_from_utf8_string (converted_str_p, new_length); rcs_record_type_t type = rcs_record_get_type (lit); JERRY_ASSERT (RCS_RECORD_TYPE_IS_CHARSET (type) || RCS_RECORD_TYPE_IS_MAGIC_STR (type) || RCS_RECORD_TYPE_IS_MAGIC_STR_EX (type)); if (unlikely (should_convert)) { jsp_mm_free (converted_str_p); } return create_token_from_lit (tt, lit); } /* lexer_create_token_for_charset */