static grn_obj * ngram_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data, uint8_t ngram_unit, uint8_t uni_alpha, uint8_t uni_digit, uint8_t uni_symbol, uint8_t ignore_blank) { unsigned int normalize_flags = GRN_STRING_REMOVE_BLANK | GRN_STRING_WITH_TYPES | GRN_STRING_REMOVE_TOKENIZED_DELIMITER; grn_tokenizer_query *query; const char *normalized; unsigned int normalized_length_in_bytes; grn_ngram_tokenizer *tokenizer; query = grn_tokenizer_query_open(ctx, nargs, args, normalize_flags); if (!query) { return NULL; } if (!(tokenizer = GRN_MALLOC(sizeof(grn_ngram_tokenizer)))) { grn_tokenizer_query_close(ctx, query); ERR(GRN_NO_MEMORY_AVAILABLE, "[tokenizer][ngram] " "memory allocation to grn_ngram_tokenizer failed"); return NULL; } user_data->ptr = tokenizer; grn_tokenizer_token_init(ctx, &(tokenizer->token)); tokenizer->query = query; tokenizer->uni_alpha = uni_alpha; tokenizer->uni_digit = uni_digit; tokenizer->uni_symbol = uni_symbol; tokenizer->ngram_unit = ngram_unit; tokenizer->ignore_blank = ignore_blank; tokenizer->overlap = 0; tokenizer->pos = 0; tokenizer->skip = 0; grn_string_get_normalized(ctx, tokenizer->query->normalized_query, &normalized, &normalized_length_in_bytes, &(tokenizer->len)); tokenizer->next = (const unsigned char *)normalized; tokenizer->end = tokenizer->next + normalized_length_in_bytes; tokenizer->ctypes = grn_string_get_types(ctx, tokenizer->query->normalized_query); return NULL; }
static grn_obj * regexp_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) { unsigned int normalize_flags = GRN_STRING_WITH_TYPES; grn_tokenizer_query *query; const char *normalized; unsigned int normalized_length_in_bytes; grn_regexp_tokenizer *tokenizer; query = grn_tokenizer_query_open(ctx, nargs, args, normalize_flags); if (!query) { return NULL; } tokenizer = GRN_MALLOC(sizeof(grn_regexp_tokenizer)); if (!tokenizer) { grn_tokenizer_query_close(ctx, query); ERR(GRN_NO_MEMORY_AVAILABLE, "[tokenizer][regexp] failed to allocate memory"); return NULL; } user_data->ptr = tokenizer; grn_tokenizer_token_init(ctx, &(tokenizer->token)); tokenizer->query = query; tokenizer->get.n_skip_tokens = 0; tokenizer->is_begin = GRN_TRUE; tokenizer->is_end = GRN_FALSE; tokenizer->is_start_token = GRN_TRUE; tokenizer->is_overlapping = GRN_FALSE; grn_string_get_normalized(ctx, tokenizer->query->normalized_query, &normalized, &normalized_length_in_bytes, NULL); tokenizer->next = normalized; tokenizer->end = tokenizer->next + normalized_length_in_bytes; tokenizer->nth_char = 0; tokenizer->char_types = grn_string_get_types(ctx, tokenizer->query->normalized_query); GRN_TEXT_INIT(&(tokenizer->buffer), 0); return NULL; }
static grn_obj * yangram_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data, unsigned short ngram_unit, grn_bool ignore_blank, grn_bool split_symbol, grn_bool split_alpha, grn_bool split_digit, grn_bool skip_overlap, unsigned short use_vgram) { grn_tokenizer_query *query; unsigned int normalize_flags = GRN_STRING_WITH_TYPES | GRN_STRING_REMOVE_TOKENIZED_DELIMITER | GRN_STRING_REMOVE_BLANK; const char *normalized; unsigned int normalized_length_in_bytes; grn_yangram_tokenizer *tokenizer; query = grn_tokenizer_query_open(ctx, nargs, args, normalize_flags); if (!query) { return NULL; } if (!(tokenizer = GRN_PLUGIN_MALLOC(ctx, sizeof(grn_yangram_tokenizer)))) { GRN_PLUGIN_ERROR(ctx,GRN_NO_MEMORY_AVAILABLE, "[tokenizer][yangram] " "memory allocation to grn_yangram_tokenizer failed"); grn_tokenizer_query_close(ctx, query); return NULL; } user_data->ptr = tokenizer; grn_tokenizer_token_init(ctx, &(tokenizer->token)); tokenizer->query = query; tokenizer->skip_overlap = skip_overlap; tokenizer->ignore_blank = ignore_blank; tokenizer->ngram_unit = ngram_unit; tokenizer->split_symbol = split_symbol; tokenizer->split_alpha = split_alpha; tokenizer->split_digit = split_digit; tokenizer->use_vgram = use_vgram; if (tokenizer->use_vgram > 0) { const char *vgram_word_table_name_env; vgram_word_table_name_env = getenv("GRN_VGRAM_WORD_TABLE_NAME"); if (vgram_word_table_name_env) { tokenizer->vgram_table = grn_ctx_get(ctx, vgram_word_table_name_env, strlen(vgram_word_table_name_env)); } else { tokenizer->vgram_table = grn_ctx_get(ctx, VGRAM_WORD_TABLE_NAME, strlen(VGRAM_WORD_TABLE_NAME)); } if (!tokenizer->vgram_table) { GRN_PLUGIN_ERROR(ctx, GRN_NO_MEMORY_AVAILABLE, "[tokenizer][yangram] " "couldn't open a vgram table"); tokenizer->vgram_table = NULL; return NULL; } } else { tokenizer->vgram_table = NULL; } grn_string_get_normalized(ctx, tokenizer->query->normalized_query, &normalized, &normalized_length_in_bytes, NULL); { const char *phrase_table_name_env; phrase_table_name_env = getenv("GRN_KNOWN_PHRASE_TABLE_NAME"); if (phrase_table_name_env) { tokenizer->phrase_table = grn_ctx_get(ctx, phrase_table_name_env, strlen(phrase_table_name_env)); } else { tokenizer->phrase_table = grn_ctx_get(ctx, KNOWN_PHRASE_TABLE_NAME, strlen(KNOWN_PHRASE_TABLE_NAME)); } if (tokenizer->phrase_table) { if (!(tokenizer->hits = GRN_PLUGIN_MALLOC(ctx, sizeof(grn_pat_scan_hit) * MAX_N_HITS))) { GRN_PLUGIN_ERROR(ctx,GRN_NO_MEMORY_AVAILABLE, "[tokenizer][yangram] " "memory allocation to grn_pat_scan_hit failed"); grn_tokenizer_query_close(ctx, query); return NULL; } else { tokenizer->scan_rest = normalized; tokenizer->nhits = 0; tokenizer->current_hit = 0; } } else { tokenizer->phrase_table = NULL; } } tokenizer->next = (const unsigned char *)normalized; tokenizer->end = tokenizer->next + normalized_length_in_bytes; tokenizer->rest_length = tokenizer->end - tokenizer->next; tokenizer->ctypes = grn_string_get_types(ctx, tokenizer->query->normalized_query); tokenizer->pushed_token_tail = NULL; tokenizer->ctypes_next = 0; return NULL; }