static grn_id
grn_plugin_open_mrb(grn_ctx *ctx, const char *filename, size_t filename_size)
{
grn_id id = GRN_ID_NIL;
grn_plugin **plugin = NULL;
if (!ctx->impl->mrb.state) {
ERR(GRN_FUNCTION_NOT_IMPLEMENTED, "mruby support isn't enabled");
return GRN_ID_NIL;
}
id = grn_hash_add(&grn_gctx, grn_plugins, filename, filename_size,
(void **)&plugin, NULL);
if (!id) {
return id;
}
*plugin = GRN_GMALLOCN(grn_plugin, 1);
if (!*plugin) {
grn_hash_delete_by_id(&grn_gctx, grn_plugins, id, NULL);
return GRN_ID_NIL;
}
grn_memcpy((*plugin)->path, filename, filename_size);
(*plugin)->dl = NULL;
(*plugin)->init_func = NULL;
(*plugin)->register_func = NULL;
(*plugin)->fin_func = NULL;
(*plugin)->refcount = 1;
return id;
}
void
test_read_write(gconstpointer *data)
{
gint i, key;
int added;
grn_ctx *context;
grn_hash *hash;
const gchar *path;
const gchar *value_string;
gint process_number = 0;
const gchar *process_number_string;
void *value;
grn_id id = GRN_ID_NIL;
grn_rc rc;
i = GPOINTER_TO_INT(data);
process_number_string = g_getenv(GRN_TEST_ENV_PROCESS_NUMBER);
if (process_number_string)
process_number = atoi(process_number_string);
key = i + process_number * N_THREADS;
rc = grn_ctx_init(contexts[i], GRN_CTX_USE_QL);
grn_test_assert(rc, cut_message("context: %d (%d)", i, process_number));
context = contexts[i];
path = g_getenv(GRN_TEST_ENV_HASH_PATH);
cut_assert_not_null(path);
hashes[i] = grn_hash_open(context, path);
cut_assert_not_null(hashes[i],
cut_message("hash: %d (%d)", i, process_number));
hash = hashes[i];
grn_test_assert_nil(
grn_hash_get(context, hash, &key, sizeof(key), &value),
cut_message("lookup - fail: %d (%d:%d)", key, i, process_number));
value_string = cut_take_printf("value: %d (%d:%d)", key, i, process_number);
rc = grn_io_lock(context, hash->io, -1);
if (rc != GRN_SUCCESS)
grn_test_assert(rc);
id = grn_hash_add(context, hash, &key, sizeof(key), &value, &added);
grn_io_unlock(hash->io);
grn_test_assert_not_nil(id);
cut_assert_equal_int(1, added);
strcpy(value, value_string);
value = NULL;
id = grn_hash_get(context, hash, &key, sizeof(key), &value);
grn_test_assert_not_nil(
id,
cut_message("lookup - success: %d (%d:%d)", key, i, process_number));
cut_assert_equal_string(value_string, value);
hashes[i] = NULL;
grn_test_assert(grn_hash_close(context, hash));
contexts[i] = NULL;
grn_test_assert(grn_ctx_fin(context));
}
int
hash_put(const char *path)
{
int i;
grn_obj buf;
grn_hash *hash = grn_hash_create(&ctx, path, key_size, value_size,
GRN_OBJ_PERSISTENT|GRN_OBJ_KEY_VAR_SIZE);
if (!hash) { return -1; }
GRN_TEXT_INIT(&buf, 0);
for (i = 0; i < nloops; i++) {
int key = GENKEY(i);
GRN_BULK_REWIND(&buf);
grn_text_itoh(&ctx, &buf, key, key_size);
{
void *value;
grn_id rid = grn_hash_add(&ctx, hash, GRN_BULK_HEAD(&buf), key_size, &value, NULL);
if (!rid) {
fprintf(stderr, "table_lookup failed");
} else {
memcpy(value, GRN_BULK_HEAD(&buf), value_size);
}
}
}
grn_obj_close(&ctx, &buf);
grn_hash_close(&ctx, hash);
return 0;
}
static int
rb_grn_hash_from_ruby_hash_body (VALUE rb_key,
VALUE rb_value,
VALUE user_data)
{
RbGrnHashFromRubyHashData *data = (RbGrnHashFromRubyHashData *)user_data;
grn_obj *value;
int added;
rb_key = rb_grn_convert_to_string(rb_key);
grn_hash_add(data->context,
data->hash,
RSTRING_PTR(rb_key),
RSTRING_LEN(rb_key),
(void **)&value,
&added);
rb_grn_context_check(data->context, data->rb_hash);
if (added) {
GRN_VOID_INIT(value);
}
RVAL2GRNBULK(rb_value, data->context, value);
return ST_CONTINUE;
}
grn_obj *
grn_command_input_add(grn_ctx *ctx,
grn_command_input *input,
const char *name,
int name_size,
grn_bool *added)
{
grn_obj *argument = NULL;
/* TODO: Use grn_bool */
int internal_added = GRN_FALSE;
GRN_API_ENTER;
if (name_size == -1) {
name_size = strlen(name);
}
if (input->arguments) {
grn_hash_add(ctx, input->arguments, name, name_size, (void **)&argument,
&internal_added);
if (internal_added) {
GRN_TEXT_INIT(argument, 0);
}
}
if (added) {
*added = internal_added;
}
GRN_API_RETURN(argument);
}
static grn_rc
grn_plugin_call_register_mrb(grn_ctx *ctx, grn_id id, grn_plugin *plugin)
{
grn_mrb_data *data = &(ctx->impl->mrb);
mrb_state *mrb = data->state;
struct RClass *module = data->module;
struct RClass *plugin_loader_class;
int arena_index;
{
int added;
grn_hash_add(ctx, ctx->impl->mrb.registered_plugins,
&id, sizeof(grn_id), NULL, &added);
if (!added) {
return ctx->rc;
}
}
arena_index = mrb_gc_arena_save(mrb);
plugin_loader_class = mrb_class_get_under(mrb, module, "PluginLoader");
mrb_funcall(mrb, mrb_obj_value(plugin_loader_class),
"load_file", 1, mrb_str_new_cstr(mrb, ctx->impl->plugin_path));
mrb_gc_arena_restore(mrb, arena_index);
return ctx->rc;
}
static void
parse_synonyms_file_line(grn_ctx *ctx, const char *line, size_t line_length,
grn_obj *key, grn_obj *value)
{
size_t i = 0;
if (is_comment_mark(line[i])) {
return;
}
while (i < line_length) {
char character = line[i];
i++;
if (character == '\t') {
break;
}
GRN_TEXT_PUTC(ctx, key, character);
}
if (i == line_length) {
return;
}
GRN_TEXT_PUTS(ctx, value, "((");
while (i < line_length) {
char character = line[i];
i++;
if (character == '\t') {
GRN_TEXT_PUTS(ctx, value, ") OR (");
} else {
GRN_TEXT_PUTC(ctx, value, character);
}
}
GRN_TEXT_PUTS(ctx, value, "))");
{
grn_id id;
void *value_location = NULL;
id = grn_hash_add(ctx, synonyms, GRN_TEXT_VALUE(key), GRN_TEXT_LEN(key),
&value_location, NULL);
if (id == GRN_ID_NIL) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_WARNING,
"[plugin][query-expander][tsv] "
"failed to register key: <%.*s>",
(int)GRN_TEXT_LEN(key), GRN_TEXT_VALUE(key));
return;
}
if (GRN_TEXT_LEN(value) <= MAX_SYNONYM_BYTES - 1) {
GRN_TEXT_PUTC(ctx, value, '\0');
} else {
grn_bulk_truncate(ctx, value, MAX_SYNONYM_BYTES - 1);
GRN_TEXT_PUTC(ctx, value, '\0');
}
grn_memcpy(value_location, GRN_TEXT_VALUE(value), GRN_TEXT_LEN(value));
}
}
grn_id
grn_plugin_open(grn_ctx *ctx, const char *filename)
{
grn_id id;
grn_dl dl;
grn_plugin **plugin = NULL;
CRITICAL_SECTION_ENTER(grn_plugins_lock);
if ((id = grn_hash_get(&grn_gctx, grn_plugins, filename, PATHLEN(filename),
(void **)&plugin))) {
(*plugin)->refcount++;
goto exit;
}
if ((dl = grn_dl_open(filename))) {
if ((id = grn_hash_add(&grn_gctx, grn_plugins, filename, PATHLEN(filename),
(void **)&plugin, NULL))) {
*plugin = GRN_GMALLOCN(grn_plugin, 1);
if (*plugin) {
if (grn_plugin_initialize(ctx, *plugin, dl, id, filename)) {
GRN_GFREE(*plugin);
*plugin = NULL;
}
}
if (!*plugin) {
grn_hash_delete_by_id(&grn_gctx, grn_plugins, id, NULL);
if (grn_dl_close(dl)) {
/* Now, __FILE__ set in plugin is invalid. */
ctx->errline = 0;
ctx->errfile = NULL;
} else {
const char *label;
label = grn_dl_close_error_label();
SERR(label);
}
id = GRN_ID_NIL;
} else {
(*plugin)->refcount = 1;
}
} else {
if (!grn_dl_close(dl)) {
const char *label;
label = grn_dl_close_error_label();
SERR(label);
}
}
} else {
const char *label;
label = grn_dl_open_error_label();
SERR(label);
}
exit:
CRITICAL_SECTION_LEAVE(grn_plugins_lock);
return id;
}
static session *
session_open(grn_ctx *ctx, grn_slap_dest *dest)
{
grn_id id;
session *s;
grn_com *com;
if (!(com = grn_com_copen(ctx, &ev, dest->host, dest->port))) { return NULL; }
id = grn_hash_add(ctx, sessions, &com->fd, sizeof(grn_sock), (void **)&s, NULL);
com->opaque = s;
s->com = com;
s->id = id;
s->stat = 1;
return s;
}
static const char *
get_weight_vector(grn_ctx *ctx, grn_query *query, const char *source)
{
const char *p;
if (!query->opt.weight_vector &&
!query->weight_set &&
!(query->opt.weight_vector = GRN_CALLOC(sizeof(int) * DEFAULT_WEIGHT_VECTOR_SIZE))) {
GRN_LOG(ctx, GRN_LOG_ALERT, "get_weight_vector malloc fail");
return source;
}
for (p = source; p < query->str_end; ) {
unsigned int key;
int value;
/* key, key is not zero */
key = grn_atoui(p, query->str_end, &p);
if (!key || key > GRN_ID_MAX) { break; }
/* value */
if (*p == ':') {
p++;
value = grn_atoi(p, query->str_end, &p);
} else {
value = 1;
}
if (query->weight_set) {
int *pval;
if (grn_hash_add(ctx, query->weight_set, &key, sizeof(unsigned int), (void **)&pval, NULL)) {
*pval = value;
}
} else if (key < DEFAULT_WEIGHT_VECTOR_SIZE) {
query->opt.weight_vector[key - 1] = value;
} else {
GRN_FREE(query->opt.weight_vector);
query->opt.weight_vector = NULL;
if (!(query->weight_set = grn_hash_create(ctx, NULL, sizeof(unsigned int), sizeof(int),
0))) {
return source;
}
p = source; /* reparse */
continue;
}
if (*p != ',') { break; }
p++;
}
return p;
}
grn_rc
grn_com_event_add(grn_ctx *ctx, grn_com_event *ev, grn_sock fd, int events, grn_com **com)
{
grn_com *c;
/* todo : expand events */
if (!ev || *ev->hash->n_entries == ev->max_nevents) {
if (ev) {
GRN_LOG(ctx, GRN_LOG_ERROR, "too many connections (%d)", ev->max_nevents);
}
return GRN_INVALID_ARGUMENT;
}
#ifdef USE_EPOLL
{
struct epoll_event e;
memset(&e, 0, sizeof(struct epoll_event));
e.data.fd = (fd);
e.events = (uint32_t) events;
if (epoll_ctl(ev->epfd, EPOLL_CTL_ADD, (fd), &e) == -1) {
SERR("epoll_ctl");
return ctx->rc;
}
}
#endif /* USE_EPOLL*/
#ifdef USE_KQUEUE
{
struct kevent e;
/* todo: udata should have fd */
EV_SET(&e, (fd), events, EV_ADD, 0, 0, NULL);
if (kevent(ev->kqfd, &e, 1, NULL, 0, NULL) == -1) {
SERR("kevent");
return ctx->rc;
}
}
#endif /* USE_KQUEUE */
{
if (grn_hash_add(ctx, ev->hash, &fd, sizeof(grn_sock), (void **)&c, NULL)) {
c->ev = ev;
c->fd = fd;
c->events = events;
if (com) {
*com = c;
}
}
}
return ctx->rc;
}
bool mrn_hash_put(grn_ctx *ctx, grn_hash *hash, const char *key, grn_obj *value)
{
int added;
bool succeed;
void *buf;
grn_hash_add(ctx, hash, (const char *)key, strlen(key), &buf, &added);
// duplicate check
if (added == 0) {
GRN_LOG(ctx, GRN_LOG_WARNING, "hash put duplicated (key=%s)", key);
succeed = false;
} else {
// store address of value
memcpy(buf, &value, sizeof(grn_obj *));
GRN_LOG(ctx, GRN_LOG_DEBUG, "hash put (key=%s)", key);
succeed = true;
}
return succeed;
}
void
rb_grn_context_register_floating_object (RbGrnObject *rb_grn_object)
{
RbGrnContext *rb_grn_context;
grn_ctx *context;
grn_hash *floating_objects;
rb_grn_context = rb_grn_object->rb_grn_context;
context = rb_grn_context->context;
floating_objects = rb_grn_context->floating_objects;
if (!floating_objects) {
rb_grn_context_reset_floating_objects(rb_grn_context);
floating_objects = rb_grn_context->floating_objects;
}
grn_hash_add(context, floating_objects,
(const void *)(&rb_grn_object), sizeof(RbGrnObject *),
NULL, NULL);
rb_grn_object->floating = GRN_TRUE;
rb_grn_context_object_created(rb_grn_context->self, rb_grn_object->self);
}
grn_id
grn_token_cursor_next(grn_ctx *ctx, grn_token_cursor *token_cursor)
{
int status;
grn_id tid = GRN_ID_NIL;
grn_obj *table = token_cursor->table;
grn_obj *tokenizer = token_cursor->tokenizer;
while (token_cursor->status != GRN_TOKEN_CURSOR_DONE) {
if (tokenizer) {
grn_obj *curr_, *stat_;
((grn_proc *)tokenizer)->funcs[PROC_NEXT](ctx, 1, &table, &token_cursor->pctx.user_data);
stat_ = grn_ctx_pop(ctx);
curr_ = grn_ctx_pop(ctx);
status = grn_token_cursor_next_apply_token_filters(ctx, token_cursor,
curr_, stat_);
token_cursor->status =
((status & GRN_TOKEN_LAST) ||
(token_cursor->mode == GRN_TOKENIZE_GET &&
(status & GRN_TOKEN_REACH_END)))
? GRN_TOKEN_CURSOR_DONE : GRN_TOKEN_CURSOR_DOING;
token_cursor->force_prefix = GRN_FALSE;
#define SKIP_FLAGS \
(GRN_TOKEN_SKIP | GRN_TOKEN_SKIP_WITH_POSITION)
if (status & SKIP_FLAGS) {
if (status & GRN_TOKEN_SKIP) {
token_cursor->pos++;
}
if (token_cursor->status == GRN_TOKEN_CURSOR_DONE && tid == GRN_ID_NIL) {
token_cursor->status = GRN_TOKEN_CURSOR_DONE_SKIP;
break;
} else {
continue;
}
}
#undef SKIP_FLAGS
if (status & GRN_TOKEN_FORCE_PREFIX) {
token_cursor->force_prefix = GRN_TRUE;
}
if (token_cursor->curr_size == 0) {
if (token_cursor->status != GRN_TOKEN_CURSOR_DONE) {
char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE];
int tokenizer_name_length;
tokenizer_name_length =
grn_obj_name(ctx, token_cursor->tokenizer,
tokenizer_name, GRN_TABLE_MAX_KEY_SIZE);
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore an empty token: <%.*s>: <%.*s>",
tokenizer_name_length, tokenizer_name,
token_cursor->orig_blen, token_cursor->orig);
}
continue;
}
if (token_cursor->curr_size > GRN_TABLE_MAX_KEY_SIZE) {
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore too long token. "
"Token must be less than or equal to %d: <%d>(<%.*s>)",
GRN_TABLE_MAX_KEY_SIZE,
token_cursor->curr_size,
token_cursor->curr_size, token_cursor->curr);
continue;
}
if (status & GRN_TOKEN_UNMATURED) {
if (status & GRN_TOKEN_OVERLAP) {
if (token_cursor->mode == GRN_TOKENIZE_GET) {
token_cursor->pos++;
continue;
}
} else {
if (status & GRN_TOKEN_REACH_END) {
token_cursor->force_prefix = GRN_TRUE;
}
}
}
} else {
token_cursor->status = GRN_TOKEN_CURSOR_DONE;
}
if (token_cursor->mode == GRN_TOKENIZE_ADD) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
if (grn_io_lock(ctx, ((grn_pat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_pat_add(ctx, (grn_pat *)table, token_cursor->curr, token_cursor->curr_size,
NULL, NULL);
grn_io_unlock(((grn_pat *)table)->io);
}
break;
case GRN_TABLE_DAT_KEY :
if (grn_io_lock(ctx, ((grn_dat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_dat_add(ctx, (grn_dat *)table, token_cursor->curr, token_cursor->curr_size,
NULL, NULL);
grn_io_unlock(((grn_dat *)table)->io);
}
break;
case GRN_TABLE_HASH_KEY :
if (grn_io_lock(ctx, ((grn_hash *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_hash_add(ctx, (grn_hash *)table, token_cursor->curr, token_cursor->curr_size,
NULL, NULL);
grn_io_unlock(((grn_hash *)table)->io);
}
break;
case GRN_TABLE_NO_KEY :
if (token_cursor->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token_cursor->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
} else if (token_cursor->mode != GRN_TOKENIZE_ONLY) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
tid = grn_pat_get(ctx, (grn_pat *)table, token_cursor->curr, token_cursor->curr_size, NULL);
break;
case GRN_TABLE_DAT_KEY :
tid = grn_dat_get(ctx, (grn_dat *)table, token_cursor->curr, token_cursor->curr_size, NULL);
break;
case GRN_TABLE_HASH_KEY :
tid = grn_hash_get(ctx, (grn_hash *)table, token_cursor->curr, token_cursor->curr_size, NULL);
break;
case GRN_TABLE_NO_KEY :
if (token_cursor->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token_cursor->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
}
if (token_cursor->mode != GRN_TOKENIZE_ONLY &&
tid == GRN_ID_NIL && token_cursor->status != GRN_TOKEN_CURSOR_DONE) {
token_cursor->status = GRN_TOKEN_CURSOR_NOT_FOUND;
}
token_cursor->pos++;
break;
}
return tid;
}
grn_id
grn_token_next(grn_ctx *ctx, grn_token *token)
{
int status;
grn_id tid = GRN_ID_NIL;
grn_obj *table = token->table;
grn_obj *tokenizer = token->tokenizer;
while (token->status != grn_token_done) {
if (tokenizer) {
grn_obj *curr_, *stat_;
((grn_proc *)tokenizer)->funcs[PROC_NEXT](ctx, 1, &table, &token->pctx.user_data);
stat_ = grn_ctx_pop(ctx);
curr_ = grn_ctx_pop(ctx);
token->curr = GRN_TEXT_VALUE(curr_);
token->curr_size = GRN_TEXT_LEN(curr_);
status = GRN_UINT32_VALUE(stat_);
token->status = ((status & GRN_TOKEN_LAST) ||
(!token->add && (status & GRN_TOKEN_REACH_END)))
? grn_token_done : grn_token_doing;
token->force_prefix = 0;
if (status & GRN_TOKEN_UNMATURED) {
if (status & GRN_TOKEN_OVERLAP) {
if (!token->add) { token->pos++; continue; }
} else {
if (status & GRN_TOKEN_LAST) { token->force_prefix = 1; }
}
}
} else {
token->curr = token->orig;
token->curr_size = token->orig_blen;
token->status = grn_token_done;
}
if (token->add) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
if (grn_io_lock(ctx, ((grn_pat *)table)->io, 10000000)) {
tid = GRN_ID_NIL;
} else {
tid = grn_pat_add(ctx, (grn_pat *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_pat *)table)->io);
}
break;
case GRN_TABLE_HASH_KEY :
if (grn_io_lock(ctx, ((grn_hash *)table)->io, 10000000)) {
tid = GRN_ID_NIL;
} else {
tid = grn_hash_add(ctx, (grn_hash *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_hash *)table)->io);
}
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
} else {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
tid = grn_pat_get(ctx, (grn_pat *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_HASH_KEY :
tid = grn_hash_get(ctx, (grn_hash *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
}
if (tid == GRN_ID_NIL && token->status != grn_token_done) {
token->status = grn_token_not_found;
}
token->pos++;
break;
}
return tid;
}
grn_id
grn_plugin_open(grn_ctx *ctx, const char *filename)
{
grn_id id = GRN_ID_NIL;
grn_dl dl;
grn_plugin **plugin = NULL;
size_t filename_size;
filename_size = GRN_PLUGIN_KEY_SIZE(filename);
CRITICAL_SECTION_ENTER(grn_plugins_lock);
if ((id = grn_hash_get(&grn_gctx, grn_plugins, filename, filename_size,
(void **)&plugin))) {
(*plugin)->refcount++;
goto exit;
}
#ifdef GRN_WITH_MRUBY
{
const char *mrb_suffix;
mrb_suffix = grn_plugin_get_ruby_suffix();
if (filename_size > strlen(mrb_suffix) &&
strcmp(filename + (strlen(filename) - strlen(mrb_suffix)),
mrb_suffix) == 0) {
id = grn_plugin_open_mrb(ctx, filename, filename_size);
goto exit;
}
}
#endif /* GRN_WITH_MRUBY */
if ((dl = grn_dl_open(filename))) {
if ((id = grn_hash_add(&grn_gctx, grn_plugins, filename, filename_size,
(void **)&plugin, NULL))) {
*plugin = GRN_GMALLOCN(grn_plugin, 1);
if (*plugin) {
grn_memcpy((*plugin)->path, filename, filename_size);
if (grn_plugin_initialize(ctx, *plugin, dl, id, filename)) {
GRN_GFREE(*plugin);
*plugin = NULL;
}
}
if (!*plugin) {
grn_hash_delete_by_id(&grn_gctx, grn_plugins, id, NULL);
if (grn_dl_close(dl)) {
/* Now, __FILE__ set in plugin is invalid. */
ctx->errline = 0;
ctx->errfile = NULL;
} else {
const char *label;
label = grn_dl_close_error_label();
SERR("%s", label);
}
id = GRN_ID_NIL;
} else {
(*plugin)->refcount = 1;
}
} else {
if (!grn_dl_close(dl)) {
const char *label;
label = grn_dl_close_error_label();
SERR("%s", label);
}
}
} else {
const char *label;
label = grn_dl_open_error_label();
SERR("%s", label);
}
exit:
CRITICAL_SECTION_LEAVE(grn_plugins_lock);
return id;
}
grn_id
grn_token_next(grn_ctx *ctx, grn_token *token)
{
int status;
grn_id tid = GRN_ID_NIL;
grn_obj *table = token->table;
grn_obj *tokenizer = token->tokenizer;
while (token->status != GRN_TOKEN_DONE) {
if (tokenizer) {
grn_obj *curr_, *stat_;
((grn_proc *)tokenizer)->funcs[PROC_NEXT](ctx, 1, &table, &token->pctx.user_data);
stat_ = grn_ctx_pop(ctx);
curr_ = grn_ctx_pop(ctx);
token->curr = (const unsigned char *)GRN_TEXT_VALUE(curr_);
token->curr_size = GRN_TEXT_LEN(curr_);
status = GRN_UINT32_VALUE(stat_);
token->status = ((status & GRN_TOKENIZER_TOKEN_LAST) ||
(token->mode == GRN_TOKEN_GET &&
(status & GRN_TOKENIZER_TOKEN_REACH_END)))
? GRN_TOKEN_DONE : GRN_TOKEN_DOING;
token->force_prefix = 0;
if (token->curr_size == 0) {
char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE];
int tokenizer_name_length;
tokenizer_name_length =
grn_obj_name(ctx, token->tokenizer,
tokenizer_name, GRN_TABLE_MAX_KEY_SIZE);
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore an empty token: <%.*s>: <%.*s>",
tokenizer_name_length, tokenizer_name,
token->orig_blen, token->orig);
continue;
}
if (token->curr_size > GRN_TABLE_MAX_KEY_SIZE) {
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore too long token. "
"Token must be less than or equal to %d: <%d>(<%.*s>)",
GRN_TABLE_MAX_KEY_SIZE,
token->curr_size,
token->curr_size, token->curr);
continue;
}
if (status & GRN_TOKENIZER_TOKEN_UNMATURED) {
if (status & GRN_TOKENIZER_TOKEN_OVERLAP) {
if (token->mode == GRN_TOKEN_GET) { token->pos++; continue; }
} else {
if (status & GRN_TOKENIZER_TOKEN_LAST) { token->force_prefix = 1; }
}
}
} else {
token->status = GRN_TOKEN_DONE;
}
if (token->mode == GRN_TOKEN_ADD) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
if (grn_io_lock(ctx, ((grn_pat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_pat_add(ctx, (grn_pat *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_pat *)table)->io);
}
break;
case GRN_TABLE_DAT_KEY :
if (grn_io_lock(ctx, ((grn_dat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_dat_add(ctx, (grn_dat *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_dat *)table)->io);
}
break;
case GRN_TABLE_HASH_KEY :
if (grn_io_lock(ctx, ((grn_hash *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_hash_add(ctx, (grn_hash *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_hash *)table)->io);
}
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
} else {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
tid = grn_pat_get(ctx, (grn_pat *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_DAT_KEY :
tid = grn_dat_get(ctx, (grn_dat *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_HASH_KEY :
tid = grn_hash_get(ctx, (grn_hash *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
}
if (tid == GRN_ID_NIL && token->status != GRN_TOKEN_DONE) {
token->status = GRN_TOKEN_NOT_FOUND;
}
token->pos++;
break;
}
return tid;
}
