static grn_id
parse_id_value(grn_ctx *ctx, grn_obj *value)
{
switch (value->header.type) {
case GRN_DB_UINT32 :
return GRN_UINT32_VALUE(value);
case GRN_DB_INT32 :
return GRN_INT32_VALUE(value);
default :
{
grn_id id = GRN_ID_NIL;
grn_obj casted_value;
GRN_UINT32_INIT(&casted_value, 0);
if (grn_obj_cast(ctx, value, &casted_value, GRN_FALSE) != GRN_SUCCESS) {
grn_obj inspected;
GRN_TEXT_INIT(&inspected, 0);
grn_inspect(ctx, &inspected, value);
ERR(GRN_INVALID_ARGUMENT,
"<%s>: failed to cast to <UInt32>: <%.*s>",
GRN_COLUMN_NAME_ID,
(int)GRN_TEXT_LEN(&inspected),
GRN_TEXT_VALUE(&inspected));
GRN_OBJ_FIN(ctx, &inspected);
} else {
id = GRN_UINT32_VALUE(&casted_value);
}
GRN_OBJ_FIN(ctx, &casted_value);
return id;
}
}
}
grn_rc
grn_db_init_builtin_tokenizers(grn_ctx *ctx)
{
grn_obj *obj, results[2];
GRN_UINT32_INIT(&results[0], 0);
GRN_TEXT_INIT(&results[1], 0);
obj = grn_proc_create(ctx, "<token:delimit>", 15, NULL,
delimit_init, delimited_next, delimited_fin, 1, 2, results);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_DELIMIT) { return GRN_FILE_CORRUPT; }
obj = grn_proc_create(ctx, "<token:unigram>", 15, NULL,
unigram_init, ngram_next, ngram_fin, 1, 2, results);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_UNIGRAM) { return GRN_FILE_CORRUPT; }
obj = grn_proc_create(ctx, "<token:bigram>", 14, NULL,
bigram_init, ngram_next, ngram_fin, 1, 2, results);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_BIGRAM) { return GRN_FILE_CORRUPT; }
obj = grn_proc_create(ctx, "<token:trigram>", 15, NULL,
trigram_init, ngram_next, ngram_fin, 1, 2, results);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_TRIGRAM) { return GRN_FILE_CORRUPT; }
#ifndef NO_MECAB
obj = grn_proc_create(ctx, "<token:mecab>", 13, NULL,
mecab_init, mecab_next, mecab_fin, 1, 2, results);
#endif /* NO_MECAB */
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_MECAB) { return GRN_FILE_CORRUPT; }
return GRN_SUCCESS;
}
grn_rc
grn_tokenizer_register(grn_ctx *ctx, const char *plugin_name_ptr,
unsigned int plugin_name_length,
grn_proc_func *init, grn_proc_func *next,
grn_proc_func *fin)
{
grn_expr_var vars[] = {
{ NULL, 0 },
{ NULL, 0 },
{ NULL, 0 }
};
GRN_TEXT_INIT(&vars[0].value, 0);
GRN_TEXT_INIT(&vars[1].value, 0);
GRN_UINT32_INIT(&vars[2].value, 0);
{
/*
grn_proc_create() registers a plugin to the database which is associated
with `ctx'. A returned object must not be finalized here.
*/
grn_obj * const obj = grn_proc_create(ctx, plugin_name_ptr,
plugin_name_length,
GRN_PROC_TOKENIZER,
init, next, fin, 3, vars);
if (obj == NULL) {
GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR, "grn_proc_create() failed");
return ctx->rc;
}
}
return GRN_SUCCESS;
}
static grn_rc
ngram_init(grn_ctx *ctx, grn_obj *table, grn_proc_data *user_data, uint8_t ngram_unit)
{
grn_obj *str;
int nflags = GRN_STR_REMOVEBLANK|GRN_STR_WITH_CTYPES;
grn_ngram_tokenizer *token;
grn_obj_flags table_flags;
if (!(str = grn_ctx_pop(ctx))) { return GRN_INVALID_ARGUMENT; }
if (!(token = GRN_MALLOC(sizeof(grn_ngram_tokenizer)))) { return ctx->rc; }
user_data->ptr = token;
token->uni_alpha = 1;
token->uni_digit = 1;
token->uni_symbol = 1;
token->ngram_unit = ngram_unit;
token->overlap = 0;
token->pos = 0;
token->skip = 0;
grn_table_get_info(ctx, table, &table_flags, &token->encoding, NULL);
nflags |= (table_flags & GRN_OBJ_KEY_NORMALIZE);
if (!(token->nstr = grn_str_open_(ctx, GRN_TEXT_VALUE(str), GRN_TEXT_LEN(str),
nflags, token->encoding))) {
GRN_LOG(ctx, GRN_LOG_ALERT, "grn_str_open failed at grn_token_open");
return GRN_TOKENIZER_ERROR;
}
token->next = (unsigned char *)token->nstr->norm;
token->end = token->next + token->nstr->norm_blen;
token->ctypes = token->nstr->ctypes;
token->len = token->nstr->length;
GRN_TEXT_INIT(&token->curr_, GRN_OBJ_DO_SHALLOW_COPY);
GRN_UINT32_INIT(&token->stat_, 0);
return GRN_SUCCESS;
}
void
test_table_scan(void)
{
grn_obj *cond, *v, *res, textbuf, intbuf;
GRN_TEXT_INIT(&textbuf, 0);
GRN_UINT32_INIT(&intbuf, 0);
prepare_data(&textbuf, &intbuf);
cut_assert_not_null((cond = grn_expr_create(&context, NULL, 0)));
v = grn_expr_add_var(&context, cond, NULL, 0);
GRN_RECORD_INIT(v, 0, grn_obj_id(&context, docs));
grn_expr_append_obj(&context, cond, v);
GRN_TEXT_SETS(&context, &textbuf, "size");
grn_expr_append_const(&context, cond, &textbuf);
grn_expr_append_op(&context, cond, GRN_OP_OBJ_GET_VALUE, 2);
GRN_UINT32_SET(&context, &intbuf, 14);
grn_expr_append_const(&context, cond, &intbuf);
grn_expr_append_op(&context, cond, GRN_OP_EQUAL, 2);
grn_expr_compile(&context, cond);
res = grn_table_create(&context, NULL, 0, NULL,
GRN_TABLE_HASH_KEY|GRN_OBJ_WITH_SUBREC, docs, 0);
cut_assert_not_null(res);
grn_test_assert(grn_table_scan(&context, docs, cond, res, GRN_SEL_OR));
cut_assert_equal_uint(3, grn_table_size(&context, res));
grn_test_assert(grn_obj_close(&context, res));
grn_test_assert(grn_obj_close(&context, cond));
grn_test_assert(grn_obj_close(&context, &textbuf));
grn_test_assert(grn_obj_close(&context, &intbuf));
}
static grn_obj *
delimited_init(grn_ctx *ctx, grn_obj *table, grn_user_data *user_data,
uint8_t *delimiter, uint32_t delimiter_len)
{
grn_obj *str;
int nflags = 0;
grn_delimited_tokenizer *token;
grn_obj_flags table_flags;
if (!(str = grn_ctx_pop(ctx))) {
ERR(GRN_INVALID_ARGUMENT, "missing argument");
return NULL;
}
if (!(token = GRN_MALLOC(sizeof(grn_delimited_tokenizer)))) { return NULL; }
user_data->ptr = token;
token->delimiter = delimiter;
token->delimiter_len = delimiter_len;
token->pos = 0;
grn_table_get_info(ctx, table, &table_flags, &token->encoding, NULL);
nflags |= (table_flags & GRN_OBJ_KEY_NORMALIZE);
if (!(token->nstr = grn_str_open_(ctx, GRN_TEXT_VALUE(str), GRN_TEXT_LEN(str),
nflags, token->encoding))) {
GRN_FREE(token);
ERR(GRN_TOKENIZER_ERROR, "grn_str_open failed at grn_token_open");
return NULL;
}
token->next = (unsigned char *)token->nstr->norm;
token->end = token->next + token->nstr->norm_blen;
token->len = token->nstr->length;
GRN_TEXT_INIT(&token->curr_, GRN_OBJ_DO_SHALLOW_COPY);
GRN_UINT32_INIT(&token->stat_, 0);
return NULL;
}
static void
output_tokens(grn_ctx *ctx, grn_obj *tokens, grn_obj *lexicon, grn_obj *index_column)
{
int i, n_tokens, n_elements;
grn_obj estimated_size;
n_tokens = GRN_BULK_VSIZE(tokens) / sizeof(tokenize_token);
n_elements = 3;
if (index_column) {
n_elements++;
GRN_UINT32_INIT(&estimated_size, 0);
}
grn_ctx_output_array_open(ctx, "TOKENS", n_tokens);
for (i = 0; i < n_tokens; i++) {
tokenize_token *token;
char value[GRN_TABLE_MAX_KEY_SIZE];
unsigned int value_size;
token = ((tokenize_token *)(GRN_BULK_HEAD(tokens))) + i;
grn_ctx_output_map_open(ctx, "TOKEN", n_elements);
grn_ctx_output_cstr(ctx, "value");
value_size = grn_table_get_key(ctx, lexicon, token->id,
value, GRN_TABLE_MAX_KEY_SIZE);
grn_ctx_output_str(ctx, value, value_size);
grn_ctx_output_cstr(ctx, "position");
grn_ctx_output_int32(ctx, token->position);
grn_ctx_output_cstr(ctx, "force_prefix");
grn_ctx_output_bool(ctx, token->force_prefix);
if (index_column) {
GRN_BULK_REWIND(&estimated_size);
grn_obj_get_value(ctx, index_column, token->id, &estimated_size);
grn_ctx_output_cstr(ctx, "estimated_size");
grn_ctx_output_int64(ctx, GRN_UINT32_VALUE(&estimated_size));
}
grn_ctx_output_map_close(ctx);
}
if (index_column) {
GRN_OBJ_FIN(ctx, &estimated_size);
}
grn_ctx_output_array_close(ctx);
}
static grn_obj *
proc_column_create(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
{
uint32_t nvars;
grn_obj *buf = args[0];
grn_expr_var *vars;
grn_proc_get_info(ctx, user_data, &vars, &nvars, NULL);
if (nvars == 6) {
grn_obj_flags flags = grn_atoi(GRN_TEXT_VALUE(&vars[2].value),
GRN_BULK_CURR(&vars[2].value), NULL);
grn_obj *column, *table = grn_ctx_get(ctx, GRN_TEXT_VALUE(&vars[0].value),
GRN_TEXT_LEN(&vars[0].value));
grn_obj *type = grn_ctx_get(ctx, GRN_TEXT_VALUE(&vars[3].value),
GRN_TEXT_LEN(&vars[3].value));
if (GRN_TEXT_LEN(&vars[1].value)) {
flags |= GRN_OBJ_PERSISTENT;
}
column = grn_column_create(ctx, table,
GRN_TEXT_VALUE(&vars[1].value),
GRN_TEXT_LEN(&vars[1].value),
NULL, flags, type);
if (column) {
if (GRN_TEXT_LEN(&vars[4].value)) {
grn_obj sources, source_ids, **p, **pe;
GRN_PTR_INIT(&sources, GRN_OBJ_VECTOR, GRN_ID_NIL);
GRN_UINT32_INIT(&source_ids, GRN_OBJ_VECTOR);
grn_obj_columns(ctx, type,
GRN_TEXT_VALUE(&vars[4].value),
GRN_TEXT_LEN(&vars[4].value),
&sources);
p = (grn_obj **)GRN_BULK_HEAD(&sources);
pe = (grn_obj **)GRN_BULK_CURR(&sources);
while (p < pe) {
grn_id source_id = grn_obj_id(ctx, *p++);
if (source_id) {
GRN_UINT32_PUT(ctx, &source_ids, source_id);
}
}
if (GRN_BULK_VSIZE(&source_ids)) {
grn_obj_set_info(ctx, column, GRN_INFO_SOURCE, &source_ids);
}
GRN_OBJ_FIN(ctx, &source_ids);
GRN_OBJ_FIN(ctx, &sources);
}
grn_obj_unlink(ctx, column);
}
GRN_TEXT_PUTS(ctx, buf, ctx->rc ? "false" : "true");
}
return buf;
}
static grn_rc
uvector_init(grn_ctx *ctx, grn_obj *table, grn_proc_data *user_data)
{
grn_obj *str;
grn_uvector_tokenizer_info *token;
if (!(str = grn_ctx_pop(ctx))) { return GRN_INVALID_ARGUMENT; }
if (!(token = GRN_MALLOC(sizeof(grn_uvector_tokenizer_info)))) { return ctx->rc; }
user_data->ptr = token;
token->curr = GRN_TEXT_VALUE(str);
token->tail = token->curr + GRN_TEXT_LEN(str);
token->unit = sizeof(grn_id);
GRN_TEXT_INIT(&token->curr_, GRN_OBJ_DO_SHALLOW_COPY);
GRN_UINT32_INIT(&token->stat_, 0);
return GRN_SUCCESS;
}
void
cut_setup(void)
{
cut_remove_path(tmp_directory, NULL);
g_mkdir_with_parents(tmp_directory, 0700);
path = g_build_filename(tmp_directory, "text-expr", NULL);
context = g_new0(grn_ctx, 1);
grn_ctx_init(context, 0);
database = grn_db_create(context, path, NULL);
expr = NULL;
GRN_TEXT_INIT(&textbuf, 0);
GRN_UINT32_INIT(&intbuf, 0);
}
void
cut_setup(void)
{
cut_remove_path(tmp_directory, NULL);
g_mkdir_with_parents(tmp_directory, 0700);
path = g_build_filename(tmp_directory, "text-expr", NULL);
grn_ctx_init(&context, 0);
database = grn_db_create(&context, path, NULL);
expr = NULL;
res = NULL;
GRN_TEXT_INIT(&textbuf, 0);
GRN_UINT32_INIT(&intbuf, 0);
GRN_FLOAT_INIT(&floatbuf, 0);
GRN_TIME_INIT(&timebuf, 0);
}
void
cut_setup(void)
{
cut_remove_path(tmp_directory, NULL);
g_mkdir_with_parents(tmp_directory, 0700);
path = g_build_filename(tmp_directory, "text-table-select", NULL);
grn_ctx_init(&context, 0);
database = grn_db_create(&context, path, NULL);
cond = NULL;
expr = NULL;
res = NULL;
GRN_TEXT_INIT(&text_buf, 0);
GRN_UINT32_INIT(&int_buf, 0);
GRN_PTR_INIT(&ptr_buf, 0, GRN_ID_NIL);
}
grn_rc
grn_db_init_builtin_tokenizers(grn_ctx *ctx)
{
grn_obj *obj;
grn_expr_var vars[] = {
{NULL, 0},
{NULL, 0},
{NULL, 0}
};
GRN_TEXT_INIT(&vars[0].value, 0);
GRN_TEXT_INIT(&vars[1].value, 0);
GRN_UINT32_INIT(&vars[2].value, 0);
obj = DEF_TOKENIZER("TokenDelimit",
delimit_init, delimited_next, delimited_fin, vars);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_DELIMIT) { return GRN_FILE_CORRUPT; }
obj = DEF_TOKENIZER("TokenUnigram",
unigram_init, ngram_next, ngram_fin, vars);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_UNIGRAM) { return GRN_FILE_CORRUPT; }
obj = DEF_TOKENIZER("TokenBigram",
bigram_init, ngram_next, ngram_fin, vars);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_BIGRAM) { return GRN_FILE_CORRUPT; }
obj = DEF_TOKENIZER("TokenTrigram",
trigram_init, ngram_next, ngram_fin, vars);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_TRIGRAM) { return GRN_FILE_CORRUPT; }
DEF_TOKENIZER("TokenBigramSplitSymbol",
bigrams_init, ngram_next, ngram_fin, vars);
DEF_TOKENIZER("TokenBigramSplitSymbolAlpha",
bigramsa_init, ngram_next, ngram_fin, vars);
DEF_TOKENIZER("TokenBigramSplitSymbolAlphaDigit",
bigramsad_init, ngram_next, ngram_fin, vars);
DEF_TOKENIZER("TokenBigramIgnoreBlank",
bigrami_init, ngram_next, ngram_fin, vars);
DEF_TOKENIZER("TokenBigramIgnoreBlankSplitSymbol",
bigramis_init, ngram_next, ngram_fin, vars);
DEF_TOKENIZER("TokenBigramIgnoreBlankSplitSymbolAlpha",
bigramisa_init, ngram_next, ngram_fin, vars);
DEF_TOKENIZER("TokenBigramIgnoreBlankSplitSymbolAlphaDigit",
bigramisad_init, ngram_next, ngram_fin, vars);
DEF_TOKENIZER("TokenDelimitNull",
delimit_null_init, delimited_next, delimited_fin, vars);
DEF_TOKENIZER("TokenRegexp",
regexp_init, regexp_next, regexp_fin, vars);
return GRN_SUCCESS;
}
static grn_obj *
uvector_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
{
grn_obj *str;
grn_uvector_tokenizer_info *token;
if (!(str = grn_ctx_pop(ctx))) {
ERR(GRN_INVALID_ARGUMENT, "missing argument");
return NULL;
}
if (!(token = GRN_MALLOC(sizeof(grn_uvector_tokenizer_info)))) { return NULL; }
user_data->ptr = token;
token->curr = GRN_TEXT_VALUE(str);
token->tail = token->curr + GRN_TEXT_LEN(str);
token->unit = sizeof(grn_id);
GRN_TEXT_INIT(&token->curr_, GRN_OBJ_DO_SHALLOW_COPY);
GRN_UINT32_INIT(&token->stat_, 0);
return NULL;
}
static void
grn_loader_init(grn_loader *loader)
{
GRN_TEXT_INIT(&loader->values, 0);
GRN_UINT32_INIT(&loader->level, GRN_OBJ_VECTOR);
GRN_PTR_INIT(&loader->columns, GRN_OBJ_VECTOR, GRN_ID_NIL);
loader->id_offset = -1;
loader->key_offset = -1;
loader->table = NULL;
loader->last = NULL;
loader->ifexists = NULL;
loader->each = NULL;
loader->values_size = 0;
loader->nrecords = 0;
loader->stat = GRN_LOADER_BEGIN;
loader->columns_status = GRN_LOADER_COLUMNS_UNSET;
loader->rc = GRN_SUCCESS;
loader->errbuf[0] = '\0';
}
static grn_obj *
ngram_init(grn_ctx *ctx, grn_obj *table, grn_user_data *user_data, uint8_t ngram_unit,
uint8_t uni_alpha, uint8_t uni_digit, uint8_t uni_symbol, uint8_t ignore_blank)
{
grn_obj *str;
int nflags = GRN_STR_REMOVEBLANK|GRN_STR_WITH_CTYPES;
grn_ngram_tokenizer *token;
grn_obj_flags table_flags;
if (!(str = grn_ctx_pop(ctx))) {
ERR(GRN_INVALID_ARGUMENT, "missing argument");
return NULL;
}
if (!(token = GRN_MALLOC(sizeof(grn_ngram_tokenizer)))) { return NULL; }
user_data->ptr = token;
token->uni_alpha = uni_alpha;
token->uni_digit = uni_digit;
token->uni_symbol = uni_symbol;
token->ngram_unit = ngram_unit;
token->ignore_blank = ignore_blank;
token->overlap = 0;
token->pos = 0;
token->skip = 0;
grn_table_get_info(ctx, table, &table_flags, &token->encoding, NULL);
nflags |= (table_flags & GRN_OBJ_KEY_NORMALIZE);
if (!(token->nstr = grn_str_open_(ctx, GRN_TEXT_VALUE(str), GRN_TEXT_LEN(str),
nflags, token->encoding))) {
GRN_FREE(token);
ERR(GRN_TOKENIZER_ERROR, "grn_str_open failed at grn_token_open");
return NULL;
}
token->next = (unsigned char *)token->nstr->norm;
token->end = token->next + token->nstr->norm_blen;
token->ctypes = token->nstr->ctypes;
token->len = token->nstr->length;
GRN_TEXT_INIT(&token->curr_, GRN_OBJ_DO_SHALLOW_COPY);
GRN_UINT32_INIT(&token->stat_, 0);
return NULL;
}
/*
This function registers a plugin to a database.
*/
grn_rc
GRN_PLUGIN_REGISTER(grn_ctx *ctx)
{
grn_obj *obj;
grn_expr_var vars[] = {
{NULL, 0},
{NULL, 0},
{NULL, 0}
};
GRN_TEXT_INIT(&vars[0].value, 0);
GRN_TEXT_INIT(&vars[1].value, 0);
GRN_UINT32_INIT(&vars[2].value, 0);
/*
grn_proc_create() registers a plugin to a database which is associated
with `ctx'. A returned object must not be finalized here.
*/
obj = grn_proc_create(ctx, "TokenMecab", 10, GRN_PROC_TOKENIZER,
mecab_init, mecab_next, mecab_fin, 3, vars);
if (!obj || ((grn_db_obj *)obj)->id != GRN_DB_MECAB) { return GRN_FILE_CORRUPT; }
return GRN_SUCCESS;
}
static void
add_weight_vector(grn_ctx *ctx,
grn_obj *column,
grn_obj *value,
grn_obj *vector)
{
unsigned int i, n;
grn_obj weight_buffer;
n = GRN_UINT32_VALUE(value);
GRN_UINT32_INIT(&weight_buffer, 0);
for (i = 0; i < n; i += 2) {
grn_rc rc;
grn_obj *key, *weight;
key = value + 1 + i;
weight = key + 1;
GRN_BULK_REWIND(&weight_buffer);
rc = grn_obj_cast(ctx, weight, &weight_buffer, GRN_TRUE);
if (rc != GRN_SUCCESS) {
grn_obj *range;
range = grn_ctx_at(ctx, weight_buffer.header.domain);
ERR_CAST(column, range, weight);
grn_obj_unlink(ctx, range);
break;
}
grn_vector_add_element(ctx,
vector,
GRN_BULK_HEAD(key),
GRN_BULK_VSIZE(key),
GRN_UINT32_VALUE(&weight_buffer),
key->header.domain);
}
GRN_OBJ_FIN(ctx, &weight_buffer);
}
grn_token *
grn_token_open(grn_ctx *ctx, grn_obj *table, const char *str, size_t str_len,
grn_token_mode mode, unsigned int flags)
{
grn_token *token;
grn_encoding encoding;
grn_obj *tokenizer;
grn_obj *normalizer;
grn_obj_flags table_flags;
if (grn_table_get_info(ctx, table, &table_flags, &encoding, &tokenizer,
&normalizer)) {
return NULL;
}
if (!(token = GRN_MALLOC(sizeof(grn_token)))) { return NULL; }
token->table = table;
token->mode = mode;
token->encoding = encoding;
token->tokenizer = tokenizer;
token->orig = str;
token->orig_blen = str_len;
token->curr = NULL;
token->nstr = NULL;
token->curr_size = 0;
token->pos = -1;
token->status = GRN_TOKEN_DOING;
token->force_prefix = 0;
if (tokenizer) {
grn_obj str_, flags_;
GRN_TEXT_INIT(&str_, GRN_OBJ_DO_SHALLOW_COPY);
GRN_TEXT_SET_REF(&str_, str, str_len);
GRN_UINT32_INIT(&flags_, 0);
GRN_UINT32_SET(ctx, &flags_, flags);
token->pctx.caller = NULL;
token->pctx.user_data.ptr = NULL;
token->pctx.proc = (grn_proc *)tokenizer;
token->pctx.hooks = NULL;
token->pctx.currh = NULL;
token->pctx.phase = PROC_INIT;
grn_ctx_push(ctx, &str_);
grn_ctx_push(ctx, &flags_);
((grn_proc *)tokenizer)->funcs[PROC_INIT](ctx, 1, &table, &token->pctx.user_data);
grn_obj_close(ctx, &flags_);
grn_obj_close(ctx, &str_);
} else {
int nflags = 0;
token->nstr = grn_string_open_(ctx, str, str_len,
normalizer, nflags, token->encoding);
if (token->nstr) {
const char *normalized;
grn_string_get_normalized(ctx, token->nstr,
&normalized, &(token->curr_size), NULL);
token->curr = (const unsigned char *)normalized;
} else {
ERR(GRN_TOKENIZER_ERROR, "grn_string_open failed at grn_token_open");
}
}
if (ctx->rc) {
grn_token_close(ctx, token);
token = NULL;
}
return token;
}
void
test_mroonga_index_score(void)
{
grn_obj *t1,*c1,*lc,*ft;
grn_obj buff;
grn_id r1,r2,r3,r4;
remove_tmp_directory();
g_mkdir_with_parents(tmp_directory,0700);
g_chdir(tmp_directory);
g_mkdir_with_parents("mrn",0700);
db = grn_db_create(context,"mroonga.grn",NULL);
cut_assert_not_null(db);
/* actual table */
t1 = grn_table_create(context,"t1",2,"mrn/t1.grn",
GRN_OBJ_TABLE_NO_KEY|GRN_OBJ_PERSISTENT,NULL,0);
cut_assert_not_null(t1);
/* lexicon table */
lc = grn_table_create(context,"lc",2,"mrn/lc.grn",
GRN_OBJ_TABLE_PAT_KEY|GRN_OBJ_PERSISTENT,
grn_ctx_at(context, GRN_DB_SHORT_TEXT), 0);
cut_assert_not_null(lc);
grn_test_assert(grn_obj_set_info(context, lc, GRN_INFO_DEFAULT_TOKENIZER,
grn_ctx_at(context, GRN_DB_BIGRAM)));
/* actual column */
c1 = grn_column_create(context,t1,"c1",2,"mrn/t1.c1.grn",
GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT,
grn_ctx_at(context, GRN_DB_TEXT));
cut_assert_not_null(c1);
/* fulltext index */
ft = grn_column_create(context,lc,"ft",2,"mrn/lc.ft.grn",
GRN_OBJ_COLUMN_INDEX|GRN_OBJ_PERSISTENT,t1);
cut_assert_not_null(ft);
GRN_TEXT_INIT(&buff,0);
/* link between actual column and fulltext index */
GRN_UINT32_SET(context, &buff, grn_obj_id(context, c1));
grn_obj_set_info(context, ft, GRN_INFO_SOURCE, &buff); /* need to use grn_id */
/* insert row */
r1 = grn_table_add(context, t1, NULL, 0, NULL);
cut_assert_equal_int(1,r1);
GRN_TEXT_SETS(context, &buff, "abcde");
grn_test_assert(grn_obj_set_value(context, c1, r1, &buff, GRN_OBJ_SET));
r2 = grn_table_add(context, t1, NULL, 0, NULL);
cut_assert_equal_int(2,r2);
GRN_TEXT_SETS(context, &buff, "fghij");
grn_test_assert(grn_obj_set_value(context, c1, r2, &buff, GRN_OBJ_SET));
r3 = grn_table_add(context, t1, NULL, 0, NULL);
cut_assert_equal_int(3,r3);
GRN_TEXT_SETS(context, &buff, "11 22 33");
grn_test_assert(grn_obj_set_value(context, c1, r3, &buff, GRN_OBJ_SET));
r4 = grn_table_add(context, t1, NULL, 0, NULL);
cut_assert_equal_int(4,r4);
GRN_TEXT_SETS(context, &buff, "44 22 55");
grn_test_assert(grn_obj_set_value(context, c1, r4, &buff, GRN_OBJ_SET));
/* confirm record are inserted in both column and index */
cut_assert_equal_int(4,grn_table_size(context,t1));
cut_assert_equal_int(23,grn_table_size(context,lc));
/* nlq search */
{
grn_id id, docid;
grn_obj *res;
grn_table_cursor *tc;
grn_obj score, *score_column;
res = grn_table_create(context, NULL, 0, NULL,
GRN_TABLE_HASH_KEY|GRN_OBJ_WITH_SUBREC, t1, 0);
GRN_UINT32_INIT(&score, 0);
GRN_BULK_REWIND(&buff);
GRN_TEXT_SETS(context, &buff, "hij");
grn_obj_search(context, ft, &buff, res, GRN_OP_OR, NULL);
cut_assert_equal_int(1, grn_table_size(context, res));
score_column = grn_obj_column(context, res, ".:score", 7);
tc = grn_table_cursor_open(context, res, NULL, 0, NULL, 0, 0, 0, 0);
while ((id = grn_table_cursor_next(context, tc))) {
GRN_BULK_REWIND(&buff);
grn_table_get_key(context, res, id, &docid, sizeof(grn_id));
cut_assert_equal_int(2, docid);
cut_assert_not_null(grn_obj_get_value(context, c1, docid, &buff));
cut_assert_equal_int(5 ,GRN_TEXT_LEN(&buff));
cut_assert_equal_substring("fghij", (char*) GRN_BULK_HEAD(&buff),GRN_TEXT_LEN(&buff));
grn_obj_get_value(context, score_column, id, &score);
cut_assert_equal_uint(1, GRN_UINT32_VALUE(&score));
}
grn_table_cursor_close(context, tc);
grn_obj_close(context, score_column);
grn_obj_close(context, res);
}
/* boolean search */
{
grn_id id, docid;
grn_obj *res;
grn_query *query;
grn_table_cursor *tc;
grn_obj score, *score_column;
const char *qstr = "+22 -55";
res = grn_table_create(context, NULL, 0, NULL,
GRN_TABLE_HASH_KEY|GRN_OBJ_WITH_SUBREC, t1, 0);
GRN_UINT32_INIT(&score, 0);
query = grn_query_open(context, qstr, strlen(qstr), GRN_OP_OR, 32);
grn_obj_search(context, ft, (grn_obj*) query, res, GRN_OP_OR, NULL);
cut_assert_equal_int(1, grn_table_size(context, res));
score_column = grn_obj_column(context, res, ".:score", 7);
tc = grn_table_cursor_open(context, res, NULL, 0, NULL, 0, 0, 0, 0);
while ((id = grn_table_cursor_next(context, tc))) {
GRN_BULK_REWIND(&buff);
grn_table_get_key(context, res, id, &docid, sizeof(grn_id));
cut_assert_equal_int(3, docid);
cut_assert_not_null(grn_obj_get_value(context, c1, docid, &buff));
cut_assert_equal_int(8 ,GRN_TEXT_LEN(&buff));
cut_assert_equal_substring("11 22 33", (char*) GRN_BULK_HEAD(&buff),GRN_TEXT_LEN(&buff));
grn_obj_get_value(context, score_column, id, &score);
cut_assert_equal_uint(5, GRN_UINT32_VALUE(&score));
}
grn_query_close(context, query);
grn_table_cursor_close(context ,tc);
grn_obj_close(context, score_column);
grn_obj_close(context, res);
}
grn_obj_close(context, &buff);
grn_obj_close(context, ft);
grn_obj_close(context, c1);
grn_obj_close(context, lc);
grn_obj_close(context, t1);
}
void
test_scan_search(void)
{
grn_obj *cond, *expr, *v, textbuf, intbuf;
GRN_TEXT_INIT(&textbuf, 0);
GRN_UINT32_INIT(&intbuf, 0);
prepare_data(&textbuf, &intbuf);
cut_assert_not_null((cond = grn_expr_create(&context, NULL, 0)));
v = grn_expr_add_var(&context, cond, NULL, 0);
GRN_RECORD_INIT(v, 0, grn_obj_id(&context, docs));
grn_expr_append_obj(&context, cond, v);
GRN_TEXT_SETS(&context, &textbuf, "size");
grn_expr_append_const(&context, cond, &textbuf);
grn_expr_append_op(&context, cond, GRN_OP_OBJ_GET_VALUE, 2);
GRN_UINT32_SET(&context, &intbuf, 14);
grn_expr_append_const(&context, cond, &intbuf);
grn_expr_append_op(&context, cond, GRN_OP_EQUAL, 2);
grn_expr_compile(&context, cond);
cut_assert_not_null((expr = grn_expr_create(&context, NULL, 0)));
v = grn_expr_add_var(&context, expr, NULL, 0);
GRN_BULK_REWIND(&textbuf);
grn_expr_append_const(&context, expr, &textbuf);
GRN_UINT32_SET(&context, &intbuf, GRN_TABLE_HASH_KEY|GRN_OBJ_WITH_SUBREC);
grn_expr_append_const(&context, expr, &intbuf);
grn_expr_append_obj(&context, expr, docs);
GRN_UINT32_SET(&context, &intbuf, 0);
grn_expr_append_const(&context, expr, &intbuf);
grn_expr_append_op(&context, expr, GRN_OP_TABLE_CREATE, 4);
grn_expr_append_obj(&context, expr, v);
grn_expr_append_op(&context, expr, GRN_OP_VAR_SET_VALUE, 2);
grn_expr_append_obj(&context, expr, docs);
grn_expr_append_obj(&context, expr, cond);
grn_expr_append_obj(&context, expr, v);
GRN_UINT32_SET(&context, &intbuf, GRN_SEL_OR);
grn_expr_append_const(&context, expr, &intbuf);
grn_expr_append_op(&context, expr, GRN_OP_TABLE_SCAN, 4);
grn_expr_append_obj(&context, expr, index_body);
GRN_TEXT_SETS(&context, &textbuf, "moge");
grn_expr_append_const(&context, expr, &textbuf);
grn_expr_append_obj(&context, expr, v);
GRN_UINT32_SET(&context, &intbuf, GRN_SEL_AND);
grn_expr_append_const(&context, expr, &intbuf);
grn_expr_append_op(&context, expr, GRN_OP_OBJ_SEARCH, 4);
grn_expr_append_obj(&context, expr, v);
GRN_TEXT_SETS(&context, &textbuf, ".size .:score .body");
grn_expr_append_const(&context, expr, &textbuf);
GRN_BULK_REWIND(&textbuf);
grn_expr_append_obj(&context, expr, &textbuf);
grn_expr_append_op(&context, expr, GRN_OP_JSON_PUT, 3);
grn_expr_exec(&context, expr);
cut_assert_equal_substring("[[14, 4, \"moge moge moge\"], [14, 2, \"moge hoge hoge\"]]",
GRN_TEXT_VALUE(&textbuf), GRN_TEXT_LEN(&textbuf));
grn_test_assert(grn_obj_close(&context, expr));
grn_test_assert(grn_obj_close(&context, cond));
grn_test_assert(grn_obj_close(&context, &textbuf));
grn_test_assert(grn_obj_close(&context, &intbuf));
}
void
test_expr_query(void)
{
grn_obj *t1, *c1, *lc, *ft, *v, *expr;
grn_obj textbuf, intbuf;
grn_id r1, r2, r3, r4;
/* actual table */
t1 = grn_table_create(&context, "t1", 2, NULL,
GRN_OBJ_TABLE_NO_KEY|GRN_OBJ_PERSISTENT, NULL, 0);
cut_assert_not_null(t1);
/* lexicon table */
lc = grn_table_create(&context, "lc", 2, NULL,
GRN_OBJ_TABLE_PAT_KEY|GRN_OBJ_PERSISTENT,
grn_ctx_at(&context, GRN_DB_SHORTTEXT), 0);
cut_assert_not_null(lc);
grn_test_assert(grn_obj_set_info(&context, lc, GRN_INFO_DEFAULT_TOKENIZER,
grn_ctx_at(&context, GRN_DB_BIGRAM)));
/* actual column */
c1 = grn_column_create(&context, t1, "c1", 2, NULL,
GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT,
grn_ctx_at(&context, GRN_DB_TEXT));
cut_assert_not_null(c1);
/* fulltext index */
ft = grn_column_create(&context, lc, "ft", 2, NULL,
GRN_OBJ_COLUMN_INDEX|GRN_OBJ_PERSISTENT|GRN_OBJ_WITH_POSITION, t1);
cut_assert_not_null(ft);
GRN_TEXT_INIT(&textbuf, 0);
GRN_UINT32_INIT(&intbuf, 0);
/* link between actual column and fulltext index */
GRN_UINT32_SET(&context, &intbuf, grn_obj_id(&context, c1));
grn_obj_set_info(&context, ft, GRN_INFO_SOURCE, &intbuf); /* need to use grn_id */
/* insert row */
r1 = grn_table_add(&context, t1, NULL, 0, NULL);
cut_assert_equal_int(1, r1);
GRN_TEXT_SETS(&context, &textbuf, "abhij");
grn_test_assert(grn_obj_set_value(&context, c1, r1, &textbuf, GRN_OBJ_SET));
r2 = grn_table_add(&context, t1, NULL, 0, NULL);
cut_assert_equal_int(2, r2);
GRN_TEXT_SETS(&context, &textbuf, "fghij");
grn_test_assert(grn_obj_set_value(&context, c1, r2, &textbuf, GRN_OBJ_SET));
r3 = grn_table_add(&context, t1, NULL, 0, NULL);
cut_assert_equal_int(3, r3);
GRN_TEXT_SETS(&context, &textbuf, "11 22 33");
grn_test_assert(grn_obj_set_value(&context, c1, r3, &textbuf, GRN_OBJ_SET));
r4 = grn_table_add(&context, t1, NULL, 0, NULL);
cut_assert_equal_int(4, r4);
GRN_TEXT_SETS(&context, &textbuf, "44 22 55");
grn_test_assert(grn_obj_set_value(&context, c1, r4, &textbuf, GRN_OBJ_SET));
/* confirm record are inserted in both column and index */
cut_assert_equal_int(4, grn_table_size(&context, t1));
cut_assert_equal_int(17, grn_table_size(&context, lc));
cut_assert_not_null((expr = grn_expr_create(&context, NULL, 0)));
v = grn_expr_add_var(&context, expr, NULL, 0);
GRN_BULK_REWIND(&textbuf);
grn_expr_append_const(&context, expr, &textbuf);
GRN_UINT32_SET(&context, &intbuf, GRN_TABLE_HASH_KEY|GRN_OBJ_WITH_SUBREC);
grn_expr_append_const(&context, expr, &intbuf);
grn_expr_append_obj(&context, expr, t1);
GRN_UINT32_SET(&context, &intbuf, 0);
grn_expr_append_const(&context, expr, &intbuf);
grn_expr_append_op(&context, expr, GRN_OP_TABLE_CREATE, 4);
grn_expr_append_obj(&context, expr, v);
grn_expr_append_op(&context, expr, GRN_OP_VAR_SET_VALUE, 2);
grn_expr_append_obj(&context, expr, ft);
GRN_TEXT_SETS(&context, &textbuf, "hij");
grn_expr_append_const(&context, expr, &textbuf);
grn_expr_append_obj(&context, expr, v);
GRN_UINT32_SET(&context, &intbuf, GRN_SEL_OR);
grn_expr_append_const(&context, expr, &intbuf);
grn_expr_append_op(&context, expr, GRN_OP_OBJ_SEARCH, 4);
grn_expr_append_obj(&context, expr, v);
GRN_TEXT_SETS(&context, &textbuf, ".c1 .:score");
grn_expr_append_const(&context, expr, &textbuf);
GRN_BULK_REWIND(&textbuf);
grn_expr_append_obj(&context, expr, &textbuf);
grn_expr_append_op(&context, expr, GRN_OP_JSON_PUT, 3);
grn_expr_compile(&context, expr);
grn_expr_exec(&context, expr);
cut_assert_equal_uint(0, grn_obj_close(&context, expr));
cut_assert_equal_substring("[[\"abhij\", 1], [\"fghij\", 1]]",
GRN_TEXT_VALUE(&textbuf), GRN_TEXT_LEN(&textbuf));
grn_obj_close(&context, &textbuf);
grn_obj_close(&context, ft);
grn_obj_close(&context, c1);
grn_obj_close(&context, lc);
grn_obj_close(&context, t1);
}
static grn_rc
grn_ctx_impl_init(grn_ctx *ctx)
{
grn_io_mapinfo mi;
if (!(ctx->impl = grn_io_anon_map(ctx, &mi, IMPL_SIZE))) {
return ctx->rc;
}
grn_alloc_init_ctx_impl(ctx);
ctx->impl->encoding = ctx->encoding;
ctx->impl->lifoseg = -1;
ctx->impl->currseg = -1;
CRITICAL_SECTION_INIT(ctx->impl->lock);
if (!(ctx->impl->values = grn_array_create(ctx, NULL, sizeof(grn_db_obj *),
GRN_ARRAY_TINY))) {
CRITICAL_SECTION_FIN(ctx->impl->lock);
grn_io_anon_unmap(ctx, &mi, IMPL_SIZE);
ctx->impl = NULL;
return ctx->rc;
}
if (!(ctx->impl->temporary_columns = grn_pat_create(ctx, NULL,
GRN_TABLE_MAX_KEY_SIZE,
sizeof(grn_obj *),
0))) {
grn_array_close(ctx, ctx->impl->values);
CRITICAL_SECTION_FIN(ctx->impl->lock);
grn_io_anon_unmap(ctx, &mi, IMPL_SIZE);
ctx->impl = NULL;
return ctx->rc;
}
if (!(ctx->impl->ios = grn_hash_create(ctx, NULL, GRN_TABLE_MAX_KEY_SIZE,
sizeof(grn_io *),
GRN_OBJ_KEY_VAR_SIZE|GRN_HASH_TINY))) {
grn_array_close(ctx, ctx->impl->values);
grn_pat_close(ctx, ctx->impl->temporary_columns);
CRITICAL_SECTION_FIN(ctx->impl->lock);
grn_io_anon_unmap(ctx, &mi, IMPL_SIZE);
ctx->impl = NULL;
return ctx->rc;
}
ctx->impl->db = NULL;
ctx->impl->expr_vars = grn_hash_create(ctx, NULL, sizeof(grn_id), sizeof(grn_obj *), 0);
ctx->impl->stack_curr = 0;
ctx->impl->curr_expr = NULL;
ctx->impl->qe_next = NULL;
GRN_TEXT_INIT(&ctx->impl->current_request_id, 0);
ctx->impl->current_request_timer_id = NULL;
ctx->impl->parser = NULL;
GRN_TEXT_INIT(&ctx->impl->output.names, GRN_OBJ_VECTOR);
GRN_UINT32_INIT(&ctx->impl->output.levels, GRN_OBJ_VECTOR);
if (ctx == &grn_gctx) {
ctx->impl->command_version = GRN_COMMAND_VERSION_STABLE;
} else {
ctx->impl->command_version = grn_get_default_command_version();
}
if (ctx == &grn_gctx) {
ctx->impl->match_escalation_threshold =
GRN_DEFAULT_MATCH_ESCALATION_THRESHOLD;
} else {
ctx->impl->match_escalation_threshold =
grn_get_default_match_escalation_threshold();
}
ctx->impl->finalizer = NULL;
ctx->impl->com = NULL;
ctx->impl->output.buf = grn_obj_open(ctx, GRN_BULK, 0, GRN_DB_TEXT);
ctx->impl->output.func = NULL;
ctx->impl->output.data.ptr = NULL;
#ifdef GRN_WITH_MESSAGE_PACK
msgpack_packer_init(&ctx->impl->output.msgpacker,
ctx, grn_msgpack_buffer_write);
#endif
ctx->impl->tv.tv_sec = 0;
ctx->impl->tv.tv_nsec = 0;
ctx->impl->edge = NULL;
grn_loader_init(&ctx->impl->loader);
ctx->impl->plugin_path = NULL;
GRN_TEXT_INIT(&ctx->impl->query_log_buf, 0);
ctx->impl->previous_errbuf[0] = '\0';
ctx->impl->n_same_error_messages = 0;
grn_ctx_impl_mrb_init(ctx);
return ctx->rc;
}
/*
This function is called for a full text search query or a document to be
indexed. This means that both short/long strings are given.
The return value of this function is ignored. When an error occurs in this
function, `ctx->rc' is overwritten with an error code (not GRN_SUCCESS).
*/
static grn_obj *
mecab_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
{
grn_obj *str;
int nflags = 0;
char *buf, *p;
const char *s;
grn_obj *table = args[0];
grn_obj_flags table_flags;
grn_encoding table_encoding;
grn_mecab_tokenizer *token;
unsigned int bufsize, len;
if (!(str = grn_ctx_pop(ctx))) {
ERR(GRN_INVALID_ARGUMENT, "missing argument");
return NULL;
}
if (!sole_mecab) {
CRITICAL_SECTION_ENTER(sole_mecab_lock);
if (!sole_mecab) {
sole_mecab = mecab_new2("-Owakati");
if (!sole_mecab) {
ERR(GRN_TOKENIZER_ERROR, "mecab_new2 failed on grn_mecab_init: %s",
mecab_strerror(NULL));
} else {
sole_mecab_encoding = get_mecab_encoding(sole_mecab);
}
}
CRITICAL_SECTION_LEAVE(sole_mecab_lock);
}
if (!sole_mecab) {
return NULL;
}
grn_table_get_info(ctx, table, &table_flags, &table_encoding, NULL);
if (table_encoding != sole_mecab_encoding) {
ERR(GRN_TOKENIZER_ERROR,
"MeCab dictionary charset (%s) does not match the context encoding: <%s>",
grn_enctostr(sole_mecab_encoding), grn_enctostr(table_encoding));
return NULL;
}
if (!(token = GRN_MALLOC(sizeof(grn_mecab_tokenizer)))) { return NULL; }
token->mecab = sole_mecab;
token->encoding = table_encoding;
nflags |= (table_flags & GRN_OBJ_KEY_NORMALIZE);
if (!(token->nstr = grn_str_open_(ctx, GRN_TEXT_VALUE(str), GRN_TEXT_LEN(str),
nflags, token->encoding))) {
GRN_FREE(token);
ERR(GRN_TOKENIZER_ERROR, "grn_str_open failed at grn_token_open");
return NULL;
}
len = token->nstr->norm_blen;
CRITICAL_SECTION_ENTER(sole_mecab_lock);
s = mecab_sparse_tostr2(token->mecab, token->nstr->norm, len);
if (!s) {
ERR(GRN_TOKENIZER_ERROR, "mecab_sparse_tostr failed len=%d err=%s",
len, mecab_strerror(token->mecab));
} else {
bufsize = strlen(s) + 1;
if (!(buf = GRN_MALLOC(bufsize))) {
GRN_LOG(ctx, GRN_LOG_ALERT, "buffer allocation on mecab_init failed !");
} else {
memcpy(buf, s, bufsize);
}
}
CRITICAL_SECTION_LEAVE(sole_mecab_lock);
if (!s || !buf) {
grn_str_close(ctx, token->nstr);
GRN_FREE(token);
return NULL;
}
/* A certain version of mecab returns trailing lf or spaces. */
for (p = buf + bufsize - 2;
buf <= p && isspace(*(unsigned char *)p);
p--) { *p = '\0'; }
user_data->ptr = token;
token->buf = buf;
token->next = buf;
token->end = p + 1;
GRN_TEXT_INIT(&token->curr_, GRN_OBJ_DO_SHALLOW_COPY);
GRN_UINT32_INIT(&token->stat_, 0);
return NULL;
}
static grn_rc
mecab_init(grn_ctx *ctx, grn_obj *table, grn_proc_data *user_data)
{
grn_obj *str;
int nflags = 0;
char *buf, *s, *p;
char mecab_err[256];
grn_obj_flags table_flags;
grn_mecab_tokenizer *token;
unsigned int bufsize, maxtrial = 10, len;
if (!(str = grn_ctx_pop(ctx))) { return GRN_INVALID_ARGUMENT; }
SOLE_MECAB_CONFIRM;
if (!sole_mecab) {
GRN_LOG(ctx, GRN_LOG_ALERT, "mecab_new failed on grn_mecab_init");
return GRN_TOKENIZER_ERROR;
}
if (!(token = GRN_MALLOC(sizeof(grn_mecab_tokenizer)))) { return ctx->rc; }
user_data->ptr = token;
token->mecab = sole_mecab;
// if (!(token->mecab = mecab_new3())) {
grn_table_get_info(ctx, table, &table_flags, &token->encoding, NULL);
nflags |= (table_flags & GRN_OBJ_KEY_NORMALIZE);
if (!(token->nstr = grn_str_open_(ctx, GRN_TEXT_VALUE(str), GRN_TEXT_LEN(str),
nflags, token->encoding))) {
GRN_LOG(ctx, GRN_LOG_ALERT, "grn_str_open failed at grn_token_open");
return GRN_TOKENIZER_ERROR;
}
len = token->nstr->norm_blen;
mecab_err[sizeof(mecab_err) - 1] = '\0';
for (bufsize = len * 2 + 1; maxtrial; bufsize *= 2, maxtrial--) {
if(!(buf = GRN_MALLOC(bufsize + 1))) {
GRN_LOG(ctx, GRN_LOG_ALERT, "buffer allocation on mecab_init failed !");
GRN_FREE(token);
return ctx->rc;
}
MUTEX_LOCK(sole_mecab_lock);
s = mecab_sparse_tostr3(token->mecab, token->nstr->norm, len, buf, bufsize);
if (!s) {
strncpy(mecab_err, mecab_strerror(token->mecab), sizeof(mecab_err) - 1);
}
MUTEX_UNLOCK(sole_mecab_lock);
if (s) { break; }
GRN_FREE(buf);
if (strstr(mecab_err, "output buffer overflow") == NULL) { break; }
}
if (!s) {
GRN_LOG(ctx, GRN_LOG_ALERT, "mecab_sparse_tostr failed len=%d bufsize=%d err=%s",
len, bufsize, mecab_err);
GRN_FREE(token);
return GRN_TOKENIZER_ERROR;
}
// certain version of mecab returns trailing lf or spaces.
for (p = buf + strlen(buf) - 1;
buf <= p && (*p == '\n' || isspace(*(unsigned char *)p));
p--) { *p = '\0'; }
//grn_log("sparsed='%s'", s);
token->buf = (unsigned char *)buf;
token->next = (unsigned char *)buf;
token->end = (unsigned char *)buf + strlen(buf);
GRN_TEXT_INIT(&token->curr_, GRN_OBJ_DO_SHALLOW_COPY);
GRN_UINT32_INIT(&token->stat_, 0);
return GRN_SUCCESS;
}
static grn_obj *
command_tag_synonym(grn_ctx *ctx, GNUC_UNUSED int nargs, GNUC_UNUSED grn_obj **args,
GNUC_UNUSED grn_user_data *user_data)
{
GNUC_UNUSED grn_obj *flags = grn_ctx_pop(ctx);
grn_obj *newvalue = grn_ctx_pop(ctx);
grn_obj *oldvalue = grn_ctx_pop(ctx);
GNUC_UNUSED grn_obj *id = grn_ctx_pop(ctx);
grn_obj buf;
grn_obj record;
grn_obj *domain;
grn_obj *table;
grn_obj *column;
int i,n;
if (GRN_BULK_VSIZE(newvalue) == 0 || GRN_INT32_VALUE(flags) == 0) {
return NULL;
}
table = grn_ctx_at(ctx, oldvalue->header.domain);
if (table && !is_table(table)) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_WARNING,
"[tag-synonym] "
"hooked column must be reference type");
return NULL;
}
column = grn_obj_column(ctx,
table,
SYNONYM_COLUMN_NAME,
SYNONYM_COLUMN_NAME_LEN);
if (!column) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_WARNING,
"[tag-synonym] "
"couldn't open synonym column");
return NULL;
}
GRN_TEXT_INIT(&buf, 0);
domain = grn_ctx_at(ctx, newvalue->header.domain);
if (domain && is_string(domain)) {
GRN_RECORD_INIT(&record, GRN_OBJ_VECTOR, oldvalue->header.domain);
grn_table_tokenize(ctx, table, GRN_TEXT_VALUE(newvalue), GRN_TEXT_LEN(newvalue), &record, GRN_TRUE);
} else if (newvalue->header.type == GRN_UVECTOR) {
record = *newvalue;
}
if (is_string(domain) || newvalue->header.type == GRN_UVECTOR) {
grn_obj value;
GRN_RECORD_INIT(newvalue, GRN_OBJ_VECTOR, oldvalue->header.domain);
GRN_UINT32_INIT(&value, 0);
n = grn_vector_size(ctx, &record);
for (i = 0; i < n; i++) {
grn_id tid;
tid = grn_uvector_get_element(ctx, &record, i, NULL);
GRN_BULK_REWIND(&value);
grn_obj_get_value(ctx, column, tid, &value);
if (GRN_UINT32_VALUE(&value)) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_INFO,
"[tag-synonym] "
"changed: tid %d -> %d", tid, GRN_UINT32_VALUE(&value));
tid = GRN_UINT32_VALUE(&value);
}
grn_uvector_add_element(ctx, newvalue, tid, 0);
}
grn_obj_unlink(ctx, &value);
} else {
grn_id tid;
grn_obj value;
tid = GRN_RECORD_VALUE(newvalue);
GRN_UINT32_INIT(&value, 0);
grn_obj_get_value(ctx, column, tid, &value);
if (GRN_UINT32_VALUE(&value)) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_INFO,
"[tag-synonym] "
"changed: tid %d -> %d", tid, GRN_UINT32_VALUE(&value));
tid = GRN_UINT32_VALUE(&value);
GRN_BULK_REWIND(newvalue);
GRN_RECORD_SET(ctx, newvalue, tid);
}
grn_obj_unlink(ctx, &value);
}
grn_obj_unlink(ctx, &buf);
return NULL;
}
void
grn_output_obj(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type,
grn_obj *obj, grn_obj_format *format)
{
grn_obj buf;
GRN_TEXT_INIT(&buf, 0);
switch (obj->header.type) {
case GRN_BULK :
switch (obj->header.domain) {
case GRN_DB_VOID :
grn_output_void(ctx, outbuf, output_type, GRN_BULK_HEAD(obj), GRN_BULK_VSIZE(obj));
break;
case GRN_DB_SHORT_TEXT :
case GRN_DB_TEXT :
case GRN_DB_LONG_TEXT :
grn_output_str(ctx, outbuf, output_type, GRN_BULK_HEAD(obj), GRN_BULK_VSIZE(obj));
break;
case GRN_DB_BOOL :
grn_output_bool(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT8_VALUE(obj) : 0);
break;
case GRN_DB_INT8 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT8_VALUE(obj) : 0);
break;
case GRN_DB_UINT8 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT8_VALUE(obj) : 0);
break;
case GRN_DB_INT16 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT16_VALUE(obj) : 0);
break;
case GRN_DB_UINT16 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT16_VALUE(obj) : 0);
break;
case GRN_DB_INT32 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT32_VALUE(obj) : 0);
break;
case GRN_DB_UINT32 :
grn_output_int64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT32_VALUE(obj) : 0);
break;
case GRN_DB_INT64 :
grn_output_int64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT64_VALUE(obj) : 0);
break;
case GRN_DB_UINT64 :
grn_output_uint64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT64_VALUE(obj) : 0);
break;
case GRN_DB_FLOAT :
grn_output_float(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_FLOAT_VALUE(obj) : 0);
break;
case GRN_DB_TIME :
grn_output_time(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT64_VALUE(obj) : 0);
break;
case GRN_DB_TOKYO_GEO_POINT :
case GRN_DB_WGS84_GEO_POINT :
grn_output_geo_point(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? (grn_geo_point *)GRN_BULK_HEAD(obj) : NULL);
break;
default :
if (format) {
int j;
int ncolumns = GRN_BULK_VSIZE(&format->columns)/sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", 2);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_open(ctx, outbuf, output_type, "HIT", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_text_atoj_o(ctx, outbuf, output_type, columns[j], obj);
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_obj *table = grn_ctx_at(ctx, obj->header.domain);
grn_id id = *((grn_id *)GRN_BULK_HEAD(obj));
if (table && table->header.type != GRN_TABLE_NO_KEY) {
grn_obj *accessor = grn_obj_column(ctx, table, "_key", 4);
if (accessor) {
grn_obj_get_value(ctx, accessor, id, &buf);
grn_obj_unlink(ctx, accessor);
}
grn_output_obj(ctx, outbuf, output_type, &buf, format);
} else {
grn_output_int64(ctx, outbuf, output_type, id);
}
}
break;
}
break;
case GRN_UVECTOR :
if (format) {
int i, j;
grn_id *v = (grn_id *)GRN_BULK_HEAD(obj), *ve = (grn_id *)GRN_BULK_CURR(obj);
int ncolumns = GRN_BULK_VSIZE(&format->columns) / sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
grn_output_array_open(ctx, outbuf, output_type, "RESULTSET", -1);
grn_output_array_open(ctx, outbuf, output_type, "NHITS", 1);
grn_text_itoa(ctx, outbuf, ve - v);
grn_output_array_close(ctx, outbuf, output_type);
if (v < ve) {
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", -1);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", -1);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
for (i = 0;; i++) {
grn_output_array_open(ctx, outbuf, output_type, "HITS", -1);
for (j = 0; j < ncolumns; j++) {
GRN_BULK_REWIND(&buf);
grn_obj_get_value(ctx, columns[j], *v, &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
v++;
if (v < ve) {
} else {
break;
}
}
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_obj *range = grn_ctx_at(ctx, obj->header.domain);
if (range && range->header.type == GRN_TYPE) {
int value_size = ((struct _grn_type *)range)->obj.range;
char *v = (char *)GRN_BULK_HEAD(obj),
*ve = (char *)GRN_BULK_CURR(obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
if (v < ve) {
for (;;) {
grn_obj value;
GRN_OBJ_INIT(&value, GRN_BULK, 0, obj->header.domain);
grn_bulk_write_from(ctx, &value, v, 0, value_size);
grn_output_obj(ctx, outbuf, output_type, &value, NULL);
v += value_size;
if (v < ve) {
} else {
break;
}
}
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_id *v = (grn_id *)GRN_BULK_HEAD(obj),
*ve = (grn_id *)GRN_BULK_CURR(obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", ve - v);
if (v < ve) {
grn_obj key;
GRN_OBJ_INIT(&key, GRN_BULK, 0, range->header.domain);
for (;;) {
if (range->header.type != GRN_TABLE_NO_KEY) {
grn_table_get_key2(ctx, range, *v, &key);
grn_output_obj(ctx, outbuf, output_type, &key, NULL);
GRN_BULK_REWIND(&key);
} else {
grn_obj id;
GRN_UINT32_INIT(&id, 0);
GRN_UINT32_SET(ctx, &id, *v);
grn_output_obj(ctx, outbuf, output_type, &id, NULL);
GRN_OBJ_FIN(ctx, &id);
}
v++;
if (v < ve) {
} else {
break;
}
}
GRN_OBJ_FIN(ctx, &key);
}
grn_output_array_close(ctx, outbuf, output_type);
}
}
break;
case GRN_VECTOR :
if (obj->header.domain == GRN_DB_VOID) {
ERR(GRN_INVALID_ARGUMENT, "invalid obj->header.domain");
}
if (format) {
ERR(GRN_FUNCTION_NOT_IMPLEMENTED,
"cannot print GRN_VECTOR using grn_obj_format");
} else {
unsigned int i, n;
grn_obj value;
GRN_VOID_INIT(&value);
n = grn_vector_size(ctx, obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
for (i = 0; i < n; i++) {
const char *_value;
unsigned int weight, length;
grn_id domain;
length = grn_vector_get_element(ctx, obj, i,
&_value, &weight, &domain);
if (domain != GRN_DB_VOID) {
grn_obj_reinit(ctx, &value, domain, 0);
} else {
grn_obj_reinit(ctx, &value, obj->header.domain, 0);
}
grn_bulk_write(ctx, &value, _value, length);
grn_output_obj(ctx, outbuf, output_type, &value, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
GRN_OBJ_FIN(ctx, &value);
}
break;
case GRN_PVECTOR :
if (format) {
ERR(GRN_FUNCTION_NOT_IMPLEMENTED,
"cannot print GRN_PVECTOR using grn_obj_format");
} else {
unsigned int i, n;
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
n = GRN_BULK_VSIZE(obj) / sizeof(grn_obj *);
for (i = 0; i < n; i++) {
grn_obj *value;
value = GRN_PTR_VALUE_AT(obj, i);
grn_output_obj(ctx, outbuf, output_type, value, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
break;
case GRN_TABLE_HASH_KEY :
case GRN_TABLE_PAT_KEY :
case GRN_TABLE_NO_KEY :
case GRN_TABLE_VIEW :
if (format) {
int i, j;
int ncolumns = GRN_BULK_VSIZE(&format->columns)/sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
grn_table_cursor *tc = grn_table_cursor_open(ctx, obj, NULL, 0, NULL, 0,
format->offset, format->limit,
GRN_CURSOR_ASCENDING);
int resultset_size = -1;
if (!tc) { ERRCLR(ctx); }
#ifdef HAVE_MESSAGE_PACK
resultset_size = 1; /* [NHITS, (COLUMNS), (HITS)] */
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
resultset_size++;
}
resultset_size += format->limit;
#endif
grn_output_array_open(ctx, outbuf, output_type, "RESULTSET", resultset_size);
grn_output_array_open(ctx, outbuf, output_type, "NHITS", 1);
if (output_type == GRN_CONTENT_XML) {
grn_text_itoa(ctx, outbuf, format->nhits);
} else {
grn_output_int32(ctx, outbuf, output_type, format->nhits);
}
grn_output_array_close(ctx, outbuf, output_type);
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", 2);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
if (tc) {
grn_obj id;
GRN_TEXT_INIT(&id, 0);
for (i = 0; !grn_table_cursor_next_o(ctx, tc, &id); i++) {
grn_output_array_open(ctx, outbuf, output_type, "HIT", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_text_atoj_o(ctx, outbuf, output_type, columns[j], &id);
}
grn_output_array_close(ctx, outbuf, output_type);
}
GRN_OBJ_FIN(ctx, &id);
grn_table_cursor_close(ctx, tc);
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
int i;
grn_obj *column = grn_obj_column(ctx, obj, "_key", 4);
grn_table_cursor *tc = grn_table_cursor_open(ctx, obj, NULL, 0, NULL, 0,
0, -1, GRN_CURSOR_ASCENDING);
grn_output_array_open(ctx, outbuf, output_type, "HIT", -1);
if (tc) {
grn_obj id;
GRN_TEXT_INIT(&id, 0);
for (i = 0; !grn_table_cursor_next_o(ctx, tc, &id); i++) {
/* todo:
grn_text_atoj_o(ctx, outbuf, output_type, column, &id);
*/
GRN_BULK_REWIND(&buf);
grn_obj_get_value_o(ctx, column, &id, &buf);
grn_text_esc(ctx, outbuf, GRN_BULK_HEAD(&buf), GRN_BULK_VSIZE(&buf));
}
GRN_OBJ_FIN(ctx, &id);
grn_table_cursor_close(ctx, tc);
}
grn_output_array_close(ctx, outbuf, output_type);
grn_obj_unlink(ctx, column);
}
break;
}
GRN_OBJ_FIN(ctx, &buf);
}
static void
construct_object(gconstpointer data, grn_builtin_type type, grn_obj *object)
{
switch (type) {
case GRN_DB_VOID:
GRN_VOID_INIT(object);
break;
case GRN_DB_BOOL:
GRN_BOOL_INIT(object, 0);
GRN_BOOL_SET(&context, object, gcut_data_get_boolean(data, "value"));
break;
case GRN_DB_INT8:
GRN_INT8_INIT(object, 0);
GRN_INT8_SET(&context, object, gcut_data_get_int(data, "value"));
break;
case GRN_DB_UINT8:
GRN_UINT8_INIT(object, 0);
GRN_UINT8_SET(&context, object, gcut_data_get_uint(data, "value"));
break;
case GRN_DB_INT16:
GRN_INT16_INIT(object, 0);
GRN_INT16_SET(&context, object, gcut_data_get_int(data, "value"));
break;
case GRN_DB_UINT16:
GRN_UINT16_INIT(object, 0);
GRN_UINT16_SET(&context, object, gcut_data_get_uint(data, "value"));
break;
case GRN_DB_INT32:
GRN_INT32_INIT(object, 0);
GRN_INT32_SET(&context, object, gcut_data_get_int(data, "value"));
break;
case GRN_DB_UINT32:
GRN_UINT32_INIT(object, 0);
GRN_UINT32_SET(&context, object, gcut_data_get_uint(data, "value"));
break;
case GRN_DB_INT64:
GRN_INT64_INIT(object, 0);
GRN_INT64_SET(&context, object, gcut_data_get_int64(data, "value"));
break;
case GRN_DB_UINT64:
GRN_UINT64_INIT(object, 0);
GRN_UINT64_SET(&context, object, gcut_data_get_uint64(data, "value"));
break;
case GRN_DB_FLOAT:
GRN_FLOAT_INIT(object, 0);
GRN_FLOAT_SET(&context, object, gcut_data_get_double(data, "value"));
break;
case GRN_DB_TIME:
GRN_TIME_INIT(object, 0);
GRN_TIME_SET(&context, object, gcut_data_get_int64(data, "value"));
break;
case GRN_DB_SHORT_TEXT:
GRN_SHORT_TEXT_INIT(object, 0);
GRN_TEXT_SETS(&context, object, gcut_data_get_string(data, "value"));
break;
case GRN_DB_TEXT:
GRN_TEXT_INIT(object, 0);
GRN_TEXT_SETS(&context, object, gcut_data_get_string(data, "value"));
break;
case GRN_DB_LONG_TEXT:
GRN_LONG_TEXT_INIT(object, 0);
GRN_TEXT_SETS(&context, object, gcut_data_get_string(data, "value"));
break;
case GRN_DB_TOKYO_GEO_POINT:
GRN_TOKYO_GEO_POINT_INIT(object, 0);
GRN_GEO_POINT_SET(&context, object,
gcut_data_get_int(data, "latitude"),
gcut_data_get_int(data, "longitude"));
break;
case GRN_DB_WGS84_GEO_POINT:
GRN_WGS84_GEO_POINT_INIT(object, 0);
GRN_GEO_POINT_SET(&context, object,
gcut_data_get_int(data, "latitude"),
gcut_data_get_int(data, "longitude"));
break;
default:
cut_fail("unknown type: %d", type);
break;
}
}
void
grn_tokenizer_token_init(grn_ctx *ctx, grn_tokenizer_token *token)
{
GRN_TEXT_INIT(&token->str, GRN_OBJ_DO_SHALLOW_COPY);
GRN_UINT32_INIT(&token->status, 0);
}
static void
grn_text_atoj(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type,
grn_obj *obj, grn_id id)
{
int vs;
grn_obj buf;
if (obj->header.type == GRN_ACCESSOR) {
grn_accessor *a = (grn_accessor *)obj;
GRN_TEXT_INIT(&buf, 0);
for (;;) {
buf.header.domain = grn_obj_get_range(ctx, obj);
GRN_BULK_REWIND(&buf);
switch (a->action) {
case GRN_ACCESSOR_GET_ID :
GRN_UINT32_PUT(ctx, &buf, id);
buf.header.domain = GRN_DB_UINT32;
break;
case GRN_ACCESSOR_GET_KEY :
grn_table_get_key2(ctx, a->obj, id, &buf);
buf.header.domain = DB_OBJ(a->obj)->header.domain;
break;
case GRN_ACCESSOR_GET_VALUE :
grn_obj_get_value(ctx, a->obj, id, &buf);
buf.header.domain = GRN_DB_INT32; /* fix me */
break;
case GRN_ACCESSOR_GET_SCORE :
grn_obj_get_value(ctx, a->obj, id, &buf);
{
grn_rset_recinfo *ri = (grn_rset_recinfo *)grn_obj_get_value_(ctx, a->obj, id, &vs);
GRN_INT32_PUT(ctx, &buf, ri->score);
}
buf.header.domain = GRN_DB_INT32;
break;
case GRN_ACCESSOR_GET_NSUBRECS :
{
grn_rset_recinfo *ri = (grn_rset_recinfo *)grn_obj_get_value_(ctx, a->obj, id, &vs);
GRN_INT32_PUT(ctx, &buf, ri->n_subrecs);
}
buf.header.domain = GRN_DB_INT32;
break;
case GRN_ACCESSOR_GET_COLUMN_VALUE :
if ((a->obj->header.flags & GRN_OBJ_COLUMN_TYPE_MASK) == GRN_OBJ_COLUMN_VECTOR) {
if (a->next) {
grn_id *idp;
grn_obj_get_value(ctx, a->obj, id, &buf);
idp = (grn_id *)GRN_BULK_HEAD(&buf);
vs = GRN_BULK_VSIZE(&buf) / sizeof(grn_id);
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", vs);
for (; vs--; idp++) {
grn_text_atoj(ctx, outbuf, output_type, (grn_obj *)a->next, *idp);
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_text_atoj(ctx, outbuf, output_type, a->obj, id);
}
goto exit;
} else {
grn_obj_get_value(ctx, a->obj, id, &buf);
}
break;
case GRN_ACCESSOR_GET_DB_OBJ :
/* todo */
break;
case GRN_ACCESSOR_LOOKUP :
/* todo */
break;
case GRN_ACCESSOR_FUNCALL :
/* todo */
break;
}
if (a->next) {
a = a->next;
if (GRN_BULK_VSIZE(&buf) >= sizeof(grn_id)) {
id = *((grn_id *)GRN_BULK_HEAD(&buf));
} else {
id = GRN_ID_NIL;
}
} else {
break;
}
}
} else {
switch (obj->header.type) {
case GRN_COLUMN_FIX_SIZE :
GRN_VALUE_FIX_SIZE_INIT(&buf, 0, DB_OBJ(obj)->range);
break;
case GRN_COLUMN_VAR_SIZE :
if ((obj->header.flags & GRN_OBJ_COLUMN_TYPE_MASK) == GRN_OBJ_COLUMN_VECTOR) {
grn_obj *range = grn_ctx_at(ctx, DB_OBJ(obj)->range);
if (range->header.flags & GRN_OBJ_KEY_VAR_SIZE) {
GRN_VALUE_VAR_SIZE_INIT(&buf, GRN_OBJ_VECTOR, DB_OBJ(obj)->range);
} else {
GRN_VALUE_FIX_SIZE_INIT(&buf, GRN_OBJ_VECTOR, DB_OBJ(obj)->range);
}
} else {
GRN_VALUE_VAR_SIZE_INIT(&buf, 0, DB_OBJ(obj)->range);
}
break;
case GRN_COLUMN_INDEX :
GRN_UINT32_INIT(&buf, 0);
break;
default:
GRN_TEXT_INIT(&buf, 0);
break;
}
grn_obj_get_value(ctx, obj, id, &buf);
}
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
exit :
grn_obj_close(ctx, &buf);
}
