static grn_id
parse_id_value(grn_ctx *ctx, grn_obj *value)
{
switch (value->header.type) {
case GRN_DB_UINT32 :
return GRN_UINT32_VALUE(value);
case GRN_DB_INT32 :
return GRN_INT32_VALUE(value);
default :
{
grn_id id = GRN_ID_NIL;
grn_obj casted_value;
GRN_UINT32_INIT(&casted_value, 0);
if (grn_obj_cast(ctx, value, &casted_value, GRN_FALSE) != GRN_SUCCESS) {
grn_obj inspected;
GRN_TEXT_INIT(&inspected, 0);
grn_inspect(ctx, &inspected, value);
ERR(GRN_INVALID_ARGUMENT,
"<%s>: failed to cast to <UInt32>: <%.*s>",
GRN_COLUMN_NAME_ID,
(int)GRN_TEXT_LEN(&inspected),
GRN_TEXT_VALUE(&inspected));
GRN_OBJ_FIN(ctx, &inspected);
} else {
id = GRN_UINT32_VALUE(&casted_value);
}
GRN_OBJ_FIN(ctx, &casted_value);
return id;
}
}
}
grn_tokenizer_query *
grn_tokenizer_query_open(grn_ctx *ctx, int num_args, grn_obj **args,
unsigned int normalize_flags)
{
grn_obj *flags;
grn_obj *query_str;
grn_obj *tokenize_mode;
GRN_API_ENTER;
flags = grn_ctx_pop(ctx);
query_str = grn_ctx_pop(ctx);
tokenize_mode = grn_ctx_pop(ctx);
if (query_str == NULL) {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, "missing argument");
GRN_API_RETURN(NULL);
}
if ((num_args < 1) || (args == NULL) || (args[0] == NULL)) {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, "invalid NULL pointer");
GRN_API_RETURN(NULL);
}
{
grn_tokenizer_query * const query =
GRN_PLUGIN_MALLOC(ctx, sizeof(grn_tokenizer_query));
if (!query) {
GRN_API_RETURN(NULL);
}
grn_tokenizer_query_init(ctx, query);
grn_tokenizer_query_set_raw_string(ctx,
query,
GRN_TEXT_VALUE(query_str),
GRN_TEXT_LEN(query_str));
if (ctx->rc != GRN_SUCCESS) {
GRN_PLUGIN_FREE(ctx, query);
GRN_API_RETURN(NULL);
}
if (flags) {
grn_tokenizer_query_set_flags(ctx, query, GRN_UINT32_VALUE(flags));
}
if (tokenize_mode) {
grn_tokenizer_query_set_mode(ctx, query, GRN_UINT32_VALUE(tokenize_mode));
}
grn_tokenizer_query_set_normalize_flags(ctx, query, normalize_flags);
grn_tokenizer_query_set_lexicon(ctx, query, args[0]);
grn_tokenizer_query_ensure_have_tokenized_delimiter(ctx, query);
GRN_API_RETURN(query);
}
}
static mrb_value
mrb_grn_bulk_get_value(mrb_state *mrb, mrb_value self)
{
grn_obj *bulk;
mrb_value mrb_value_;
bulk = DATA_PTR(self);
switch (bulk->header.domain) {
case GRN_DB_UINT32 :
{
int64_t value;
value = GRN_UINT32_VALUE(bulk);
if (!FIXABLE(value)) {
mrb_raisef(mrb, E_RANGE_ERROR,
"can't handle large number: <%S>: max: <%S>",
mrb_fixnum_value(value), /* TODO: This will cause overflow */
mrb_fixnum_value(MRB_INT_MAX));
}
mrb_value_ = mrb_fixnum_value(value);
}
break;
}
return mrb_value_;
}
void
test_uint64_to_uint32(void)
{
grn_obj_reinit(&context, &dest, GRN_DB_UINT32, 0);
cast_uint64(29292929);
cut_assert_equal_uint(29292929, GRN_UINT32_VALUE(&dest));
}
static grn_obj *
values_next(grn_ctx *ctx, grn_obj *value)
{
if (value->header.domain == GRN_JSON_LOAD_OPEN_BRACKET ||
value->header.domain == GRN_JSON_LOAD_OPEN_BRACE) {
value += GRN_UINT32_VALUE(value);
}
return value + 1;
}
static VALUE
rb_grn_bulk_to_ruby_object_by_range_id (grn_ctx *context, grn_obj *bulk,
grn_id range_id,
VALUE related_object, VALUE *rb_value)
{
grn_bool success = GRN_TRUE;
switch (range_id) {
case GRN_DB_VOID:
*rb_value = rb_str_new(GRN_TEXT_VALUE(bulk), GRN_TEXT_LEN(bulk));
break;
case GRN_DB_BOOL:
*rb_value = GRN_BOOL_VALUE(bulk) ? Qtrue : Qfalse;
break;
case GRN_DB_INT32:
*rb_value = INT2NUM(GRN_INT32_VALUE(bulk));
break;
case GRN_DB_UINT32:
*rb_value = UINT2NUM(GRN_UINT32_VALUE(bulk));
break;
case GRN_DB_INT64:
*rb_value = LL2NUM(GRN_INT64_VALUE(bulk));
break;
case GRN_DB_UINT64:
*rb_value = ULL2NUM(GRN_UINT64_VALUE(bulk));
break;
case GRN_DB_FLOAT:
*rb_value = rb_float_new(GRN_FLOAT_VALUE(bulk));
break;
case GRN_DB_TIME:
{
int64_t time_value, sec, usec;
time_value = GRN_TIME_VALUE(bulk);
GRN_TIME_UNPACK(time_value, sec, usec);
*rb_value = rb_funcall(rb_cTime, rb_intern("at"), 2,
LL2NUM(sec), LL2NUM(usec));
}
break;
case GRN_DB_SHORT_TEXT:
case GRN_DB_TEXT:
case GRN_DB_LONG_TEXT:
*rb_value = rb_grn_context_rb_string_new(context,
GRN_TEXT_VALUE(bulk),
GRN_TEXT_LEN(bulk));
break;
default:
success = GRN_FALSE;
break;
}
return success;
}
static void
output_tokens(grn_ctx *ctx, grn_obj *tokens, grn_obj *lexicon, grn_obj *index_column)
{
int i, n_tokens, n_elements;
grn_obj estimated_size;
n_tokens = GRN_BULK_VSIZE(tokens) / sizeof(tokenize_token);
n_elements = 3;
if (index_column) {
n_elements++;
GRN_UINT32_INIT(&estimated_size, 0);
}
grn_ctx_output_array_open(ctx, "TOKENS", n_tokens);
for (i = 0; i < n_tokens; i++) {
tokenize_token *token;
char value[GRN_TABLE_MAX_KEY_SIZE];
unsigned int value_size;
token = ((tokenize_token *)(GRN_BULK_HEAD(tokens))) + i;
grn_ctx_output_map_open(ctx, "TOKEN", n_elements);
grn_ctx_output_cstr(ctx, "value");
value_size = grn_table_get_key(ctx, lexicon, token->id,
value, GRN_TABLE_MAX_KEY_SIZE);
grn_ctx_output_str(ctx, value, value_size);
grn_ctx_output_cstr(ctx, "position");
grn_ctx_output_int32(ctx, token->position);
grn_ctx_output_cstr(ctx, "force_prefix");
grn_ctx_output_bool(ctx, token->force_prefix);
if (index_column) {
GRN_BULK_REWIND(&estimated_size);
grn_obj_get_value(ctx, index_column, token->id, &estimated_size);
grn_ctx_output_cstr(ctx, "estimated_size");
grn_ctx_output_int64(ctx, GRN_UINT32_VALUE(&estimated_size));
}
grn_ctx_output_map_close(ctx);
}
if (index_column) {
GRN_OBJ_FIN(ctx, &estimated_size);
}
grn_ctx_output_array_close(ctx);
}
static void
add_weight_vector(grn_ctx *ctx,
grn_obj *column,
grn_obj *value,
grn_obj *vector)
{
unsigned int i, n;
grn_obj weight_buffer;
n = GRN_UINT32_VALUE(value);
GRN_UINT32_INIT(&weight_buffer, 0);
for (i = 0; i < n; i += 2) {
grn_rc rc;
grn_obj *key, *weight;
key = value + 1 + i;
weight = key + 1;
GRN_BULK_REWIND(&weight_buffer);
rc = grn_obj_cast(ctx, weight, &weight_buffer, GRN_TRUE);
if (rc != GRN_SUCCESS) {
grn_obj *range;
range = grn_ctx_at(ctx, weight_buffer.header.domain);
ERR_CAST(column, range, weight);
grn_obj_unlink(ctx, range);
break;
}
grn_vector_add_element(ctx,
vector,
GRN_BULK_HEAD(key),
GRN_BULK_VSIZE(key),
GRN_UINT32_VALUE(&weight_buffer),
key->header.domain);
}
GRN_OBJ_FIN(ctx, &weight_buffer);
}
grn_bool
grn_obj_is_true(grn_ctx *ctx, grn_obj *obj)
{
if (!obj) {
return GRN_FALSE;
}
switch (obj->header.type) {
case GRN_BULK :
switch (obj->header.domain) {
case GRN_DB_BOOL :
return GRN_BOOL_VALUE(obj);
break;
case GRN_DB_INT32 :
return GRN_INT32_VALUE(obj) != 0;
break;
case GRN_DB_UINT32 :
return GRN_UINT32_VALUE(obj) != 0;
break;
case GRN_DB_FLOAT : {
double float_value;
float_value = GRN_FLOAT_VALUE(obj);
return (float_value < -DBL_EPSILON ||
DBL_EPSILON < float_value);
break;
}
case GRN_DB_SHORT_TEXT :
case GRN_DB_TEXT :
case GRN_DB_LONG_TEXT :
return GRN_TEXT_LEN(obj) != 0;
break;
default :
return GRN_FALSE;
break;
}
break;
case GRN_VECTOR :
return GRN_TRUE;
break;
default :
return GRN_FALSE;
break;
}
}
void
test_aton(gconstpointer data)
{
const gchar *input, *input_end, *rest;
grn_builtin_type type;
grn_rc rc;
type = gcut_data_get_int(data, "type");
input = gcut_data_get_string(data, "input");
input_end = strchr(input, '\0');
rc = grn_aton(&context, input, input_end, &rest, &buffer);
grn_test_assert(rc);
cut_assert_equal_string(input_end, rest);
cut_assert_equal_int(type, buffer.header.domain);
switch (type) {
case GRN_DB_INT32 :
cut_assert_equal_int(gcut_data_get_int(data, "expected"),
GRN_INT32_VALUE(&buffer));
break;
case GRN_DB_UINT32 :
cut_assert_equal_uint(gcut_data_get_uint(data, "expected"),
GRN_UINT32_VALUE(&buffer));
break;
case GRN_DB_INT64 :
gcut_assert_equal_int64(gcut_data_get_int64(data, "expected"),
GRN_INT64_VALUE(&buffer));
break;
case GRN_DB_UINT64 :
gcut_assert_equal_uint64(gcut_data_get_uint64(data, "expected"),
GRN_UINT64_VALUE(&buffer));
break;
case GRN_DB_FLOAT :
cut_assert_equal_double(gcut_data_get_double(data, "expected"),
0.000001,
GRN_FLOAT_VALUE(&buffer));
break;
default :
cut_error("unknown type: %d", type);
break;
}
}
/* TODO: support caching for the same parameter. */
static grn_obj *
func_snippet(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)
{
grn_obj *snippets = NULL;
#define N_REQUIRED_ARGS 1
#define KEYWORD_SET_SIZE 3
if (nargs > N_REQUIRED_ARGS) {
grn_obj *text = args[0];
grn_obj *end_arg = args[nargs - 1];
grn_obj *snip = NULL;
unsigned int width = 200;
unsigned int max_n_results = 3;
grn_snip_mapping *mapping = NULL;
int flags = GRN_SNIP_SKIP_LEADING_SPACES;
const char *prefix = NULL;
int prefix_length = 0;
const char *suffix = NULL;
int suffix_length = 0;
const char *normalizer_name = NULL;
int normalizer_name_length = 0;
const char *default_open_tag = NULL;
int default_open_tag_length = 0;
const char *default_close_tag = NULL;
int default_close_tag_length = 0;
int n_args_without_option = nargs;
if (end_arg->header.type == GRN_TABLE_HASH_KEY) {
grn_obj *options = end_arg;
grn_hash_cursor *cursor;
void *key;
int key_size;
grn_obj *value;
n_args_without_option--;
cursor = grn_hash_cursor_open(ctx, (grn_hash *)options,
NULL, 0, NULL, 0,
0, -1, 0);
if (!cursor) {
GRN_PLUGIN_ERROR(ctx, GRN_NO_MEMORY_AVAILABLE,
"snippet(): couldn't open cursor");
goto exit;
}
while (grn_hash_cursor_next(ctx, cursor) != GRN_ID_NIL) {
grn_hash_cursor_get_key_value(ctx, cursor,
&key, &key_size,
(void **)&value);
if (key_size == 5 && !memcmp(key, "width", 5)) {
width = GRN_UINT32_VALUE(value);
} else if (key_size == 13 && !memcmp(key, "max_n_results", 13)) {
max_n_results = GRN_UINT32_VALUE(value);
} else if (key_size == 19 && !memcmp(key, "skip_leading_spaces", 19)) {
if (GRN_BOOL_VALUE(value) == GRN_FALSE) {
flags &= ~GRN_SNIP_SKIP_LEADING_SPACES;
}
} else if (key_size == 11 && !memcmp(key, "html_escape", 11)) {
if (GRN_BOOL_VALUE(value)) {
mapping = GRN_SNIP_MAPPING_HTML_ESCAPE;
}
} else if (key_size == 6 && !memcmp(key, "prefix", 6)) {
prefix = GRN_TEXT_VALUE(value);
prefix_length = GRN_TEXT_LEN(value);
} else if (key_size == 6 && !memcmp(key, "suffix", 6)) {
suffix = GRN_TEXT_VALUE(value);
suffix_length = GRN_TEXT_LEN(value);
} else if (key_size == 10 && !memcmp(key, "normalizer", 10)) {
normalizer_name = GRN_TEXT_VALUE(value);
normalizer_name_length = GRN_TEXT_LEN(value);
} else if (key_size == 16 && !memcmp(key, "default_open_tag", 16)) {
default_open_tag = GRN_TEXT_VALUE(value);
default_open_tag_length = GRN_TEXT_LEN(value);
} else if (key_size == 17 && !memcmp(key, "default_close_tag", 17)) {
default_close_tag = GRN_TEXT_VALUE(value);
default_close_tag_length = GRN_TEXT_LEN(value);
} else {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"invalid option name: <%.*s>",
key_size, (char *)key);
grn_hash_cursor_close(ctx, cursor);
goto exit;
}
}
grn_hash_cursor_close(ctx, cursor);
}
snip = grn_snip_open(ctx, flags, width, max_n_results,
default_open_tag, default_open_tag_length,
default_close_tag, default_close_tag_length, mapping);
if (snip) {
grn_rc rc;
unsigned int i;
if (!normalizer_name) {
grn_snip_set_normalizer(ctx, snip, GRN_NORMALIZER_AUTO);
} else if (normalizer_name_length > 0) {
grn_obj *normalizer;
normalizer = grn_ctx_get(ctx, normalizer_name, normalizer_name_length);
if (!grn_obj_is_normalizer_proc(ctx, normalizer)) {
grn_obj inspected;
GRN_TEXT_INIT(&inspected, 0);
grn_inspect(ctx, &inspected, normalizer);
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"snippet(): not normalizer: <%.*s>",
(int)GRN_TEXT_LEN(&inspected),
GRN_TEXT_VALUE(&inspected));
GRN_OBJ_FIN(ctx, &inspected);
grn_obj_unlink(ctx, normalizer);
goto exit;
}
grn_snip_set_normalizer(ctx, snip, normalizer);
grn_obj_unlink(ctx, normalizer);
}
if (default_open_tag_length == 0 && default_close_tag_length == 0) {
unsigned int n_keyword_sets =
(n_args_without_option - N_REQUIRED_ARGS) / KEYWORD_SET_SIZE;
grn_obj **keyword_set_args = args + N_REQUIRED_ARGS;
for (i = 0; i < n_keyword_sets; i++) {
rc = grn_snip_add_cond(ctx, snip,
GRN_TEXT_VALUE(keyword_set_args[i * KEYWORD_SET_SIZE]),
GRN_TEXT_LEN(keyword_set_args[i * KEYWORD_SET_SIZE]),
GRN_TEXT_VALUE(keyword_set_args[i * KEYWORD_SET_SIZE + 1]),
GRN_TEXT_LEN(keyword_set_args[i * KEYWORD_SET_SIZE + 1]),
GRN_TEXT_VALUE(keyword_set_args[i * KEYWORD_SET_SIZE + 2]),
GRN_TEXT_LEN(keyword_set_args[i * KEYWORD_SET_SIZE + 2]));
}
} else {
unsigned int n_keywords = n_args_without_option - N_REQUIRED_ARGS;
grn_obj **keyword_args = args + N_REQUIRED_ARGS;
for (i = 0; i < n_keywords; i++) {
rc = grn_snip_add_cond(ctx, snip,
GRN_TEXT_VALUE(keyword_args[i]),
GRN_TEXT_LEN(keyword_args[i]),
NULL, 0,
NULL, 0);
}
}
snippets = snippet_exec(ctx, snip, text, user_data,
prefix, prefix_length,
suffix, suffix_length);
}
}
#undef KEYWORD_SET_SIZE
#undef N_REQUIRED_ARGS
exit :
if (!snippets) {
snippets = grn_plugin_proc_alloc(ctx, user_data, GRN_DB_VOID, 0);
}
return snippets;
}
static VALUE
rb_grn_bulk_to_ruby_object_by_range_type (grn_ctx *context, grn_obj *bulk,
grn_obj *range, grn_id range_id,
VALUE related_object, VALUE *rb_value)
{
grn_bool success = GRN_TRUE;
if (!range && range_id != GRN_ID_NIL) {
range = grn_ctx_at(context, range_id);
}
if (!range)
return GRN_FALSE;
switch (range->header.type) {
case GRN_TABLE_HASH_KEY:
case GRN_TABLE_PAT_KEY:
case GRN_TABLE_DAT_KEY:
case GRN_TABLE_NO_KEY: {
grn_id id;
id = *((grn_id *)GRN_BULK_HEAD(bulk));
if (id == GRN_ID_NIL) {
*rb_value = Qnil;
} else {
VALUE rb_range;
rb_range = GRNOBJECT2RVAL(Qnil, context, range, GRN_FALSE);
*rb_value = rb_grn_record_new(rb_range, id, Qnil);
}
break;
}
case GRN_TYPE:
if (range->header.flags & GRN_OBJ_KEY_VAR_SIZE) {
*rb_value = rb_grn_context_rb_string_new(context,
GRN_BULK_HEAD(bulk),
GRN_BULK_VSIZE(bulk));
} else {
switch (range->header.flags & GRN_OBJ_KEY_MASK) {
case GRN_OBJ_KEY_UINT:
*rb_value = INT2NUM(GRN_UINT32_VALUE(bulk));
break;
case GRN_OBJ_KEY_INT:
*rb_value = INT2NUM(GRN_INT32_VALUE(bulk));
break;
case GRN_OBJ_KEY_FLOAT:
*rb_value = rb_float_new(GRN_FLOAT_VALUE(bulk));
break;
default:
success = GRN_FALSE;
}
break;
}
break;
default:
success = GRN_FALSE;
break;
}
return success;
}
mrb_value
grn_mrb_value_from_bulk(mrb_state *mrb, grn_obj *bulk)
{
mrb_value mrb_value_;
grn_ctx *ctx = (grn_ctx *)mrb->ud;
switch (bulk->header.domain) {
case GRN_DB_INT32 :
{
int32_t value;
value = GRN_INT32_VALUE(bulk);
mrb_value_ = mrb_fixnum_value(value);
}
break;
case GRN_DB_UINT32 :
{
int64_t value;
value = GRN_UINT32_VALUE(bulk);
if (!FIXABLE(value)) {
mrb_raisef(mrb, E_RANGE_ERROR,
"can't handle large number: <%S>: max: <%S>",
mrb_fixnum_value(value), /* TODO: This will cause overflow */
mrb_fixnum_value(MRB_INT_MAX));
}
mrb_value_ = mrb_fixnum_value(value);
}
break;
case GRN_DB_TIME :
{
int64_t value;
int32_t sec;
int32_t usec;
value = GRN_TIME_VALUE(bulk);
GRN_TIME_UNPACK(value, sec, usec);
mrb_value_ = mrb_funcall(mrb,
mrb_obj_value(ctx->impl->mrb.builtin.time_class),
"at",
2,
mrb_fixnum_value(sec),
mrb_fixnum_value(usec));
}
break;
case GRN_DB_SHORT_TEXT :
case GRN_DB_TEXT :
case GRN_DB_LONG_TEXT :
mrb_value_ = mrb_str_new(mrb,
GRN_TEXT_VALUE(bulk),
GRN_TEXT_LEN(bulk));
break;
default :
{
grn_obj *domain;
grn_bool is_record = GRN_FALSE;
domain = grn_ctx_at(ctx, bulk->header.domain);
if (domain) {
switch (domain->header.type) {
case GRN_TABLE_HASH_KEY :
case GRN_TABLE_PAT_KEY :
case GRN_TABLE_DAT_KEY :
case GRN_TABLE_NO_KEY :
is_record = GRN_TRUE;
break;
default :
break;
}
}
if (is_record) {
mrb_value_ = mrb_fixnum_value(GRN_RECORD_VALUE(bulk));
grn_obj_unlink(ctx, domain);
} else {
#define MESSAGE_SIZE 4096
char message[MESSAGE_SIZE];
char domain_name[GRN_TABLE_MAX_KEY_SIZE];
int domain_name_size;
if (domain) {
domain_name_size = grn_obj_name(ctx, domain,
domain_name, GRN_TABLE_MAX_KEY_SIZE);
grn_obj_unlink(ctx, domain);
} else {
grn_strcpy(domain_name, GRN_TABLE_MAX_KEY_SIZE, "unknown");
domain_name_size = strlen(domain_name);
}
grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE,
"unsupported bulk value type: <%d>(%.*s)",
bulk->header.domain,
domain_name_size,
domain_name);
mrb_raise(mrb, E_RANGE_ERROR, message);
}
#undef MESSAGE_SIZE
}
break;
}
return mrb_value_;
}
grn_id
grn_token_next(grn_ctx *ctx, grn_token *token)
{
int status;
grn_id tid = GRN_ID_NIL;
grn_obj *table = token->table;
grn_obj *tokenizer = token->tokenizer;
while (token->status != grn_token_done) {
if (tokenizer) {
grn_obj *curr_, *stat_;
((grn_proc *)tokenizer)->funcs[PROC_NEXT](ctx, 1, &table, &token->pctx.user_data);
stat_ = grn_ctx_pop(ctx);
curr_ = grn_ctx_pop(ctx);
token->curr = GRN_TEXT_VALUE(curr_);
token->curr_size = GRN_TEXT_LEN(curr_);
status = GRN_UINT32_VALUE(stat_);
token->status = ((status & GRN_TOKEN_LAST) ||
(!token->add && (status & GRN_TOKEN_REACH_END)))
? grn_token_done : grn_token_doing;
token->force_prefix = 0;
if (status & GRN_TOKEN_UNMATURED) {
if (status & GRN_TOKEN_OVERLAP) {
if (!token->add) { token->pos++; continue; }
} else {
if (status & GRN_TOKEN_LAST) { token->force_prefix = 1; }
}
}
} else {
token->curr = token->orig;
token->curr_size = token->orig_blen;
token->status = grn_token_done;
}
if (token->add) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
if (grn_io_lock(ctx, ((grn_pat *)table)->io, 10000000)) {
tid = GRN_ID_NIL;
} else {
tid = grn_pat_add(ctx, (grn_pat *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_pat *)table)->io);
}
break;
case GRN_TABLE_HASH_KEY :
if (grn_io_lock(ctx, ((grn_hash *)table)->io, 10000000)) {
tid = GRN_ID_NIL;
} else {
tid = grn_hash_add(ctx, (grn_hash *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_hash *)table)->io);
}
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
} else {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
tid = grn_pat_get(ctx, (grn_pat *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_HASH_KEY :
tid = grn_hash_get(ctx, (grn_hash *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
}
if (tid == GRN_ID_NIL && token->status != grn_token_done) {
token->status = grn_token_not_found;
}
token->pos++;
break;
}
return tid;
}
static grn_obj *
command_tag_synonym(grn_ctx *ctx, GNUC_UNUSED int nargs, GNUC_UNUSED grn_obj **args,
GNUC_UNUSED grn_user_data *user_data)
{
GNUC_UNUSED grn_obj *flags = grn_ctx_pop(ctx);
grn_obj *newvalue = grn_ctx_pop(ctx);
grn_obj *oldvalue = grn_ctx_pop(ctx);
GNUC_UNUSED grn_obj *id = grn_ctx_pop(ctx);
grn_obj buf;
grn_obj record;
grn_obj *domain;
grn_obj *table;
grn_obj *column;
int i,n;
if (GRN_BULK_VSIZE(newvalue) == 0 || GRN_INT32_VALUE(flags) == 0) {
return NULL;
}
table = grn_ctx_at(ctx, oldvalue->header.domain);
if (table && !is_table(table)) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_WARNING,
"[tag-synonym] "
"hooked column must be reference type");
return NULL;
}
column = grn_obj_column(ctx,
table,
SYNONYM_COLUMN_NAME,
SYNONYM_COLUMN_NAME_LEN);
if (!column) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_WARNING,
"[tag-synonym] "
"couldn't open synonym column");
return NULL;
}
GRN_TEXT_INIT(&buf, 0);
domain = grn_ctx_at(ctx, newvalue->header.domain);
if (domain && is_string(domain)) {
GRN_RECORD_INIT(&record, GRN_OBJ_VECTOR, oldvalue->header.domain);
grn_table_tokenize(ctx, table, GRN_TEXT_VALUE(newvalue), GRN_TEXT_LEN(newvalue), &record, GRN_TRUE);
} else if (newvalue->header.type == GRN_UVECTOR) {
record = *newvalue;
}
if (is_string(domain) || newvalue->header.type == GRN_UVECTOR) {
grn_obj value;
GRN_RECORD_INIT(newvalue, GRN_OBJ_VECTOR, oldvalue->header.domain);
GRN_UINT32_INIT(&value, 0);
n = grn_vector_size(ctx, &record);
for (i = 0; i < n; i++) {
grn_id tid;
tid = grn_uvector_get_element(ctx, &record, i, NULL);
GRN_BULK_REWIND(&value);
grn_obj_get_value(ctx, column, tid, &value);
if (GRN_UINT32_VALUE(&value)) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_INFO,
"[tag-synonym] "
"changed: tid %d -> %d", tid, GRN_UINT32_VALUE(&value));
tid = GRN_UINT32_VALUE(&value);
}
grn_uvector_add_element(ctx, newvalue, tid, 0);
}
grn_obj_unlink(ctx, &value);
} else {
grn_id tid;
grn_obj value;
tid = GRN_RECORD_VALUE(newvalue);
GRN_UINT32_INIT(&value, 0);
grn_obj_get_value(ctx, column, tid, &value);
if (GRN_UINT32_VALUE(&value)) {
GRN_PLUGIN_LOG(ctx, GRN_LOG_INFO,
"[tag-synonym] "
"changed: tid %d -> %d", tid, GRN_UINT32_VALUE(&value));
tid = GRN_UINT32_VALUE(&value);
GRN_BULK_REWIND(newvalue);
GRN_RECORD_SET(ctx, newvalue, tid);
}
grn_obj_unlink(ctx, &value);
}
grn_obj_unlink(ctx, &buf);
return NULL;
}
static grn_rc
selector_fuzzy_search(grn_ctx *ctx, grn_obj *table, grn_obj *index,
int nargs, grn_obj **args,
grn_obj *res, grn_operator op)
{
grn_rc rc = GRN_SUCCESS;
grn_obj *target = NULL;
grn_obj *obj;
grn_obj *query;
uint32_t max_distance = 1;
uint32_t prefix_length = 0;
uint32_t prefix_match_size = 0;
uint32_t max_expansion = 0;
int flags = 0;
grn_bool use_sequential_search = GRN_FALSE;
if ((nargs - 1) < 2) {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"fuzzy_search(): wrong number of arguments (%d ...)",
nargs - 1);
rc = ctx->rc;
goto exit;
}
obj = args[1];
query = args[2];
if (nargs == 4) {
grn_obj *options = args[3];
grn_hash_cursor *cursor;
void *key;
grn_obj *value;
int key_size;
if (options->header.type != GRN_TABLE_HASH_KEY) {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"fuzzy_search(): "
"3rd argument must be object literal: <%.*s>",
(int)GRN_TEXT_LEN(options),
GRN_TEXT_VALUE(options));
goto exit;
}
cursor = grn_hash_cursor_open(ctx, (grn_hash *)options,
NULL, 0, NULL, 0,
0, -1, 0);
if (!cursor) {
GRN_PLUGIN_ERROR(ctx, GRN_NO_MEMORY_AVAILABLE,
"fuzzy_search(): couldn't open cursor");
goto exit;
}
while (grn_hash_cursor_next(ctx, cursor) != GRN_ID_NIL) {
grn_hash_cursor_get_key_value(ctx, cursor, &key, &key_size,
(void **)&value);
if (key_size == 12 && !memcmp(key, "max_distance", 12)) {
max_distance = GRN_UINT32_VALUE(value);
} else if (key_size == 13 && !memcmp(key, "prefix_length", 13)) {
prefix_length = GRN_UINT32_VALUE(value);
} else if (key_size == 13 && !memcmp(key, "max_expansion", 13)) {
max_expansion = GRN_UINT32_VALUE(value);
} else if (key_size == 18 && !memcmp(key, "with_transposition", 18)) {
if (GRN_BOOL_VALUE(value)) {
flags |= GRN_TABLE_FUZZY_SEARCH_WITH_TRANSPOSITION;
}
} else {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"invalid option name: <%.*s>",
key_size, (char *)key);
grn_hash_cursor_close(ctx, cursor);
goto exit;
}
}
grn_hash_cursor_close(ctx, cursor);
}
if (index) {
target = index;
} else {
if (obj->header.type == GRN_COLUMN_INDEX) {
target = obj;
} else {
grn_column_index(ctx, obj, GRN_OP_FUZZY, &target, 1, NULL);
}
}
if (target) {
grn_obj *lexicon;
use_sequential_search = GRN_TRUE;
lexicon = grn_ctx_at(ctx, target->header.domain);
if (lexicon) {
if (lexicon->header.type == GRN_TABLE_PAT_KEY) {
use_sequential_search = GRN_FALSE;
}
grn_obj_unlink(ctx, lexicon);
}
} else {
if (grn_obj_is_key_accessor(ctx, obj) &&
table->header.type == GRN_TABLE_PAT_KEY) {
target = table;
} else {
use_sequential_search = GRN_TRUE;
}
}
if (prefix_length) {
const char *s = GRN_TEXT_VALUE(query);
const char *e = GRN_BULK_CURR(query);
const char *p;
unsigned int cl = 0;
unsigned int length = 0;
for (p = s; p < e && (cl = grn_charlen(ctx, p, e)); p += cl) {
length++;
if (length > prefix_length) {
break;
}
}
prefix_match_size = p - s;
}
if (use_sequential_search) {
rc = sequential_fuzzy_search(ctx, table, obj, query,
max_distance, prefix_match_size,
max_expansion, flags, res, op);
goto exit;
}
if (!target) {
grn_obj inspected;
GRN_TEXT_INIT(&inspected, 0);
grn_inspect(ctx, &inspected, target);
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"fuzzy_search(): "
"column must be COLUMN_INDEX or TABLE_PAT_KEY: <%.*s>",
(int)GRN_TEXT_LEN(&inspected),
GRN_TEXT_VALUE(&inspected));
rc = ctx->rc;
GRN_OBJ_FIN(ctx, &inspected);
} else {
grn_search_optarg options = {0};
options.mode = GRN_OP_FUZZY;
options.fuzzy.prefix_match_size = prefix_match_size;
options.fuzzy.max_distance = max_distance;
options.fuzzy.max_expansion = max_expansion;
options.fuzzy.flags = flags;
grn_obj_search(ctx, target, query, res, op, &options);
}
exit :
return rc;
}
grn_tokenizer_query *
grn_tokenizer_query_open(grn_ctx *ctx, int num_args, grn_obj **args,
unsigned int normalize_flags)
{
grn_obj *flags = grn_ctx_pop(ctx);
grn_obj *query_str = grn_ctx_pop(ctx);
grn_obj *tokenize_mode = grn_ctx_pop(ctx);
if (query_str == NULL) {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, "missing argument");
return NULL;
}
if ((num_args < 1) || (args == NULL) || (args[0] == NULL)) {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, "invalid NULL pointer");
return NULL;
}
{
grn_tokenizer_query * const query =
GRN_PLUGIN_MALLOC(ctx, sizeof(grn_tokenizer_query));
if (query == NULL) {
return NULL;
}
query->normalized_query = NULL;
query->query_buf = NULL;
if (flags) {
query->flags = GRN_UINT32_VALUE(flags);
} else {
query->flags = 0;
}
if (tokenize_mode) {
query->tokenize_mode = GRN_UINT32_VALUE(tokenize_mode);
} else {
query->tokenize_mode = GRN_TOKENIZE_ADD;
}
query->token_mode = query->tokenize_mode;
{
grn_obj * const table = args[0];
grn_obj_flags table_flags;
grn_encoding table_encoding;
unsigned int query_length = GRN_TEXT_LEN(query_str);
char *query_buf = (char *)GRN_PLUGIN_MALLOC(ctx, query_length + 1);
grn_obj *normalizer = NULL;
if (query_buf == NULL) {
GRN_PLUGIN_FREE(ctx, query);
GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR,
"[tokenizer] failed to duplicate query");
return NULL;
}
grn_table_get_info(ctx, table, &table_flags, &table_encoding, NULL,
&normalizer, NULL);
{
grn_obj *normalized_query;
if (table_flags & GRN_OBJ_KEY_NORMALIZE) {
normalizer = GRN_NORMALIZER_AUTO;
}
normalized_query = grn_string_open_(ctx,
GRN_TEXT_VALUE(query_str),
GRN_TEXT_LEN(query_str),
normalizer,
normalize_flags,
table_encoding);
if (!normalized_query) {
GRN_PLUGIN_FREE(ctx, query_buf);
GRN_PLUGIN_FREE(ctx, query);
GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR,
"[tokenizer] failed to open normalized string");
return NULL;
}
query->normalized_query = normalized_query;
grn_memcpy(query_buf, GRN_TEXT_VALUE(query_str), query_length);
query_buf[query_length] = '\0';
query->query_buf = query_buf;
query->ptr = query_buf;
query->length = query_length;
}
query->encoding = table_encoding;
if (query->flags & GRN_TOKEN_CURSOR_ENABLE_TOKENIZED_DELIMITER) {
const char *normalized_string;
unsigned int normalized_string_length;
grn_string_get_normalized(ctx,
query->normalized_query,
&normalized_string,
&normalized_string_length,
NULL);
query->have_tokenized_delimiter =
grn_tokenizer_have_tokenized_delimiter(ctx,
normalized_string,
normalized_string_length,
query->encoding);
} else {
query->have_tokenized_delimiter = GRN_FALSE;
}
}
return query;
}
}
void
test_mroonga_index_score(void)
{
grn_obj *t1,*c1,*lc,*ft;
grn_obj buff;
grn_id r1,r2,r3,r4;
remove_tmp_directory();
g_mkdir_with_parents(tmp_directory,0700);
g_chdir(tmp_directory);
g_mkdir_with_parents("mrn",0700);
db = grn_db_create(context,"mroonga.grn",NULL);
cut_assert_not_null(db);
/* actual table */
t1 = grn_table_create(context,"t1",2,"mrn/t1.grn",
GRN_OBJ_TABLE_NO_KEY|GRN_OBJ_PERSISTENT,NULL,0);
cut_assert_not_null(t1);
/* lexicon table */
lc = grn_table_create(context,"lc",2,"mrn/lc.grn",
GRN_OBJ_TABLE_PAT_KEY|GRN_OBJ_PERSISTENT,
grn_ctx_at(context, GRN_DB_SHORT_TEXT), 0);
cut_assert_not_null(lc);
grn_test_assert(grn_obj_set_info(context, lc, GRN_INFO_DEFAULT_TOKENIZER,
grn_ctx_at(context, GRN_DB_BIGRAM)));
/* actual column */
c1 = grn_column_create(context,t1,"c1",2,"mrn/t1.c1.grn",
GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT,
grn_ctx_at(context, GRN_DB_TEXT));
cut_assert_not_null(c1);
/* fulltext index */
ft = grn_column_create(context,lc,"ft",2,"mrn/lc.ft.grn",
GRN_OBJ_COLUMN_INDEX|GRN_OBJ_PERSISTENT,t1);
cut_assert_not_null(ft);
GRN_TEXT_INIT(&buff,0);
/* link between actual column and fulltext index */
GRN_UINT32_SET(context, &buff, grn_obj_id(context, c1));
grn_obj_set_info(context, ft, GRN_INFO_SOURCE, &buff); /* need to use grn_id */
/* insert row */
r1 = grn_table_add(context, t1, NULL, 0, NULL);
cut_assert_equal_int(1,r1);
GRN_TEXT_SETS(context, &buff, "abcde");
grn_test_assert(grn_obj_set_value(context, c1, r1, &buff, GRN_OBJ_SET));
r2 = grn_table_add(context, t1, NULL, 0, NULL);
cut_assert_equal_int(2,r2);
GRN_TEXT_SETS(context, &buff, "fghij");
grn_test_assert(grn_obj_set_value(context, c1, r2, &buff, GRN_OBJ_SET));
r3 = grn_table_add(context, t1, NULL, 0, NULL);
cut_assert_equal_int(3,r3);
GRN_TEXT_SETS(context, &buff, "11 22 33");
grn_test_assert(grn_obj_set_value(context, c1, r3, &buff, GRN_OBJ_SET));
r4 = grn_table_add(context, t1, NULL, 0, NULL);
cut_assert_equal_int(4,r4);
GRN_TEXT_SETS(context, &buff, "44 22 55");
grn_test_assert(grn_obj_set_value(context, c1, r4, &buff, GRN_OBJ_SET));
/* confirm record are inserted in both column and index */
cut_assert_equal_int(4,grn_table_size(context,t1));
cut_assert_equal_int(23,grn_table_size(context,lc));
/* nlq search */
{
grn_id id, docid;
grn_obj *res;
grn_table_cursor *tc;
grn_obj score, *score_column;
res = grn_table_create(context, NULL, 0, NULL,
GRN_TABLE_HASH_KEY|GRN_OBJ_WITH_SUBREC, t1, 0);
GRN_UINT32_INIT(&score, 0);
GRN_BULK_REWIND(&buff);
GRN_TEXT_SETS(context, &buff, "hij");
grn_obj_search(context, ft, &buff, res, GRN_OP_OR, NULL);
cut_assert_equal_int(1, grn_table_size(context, res));
score_column = grn_obj_column(context, res, ".:score", 7);
tc = grn_table_cursor_open(context, res, NULL, 0, NULL, 0, 0, 0, 0);
while ((id = grn_table_cursor_next(context, tc))) {
GRN_BULK_REWIND(&buff);
grn_table_get_key(context, res, id, &docid, sizeof(grn_id));
cut_assert_equal_int(2, docid);
cut_assert_not_null(grn_obj_get_value(context, c1, docid, &buff));
cut_assert_equal_int(5 ,GRN_TEXT_LEN(&buff));
cut_assert_equal_substring("fghij", (char*) GRN_BULK_HEAD(&buff),GRN_TEXT_LEN(&buff));
grn_obj_get_value(context, score_column, id, &score);
cut_assert_equal_uint(1, GRN_UINT32_VALUE(&score));
}
grn_table_cursor_close(context, tc);
grn_obj_close(context, score_column);
grn_obj_close(context, res);
}
/* boolean search */
{
grn_id id, docid;
grn_obj *res;
grn_query *query;
grn_table_cursor *tc;
grn_obj score, *score_column;
const char *qstr = "+22 -55";
res = grn_table_create(context, NULL, 0, NULL,
GRN_TABLE_HASH_KEY|GRN_OBJ_WITH_SUBREC, t1, 0);
GRN_UINT32_INIT(&score, 0);
query = grn_query_open(context, qstr, strlen(qstr), GRN_OP_OR, 32);
grn_obj_search(context, ft, (grn_obj*) query, res, GRN_OP_OR, NULL);
cut_assert_equal_int(1, grn_table_size(context, res));
score_column = grn_obj_column(context, res, ".:score", 7);
tc = grn_table_cursor_open(context, res, NULL, 0, NULL, 0, 0, 0, 0);
while ((id = grn_table_cursor_next(context, tc))) {
GRN_BULK_REWIND(&buff);
grn_table_get_key(context, res, id, &docid, sizeof(grn_id));
cut_assert_equal_int(3, docid);
cut_assert_not_null(grn_obj_get_value(context, c1, docid, &buff));
cut_assert_equal_int(8 ,GRN_TEXT_LEN(&buff));
cut_assert_equal_substring("11 22 33", (char*) GRN_BULK_HEAD(&buff),GRN_TEXT_LEN(&buff));
grn_obj_get_value(context, score_column, id, &score);
cut_assert_equal_uint(5, GRN_UINT32_VALUE(&score));
}
grn_query_close(context, query);
grn_table_cursor_close(context ,tc);
grn_obj_close(context, score_column);
grn_obj_close(context, res);
}
grn_obj_close(context, &buff);
grn_obj_close(context, ft);
grn_obj_close(context, c1);
grn_obj_close(context, lc);
grn_obj_close(context, t1);
}
grn_id
grn_token_next(grn_ctx *ctx, grn_token *token)
{
int status;
grn_id tid = GRN_ID_NIL;
grn_obj *table = token->table;
grn_obj *tokenizer = token->tokenizer;
while (token->status != GRN_TOKEN_DONE) {
if (tokenizer) {
grn_obj *curr_, *stat_;
((grn_proc *)tokenizer)->funcs[PROC_NEXT](ctx, 1, &table, &token->pctx.user_data);
stat_ = grn_ctx_pop(ctx);
curr_ = grn_ctx_pop(ctx);
token->curr = (const unsigned char *)GRN_TEXT_VALUE(curr_);
token->curr_size = GRN_TEXT_LEN(curr_);
status = GRN_UINT32_VALUE(stat_);
token->status = ((status & GRN_TOKENIZER_TOKEN_LAST) ||
(token->mode == GRN_TOKEN_GET &&
(status & GRN_TOKENIZER_TOKEN_REACH_END)))
? GRN_TOKEN_DONE : GRN_TOKEN_DOING;
token->force_prefix = 0;
if (token->curr_size == 0) {
char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE];
int tokenizer_name_length;
tokenizer_name_length =
grn_obj_name(ctx, token->tokenizer,
tokenizer_name, GRN_TABLE_MAX_KEY_SIZE);
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore an empty token: <%.*s>: <%.*s>",
tokenizer_name_length, tokenizer_name,
token->orig_blen, token->orig);
continue;
}
if (token->curr_size > GRN_TABLE_MAX_KEY_SIZE) {
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore too long token. "
"Token must be less than or equal to %d: <%d>(<%.*s>)",
GRN_TABLE_MAX_KEY_SIZE,
token->curr_size,
token->curr_size, token->curr);
continue;
}
if (status & GRN_TOKENIZER_TOKEN_UNMATURED) {
if (status & GRN_TOKENIZER_TOKEN_OVERLAP) {
if (token->mode == GRN_TOKEN_GET) { token->pos++; continue; }
} else {
if (status & GRN_TOKENIZER_TOKEN_LAST) { token->force_prefix = 1; }
}
}
} else {
token->status = GRN_TOKEN_DONE;
}
if (token->mode == GRN_TOKEN_ADD) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
if (grn_io_lock(ctx, ((grn_pat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_pat_add(ctx, (grn_pat *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_pat *)table)->io);
}
break;
case GRN_TABLE_DAT_KEY :
if (grn_io_lock(ctx, ((grn_dat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_dat_add(ctx, (grn_dat *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_dat *)table)->io);
}
break;
case GRN_TABLE_HASH_KEY :
if (grn_io_lock(ctx, ((grn_hash *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_hash_add(ctx, (grn_hash *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_hash *)table)->io);
}
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
} else {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
tid = grn_pat_get(ctx, (grn_pat *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_DAT_KEY :
tid = grn_dat_get(ctx, (grn_dat *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_HASH_KEY :
tid = grn_hash_get(ctx, (grn_hash *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
}
if (tid == GRN_ID_NIL && token->status != GRN_TOKEN_DONE) {
token->status = GRN_TOKEN_NOT_FOUND;
}
token->pos++;
break;
}
return tid;
}
static void
set_vector(grn_ctx *ctx, grn_obj *column, grn_id id, grn_obj *vector)
{
int n = GRN_UINT32_VALUE(vector);
grn_obj buf, *v = vector + 1;
grn_id range_id;
grn_obj *range;
range_id = DB_OBJ(column)->range;
range = grn_ctx_at(ctx, range_id);
if (grn_obj_is_table(ctx, range)) {
GRN_RECORD_INIT(&buf, GRN_OBJ_VECTOR, range_id);
while (n--) {
grn_bool cast_failed = GRN_FALSE;
grn_obj record, *element = v;
if (range_id != element->header.domain) {
GRN_RECORD_INIT(&record, 0, range_id);
if (grn_obj_cast(ctx, element, &record, GRN_TRUE)) {
cast_failed = GRN_TRUE;
ERR_CAST(column, range, element);
}
element = &record;
}
if (!cast_failed) {
GRN_UINT32_PUT(ctx, &buf, GRN_RECORD_VALUE(element));
}
if (element == &record) { GRN_OBJ_FIN(ctx, element); }
v = values_next(ctx, v);
}
} else {
if (((struct _grn_type *)range)->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) {
GRN_TEXT_INIT(&buf, GRN_OBJ_VECTOR);
while (n--) {
switch (v->header.domain) {
case GRN_DB_TEXT :
{
grn_bool cast_failed = GRN_FALSE;
grn_obj casted_element, *element = v;
if (range_id != element->header.domain) {
GRN_OBJ_INIT(&casted_element, GRN_BULK, 0, range_id);
if (grn_obj_cast(ctx, element, &casted_element, GRN_TRUE)) {
cast_failed = GRN_TRUE;
ERR_CAST(column, range, element);
}
element = &casted_element;
}
if (!cast_failed) {
grn_vector_add_element(ctx, &buf,
GRN_TEXT_VALUE(element),
GRN_TEXT_LEN(element),
0,
element->header.domain);
}
if (element == &casted_element) { GRN_OBJ_FIN(ctx, element); }
break;
}
case GRN_JSON_LOAD_OPEN_BRACE :
add_weight_vector(ctx, column, v, &buf);
n -= GRN_UINT32_VALUE(v);
break;
default :
ERR(GRN_INVALID_ARGUMENT, "array must contain string or object");
break;
}
v = values_next(ctx, v);
}
} else {
grn_id value_size = ((grn_db_obj *)range)->range;
GRN_VALUE_FIX_SIZE_INIT(&buf, GRN_OBJ_VECTOR, range_id);
while (n--) {
grn_bool cast_failed = GRN_FALSE;
grn_obj casted_element, *element = v;
if (range_id != element->header.domain) {
GRN_OBJ_INIT(&casted_element, GRN_BULK, 0, range_id);
if (grn_obj_cast(ctx, element, &casted_element, GRN_TRUE)) {
cast_failed = GRN_TRUE;
ERR_CAST(column, range, element);
}
element = &casted_element;
}
if (!cast_failed) {
grn_bulk_write(ctx, &buf, GRN_TEXT_VALUE(element), value_size);
}
if (element == &casted_element) { GRN_OBJ_FIN(ctx, element); }
v = values_next(ctx, v);
}
}
}
grn_obj_set_value(ctx, column, id, &buf, GRN_OBJ_SET);
GRN_OBJ_FIN(ctx, &buf);
}
static grn_obj *
func_time_classify_raw(grn_ctx *ctx,
int n_args,
grn_obj **args,
grn_user_data *user_data,
const char *function_name,
grn_time_classify_unit unit)
{
grn_obj *time;
uint32_t interval_raw = 1;
grn_obj *classed_time;
grn_bool accept_interval = GRN_TRUE;
switch (unit) {
case GRN_TIME_CLASSIFY_UNIT_SECOND :
case GRN_TIME_CLASSIFY_UNIT_MINUTE :
case GRN_TIME_CLASSIFY_UNIT_HOUR :
accept_interval = GRN_TRUE;
break;
case GRN_TIME_CLASSIFY_UNIT_DAY :
case GRN_TIME_CLASSIFY_UNIT_WEEK :
accept_interval = GRN_FALSE;
break;
case GRN_TIME_CLASSIFY_UNIT_MONTH :
case GRN_TIME_CLASSIFY_UNIT_YEAR :
accept_interval = GRN_TRUE;
break;
}
if (accept_interval) {
if (!(n_args == 1 || n_args == 2)) {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"%s(): "
"wrong number of arguments (%d for 1..2)",
function_name,
n_args);
return NULL;
}
} else {
if (n_args != 1) {
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"%s(): "
"wrong number of arguments (%d for 1)",
function_name,
n_args);
return NULL;
}
}
time = args[0];
if (!(time->header.type == GRN_BULK &&
time->header.domain == GRN_DB_TIME)) {
grn_obj inspected;
GRN_TEXT_INIT(&inspected, 0);
grn_inspect(ctx, &inspected, time);
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"%s(): "
"the first argument must be a time: "
"<%.*s>",
function_name,
(int)GRN_TEXT_LEN(&inspected),
GRN_TEXT_VALUE(&inspected));
GRN_OBJ_FIN(ctx, &inspected);
return NULL;
}
if (n_args == 2) {
grn_obj *interval;
grn_obj casted_interval;
interval = args[1];
if (!(interval->header.type == GRN_BULK &&
grn_type_id_is_number_family(ctx, interval->header.domain))) {
grn_obj inspected;
GRN_TEXT_INIT(&inspected, 0);
grn_inspect(ctx, &inspected, interval);
GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
"%s(): "
"the second argument must be a number: "
"<%.*s>",
function_name,
(int)GRN_TEXT_LEN(&inspected),
GRN_TEXT_VALUE(&inspected));
GRN_OBJ_FIN(ctx, &inspected);
return NULL;
}
GRN_VALUE_FIX_SIZE_INIT(&casted_interval, 0, GRN_DB_UINT32);
grn_obj_cast(ctx, interval, &casted_interval, GRN_FALSE);
interval_raw = GRN_UINT32_VALUE(&casted_interval);
GRN_OBJ_FIN(ctx, &casted_interval);
}
{
int64_t time_raw;
struct tm tm;
int64_t classed_time_raw;
time_raw = GRN_TIME_VALUE(time);
if (!grn_time_to_tm(ctx, time_raw, &tm)) {
return NULL;
}
switch (unit) {
case GRN_TIME_CLASSIFY_UNIT_SECOND :
tm.tm_sec = (tm.tm_sec / interval_raw) * interval_raw;
break;
case GRN_TIME_CLASSIFY_UNIT_MINUTE :
tm.tm_min = (tm.tm_min / interval_raw) * interval_raw;
tm.tm_sec = 0;
break;
case GRN_TIME_CLASSIFY_UNIT_HOUR :
tm.tm_hour = (tm.tm_hour / interval_raw) * interval_raw;
tm.tm_min = 0;
tm.tm_sec = 0;
break;
case GRN_TIME_CLASSIFY_UNIT_DAY :
tm.tm_hour = 0;
tm.tm_min = 0;
tm.tm_sec = 0;
break;
case GRN_TIME_CLASSIFY_UNIT_WEEK :
if ((tm.tm_mday - tm.tm_wday) >= 0) {
tm.tm_mday -= tm.tm_wday;
} else {
int n_underflowed_mday = -(tm.tm_mday - tm.tm_wday);
int mday;
int max_mday = 31;
if (tm.tm_mon == 0) {
tm.tm_year--;
tm.tm_mon = 11;
} else {
tm.tm_mon--;
}
for (mday = max_mday; mday > n_underflowed_mday; mday--) {
int64_t unused;
tm.tm_mday = mday;
if (grn_time_from_tm(ctx, &unused, &tm)) {
break;
}
}
tm.tm_mday -= n_underflowed_mday;
}
tm.tm_hour = 0;
tm.tm_min = 0;
tm.tm_sec = 0;
break;
case GRN_TIME_CLASSIFY_UNIT_MONTH :
tm.tm_mon = (tm.tm_mon / interval_raw) * interval_raw;
tm.tm_mday = 1;
tm.tm_hour = 0;
tm.tm_min = 0;
tm.tm_sec = 0;
break;
case GRN_TIME_CLASSIFY_UNIT_YEAR :
tm.tm_year = (((1900 + tm.tm_year) / interval_raw) * interval_raw) - 1900;
tm.tm_mon = 0;
tm.tm_mday = 1;
tm.tm_hour = 0;
tm.tm_min = 0;
tm.tm_sec = 0;
break;
}
if (!grn_time_from_tm(ctx, &classed_time_raw, &tm)) {
return NULL;
}
classed_time = grn_plugin_proc_alloc(ctx,
user_data,
time->header.domain,
0);
if (!classed_time) {
return NULL;
}
GRN_TIME_SET(ctx, classed_time, classed_time_raw);
return classed_time;
}
}
void
grn_output_obj(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type,
grn_obj *obj, grn_obj_format *format)
{
grn_obj buf;
GRN_TEXT_INIT(&buf, 0);
switch (obj->header.type) {
case GRN_BULK :
switch (obj->header.domain) {
case GRN_DB_VOID :
grn_output_void(ctx, outbuf, output_type, GRN_BULK_HEAD(obj), GRN_BULK_VSIZE(obj));
break;
case GRN_DB_SHORT_TEXT :
case GRN_DB_TEXT :
case GRN_DB_LONG_TEXT :
grn_output_str(ctx, outbuf, output_type, GRN_BULK_HEAD(obj), GRN_BULK_VSIZE(obj));
break;
case GRN_DB_BOOL :
grn_output_bool(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT8_VALUE(obj) : 0);
break;
case GRN_DB_INT8 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT8_VALUE(obj) : 0);
break;
case GRN_DB_UINT8 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT8_VALUE(obj) : 0);
break;
case GRN_DB_INT16 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT16_VALUE(obj) : 0);
break;
case GRN_DB_UINT16 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT16_VALUE(obj) : 0);
break;
case GRN_DB_INT32 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT32_VALUE(obj) : 0);
break;
case GRN_DB_UINT32 :
grn_output_int64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT32_VALUE(obj) : 0);
break;
case GRN_DB_INT64 :
grn_output_int64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT64_VALUE(obj) : 0);
break;
case GRN_DB_UINT64 :
grn_output_uint64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT64_VALUE(obj) : 0);
break;
case GRN_DB_FLOAT :
grn_output_float(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_FLOAT_VALUE(obj) : 0);
break;
case GRN_DB_TIME :
grn_output_time(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT64_VALUE(obj) : 0);
break;
case GRN_DB_TOKYO_GEO_POINT :
case GRN_DB_WGS84_GEO_POINT :
grn_output_geo_point(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? (grn_geo_point *)GRN_BULK_HEAD(obj) : NULL);
break;
default :
if (format) {
int j;
int ncolumns = GRN_BULK_VSIZE(&format->columns)/sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", 2);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_open(ctx, outbuf, output_type, "HIT", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_text_atoj_o(ctx, outbuf, output_type, columns[j], obj);
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_obj *table = grn_ctx_at(ctx, obj->header.domain);
grn_id id = *((grn_id *)GRN_BULK_HEAD(obj));
if (table && table->header.type != GRN_TABLE_NO_KEY) {
grn_obj *accessor = grn_obj_column(ctx, table, "_key", 4);
if (accessor) {
grn_obj_get_value(ctx, accessor, id, &buf);
grn_obj_unlink(ctx, accessor);
}
grn_output_obj(ctx, outbuf, output_type, &buf, format);
} else {
grn_output_int64(ctx, outbuf, output_type, id);
}
}
break;
}
break;
case GRN_UVECTOR :
if (format) {
int i, j;
grn_id *v = (grn_id *)GRN_BULK_HEAD(obj), *ve = (grn_id *)GRN_BULK_CURR(obj);
int ncolumns = GRN_BULK_VSIZE(&format->columns) / sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
grn_output_array_open(ctx, outbuf, output_type, "RESULTSET", -1);
grn_output_array_open(ctx, outbuf, output_type, "NHITS", 1);
grn_text_itoa(ctx, outbuf, ve - v);
grn_output_array_close(ctx, outbuf, output_type);
if (v < ve) {
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", -1);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", -1);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
for (i = 0;; i++) {
grn_output_array_open(ctx, outbuf, output_type, "HITS", -1);
for (j = 0; j < ncolumns; j++) {
GRN_BULK_REWIND(&buf);
grn_obj_get_value(ctx, columns[j], *v, &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
v++;
if (v < ve) {
} else {
break;
}
}
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_obj *range = grn_ctx_at(ctx, obj->header.domain);
if (range && range->header.type == GRN_TYPE) {
int value_size = ((struct _grn_type *)range)->obj.range;
char *v = (char *)GRN_BULK_HEAD(obj),
*ve = (char *)GRN_BULK_CURR(obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
if (v < ve) {
for (;;) {
grn_obj value;
GRN_OBJ_INIT(&value, GRN_BULK, 0, obj->header.domain);
grn_bulk_write_from(ctx, &value, v, 0, value_size);
grn_output_obj(ctx, outbuf, output_type, &value, NULL);
v += value_size;
if (v < ve) {
} else {
break;
}
}
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_id *v = (grn_id *)GRN_BULK_HEAD(obj),
*ve = (grn_id *)GRN_BULK_CURR(obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", ve - v);
if (v < ve) {
grn_obj key;
GRN_OBJ_INIT(&key, GRN_BULK, 0, range->header.domain);
for (;;) {
if (range->header.type != GRN_TABLE_NO_KEY) {
grn_table_get_key2(ctx, range, *v, &key);
grn_output_obj(ctx, outbuf, output_type, &key, NULL);
GRN_BULK_REWIND(&key);
} else {
grn_obj id;
GRN_UINT32_INIT(&id, 0);
GRN_UINT32_SET(ctx, &id, *v);
grn_output_obj(ctx, outbuf, output_type, &id, NULL);
GRN_OBJ_FIN(ctx, &id);
}
v++;
if (v < ve) {
} else {
break;
}
}
GRN_OBJ_FIN(ctx, &key);
}
grn_output_array_close(ctx, outbuf, output_type);
}
}
break;
case GRN_VECTOR :
if (obj->header.domain == GRN_DB_VOID) {
ERR(GRN_INVALID_ARGUMENT, "invalid obj->header.domain");
}
if (format) {
ERR(GRN_FUNCTION_NOT_IMPLEMENTED,
"cannot print GRN_VECTOR using grn_obj_format");
} else {
unsigned int i, n;
grn_obj value;
GRN_VOID_INIT(&value);
n = grn_vector_size(ctx, obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
for (i = 0; i < n; i++) {
const char *_value;
unsigned int weight, length;
grn_id domain;
length = grn_vector_get_element(ctx, obj, i,
&_value, &weight, &domain);
if (domain != GRN_DB_VOID) {
grn_obj_reinit(ctx, &value, domain, 0);
} else {
grn_obj_reinit(ctx, &value, obj->header.domain, 0);
}
grn_bulk_write(ctx, &value, _value, length);
grn_output_obj(ctx, outbuf, output_type, &value, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
GRN_OBJ_FIN(ctx, &value);
}
break;
case GRN_PVECTOR :
if (format) {
ERR(GRN_FUNCTION_NOT_IMPLEMENTED,
"cannot print GRN_PVECTOR using grn_obj_format");
} else {
unsigned int i, n;
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
n = GRN_BULK_VSIZE(obj) / sizeof(grn_obj *);
for (i = 0; i < n; i++) {
grn_obj *value;
value = GRN_PTR_VALUE_AT(obj, i);
grn_output_obj(ctx, outbuf, output_type, value, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
break;
case GRN_TABLE_HASH_KEY :
case GRN_TABLE_PAT_KEY :
case GRN_TABLE_NO_KEY :
case GRN_TABLE_VIEW :
if (format) {
int i, j;
int ncolumns = GRN_BULK_VSIZE(&format->columns)/sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
grn_table_cursor *tc = grn_table_cursor_open(ctx, obj, NULL, 0, NULL, 0,
format->offset, format->limit,
GRN_CURSOR_ASCENDING);
int resultset_size = -1;
if (!tc) { ERRCLR(ctx); }
#ifdef HAVE_MESSAGE_PACK
resultset_size = 1; /* [NHITS, (COLUMNS), (HITS)] */
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
resultset_size++;
}
resultset_size += format->limit;
#endif
grn_output_array_open(ctx, outbuf, output_type, "RESULTSET", resultset_size);
grn_output_array_open(ctx, outbuf, output_type, "NHITS", 1);
if (output_type == GRN_CONTENT_XML) {
grn_text_itoa(ctx, outbuf, format->nhits);
} else {
grn_output_int32(ctx, outbuf, output_type, format->nhits);
}
grn_output_array_close(ctx, outbuf, output_type);
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", 2);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
if (tc) {
grn_obj id;
GRN_TEXT_INIT(&id, 0);
for (i = 0; !grn_table_cursor_next_o(ctx, tc, &id); i++) {
grn_output_array_open(ctx, outbuf, output_type, "HIT", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_text_atoj_o(ctx, outbuf, output_type, columns[j], &id);
}
grn_output_array_close(ctx, outbuf, output_type);
}
GRN_OBJ_FIN(ctx, &id);
grn_table_cursor_close(ctx, tc);
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
int i;
grn_obj *column = grn_obj_column(ctx, obj, "_key", 4);
grn_table_cursor *tc = grn_table_cursor_open(ctx, obj, NULL, 0, NULL, 0,
0, -1, GRN_CURSOR_ASCENDING);
grn_output_array_open(ctx, outbuf, output_type, "HIT", -1);
if (tc) {
grn_obj id;
GRN_TEXT_INIT(&id, 0);
for (i = 0; !grn_table_cursor_next_o(ctx, tc, &id); i++) {
/* todo:
grn_text_atoj_o(ctx, outbuf, output_type, column, &id);
*/
GRN_BULK_REWIND(&buf);
grn_obj_get_value_o(ctx, column, &id, &buf);
grn_text_esc(ctx, outbuf, GRN_BULK_HEAD(&buf), GRN_BULK_VSIZE(&buf));
}
GRN_OBJ_FIN(ctx, &id);
grn_table_cursor_close(ctx, tc);
}
grn_output_array_close(ctx, outbuf, output_type);
grn_obj_unlink(ctx, column);
}
break;
}
GRN_OBJ_FIN(ctx, &buf);
}
