grn_obj *
grn_inspect_name(grn_ctx *ctx, grn_obj *buf, grn_obj *obj)
{
int name_size;
name_size = grn_obj_name(ctx, obj, NULL, 0);
if (name_size) {
grn_bulk_space(ctx, buf, name_size);
grn_obj_name(ctx, obj, GRN_BULK_CURR(buf) - name_size, name_size);
} else {
GRN_TEXT_PUTS(ctx, buf, "(nil)");
}
return buf;
}
static void
command_schema_column_output_sources(grn_ctx *ctx, grn_obj *column)
{
grn_obj *source_table;
grn_obj source_ids;
unsigned int i, n_ids;
source_table = grn_ctx_at(ctx, grn_obj_get_range(ctx, column));
GRN_RECORD_INIT(&source_ids, GRN_OBJ_VECTOR, GRN_ID_NIL);
if (column->header.type == GRN_COLUMN_INDEX) {
grn_obj_get_info(ctx, column, GRN_INFO_SOURCE, &source_ids);
}
n_ids = GRN_BULK_VSIZE(&source_ids) / sizeof(grn_id);
grn_ctx_output_array_open(ctx, "sources", n_ids);
for (i = 0; i < n_ids; i++) {
grn_id source_id;
grn_obj *source;
source_id = GRN_RECORD_VALUE_AT(&source_ids, i);
source = grn_ctx_at(ctx, source_id);
grn_ctx_output_map_open(ctx, "source", 3);
grn_ctx_output_cstr(ctx, "name");
if (grn_obj_is_table(ctx, source)) {
grn_ctx_output_cstr(ctx, "_key");
} else {
command_schema_output_column_name(ctx, source);
}
grn_ctx_output_cstr(ctx, "table");
command_schema_output_name(ctx, source_table);
grn_ctx_output_cstr(ctx, "full_name");
if (grn_obj_is_table(ctx, source)) {
char name[GRN_TABLE_MAX_KEY_SIZE];
unsigned int name_size;
name_size = grn_obj_name(ctx, source, name, GRN_TABLE_MAX_KEY_SIZE);
name[name_size] = '\0';
grn_strcat(name, GRN_TABLE_MAX_KEY_SIZE, "._key");
grn_ctx_output_cstr(ctx, name);
} else {
command_schema_output_name(ctx, source);
}
grn_ctx_output_map_close(ctx);
}
grn_ctx_output_array_close(ctx);
GRN_OBJ_FIN(ctx, &source_ids);
}
static void
command_schema_output_name(grn_ctx *ctx, grn_obj *obj)
{
if (obj) {
char name[GRN_TABLE_MAX_KEY_SIZE];
unsigned int name_size;
name_size = grn_obj_name(ctx, obj, name, GRN_TABLE_MAX_KEY_SIZE);
grn_ctx_output_str(ctx, name, name_size);
} else {
grn_ctx_output_null(ctx);
}
}
static int
print_tableinfo(grn_ctx *ctx, grn_obj *table, grn_obj *buf, grn_content_type otype)
{
grn_id id;
char name[GRN_TABLE_MAX_KEY_SIZE];
const char *path;
int name_len;
switch (table->header.type) {
case GRN_TABLE_HASH_KEY:
case GRN_TABLE_PAT_KEY:
case GRN_TABLE_NO_KEY:
case GRN_TABLE_VIEW:
break;
default:
return 0;
}
id = grn_obj_id(ctx, table);
name_len = grn_obj_name(ctx, table, name, GRN_TABLE_MAX_KEY_SIZE);
path = grn_obj_path(ctx, table);
switch (otype) {
case GRN_CONTENT_TSV:
grn_text_itoa(ctx, buf, id);
GRN_TEXT_PUTC(ctx, buf, '\t');
grn_text_esc(ctx, buf, name, name_len);
GRN_TEXT_PUTC(ctx, buf, '\t');
grn_text_esc(ctx, buf, path, GRN_STRLEN(path));
GRN_TEXT_PUTC(ctx, buf, '\t');
grn_text_itoa(ctx, buf, table->header.flags);
GRN_TEXT_PUTC(ctx, buf, '\t');
grn_text_itoa(ctx, buf, table->header.domain);
/* TODO: domain to str */
break;
case GRN_CONTENT_JSON:
GRN_TEXT_PUTC(ctx, buf, '[');
grn_text_itoa(ctx, buf, id);
GRN_TEXT_PUTC(ctx, buf, ',');
grn_text_esc(ctx, buf, name, name_len);
GRN_TEXT_PUTC(ctx, buf, ',');
grn_text_esc(ctx, buf, path, GRN_STRLEN(path));
GRN_TEXT_PUTC(ctx, buf, ',');
grn_text_itoa(ctx, buf, table->header.flags);
GRN_TEXT_PUTC(ctx, buf, ',');
grn_text_itoa(ctx, buf, table->header.domain);
/* TODO: domain to str */
GRN_TEXT_PUTC(ctx, buf, ']');
break;
}
return 1;
}
static mrb_value
object_get_name(mrb_state *mrb, mrb_value self)
{
grn_ctx *ctx = (grn_ctx *)mrb->ud;
grn_obj *object;
char name[GRN_TABLE_MAX_KEY_SIZE];
int name_length;
object = DATA_PTR(self);
name_length = grn_obj_name(ctx, object, name, GRN_TABLE_MAX_KEY_SIZE);
return mrb_str_new(mrb, name, name_length);
}
grn_obj *
grn_inspect_name(grn_ctx *ctx, grn_obj *buf, grn_obj *obj)
{
int name_size;
name_size = grn_obj_name(ctx, obj, NULL, 0);
if (name_size > 0) {
grn_bulk_space(ctx, buf, name_size);
grn_obj_name(ctx, obj, GRN_BULK_CURR(buf) - name_size, name_size);
} else {
grn_id id;
id = grn_obj_id(ctx, obj);
if (id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, buf, "(nil)");
} else {
GRN_TEXT_PUTS(ctx, buf, "(anonymous:");
grn_text_lltoa(ctx, buf, id);
GRN_TEXT_PUTS(ctx, buf, ")");
}
}
return buf;
}
static grn_id
resolve_source_id (grn_ctx *context, grn_obj *column, VALUE rb_source)
{
grn_id source_id;
if (CBOOL2RVAL(rb_obj_is_kind_of(rb_source, rb_cInteger))) {
source_id = NUM2UINT(rb_source);
} else {
grn_obj *source;
if (TYPE(rb_source) == T_STRING) {
grn_obj *table;
const char *name;
const char *dot_point;
int length;
table = grn_ctx_at(context, grn_obj_get_range(context, column));
name = StringValueCStr(rb_source);
length = RSTRING_LEN(rb_source);
dot_point = strstr(name, ".");
if (dot_point) {
char table_name[4096];
int table_name_length;
table_name_length = grn_obj_name(context, table,
table_name, sizeof(table_name));
table_name[table_name_length] = '\0';
if (strncmp(table_name, name, dot_point - name) != 0) {
rb_raise(rb_eArgError,
"wrong table's column: <%s>: "
"expected table: <%s>",
name, table_name);
}
length -= (dot_point - name) + 1;
name = dot_point + 1;
}
source = grn_obj_column(context, table, name, length);
} else {
source = RVAL2GRNOBJECT(rb_source, &context);
}
rb_grn_context_check(context, rb_source);
source_id = grn_obj_id(context, source);
}
return source_id;
}
void
grn_report_index(grn_ctx *ctx,
const char *action,
const char *tag,
grn_obj *index)
{
char index_name[GRN_TABLE_MAX_KEY_SIZE];
int index_name_size;
if (!grn_logger_pass(ctx, GRN_REPORT_INDEX_LOG_LEVEL)) {
return;
}
index_name_size = grn_obj_name(ctx, index, index_name, GRN_TABLE_MAX_KEY_SIZE);
GRN_LOG(ctx, GRN_REPORT_INDEX_LOG_LEVEL,
"%s[index]%s <%.*s>",
action, tag, index_name_size, index_name);
}
void
test_temporary_table_default_tokenizer(gpointer data)
{
grn_obj *table;
grn_obj_flags flags = GPOINTER_TO_INT(data);
grn_obj *tokenizer = NULL;
char name[1024];
int name_size;
table = grn_table_create(context, NULL, 0, NULL,
flags,
NULL, NULL);
grn_obj_set_info(context, table, GRN_INFO_DEFAULT_TOKENIZER,
get_object("TokenTrigram"));
tokenizer = grn_obj_get_info(context, table, GRN_INFO_DEFAULT_TOKENIZER, NULL);
name_size = grn_obj_name(context, tokenizer, name, sizeof(name));
name[name_size] = '\0';
cut_assert_equal_string("TokenTrigram", name);
}
static void
set_weight_vector(grn_ctx *ctx, grn_obj *column, grn_id id, grn_obj *value)
{
if (!grn_obj_is_weight_vector_column(ctx, column)) {
char column_name[GRN_TABLE_MAX_KEY_SIZE];
int column_name_size;
column_name_size = grn_obj_name(ctx, column, column_name,
GRN_TABLE_MAX_KEY_SIZE);
ERR(GRN_INVALID_ARGUMENT,
"<%.*s>: columns except weight vector column don't support object value",
column_name_size, column_name);
return;
}
{
grn_obj vector;
GRN_TEXT_INIT(&vector, GRN_OBJ_VECTOR);
add_weight_vector(ctx, column, value, &vector);
grn_obj_set_value(ctx, column, id, &vector, GRN_OBJ_SET);
GRN_OBJ_FIN(ctx, &vector);
}
}
void
grn_report_table(grn_ctx *ctx,
const char *action,
const char *tag,
grn_obj *table)
{
grn_obj description;
grn_obj *target;
if (!grn_logger_pass(ctx, GRN_REPORT_INDEX_LOG_LEVEL)) {
return;
}
GRN_TEXT_INIT(&description, 0);
for (target = table; target; target = grn_ctx_at(ctx, target->header.domain)) {
char name[GRN_TABLE_MAX_KEY_SIZE];
int name_size;
name_size = grn_obj_name(ctx, target, name, GRN_TABLE_MAX_KEY_SIZE);
if (GRN_TEXT_LEN(&description) > 0) {
GRN_TEXT_PUTS(ctx, &description, " -> ");
}
if (name_size == 0) {
GRN_TEXT_PUTS(ctx, &description, "(temporary)");
} else {
GRN_TEXT_PUTS(ctx, &description, "<");
GRN_TEXT_PUT(ctx, &description, name, name_size);
GRN_TEXT_PUTS(ctx, &description, ">");
}
}
GRN_LOG(ctx, GRN_REPORT_INDEX_LOG_LEVEL,
"%s[table]%s %.*s",
action, tag,
(int)GRN_TEXT_LEN(&description),
GRN_TEXT_VALUE(&description));
GRN_OBJ_FIN(ctx, &description);
}
/* grn_ts_writer_output_header() outputs names and data types. */
static grn_rc
grn_ts_writer_output_header(grn_ctx *ctx, grn_ts_writer *writer)
{
grn_rc rc;
GRN_OUTPUT_ARRAY_OPEN("COLUMNS", writer->n_exprs);
for (size_t i = 0; i < writer->n_exprs; ++i) {
GRN_OUTPUT_ARRAY_OPEN("COLUMN", 2);
rc = grn_text_esc(ctx, ctx->impl->output.buf,
writer->names[i].ptr, writer->names[i].size);
if (rc != GRN_SUCCESS) {
return rc;
}
GRN_TEXT_PUT(ctx, ctx->impl->output.buf, ",\"", 2);
switch (writer->exprs[i]->data_type) {
case GRN_DB_VOID: {
if (writer->exprs[i]->data_kind == GRN_TS_GEO) {
GRN_TEXT_PUTS(ctx, ctx->impl->output.buf, "GeoPoint");
} else {
GRN_TEXT_PUTS(ctx, ctx->impl->output.buf, "Void");
}
break;
}
GRN_TS_WRITER_OUTPUT_HEADER_CASE(BOOL, "Bool")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(INT8, "Int8")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(INT16, "Int16")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(INT32, "Int32")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(INT64, "Int64")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(UINT8, "UInt8")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(UINT16, "UInt16")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(UINT32, "UInt32")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(UINT64, "UInt64")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(FLOAT, "Float")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(TIME, "Time")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(SHORT_TEXT, "ShortText")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(TEXT, "Text")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(LONG_TEXT, "LongText")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(TOKYO_GEO_POINT, "TokyoGeoPoint")
GRN_TS_WRITER_OUTPUT_HEADER_CASE(WGS84_GEO_POINT, "WGS84GeoPoint")
default: {
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
size_t name_size;
grn_obj *obj = grn_ctx_at(ctx, writer->exprs[i]->data_type);
if (!obj) {
GRN_TS_ERR_RETURN(GRN_UNKNOWN_ERROR, "grn_ctx_at failed: %d",
writer->exprs[i]->data_type);
}
if (!grn_ts_obj_is_table(ctx, obj)) {
grn_obj_unlink(ctx, obj);
GRN_TS_ERR_RETURN(GRN_UNKNOWN_ERROR, "not table: %d",
writer->exprs[i]->data_type);
}
name_size = grn_obj_name(ctx, obj, name_buf, sizeof(name_buf));
GRN_TEXT_PUT(ctx, ctx->impl->output.buf, name_buf, name_size);
grn_obj_unlink(ctx, obj);
break;
}
}
GRN_TEXT_PUTC(ctx, ctx->impl->output.buf, '"');
GRN_OUTPUT_ARRAY_CLOSE();
}
GRN_OUTPUT_ARRAY_CLOSE(); /* COLUMNS. */
return GRN_SUCCESS;
}
static grn_id
resolve_source_id (grn_ctx *context, grn_obj *column, grn_id range_id,
VALUE rb_source)
{
grn_id source_id;
if (CBOOL2RVAL(rb_obj_is_kind_of(rb_source, rb_cInteger))) {
source_id = NUM2UINT(rb_source);
} else {
grn_obj *source;
grn_bool need_source_unlink = GRN_FALSE;
if (TYPE(rb_source) == T_STRING) {
grn_obj *table;
const char *name;
const char *dot_point;
int length;
table = grn_ctx_at(context, grn_obj_get_range(context, column));
name = StringValueCStr(rb_source);
length = RSTRING_LEN(rb_source);
dot_point = strstr(name, ".");
if (dot_point) {
char table_name[4096];
int table_name_length;
table_name_length = grn_obj_name(context, table,
table_name, sizeof(table_name));
table_name[table_name_length] = '\0';
if (strncmp(table_name, name, dot_point - name) != 0) {
rb_raise(rb_eArgError,
"wrong table's column: <%s>: "
"expected table: <%s>",
name, table_name);
}
length -= (dot_point - name) + 1;
name = dot_point + 1;
}
source = grn_obj_column(context, table, name, length);
need_source_unlink = GRN_TRUE;
} else {
source = RVAL2GRNOBJECT(rb_source, &context);
}
rb_grn_context_check(context, rb_source);
if (source->header.type == GRN_ACCESSOR) {
char name[256];
int length;
length = grn_column_name(context, source, name, sizeof(name));
name[length] = '\0';
if (strcmp(name, "_key") != 0) {
rb_raise(rb_eArgError,
"source accessor must be '_key': <%s>", name);
}
source_id = range_id;
} else {
source_id = grn_obj_id(context, source);
}
if (need_source_unlink) {
grn_obj_unlink(context, source);
}
}
return source_id;
}
static void
command_schema_table_command_collect_arguments(grn_ctx *ctx,
grn_obj *table,
grn_obj *arguments)
{
#define ADD(name_, value_) \
grn_vector_add_element(ctx, arguments, \
name_, strlen(name_), \
0, GRN_DB_TEXT); \
grn_vector_add_element(ctx, arguments, \
value_, strlen(value_), \
0, GRN_DB_TEXT)
#define ADD_OBJECT_NAME(name_, object_) do { \
char object_name[GRN_TABLE_MAX_KEY_SIZE]; \
unsigned int object_name_size; \
object_name_size = grn_obj_name(ctx, object_, \
object_name, \
GRN_TABLE_MAX_KEY_SIZE); \
object_name[object_name_size] = '\0'; \
ADD(name_, object_name); \
} while (GRN_FALSE)
ADD_OBJECT_NAME("name", table);
{
grn_obj flags;
grn_table_flags table_flags;
grn_table_flags ignored_flags = GRN_OBJ_KEY_NORMALIZE | GRN_OBJ_PERSISTENT;
GRN_TEXT_INIT(&flags, 0);
grn_table_get_info(ctx, table, &table_flags, NULL, NULL, NULL, NULL);
grn_dump_table_create_flags(ctx,
table_flags & ~ignored_flags,
&flags);
GRN_TEXT_PUTC(ctx, &flags, '\0');
ADD("flags", GRN_TEXT_VALUE(&flags));
GRN_OBJ_FIN(ctx, &flags);
}
{
grn_obj *key_type = NULL;
if (table->header.type != GRN_TABLE_NO_KEY &&
table->header.domain != GRN_ID_NIL) {
key_type = grn_ctx_at(ctx, table->header.domain);
}
if (key_type) {
ADD_OBJECT_NAME("key_type", key_type);
}
}
{
grn_obj *value_type = NULL;
grn_id range = GRN_ID_NIL;
if (table->header.type != GRN_TABLE_DAT_KEY) {
range = grn_obj_get_range(ctx, table);
}
if (range != GRN_ID_NIL) {
value_type = grn_ctx_at(ctx, range);
}
if (value_type) {
ADD_OBJECT_NAME("value_type", value_type);
}
}
{
grn_obj *tokenizer;
tokenizer = grn_obj_get_info(ctx, table, GRN_INFO_DEFAULT_TOKENIZER, NULL);
if (tokenizer) {
ADD_OBJECT_NAME("default_tokenizer", tokenizer);
}
}
{
grn_obj *normalizer;
normalizer = grn_obj_get_info(ctx, table, GRN_INFO_NORMALIZER, NULL);
if (!normalizer && (table->header.flags & GRN_OBJ_KEY_NORMALIZE)) {
normalizer = grn_ctx_get(ctx, "NormalizerAuto", -1);
}
if (normalizer) {
ADD_OBJECT_NAME("normalizer", normalizer);
}
}
if (table->header.type != GRN_TABLE_NO_KEY) {
grn_obj token_filters;
int n;
GRN_PTR_INIT(&token_filters, GRN_OBJ_VECTOR, GRN_DB_OBJECT);
grn_obj_get_info(ctx, table, GRN_INFO_TOKEN_FILTERS, &token_filters);
n = GRN_BULK_VSIZE(&token_filters) / sizeof(grn_obj *);
if (n > 0) {
grn_obj token_filter_names;
int i;
GRN_TEXT_INIT(&token_filter_names, 0);
for (i = 0; i < n; i++) {
grn_obj *token_filter;
char name[GRN_TABLE_MAX_KEY_SIZE];
int name_size;
token_filter = GRN_PTR_VALUE_AT(&token_filters, i);
name_size = grn_obj_name(ctx, token_filter,
name, GRN_TABLE_MAX_KEY_SIZE);
if (i > 0) {
GRN_TEXT_PUTC(ctx, &token_filter_names, ',');
}
GRN_TEXT_PUT(ctx, &token_filter_names, name, name_size);
}
GRN_TEXT_PUTC(ctx, &token_filter_names, '\0');
ADD("token_filters", GRN_TEXT_VALUE(&token_filter_names));
GRN_OBJ_FIN(ctx, &token_filter_names);
}
GRN_OBJ_FIN(ctx, &token_filters);
}
#undef ADD_OBJECT_NAME
#undef ADD
}
static void
bracket_close(grn_ctx *ctx, grn_loader *loader)
{
grn_id id = GRN_ID_NIL;
grn_obj *value, *value_end, *id_value = NULL, *key_value = NULL;
grn_obj *col, **cols; /* Columns except _id and _key. */
uint32_t i, begin;
uint32_t ncols; /* Number of columns except _id and _key. */
uint32_t nvalues; /* Number of values in brackets. */
uint32_t depth;
grn_bool is_record_load = GRN_FALSE;
cols = (grn_obj **)GRN_BULK_HEAD(&loader->columns);
ncols = GRN_BULK_VSIZE(&loader->columns) / sizeof(grn_obj *);
GRN_UINT32_POP(&loader->level, begin);
value = (grn_obj *)GRN_TEXT_VALUE(&loader->values) + begin;
value_end = (grn_obj *)GRN_TEXT_VALUE(&loader->values) + loader->values_size;
GRN_ASSERT(value->header.domain == GRN_JSON_LOAD_OPEN_BRACKET);
GRN_UINT32_SET(ctx, value, loader->values_size - begin - 1);
value++;
depth = GRN_BULK_VSIZE(&loader->level);
if (depth > sizeof(uint32_t) * loader->emit_level) {
return;
}
if (depth == 0 || !loader->table ||
loader->columns_status == GRN_LOADER_COLUMNS_BROKEN) {
goto exit;
}
nvalues = values_len(ctx, value, value_end);
if (loader->columns_status == GRN_LOADER_COLUMNS_UNSET) {
/*
* Target columns and _id or _key are not specified yet and values are
* handled as column names and "_id" or "_key".
*/
for (i = 0; i < nvalues; i++) {
const char *col_name;
unsigned int col_name_size;
if (value->header.domain != GRN_DB_TEXT) {
grn_obj buffer;
GRN_TEXT_INIT(&buffer, 0);
grn_inspect(ctx, &buffer, value);
ERR(GRN_INVALID_ARGUMENT,
"column name must be string: <%.*s>",
(int)GRN_TEXT_LEN(&buffer), GRN_TEXT_VALUE(&buffer));
grn_loader_save_error(ctx, loader);
GRN_OBJ_FIN(ctx, &buffer);
loader->columns_status = GRN_LOADER_COLUMNS_BROKEN;
goto exit;
}
col_name = GRN_TEXT_VALUE(value);
col_name_size = GRN_TEXT_LEN(value);
col = grn_obj_column(ctx, loader->table, col_name, col_name_size);
if (!col) {
ERR(GRN_INVALID_ARGUMENT, "nonexistent column: <%.*s>",
col_name_size, col_name);
grn_loader_save_error(ctx, loader);
loader->columns_status = GRN_LOADER_COLUMNS_BROKEN;
goto exit;
}
if (name_equal(col_name, col_name_size, GRN_COLUMN_NAME_ID)) {
grn_obj_unlink(ctx, col);
if (loader->id_offset != -1 || loader->key_offset != -1) {
/* _id and _key must not appear more than once. */
if (loader->id_offset != -1) {
ERR(GRN_INVALID_ARGUMENT,
"duplicated id and key columns: <%s> at %d and <%s> at %d",
GRN_COLUMN_NAME_ID, i,
GRN_COLUMN_NAME_ID, loader->id_offset);
} else {
ERR(GRN_INVALID_ARGUMENT,
"duplicated id and key columns: <%s> at %d and <%s> at %d",
GRN_COLUMN_NAME_ID, i,
GRN_COLUMN_NAME_KEY, loader->key_offset);
}
grn_loader_save_error(ctx, loader);
loader->columns_status = GRN_LOADER_COLUMNS_BROKEN;
goto exit;
}
loader->id_offset = i;
} else if (name_equal(col_name, col_name_size, GRN_COLUMN_NAME_KEY)) {
grn_obj_unlink(ctx, col);
if (loader->id_offset != -1 || loader->key_offset != -1) {
/* _id and _key must not appear more than once. */
if (loader->id_offset != -1) {
ERR(GRN_INVALID_ARGUMENT,
"duplicated id and key columns: <%s> at %d and <%s> at %d",
GRN_COLUMN_NAME_KEY, i,
GRN_COLUMN_NAME_ID, loader->id_offset);
} else {
ERR(GRN_INVALID_ARGUMENT,
"duplicated id and key columns: <%s> at %d and <%s> at %d",
GRN_COLUMN_NAME_KEY, i,
GRN_COLUMN_NAME_KEY, loader->key_offset);
}
grn_loader_save_error(ctx, loader);
loader->columns_status = GRN_LOADER_COLUMNS_BROKEN;
goto exit;
}
loader->key_offset = i;
} else {
GRN_PTR_PUT(ctx, &loader->columns, col);
}
value++;
}
switch (loader->table->header.type) {
case GRN_TABLE_HASH_KEY :
case GRN_TABLE_PAT_KEY :
case GRN_TABLE_DAT_KEY :
if (loader->id_offset == -1 && loader->key_offset == -1) {
ERR(GRN_INVALID_ARGUMENT, "missing id or key column");
grn_loader_save_error(ctx, loader);
loader->columns_status = GRN_LOADER_COLUMNS_BROKEN;
goto exit;
}
break;
}
loader->columns_status = GRN_LOADER_COLUMNS_SET;
goto exit;
}
is_record_load = GRN_TRUE;
/* Target columns and _id or _key are already specified. */
if (!nvalues) {
/*
* Accept empty arrays because a dump command may output a load command
* which contains empty arrays for a table with deleted records.
*/
id = grn_table_add(ctx, loader->table, NULL, 0, NULL);
} else {
uint32_t expected_nvalues = ncols;
if (loader->id_offset != -1 || loader->key_offset != -1) {
expected_nvalues++;
}
if (nvalues != expected_nvalues) {
ERR(GRN_INVALID_ARGUMENT,
"unexpected #values: expected:%u, actual:%u",
expected_nvalues, nvalues);
grn_loader_save_error(ctx, loader);
goto exit;
}
if (loader->id_offset != -1) {
id_value = value + loader->id_offset;
id = parse_id_value(ctx, id_value);
if (grn_table_at(ctx, loader->table, id) == GRN_ID_NIL) {
id = grn_table_add(ctx, loader->table, NULL, 0, NULL);
}
} else if (loader->key_offset != -1) {
key_value = value + loader->key_offset;
id = loader_add(ctx, key_value);
} else {
id = grn_table_add(ctx, loader->table, NULL, 0, NULL);
}
}
if (id == GRN_ID_NIL) {
/* Target record is not available. */
goto exit;
}
for (i = 0; i < nvalues; i++, value = values_next(ctx, value)) {
if (i == loader->id_offset || i == loader->key_offset) {
/* Skip _id and _key, because it's already used to get id. */
continue;
}
col = *cols;
if (value->header.domain == GRN_JSON_LOAD_OPEN_BRACKET) {
set_vector(ctx, col, id, value);
} else if (value->header.domain == GRN_JSON_LOAD_OPEN_BRACE) {
set_weight_vector(ctx, col, id, value);
} else {
grn_obj_set_value(ctx, col, id, value, GRN_OBJ_SET);
}
if (ctx->rc != GRN_SUCCESS) {
char column_name[GRN_TABLE_MAX_KEY_SIZE];
unsigned int column_name_size;
grn_loader_save_error(ctx, loader);
column_name_size = grn_obj_name(ctx, col, column_name,
GRN_TABLE_MAX_KEY_SIZE);
report_set_column_value_failure(ctx, key_value,
column_name, column_name_size,
value);
loader->n_column_errors++;
ERRCLR(ctx);
}
cols++;
}
if (loader->each) {
grn_obj *v = grn_expr_get_var_by_offset(ctx, loader->each, 0);
GRN_RECORD_SET(ctx, v, id);
grn_expr_exec(ctx, loader->each, 0);
}
loader->nrecords++;
exit:
if (is_record_load) {
if (ctx->rc != GRN_SUCCESS) {
loader->n_record_errors++;
}
if (loader->output_ids) {
GRN_UINT32_PUT(ctx, &(loader->ids), id);
}
if (loader->output_errors) {
GRN_INT32_PUT(ctx, &(loader->return_codes), ctx->rc);
grn_vector_add_element(ctx,
&(loader->error_messages),
ctx->errbuf,
strlen(ctx->errbuf),
0,
GRN_DB_TEXT);
}
}
loader->values_size = begin;
ERRCLR(ctx);
}
grn_id
grn_token_next(grn_ctx *ctx, grn_token *token)
{
int status;
grn_id tid = GRN_ID_NIL;
grn_obj *table = token->table;
grn_obj *tokenizer = token->tokenizer;
while (token->status != GRN_TOKEN_DONE) {
if (tokenizer) {
grn_obj *curr_, *stat_;
((grn_proc *)tokenizer)->funcs[PROC_NEXT](ctx, 1, &table, &token->pctx.user_data);
stat_ = grn_ctx_pop(ctx);
curr_ = grn_ctx_pop(ctx);
token->curr = (const unsigned char *)GRN_TEXT_VALUE(curr_);
token->curr_size = GRN_TEXT_LEN(curr_);
status = GRN_UINT32_VALUE(stat_);
token->status = ((status & GRN_TOKENIZER_TOKEN_LAST) ||
(token->mode == GRN_TOKEN_GET &&
(status & GRN_TOKENIZER_TOKEN_REACH_END)))
? GRN_TOKEN_DONE : GRN_TOKEN_DOING;
token->force_prefix = 0;
if (token->curr_size == 0) {
char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE];
int tokenizer_name_length;
tokenizer_name_length =
grn_obj_name(ctx, token->tokenizer,
tokenizer_name, GRN_TABLE_MAX_KEY_SIZE);
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore an empty token: <%.*s>: <%.*s>",
tokenizer_name_length, tokenizer_name,
token->orig_blen, token->orig);
continue;
}
if (token->curr_size > GRN_TABLE_MAX_KEY_SIZE) {
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore too long token. "
"Token must be less than or equal to %d: <%d>(<%.*s>)",
GRN_TABLE_MAX_KEY_SIZE,
token->curr_size,
token->curr_size, token->curr);
continue;
}
if (status & GRN_TOKENIZER_TOKEN_UNMATURED) {
if (status & GRN_TOKENIZER_TOKEN_OVERLAP) {
if (token->mode == GRN_TOKEN_GET) { token->pos++; continue; }
} else {
if (status & GRN_TOKENIZER_TOKEN_LAST) { token->force_prefix = 1; }
}
}
} else {
token->status = GRN_TOKEN_DONE;
}
if (token->mode == GRN_TOKEN_ADD) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
if (grn_io_lock(ctx, ((grn_pat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_pat_add(ctx, (grn_pat *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_pat *)table)->io);
}
break;
case GRN_TABLE_DAT_KEY :
if (grn_io_lock(ctx, ((grn_dat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_dat_add(ctx, (grn_dat *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_dat *)table)->io);
}
break;
case GRN_TABLE_HASH_KEY :
if (grn_io_lock(ctx, ((grn_hash *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_hash_add(ctx, (grn_hash *)table, token->curr, token->curr_size,
NULL, NULL);
grn_io_unlock(((grn_hash *)table)->io);
}
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
} else {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
tid = grn_pat_get(ctx, (grn_pat *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_DAT_KEY :
tid = grn_dat_get(ctx, (grn_dat *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_HASH_KEY :
tid = grn_hash_get(ctx, (grn_hash *)table, token->curr, token->curr_size, NULL);
break;
case GRN_TABLE_NO_KEY :
if (token->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
}
if (tid == GRN_ID_NIL && token->status != GRN_TOKEN_DONE) {
token->status = GRN_TOKEN_NOT_FOUND;
}
token->pos++;
break;
}
return tid;
}
void
grn_output_obj(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type,
grn_obj *obj, grn_obj_format *format)
{
grn_obj buf;
GRN_TEXT_INIT(&buf, 0);
switch (obj->header.type) {
case GRN_BULK :
switch (obj->header.domain) {
case GRN_DB_VOID :
grn_output_void(ctx, outbuf, output_type, GRN_BULK_HEAD(obj), GRN_BULK_VSIZE(obj));
break;
case GRN_DB_SHORT_TEXT :
case GRN_DB_TEXT :
case GRN_DB_LONG_TEXT :
grn_output_str(ctx, outbuf, output_type, GRN_BULK_HEAD(obj), GRN_BULK_VSIZE(obj));
break;
case GRN_DB_BOOL :
grn_output_bool(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT8_VALUE(obj) : 0);
break;
case GRN_DB_INT8 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT8_VALUE(obj) : 0);
break;
case GRN_DB_UINT8 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT8_VALUE(obj) : 0);
break;
case GRN_DB_INT16 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT16_VALUE(obj) : 0);
break;
case GRN_DB_UINT16 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT16_VALUE(obj) : 0);
break;
case GRN_DB_INT32 :
grn_output_int32(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT32_VALUE(obj) : 0);
break;
case GRN_DB_UINT32 :
grn_output_int64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT32_VALUE(obj) : 0);
break;
case GRN_DB_INT64 :
grn_output_int64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT64_VALUE(obj) : 0);
break;
case GRN_DB_UINT64 :
grn_output_uint64(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_UINT64_VALUE(obj) : 0);
break;
case GRN_DB_FLOAT :
grn_output_float(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_FLOAT_VALUE(obj) : 0);
break;
case GRN_DB_TIME :
grn_output_time(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? GRN_INT64_VALUE(obj) : 0);
break;
case GRN_DB_TOKYO_GEO_POINT :
case GRN_DB_WGS84_GEO_POINT :
grn_output_geo_point(ctx, outbuf, output_type,
GRN_BULK_VSIZE(obj) ? (grn_geo_point *)GRN_BULK_HEAD(obj) : NULL);
break;
default :
if (format) {
int j;
int ncolumns = GRN_BULK_VSIZE(&format->columns)/sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", 2);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_open(ctx, outbuf, output_type, "HIT", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_text_atoj_o(ctx, outbuf, output_type, columns[j], obj);
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_obj *table = grn_ctx_at(ctx, obj->header.domain);
grn_id id = *((grn_id *)GRN_BULK_HEAD(obj));
if (table && table->header.type != GRN_TABLE_NO_KEY) {
grn_obj *accessor = grn_obj_column(ctx, table, "_key", 4);
if (accessor) {
grn_obj_get_value(ctx, accessor, id, &buf);
grn_obj_unlink(ctx, accessor);
}
grn_output_obj(ctx, outbuf, output_type, &buf, format);
} else {
grn_output_int64(ctx, outbuf, output_type, id);
}
}
break;
}
break;
case GRN_UVECTOR :
if (format) {
int i, j;
grn_id *v = (grn_id *)GRN_BULK_HEAD(obj), *ve = (grn_id *)GRN_BULK_CURR(obj);
int ncolumns = GRN_BULK_VSIZE(&format->columns) / sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
grn_output_array_open(ctx, outbuf, output_type, "RESULTSET", -1);
grn_output_array_open(ctx, outbuf, output_type, "NHITS", 1);
grn_text_itoa(ctx, outbuf, ve - v);
grn_output_array_close(ctx, outbuf, output_type);
if (v < ve) {
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", -1);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", -1);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
for (i = 0;; i++) {
grn_output_array_open(ctx, outbuf, output_type, "HITS", -1);
for (j = 0; j < ncolumns; j++) {
GRN_BULK_REWIND(&buf);
grn_obj_get_value(ctx, columns[j], *v, &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
v++;
if (v < ve) {
} else {
break;
}
}
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_obj *range = grn_ctx_at(ctx, obj->header.domain);
if (range && range->header.type == GRN_TYPE) {
int value_size = ((struct _grn_type *)range)->obj.range;
char *v = (char *)GRN_BULK_HEAD(obj),
*ve = (char *)GRN_BULK_CURR(obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
if (v < ve) {
for (;;) {
grn_obj value;
GRN_OBJ_INIT(&value, GRN_BULK, 0, obj->header.domain);
grn_bulk_write_from(ctx, &value, v, 0, value_size);
grn_output_obj(ctx, outbuf, output_type, &value, NULL);
v += value_size;
if (v < ve) {
} else {
break;
}
}
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
grn_id *v = (grn_id *)GRN_BULK_HEAD(obj),
*ve = (grn_id *)GRN_BULK_CURR(obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", ve - v);
if (v < ve) {
grn_obj key;
GRN_OBJ_INIT(&key, GRN_BULK, 0, range->header.domain);
for (;;) {
if (range->header.type != GRN_TABLE_NO_KEY) {
grn_table_get_key2(ctx, range, *v, &key);
grn_output_obj(ctx, outbuf, output_type, &key, NULL);
GRN_BULK_REWIND(&key);
} else {
grn_obj id;
GRN_UINT32_INIT(&id, 0);
GRN_UINT32_SET(ctx, &id, *v);
grn_output_obj(ctx, outbuf, output_type, &id, NULL);
GRN_OBJ_FIN(ctx, &id);
}
v++;
if (v < ve) {
} else {
break;
}
}
GRN_OBJ_FIN(ctx, &key);
}
grn_output_array_close(ctx, outbuf, output_type);
}
}
break;
case GRN_VECTOR :
if (obj->header.domain == GRN_DB_VOID) {
ERR(GRN_INVALID_ARGUMENT, "invalid obj->header.domain");
}
if (format) {
ERR(GRN_FUNCTION_NOT_IMPLEMENTED,
"cannot print GRN_VECTOR using grn_obj_format");
} else {
unsigned int i, n;
grn_obj value;
GRN_VOID_INIT(&value);
n = grn_vector_size(ctx, obj);
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
for (i = 0; i < n; i++) {
const char *_value;
unsigned int weight, length;
grn_id domain;
length = grn_vector_get_element(ctx, obj, i,
&_value, &weight, &domain);
if (domain != GRN_DB_VOID) {
grn_obj_reinit(ctx, &value, domain, 0);
} else {
grn_obj_reinit(ctx, &value, obj->header.domain, 0);
}
grn_bulk_write(ctx, &value, _value, length);
grn_output_obj(ctx, outbuf, output_type, &value, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
GRN_OBJ_FIN(ctx, &value);
}
break;
case GRN_PVECTOR :
if (format) {
ERR(GRN_FUNCTION_NOT_IMPLEMENTED,
"cannot print GRN_PVECTOR using grn_obj_format");
} else {
unsigned int i, n;
grn_output_array_open(ctx, outbuf, output_type, "VECTOR", -1);
n = GRN_BULK_VSIZE(obj) / sizeof(grn_obj *);
for (i = 0; i < n; i++) {
grn_obj *value;
value = GRN_PTR_VALUE_AT(obj, i);
grn_output_obj(ctx, outbuf, output_type, value, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
break;
case GRN_TABLE_HASH_KEY :
case GRN_TABLE_PAT_KEY :
case GRN_TABLE_NO_KEY :
case GRN_TABLE_VIEW :
if (format) {
int i, j;
int ncolumns = GRN_BULK_VSIZE(&format->columns)/sizeof(grn_obj *);
grn_obj **columns = (grn_obj **)GRN_BULK_HEAD(&format->columns);
grn_table_cursor *tc = grn_table_cursor_open(ctx, obj, NULL, 0, NULL, 0,
format->offset, format->limit,
GRN_CURSOR_ASCENDING);
int resultset_size = -1;
if (!tc) { ERRCLR(ctx); }
#ifdef HAVE_MESSAGE_PACK
resultset_size = 1; /* [NHITS, (COLUMNS), (HITS)] */
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
resultset_size++;
}
resultset_size += format->limit;
#endif
grn_output_array_open(ctx, outbuf, output_type, "RESULTSET", resultset_size);
grn_output_array_open(ctx, outbuf, output_type, "NHITS", 1);
if (output_type == GRN_CONTENT_XML) {
grn_text_itoa(ctx, outbuf, format->nhits);
} else {
grn_output_int32(ctx, outbuf, output_type, format->nhits);
}
grn_output_array_close(ctx, outbuf, output_type);
if (format->flags & GRN_OBJ_FORMAT_WITH_COLUMN_NAMES) {
grn_output_array_open(ctx, outbuf, output_type, "COLUMNS", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_id range_id;
grn_output_array_open(ctx, outbuf, output_type, "COLUMN", 2);
GRN_BULK_REWIND(&buf);
grn_column_name_(ctx, columns[j], &buf);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
/* column range */
range_id = grn_obj_get_range(ctx, columns[j]);
if (range_id == GRN_ID_NIL) {
GRN_TEXT_PUTS(ctx, outbuf, "null");
} else {
int name_len;
grn_obj *range_obj;
char name_buf[GRN_TABLE_MAX_KEY_SIZE];
range_obj = grn_ctx_at(ctx, range_id);
name_len = grn_obj_name(ctx, range_obj, name_buf,
GRN_TABLE_MAX_KEY_SIZE);
GRN_BULK_REWIND(&buf);
GRN_TEXT_PUT(ctx, &buf, name_buf, name_len);
grn_output_obj(ctx, outbuf, output_type, &buf, NULL);
}
grn_output_array_close(ctx, outbuf, output_type);
}
grn_output_array_close(ctx, outbuf, output_type);
}
if (tc) {
grn_obj id;
GRN_TEXT_INIT(&id, 0);
for (i = 0; !grn_table_cursor_next_o(ctx, tc, &id); i++) {
grn_output_array_open(ctx, outbuf, output_type, "HIT", ncolumns);
for (j = 0; j < ncolumns; j++) {
grn_text_atoj_o(ctx, outbuf, output_type, columns[j], &id);
}
grn_output_array_close(ctx, outbuf, output_type);
}
GRN_OBJ_FIN(ctx, &id);
grn_table_cursor_close(ctx, tc);
}
grn_output_array_close(ctx, outbuf, output_type);
} else {
int i;
grn_obj *column = grn_obj_column(ctx, obj, "_key", 4);
grn_table_cursor *tc = grn_table_cursor_open(ctx, obj, NULL, 0, NULL, 0,
0, -1, GRN_CURSOR_ASCENDING);
grn_output_array_open(ctx, outbuf, output_type, "HIT", -1);
if (tc) {
grn_obj id;
GRN_TEXT_INIT(&id, 0);
for (i = 0; !grn_table_cursor_next_o(ctx, tc, &id); i++) {
/* todo:
grn_text_atoj_o(ctx, outbuf, output_type, column, &id);
*/
GRN_BULK_REWIND(&buf);
grn_obj_get_value_o(ctx, column, &id, &buf);
grn_text_esc(ctx, outbuf, GRN_BULK_HEAD(&buf), GRN_BULK_VSIZE(&buf));
}
GRN_OBJ_FIN(ctx, &id);
grn_table_cursor_close(ctx, tc);
}
grn_output_array_close(ctx, outbuf, output_type);
grn_obj_unlink(ctx, column);
}
break;
}
GRN_OBJ_FIN(ctx, &buf);
}
void
grn_mrb_value_to_raw_data(mrb_state *mrb,
const char *context,
mrb_value mrb_value_,
grn_id domain_id,
grn_mrb_value_to_raw_data_buffer *buffer,
void **raw_value,
unsigned int *raw_value_size)
{
grn_ctx *ctx = (grn_ctx *)mrb->ud;
enum mrb_vtype mrb_value_type;
grn_bool try_cast = GRN_FALSE;
grn_obj *from_bulk = NULL;
if (mrb_nil_p(mrb_value_)) {
*raw_value = NULL;
*raw_value_size = 0;
return;
}
mrb_value_type = mrb_type(mrb_value_);
switch (mrb_value_type) {
case MRB_TT_STRING :
switch (domain_id) {
case GRN_DB_SHORT_TEXT :
case GRN_DB_TEXT :
case GRN_DB_LONG_TEXT :
*raw_value = RSTRING_PTR(mrb_value_);
*raw_value_size = RSTRING_LEN(mrb_value_);
break;
default :
try_cast = GRN_TRUE;
break;
}
break;
default :
{
struct RClass *klass;
grn_mrb_data *data = &(ctx->impl->mrb);
klass = mrb_class(mrb, mrb_value_);
if (domain_id == GRN_DB_TIME &&
klass == data->builtin.time_class) {
mrb_value mrb_sec;
mrb_value mrb_usec;
mrb_sec = mrb_funcall(mrb, mrb_value_, "to_i", 0);
mrb_usec = mrb_funcall(mrb, mrb_value_, "usec", 0);
buffer->value.time_value = GRN_TIME_PACK(mrb_fixnum(mrb_sec),
mrb_fixnum(mrb_usec));
*raw_value = &(buffer->value.time_value);
*raw_value_size = sizeof(buffer->value.time_value);
} else {
try_cast = GRN_TRUE;
if (mrb_value_type == MRB_TT_DATA &&
klass == mrb_class_get_under(mrb, data->module, "Bulk")) {
from_bulk = DATA_PTR(mrb_value_);
}
}
}
break;
}
if (!try_cast) {
return;
}
if (!from_bulk) {
from_bulk = &(buffer->from);
grn_mrb_value_to_bulk(mrb, mrb_value_, from_bulk);
}
if (!grn_mrb_bulk_cast(mrb, from_bulk, &(buffer->to), domain_id)) {
grn_obj *domain;
char domain_name[GRN_TABLE_MAX_KEY_SIZE];
int domain_name_size;
domain = grn_ctx_at(ctx, domain_id);
domain_name_size = grn_obj_name(ctx, domain, domain_name,
GRN_TABLE_MAX_KEY_SIZE);
mrb_raisef(mrb, E_ARGUMENT_ERROR,
"%S: failed to convert to %S: %S",
mrb_str_new_static(mrb, context, strlen(context)),
mrb_str_new_static(mrb, domain_name, domain_name_size),
mrb_funcall(mrb, mrb_value_, "inspect", 0));
}
*raw_value = GRN_BULK_HEAD(&(buffer->to));
*raw_value_size = GRN_BULK_VSIZE(&(buffer->to));
}
mrb_value
grn_mrb_value_from_bulk(mrb_state *mrb, grn_obj *bulk)
{
mrb_value mrb_value_;
grn_ctx *ctx = (grn_ctx *)mrb->ud;
switch (bulk->header.domain) {
case GRN_DB_INT32 :
{
int32_t value;
value = GRN_INT32_VALUE(bulk);
mrb_value_ = mrb_fixnum_value(value);
}
break;
case GRN_DB_UINT32 :
{
int64_t value;
value = GRN_UINT32_VALUE(bulk);
if (!FIXABLE(value)) {
mrb_raisef(mrb, E_RANGE_ERROR,
"can't handle large number: <%S>: max: <%S>",
mrb_fixnum_value(value), /* TODO: This will cause overflow */
mrb_fixnum_value(MRB_INT_MAX));
}
mrb_value_ = mrb_fixnum_value(value);
}
break;
case GRN_DB_TIME :
{
int64_t value;
int32_t sec;
int32_t usec;
value = GRN_TIME_VALUE(bulk);
GRN_TIME_UNPACK(value, sec, usec);
mrb_value_ = mrb_funcall(mrb,
mrb_obj_value(ctx->impl->mrb.builtin.time_class),
"at",
2,
mrb_fixnum_value(sec),
mrb_fixnum_value(usec));
}
break;
case GRN_DB_SHORT_TEXT :
case GRN_DB_TEXT :
case GRN_DB_LONG_TEXT :
mrb_value_ = mrb_str_new(mrb,
GRN_TEXT_VALUE(bulk),
GRN_TEXT_LEN(bulk));
break;
default :
{
grn_obj *domain;
grn_bool is_record = GRN_FALSE;
domain = grn_ctx_at(ctx, bulk->header.domain);
if (domain) {
switch (domain->header.type) {
case GRN_TABLE_HASH_KEY :
case GRN_TABLE_PAT_KEY :
case GRN_TABLE_DAT_KEY :
case GRN_TABLE_NO_KEY :
is_record = GRN_TRUE;
break;
default :
break;
}
}
if (is_record) {
mrb_value_ = mrb_fixnum_value(GRN_RECORD_VALUE(bulk));
grn_obj_unlink(ctx, domain);
} else {
#define MESSAGE_SIZE 4096
char message[MESSAGE_SIZE];
char domain_name[GRN_TABLE_MAX_KEY_SIZE];
int domain_name_size;
if (domain) {
domain_name_size = grn_obj_name(ctx, domain,
domain_name, GRN_TABLE_MAX_KEY_SIZE);
grn_obj_unlink(ctx, domain);
} else {
grn_strcpy(domain_name, GRN_TABLE_MAX_KEY_SIZE, "unknown");
domain_name_size = strlen(domain_name);
}
grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE,
"unsupported bulk value type: <%d>(%.*s)",
bulk->header.domain,
domain_name_size,
domain_name);
mrb_raise(mrb, E_RANGE_ERROR, message);
}
#undef MESSAGE_SIZE
}
break;
}
return mrb_value_;
}
grn_id
grn_token_cursor_next(grn_ctx *ctx, grn_token_cursor *token_cursor)
{
int status;
grn_id tid = GRN_ID_NIL;
grn_obj *table = token_cursor->table;
grn_obj *tokenizer = token_cursor->tokenizer;
while (token_cursor->status != GRN_TOKEN_CURSOR_DONE) {
if (tokenizer) {
grn_obj *curr_, *stat_;
((grn_proc *)tokenizer)->funcs[PROC_NEXT](ctx, 1, &table, &token_cursor->pctx.user_data);
stat_ = grn_ctx_pop(ctx);
curr_ = grn_ctx_pop(ctx);
status = grn_token_cursor_next_apply_token_filters(ctx, token_cursor,
curr_, stat_);
token_cursor->status =
((status & GRN_TOKEN_LAST) ||
(token_cursor->mode == GRN_TOKENIZE_GET &&
(status & GRN_TOKEN_REACH_END)))
? GRN_TOKEN_CURSOR_DONE : GRN_TOKEN_CURSOR_DOING;
token_cursor->force_prefix = GRN_FALSE;
#define SKIP_FLAGS \
(GRN_TOKEN_SKIP | GRN_TOKEN_SKIP_WITH_POSITION)
if (status & SKIP_FLAGS) {
if (status & GRN_TOKEN_SKIP) {
token_cursor->pos++;
}
if (token_cursor->status == GRN_TOKEN_CURSOR_DONE && tid == GRN_ID_NIL) {
token_cursor->status = GRN_TOKEN_CURSOR_DONE_SKIP;
break;
} else {
continue;
}
}
#undef SKIP_FLAGS
if (status & GRN_TOKEN_FORCE_PREFIX) {
token_cursor->force_prefix = GRN_TRUE;
}
if (token_cursor->curr_size == 0) {
if (token_cursor->status != GRN_TOKEN_CURSOR_DONE) {
char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE];
int tokenizer_name_length;
tokenizer_name_length =
grn_obj_name(ctx, token_cursor->tokenizer,
tokenizer_name, GRN_TABLE_MAX_KEY_SIZE);
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore an empty token: <%.*s>: <%.*s>",
tokenizer_name_length, tokenizer_name,
token_cursor->orig_blen, token_cursor->orig);
}
continue;
}
if (token_cursor->curr_size > GRN_TABLE_MAX_KEY_SIZE) {
GRN_LOG(ctx, GRN_WARN,
"[token_next] ignore too long token. "
"Token must be less than or equal to %d: <%d>(<%.*s>)",
GRN_TABLE_MAX_KEY_SIZE,
token_cursor->curr_size,
token_cursor->curr_size, token_cursor->curr);
continue;
}
if (status & GRN_TOKEN_UNMATURED) {
if (status & GRN_TOKEN_OVERLAP) {
if (token_cursor->mode == GRN_TOKENIZE_GET) {
token_cursor->pos++;
continue;
}
} else {
if (status & GRN_TOKEN_REACH_END) {
token_cursor->force_prefix = GRN_TRUE;
}
}
}
} else {
token_cursor->status = GRN_TOKEN_CURSOR_DONE;
}
if (token_cursor->mode == GRN_TOKENIZE_ADD) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
if (grn_io_lock(ctx, ((grn_pat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_pat_add(ctx, (grn_pat *)table, token_cursor->curr, token_cursor->curr_size,
NULL, NULL);
grn_io_unlock(((grn_pat *)table)->io);
}
break;
case GRN_TABLE_DAT_KEY :
if (grn_io_lock(ctx, ((grn_dat *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_dat_add(ctx, (grn_dat *)table, token_cursor->curr, token_cursor->curr_size,
NULL, NULL);
grn_io_unlock(((grn_dat *)table)->io);
}
break;
case GRN_TABLE_HASH_KEY :
if (grn_io_lock(ctx, ((grn_hash *)table)->io, grn_lock_timeout)) {
tid = GRN_ID_NIL;
} else {
tid = grn_hash_add(ctx, (grn_hash *)table, token_cursor->curr, token_cursor->curr_size,
NULL, NULL);
grn_io_unlock(((grn_hash *)table)->io);
}
break;
case GRN_TABLE_NO_KEY :
if (token_cursor->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token_cursor->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
} else if (token_cursor->mode != GRN_TOKENIZE_ONLY) {
switch (table->header.type) {
case GRN_TABLE_PAT_KEY :
tid = grn_pat_get(ctx, (grn_pat *)table, token_cursor->curr, token_cursor->curr_size, NULL);
break;
case GRN_TABLE_DAT_KEY :
tid = grn_dat_get(ctx, (grn_dat *)table, token_cursor->curr, token_cursor->curr_size, NULL);
break;
case GRN_TABLE_HASH_KEY :
tid = grn_hash_get(ctx, (grn_hash *)table, token_cursor->curr, token_cursor->curr_size, NULL);
break;
case GRN_TABLE_NO_KEY :
if (token_cursor->curr_size == sizeof(grn_id)) {
tid = *((grn_id *)token_cursor->curr);
} else {
tid = GRN_ID_NIL;
}
break;
}
}
if (token_cursor->mode != GRN_TOKENIZE_ONLY &&
tid == GRN_ID_NIL && token_cursor->status != GRN_TOKEN_CURSOR_DONE) {
token_cursor->status = GRN_TOKEN_CURSOR_NOT_FOUND;
}
token_cursor->pos++;
break;
}
return tid;
}
