void cut_startup()
{
ctx = (grn_ctx *)malloc(sizeof(grn_ctx));
grn_init();
grn_ctx_init(ctx, 0);
db = grn_db_create(ctx, NULL, NULL);
grn_ctx_use(ctx, db);
}
void release(grn_ctx *ctx) {
MRN_DBUG_ENTER_METHOD();
{
mrn::Lock lock(mutex_);
list_push(pool_, ctx);
grn_ctx_use(ctx, NULL);
}
DBUG_VOID_RETURN;
}
void
cut_setup(void)
{
gchar *table_path, *vgram_path;
const gchar *type_name, *table_name;
cut_set_fixture_data_dir(grn_test_get_base_dir(),
"fixtures",
"inverted-index",
NULL);
logger = setup_grn_logger();
expected_messages = NULL;
record_ids = NULL;
remove_tmp_directory();
g_mkdir_with_parents(tmp_directory, 0700);
path = g_build_filename(tmp_directory, "inverted-index", NULL);
context = g_new0(grn_ctx, 1);
grn_test_assert(grn_ctx_init(context, GRN_CTX_USE_QL));
GRN_CTX_SET_ENCODING(context, GRN_ENC_UTF8);
db = grn_db_create(context, NULL, NULL);
grn_ctx_use(context, db);
type_name = "name";
type = grn_type_create(context, type_name, strlen(type_name),
GRN_OBJ_KEY_VAR_SIZE, TYPE_SIZE);
table_name = "lexicon";
table_path = g_build_filename(tmp_directory, "lexicon-table", NULL);
lexicon = grn_table_create(context,
table_name, strlen(table_name),
table_path,
GRN_OBJ_PERSISTENT | GRN_OBJ_TABLE_PAT_KEY,
type, NULL);
grn_obj_set_info(context, lexicon, GRN_INFO_DEFAULT_TOKENIZER,
grn_ctx_at(context, GRN_DB_BIGRAM));
g_free(table_path);
vgram_path = g_build_filename(tmp_directory, "vgram", NULL);
/*
vgram = grn_vgram_create(vgram_path);
*/
g_free(vgram_path);
inverted_index = NULL;
}
static ngx_int_t
ngx_http_groonga_handler_create_data(ngx_http_request_t *r,
ngx_http_groonga_handler_data_t **data_return)
{
ngx_int_t rc;
ngx_http_groonga_loc_conf_t *location_conf;
ngx_http_cleanup_t *cleanup;
ngx_http_groonga_handler_data_t *data;
grn_ctx *context;
location_conf = ngx_http_get_module_loc_conf(r, ngx_http_groonga_module);
cleanup = ngx_http_cleanup_add(r, sizeof(ngx_http_groonga_handler_data_t));
cleanup->handler = ngx_http_groonga_handler_cleanup;
data = cleanup->data;
*data_return = data;
context = &(data->context);
rc = ngx_http_groonga_context_init(context, location_conf,
r->pool, r->connection->log);
if (rc != NGX_OK) {
return rc;
}
data->initialized = GRN_TRUE;
data->raw.processed = GRN_FALSE;
data->raw.header_sent = GRN_FALSE;
data->raw.r = r;
data->raw.rc = NGX_OK;
data->raw.free_chain = NULL;
data->raw.busy_chain = NULL;
GRN_TEXT_INIT(&(data->typed.head), GRN_NO_FLAGS);
GRN_TEXT_INIT(&(data->typed.body), GRN_NO_FLAGS);
GRN_TEXT_INIT(&(data->typed.foot), GRN_NO_FLAGS);
grn_ctx_use(context, grn_ctx_db(&(location_conf->context)));
rc = ngx_http_groonga_context_check_error(r->connection->log, context);
if (rc != NGX_OK) {
return rc;
}
grn_ctx_recv_handler_set(context,
ngx_http_groonga_context_receive_handler,
data);
return NGX_OK;
}
void
rb_grn_database_finalizer (grn_ctx *context,
RbGrnContext *rb_grn_context,
grn_obj *column,
RbGrnObject *rb_grn_database)
{
if (rb_grn_context) {
rb_grn_context_close_floating_objects(rb_grn_context);
}
if (!(context->flags & GRN_CTX_PER_DB)) {
grn_ctx_use(context, NULL);
}
}
static void *
recv_from_learner(void *arg)
{
void *zmq_recv_sock;
recv_thd_data *thd = arg;
if ((zmq_recv_sock = zmq_socket(thd->zmq_ctx, ZMQ_SUB))) {
if (!zmq_connect(zmq_recv_sock, thd->recv_endpoint)) {
grn_ctx ctx;
if (!grn_ctx_init(&ctx, 0)) {
if ((!grn_ctx_use(&ctx, db))) {
msgpack_zone *mempool;
if ((mempool = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE))) {
grn_obj cmd_buf;
zmq_pollitem_t items[] = {
{ zmq_recv_sock, 0, ZMQ_POLLIN, 0}
};
GRN_TEXT_INIT(&cmd_buf, 0);
zmq_setsockopt(zmq_recv_sock, ZMQ_SUBSCRIBE, "", 0);
while (loop) {
zmq_poll(items, 1, 10000);
if (items[0].revents & ZMQ_POLLIN) {
recv_handler(&ctx, zmq_recv_sock, mempool, &cmd_buf);
}
}
grn_obj_unlink(&ctx, &cmd_buf);
msgpack_zone_free(mempool);
} else {
print_error("cannot create msgpack zone.");
}
/* db_close */
} else {
print_error("error in grn_db_open() on recv thread.");
}
grn_ctx_fin(&ctx);
} else {
print_error("error in grn_ctx_init() on recv thread.");
}
} else {
print_error("cannot create recv zmq_socket.");
}
} else {
print_error("cannot connect zmq_socket.");
}
return NULL;
}
static ngx_int_t
ngx_http_groonga_context_init(ngx_http_groonga_loc_conf_t *location_conf,
ngx_pool_t *pool,
ngx_log_t *log)
{
ngx_int_t status;
if (location_conf == ngx_http_groonga_current_location_conf) {
return NGX_OK;
}
status = ngx_http_groonga_context_init_logger(location_conf,
pool,
log);
if (status == NGX_ERROR) {
return status;
}
status = ngx_http_groonga_context_init_query_logger(location_conf,
pool,
log);
if (status == NGX_ERROR) {
return status;
}
grn_ctx_use(context, location_conf->database);
grn_cache_current_set(context, location_conf->cache);
/* TODO: It doesn't work yet. We need to implement request timeout
* handler. */
if (location_conf->default_request_timeout_msec == NGX_CONF_UNSET_MSEC) {
grn_set_default_request_timeout(0.0);
} else {
double timeout;
timeout = location_conf->default_request_timeout_msec / 1000.0;
grn_set_default_request_timeout(timeout);
}
ngx_http_groonga_current_location_conf = location_conf;
return status;
}
static ngx_int_t
ngx_http_groonga_handler_create_data(ngx_http_request_t *r,
ngx_http_groonga_handler_data_t **data_return)
{
ngx_int_t rc;
ngx_http_groonga_loc_conf_t *location_conf;
ngx_http_cleanup_t *cleanup;
ngx_http_groonga_handler_data_t *data;
grn_ctx *context;
location_conf = ngx_http_get_module_loc_conf(r, ngx_http_groonga_module);
cleanup = ngx_http_cleanup_add(r, sizeof(ngx_http_groonga_handler_data_t));
cleanup->handler = ngx_http_groonga_handler_cleanup;
data = cleanup->data;
*data_return = data;
context = &(data->context);
grn_ctx_init(context, GRN_NO_FLAGS);
GRN_TEXT_INIT(&(data->head), GRN_NO_FLAGS);
GRN_TEXT_INIT(&(data->body), GRN_NO_FLAGS);
GRN_TEXT_INIT(&(data->foot), GRN_NO_FLAGS);
grn_ctx_use(context, grn_ctx_db(&(location_conf->context)));
rc = ngx_http_groonga_context_check_error(r->connection->log, context);
if (rc != NGX_OK) {
return rc;
}
grn_ctx_recv_handler_set(context,
ngx_http_groonga_context_receive_handler,
data);
return NGX_OK;
}
MRN_API my_bool mroonga_normalize_init(UDF_INIT *initid, UDF_ARGS *args,
char *message)
{
st_mrn_normalize_info *info = NULL;
String *result_str = NULL;
initid->ptr = NULL;
if (!(1 <= args->arg_count && args->arg_count <= 2)) {
sprintf(message,
"mroonga_normalize(): Incorrect number of arguments: %u for 1..2",
args->arg_count);
goto error;
}
if (args->arg_type[0] != STRING_RESULT) {
strcpy(message,
"mroonga_normalize(): The 1st argument must be query as string");
goto error;
}
if (args->arg_count == 2) {
if (args->arg_type[1] != STRING_RESULT) {
strcpy(message,
"mroonga_normalize(): "
"The 2st argument must be normalizer name as string");
goto error;
}
}
initid->maybe_null = 1;
info = (st_mrn_normalize_info *)mrn_my_malloc(sizeof(st_mrn_normalize_info),
MYF(MY_WME | MY_ZEROFILL));
if (!info) {
strcpy(message, "mroonga_normalize(): out of memory");
goto error;
}
info->ctx = mrn_context_pool->pull();
{
const char *current_db_path = MRN_THD_DB_PATH(current_thd);
const char *action;
if (current_db_path) {
action = "open database";
mrn::Database *db;
int error = mrn_db_manager->open(current_db_path, &db);
if (error == 0) {
info->db = db->get();
grn_ctx_use(info->ctx, info->db);
info->use_shared_db = true;
}
} else {
action = "create anonymous database";
info->db = grn_db_create(info->ctx, NULL, NULL);
info->use_shared_db = false;
}
if (!info->db) {
sprintf(message,
"mroonga_normalize(): failed to %s: %s",
action,
info->ctx->errbuf);
goto error;
}
}
if (args->arg_count == 1) {
info->normalizer = grn_ctx_get(info->ctx, DEFAULT_NORMALIZER_NAME, -1);
} else {
info->normalizer = grn_ctx_get(info->ctx, args->args[1], args->lengths[1]);
}
if (!info->normalizer) {
sprintf(message, "mroonga_normalize(): nonexistent normalizer %.*s",
(int)args->lengths[1], args->args[1]);
goto error;
}
info->flags = 0;
result_str = &(info->result_str);
mrn::encoding::set_raw(info->ctx, system_charset_info);
result_str->set_charset(system_charset_info);
initid->ptr = (char *)info;
return FALSE;
error:
if (info) {
if (!info->use_shared_db) {
grn_obj_close(info->ctx, info->db);
}
mrn_context_pool->release(info->ctx);
my_free(info);
}
return TRUE;
}
static int
serve_threads(int nthreads, int port, const char *db_path, void *zmq_ctx,
const char *send_endpoint, const char *recv_endpoint,
const char *log_base_path)
{
int nfd;
uint32_t i;
thd_data thds[nthreads];
if ((nfd = bind_socket(port)) < 0) {
print_error("cannot bind socket. please check port number with netstat.");
return -1;
}
for (i = 0; i < nthreads; i++) {
memset(&thds[i], 0, sizeof(thds[i]));
if (!(thds[i].base = event_init())) {
print_error("error in event_init() on thread %d.", i);
} else {
if (!(thds[i].httpd = evhttp_new(thds[i].base))) {
print_error("error in evhttp_new() on thread %d.", i);
} else {
int r;
if ((r = evhttp_accept_socket(thds[i].httpd, nfd))) {
print_error("error in evhttp_accept_socket() on thread %d.", i);
} else {
if (send_endpoint) {
if (!(thds[i].zmq_sock = zmq_socket(zmq_ctx, ZMQ_PUB))) {
print_error("cannot create zmq_socket.");
} else if (zmq_connect(thds[i].zmq_sock, send_endpoint)) {
print_error("cannot connect zmq_socket.");
zmq_close(thds[i].zmq_sock);
thds[i].zmq_sock = NULL;
} else {
uint64_t hwm = 1;
zmq_setsockopt(thds[i].zmq_sock, ZMQ_HWM, &hwm, sizeof(uint64_t));
}
} else {
thds[i].zmq_sock = NULL;
}
if (!(thds[i].ctx = grn_ctx_open(0))) {
print_error("error in grn_ctx_open() on thread %d.", i);
} else if (grn_ctx_use(thds[i].ctx, db)) {
print_error("error in grn_db_open() on thread %d.", i);
} else {
GRN_TEXT_INIT(&(thds[i].cmd_buf), 0);
thds[i].log_base_path = log_base_path;
thds[i].thread_id = i;
evhttp_set_gencb(thds[i].httpd, generic_handler, &thds[i]);
evhttp_set_timeout(thds[i].httpd, 10);
{
struct timeval tv = {1, 0};
evtimer_set(&(thds[i].pulse), timeout_handler, &thds[i]);
evtimer_add(&(thds[i].pulse), &tv);
}
if ((r = pthread_create(&(thds[i].thd), NULL, dispatch, thds[i].base))) {
print_error("error in pthread_create() on thread %d.", i);
}
}
}
}
}
}
/* recv thread from learner */
if (recv_endpoint) {
recv_thd_data rthd;
rthd.db_path = db_path;
rthd.recv_endpoint = recv_endpoint;
rthd.zmq_ctx = zmq_ctx;
if (pthread_create(&(rthd.thd), NULL, recv_from_learner, &rthd)) {
print_error("error in pthread_create() on thread %d.", i);
}
pthread_join(rthd.thd, NULL);
} else {
while (loop) { sleep(1000); }
}
/* join all httpd thread */
for (i = 0; i < nthreads; i++) {
if (thds[i].thd) {
pthread_join(thds[i].thd, NULL);
}
cleanup_httpd_thread(&(thds[i]));
}
return 0;
}
MRN_API mrn_bool mroonga_snippet_html_init(UDF_INIT *init,
UDF_ARGS *args,
char *message)
{
MRN_DBUG_ENTER_FUNCTION();
mrn_snippet_html_info *info = NULL;
init->ptr = NULL;
if (args->arg_count < 1) {
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_snippet_html(): wrong number of arguments: %u for 1+",
args->arg_count);
goto error;
}
for (unsigned int i = 0; i < args->arg_count; ++i) {
switch (args->arg_type[i]) {
case STRING_RESULT:
/* OK */
break;
case REAL_RESULT:
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_snippet_html(): all arguments must be string: "
"<%u>=<%g>",
i, *((double *)(args->args[i])));
goto error;
break;
case INT_RESULT:
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_snippet_html(): all arguments must be string: "
"<%u>=<%lld>",
i, *((longlong *)(args->args[i])));
goto error;
break;
default:
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_snippet_html(): all arguments must be string: <%u>",
i);
goto error;
break;
}
}
init->maybe_null = 1;
info = (mrn_snippet_html_info *)mrn_my_malloc(sizeof(mrn_snippet_html_info),
MYF(MY_WME | MY_ZEROFILL));
if (!info) {
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_snippet_html(): failed to allocate memory");
goto error;
}
info->ctx = mrn_context_pool->pull();
{
const char *current_db_path = MRN_THD_DB_PATH(current_thd);
const char *action;
if (current_db_path) {
action = "open database";
mrn::Database *db;
int error = mrn_db_manager->open(current_db_path, &db);
if (error == 0) {
info->db = db->get();
grn_ctx_use(info->ctx, info->db);
info->use_shared_db = true;
}
} else {
action = "create anonymous database";
info->db = grn_db_create(info->ctx, NULL, NULL);
info->use_shared_db = false;
}
if (!info->db) {
sprintf(message,
"mroonga_snippet_html(): failed to %s: %s",
action,
info->ctx->errbuf);
goto error;
}
}
info->query_mode.used = false;
if (args->arg_count == 2 &&
args->attribute_lengths[1] == strlen("query") &&
strncmp(args->attributes[1], "query", strlen("query")) == 0) {
info->query_mode.used = true;
info->query_mode.table = NULL;
info->query_mode.default_column = NULL;
}
{
bool all_keywords_are_constant = true;
for (unsigned int i = 1; i < args->arg_count; ++i) {
if (!args->args[i]) {
all_keywords_are_constant = false;
break;
}
}
if (all_keywords_are_constant) {
if (mrn_snippet_html_prepare(info, args, message, &(info->snippet))) {
goto error;
}
} else {
info->snippet = NULL;
}
}
init->ptr = (char *)info;
DBUG_RETURN(false);
error:
if (info) {
if (!info->use_shared_db) {
grn_obj_close(info->ctx, info->db);
}
mrn_context_pool->release(info->ctx);
my_free(info);
}
DBUG_RETURN(true);
}
MRN_API mrn_bool mroonga_query_expand_init(UDF_INIT *init,
UDF_ARGS *args,
char *message)
{
mrn::QueryExpandInfo *info = NULL;
MRN_DBUG_ENTER_FUNCTION();
init->ptr = NULL;
if (args->arg_count != 4) {
sprintf(message,
"mroonga_query_expand(): wrong number of arguments: %u for 4",
args->arg_count);
goto error;
}
if (args->arg_type[0] != STRING_RESULT) {
strcpy(message,
"mroonga_query_expand(): "
"the 1st argument must be table name as string");
goto error;
}
if (args->arg_type[1] != STRING_RESULT) {
strcpy(message,
"mroonga_query_expand(): "
"the 2nd argument must be term column name as string");
goto error;
}
if (args->arg_type[2] != STRING_RESULT) {
strcpy(message,
"mroonga_query_expand(): "
"the 3nd argument must be expanded term column name as string");
goto error;
}
if (args->arg_type[3] != STRING_RESULT) {
strcpy(message,
"mroonga_query_expand(): "
"the 4th argument must be query as string");
goto error;
}
init->maybe_null = 1;
info = static_cast<mrn::QueryExpandInfo *>(
mrn_my_malloc(sizeof(mrn::QueryExpandInfo),
MYF(MY_WME | MY_ZEROFILL)));
if (!info) {
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_query_expand(): failed to allocate memory");
goto error;
}
{
const char *current_db_path = MRN_THD_DB_PATH(current_thd);
if (!current_db_path) {
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_query_expand(): no current database");
goto error;
}
mrn::Database *db;
int error = mrn_db_manager->open(current_db_path, &db);
if (error != 0) {
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_query_expand(): failed to open database: %s",
mrn_db_manager->error_message());
goto error;
}
info->ctx = mrn_context_pool->pull();
grn_ctx_use(info->ctx, db->get());
}
GRN_TEXT_INIT(&(info->expanded_query), 0);
{
const char *table_name = args->args[0];
unsigned int table_name_length = args->lengths[0];
grn_obj *table = grn_ctx_get(info->ctx,
table_name,
table_name_length);
if (!table) {
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_query_expand(): table doesn't exist: <%.*s>",
static_cast<int>(table_name_length),
table_name);
goto error;
}
const char *term_column_name = args->args[1];
unsigned int term_column_name_length = args->lengths[1];
info->term_column = grn_obj_column(info->ctx,
table,
term_column_name,
term_column_name_length);
if (!info->term_column) {
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_query_expand(): term column doesn't exist: <%.*s.%.*s>",
static_cast<int>(table_name_length),
table_name,
static_cast<int>(term_column_name_length),
term_column_name);
goto error;
}
const char *expanded_term_column_name = args->args[2];
unsigned int expanded_term_column_name_length = args->lengths[2];
info->expanded_term_column = grn_obj_column(info->ctx,
table,
expanded_term_column_name,
expanded_term_column_name_length);
if (!info->expanded_term_column) {
snprintf(message, MYSQL_ERRMSG_SIZE,
"mroonga_query_expand(): "
"expanded term column doesn't exist: <%.*s.%.*s>",
static_cast<int>(table_name_length),
table_name,
static_cast<int>(expanded_term_column_name_length),
expanded_term_column_name);
goto error;
}
}
init->ptr = reinterpret_cast<char *>(info);
DBUG_RETURN(false);
error:
mrn_query_expand_info_free(info);
DBUG_RETURN(true);
}
MRN_API my_bool mroonga_command_init(UDF_INIT *initid, UDF_ARGS *args,
char *message)
{
CommandInfo *info = NULL;
initid->ptr = NULL;
if (args->arg_count != 1) {
sprintf(message,
"mroonga_command(): Incorrect number of arguments: %u for 1",
args->arg_count);
goto error;
}
if (args->arg_type[0] != STRING_RESULT) {
strcpy(message,
"mroonga_command(): The 1st argument must be command as string");
goto error;
}
initid->maybe_null = 1;
initid->const_item = 0;
info = (CommandInfo *)mrn_my_malloc(sizeof(CommandInfo),
MYF(MY_WME | MY_ZEROFILL));
if (!info) {
strcpy(message, "mroonga_command(): out of memory");
goto error;
}
info->ctx = mrn_context_pool->pull();
{
const char *current_db_path = MRN_THD_DB_PATH(current_thd);
const char *action;
if (current_db_path) {
action = "open database";
mrn::Database *db;
int error = mrn_db_manager->open(current_db_path, &db);
if (error == 0) {
info->db= db->get();
grn_ctx_use(info->ctx, info->db);
info->use_shared_db = true;
}
} else {
action = "create anonymous database";
info->db = grn_db_create(info->ctx, NULL, NULL);
info->use_shared_db = false;
}
if (!info->db) {
sprintf(message,
"mroonga_command(): failed to %s: %s",
action,
info->ctx->errbuf);
goto error;
}
}
initid->ptr = (char *)info;
return FALSE;
error:
if (info) {
if (!info->use_shared_db) {
grn_obj_close(info->ctx, info->db);
}
mrn_context_pool->release(info->ctx);
my_free(info);
}
return TRUE;
}
