static gpointer
afmongodb_worker_thread (gpointer arg)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)arg;
msg_debug ("Worker thread started",
evt_tag_str("driver", self->super.super.id),
NULL);
afmongodb_dd_connect(self, FALSE);
self->ns = g_strconcat (self->db, ".", self->coll, NULL);
self->current_value = g_string_sized_new(256);
self->bson = bson_new_sized(4096);
while (!self->writer_thread_terminate)
{
g_mutex_lock(self->suspend_mutex);
if (self->writer_thread_suspended)
{
g_cond_timed_wait(self->writer_thread_wakeup_cond,
self->suspend_mutex,
&self->writer_thread_suspend_target);
self->writer_thread_suspended = FALSE;
g_mutex_unlock(self->suspend_mutex);
}
else if (!log_queue_check_items(self->queue, NULL, afmongodb_dd_message_became_available_in_the_queue, self, NULL))
{
g_cond_wait(self->writer_thread_wakeup_cond, self->suspend_mutex);
g_mutex_unlock(self->suspend_mutex);
}
else
g_mutex_unlock(self->suspend_mutex);
if (self->writer_thread_terminate)
break;
if (!afmongodb_worker_insert (self))
{
afmongodb_dd_disconnect(self);
afmongodb_dd_suspend(self);
}
}
afmongodb_dd_disconnect(self);
g_free (self->ns);
g_string_free (self->current_value, TRUE);
bson_free (self->bson);
msg_debug ("Worker thread finished",
evt_tag_str("driver", self->super.super.id),
NULL);
return NULL;
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson, "_id", oid);
g_free (oid);
value_pairs_walk(self->vp,
afmongodb_vp_obj_start,
afmongodb_vp_process_value,
afmongodb_vp_obj_end,
msg, self->seq_num, self->bson);
bson_finish (self->bson);
if (!mongo_sync_cmd_insert_n(self->conn, self->ns, 1,
(const bson **)&self->bson))
{
msg_error("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
log_queue_push_head(self->queue, msg, &path_options);
}
return success;
}
static void
afmongodb_worker_thread_init(LogThrDestDriver *d)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)d;
afmongodb_dd_connect(self, FALSE);
self->ns = g_strconcat (self->db, ".", self->coll, NULL);
self->current_value = g_string_sized_new(256);
self->bson = bson_new_sized(4096);
}
static gpointer
afmongodb_worker_thread (gpointer arg)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)arg;
gboolean success;
msg_debug ("Worker thread started",
evt_tag_str("driver", self->super.super.id),
NULL);
success = afmongodb_dd_connect(self, FALSE);
self->ns = g_strconcat (self->db, ".", self->coll, NULL);
self->current_value = g_string_sized_new(256);
self->bson_sel = bson_new_sized(64);
self->bson_upd = bson_new_sized(512);
self->bson_set = bson_new_sized(512);
while (!self->writer_thread_terminate)
{
g_mutex_lock(self->suspend_mutex);
if (self->writer_thread_suspended)
{
g_cond_timed_wait(self->writer_thread_wakeup_cond,
self->suspend_mutex,
&self->writer_thread_suspend_target);
self->writer_thread_suspended = FALSE;
g_mutex_unlock(self->suspend_mutex);
}
else
{
g_mutex_unlock(self->suspend_mutex);
g_mutex_lock(self->queue_mutex);
if (log_queue_get_length(self->queue) == 0)
{
g_cond_wait(self->writer_thread_wakeup_cond, self->queue_mutex);
}
g_mutex_unlock(self->queue_mutex);
}
if (self->writer_thread_terminate)
break;
if (!afmongodb_worker_insert (self))
{
afmongodb_dd_disconnect(self);
afmongodb_dd_suspend(self);
}
}
afmongodb_dd_disconnect(self);
g_free (self->ns);
g_string_free (self->current_value, TRUE);
bson_free (self->bson_sel);
bson_free (self->bson_upd);
bson_free (self->bson_set);
msg_debug ("Worker thread finished",
evt_tag_str("driver", self->super.super.id),
NULL);
return NULL;
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
g_mutex_lock(self->queue_mutex);
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
g_mutex_unlock(self->queue_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson_sel);
bson_reset (self->bson_upd);
bson_reset (self->bson_set);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson_sel, "_id", oid);
g_free (oid);
bson_finish (self->bson_sel);
value_pairs_foreach (self->vp, afmongodb_vp_foreach,
msg, self->seq_num, self->bson_set);
bson_finish (self->bson_set);
bson_append_document (self->bson_upd, "$set", self->bson_set);
bson_finish (self->bson_upd);
if (!mongo_sync_cmd_update (self->conn, self->ns, MONGO_WIRE_FLAG_UPDATE_UPSERT,
self->bson_sel, self->bson_upd))
{
msg_error ("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
g_mutex_lock(self->queue_mutex);
log_queue_push_head(self->queue, msg, &path_options);
g_mutex_unlock(self->queue_mutex);
}
return success;
}
static worker_insert_result_t
afmongodb_worker_insert (LogThrDestDriver *s, LogMessage *msg)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)s;
gboolean success;
gboolean drop_silently = self->template_options.on_error & ON_ERROR_SILENT;
if (!afmongodb_dd_connect(self, TRUE))
return WORKER_INSERT_RESULT_NOT_CONNECTED;
bson_reset (self->bson);
success = value_pairs_walk(self->vp,
afmongodb_vp_obj_start,
afmongodb_vp_process_value,
afmongodb_vp_obj_end,
msg, self->super.seq_num,
LTZ_SEND,
&self->template_options,
self);
bson_finish (self->bson);
if (!success)
{
if (!drop_silently)
{
msg_error("Failed to format message for MongoDB, dropping message",
evt_tag_value_pairs("message", self->vp, msg,
self->super.seq_num,
LTZ_SEND, &self->template_options),
evt_tag_str("driver", self->super.super.super.id),
NULL);
}
return WORKER_INSERT_RESULT_DROP;
}
else
{
msg_debug("Outgoing message to MongoDB destination",
evt_tag_value_pairs("message", self->vp, msg,
self->super.seq_num,
LTZ_SEND, &self->template_options),
evt_tag_str("driver", self->super.super.super.id),
NULL);
if (!mongo_sync_cmd_insert_n(self->conn, self->ns, 1,
(const bson **)&self->bson))
{
msg_error("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->super.time_reopen),
evt_tag_str("reason", mongo_sync_conn_get_last_error(self->conn)),
evt_tag_str("driver", self->super.super.super.id),
NULL);
success = FALSE;
}
}
if (!success && (errno == ENOTCONN))
return WORKER_INSERT_RESULT_NOT_CONNECTED;
if (!success)
return WORKER_INSERT_RESULT_ERROR;
return WORKER_INSERT_RESULT_SUCCESS;
}
