gboolean
log_threaded_dest_driver_deinit_method(LogPipe *s)
{
LogThrDestDriver *self = (LogThrDestDriver *)s;
log_queue_reset_parallel_push(self->queue);
log_queue_set_counters(self->queue, NULL, NULL);
cfg_persist_config_add(log_pipe_get_config(s),
log_threaded_dest_driver_format_seqnum_for_persist(self),
GINT_TO_POINTER(self->seq_num), NULL, FALSE);
stats_lock();
stats_unregister_counter(self->stats_source | SCS_DESTINATION, self->super.super.id,
self->format.stats_instance(self),
SC_TYPE_STORED, &self->stored_messages);
stats_unregister_counter(self->stats_source | SCS_DESTINATION, self->super.super.id,
self->format.stats_instance(self),
SC_TYPE_DROPPED, &self->dropped_messages);
stats_unregister_counter(self->stats_source | SCS_DESTINATION, self->super.super.id,
self->format.stats_instance(self),
SC_TYPE_PROCESSED, &self->processed_messages);
stats_unlock();
if (!log_dest_driver_deinit_method(s))
return FALSE;
return TRUE;
}
static void
log_threaded_dest_driver_do_work(gpointer data)
{
LogThrDestDriver *self = (LogThrDestDriver *)data;
gint timeout_msec = 0;
self->suspended = FALSE;
log_threaded_dest_driver_stop_watches(self);
if (!self->worker.connected)
{
__connect(self);
}
else if (log_queue_check_items(self->queue, &timeout_msec,
log_threaded_dest_driver_message_became_available_in_the_queue,
self, NULL))
{
log_threaded_dest_driver_do_insert(self);
if (!self->suspended)
log_threaded_dest_driver_start_watches(self);
}
else if (timeout_msec != 0)
{
log_queue_reset_parallel_push(self->queue);
iv_validate_now();
self->timer_throttle.expires = iv_now;
timespec_add_msec(&self->timer_throttle.expires, timeout_msec);
iv_timer_register(&self->timer_throttle);
}
}
/**
* afsql_dd_begin_txn:
*
* Commit SQL transaction.
*
* NOTE: This function can only be called from the database thread.
**/
static gboolean
afsql_dd_commit_txn(AFSqlDestDriver *self, gboolean lock)
{
gboolean success;
success = afsql_dd_run_query(self, "COMMIT", FALSE, NULL);
if (lock)
g_mutex_lock(self->db_thread_mutex);
/* FIXME: this is a workaround because of the non-proper locking semantics
* of the LogQueue. It might happen that the _queue() method sees 0
* elements in the queue, while the thread is still busy processing the
* previous message. In that case arming the parallel push callback is
* not needed and will cause assertions to fail. This is ugly and should
* be fixed by properly defining the "blocking" semantics of the LogQueue
* object w/o having to rely on user-code messing with parallel push
* callbacks. */
log_queue_reset_parallel_push(self->queue);
if (success)
{
log_queue_ack_backlog(self->queue, self->flush_lines_queued);
}
else
{
msg_notice("SQL transaction commit failed, rewinding backlog and starting again",
NULL);
log_queue_rewind_backlog(self->queue);
}
if (lock)
g_mutex_unlock(self->db_thread_mutex);
self->flush_lines_queued = 0;
return success;
}
static void
_disconnect_and_suspend(LogThrDestDriver *self)
{
self->suspended = TRUE;
__disconnect(self);
log_queue_reset_parallel_push(self->queue);
log_threaded_dest_driver_suspend(self);
}
static void
afmongodb_dd_queue_notify(gpointer user_data)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)user_data;
g_mutex_lock(self->queue_mutex);
g_cond_signal(self->writer_thread_wakeup_cond);
log_queue_reset_parallel_push(self->queue);
g_mutex_unlock(self->queue_mutex);
}
static void
log_writer_stop_watches(LogWriter *self)
{
if (self->watches_running)
{
if (iv_timer_registered(&self->suspend_timer))
iv_timer_unregister(&self->suspend_timer);
if (iv_fd_registered(&self->fd_watch))
iv_fd_unregister(&self->fd_watch);
if (iv_task_registered(&self->immed_io_task))
iv_task_unregister(&self->immed_io_task);
log_queue_reset_parallel_push(self->queue);
self->watches_running = FALSE;
}
}
static void
__connect(LogThrDestDriver *self)
{
self->worker.connected = TRUE;
if (self->worker.connect)
{
self->worker.connected = self->worker.connect(self);
}
if (!self->worker.connected)
{
log_queue_reset_parallel_push(self->queue);
log_threaded_dest_driver_suspend(self);
}
else
{
log_threaded_dest_driver_start_watches(self);
}
}
static gboolean
afamqp_dd_deinit(LogPipe *s)
{
AMQPDestDriver *self = (AMQPDestDriver *) s;
afamqp_dd_stop_thread(self);
log_queue_reset_parallel_push(self->queue);
log_queue_set_counters(self->queue, NULL, NULL);
stats_lock();
stats_unregister_counter(SCS_AMQP | SCS_DESTINATION,
self->super.super.id, afamqp_dd_format_stats_instance(self),
SC_TYPE_STORED, &self->stored_messages);
stats_unregister_counter(SCS_AMQP | SCS_DESTINATION,
self->super.super.id, afamqp_dd_format_stats_instance(self),
SC_TYPE_DROPPED, &self->dropped_messages);
stats_unlock();
if (!log_dest_driver_deinit_method(s))
return FALSE;
return TRUE;
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
g_mutex_lock(self->queue_mutex);
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
g_mutex_unlock(self->queue_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson_sel);
bson_reset (self->bson_upd);
bson_reset (self->bson_set);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson_sel, "_id", oid);
g_free (oid);
bson_finish (self->bson_sel);
value_pairs_foreach (self->vp, afmongodb_vp_foreach,
msg, self->seq_num, self->bson_set);
bson_finish (self->bson_set);
bson_append_document (self->bson_upd, "$set", self->bson_set);
bson_finish (self->bson_upd);
if (!mongo_sync_cmd_update (self->conn, self->ns, MONGO_WIRE_FLAG_UPDATE_UPSERT,
self->bson_sel, self->bson_upd))
{
msg_error ("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
g_mutex_lock(self->queue_mutex);
log_queue_push_head(self->queue, msg, &path_options);
g_mutex_unlock(self->queue_mutex);
}
return success;
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table, *query_string;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afsql_dd_connect(self);
g_mutex_lock(self->db_thread_mutex);
/* FIXME: this is a workaround because of the non-proper locking semantics
* of the LogQueue. It might happen that the _queue() method sees 0
* elements in the queue, while the thread is still busy processing the
* previous message. In that case arming the parallel push callback is
* not needed and will cause assertions to fail. This is ugly and should
* be fixed by properly defining the "blocking" semantics of the LogQueue
* object w/o having to rely on user-code messing with parallel push
* callbacks. */
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
g_mutex_unlock(self->db_thread_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_validate_table(self, msg);
if (!table)
{
/* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
msg_error("Error checking table, disconnecting from database, trying again shortly",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
msg_set_context(NULL);
g_string_free(table, TRUE);
return afsql_dd_insert_fail_handler(self, msg, &path_options);
}
query_string = afsql_dd_construct_query(self, table, msg);
if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
return FALSE;
success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self, TRUE))
return FALSE;
}
g_string_free(table, TRUE);
g_string_free(query_string, TRUE);
msg_set_context(NULL);
if (!success)
return afsql_dd_insert_fail_handler(self, msg, &path_options);
/* we only ACK if each INSERT is a separate transaction */
if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
return TRUE;
}
