static void
log_threaded_dest_driver_do_work(gpointer data)
{
LogThrDestDriver *self = (LogThrDestDriver *)data;
gint timeout_msec = 0;
self->suspended = FALSE;
log_threaded_dest_driver_stop_watches(self);
if (!self->worker.connected)
{
__connect(self);
}
else if (log_queue_check_items(self->queue, &timeout_msec,
log_threaded_dest_driver_message_became_available_in_the_queue,
self, NULL))
{
log_threaded_dest_driver_do_insert(self);
if (!self->suspended)
log_threaded_dest_driver_start_watches(self);
}
else if (timeout_msec != 0)
{
log_queue_reset_parallel_push(self->queue);
iv_validate_now();
self->timer_throttle.expires = iv_now;
timespec_add_msec(&self->timer_throttle.expires, timeout_msec);
iv_timer_register(&self->timer_throttle);
}
}
static gpointer
afmongodb_worker_thread (gpointer arg)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)arg;
msg_debug ("Worker thread started",
evt_tag_str("driver", self->super.super.id),
NULL);
afmongodb_dd_connect(self, FALSE);
self->ns = g_strconcat (self->db, ".", self->coll, NULL);
self->current_value = g_string_sized_new(256);
self->bson = bson_new_sized(4096);
while (!self->writer_thread_terminate)
{
g_mutex_lock(self->suspend_mutex);
if (self->writer_thread_suspended)
{
g_cond_timed_wait(self->writer_thread_wakeup_cond,
self->suspend_mutex,
&self->writer_thread_suspend_target);
self->writer_thread_suspended = FALSE;
g_mutex_unlock(self->suspend_mutex);
}
else if (!log_queue_check_items(self->queue, NULL, afmongodb_dd_message_became_available_in_the_queue, self, NULL))
{
g_cond_wait(self->writer_thread_wakeup_cond, self->suspend_mutex);
g_mutex_unlock(self->suspend_mutex);
}
else
g_mutex_unlock(self->suspend_mutex);
if (self->writer_thread_terminate)
break;
if (!afmongodb_worker_insert (self))
{
afmongodb_dd_disconnect(self);
afmongodb_dd_suspend(self);
}
}
afmongodb_dd_disconnect(self);
g_free (self->ns);
g_string_free (self->current_value, TRUE);
bson_free (self->bson);
msg_debug ("Worker thread finished",
evt_tag_str("driver", self->super.super.id),
NULL);
return NULL;
}
static gpointer
afamqp_worker_thread(gpointer arg)
{
AMQPDestDriver *self = (AMQPDestDriver *) arg;
msg_debug("Worker thread started",
evt_tag_str("driver", self->super.super.id), NULL);
afamqp_dd_connect(self, FALSE);
while (!self->writer_thread_terminate)
{
g_mutex_lock(self->suspend_mutex);
if (self->writer_thread_suspended)
{
g_cond_timed_wait(self->writer_thread_wakeup_cond,
self->suspend_mutex, &self->writer_thread_suspend_target);
self->writer_thread_suspended = FALSE;
g_mutex_unlock(self->suspend_mutex);
}
else if (!log_queue_check_items(self->queue, NULL, afamqp_dd_message_became_available_in_the_queue, self, NULL))
{
g_cond_wait(self->writer_thread_wakeup_cond, self->suspend_mutex);
g_mutex_unlock(self->suspend_mutex);
}
else
g_mutex_unlock(self->suspend_mutex);
if (self->writer_thread_terminate)
break;
if (!afamqp_worker_insert(self))
{
afamqp_dd_disconnect(self);
afamqp_dd_suspend(self);
}
}
afamqp_dd_disconnect(self);
msg_debug("Worker thread finished",
evt_tag_str("driver", self->super.super.id), NULL);
return NULL;
}
static void
log_writer_update_watches(LogWriter *self)
{
gint fd;
GIOCondition cond = 0;
gboolean partial_batch;
gint timeout_msec = 0;
main_loop_assert_main_thread();
/* NOTE: we either start the suspend_timer or enable the fd_watch. The two MUST not happen at the same time. */
if (log_proto_prepare(self->proto, &fd, &cond) ||
self->flush_waiting_for_timeout ||
log_queue_check_items(self->queue, self->options->flush_lines, &partial_batch, &timeout_msec,
(LogQueuePushNotifyFunc) log_writer_schedule_update_watches, self, NULL))
{
/* flush_lines number of element is already available and throttle would permit us to send. */
log_writer_update_fd_callbacks(self, cond);
}
else if (partial_batch || timeout_msec)
{
/* few elements are available, but less than flush_lines, we need to start a timer to initiate a flush */
log_writer_update_fd_callbacks(self, 0);
self->flush_waiting_for_timeout = TRUE;
log_writer_arm_suspend_timer(self, (void (*)(void *)) log_writer_update_watches, timeout_msec ? timeout_msec : self->options->flush_timeout);
}
else
{
/* no elements or no throttle space, wait for a wakeup by the queue
* when the required number of items are added. see the
* log_queue_check_items and its parallel_push argument above
*/
log_writer_update_fd_callbacks(self, 0);
}
}
/**
* afsql_dd_database_thread:
*
* This is the thread inserting records into the database.
**/
static void
afsql_dd_database_thread(gpointer arg)
{
AFSqlDestDriver *self = (AFSqlDestDriver *) arg;
msg_verbose("Database thread started",
evt_tag_str("driver", self->super.super.id),
NULL);
while (!self->db_thread_terminate)
{
g_mutex_lock(self->db_thread_mutex);
if (self->db_thread_suspended)
{
afsql_dd_wait_for_suspension_wakeup(self);
/* we loop back to check if the thread was requested to terminate */
}
else if (!log_queue_check_items(self->queue, NULL, afsql_dd_message_became_available_in_the_queue, self, NULL))
{
/* we have nothing to INSERT into the database, let's wait we get some new stuff */
if (self->flush_lines_queued > 0)
{
if (!afsql_dd_commit_txn(self))
{
afsql_dd_disconnect(self);
afsql_dd_suspend(self);
g_mutex_unlock(self->db_thread_mutex);
continue;
}
}
else if (!self->db_thread_terminate)
{
g_cond_wait(self->db_thread_wakeup_cond, self->db_thread_mutex);
}
/* we loop back to check if the thread was requested to terminate */
}
g_mutex_unlock(self->db_thread_mutex);
if (self->db_thread_terminate)
break;
if (!afsql_dd_insert_db(self))
{
afsql_dd_disconnect(self);
afsql_dd_suspend(self);
}
}
while (log_queue_get_length(self->queue) > 0)
{
if (!afsql_dd_insert_db(self))
{
goto exit;
}
}
if (self->flush_lines_queued > 0)
{
/* we can't do anything with the return value here. if commit isn't
* successful, we get our backlog back, but we have no chance
* submitting that back to the SQL engine.
*/
afsql_dd_commit_txn(self);
}
exit:
afsql_dd_disconnect(self);
msg_verbose("Database thread finished",
evt_tag_str("driver", self->super.super.id),
NULL);
}
