/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table, *query_string;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afsql_dd_connect(self);
success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_validate_table(self, msg);
if (!table)
{
/* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
msg_error("Error checking table, disconnecting from database, trying again shortly",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
msg_set_context(NULL);
g_string_free(table, TRUE);
return afsql_dd_insert_fail_handler(self, msg, &path_options);
}
query_string = afsql_dd_construct_query(self, table, msg);
if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
return FALSE;
success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self))
return FALSE;
}
g_string_free(table, TRUE);
g_string_free(query_string, TRUE);
msg_set_context(NULL);
if (!success)
return afsql_dd_insert_fail_handler(self, msg, &path_options);
/* we only ACK if each INSERT is a separate transaction */
if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
return TRUE;
}
/**
* afsql_dd_database_thread:
*
* This is the thread inserting records into the database.
**/
static void
afsql_dd_database_thread(gpointer arg)
{
AFSqlDestDriver *self = (AFSqlDestDriver *) arg;
msg_verbose("Database thread started",
evt_tag_str("driver", self->super.super.id),
NULL);
while (!self->db_thread_terminate)
{
g_mutex_lock(self->db_thread_mutex);
if (self->db_thread_suspended)
{
afsql_dd_wait_for_suspension_wakeup(self);
/* we loop back to check if the thread was requested to terminate */
}
else if (!log_queue_check_items(self->queue, NULL, afsql_dd_message_became_available_in_the_queue, self, NULL))
{
/* we have nothing to INSERT into the database, let's wait we get some new stuff */
if (self->flush_lines_queued > 0)
{
if (!afsql_dd_commit_txn(self))
{
afsql_dd_disconnect(self);
afsql_dd_suspend(self);
g_mutex_unlock(self->db_thread_mutex);
continue;
}
}
else if (!self->db_thread_terminate)
{
g_cond_wait(self->db_thread_wakeup_cond, self->db_thread_mutex);
}
/* we loop back to check if the thread was requested to terminate */
}
g_mutex_unlock(self->db_thread_mutex);
if (self->db_thread_terminate)
break;
if (!afsql_dd_insert_db(self))
{
afsql_dd_disconnect(self);
afsql_dd_suspend(self);
}
}
while (log_queue_get_length(self->queue) > 0)
{
if (!afsql_dd_insert_db(self))
{
goto exit;
}
}
if (self->flush_lines_queued > 0)
{
/* we can't do anything with the return value here. if commit isn't
* successful, we get our backlog back, but we have no chance
* submitting that back to the SQL engine.
*/
afsql_dd_commit_txn(self);
}
exit:
afsql_dd_disconnect(self);
msg_verbose("Database thread finished",
evt_tag_str("driver", self->super.super.id),
NULL);
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table = NULL;
GString *insert_command = NULL;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
if (!afsql_dd_ensure_initialized_connection(self))
return FALSE;
/* connection established, try to insert a message */
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, self->flags & AFSQL_DDF_EXPLICIT_COMMITS);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_ensure_accessible_database_table(self, msg);
if (!table)
{
success = FALSE;
goto out;
}
if (afsql_dd_should_start_new_transaction(self) && !afsql_dd_begin_txn(self))
{
success = FALSE;
goto out;
}
insert_command = afsql_dd_build_insert_command(self, msg, table);
success = afsql_dd_run_query(self, insert_command->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (afsql_dd_should_commit_transaction(self) && !afsql_dd_commit_txn(self))
{
/* Assuming that in case of error, the queue is rewound by afsql_dd_commit_txn() */
g_string_free(insert_command, TRUE);
msg_set_context(NULL);
return FALSE;
}
}
out:
if (table != NULL)
g_string_free(table, TRUE);
if (insert_command != NULL)
g_string_free(insert_command, TRUE);
msg_set_context(NULL);
if (success)
{
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
}
else
{
if (self->failed_message_counter < self->num_retries - 1)
{
if (!afsql_dd_handle_insert_row_error_depending_on_connection_availability(self, msg, &path_options))
return FALSE;
self->failed_message_counter++;
}
else
{
msg_error("Multiple failures while inserting this record into the database, message dropped",
evt_tag_int("attempts", self->num_retries),
NULL);
stats_counter_inc(self->dropped_messages);
log_msg_drop(msg, &path_options);
self->failed_message_counter = 0;
success = TRUE;
}
}
return success;
}
/**
* afsql_dd_database_thread:
*
* This is the thread inserting records into the database.
**/
static gpointer
afsql_dd_database_thread(gpointer arg)
{
AFSqlDestDriver *self = (AFSqlDestDriver *) arg;
msg_verbose("Database thread started",
evt_tag_str("driver", self->super.super.id),
NULL);
while (!self->db_thread_terminate)
{
g_mutex_lock(self->db_thread_mutex);
if (self->db_thread_suspended)
{
/* we got suspended, probably because of a connection error,
* during this time we only get wakeups if we need to be
* terminated. */
if (!self->db_thread_terminate)
g_cond_timed_wait(self->db_thread_wakeup_cond, self->db_thread_mutex, &self->db_thread_suspend_target);
self->db_thread_suspended = FALSE;
g_mutex_unlock(self->db_thread_mutex);
/* we loop back to check if the thread was requested to terminate */
}
else if (log_queue_get_length(self->queue) == 0)
{
/* we have nothing to INSERT into the database, let's wait we get some new stuff */
if (self->flush_lines_queued > 0 && self->flush_timeout > 0)
{
GTimeVal flush_target;
g_get_current_time(&flush_target);
g_time_val_add(&flush_target, self->flush_timeout * 1000);
if (!self->db_thread_terminate && !g_cond_timed_wait(self->db_thread_wakeup_cond, self->db_thread_mutex, &flush_target))
{
/* timeout elapsed */
if (!afsql_dd_commit_txn(self, FALSE))
{
afsql_dd_disconnect(self);
afsql_dd_suspend(self);
g_mutex_unlock(self->db_thread_mutex);
continue;
}
}
}
else if (!self->db_thread_terminate)
{
g_cond_wait(self->db_thread_wakeup_cond, self->db_thread_mutex);
}
g_mutex_unlock(self->db_thread_mutex);
/* we loop back to check if the thread was requested to terminate */
}
else
g_mutex_unlock(self->db_thread_mutex);
if (self->db_thread_terminate)
break;
if (!afsql_dd_insert_db(self))
{
afsql_dd_disconnect(self);
afsql_dd_suspend(self);
}
}
if (self->flush_lines_queued > 0)
{
/* we can't do anything with the return value here. if commit isn't
* successful, we get our backlog back, but we have no chance
* submitting that back to the SQL engine.
*/
afsql_dd_commit_txn(self, TRUE);
}
afsql_dd_disconnect(self);
msg_verbose("Database thread finished",
evt_tag_str("driver", self->super.super.id),
NULL);
return NULL;
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table, *query_string;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afsql_dd_connect(self);
g_mutex_lock(self->db_thread_mutex);
/* FIXME: this is a workaround because of the non-proper locking semantics
* of the LogQueue. It might happen that the _queue() method sees 0
* elements in the queue, while the thread is still busy processing the
* previous message. In that case arming the parallel push callback is
* not needed and will cause assertions to fail. This is ugly and should
* be fixed by properly defining the "blocking" semantics of the LogQueue
* object w/o having to rely on user-code messing with parallel push
* callbacks. */
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
g_mutex_unlock(self->db_thread_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_validate_table(self, msg);
if (!table)
{
/* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
msg_error("Error checking table, disconnecting from database, trying again shortly",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
msg_set_context(NULL);
g_string_free(table, TRUE);
return afsql_dd_insert_fail_handler(self, msg, &path_options);
}
query_string = afsql_dd_construct_query(self, table, msg);
if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
return FALSE;
success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self, TRUE))
return FALSE;
}
g_string_free(table, TRUE);
g_string_free(query_string, TRUE);
msg_set_context(NULL);
if (!success)
return afsql_dd_insert_fail_handler(self, msg, &path_options);
/* we only ACK if each INSERT is a separate transaction */
if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
return TRUE;
}