static gboolean
afamqp_worker_insert(LogThrDestDriver *s)
{
AMQPDestDriver *self = (AMQPDestDriver *)s;
gboolean success;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afamqp_dd_connect(self, TRUE);
success = log_queue_pop_head(s->queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
success = afamqp_worker_publish (self, msg);
msg_set_context(NULL);
if (success)
{
stats_counter_inc(s->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
log_queue_push_head(s->queue, msg, &path_options);
return success;
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table, *query_string;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afsql_dd_connect(self);
success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_validate_table(self, msg);
if (!table)
{
/* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
msg_error("Error checking table, disconnecting from database, trying again shortly",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
msg_set_context(NULL);
g_string_free(table, TRUE);
return afsql_dd_insert_fail_handler(self, msg, &path_options);
}
query_string = afsql_dd_construct_query(self, table, msg);
if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
return FALSE;
success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self))
return FALSE;
}
g_string_free(table, TRUE);
g_string_free(query_string, TRUE);
msg_set_context(NULL);
if (!success)
return afsql_dd_insert_fail_handler(self, msg, &path_options);
/* we only ACK if each INSERT is a separate transaction */
if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
return TRUE;
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson, "_id", oid);
g_free (oid);
value_pairs_walk(self->vp,
afmongodb_vp_obj_start,
afmongodb_vp_process_value,
afmongodb_vp_obj_end,
msg, self->seq_num, self->bson);
bson_finish (self->bson);
if (!mongo_sync_cmd_insert_n(self->conn, self->ns, 1,
(const bson **)&self->bson))
{
msg_error("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
log_queue_push_head(self->queue, msg, &path_options);
}
return success;
}
static void
log_threaded_dest_driver_do_insert(LogThrDestDriver *self)
{
LogMessage *msg;
worker_insert_result_t result;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
while (!self->suspended &&
(msg = log_queue_pop_head(self->queue, &path_options)) != NULL)
{
msg_set_context(msg);
log_msg_refcache_start_consumer(msg, &path_options);
result = self->worker.insert(self, msg);
switch (result)
{
case WORKER_INSERT_RESULT_DROP:
log_threaded_dest_driver_message_drop(self, msg);
_disconnect_and_suspend(self);
break;
case WORKER_INSERT_RESULT_ERROR:
self->retries.counter++;
if (self->retries.counter >= self->retries.max)
{
if (self->messages.retry_over)
self->messages.retry_over(self, msg);
log_threaded_dest_driver_message_drop(self, msg);
}
else
{
log_threaded_dest_driver_message_rewind(self, msg);
_disconnect_and_suspend(self);
}
break;
case WORKER_INSERT_RESULT_REWIND:
log_threaded_dest_driver_message_rewind(self, msg);
break;
case WORKER_INSERT_RESULT_SUCCESS:
log_threaded_dest_driver_message_accept(self, msg);
break;
default:
break;
}
msg_set_context(NULL);
log_msg_refcache_stop();
}
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
g_mutex_lock(self->queue_mutex);
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
g_mutex_unlock(self->queue_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson_sel);
bson_reset (self->bson_upd);
bson_reset (self->bson_set);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson_sel, "_id", oid);
g_free (oid);
bson_finish (self->bson_sel);
value_pairs_foreach (self->vp, afmongodb_vp_foreach,
msg, self->seq_num, self->bson_set);
bson_finish (self->bson_set);
bson_append_document (self->bson_upd, "$set", self->bson_set);
bson_finish (self->bson_upd);
if (!mongo_sync_cmd_update (self->conn, self->ns, MONGO_WIRE_FLAG_UPDATE_UPSERT,
self->bson_sel, self->bson_upd))
{
msg_error ("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
g_mutex_lock(self->queue_mutex);
log_queue_push_head(self->queue, msg, &path_options);
g_mutex_unlock(self->queue_mutex);
}
return success;
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table = NULL;
GString *insert_command = NULL;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
if (!afsql_dd_ensure_initialized_connection(self))
return FALSE;
/* connection established, try to insert a message */
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, self->flags & AFSQL_DDF_EXPLICIT_COMMITS);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_ensure_accessible_database_table(self, msg);
if (!table)
{
success = FALSE;
goto out;
}
if (afsql_dd_should_start_new_transaction(self) && !afsql_dd_begin_txn(self))
{
success = FALSE;
goto out;
}
insert_command = afsql_dd_build_insert_command(self, msg, table);
success = afsql_dd_run_query(self, insert_command->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (afsql_dd_should_commit_transaction(self) && !afsql_dd_commit_txn(self))
{
/* Assuming that in case of error, the queue is rewound by afsql_dd_commit_txn() */
g_string_free(insert_command, TRUE);
msg_set_context(NULL);
return FALSE;
}
}
out:
if (table != NULL)
g_string_free(table, TRUE);
if (insert_command != NULL)
g_string_free(insert_command, TRUE);
msg_set_context(NULL);
if (success)
{
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
}
else
{
if (self->failed_message_counter < self->num_retries - 1)
{
if (!afsql_dd_handle_insert_row_error_depending_on_connection_availability(self, msg, &path_options))
return FALSE;
self->failed_message_counter++;
}
else
{
msg_error("Multiple failures while inserting this record into the database, message dropped",
evt_tag_int("attempts", self->num_retries),
NULL);
stats_counter_inc(self->dropped_messages);
log_msg_drop(msg, &path_options);
self->failed_message_counter = 0;
success = TRUE;
}
}
return success;
}
static gboolean
perl_worker_eval(LogThrDestDriver *d)
{
PerlDestDriver *self = (PerlDestDriver *)d;
gboolean success, vp_ok;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
PerlInterpreter *my_perl = self->perl;
int count;
HV *kvmap;
gpointer args[3];
dSP;
success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
ENTER;
SAVETMPS;
PUSHMARK(SP);
kvmap = newHV();
args[0] = self->perl;
args[1] = kvmap;
args[2] = self;
vp_ok = value_pairs_foreach(self->vp, perl_worker_vp_add_one,
msg, self->seq_num, LTZ_SEND,
&self->template_options,
args);
if (!vp_ok && (self->template_options.on_error & ON_ERROR_DROP_MESSAGE))
goto exit;
XPUSHs(sv_2mortal(newRV_noinc((SV *)kvmap)));
PUTBACK;
count = call_pv(self->queue_func_name, G_EVAL | G_SCALAR);
SPAGAIN;
msg_set_context(NULL);
if (SvTRUE(ERRSV))
{
msg_error("Error while calling a Perl function",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_str("script", self->filename),
evt_tag_str("function", self->queue_func_name),
evt_tag_str("error-message", SvPV_nolen(ERRSV)),
NULL);
(void) POPs;
success = FALSE;
}
if (count != 1)
{
msg_error("Too many values returned by a Perl function",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_str("script", self->filename),
evt_tag_str("function", self->queue_func_name),
evt_tag_int("returned-values", count),
evt_tag_int("expected-values", 1),
NULL);
success = FALSE;
}
else
{
int r = POPi;
success = (r != 0);
}
exit:
PUTBACK;
FREETMPS;
LEAVE;
if (success && vp_ok)
{
stats_counter_inc(self->super.stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
stats_counter_inc(self->super.dropped_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
return success;
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table, *query_string;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afsql_dd_connect(self);
g_mutex_lock(self->db_thread_mutex);
/* FIXME: this is a workaround because of the non-proper locking semantics
* of the LogQueue. It might happen that the _queue() method sees 0
* elements in the queue, while the thread is still busy processing the
* previous message. In that case arming the parallel push callback is
* not needed and will cause assertions to fail. This is ugly and should
* be fixed by properly defining the "blocking" semantics of the LogQueue
* object w/o having to rely on user-code messing with parallel push
* callbacks. */
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
g_mutex_unlock(self->db_thread_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_validate_table(self, msg);
if (!table)
{
/* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
msg_error("Error checking table, disconnecting from database, trying again shortly",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
msg_set_context(NULL);
g_string_free(table, TRUE);
return afsql_dd_insert_fail_handler(self, msg, &path_options);
}
query_string = afsql_dd_construct_query(self, table, msg);
if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
return FALSE;
success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self, TRUE))
return FALSE;
}
g_string_free(table, TRUE);
g_string_free(query_string, TRUE);
msg_set_context(NULL);
if (!success)
return afsql_dd_insert_fail_handler(self, msg, &path_options);
/* we only ACK if each INSERT is a separate transaction */
if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
return TRUE;
}
static gboolean
redis_worker_insert(LogThrDestDriver *s)
{
RedisDriver *self = (RedisDriver *)s;
gboolean success;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
redisReply *reply;
const char *argv[5];
size_t argvlen[5];
int argc = 2;
redis_dd_connect(self, TRUE);
if (self->c->err)
return FALSE;
success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
log_template_format(self->key, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->key_str);
if (self->param1)
log_template_format(self->param1, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->param1_str);
if (self->param2)
log_template_format(self->param2, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->param2_str);
argv[0] = self->command->str;
argvlen[0] = self->command->len;
argv[1] = self->key_str->str;
argvlen[1] = self->key_str->len;
if (self->param1)
{
argv[2] = self->param1_str->str;
argvlen[2] = self->param1_str->len;
argc++;
}
if (self->param2)
{
argv[3] = self->param2_str->str;
argvlen[3] = self->param2_str->len;
argc++;
}
reply = redisCommandArgv(self->c, argc, argv, argvlen);
msg_debug("REDIS command sent",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_str("command", self->command->str),
evt_tag_str("key", self->key_str->str),
evt_tag_str("param1", self->param1_str->str),
evt_tag_str("param2", self->param2_str->str),
NULL);
success = TRUE;
freeReplyObject(reply);
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->super.stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
log_queue_push_head(self->super.queue, msg, &path_options);
}
return success;
}
static void
log_source_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
LogSource *self = (LogSource *) s;
LogPathOptions local_options = *path_options;
gint old_window_size;
gint i;
msg_set_context(msg);
if (msg->timestamps[LM_TS_STAMP].tv_sec == -1 || !self->options->keep_timestamp)
msg->timestamps[LM_TS_STAMP] = msg->timestamps[LM_TS_RECVD];
g_assert(msg->timestamps[LM_TS_STAMP].zone_offset != -1);
ack_tracker_track_msg(self->ack_tracker, msg);
/* $HOST setup */
log_source_mangle_hostname(self, msg);
/* $PROGRAM override */
if (self->options->program_override)
{
if (self->options->program_override_len < 0)
self->options->program_override_len = strlen(self->options->program_override);
log_msg_set_value(msg, LM_V_PROGRAM, self->options->program_override, self->options->program_override_len);
}
/* $HOST override */
if (self->options->host_override)
{
if (self->options->host_override_len < 0)
self->options->host_override_len = strlen(self->options->host_override);
log_msg_set_value(msg, LM_V_HOST, self->options->host_override, self->options->host_override_len);
}
/* source specific tags */
if (self->options->tags)
{
for (i = 0; i < self->options->tags->len; i++)
{
log_msg_set_tag_by_id(msg, g_array_index(self->options->tags, LogTagId, i));
}
}
log_msg_set_tag_by_id(msg, self->options->source_group_tag);
/* stats counters */
if (stats_check_level(2))
{
stats_lock();
stats_register_and_increment_dynamic_counter(2, SCS_HOST | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_HOST, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
if (stats_check_level(3))
{
stats_register_and_increment_dynamic_counter(3, SCS_SENDER | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_HOST_FROM, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
stats_register_and_increment_dynamic_counter(3, SCS_PROGRAM | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_PROGRAM, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
}
stats_unlock();
}
stats_syslog_process_message_pri(msg->pri);
/* message setup finished, send it out */
/* NOTE: we start by enabling flow-control, thus we need an acknowledgement */
local_options.ack_needed = TRUE;
log_msg_ref(msg);
log_msg_add_ack(msg, &local_options);
msg->ack_func = log_source_msg_ack;
old_window_size = g_atomic_counter_exchange_and_add(&self->window_size, -1);
/*
* NOTE: this assertion validates that the source is not overflowing its
* own flow-control window size, decreased above, by the atomic statement.
*
* If the _old_ value is zero, that means that the decrement operation
* above has decreased the value to -1.
*/
g_assert(old_window_size > 0);
stats_counter_inc(self->recvd_messages);
stats_counter_set(self->last_message_seen, msg->timestamps[LM_TS_RECVD].tv_sec);
log_pipe_forward_msg(s, msg, &local_options);
msg_set_context(NULL);
if (accurate_nanosleep && self->threaded && self->window_full_sleep_nsec > 0 && !log_source_free_to_send(self))
{
struct timespec ts;
/* wait one 0.1msec in the hope that the buffer clears up */
ts.tv_sec = 0;
ts.tv_nsec = self->window_full_sleep_nsec;
nanosleep(&ts, NULL);
}
}
static void
log_source_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
LogSource *self = (LogSource *) s;
gint i;
msg_set_context(msg);
if (msg->timestamps[LM_TS_STAMP].tv_sec == -1 || !self->options->keep_timestamp)
msg->timestamps[LM_TS_STAMP] = msg->timestamps[LM_TS_RECVD];
g_assert(msg->timestamps[LM_TS_STAMP].zone_offset != -1);
/* $HOST setup */
log_source_mangle_hostname(self, msg);
/* $PROGRAM override */
if (self->options->program_override)
{
if (self->options->program_override_len < 0)
self->options->program_override_len = strlen(self->options->program_override);
log_msg_set_value(msg, LM_V_PROGRAM, self->options->program_override, self->options->program_override_len);
}
/* $HOST override */
if (self->options->host_override)
{
if (self->options->host_override_len < 0)
self->options->host_override_len = strlen(self->options->host_override);
log_msg_set_value(msg, LM_V_HOST, self->options->host_override, self->options->host_override_len);
}
/* source specific tags */
if (self->options->tags)
{
for (i = 0; i < self->options->tags->len; i++)
{
log_msg_set_tag_by_id(msg, g_array_index(self->options->tags, LogTagId, i));
}
}
log_msg_set_tag_by_id(msg, self->options->source_group_tag);
/* stats counters */
if (stats_check_level(2))
{
stats_lock();
stats_register_and_increment_dynamic_counter(2, SCS_HOST | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_HOST, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
if (stats_check_level(3))
{
stats_register_and_increment_dynamic_counter(3, SCS_SENDER | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_HOST_FROM, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
stats_register_and_increment_dynamic_counter(3, SCS_PROGRAM | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_PROGRAM, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
}
stats_unlock();
}
stats_syslog_process_message_pri(msg->pri);
/* message setup finished, send it out */
stats_counter_inc(self->recvd_messages);
stats_counter_set(self->last_message_seen, msg->timestamps[LM_TS_RECVD].tv_sec);
log_pipe_forward_msg(s, msg, path_options);
msg_set_context(NULL);
if (accurate_nanosleep && self->threaded && self->window_full_sleep_nsec > 0 && !log_source_free_to_send(self))
{
struct timespec ts;
/* wait one 0.1msec in the hope that the buffer clears up */
ts.tv_sec = 0;
ts.tv_nsec = self->window_full_sleep_nsec;
nanosleep(&ts, NULL);
}
}
