static gboolean
afamqp_worker_insert(LogThrDestDriver *s)
{
AMQPDestDriver *self = (AMQPDestDriver *)s;
gboolean success;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afamqp_dd_connect(self, TRUE);
success = log_queue_pop_head(s->queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
success = afamqp_worker_publish (self, msg);
msg_set_context(NULL);
if (success)
{
stats_counter_inc(s->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
log_queue_push_head(s->queue, msg, &path_options);
return success;
}
static inline void
afsql_dd_rollback_msg(AFSqlDestDriver *self, LogMessage *msg, LogPathOptions *path_options)
{
if (self->flags & AFSQL_DDF_EXPLICIT_COMMITS)
log_queue_rewind_backlog(self->queue, 1);
else
log_queue_push_head(self->queue, msg, path_options);
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson, "_id", oid);
g_free (oid);
value_pairs_walk(self->vp,
afmongodb_vp_obj_start,
afmongodb_vp_process_value,
afmongodb_vp_obj_end,
msg, self->seq_num, self->bson);
bson_finish (self->bson);
if (!mongo_sync_cmd_insert_n(self->conn, self->ns, 1,
(const bson **)&self->bson))
{
msg_error("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
log_queue_push_head(self->queue, msg, &path_options);
}
return success;
}
static gpointer
_threaded_consume(gpointer st)
{
LogQueue *q = (LogQueue *) st;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
gint loops = 0;
gint msg_count = 0;
/* just to make sure time is properly cached */
iv_init();
while (msg_count < MESSAGES_SUM)
{
gint slept = 0;
msg = NULL;
while((msg = log_queue_pop_head(q, &path_options)) == NULL)
{
struct timespec ns;
/* sleep 1 msec */
ns.tv_sec = 0;
ns.tv_nsec = 1000000;
nanosleep(&ns, NULL);
slept++;
if (slept > 10000)
{
/* slept for more than 10 seconds */
fprintf(stderr, "The wait for messages took too much time, loops=%d, msg_count=%d\n", loops, msg_count);
return GUINT_TO_POINTER(1);
}
}
if ((loops % 10) == 0)
{
/* push the message back to the queue */
log_queue_push_head(q, msg, &path_options);
}
else
{
log_msg_ack(msg, &path_options, AT_PROCESSED);
log_msg_unref(msg);
msg_count++;
}
loops++;
}
iv_deinit();
return NULL;
}
static inline gboolean
afsql_dd_handle_insert_row_error_depending_on_connection_availability(AFSqlDestDriver *self,
LogMessage *msg,
LogPathOptions *path_options)
{
const gchar *dbi_error, *error_message;
if (dbi_conn_ping(self->dbi_ctx) == 1)
{
log_queue_push_head(self->queue, msg, path_options);
return TRUE;
}
if (afsql_dd_is_transaction_handling_enabled(self))
{
error_message = "SQL connection lost in the middle of a transaction,"
" rewinding backlog and starting again";
afsql_dd_handle_transaction_error(self);
}
else
{
error_message = "Error, no SQL connection after failed query attempt";
log_queue_push_head(self->queue, msg, path_options);
}
dbi_conn_error(self->dbi_ctx, &dbi_error);
msg_error(error_message,
evt_tag_str("type", self->type),
evt_tag_str("host", self->host),
evt_tag_str("port", self->port),
evt_tag_str("username", self->user),
evt_tag_str("database", self->database),
evt_tag_str("error", dbi_error),
NULL);
return FALSE;
}
static gboolean
afsql_dd_insert_fail_handler(AFSqlDestDriver *self, LogMessage *msg,
LogPathOptions *path_options)
{
if (self->failed_message_counter < self->num_retries - 1)
{
log_queue_push_head(self->queue, msg, path_options);
/* database connection status sanity check after failed query */
if (dbi_conn_ping(self->dbi_ctx) != 1)
{
const gchar *dbi_error;
dbi_conn_error(self->dbi_ctx, &dbi_error);
msg_error("Error, no SQL connection after failed query attempt",
evt_tag_str("type", self->type),
evt_tag_str("host", self->host),
evt_tag_str("port", self->port),
evt_tag_str("username", self->user),
evt_tag_str("database", self->database),
evt_tag_str("error", dbi_error),
NULL);
return FALSE;
}
self->failed_message_counter++;
return FALSE;
}
msg_error("Multiple failures while inserting this record into the database, message dropped",
evt_tag_int("attempts", self->num_retries),
NULL);
stats_counter_inc(self->dropped_messages);
log_msg_drop(msg, path_options);
self->failed_message_counter = 0;
return TRUE;
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
g_mutex_lock(self->queue_mutex);
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
g_mutex_unlock(self->queue_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson_sel);
bson_reset (self->bson_upd);
bson_reset (self->bson_set);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson_sel, "_id", oid);
g_free (oid);
bson_finish (self->bson_sel);
value_pairs_foreach (self->vp, afmongodb_vp_foreach,
msg, self->seq_num, self->bson_set);
bson_finish (self->bson_set);
bson_append_document (self->bson_upd, "$set", self->bson_set);
bson_finish (self->bson_upd);
if (!mongo_sync_cmd_update (self->conn, self->ns, MONGO_WIRE_FLAG_UPDATE_UPSERT,
self->bson_sel, self->bson_upd))
{
msg_error ("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
g_mutex_lock(self->queue_mutex);
log_queue_push_head(self->queue, msg, &path_options);
g_mutex_unlock(self->queue_mutex);
}
return success;
}
static gboolean
redis_worker_insert(LogThrDestDriver *s)
{
RedisDriver *self = (RedisDriver *)s;
gboolean success;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
redisReply *reply;
const char *argv[5];
size_t argvlen[5];
int argc = 2;
redis_dd_connect(self, TRUE);
if (self->c->err)
return FALSE;
success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
log_template_format(self->key, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->key_str);
if (self->param1)
log_template_format(self->param1, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->param1_str);
if (self->param2)
log_template_format(self->param2, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->param2_str);
argv[0] = self->command->str;
argvlen[0] = self->command->len;
argv[1] = self->key_str->str;
argvlen[1] = self->key_str->len;
if (self->param1)
{
argv[2] = self->param1_str->str;
argvlen[2] = self->param1_str->len;
argc++;
}
if (self->param2)
{
argv[3] = self->param2_str->str;
argvlen[3] = self->param2_str->len;
argc++;
}
reply = redisCommandArgv(self->c, argc, argv, argvlen);
msg_debug("REDIS command sent",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_str("command", self->command->str),
evt_tag_str("key", self->key_str->str),
evt_tag_str("param1", self->param1_str->str),
evt_tag_str("param2", self->param2_str->str),
NULL);
success = TRUE;
freeReplyObject(reply);
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->super.stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
log_queue_push_head(self->super.queue, msg, &path_options);
}
return success;
}