static void
_push_tail(LogQueue *s, LogMessage *msg, const LogPathOptions *path_options)
{
LogQueueDisk *self = (LogQueueDisk *) s;
LogPathOptions local_options = *path_options;
g_static_mutex_lock(&self->super.lock);
if (self->push_tail)
{
if (self->push_tail(self, msg, &local_options, path_options))
{
log_queue_push_notify (&self->super);
stats_counter_inc(self->super.stored_messages);
log_msg_ack(msg, &local_options, AT_PROCESSED);
log_msg_unref(msg);
g_static_mutex_unlock(&self->super.lock);
return;
}
}
stats_counter_inc (self->super.dropped_messages);
if (path_options->flow_control_requested)
log_msg_ack(msg, path_options, AT_SUSPENDED);
else
log_msg_drop(msg, path_options, AT_PROCESSED);
g_static_mutex_unlock(&self->super.lock);
}
/*
* Can only run from the output thread.
*/
static void
log_queue_fifo_ack_backlog(LogQueue *s, gint n)
{
LogQueueFifo *self = (LogQueueFifo *) s;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
gint i;
log_queue_assert_output_thread(s);
for (i = 0; i < n && self->qbacklog_len > 0; i++)
{
LogMessageQueueNode *node;
node = iv_list_entry(self->qbacklog.next, LogMessageQueueNode, list);
msg = node->msg;
path_options.ack_needed = node->ack_needed;
iv_list_del(&node->list);
log_msg_free_queue_node(node);
self->qbacklog_len--;
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
}
static void
log_dest_group_queue(LogPipe *s, LogMessage *msg, gint path_flags)
{
LogDestGroup *self = (LogDestGroup *) s;
LogDriver *p;
if ((path_flags & PF_FLOW_CTL_OFF) == 0)
{
log_msg_ref(msg);
log_msg_ack_block_start(msg, log_dest_group_ack, NULL);
}
for (p = self->drivers; p; p = p->drv_next)
{
#if 1 /* start dongshu */
if(p->processed_limit !=0 &&
((p->processed_messages > 0) && (p->processed_messages % p->processed_limit == 0))){
p->flush = TRUE;
}
#endif /* end */
if ((path_flags & PF_FLOW_CTL_OFF) == 0)
log_msg_ack_block_inc(msg);
log_pipe_queue(&p->super, log_msg_ref(msg), path_flags); /* call affile_dd_queue()... */
p->processed_messages++;
}
(*self->processed_messages)++;
if ((path_flags & PF_FLOW_CTL_OFF) == 0)
log_msg_ack(msg);
log_msg_unref(msg);
}
static void
log_center_ack(LogMessage *msg, gpointer user_data)
{
log_msg_ack_block_end(msg);
log_msg_ack(msg);
log_msg_unref(msg);
}
static void
afinet_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
#if SYSLOG_NG_ENABLE_SPOOF_SOURCE
AFInetDestDriver *self = (AFInetDestDriver *) s;
/* NOTE: this code should probably become a LogTransport instance so that
* spoofed packets are also going through the LogWriter queue */
if (self->spoof_source && self->lnet_ctx && msg->saddr && (msg->saddr->sa.sa_family == AF_INET || msg->saddr->sa.sa_family == AF_INET6) && log_writer_opened(self->super.writer))
{
gboolean success = FALSE;
g_assert(self->super.transport_mapper->sock_type == SOCK_DGRAM);
g_static_mutex_lock(&self->lnet_lock);
if (!self->lnet_buffer)
self->lnet_buffer = g_string_sized_new(self->spoof_source_maxmsglen);
log_writer_format_log(self->super.writer, msg, self->lnet_buffer);
if (self->lnet_buffer->len > self->spoof_source_maxmsglen)
g_string_truncate(self->lnet_buffer, self->spoof_source_maxmsglen);
switch (self->super.dest_addr->sa.sa_family)
{
case AF_INET:
success = afinet_dd_construct_ipv4_packet(self, msg, self->lnet_buffer);
break;
#if SYSLOG_NG_ENABLE_IPV6
case AF_INET6:
success = afinet_dd_construct_ipv6_packet(self, msg, self->lnet_buffer);
break;
#endif
default:
g_assert_not_reached();
}
if (success)
{
if (libnet_write(self->lnet_ctx) >= 0)
{
/* we have finished processing msg */
log_msg_ack(msg, path_options, AT_PROCESSED);
log_msg_unref(msg);
g_static_mutex_unlock(&self->lnet_lock);
return;
}
else
{
msg_error("Error sending raw frame",
evt_tag_str("error", libnet_geterror(self->lnet_ctx)));
}
}
g_static_mutex_unlock(&self->lnet_lock);
}
#endif
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
gpointer
threaded_consume(gpointer st)
{
LogQueue *q = (LogQueue *) st;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
gboolean success;
gint i;
for (i = 0; i < 100000; i++)
{
g_static_mutex_lock(&threaded_lock);
msg = NULL;
success = log_queue_pop_head(q, &msg, &path_options, FALSE);
g_static_mutex_unlock(&threaded_lock);
g_assert(!success || (success && msg != NULL));
if (!success)
{
fprintf(stderr, "Queue didn't return enough messages: i=%d\n", i);
return GUINT_TO_POINTER(1);
}
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
return NULL;
}
static gboolean
afamqp_worker_insert(LogThrDestDriver *s)
{
AMQPDestDriver *self = (AMQPDestDriver *)s;
gboolean success;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afamqp_dd_connect(self, TRUE);
success = log_queue_pop_head(s->queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
success = afamqp_worker_publish (self, msg);
msg_set_context(NULL);
if (success)
{
stats_counter_inc(s->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
log_queue_push_head(s->queue, msg, &path_options);
return success;
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table, *query_string;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afsql_dd_connect(self);
success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_validate_table(self, msg);
if (!table)
{
/* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
msg_error("Error checking table, disconnecting from database, trying again shortly",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
msg_set_context(NULL);
g_string_free(table, TRUE);
return afsql_dd_insert_fail_handler(self, msg, &path_options);
}
query_string = afsql_dd_construct_query(self, table, msg);
if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
return FALSE;
success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self))
return FALSE;
}
g_string_free(table, TRUE);
g_string_free(query_string, TRUE);
msg_set_context(NULL);
if (!success)
return afsql_dd_insert_fail_handler(self, msg, &path_options);
/* we only ACK if each INSERT is a separate transaction */
if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
return TRUE;
}
static void
_add_message_to_qout(LogQueueDiskNonReliable *self, LogMessage *msg, LogPathOptions *path_options)
{
/* NOTE: we always generate flow-control disabled entries into
* qout, they only get there via backlog rewind */
g_queue_push_tail (self->qout, msg);
g_queue_push_tail (self->qout, LOG_PATH_OPTIONS_FOR_BACKLOG);
log_msg_ack (msg, path_options, AT_PROCESSED);
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson, "_id", oid);
g_free (oid);
value_pairs_walk(self->vp,
afmongodb_vp_obj_start,
afmongodb_vp_process_value,
afmongodb_vp_obj_end,
msg, self->seq_num, self->bson);
bson_finish (self->bson);
if (!mongo_sync_cmd_insert_n(self->conn, self->ns, 1,
(const bson **)&self->bson))
{
msg_error("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
log_queue_push_head(self->queue, msg, &path_options);
}
return success;
}
static void
dummy_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
DummyDestDriver *self = (DummyDestDriver *) s;
msg_notice("Dummy plugin received a message",
evt_tag_str("msg", log_msg_get_value(msg, LM_V_MESSAGE, NULL)),
evt_tag_int("opt", self->opt),
NULL);
log_msg_ack(msg, path_options);
log_msg_unref(msg);
}
static void
afinet_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options)
{
#if ENABLE_SPOOF_SOURCE
AFInetDestDriver *self = (AFInetDestDriver *) s;
if (self->spoof_source && self->lnet_ctx && msg->saddr && (msg->saddr->sa.sa_family == AF_INET || msg->saddr->sa.sa_family == AF_INET6))
{
gboolean success = FALSE;
GString *msg_line = g_string_sized_new(256);
g_assert((self->super.flags & AFSOCKET_DGRAM) != 0);
log_writer_format_log((LogWriter *) self->super.writer, msg, msg_line);
switch (self->super.dest_addr->sa.sa_family)
{
case AF_INET:
success = afinet_dd_construct_ipv4_packet(self, msg, msg_line);
break;
#if ENABLE_IPV6
case AF_INET6:
success = afinet_dd_construct_ipv6_packet(self, msg, msg_line);
break;
#endif
default:
g_assert_not_reached();
}
if (success)
{
if (libnet_write(self->lnet_ctx) >= 0)
{
/* we have finished processing msg */
log_msg_ack(msg, path_options);
log_msg_unref(msg);
g_string_free(msg_line, TRUE);
return;
}
else
{
msg_error("Error sending raw frame",
evt_tag_str("error", libnet_geterror(self->lnet_ctx)),
NULL);
}
}
g_string_free(msg_line, TRUE);
}
#endif
log_pipe_forward_msg(s, msg, path_options);
}
static gpointer
_threaded_consume(gpointer st)
{
LogQueue *q = (LogQueue *) st;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
gint loops = 0;
gint msg_count = 0;
/* just to make sure time is properly cached */
iv_init();
while (msg_count < MESSAGES_SUM)
{
gint slept = 0;
msg = NULL;
while((msg = log_queue_pop_head(q, &path_options)) == NULL)
{
struct timespec ns;
/* sleep 1 msec */
ns.tv_sec = 0;
ns.tv_nsec = 1000000;
nanosleep(&ns, NULL);
slept++;
if (slept > 10000)
{
/* slept for more than 10 seconds */
fprintf(stderr, "The wait for messages took too much time, loops=%d, msg_count=%d\n", loops, msg_count);
return GUINT_TO_POINTER(1);
}
}
if ((loops % 10) == 0)
{
/* push the message back to the queue */
log_queue_push_head(q, msg, &path_options);
}
else
{
log_msg_ack(msg, &path_options, AT_PROCESSED);
log_msg_unref(msg);
msg_count++;
}
loops++;
}
iv_deinit();
return NULL;
}
void
send_some_messages(LogQueue *q, gint n, gboolean use_app_acks)
{
gint i;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
for (i = 0; i < n; i++)
{
log_queue_pop_head(q, &msg, &path_options, use_app_acks);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
}
void
send_some_messages(LogQueue *q, gint n)
{
gint i;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
for (i = 0; i < n; i++)
{
msg = log_queue_pop_head(q, &path_options);
log_msg_ack(msg, &path_options, AT_PROCESSED);
log_msg_unref(msg);
}
}
static void
_free_queue (GQueue *q)
{
while (!g_queue_is_empty (q))
{
LogMessage *lm;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
lm = g_queue_pop_head (q);
POINTER_TO_LOG_PATH_OPTIONS (g_queue_pop_head (q), &path_options);
log_msg_ack (lm, &path_options, AT_PROCESSED);
log_msg_unref (lm);
}
g_queue_free (q);
}
static void
_ack_backlog (LogQueueDisk *s, guint num_msg_to_ack)
{
LogQueueDiskNonReliable *self = (LogQueueDiskNonReliable *) s;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
guint i;
for (i = 0; i < num_msg_to_ack; i++)
{
if (self->qbacklog->length < ITEM_NUMBER_PER_MESSAGE)
return;
msg = g_queue_pop_head (self->qbacklog);
POINTER_TO_LOG_PATH_OPTIONS (g_queue_pop_head (self->qbacklog), &path_options);
log_msg_unref (msg);
log_msg_ack (msg, &path_options, AT_PROCESSED);
}
}
static gboolean
_save_queue(QDisk *self, GQueue *q, gint64 *q_ofs, gint32 *q_len)
{
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
SerializeArchive *sa;
GString *serialized;
if (q->length == 0)
{
*q_ofs = 0;
*q_len = 0;
return TRUE;
}
serialized = g_string_sized_new(4096);
sa = serialize_string_archive_new(serialized);
while ((msg = g_queue_pop_head(q)))
{
/* NOTE: we might have some flow-controlled events on qout, when
* saving them to disk, we ack them, they are restored as
* non-flow-controlled entries later, but then we've saved them to
* disk anyway. */
POINTER_TO_LOG_PATH_OPTIONS(g_queue_pop_head(q), &path_options);
log_msg_serialize(msg, sa);
log_msg_ack(msg, &path_options, AT_PROCESSED);
log_msg_unref(msg);
}
serialize_archive_free(sa);
*q_ofs = lseek(self->fd, 0, SEEK_END);
if (!pwrite_strict(self->fd, serialized->str, serialized->len, *q_ofs))
{
msg_error("Error writing in-memory buffer of disk-queue to disk",
evt_tag_str("filename", self->filename),
evt_tag_errno("error", errno));
g_string_free(serialized, TRUE);
return FALSE;
}
*q_len = serialized->len;
g_string_free(serialized, TRUE);
return TRUE;
}
static void
log_queue_fifo_free_queue(struct iv_list_head *q)
{
while (!iv_list_empty(q))
{
LogMessageQueueNode *node;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg;
node = iv_list_entry(q->next, LogMessageQueueNode, list);
iv_list_del(&node->list);
path_options.ack_needed = node->ack_needed;
msg = node->msg;
log_msg_free_queue_node(node);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
}
static void
afuser_dd_queue(LogPipe *s, LogMessage *msg, gint path_flags)
{
AFUserDestDriver *self = (AFUserDestDriver *) s;
gchar buf[8192];
struct utmp *ut;
g_snprintf(buf, sizeof(buf), "%s %s %s\n", msg->date->str, msg->host->str, msg->msg->str);
while ((ut = getutent()))
{
#if HAVE_MODERN_UTMP
if (ut->ut_type == USER_PROCESS &&
((self->username->len == 1 &&
self->username->str[0] == '*') ||
(self->username->len <= sizeof(ut->ut_user) &&
memcmp(self->username->str, ut->ut_user, self->username->len) == 0)))
#else
if ((self->username->len == 1 &&
self->username->str[0] == '*') ||
(self->username->len <= sizeof(ut->ut_name) &&
memcmp(self->username->str, ut->ut_name, self->username->len) == 0))
#endif
{
char line[128];
int fd;
strcpy(line, "/dev/");
/* CID 12139 : Out-of-bounds access(overrun-buffer-arg: Overrunning array "line" of 128 bytes by passing it to a function which accesses it at byte offset 128 using argument "128U") */
//strncat(line, ut->ut_line, sizeof(line));
strncat(line, ut->ut_line, sizeof(ut->ut_line));
/* CID 12139 end*/
fd = open(line, O_NOCTTY | O_APPEND | O_WRONLY);
if (fd != -1)
{
write(fd, buf, strlen(buf));
close(fd);
}
}
}
log_msg_ack(msg);
log_msg_unref(msg);
}
/*
* Can only run from the output thread.
*/
static void
log_queue_fifo_ack_backlog(LogQueue *s, gint rewind_count)
{
LogQueueFifo *self = (LogQueueFifo *) s;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
gint pos;
for (pos = 0; pos < rewind_count && self->qbacklog_len > 0; pos++)
{
LogMessageQueueNode *node;
node = iv_list_entry(self->qbacklog.next, LogMessageQueueNode, list);
msg = node->msg;
iv_list_del(&node->list);
self->qbacklog_len--;
path_options.ack_needed = node->ack_needed;
log_msg_ack(msg, &path_options, AT_PROCESSED);
log_msg_free_queue_node(node);
log_msg_unref(msg);
}
}
static void
_move_messages_from_overflow(LogQueueDiskNonReliable *self)
{
LogMessage *msg;
LogPathOptions path_options;
/* move away as much entries from the overflow area as possible */
while (_has_movable_message(self))
{
msg = g_queue_pop_head (self->qoverflow);
POINTER_TO_LOG_PATH_OPTIONS (g_queue_pop_head (self->qoverflow), &path_options);
if (qdisk_get_length (self->super.qdisk) == 0 && HAS_SPACE_IN_QUEUE(self->qout))
{
/* we can skip qdisk, go straight to qout */
g_queue_push_tail (self->qout, msg);
g_queue_push_tail (self->qout, LOG_PATH_OPTIONS_FOR_BACKLOG);
log_msg_ref (msg);
}
else
{
if (!self->super.write_message(&self->super, msg))
{
/* oops, altough there seemed to be some free space available,
* we failed saving this message, (it might have needed more
* than 4096 bytes than we ensured), push back and break
*/
g_queue_push_head (self->qoverflow, LOG_PATH_OPTIONS_TO_POINTER (&path_options));
g_queue_push_head (self->qoverflow, msg);
log_msg_ref (msg);
break;
}
}
log_msg_ack (msg, &path_options, AT_PROCESSED);
log_msg_unref (msg);
}
}
static gboolean
redis_worker_insert(LogThrDestDriver *s)
{
RedisDriver *self = (RedisDriver *)s;
gboolean success;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
redisReply *reply;
const char *argv[5];
size_t argvlen[5];
int argc = 2;
redis_dd_connect(self, TRUE);
if (self->c->err)
return FALSE;
success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
log_template_format(self->key, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->key_str);
if (self->param1)
log_template_format(self->param1, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->param1_str);
if (self->param2)
log_template_format(self->param2, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->param2_str);
argv[0] = self->command->str;
argvlen[0] = self->command->len;
argv[1] = self->key_str->str;
argvlen[1] = self->key_str->len;
if (self->param1)
{
argv[2] = self->param1_str->str;
argvlen[2] = self->param1_str->len;
argc++;
}
if (self->param2)
{
argv[3] = self->param2_str->str;
argvlen[3] = self->param2_str->len;
argc++;
}
reply = redisCommandArgv(self->c, argc, argv, argvlen);
msg_debug("REDIS command sent",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_str("command", self->command->str),
evt_tag_str("key", self->key_str->str),
evt_tag_str("param1", self->param1_str->str),
evt_tag_str("param2", self->param2_str->str),
NULL);
success = TRUE;
freeReplyObject(reply);
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->super.stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
log_queue_push_head(self->super.queue, msg, &path_options);
}
return success;
}
static void
afuser_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options)
{
AFUserDestDriver *self = (AFUserDestDriver *) s;
gchar buf[8192];
struct utmp *ut;
GString *timestamp;
time_t now;
now = msg->timestamps[LM_TS_RECVD].time.tv_sec;
if (self->disable_until && self->disable_until > now)
goto finish;
timestamp = g_string_sized_new(0);
log_stamp_format(&msg->timestamps[LM_TS_STAMP], timestamp, TS_FMT_FULL, -1, 0);
g_snprintf(buf, sizeof(buf), "%s %s %s\n",
timestamp->str,
log_msg_get_value(msg, LM_V_HOST, NULL),
log_msg_get_value(msg, LM_V_MESSAGE, NULL));
g_string_free(timestamp, TRUE);
/* NOTE: there's a private implementations of getutent in utils.c on Systems which do not provide one. */
while ((ut = getutent()))
{
#if HAVE_MODERN_UTMP
if (ut->ut_type == USER_PROCESS &&
((self->username->len == 1 &&
self->username->str[0] == '*') ||
(self->username->len <= sizeof(ut->ut_user) &&
memcmp(self->username->str, ut->ut_user, self->username->len) == 0)))
#else
if ((self->username->len == 1 &&
self->username->str[0] == '*') ||
(self->username->len <= sizeof(ut->ut_name) &&
memcmp(self->username->str, ut->ut_name, self->username->len) == 0))
#endif
{
gchar line[128];
gchar *p = line;
int fd;
if (ut->ut_line[0] != '/')
{
strcpy(line, "/dev/");
p = line + 5;
}
else
line[0] = 0;
strncpy(p, ut->ut_line, sizeof(line) - (p - line));
fd = open(line, O_NOCTTY | O_APPEND | O_WRONLY | O_NONBLOCK);
if (fd != -1)
{
alarm_set(10);
if (write(fd, buf, strlen(buf)) < 0 && errno == EINTR && alarm_has_fired())
{
msg_notice("Writing to the user terminal has blocked for 10 seconds, disabling for 10 minutes",
evt_tag_str("user", self->username->str),
NULL);
self->disable_until = now + 600;
}
alarm_cancel();
close(fd);
}
}
}
endutent();
finish:
log_msg_ack(msg, path_options);
log_msg_unref(msg);
}
static void
log_center_queue(LogPipe *s, LogMessage *msg, gint path_flags)
{
LogCenter *self = (LogCenter *) s;
gboolean match, fallbacks, have_fallbacks = 1;
gint ci, fi, di;
(*self->received_messages)++;
afinter_postpone_mark(self->cfg->mark_freq);
log_msg_ref(msg);
log_msg_ack_block_start(msg, log_center_ack, NULL);
for (match = 0, fallbacks = 0; !match && have_fallbacks && (fallbacks <= 1); fallbacks++)
{
have_fallbacks = 0;
for (ci = 0; ci < self->cfg->connections->len; ci++)
{
LogConnection *conn = (LogConnection *) g_ptr_array_index(self->cfg->connections, ci);
if (!fallbacks && (conn->flags & LC_FALLBACK))
{
have_fallbacks = 1;
continue;
}
else if (fallbacks && !(conn->flags & LC_FALLBACK))
{
continue;
}
if (!(conn->flags & LC_CATCHALL))
{
/* check source */
if (!g_hash_table_lookup(conn->source_cache, msg->source_group->name->str))
{
goto next_connection;
}
}
else
{
/* catchall, every source matches */
;
}
for (fi = 0; fi < conn->filters->len; fi++)
{
LogEndpoint *ep = (LogEndpoint *) g_ptr_array_index(conn->filters, fi);
LogFilterRule *f;
f = (LogFilterRule *) ep->ref;
if (!log_filter_rule_eval(f, msg))
{
goto next_connection;
}
}
match = 1;
for (di = 0; di < conn->destinations->len; di++)
{
LogEndpoint *ep = (LogEndpoint *) g_ptr_array_index(conn->destinations, di);
LogDestGroup *dest;
if (conn->flags & LC_FLOW_CONTROL)
log_msg_ack_block_inc(msg);
dest = (LogDestGroup *) ep->ref;
log_pipe_queue(&dest->super, log_msg_ref(msg), path_flags | ((conn->flags & LC_FLOW_CONTROL) ? 0 : PF_FLOW_CTL_OFF));
(*self->queued_messages)++;
}
if (conn->flags & LC_FINAL)
{
break;
}
next_connection:
;
}
}
/* our own ack */
log_msg_ack(msg);
}
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
gboolean success;
guint8 *oid;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afmongodb_dd_connect(self, TRUE);
g_mutex_lock(self->queue_mutex);
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
g_mutex_unlock(self->queue_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
bson_reset (self->bson_sel);
bson_reset (self->bson_upd);
bson_reset (self->bson_set);
oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
bson_append_oid (self->bson_sel, "_id", oid);
g_free (oid);
bson_finish (self->bson_sel);
value_pairs_foreach (self->vp, afmongodb_vp_foreach,
msg, self->seq_num, self->bson_set);
bson_finish (self->bson_set);
bson_append_document (self->bson_upd, "$set", self->bson_set);
bson_finish (self->bson_upd);
if (!mongo_sync_cmd_update (self->conn, self->ns, MONGO_WIRE_FLAG_UPDATE_UPSERT,
self->bson_sel, self->bson_upd))
{
msg_error ("Network error while inserting into MongoDB",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
success = FALSE;
}
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
g_mutex_lock(self->queue_mutex);
log_queue_push_head(self->queue, msg, &path_options);
g_mutex_unlock(self->queue_mutex);
}
return success;
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table = NULL;
GString *insert_command = NULL;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
if (!afsql_dd_ensure_initialized_connection(self))
return FALSE;
/* connection established, try to insert a message */
success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, self->flags & AFSQL_DDF_EXPLICIT_COMMITS);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_ensure_accessible_database_table(self, msg);
if (!table)
{
success = FALSE;
goto out;
}
if (afsql_dd_should_start_new_transaction(self) && !afsql_dd_begin_txn(self))
{
success = FALSE;
goto out;
}
insert_command = afsql_dd_build_insert_command(self, msg, table);
success = afsql_dd_run_query(self, insert_command->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (afsql_dd_should_commit_transaction(self) && !afsql_dd_commit_txn(self))
{
/* Assuming that in case of error, the queue is rewound by afsql_dd_commit_txn() */
g_string_free(insert_command, TRUE);
msg_set_context(NULL);
return FALSE;
}
}
out:
if (table != NULL)
g_string_free(table, TRUE);
if (insert_command != NULL)
g_string_free(insert_command, TRUE);
msg_set_context(NULL);
if (success)
{
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
}
else
{
if (self->failed_message_counter < self->num_retries - 1)
{
if (!afsql_dd_handle_insert_row_error_depending_on_connection_availability(self, msg, &path_options))
return FALSE;
self->failed_message_counter++;
}
else
{
msg_error("Multiple failures while inserting this record into the database, message dropped",
evt_tag_int("attempts", self->num_retries),
NULL);
stats_counter_inc(self->dropped_messages);
log_msg_drop(msg, &path_options);
self->failed_message_counter = 0;
success = TRUE;
}
}
return success;
}
/**
* afsql_dd_insert_db:
*
* This function is running in the database thread
*
* Returns: FALSE to indicate that the connection should be closed and
* this destination suspended for time_reopen() time.
**/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
GString *table, *query_string;
LogMessage *msg;
gboolean success;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
afsql_dd_connect(self);
g_mutex_lock(self->db_thread_mutex);
/* FIXME: this is a workaround because of the non-proper locking semantics
* of the LogQueue. It might happen that the _queue() method sees 0
* elements in the queue, while the thread is still busy processing the
* previous message. In that case arming the parallel push callback is
* not needed and will cause assertions to fail. This is ugly and should
* be fixed by properly defining the "blocking" semantics of the LogQueue
* object w/o having to rely on user-code messing with parallel push
* callbacks. */
log_queue_reset_parallel_push(self->queue);
success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
g_mutex_unlock(self->db_thread_mutex);
if (!success)
return TRUE;
msg_set_context(msg);
table = afsql_dd_validate_table(self, msg);
if (!table)
{
/* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
msg_error("Error checking table, disconnecting from database, trying again shortly",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
msg_set_context(NULL);
g_string_free(table, TRUE);
return afsql_dd_insert_fail_handler(self, msg, &path_options);
}
query_string = afsql_dd_construct_query(self, table, msg);
if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
return FALSE;
success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
if (success && self->flush_lines_queued != -1)
{
self->flush_lines_queued++;
if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self, TRUE))
return FALSE;
}
g_string_free(table, TRUE);
g_string_free(query_string, TRUE);
msg_set_context(NULL);
if (!success)
return afsql_dd_insert_fail_handler(self, msg, &path_options);
/* we only ACK if each INSERT is a separate transaction */
if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
step_sequence_number(&self->seq_num);
self->failed_message_counter = 0;
return TRUE;
}
static gboolean
perl_worker_eval(LogThrDestDriver *d)
{
PerlDestDriver *self = (PerlDestDriver *)d;
gboolean success, vp_ok;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
PerlInterpreter *my_perl = self->perl;
int count;
HV *kvmap;
gpointer args[3];
dSP;
success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
ENTER;
SAVETMPS;
PUSHMARK(SP);
kvmap = newHV();
args[0] = self->perl;
args[1] = kvmap;
args[2] = self;
vp_ok = value_pairs_foreach(self->vp, perl_worker_vp_add_one,
msg, self->seq_num, LTZ_SEND,
&self->template_options,
args);
if (!vp_ok && (self->template_options.on_error & ON_ERROR_DROP_MESSAGE))
goto exit;
XPUSHs(sv_2mortal(newRV_noinc((SV *)kvmap)));
PUTBACK;
count = call_pv(self->queue_func_name, G_EVAL | G_SCALAR);
SPAGAIN;
msg_set_context(NULL);
if (SvTRUE(ERRSV))
{
msg_error("Error while calling a Perl function",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_str("script", self->filename),
evt_tag_str("function", self->queue_func_name),
evt_tag_str("error-message", SvPV_nolen(ERRSV)),
NULL);
(void) POPs;
success = FALSE;
}
if (count != 1)
{
msg_error("Too many values returned by a Perl function",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_str("script", self->filename),
evt_tag_str("function", self->queue_func_name),
evt_tag_int("returned-values", count),
evt_tag_int("expected-values", 1),
NULL);
success = FALSE;
}
else
{
int r = POPi;
success = (r != 0);
}
exit:
PUTBACK;
FREETMPS;
LEAVE;
if (success && vp_ok)
{
stats_counter_inc(self->super.stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
stats_counter_inc(self->super.dropped_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
return success;
}
