static void
_prepare_eof_test(LogQueueDiskReliable *dq, LogMessage **msg1, LogMessage **msg2)
{
LogPathOptions local_options = LOG_PATH_OPTIONS_INIT;
gint64 start_pos = TEST_DISKQ_SIZE;
*msg1 = log_msg_new_mark();
*msg2 = log_msg_new_mark();
(*msg1)->ack_func = _dummy_ack;
log_msg_add_ack(*msg1, &local_options);
(*msg2)->ack_func = _dummy_ack;
log_msg_add_ack(*msg2, &local_options);
dq->super.qdisk->hdr->write_head = start_pos;
dq->super.qdisk->hdr->read_head = QDISK_RESERVED_SPACE + mark_message_serialized_size + 1;
dq->super.qdisk->hdr->backlog_head = dq->super.qdisk->hdr->read_head;
log_queue_push_tail(&dq->super.super, *msg1, &local_options);
log_queue_push_tail(&dq->super.super, *msg2, &local_options);
assert_gint(dq->qreliable->length, NUMBER_MESSAGES_IN_QUEUE(2), ASSERTION_ERROR("Messages aren't in qreliable"));
assert_gint64(dq->super.qdisk->hdr->write_head, QDISK_RESERVED_SPACE + mark_message_serialized_size, ASSERTION_ERROR("Bad write head"));
assert_gint(num_of_ack, 0, ASSERTION_ERROR("Messages are acked"));
dq->super.qdisk->hdr->read_head = start_pos;
dq->super.qdisk->hdr->backlog_head = dq->super.qdisk->hdr->read_head;
}
/*
* The method make the following situation
* the backlog contains 6 messages
* the qbacklog contains 3 messages,
* but messages in qbacklog are the end of the backlog
*/
void
_prepare_rewind_backlog_test(LogQueueDiskReliable *dq, gint64 *start_pos)
{
gint i;
for (i = 0; i < 8; i++)
{
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *mark_message;
mark_message = log_msg_new_mark();
mark_message->ack_func = _dummy_ack;
log_queue_push_tail(&dq->super.super, mark_message, &path_options);
}
/* Lets read the messages and leave them in the backlog */
for (i = 0; i < 8; i++)
{
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
msg = log_queue_pop_head(&dq->super.super, &path_options);
log_msg_unref(msg);
}
/* Ack the messages which are not in the qbacklog */
log_queue_ack_backlog(&dq->super.super, 5);
assert_gint(dq->qbacklog->length, NUMBER_MESSAGES_IN_QUEUE(3), ASSERTION_ERROR("Incorrect number of items in the qbacklog"));
*start_pos = dq->super.qdisk->hdr->read_head;
/* Now write 3 more messages and read them from buffer
* the number of messages in the qbacklog should not be changed
* The backlog should contain 6 messages
* from these 6 messages 3 messages are cached in the qbacklog
* No readable messages are in the queue
*/
for (i = 0; i < 3; i++)
{
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *mark_message;
mark_message = log_msg_new_mark();
mark_message->ack_func = _dummy_ack;
log_queue_push_tail(&dq->super.super, mark_message, &path_options);
mark_message = log_queue_pop_head(&dq->super.super, &path_options);
assert_gint(dq->qreliable->length, 0,
ASSERTION_ERROR("Incorrect number of items in the qreliable"));
assert_gint(dq->qbacklog->length, NUMBER_MESSAGES_IN_QUEUE(3),
ASSERTION_ERROR("Incorrect number of items in the qbacklog"));
log_msg_unref(mark_message);
}
assert_gint(dq->super.qdisk->hdr->backlog_len, 6,
ASSERTION_ERROR("Incorrect number of messages in the backlog"));
assert_gint(dq->super.qdisk->hdr->length, 0,
ASSERTION_ERROR("Reliable diskq isn't empty"));
}
static void
log_writer_queue(LogPipe *s, LogMessage *lm, const LogPathOptions *path_options)
{
LogWriter *self = (LogWriter *) s;
if (self->options->suppress > 0 && log_writer_last_msg_check(self, lm, path_options))
return;
stats_counter_inc(self->processed_messages);
if (!log_queue_push_tail(self->queue, lm, path_options))
{
/* drop incoming message, we must ack here, otherwise the sender might
* block forever, however this should not happen unless the sum of
* window_sizes of sources feeding this writer exceeds log_fifo_size
* or if flow control is not turned on.
*/
/* we don't send a message here since the system is draining anyway */
stats_counter_inc(self->dropped_messages);
msg_debug("Destination queue full, dropping message",
evt_tag_int("queue_len", log_queue_get_length(self->queue)),
evt_tag_int("mem_fifo_size", self->options->mem_fifo_size),
NULL);
log_msg_drop(lm, path_options);
return;
}
}
static void
test_rewind_over_eof(LogQueueDiskReliable *dq)
{
LogMessage *msg3 = log_msg_new_mark();
LogMessage *read_message3;
LogPathOptions local_options = LOG_PATH_OPTIONS_INIT;
msg3->ack_func = _dummy_ack;
log_queue_push_tail(&dq->super.super, msg3, &local_options);
gint64 previous_read_head = dq->super.qdisk->hdr->read_head;
read_message3 = log_queue_pop_head(&dq->super.super, &local_options);
assert_true(read_message3 != NULL, ASSERTION_ERROR("Can't read message from queue"));
assert_gint(dq->super.qdisk->hdr->read_head, dq->super.qdisk->hdr->write_head, ASSERTION_ERROR("Read head in bad position"));
assert_true(msg3 == read_message3, ASSERTION_ERROR("Message 3 isn't read from qreliable"));
log_msg_unref(read_message3);
log_queue_rewind_backlog(&dq->super.super, 1);
assert_gint(dq->super.qdisk->hdr->read_head, previous_read_head, ASSERTION_ERROR("Read head is corrupted"));
read_message3 = log_queue_pop_head(&dq->super.super, &local_options);
assert_true(read_message3 != NULL, ASSERTION_ERROR("Can't read message from queue"));
assert_gint(dq->super.qdisk->hdr->read_head, dq->super.qdisk->hdr->write_head, ASSERTION_ERROR("Read head in bad position"));
assert_true(msg3 == read_message3, ASSERTION_ERROR("Message 3 isn't read from qreliable"));
log_msg_drop(msg3, &local_options, AT_PROCESSED);
}
gpointer
threaded_feed(gpointer st)
{
LogQueue *q = (LogQueue *) st;
char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
gint i;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg;
for (i = 0; i < 100000; i++)
{
msg = log_msg_new(msg_str, strlen(msg_str), g_sockaddr_inet_new("10.10.10.10", 1010), 0, NULL, -1);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
g_static_mutex_lock(&threaded_lock);
if (!log_queue_push_tail(q, msg, &path_options))
{
fprintf(stderr, "Queue unable to consume enough messages: %d\n", fed_messages);
return GUINT_TO_POINTER(1);
}
g_static_mutex_unlock(&threaded_lock);
}
return NULL;
}
/*
* NOTE: suppress_lock must be held.
*/
static void
log_writer_last_msg_flush(LogWriter *self)
{
LogMessage *m;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
gchar buf[1024];
gssize len;
const gchar *p;
msg_debug("Suppress timer elapsed, emitting suppression summary",
NULL);
m = log_msg_new_empty();
m->timestamps[LM_TS_STAMP] = m->timestamps[LM_TS_RECVD];
m->pri = self->last_msg->pri;
m->flags = LF_INTERNAL | LF_LOCAL;
p = log_msg_get_value(self->last_msg, LM_V_HOST, &len);
log_msg_set_value(m, LM_V_HOST, p, len);
p = log_msg_get_value(self->last_msg, LM_V_PROGRAM, &len);
log_msg_set_value(m, LM_V_PROGRAM, p, len);
len = g_snprintf(buf, sizeof(buf), "Last message '%.20s' repeated %d times, suppressed by syslog-ng on %s",
log_msg_get_value(self->last_msg, LM_V_MESSAGE, NULL),
self->last_msg_count,
get_local_hostname(NULL));
log_msg_set_value(m, LM_V_MESSAGE, buf, len);
path_options.ack_needed = FALSE;
log_queue_push_tail(self->queue, m, &path_options);
log_writer_last_msg_release(self);
}
static void
afmongodb_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)s;
gboolean queue_was_empty;
LogPathOptions local_options;
if (!path_options->flow_control_requested)
path_options = log_msg_break_ack(msg, path_options, &local_options);
g_mutex_lock(self->queue_mutex);
self->last_msg_stamp = cached_g_current_time_sec ();
queue_was_empty = log_queue_get_length(self->queue) == 0;
g_mutex_unlock(self->queue_mutex);
log_msg_add_ack(msg, path_options);
log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);
g_mutex_lock(self->suspend_mutex);
if (queue_was_empty && !self->writer_thread_suspended)
{
g_mutex_lock(self->queue_mutex);
log_queue_set_parallel_push(self->queue, 1, afmongodb_dd_queue_notify, self, NULL);
g_mutex_unlock(self->queue_mutex);
}
g_mutex_unlock(self->suspend_mutex);
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
static void
afamqp_dd_queue(LogPipe *s, LogMessage *msg,
const LogPathOptions *path_options, gpointer user_data)
{
AMQPDestDriver *self = (AMQPDestDriver *) s;
LogPathOptions local_options;
if (!path_options->flow_control_requested)
path_options = log_msg_break_ack(msg, path_options, &local_options);
log_msg_add_ack(msg, path_options);
log_queue_push_tail(self->queue, msg, path_options);
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
static void
afmongodb_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)s;
LogPathOptions local_options;
if (!path_options->flow_control_requested)
path_options = log_msg_break_ack(msg, path_options, &local_options);
self->last_msg_stamp = cached_g_current_time_sec ();
log_msg_add_ack(msg, path_options);
log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
gpointer
threaded_feed(gpointer args)
{
LogQueue *q = args;
char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
gint msg_len = strlen(msg_str);
gint i;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg, *tmpl;
GTimeVal start, end;
GSockAddr *sa;
glong diff;
iv_init();
/* emulate main loop for LogQueue */
main_loop_worker_thread_start(NULL);
sa = g_sockaddr_inet_new("10.10.10.10", 1010);
tmpl = log_msg_new(msg_str, msg_len, sa, &parse_options);
g_sockaddr_unref(sa);
g_get_current_time(&start);
for (i = 0; i < MESSAGES_PER_FEEDER; i++)
{
msg = log_msg_clone_cow(tmpl, &path_options);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
log_queue_push_tail(q, msg, &path_options);
if ((i & 0xFF) == 0)
main_loop_worker_invoke_batch_callbacks();
}
main_loop_worker_invoke_batch_callbacks();
g_get_current_time(&end);
diff = g_time_val_diff(&end, &start);
g_static_mutex_lock(&tlock);
sum_time += diff;
g_static_mutex_unlock(&tlock);
log_msg_unref(tmpl);
iv_deinit();
main_loop_worker_thread_stop();
return NULL;
}
static void
log_threaded_dest_driver_queue(LogPipe *s, LogMessage *msg,
const LogPathOptions *path_options,
gpointer user_data)
{
LogThrDestDriver *self = (LogThrDestDriver *)s;
LogPathOptions local_options;
if (!path_options->flow_control_requested)
path_options = log_msg_break_ack(msg, path_options, &local_options);
if (self->queue_method)
self->queue_method(self);
log_msg_add_ack(msg, path_options);
log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
static void
log_writer_last_msg_flush(LogWriter *self)
{
LogMessage *m;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
gchar hostname[256];
gchar buf[1024];
gssize len;
const gchar *p;
msg_debug("Suppress timer elapsed, emitting suppression summary",
NULL);
getlonghostname(hostname, sizeof(hostname));
m = log_msg_new_empty();
m->timestamps[LM_TS_STAMP] = m->timestamps[LM_TS_RECVD];
m->pri = self->last_msg->pri;
m->flags = LF_INTERNAL | LF_LOCAL;
p = log_msg_get_value(self->last_msg, LM_V_HOST, &len);
log_msg_set_value(m, LM_V_HOST, p, len);
p = log_msg_get_value(self->last_msg, LM_V_PROGRAM, &len);
log_msg_set_value(m, LM_V_PROGRAM, p, len);
len = g_snprintf(buf, sizeof(buf), "Last message '%.20s' repeated %d times, supressed by syslog-ng on %s",
log_msg_get_value(self->last_msg, LM_V_MESSAGE, NULL),
self->last_msg_count,
hostname);
log_msg_set_value(m, LM_V_MESSAGE, buf, len);
path_options.flow_control = FALSE;
if (!log_queue_push_tail(self->queue, m, &path_options))
{
stats_counter_inc(self->dropped_messages);
msg_debug("Destination queue full, dropping suppressed message",
evt_tag_int("queue_len", log_queue_get_length(self->queue)),
evt_tag_int("mem_fifo_size", self->options->mem_fifo_size),
NULL);
log_msg_drop(m, &path_options);
}
log_writer_last_msg_release(self);
}
/* NOTE: runs in the reader thread */
static void
log_writer_queue(LogPipe *s, LogMessage *lm, const LogPathOptions *path_options, gpointer user_data)
{
LogWriter *self = (LogWriter *) s;
LogPathOptions local_options;
if (!path_options->flow_control_requested &&
(self->suspended || !(self->flags & LW_SOFT_FLOW_CONTROL)))
{
/* NOTE: this code ACKs the message back if there's a write error in
* order not to hang the client in case of a disk full */
path_options = log_msg_break_ack(lm, path_options, &local_options);
}
if (self->options->suppress > 0 && log_writer_last_msg_check(self, lm, path_options))
return;
stats_counter_inc(self->processed_messages);
log_queue_push_tail(self->queue, lm, path_options);
}
static gpointer
_threaded_feed(gpointer args)
{
LogQueue *q = args;
gint i;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg, *tmpl;
GTimeVal start, end;
glong diff;
iv_init();
/* emulate main loop for LogQueue */
main_loop_worker_thread_start(NULL);
tmpl = log_msg_new_empty();
g_get_current_time(&start);
for (i = 0; i < MESSAGES_PER_FEEDER; i++)
{
msg = log_msg_clone_cow(tmpl, &path_options);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
log_queue_push_tail(q, msg, &path_options);
if ((i & 0xFF) == 0)
main_loop_worker_invoke_batch_callbacks();
}
main_loop_worker_invoke_batch_callbacks();
g_get_current_time(&end);
diff = g_time_val_diff(&end, &start);
g_static_mutex_lock(&tlock);
sum_time += diff;
g_static_mutex_unlock(&tlock);
log_msg_unref(tmpl);
main_loop_worker_thread_stop();
iv_deinit();
return NULL;
}
/* NOTE: runs in the reader thread */
static void
log_writer_queue(LogPipe *s, LogMessage *lm, const LogPathOptions *path_options, gpointer user_data)
{
LogWriter *self = (LogWriter *) s;
LogPathOptions local_options;
gint mark_mode = self->options->mark_mode;
if (!path_options->flow_control_requested &&
(self->suspended || !(self->flags & LW_SOFT_FLOW_CONTROL)))
{
/* NOTE: this code ACKs the message back if there's a write error in
* order not to hang the client in case of a disk full */
path_options = log_msg_break_ack(lm, path_options, &local_options);
}
if (log_writer_is_msg_suppressed(self, lm))
{
log_msg_drop(lm, path_options);
return;
}
if (mark_mode != MM_INTERNAL && (lm->flags & LF_INTERNAL) && (lm->flags & LF_MARK))
{
/* drop MARK messages generated by internal() in case our mark-mode != internal */
log_msg_drop(lm, path_options);
return;
}
if (mark_mode == MM_DST_IDLE || (mark_mode == MM_HOST_IDLE && !(lm->flags & LF_LOCAL)))
{
/* in dst-idle and host-idle most, messages postpone the MARK itself */
log_writer_postpone_mark_timer(self);
}
stats_counter_inc(self->processed_messages);
log_queue_push_tail(self->queue, lm, path_options);
}
/* this is the callback function that gets called when the MARK timeout
* elapsed. It runs in the main thread.
*/
static void
log_writer_mark_timeout(void *cookie)
{
LogWriter *self = (LogWriter *)cookie;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
const gchar *hostname;
gsize hostname_len;
LogMessage *msg;
main_loop_assert_main_thread();
msg = log_msg_new_mark();
/* timeout: there was no new message on the writer or it is in periodical mode */
hostname = resolve_sockaddr_to_hostname(&hostname_len, msg->saddr, &self->options->host_resolve_options);
log_msg_set_value(msg, LM_V_HOST, hostname, hostname_len);
/* set the current time in the message stamp */
msg->timestamps[LM_TS_STAMP] = msg->timestamps[LM_TS_RECVD];
if (!log_writer_is_msg_suppressed(self, msg))
{
log_queue_push_tail(self->queue, msg, &path_options);
stats_counter_inc(self->processed_messages);
}
else
{
log_msg_drop(msg, &path_options);
}
/* we need to issue another MARK in all mark-mode cases that already
* triggered this callback (dst-idle, host-idle, periodical). The
* original setup of the timer is at a different location:
* - log_writer_queue() for "*-idle" modes
* - log_writer_init() for periodical mode
*/
log_writer_postpone_mark_timer(self);
}
void
feed_some_messages(LogQueue **q, int n)
{
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg;
gint i;
path_options.ack_needed = (*q)->use_backlog;
for (i = 0; i < n; i++)
{
char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
GSockAddr *sa;
sa = g_sockaddr_inet_new("10.10.10.10", 1010);
msg = log_msg_new(msg_str, strlen(msg_str), sa, &parse_options);
g_sockaddr_unref(sa);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
log_queue_push_tail((*q), msg, &path_options);
fed_messages++;
}
}
void
feed_some_messages(LogQueue **q, int n, gboolean flow_control)
{
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg;
gint i;
path_options.flow_control = flow_control;
for (i = 0; i < n; i++)
{
char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
msg = log_msg_new(msg_str, strlen(msg_str), g_sockaddr_inet_new("10.10.10.10", 1010), 0, NULL, -1, 0xFFFF);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
if (!log_queue_push_tail((*q), msg, &path_options))
{
fprintf(stderr, "Queue unable to consume enough messages: %d\n", fed_messages);
exit(1);
}
fed_messages++;
}
}
