static void
_prepare_eof_test(LogQueueDiskReliable *dq, LogMessage **msg1, LogMessage **msg2)
{
LogPathOptions local_options = LOG_PATH_OPTIONS_INIT;
gint64 start_pos = TEST_DISKQ_SIZE;
*msg1 = log_msg_new_mark();
*msg2 = log_msg_new_mark();
(*msg1)->ack_func = _dummy_ack;
log_msg_add_ack(*msg1, &local_options);
(*msg2)->ack_func = _dummy_ack;
log_msg_add_ack(*msg2, &local_options);
dq->super.qdisk->hdr->write_head = start_pos;
dq->super.qdisk->hdr->read_head = QDISK_RESERVED_SPACE + mark_message_serialized_size + 1;
dq->super.qdisk->hdr->backlog_head = dq->super.qdisk->hdr->read_head;
log_queue_push_tail(&dq->super.super, *msg1, &local_options);
log_queue_push_tail(&dq->super.super, *msg2, &local_options);
assert_gint(dq->qreliable->length, NUMBER_MESSAGES_IN_QUEUE(2), ASSERTION_ERROR("Messages aren't in qreliable"));
assert_gint64(dq->super.qdisk->hdr->write_head, QDISK_RESERVED_SPACE + mark_message_serialized_size, ASSERTION_ERROR("Bad write head"));
assert_gint(num_of_ack, 0, ASSERTION_ERROR("Messages are acked"));
dq->super.qdisk->hdr->read_head = start_pos;
dq->super.qdisk->hdr->backlog_head = dq->super.qdisk->hdr->read_head;
}
static void
afmongodb_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)s;
gboolean queue_was_empty;
LogPathOptions local_options;
if (!path_options->flow_control_requested)
path_options = log_msg_break_ack(msg, path_options, &local_options);
g_mutex_lock(self->queue_mutex);
self->last_msg_stamp = cached_g_current_time_sec ();
queue_was_empty = log_queue_get_length(self->queue) == 0;
g_mutex_unlock(self->queue_mutex);
log_msg_add_ack(msg, path_options);
log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);
g_mutex_lock(self->suspend_mutex);
if (queue_was_empty && !self->writer_thread_suspended)
{
g_mutex_lock(self->queue_mutex);
log_queue_set_parallel_push(self->queue, 1, afmongodb_dd_queue_notify, self, NULL);
g_mutex_unlock(self->queue_mutex);
}
g_mutex_unlock(self->suspend_mutex);
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
gpointer
threaded_feed(gpointer st)
{
LogQueue *q = (LogQueue *) st;
char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
gint i;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg;
for (i = 0; i < 100000; i++)
{
msg = log_msg_new(msg_str, strlen(msg_str), g_sockaddr_inet_new("10.10.10.10", 1010), 0, NULL, -1);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
g_static_mutex_lock(&threaded_lock);
if (!log_queue_push_tail(q, msg, &path_options))
{
fprintf(stderr, "Queue unable to consume enough messages: %d\n", fed_messages);
return GUINT_TO_POINTER(1);
}
g_static_mutex_unlock(&threaded_lock);
}
return NULL;
}
void
log_source_post(LogSource *self, LogMessage *msg)
{
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
gint old_window_size;
ack_tracker_track_msg(self->ack_tracker, msg);
/* NOTE: we start by enabling flow-control, thus we need an acknowledgement */
path_options.ack_needed = TRUE;
log_msg_ref(msg);
log_msg_add_ack(msg, &path_options);
msg->ack_func = log_source_msg_ack;
old_window_size = g_atomic_counter_exchange_and_add(&self->window_size, -1);
/*
* NOTE: this assertion validates that the source is not overflowing its
* own flow-control window size, decreased above, by the atomic statement.
*
* If the _old_ value is zero, that means that the decrement operation
* above has decreased the value to -1.
*/
g_assert(old_window_size > 0);
log_pipe_queue(&self->super, msg, &path_options);
}
static LogMessage *
create_clone(LogMessage *msg, LogPathOptions *path_options)
{
LogMessage *cloned = log_msg_ref(msg);
cloned = log_msg_make_writable(&cloned, path_options);
log_msg_add_ack(msg, path_options);
return cloned;
}
static void
log_multiplexer_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
LogMultiplexer *self = (LogMultiplexer *) s;
gint i;
LogPathOptions local_options = *path_options;
gboolean matched;
gboolean delivered = FALSE;
gboolean last_delivery;
gint fallback;
local_options.matched = &matched;
for (fallback = 0; (fallback == 0) || (fallback == 1 && self->fallback_exists && !delivered); fallback++)
{
for (i = 0; i < self->next_hops->len; i++)
{
LogPipe *next_hop = g_ptr_array_index(self->next_hops, i);
if (G_UNLIKELY(fallback == 0 && (next_hop->flags & PIF_BRANCH_FALLBACK) != 0))
{
continue;
}
else if (G_UNLIKELY(fallback && (next_hop->flags & PIF_BRANCH_FALLBACK) == 0))
{
continue;
}
matched = TRUE;
log_msg_add_ack(msg, &local_options);
/* NOTE: this variable indicates that the upcoming message
* delivery is the last one, thus we don't need to retain an an
* unmodified copy to be sent to further paths. The current
* delivery may modify the message at will.
*/
last_delivery = (self->super.pipe_next == NULL) &&
(i == self->next_hops->len - 1) &&
(!self->fallback_exists || delivered || fallback == 1);
if (!last_delivery)
log_msg_write_protect(msg);
log_pipe_queue(next_hop, log_msg_ref(msg), &local_options);
if (!last_delivery)
log_msg_write_unprotect(msg);
if (matched)
{
delivered = TRUE;
if (G_UNLIKELY(next_hop->flags & PIF_BRANCH_FINAL))
break;
}
}
}
log_pipe_forward_msg(s, msg, path_options);
}
static void
_init(AckRecord *self)
{
self->acked = FALSE;
log_msg_ref(self->original);
log_msg_refcache_start_producer(self->original);
log_msg_add_ack(self->original, &self->path_options);
log_msg_ref(self->original);
log_msg_write_protect(self->original);
}
static void
afamqp_dd_queue(LogPipe *s, LogMessage *msg,
const LogPathOptions *path_options, gpointer user_data)
{
AMQPDestDriver *self = (AMQPDestDriver *) s;
LogPathOptions local_options;
if (!path_options->flow_control_requested)
path_options = log_msg_break_ack(msg, path_options, &local_options);
log_msg_add_ack(msg, path_options);
log_queue_push_tail(self->queue, msg, path_options);
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
static void
afmongodb_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)s;
LogPathOptions local_options;
if (!path_options->flow_control_requested)
path_options = log_msg_break_ack(msg, path_options, &local_options);
self->last_msg_stamp = cached_g_current_time_sec ();
log_msg_add_ack(msg, path_options);
log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
gpointer
threaded_feed(gpointer args)
{
LogQueue *q = args;
char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
gint msg_len = strlen(msg_str);
gint i;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg, *tmpl;
GTimeVal start, end;
GSockAddr *sa;
glong diff;
iv_init();
/* emulate main loop for LogQueue */
main_loop_worker_thread_start(NULL);
sa = g_sockaddr_inet_new("10.10.10.10", 1010);
tmpl = log_msg_new(msg_str, msg_len, sa, &parse_options);
g_sockaddr_unref(sa);
g_get_current_time(&start);
for (i = 0; i < MESSAGES_PER_FEEDER; i++)
{
msg = log_msg_clone_cow(tmpl, &path_options);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
log_queue_push_tail(q, msg, &path_options);
if ((i & 0xFF) == 0)
main_loop_worker_invoke_batch_callbacks();
}
main_loop_worker_invoke_batch_callbacks();
g_get_current_time(&end);
diff = g_time_val_diff(&end, &start);
g_static_mutex_lock(&tlock);
sum_time += diff;
g_static_mutex_unlock(&tlock);
log_msg_unref(tmpl);
iv_deinit();
main_loop_worker_thread_stop();
return NULL;
}
static void
log_threaded_dest_driver_queue(LogPipe *s, LogMessage *msg,
const LogPathOptions *path_options,
gpointer user_data)
{
LogThrDestDriver *self = (LogThrDestDriver *)s;
LogPathOptions local_options;
if (!path_options->flow_control_requested)
path_options = log_msg_break_ack(msg, path_options, &local_options);
if (self->queue_method)
self->queue_method(self);
log_msg_add_ack(msg, path_options);
log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);
log_dest_driver_queue_method(s, msg, path_options, user_data);
}
static gpointer
_threaded_feed(gpointer args)
{
LogQueue *q = args;
gint i;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg, *tmpl;
GTimeVal start, end;
glong diff;
iv_init();
/* emulate main loop for LogQueue */
main_loop_worker_thread_start(NULL);
tmpl = log_msg_new_empty();
g_get_current_time(&start);
for (i = 0; i < MESSAGES_PER_FEEDER; i++)
{
msg = log_msg_clone_cow(tmpl, &path_options);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
log_queue_push_tail(q, msg, &path_options);
if ((i & 0xFF) == 0)
main_loop_worker_invoke_batch_callbacks();
}
main_loop_worker_invoke_batch_callbacks();
g_get_current_time(&end);
diff = g_time_val_diff(&end, &start);
g_static_mutex_lock(&tlock);
sum_time += diff;
g_static_mutex_unlock(&tlock);
log_msg_unref(tmpl);
main_loop_worker_thread_stop();
iv_deinit();
return NULL;
}
static void
log_multiplexer_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
LogMultiplexer *self = (LogMultiplexer *) s;
gint i;
LogPathOptions local_options = *path_options;
gboolean matched;
gboolean delivered = FALSE;
gint fallback;
local_options.matched = &matched;
for (fallback = 0; (fallback == 0) || (fallback == 1 && self->fallback_exists && !delivered); fallback++)
{
for (i = 0; i < self->next_hops->len; i++)
{
LogPipe *next_hop = g_ptr_array_index(self->next_hops, i);
if (G_UNLIKELY(fallback == 0 && (next_hop->flags & PIF_BRANCH_FALLBACK) != 0))
{
continue;
}
else if (G_UNLIKELY(fallback && (next_hop->flags & PIF_BRANCH_FALLBACK) == 0))
{
continue;
}
matched = TRUE;
log_msg_add_ack(msg, &local_options);
log_pipe_queue(next_hop, log_msg_ref(msg), &local_options);
if (matched)
{
delivered = TRUE;
if (G_UNLIKELY(next_hop->flags & PIF_BRANCH_FINAL))
break;
}
}
}
log_pipe_forward_msg(s, msg, path_options);
}
void
feed_some_messages(LogQueue **q, int n)
{
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg;
gint i;
path_options.ack_needed = (*q)->use_backlog;
for (i = 0; i < n; i++)
{
char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
GSockAddr *sa;
sa = g_sockaddr_inet_new("10.10.10.10", 1010);
msg = log_msg_new(msg_str, strlen(msg_str), sa, &parse_options);
g_sockaddr_unref(sa);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
log_queue_push_tail((*q), msg, &path_options);
fed_messages++;
}
}
void
feed_some_messages(LogQueue **q, int n, gboolean flow_control)
{
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
LogMessage *msg;
gint i;
path_options.flow_control = flow_control;
for (i = 0; i < n; i++)
{
char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
msg = log_msg_new(msg_str, strlen(msg_str), g_sockaddr_inet_new("10.10.10.10", 1010), 0, NULL, -1, 0xFFFF);
log_msg_add_ack(msg, &path_options);
msg->ack_func = test_ack;
if (!log_queue_push_tail((*q), msg, &path_options))
{
fprintf(stderr, "Queue unable to consume enough messages: %d\n", fed_messages);
exit(1);
}
fed_messages++;
}
}
static void
log_source_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
LogSource *self = (LogSource *) s;
LogPathOptions local_options = *path_options;
gint old_window_size;
gint i;
msg_set_context(msg);
if (msg->timestamps[LM_TS_STAMP].tv_sec == -1 || !self->options->keep_timestamp)
msg->timestamps[LM_TS_STAMP] = msg->timestamps[LM_TS_RECVD];
g_assert(msg->timestamps[LM_TS_STAMP].zone_offset != -1);
ack_tracker_track_msg(self->ack_tracker, msg);
/* $HOST setup */
log_source_mangle_hostname(self, msg);
/* $PROGRAM override */
if (self->options->program_override)
{
if (self->options->program_override_len < 0)
self->options->program_override_len = strlen(self->options->program_override);
log_msg_set_value(msg, LM_V_PROGRAM, self->options->program_override, self->options->program_override_len);
}
/* $HOST override */
if (self->options->host_override)
{
if (self->options->host_override_len < 0)
self->options->host_override_len = strlen(self->options->host_override);
log_msg_set_value(msg, LM_V_HOST, self->options->host_override, self->options->host_override_len);
}
/* source specific tags */
if (self->options->tags)
{
for (i = 0; i < self->options->tags->len; i++)
{
log_msg_set_tag_by_id(msg, g_array_index(self->options->tags, LogTagId, i));
}
}
log_msg_set_tag_by_id(msg, self->options->source_group_tag);
/* stats counters */
if (stats_check_level(2))
{
stats_lock();
stats_register_and_increment_dynamic_counter(2, SCS_HOST | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_HOST, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
if (stats_check_level(3))
{
stats_register_and_increment_dynamic_counter(3, SCS_SENDER | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_HOST_FROM, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
stats_register_and_increment_dynamic_counter(3, SCS_PROGRAM | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_PROGRAM, NULL), msg->timestamps[LM_TS_RECVD].tv_sec);
}
stats_unlock();
}
stats_syslog_process_message_pri(msg->pri);
/* message setup finished, send it out */
/* NOTE: we start by enabling flow-control, thus we need an acknowledgement */
local_options.ack_needed = TRUE;
log_msg_ref(msg);
log_msg_add_ack(msg, &local_options);
msg->ack_func = log_source_msg_ack;
old_window_size = g_atomic_counter_exchange_and_add(&self->window_size, -1);
/*
* NOTE: this assertion validates that the source is not overflowing its
* own flow-control window size, decreased above, by the atomic statement.
*
* If the _old_ value is zero, that means that the decrement operation
* above has decreased the value to -1.
*/
g_assert(old_window_size > 0);
stats_counter_inc(self->recvd_messages);
stats_counter_set(self->last_message_seen, msg->timestamps[LM_TS_RECVD].tv_sec);
log_pipe_forward_msg(s, msg, &local_options);
msg_set_context(NULL);
if (accurate_nanosleep && self->threaded && self->window_full_sleep_nsec > 0 && !log_source_free_to_send(self))
{
struct timespec ts;
/* wait one 0.1msec in the hope that the buffer clears up */
ts.tv_sec = 0;
ts.tv_nsec = self->window_full_sleep_nsec;
nanosleep(&ts, NULL);
}
}
