gboolean
log_source_init(LogPipe *s)
{
LogSource *self = (LogSource *) s;
stats_lock();
stats_register_counter(self->stats_level, self->stats_source | SCS_SOURCE, self->stats_id, self->stats_instance, SC_TYPE_PROCESSED, &self->recvd_messages);
stats_register_counter(self->stats_level, self->stats_source | SCS_SOURCE, self->stats_id, self->stats_instance, SC_TYPE_STAMP, &self->last_message_seen);
stats_unlock();
return TRUE;
}
static gboolean
afmongodb_dd_init(LogPipe *s)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)s;
GlobalConfig *cfg = log_pipe_get_config(s);
if (!log_dest_driver_init_method(s))
return FALSE;
if (cfg)
self->time_reopen = cfg->time_reopen;
if (!self->vp)
{
self->vp = value_pairs_new();
value_pairs_add_scope(self->vp, "selected-macros");
value_pairs_add_scope(self->vp, "nv-pairs");
}
self->host = NULL;
self->port = 27017;
if (!mongo_util_parse_addr(g_list_nth_data(self->servers, 0), &self->host,
&self->port))
{
msg_error("Cannot parse the primary host",
evt_tag_str("primary", g_list_nth_data(self->servers, 0)),
NULL);
return FALSE;
}
msg_verbose("Initializing MongoDB destination",
evt_tag_str("host", self->host),
evt_tag_int("port", self->port),
evt_tag_str("database", self->db),
evt_tag_str("collection", self->coll),
NULL);
self->queue = log_dest_driver_acquire_queue(&self->super, afmongodb_dd_format_persist_name(self));
stats_lock();
stats_register_counter(0, SCS_MONGODB | SCS_DESTINATION, self->super.super.id,
afmongodb_dd_format_stats_instance(self),
SC_TYPE_STORED, &self->stored_messages);
stats_register_counter(0, SCS_MONGODB | SCS_DESTINATION, self->super.super.id,
afmongodb_dd_format_stats_instance(self),
SC_TYPE_DROPPED, &self->dropped_messages);
stats_unlock();
log_queue_set_counters(self->queue, self->stored_messages, self->dropped_messages);
afmongodb_dd_start_thread(self);
return TRUE;
}
gboolean
log_threaded_dest_driver_start(LogPipe *s)
{
LogThrDestDriver *self = (LogThrDestDriver *)s;
GlobalConfig *cfg = log_pipe_get_config(s);
if (cfg && self->time_reopen == -1)
self->time_reopen = cfg->time_reopen;
self->queue = log_dest_driver_acquire_queue(&self->super,
self->format.persist_name(self));
if (self->queue == NULL)
{
return FALSE;
}
if (self->retries.max <= 0)
{
msg_warning("Wrong value for retries(), setting to default",
evt_tag_int("value", self->retries.max),
evt_tag_int("default", MAX_RETRIES_OF_FAILED_INSERT_DEFAULT),
evt_tag_str("driver", self->super.super.id),
NULL);
self->retries.max = MAX_RETRIES_OF_FAILED_INSERT_DEFAULT;
}
stats_lock();
stats_register_counter(0, self->stats_source | SCS_DESTINATION, self->super.super.id,
self->format.stats_instance(self),
SC_TYPE_STORED, &self->stored_messages);
stats_register_counter(0, self->stats_source | SCS_DESTINATION, self->super.super.id,
self->format.stats_instance(self),
SC_TYPE_DROPPED, &self->dropped_messages);
stats_register_counter(0, self->stats_source | SCS_DESTINATION, self->super.super.id,
self->format.stats_instance(self),
SC_TYPE_PROCESSED, &self->processed_messages);
stats_unlock();
log_queue_set_counters(self->queue, self->stored_messages,
self->dropped_messages);
self->seq_num = GPOINTER_TO_INT(cfg_persist_config_fetch(cfg, log_threaded_dest_driver_format_seqnum_for_persist(self)));
if (!self->seq_num)
init_sequence_number(&self->seq_num);
log_threaded_dest_driver_start_thread(self);
return TRUE;
}
void
test_reset_stats()
{
GString *reply = NULL;
GString *command = g_string_sized_new(128);
StatsCounterItem *counter = NULL;
stats_init();
stats_lock();
stats_register_counter(0, SCS_CENTER, "id", "received", SC_TYPE_PROCESSED, &counter);
stats_counter_set(counter, 666);
stats_unlock();
g_string_assign(command, "RESET_STATS");
reply = control_connection_reset_stats(command);
assert_string(reply->str, "The statistics of syslog-ng have been reset to 0.", "Bad reply");
g_string_free(reply, TRUE);
g_string_assign(command, "STATS");
reply = control_connection_send_stats(command);
assert_string(reply->str, "SourceName;SourceId;SourceInstance;State;Type;Number\ncenter;id;received;a;processed;0\n", "Bad reply");
g_string_free(reply, TRUE);
stats_destroy();
g_string_free(command, TRUE);
return;
}
void
test_stats()
{
GString *reply = NULL;
GString *command = g_string_sized_new(128);
StatsCounterItem *counter = NULL;
gchar **stats_result;
stats_init();
stats_lock();
stats_register_counter(0, SCS_CENTER, "id", "received", SC_TYPE_PROCESSED, &counter);
stats_unlock();
g_string_assign(command,"STATS");
reply = control_connection_send_stats(command);
stats_result = g_strsplit(reply->str, "\n", 2);
assert_string(stats_result[0], "SourceName;SourceId;SourceInstance;State;Type;Number",
"Bad reply");
g_strfreev(stats_result);
g_string_free(reply, TRUE);
g_string_free(command, TRUE);
stats_destroy();
return;
}
static gboolean
log_source_group_init(LogPipe *s)
{
LogSourceGroup *self = (LogSourceGroup *) s;
LogDriver *p;
GlobalConfig *cfg = log_pipe_get_config(s);
gint id = 0;
for (p = self->drivers; p; p = p->drv_next)
{
p->group = g_strdup(self->name);
if (!p->id)
p->id = g_strdup_printf("%s#%d", self->name, id++);
if (!log_pipe_init(&p->super, cfg))
{
msg_error("Error initializing source driver",
evt_tag_str("source", self->name),
evt_tag_str("id", p->id),
NULL);
goto deinit_all;
}
log_pipe_append(&p->super, s);
}
stats_register_counter(0, SCS_SOURCE | SCS_GROUP, self->name, NULL, SC_TYPE_PROCESSED, &self->processed_messages);
return TRUE;
deinit_all:
for (p = self->drivers; p; p = p->drv_next)
log_pipe_deinit(&p->super);
return FALSE;
}
static gboolean
afamqp_dd_init(LogPipe *s)
{
AMQPDestDriver *self = (AMQPDestDriver *) s;
GlobalConfig *cfg = log_pipe_get_config(s);
if (!log_dest_driver_init_method(s))
return FALSE;
if (cfg)
self->time_reopen = cfg->time_reopen;
if (!self->vp)
{
self->vp = value_pairs_new();
value_pairs_add_scope(self->vp, "selected-macros");
value_pairs_add_scope(self->vp, "nv-pairs");
value_pairs_add_scope(self->vp, "sdata");
}
msg_verbose("Initializing AMQP destination",
evt_tag_str("vhost", self->vhost),
evt_tag_str("host", self->host),
evt_tag_int("port", self->port),
evt_tag_str("exchange", self->exchange),
evt_tag_str("exchange_type", self->exchange_type),
NULL);
self->queue = log_dest_driver_acquire_queue(&self->super, afamqp_dd_format_persist_name(self));
stats_lock();
stats_register_counter(0, SCS_AMQP | SCS_DESTINATION,
self->super.super.id, afamqp_dd_format_stats_instance(self),
SC_TYPE_STORED, &self->stored_messages);
stats_register_counter(0, SCS_AMQP | SCS_DESTINATION,
self->super.super.id, afamqp_dd_format_stats_instance(self),
SC_TYPE_DROPPED, &self->dropped_messages);
stats_unlock();
log_queue_set_counters(self->queue, self->stored_messages,
self->dropped_messages);
afamqp_dd_start_thread(self);
return TRUE;
}
Test(logqueue, test_zero_diskbuf_and_normal_acks)
{
LogQueue *q;
gint i;
q = log_queue_fifo_new(OVERFLOW_SIZE, NULL);
StatsClusterKey sc_key;
stats_lock();
stats_cluster_logpipe_key_set(&sc_key, SCS_DESTINATION, q->persist_name, NULL );
stats_register_counter(0, &sc_key, SC_TYPE_QUEUED, &q->queued_messages);
stats_register_counter(1, &sc_key, SC_TYPE_MEMORY_USAGE, &q->memory_usage);
stats_unlock();
log_queue_set_use_backlog(q, TRUE);
cr_assert_eq(atomic_gssize_racy_get(&q->queued_messages->value), 0);
fed_messages = 0;
acked_messages = 0;
feed_some_messages(q, 1);
cr_assert_eq(stats_counter_get(q->queued_messages), 1);
cr_assert_neq(stats_counter_get(q->memory_usage), 0);
gint size_when_single_msg = stats_counter_get(q->memory_usage);
for (i = 0; i < 10; i++)
feed_some_messages(q, 10);
cr_assert_eq(stats_counter_get(q->queued_messages), 101);
cr_assert_eq(stats_counter_get(q->memory_usage), 101*size_when_single_msg);
send_some_messages(q, fed_messages);
log_queue_ack_backlog(q, fed_messages);
cr_assert_eq(fed_messages, acked_messages,
"did not receive enough acknowledgements: fed_messages=%d, acked_messages=%d",
fed_messages, acked_messages);
log_queue_unref(q);
}
gboolean
log_dest_driver_init_method(LogPipe *s)
{
LogDestDriver *self = (LogDestDriver *) s;
GlobalConfig *cfg = log_pipe_get_config(s);
if (!log_driver_init_method(s))
return FALSE;
if (!self->super.group)
{
self->super.group = cfg_tree_get_rule_name(&cfg->tree, ENC_DESTINATION, s->expr_node);
self->super.id = cfg_tree_get_child_id(&cfg->tree, ENC_DESTINATION, s->expr_node);
}
stats_lock();
stats_register_counter(0, SCS_DESTINATION | SCS_GROUP, self->super.group, NULL, SC_TYPE_PROCESSED, &self->super.processed_group_messages);
stats_register_counter(0, SCS_CENTER, NULL, "queued", SC_TYPE_PROCESSED, &self->queued_global_messages);
stats_unlock();
return TRUE;
}
/*
* log_tags_get_by_name
*
* Lookup a tag id by it's name. If the tag is seen for the first time
* the next tag id is assigned and the tag is added to the list.
*
* The function returns the tag id associated with the name.
*
* @name: the name of the tag
*
*/
LogTagId
log_tags_get_by_name(const gchar *name)
{
/* If log_tags_hash() is NULL, this unit is already deinitialized
but other thread may refer the tag structure.
If name is empty, it is an extremal element.
In both cases the return value is 0.
*/
guint id;
g_assert(log_tags_hash != NULL);
g_static_mutex_lock(&log_tags_lock);
id = GPOINTER_TO_UINT(g_hash_table_lookup(log_tags_hash, name)) - 1;
if (id == 0xffffffff)
{
if (log_tags_num < LOG_TAGS_MAX - 1)
{
id = log_tags_num++;
if (id == log_tags_list_size)
{
log_tags_list_size *= 2;
log_tags_list = g_renew(LogTag, log_tags_list, log_tags_list_size);
}
log_tags_list[id].id = id;
log_tags_list[id].name = g_strdup(name);
log_tags_list[id].counter = NULL;
/* NOTE: stats-level may not be set for calls that happen during
* config file parsing, those get fixed up by
* log_tags_reinit_stats() below */
stats_lock();
StatsClusterKey sc_key;
stats_cluster_logpipe_key_set(&sc_key, SCS_TAG, name, NULL );
stats_register_counter(3, &sc_key, SC_TYPE_PROCESSED, &log_tags_list[id].counter);
stats_unlock();
g_hash_table_insert(log_tags_hash, log_tags_list[id].name, GUINT_TO_POINTER(log_tags_list[id].id + 1));
}
else
id = 0;
}
g_static_mutex_unlock(&log_tags_lock);
return id;
}
static gboolean
log_dest_group_init(LogPipe *s, GlobalConfig *cfg, PersistentConfig *persist)
{
LogDestGroup *self = (LogDestGroup *) s;
LogDriver *p;
for (p = self->drivers; p; p = p->drv_next)
{
if (!p->super.init(&p->super, cfg, persist)) /* call affile_dd_init()... */
return FALSE;
}
stats_register_counter(SC_TYPE_PROCESSED, log_dest_group_format_stats_name(self), &self->processed_messages, FALSE);
return TRUE;
}
gboolean
log_src_driver_init_method(LogPipe *s)
{
LogSrcDriver *self = (LogSrcDriver *) s;
GlobalConfig *cfg = log_pipe_get_config(s);
if (!log_driver_init_method(s))
return FALSE;
if (!self->super.group)
{
self->super.group = cfg_tree_get_rule_name(&cfg->tree, ENC_SOURCE, s->expr_node);
self->group_len = strlen(self->super.group);
self->super.id = cfg_tree_get_child_id(&cfg->tree, ENC_SOURCE, s->expr_node);
}
stats_lock();
stats_register_counter(0, SCS_SOURCE | SCS_GROUP, self->super.group, NULL, SC_TYPE_PROCESSED, &self->super.processed_group_messages);
stats_register_counter(0, SCS_CENTER, NULL, "received", SC_TYPE_PROCESSED, &self->received_global_messages);
stats_unlock();
return TRUE;
}
/*
* log_tags_get_by_name
*
* Lookup a tag id by it's name. If the tag is seen for the first time
* the next tag id is assigned and the tag is added to the list.
*
* The function returns the tag id associated with the name.
*
* @name: the name of the tag
*
*/
LogTagId
log_tags_get_by_name(const gchar *name)
{
/* If log_tags_hash() is NULL, this unit is already deinitialized
but other thread may refer the tag structure.
If name is empty, it is an extremal element.
In both cases the return value is 0.
*/
guint id;
g_assert(log_tags_hash != NULL);
g_static_mutex_lock(&log_tags_lock);
id = GPOINTER_TO_UINT(g_hash_table_lookup(log_tags_hash, name)) - 1;
if (id == 0xffffffff)
{
if (log_tags_num < LOG_TAGS_MAX - 1)
{
id = log_tags_num++;
if (id == log_tags_list_size)
{
log_tags_list_size *= 2;
log_tags_list = g_renew(LogTag, log_tags_list, log_tags_list_size);
}
log_tags_list[id].id = id;
log_tags_list[id].name = g_strdup(name);
log_tags_list[id].counter = NULL;
stats_lock();
stats_register_counter(3, SCS_TAG, name, NULL, SC_TYPE_PROCESSED, &log_tags_list[id].counter);
stats_unlock();
g_hash_table_insert(log_tags_hash, log_tags_list[id].name, GUINT_TO_POINTER(log_tags_list[id].id + 1));
}
else
id = 0;
}
g_static_mutex_unlock(&log_tags_lock);
return id;
}
/*
* NOTE: this is called at cfg_init() time to update the set of counters we
* have. If stats-level is decreased, we should unregister everything we
* had earlier. If increased we need to register them again.
*
* log_tags_get_by_name() will also try to register the counter for calls
* that are _after_ cfg_init(). Early calls to log_tags_get_by_name() will
* not see a proper stats-level() in the global variable here. Those will
* get handled by this function.
*/
void
log_tags_reinit_stats(void)
{
gint id;
stats_lock();
for (id = 0; id < log_tags_num; id++)
{
const gchar *name = log_tags_list[id].name;
StatsClusterKey sc_key;
stats_cluster_logpipe_key_set(&sc_key, SCS_TAG, name, NULL );
if (stats_check_level(3))
stats_register_counter(3, &sc_key, SC_TYPE_PROCESSED, &log_tags_list[id].counter);
else
stats_unregister_counter(&sc_key, SC_TYPE_PROCESSED, &log_tags_list[id].counter);
}
stats_unlock();
}
Test(control_cmds, test_stats)
{
StatsCounterItem *counter = NULL;
gchar **stats_result;
const gchar *response;
stats_init();
stats_lock();
StatsClusterKey sc_key;
stats_cluster_logpipe_key_set(&sc_key, SCS_CENTER, "id", "received" );
stats_register_counter(0, &sc_key, SC_TYPE_PROCESSED, &counter);
stats_unlock();
_run_command("STATS", &response);
stats_result = g_strsplit(response, "\n", 2);
cr_assert_str_eq(stats_result[0], "SourceName;SourceId;SourceInstance;State;Type;Number",
"Bad reply");
g_strfreev(stats_result);
stats_destroy();
}
Test(control_cmds, test_reset_stats)
{
StatsCounterItem *counter = NULL;
const gchar *response;
stats_init();
stats_lock();
StatsClusterKey sc_key;
stats_cluster_logpipe_key_set(&sc_key, SCS_CENTER, "id", "received" );
stats_register_counter(0, &sc_key, SC_TYPE_PROCESSED, &counter);
stats_counter_set(counter, 666);
stats_unlock();
_run_command("RESET_STATS", &response);
cr_assert(first_line_eq(response, "OK The statistics of syslog-ng have been reset to 0."), "Bad reply");
_run_command("STATS", &response);
cr_assert_str_eq(response,
"SourceName;SourceId;SourceInstance;State;Type;Number\ncenter;id;received;a;processed;0\n.\n",
"Bad reply");
stats_destroy();
}
Test(logqueue, log_queue_fifo_rewind_all_and_memory_usage)
{
LogQueue *q = log_queue_fifo_new(OVERFLOW_SIZE, NULL);
log_queue_set_use_backlog(q, TRUE);
StatsClusterKey sc_key;
stats_lock();
stats_cluster_logpipe_key_set(&sc_key, SCS_DESTINATION, q->persist_name, NULL );
stats_register_counter(1, &sc_key, SC_TYPE_MEMORY_USAGE, &q->memory_usage);
stats_unlock();
feed_some_messages(q, 1);
gint size_when_single_msg = stats_counter_get(q->memory_usage);
feed_some_messages(q, 9);
cr_assert_eq(stats_counter_get(q->memory_usage), 10*size_when_single_msg);
send_some_messages(q, 10);
cr_assert_eq(stats_counter_get(q->memory_usage), 0);
log_queue_rewind_backlog_all(q);
cr_assert_eq(stats_counter_get(q->memory_usage), 10*size_when_single_msg);
log_queue_unref(q);
}
static gboolean
log_center_init(LogPipe *s, GlobalConfig *cfg, PersistentConfig *persist)
{
LogCenter *self = (LogCenter *) s;
gint i, j;
self->cfg = cfg;
self->persist = persist;
/* resolve references within the configuration */
for (i = 0; i < cfg->connections->len; i++)
{
LogConnection *conn = (LogConnection *) g_ptr_array_index(cfg->connections, i);
LogEndpoint *ep;
for (j = 0; j < conn->sources->len; j++)
{
ep = (LogEndpoint *) g_ptr_array_index(conn->sources, j);
ep->ref = g_hash_table_lookup(cfg->sources, ep->name->str);
if (!ep->ref)
{
msg_error("Error in configuration, unresolved source reference",
evt_tag_str("source", ep->name->str),
NULL);
return FALSE;
}
log_sgrp_ref(ep->ref);
}
for (j = 0; j < conn->filters->len; j++)
{
ep = (LogEndpoint *) g_ptr_array_index(conn->filters, j);
ep->ref = g_hash_table_lookup(cfg->filters, ep->name->str);
if (!ep->ref)
{
msg_error("Error in configuration, unresolved filter reference",
evt_tag_str("filter", ep->name->str),
NULL);
return FALSE;
}
log_filter_ref(ep->ref);
}
for (j = 0; j < conn->destinations->len; j++)
{
ep = (LogEndpoint *) g_ptr_array_index(conn->destinations, j);
ep->ref = g_hash_table_lookup(cfg->destinations, ep->name->str);
if (!ep->ref)
{
msg_error("Error in configuration, unresolved destination reference",
evt_tag_str("filter", ep->name->str),
NULL);
return FALSE;
}
log_dgrp_ref(ep->ref);
}
}
self->state = LC_STATE_INIT_SOURCES;
self->success = TRUE;
g_hash_table_foreach(cfg->sources, (GHFunc) log_center_init_component, self);
if (!self->success)
return FALSE;
self->success = TRUE;
self->state = LC_STATE_INIT_DESTS;
g_hash_table_foreach(cfg->destinations, (GHFunc) log_center_init_component, self);
if (!self->success)
return FALSE;
stats_register_counter(SC_TYPE_PROCESSED, "center(received)", &self->received_messages, FALSE);
stats_register_counter(SC_TYPE_PROCESSED, "center(queued)", &self->queued_messages, FALSE);
self->state = LC_STATE_WORKING;
return TRUE;
}
static gboolean
afmongodb_dd_init(LogPipe *s)
{
MongoDBDestDriver *self = (MongoDBDestDriver *)s;
GlobalConfig *cfg = log_pipe_get_config(s);
ValuePairsTransformSet *vpts;
if (!log_dest_driver_init_method(s))
return FALSE;
if (cfg)
self->time_reopen = cfg->time_reopen;
if (!self->vp)
{
self->vp = value_pairs_new();
value_pairs_add_scope(self->vp, "selected-macros");
value_pairs_add_scope(self->vp, "nv-pairs");
}
/* Always replace a leading dot with an underscore. */
vpts = value_pairs_transform_set_new(".*");
value_pairs_transform_set_add_func(vpts, value_pairs_new_transform_replace(".", "_"));
value_pairs_add_transforms(self->vp, vpts);
if (self->port != MONGO_CONN_LOCAL)
{
if (self->address)
{
gchar *srv = g_strdup_printf ("%s:%d", self->address,
(self->port) ? self->port : 27017);
self->servers = g_list_prepend (self->servers, srv);
g_free (self->address);
}
if (!self->servers)
afmongodb_dd_set_servers((LogDriver *)self, g_list_append (NULL, g_strdup ("127.0.0.1:27017")));
self->address = NULL;
self->port = 27017;
if (!mongo_util_parse_addr(g_list_nth_data(self->servers, 0),
&self->address,
&self->port))
{
msg_error("Cannot parse the primary host",
evt_tag_str("primary", g_list_nth_data(self->servers, 0)),
NULL);
return FALSE;
}
}
if (self->port == MONGO_CONN_LOCAL)
msg_verbose("Initializing MongoDB destination",
evt_tag_str("address", self->address),
evt_tag_str("database", self->db),
evt_tag_str("collection", self->coll),
NULL);
else
msg_verbose("Initializing MongoDB destination",
evt_tag_str("address", self->address),
evt_tag_int("port", self->port),
evt_tag_str("database", self->db),
evt_tag_str("collection", self->coll),
NULL);
self->queue = log_dest_driver_acquire_queue(&self->super, afmongodb_dd_format_persist_name(self));
stats_lock();
stats_register_counter(0, SCS_MONGODB | SCS_DESTINATION, self->super.super.id,
afmongodb_dd_format_stats_instance(self),
SC_TYPE_STORED, &self->stored_messages);
stats_register_counter(0, SCS_MONGODB | SCS_DESTINATION, self->super.super.id,
afmongodb_dd_format_stats_instance(self),
SC_TYPE_DROPPED, &self->dropped_messages);
stats_unlock();
log_queue_set_counters(self->queue, self->stored_messages, self->dropped_messages);
afmongodb_dd_start_thread(self);
return TRUE;
}
