static gboolean
afamqp_worker_publish(AMQPDestDriver *self, LogMessage *msg)
{
gint pos = 0, ret;
amqp_table_t table;
amqp_basic_properties_t props;
gboolean success = TRUE;
SBGString *routing_key = sb_gstring_acquire();
SBGString *body = sb_gstring_acquire();
amqp_bytes_t body_bytes = amqp_cstring_bytes("");
gpointer user_data[] = { &self->entries, &pos, &self->max_entries };
value_pairs_foreach(self->vp, afamqp_vp_foreach, msg, self->seq_num,
&self->template_options, user_data);
table.num_entries = pos;
table.entries = self->entries;
props._flags = AMQP_BASIC_CONTENT_TYPE_FLAG
| AMQP_BASIC_DELIVERY_MODE_FLAG | AMQP_BASIC_HEADERS_FLAG;
props.content_type = amqp_cstring_bytes("text/plain");
props.delivery_mode = self->persistent;
props.headers = table;
log_template_format(self->routing_key_template, msg, NULL, LTZ_LOCAL,
self->seq_num, NULL, sb_gstring_string(routing_key));
if (self->body_template)
{
log_template_format(self->body_template, msg, NULL, LTZ_LOCAL,
self->seq_num, NULL, sb_gstring_string(body));
body_bytes = amqp_cstring_bytes(sb_gstring_string(body)->str);
}
ret = amqp_basic_publish(self->conn, 1, amqp_cstring_bytes(self->exchange),
amqp_cstring_bytes(sb_gstring_string(routing_key)->str),
0, 0, &props, body_bytes);
sb_gstring_release(routing_key);
sb_gstring_release(body);
if (ret < 0)
{
msg_error("Network error while inserting into AMQP server",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_int("time_reopen", self->super.time_reopen), NULL);
success = FALSE;
}
while (--pos >= 0)
{
amqp_bytes_free(self->entries[pos].key);
amqp_bytes_free(self->entries[pos].value.value.bytes);
}
return success;
}
static void
afstomp_set_frame_body(STOMPDestDriver *self, SBGString *body, stomp_frame* frame, LogMessage* msg)
{
if (self->body_template)
{
log_template_format(self->body_template, msg, NULL, LTZ_LOCAL,
self->super.seq_num, NULL, sb_gstring_string(body));
stomp_frame_set_body(frame, sb_gstring_string(body)->str, sb_gstring_string(body)->len);
}
}
gboolean
fop_cmp_eval(FilterExprNode *s, LogMessage *msg)
{
FilterCmp *self = (FilterCmp *) s;
gboolean result = FALSE;
gint cmp;
log_template_format(self->left, msg, NULL, LTZ_LOCAL, 0, self->left_buf);
log_template_format(self->right, msg, NULL, LTZ_LOCAL, 0, self->right_buf);
if (self->cmp_op & FCMP_NUM)
{
gint l, r;
l = atoi(self->left_buf->str);
r = atoi(self->right_buf->str);
if (l == r)
cmp = 0;
else if (l > r)
cmp = -1;
else
cmp = 1;
}
else
{
cmp = strcmp(self->left_buf->str, self->right_buf->str);
}
if (cmp == 0)
{
result = self->cmp_op & FCMP_EQ;
}
else if (cmp < 0)
{
result = self->cmp_op & FCMP_LT || self->cmp_op == 0;
}
else
{
result = self->cmp_op & FCMP_GT || self->cmp_op == 0;
}
return result ^ s->comp;
}
static gchar *
_resolve(AddContextualDataSelector *s, LogMessage *msg)
{
GString *selector_str = g_string_new(NULL);
AddContextualDataTemplateSelector *self = (AddContextualDataTemplateSelector *)s;
log_template_format(self->selector_template, msg, NULL, LTZ_LOCAL, 0, NULL,
selector_str);
return g_string_free(selector_str, FALSE);
}
static void
log_rewrite_set_process(LogRewrite *s, LogMessage *msg)
{
LogRewriteSet *self = (LogRewriteSet *) s;
GString *result;
result = g_string_sized_new(64);
log_template_format(self->value_template, msg, NULL, LTZ_LOCAL, 0, result);
log_msg_set_value(msg, self->super.value_handle, result->str, result->len);
g_string_free(result, TRUE);
}
static void
log_rewrite_set_process(LogRewrite *s, LogMessage **pmsg, const LogPathOptions *path_options)
{
LogRewriteSet *self = (LogRewriteSet *) s;
GString *result;
result = g_string_sized_new(64);
log_template_format(self->value_template, *pmsg, NULL, LTZ_LOCAL, 0, NULL, result);
log_msg_make_writable(pmsg, path_options);
log_msg_set_value(*pmsg, self->super.value_handle, result->str, result->len);
g_string_free(result, TRUE);
}
static GString *
_get_body_rendered(HTTPDestinationDriver *self, LogMessage *msg)
{
GString *body_rendered = NULL;
if (self->body_template)
{
body_rendered = g_string_new(NULL);
log_template_format(self->body_template, msg, &self->template_options, LTZ_SEND,
self->super.seq_num, NULL, body_rendered);
}
return body_rendered;
}
static void
_execute_action_create_context(PatternDB *db, PDBProcessParams *process_params)
{
CorrellationKey key;
PDBAction *action = process_params->action;
PDBRule *rule = process_params->rule;
PDBContext *triggering_context = process_params->context;
LogMessage *triggering_msg = process_params->msg;
GString *buffer = process_params->buffer;
PDBContext *new_context;
LogMessage *context_msg;
SyntheticContext *syn_context;
SyntheticMessage *syn_message;
syn_context = &action->content.create_context.context;
syn_message = &action->content.create_context.message;
if (triggering_context)
{
context_msg = synthetic_message_generate_with_context(syn_message, &triggering_context->super, buffer);
log_template_format_with_context(syn_context->id_template,
(LogMessage **) triggering_context->super.messages->pdata, triggering_context->super.messages->len,
NULL, LTZ_LOCAL, 0, NULL, buffer);
}
else
{
context_msg = synthetic_message_generate_without_context(syn_message, triggering_msg, buffer);
log_template_format(syn_context->id_template,
triggering_msg,
NULL, LTZ_LOCAL, 0, NULL, buffer);
}
msg_debug("Explicit create-context action, starting a new context",
evt_tag_str("rule", rule->rule_id),
evt_tag_str("context", buffer->str),
evt_tag_int("context_timeout", syn_context->timeout),
evt_tag_int("context_expiration", timer_wheel_get_time(db->timer_wheel) + syn_context->timeout));
correllation_key_setup(&key, syn_context->scope, context_msg, buffer->str);
new_context = pdb_context_new(&key);
g_hash_table_insert(db->correllation.state, &new_context->super.key, new_context);
g_string_steal(buffer);
g_ptr_array_add(new_context->super.messages, context_msg);
new_context->super.timer = timer_wheel_add_timer(db->timer_wheel, rule->context.timeout, pattern_db_expire_entry,
correllation_context_ref(&new_context->super),
(GDestroyNotify) correllation_context_unref);
new_context->rule = pdb_rule_ref(rule);
}
static gboolean
tf_graphite_format(GString *result, ValuePairs *vp, LogMessage *msg, const LogTemplateOptions *template_options, LogTemplate *timestamp_template, gint time_zone_mode)
{
TFGraphiteForeachUserData userdata;
gboolean return_value;
userdata.result = result;
userdata.formatted_unixtime = g_string_new("");
log_template_format(timestamp_template, msg, NULL, 0, 0, NULL, userdata.formatted_unixtime);
return_value = value_pairs_foreach(vp, tf_graphite_foreach_func, msg, 0, time_zone_mode, template_options, &userdata);
g_string_free(userdata.formatted_unixtime, FALSE);
return return_value;
}
void
test_pattern(const gchar *pattern, const gchar *rule, gboolean match)
{
gboolean result;
LogMessage *msg = log_msg_new_empty();
static LogTemplate *templ;
GString *res = g_string_sized_new(128);
static TimeZoneInfo *tzinfo = NULL;
PDBInput input;
if (!tzinfo)
tzinfo = time_zone_info_new(NULL);
if (!templ)
{
templ = log_template_new(configuration, "dummy");
log_template_compile(templ, "$TEST", NULL);
}
log_msg_set_value(msg, LM_V_HOST, MYHOST, strlen(MYHOST));
log_msg_set_value(msg, LM_V_PROGRAM, "test", strlen(MYHOST));
log_msg_set_value(msg, LM_V_MESSAGE, pattern, strlen(pattern));
result = pattern_db_process(patterndb, PDB_INPUT_WRAP_MESSAGE(&input, msg));
log_template_format(templ, msg, NULL, LTZ_LOCAL, 0, NULL, res);
if (strcmp(res->str, pattern) == 0)
{
test_msg("Rule: '%s' Value '%s' is inserted into $TEST res:(%s)\n", rule, pattern, res->str);
}
if ((match && !result) || (!match && result))
{
test_fail("FAIL: Value '%s' is %smatching for pattern '%s' \n", rule, !!result ? "" : "not ", pattern);
}
else
{
test_msg("Value '%s' is %smatching for pattern '%s' \n", rule, !!result ? "" : "not ", pattern);
}
g_string_free(res, TRUE);
log_msg_unref(msg);
}
static void
testcase(gchar *msg, gchar *timezone, gchar *format, gchar *expected)
{
LogTemplate *templ;
LogMessage *logmsg;
LogParser *parser;
gboolean success;
GString *res = g_string_sized_new(128);
parser = date_parser_new (configuration);
if (format != NULL) date_parser_set_format(parser, format);
if (timezone != NULL) date_parser_set_timezone(parser, timezone);
log_pipe_init(&parser->super);
logmsg = log_msg_new_empty();
logmsg->timestamps[LM_TS_RECVD].tv_sec = 1451473200; /* Dec 30 2015 */
log_msg_set_value(logmsg, log_msg_get_value_handle("MESSAGE"), msg, -1);
success = log_parser_process(parser, &logmsg, NULL, log_msg_get_value(logmsg, LM_V_MESSAGE, NULL), -1);
if (!success && expected)
{
fprintf(stderr, "unable to parse format=%s msg=%s\n", format, msg);
exit(1);
}
else if (success && !expected)
{
fprintf(stderr, "successfully parsed but expected failure, format=%s msg=%s\n", format, msg);
exit(1);
}
else if (expected)
{
/* Convert to ISODATE */
templ = compile_template("${ISODATE}", FALSE);
log_template_format(templ, logmsg, NULL, LTZ_LOCAL, -1, NULL, res);
assert_nstring(res->str, res->len, expected, strlen(expected),
"incorrect date parsed msg=%s format=%s",
msg, format);
log_template_unref(templ);
}
g_string_free(res, TRUE);
log_pipe_unref(&parser->super);
log_msg_unref(logmsg);
return;
}
static GString *
afsql_dd_ensure_accessible_database_table(AFSqlDestDriver *self, LogMessage *msg)
{
GString *table = g_string_sized_new(32);
log_template_format(self->table, msg, &self->template_options, LTZ_LOCAL, 0, NULL, table);
if (!afsql_dd_validate_table(self, table))
{
/* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
msg_error("Error checking table, disconnecting from database, trying again shortly",
evt_tag_int("time_reopen", self->time_reopen),
NULL);
g_string_free(table, TRUE);
return NULL;
}
return table;
}
static void
testcase(gchar *msg, goffset offset, gchar *timezone, gchar *format, gchar *expected)
{
LogTemplate *templ;
LogMessage *logmsg;
NVTable *nvtable;
GlobalConfig *cfg = cfg_new (0x302);
LogParser *parser = date_parser_new (cfg);
gboolean success;
const gchar *context_id = "test-context-id";
GString *res = g_string_sized_new(128);
date_parser_set_offset(parser, offset);
if (format != NULL) date_parser_set_format(parser, format);
if (timezone != NULL) date_parser_set_timezone(parser, timezone);
parse_options.flags = 0;
logmsg = log_msg_new_empty();
logmsg->timestamps[LM_TS_RECVD].tv_sec = 1438793384; /* Wed Aug 5 2015 */
log_msg_set_value(logmsg, log_msg_get_value_handle("MESSAGE"), msg, -1);
nvtable = nv_table_ref(logmsg->payload);
success = log_parser_process(parser, &logmsg, NULL, log_msg_get_value(logmsg, LM_V_MESSAGE, NULL), -1);
nv_table_unref(nvtable);
if (!success)
{
fprintf(stderr, "unable to parse offset=%d format=%s msg=%s\n", offset, format, msg);
exit(1);
}
/* Convert to ISODATE */
templ = compile_template("${ISODATE}", FALSE);
log_template_format(templ, logmsg, NULL, LTZ_LOCAL, 999, context_id, res);
assert_nstring(res->str, res->len, expected, strlen(expected),
"incorrect date parsed msg=%s offset=%d format=%s",
msg, offset, format);
log_template_unref(templ);
g_string_free(res, TRUE);
log_pipe_unref(&parser->super);
log_msg_unref(logmsg);
return;
}
static gpointer
format_template_thread(gpointer s)
{
gpointer *args = (gpointer *) s;
LogMessage *msg = args[0];
LogTemplate *templ = args[1];
const gchar *expected = args[2];
GString *result;
gint i;
g_mutex_lock(thread_lock);
while (!thread_start)
g_cond_wait(thread_ping, thread_lock);
g_mutex_unlock(thread_lock);
result = g_string_sized_new(0);
for (i = 0; i < 10000; i++)
{
log_template_format(templ, msg, NULL, LTZ_SEND, 5555, NULL, result);
assert_string(result->str, expected, "multi-threaded formatting yielded invalid result (iteration: %d)", i);
}
g_string_free(result, TRUE);
return NULL;
}
/**
* afsql_dd_validate_table:
*
* Check if the given table exists in the database. If it doesn't
* create it, if it does, check if all the required fields are
* present and create them if they don't.
*
* NOTE: This function can only be called from the database thread.
**/
static GString *
afsql_dd_validate_table(AFSqlDestDriver *self, LogMessage *msg)
{
GString *query_string, *table;
dbi_result db_res;
gboolean success = FALSE;
gint i;
table = g_string_sized_new(32);
log_template_format(self->table, msg, &self->template_options, LTZ_LOCAL, 0, NULL, table);
if (self->flags & AFSQL_DDF_DONT_CREATE_TABLES)
return table;
afsql_dd_check_sql_identifier(table->str, TRUE);
if (g_hash_table_lookup(self->validated_tables, table->str))
return table;
query_string = g_string_sized_new(32);
g_string_printf(query_string, "SELECT * FROM %s WHERE 0=1", table->str);
if (afsql_dd_run_query(self, query_string->str, TRUE, &db_res))
{
/* table exists, check structure */
success = TRUE;
for (i = 0; success && (i < self->fields_len); i++)
{
if (dbi_result_get_field_idx(db_res, self->fields[i].name) == 0)
{
GList *l;
/* field does not exist, add this column */
g_string_printf(query_string, "ALTER TABLE %s ADD %s %s", table->str, self->fields[i].name, self->fields[i].type);
if (!afsql_dd_run_query(self, query_string->str, FALSE, NULL))
{
msg_error("Error adding missing column, giving up",
evt_tag_str("table", table->str),
evt_tag_str("column", self->fields[i].name),
NULL);
success = FALSE;
break;
}
for (l = self->indexes; l; l = l->next)
{
if (strcmp((gchar *) l->data, self->fields[i].name) == 0)
{
/* this is an indexed column, create index */
afsql_dd_create_index(self, table->str, self->fields[i].name);
}
}
}
}
dbi_result_free(db_res);
}
else
{
/* table does not exist, create it */
g_string_printf(query_string, "CREATE TABLE %s (", table->str);
for (i = 0; i < self->fields_len; i++)
{
g_string_append_printf(query_string, "%s %s", self->fields[i].name, self->fields[i].type);
if (i != self->fields_len - 1)
g_string_append(query_string, ", ");
}
g_string_append(query_string, ")");
if (afsql_dd_run_query(self, query_string->str, FALSE, NULL))
{
GList *l;
success = TRUE;
for (l = self->indexes; l; l = l->next)
{
afsql_dd_create_index(self, table->str, (gchar *) l->data);
}
}
else
{
msg_error("Error creating table, giving up",
evt_tag_str("table", table->str),
NULL);
}
}
if (success)
{
/* we have successfully created/altered the destination table, record this information */
g_hash_table_insert(self->validated_tables, g_strdup(table->str), GUINT_TO_POINTER(TRUE));
}
else
{
g_string_free(table, TRUE);
table = NULL;
}
g_string_free(query_string, TRUE);
return table;
}
static void
_pattern_db_process_matching_rule(PatternDB *self, PDBProcessParams *process_params)
{
PDBContext *context = NULL;
PDBRule *rule = process_params->rule;
LogMessage *msg = process_params->msg;
GString *buffer = g_string_sized_new(32);
g_static_rw_lock_writer_lock(&self->lock);
_advance_time_based_on_message(self, process_params, &msg->timestamps[LM_TS_STAMP]);
if (rule->context.id_template)
{
CorrellationKey key;
log_template_format(rule->context.id_template, msg, NULL, LTZ_LOCAL, 0, NULL, buffer);
log_msg_set_value(msg, context_id_handle, buffer->str, -1);
correllation_key_setup(&key, rule->context.scope, msg, buffer->str);
context = g_hash_table_lookup(self->correllation.state, &key);
if (!context)
{
msg_debug("Correllation context lookup failure, starting a new context",
evt_tag_str("rule", rule->rule_id),
evt_tag_str("context", buffer->str),
evt_tag_int("context_timeout", rule->context.timeout),
evt_tag_int("context_expiration", timer_wheel_get_time(self->timer_wheel) + rule->context.timeout));
context = pdb_context_new(&key);
g_hash_table_insert(self->correllation.state, &context->super.key, context);
g_string_steal(buffer);
}
else
{
msg_debug("Correllation context lookup successful",
evt_tag_str("rule", rule->rule_id),
evt_tag_str("context", buffer->str),
evt_tag_int("context_timeout", rule->context.timeout),
evt_tag_int("context_expiration", timer_wheel_get_time(self->timer_wheel) + rule->context.timeout),
evt_tag_int("num_messages", context->super.messages->len));
}
g_ptr_array_add(context->super.messages, log_msg_ref(msg));
if (context->super.timer)
{
timer_wheel_mod_timer(self->timer_wheel, context->super.timer, rule->context.timeout);
}
else
{
context->super.timer = timer_wheel_add_timer(self->timer_wheel, rule->context.timeout, pattern_db_expire_entry,
correllation_context_ref(&context->super),
(GDestroyNotify) correllation_context_unref);
}
if (context->rule != rule)
{
if (context->rule)
pdb_rule_unref(context->rule);
context->rule = pdb_rule_ref(rule);
}
}
else
{
context = NULL;
}
process_params->context = context;
process_params->buffer = buffer;
synthetic_message_apply(&rule->msg, &context->super, msg, buffer);
_emit_message(self, process_params, FALSE, msg);
_execute_rule_actions(self, process_params, RAT_MATCH);
pdb_rule_unref(rule);
g_static_rw_lock_writer_unlock(&self->lock);
if (context)
log_msg_write_protect(msg);
g_string_free(buffer, TRUE);
}
static GString *
afsql_dd_construct_query(AFSqlDestDriver *self, GString *table,
LogMessage *msg)
{
GString *value;
GString *query_string;
gint i;
value = g_string_sized_new(256);
query_string = g_string_sized_new(512);
g_string_printf(query_string, "INSERT INTO %s (", table->str);
for (i = 0; i < self->fields_len; i++)
{
g_string_append(query_string, self->fields[i].name);
if (i != self->fields_len - 1)
g_string_append(query_string, ", ");
}
g_string_append(query_string, ") VALUES (");
for (i = 0; i < self->fields_len; i++)
{
gchar *quoted;
if (self->fields[i].value == NULL)
{
/* the config used the 'default' value for this column -> the fields[i].value is NULL, use SQL default */
g_string_append(query_string, "DEFAULT");
}
else
{
log_template_format(self->fields[i].value, msg, &self->template_options, LTZ_SEND, self->seq_num, NULL, value);
if (self->null_value && strcmp(self->null_value, value->str) == 0)
{
g_string_append(query_string, "NULL");
}
else
{
dbi_conn_quote_string_copy(self->dbi_ctx, value->str, "ed);
if (quoted)
{
g_string_append(query_string, quoted);
free(quoted);
}
else
{
g_string_append(query_string, "''");
}
}
}
if (i != self->fields_len - 1)
g_string_append(query_string, ", ");
}
g_string_append(query_string, ")");
g_string_free(value, TRUE);
return query_string;
}
static gboolean
redis_worker_insert(LogThrDestDriver *s)
{
RedisDriver *self = (RedisDriver *)s;
gboolean success;
LogMessage *msg;
LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
redisReply *reply;
const char *argv[5];
size_t argvlen[5];
int argc = 2;
redis_dd_connect(self, TRUE);
if (self->c->err)
return FALSE;
success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
if (!success)
return TRUE;
msg_set_context(msg);
log_template_format(self->key, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->key_str);
if (self->param1)
log_template_format(self->param1, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->param1_str);
if (self->param2)
log_template_format(self->param2, msg, &self->template_options, LTZ_SEND,
self->seq_num, NULL, self->param2_str);
argv[0] = self->command->str;
argvlen[0] = self->command->len;
argv[1] = self->key_str->str;
argvlen[1] = self->key_str->len;
if (self->param1)
{
argv[2] = self->param1_str->str;
argvlen[2] = self->param1_str->len;
argc++;
}
if (self->param2)
{
argv[3] = self->param2_str->str;
argvlen[3] = self->param2_str->len;
argc++;
}
reply = redisCommandArgv(self->c, argc, argv, argvlen);
msg_debug("REDIS command sent",
evt_tag_str("driver", self->super.super.super.id),
evt_tag_str("command", self->command->str),
evt_tag_str("key", self->key_str->str),
evt_tag_str("param1", self->param1_str->str),
evt_tag_str("param2", self->param2_str->str),
NULL);
success = TRUE;
freeReplyObject(reply);
msg_set_context(NULL);
if (success)
{
stats_counter_inc(self->super.stored_messages);
step_sequence_number(&self->seq_num);
log_msg_ack(msg, &path_options);
log_msg_unref(msg);
}
else
{
log_queue_push_head(self->super.queue, msg, &path_options);
}
return success;
}
static GString *
afsql_dd_build_insert_command(AFSqlDestDriver *self, LogMessage *msg, GString *table)
{
GString *insert_command = g_string_sized_new(256);
GString *value = g_string_sized_new(512);
gint i, j;
g_string_printf(insert_command, "INSERT INTO %s (", table->str);
for (i = 0; i < self->fields_len; i++)
{
if ((self->fields[i].flags & AFSQL_FF_DEFAULT) == 0 && self->fields[i].value != NULL)
{
g_string_append(insert_command, self->fields[i].name);
j = i + 1;
while (j < self->fields_len && (self->fields[j].flags & AFSQL_FF_DEFAULT) == AFSQL_FF_DEFAULT)
j++;
if (j < self->fields_len)
g_string_append(insert_command, ", ");
}
}
g_string_append(insert_command, ") VALUES (");
for (i = 0; i < self->fields_len; i++)
{
gchar *quoted;
if ((self->fields[i].flags & AFSQL_FF_DEFAULT) == 0 && self->fields[i].value != NULL)
{
log_template_format(self->fields[i].value, msg, &self->template_options, LTZ_SEND, self->seq_num, NULL, value);
if (self->null_value && strcmp(self->null_value, value->str) == 0)
{
g_string_append(insert_command, "NULL");
}
else
{
dbi_conn_quote_string_copy(self->dbi_ctx, value->str, "ed);
if (quoted)
{
g_string_append(insert_command, quoted);
free(quoted);
}
else
{
g_string_append(insert_command, "''");
}
}
j = i + 1;
while (j < self->fields_len && (self->fields[j].flags & AFSQL_FF_DEFAULT) == AFSQL_FF_DEFAULT)
j++;
if (j < self->fields_len)
g_string_append(insert_command, ", ");
}
}
g_string_append(insert_command, ")");
g_string_free(value, TRUE);
return insert_command;
}