static void log_dest_group_queue(LogPipe *s, LogMessage *msg, gint path_flags) { LogDestGroup *self = (LogDestGroup *) s; LogDriver *p; if ((path_flags & PF_FLOW_CTL_OFF) == 0) { log_msg_ref(msg); log_msg_ack_block_start(msg, log_dest_group_ack, NULL); } for (p = self->drivers; p; p = p->drv_next) { #if 1 /* start dongshu */ if(p->processed_limit !=0 && ((p->processed_messages > 0) && (p->processed_messages % p->processed_limit == 0))){ p->flush = TRUE; } #endif /* end */ if ((path_flags & PF_FLOW_CTL_OFF) == 0) log_msg_ack_block_inc(msg); log_pipe_queue(&p->super, log_msg_ref(msg), path_flags); /* call affile_dd_queue()... */ p->processed_messages++; } (*self->processed_messages)++; if ((path_flags & PF_FLOW_CTL_OFF) == 0) log_msg_ack(msg); log_msg_unref(msg); }
static void _init(AckRecord *self) { self->acked = FALSE; log_msg_ref(self->original); log_msg_refcache_start_producer(self->original); log_msg_add_ack(self->original, &self->path_options); log_msg_ref(self->original); log_msg_write_protect(self->original); }
static void afmongodb_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data) { MongoDBDestDriver *self = (MongoDBDestDriver *)s; gboolean queue_was_empty; LogPathOptions local_options; if (!path_options->flow_control_requested) path_options = log_msg_break_ack(msg, path_options, &local_options); g_mutex_lock(self->queue_mutex); self->last_msg_stamp = cached_g_current_time_sec (); queue_was_empty = log_queue_get_length(self->queue) == 0; g_mutex_unlock(self->queue_mutex); log_msg_add_ack(msg, path_options); log_queue_push_tail(self->queue, log_msg_ref(msg), path_options); g_mutex_lock(self->suspend_mutex); if (queue_was_empty && !self->writer_thread_suspended) { g_mutex_lock(self->queue_mutex); log_queue_set_parallel_push(self->queue, 1, afmongodb_dd_queue_notify, self, NULL); g_mutex_unlock(self->queue_mutex); } g_mutex_unlock(self->suspend_mutex); log_dest_driver_queue_method(s, msg, path_options, user_data); }
void log_source_post(LogSource *self, LogMessage *msg) { LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; gint old_window_size; ack_tracker_track_msg(self->ack_tracker, msg); /* NOTE: we start by enabling flow-control, thus we need an acknowledgement */ path_options.ack_needed = TRUE; log_msg_ref(msg); log_msg_add_ack(msg, &path_options); msg->ack_func = log_source_msg_ack; old_window_size = g_atomic_counter_exchange_and_add(&self->window_size, -1); /* * NOTE: this assertion validates that the source is not overflowing its * own flow-control window size, decreased above, by the atomic statement. * * If the _old_ value is zero, that means that the decrement operation * above has decreased the value to -1. */ g_assert(old_window_size > 0); log_pipe_queue(&self->super, msg, &path_options); }
static LogMessage * create_clone(LogMessage *msg, LogPathOptions *path_options) { LogMessage *cloned = log_msg_ref(msg); cloned = log_msg_make_writable(&cloned, path_options); log_msg_add_ack(msg, path_options); return cloned; }
/* * Can only run from the output thread. * * NOTE: this returns a reference which the caller must take care to free. */ static LogMessage * log_queue_fifo_pop_head(LogQueue *s, LogPathOptions *path_options) { LogQueueFifo *self = (LogQueueFifo *) s; LogMessageQueueNode *node; LogMessage *msg = NULL; if (self->qoverflow_output_len == 0) { /* slow path, output queue is empty, get some elements from the wait queue */ g_static_mutex_lock(&self->super.lock); iv_list_splice_tail_init(&self->qoverflow_wait, &self->qoverflow_output); self->qoverflow_output_len = self->qoverflow_wait_len; self->qoverflow_wait_len = 0; g_static_mutex_unlock(&self->super.lock); } if (self->qoverflow_output_len > 0) { node = iv_list_entry(self->qoverflow_output.next, LogMessageQueueNode, list); msg = node->msg; path_options->ack_needed = node->ack_needed; self->qoverflow_output_len--; if (!self->super.use_backlog) { iv_list_del(&node->list); log_msg_free_queue_node(node); } else { iv_list_del_init(&node->list); } } else { /* no items either on the wait queue nor the output queue. * * NOTE: the input queues may contain items even in this case, * however we don't touch them here, they'll be migrated to the * wait_queue once the input threads finish their processing (or * the high watermark is reached). Also, they are unlocked, so * no way to touch them safely. */ return NULL; } stats_counter_dec(self->super.stored_messages); if (self->super.use_backlog) { log_msg_ref(msg); iv_list_add_tail(&node->list, &self->qbacklog); self->qbacklog_len++; } return msg; }
static void log_multiplexer_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data) { LogMultiplexer *self = (LogMultiplexer *) s; gint i; LogPathOptions local_options = *path_options; gboolean matched; gboolean delivered = FALSE; gboolean last_delivery; gint fallback; local_options.matched = &matched; for (fallback = 0; (fallback == 0) || (fallback == 1 && self->fallback_exists && !delivered); fallback++) { for (i = 0; i < self->next_hops->len; i++) { LogPipe *next_hop = g_ptr_array_index(self->next_hops, i); if (G_UNLIKELY(fallback == 0 && (next_hop->flags & PIF_BRANCH_FALLBACK) != 0)) { continue; } else if (G_UNLIKELY(fallback && (next_hop->flags & PIF_BRANCH_FALLBACK) == 0)) { continue; } matched = TRUE; log_msg_add_ack(msg, &local_options); /* NOTE: this variable indicates that the upcoming message * delivery is the last one, thus we don't need to retain an an * unmodified copy to be sent to further paths. The current * delivery may modify the message at will. */ last_delivery = (self->super.pipe_next == NULL) && (i == self->next_hops->len - 1) && (!self->fallback_exists || delivered || fallback == 1); if (!last_delivery) log_msg_write_protect(msg); log_pipe_queue(next_hop, log_msg_ref(msg), &local_options); if (!last_delivery) log_msg_write_unprotect(msg); if (matched) { delivered = TRUE; if (G_UNLIKELY(next_hop->flags & PIF_BRANCH_FINAL)) break; } } } log_pipe_forward_msg(s, msg, path_options); }
/** * Remember the last message for dup detection. * * NOTE: suppress_lock must be held. **/ static void log_writer_last_msg_record(LogWriter *self, LogMessage *lm) { if (self->last_msg) log_msg_unref(self->last_msg); log_msg_ref(lm); self->last_msg = lm; self->last_msg_count = 0; }
PyObject * py_log_message_new(LogMessage *msg) { PyLogMessage *self; self = PyObject_New(PyLogMessage, &py_log_message_type); if (!self) return NULL; self->msg = log_msg_ref(msg); return (PyObject *) self; }
static gboolean _push_tail (LogQueueDisk *s, LogMessage *msg, LogPathOptions *local_options, const LogPathOptions *path_options) { LogQueueDiskNonReliable *self = (LogQueueDiskNonReliable *) s; if (HAS_SPACE_IN_QUEUE(self->qout) && qdisk_get_length (self->super.qdisk) == 0) { /* simple push never generates flow-control enabled entries to qout, they only get there * when rewinding the backlog */ g_queue_push_tail (self->qout, msg); g_queue_push_tail (self->qout, LOG_PATH_OPTIONS_FOR_BACKLOG); log_msg_ref (msg); } else { if (self->qoverflow->length != 0 || !s->write_message(s, msg)) { if (HAS_SPACE_IN_QUEUE(self->qoverflow)) { g_queue_push_tail (self->qoverflow, msg); g_queue_push_tail (self->qoverflow, LOG_PATH_OPTIONS_TO_POINTER (path_options)); log_msg_ref (msg); local_options->ack_needed = FALSE; } else { msg_debug ("Destination queue full, dropping message", evt_tag_str ("filename", qdisk_get_filename (self->super.qdisk)), evt_tag_int ("queue_len", _get_length(s)), evt_tag_int ("mem_buf_length", self->qoverflow_size), evt_tag_int ("size", qdisk_get_size (self->super.qdisk)), evt_tag_str ("persist_name", self->super.super.persist_name)); return FALSE; } } } return TRUE; }
static void log_db_parser_emit(LogMessage *msg, gboolean synthetic, gpointer user_data) { LogDBParser *self = (LogDBParser *) user_data; if (synthetic) { if (self->inject_mode == LDBP_IM_PASSTHROUGH) { LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; path_options.ack_needed = FALSE; log_pipe_forward_msg(&self->super.super, log_msg_ref(msg), &path_options); } else { msg_post_message(log_msg_ref(msg)); } msg_debug("db-parser: emitting synthetic message", evt_tag_str("msg", log_msg_get_value(msg, LM_V_MESSAGE, NULL)), NULL); } }
static void _move_messages_from_overflow(LogQueueDiskNonReliable *self) { LogMessage *msg; LogPathOptions path_options; /* move away as much entries from the overflow area as possible */ while (_has_movable_message(self)) { msg = g_queue_pop_head (self->qoverflow); POINTER_TO_LOG_PATH_OPTIONS (g_queue_pop_head (self->qoverflow), &path_options); if (qdisk_get_length (self->super.qdisk) == 0 && HAS_SPACE_IN_QUEUE(self->qout)) { /* we can skip qdisk, go straight to qout */ g_queue_push_tail (self->qout, msg); g_queue_push_tail (self->qout, LOG_PATH_OPTIONS_FOR_BACKLOG); log_msg_ref (msg); } else { if (!self->super.write_message(&self->super, msg)) { /* oops, altough there seemed to be some free space available, * we failed saving this message, (it might have needed more * than 4096 bytes than we ensured), push back and break */ g_queue_push_head (self->qoverflow, LOG_PATH_OPTIONS_TO_POINTER (&path_options)); g_queue_push_head (self->qoverflow, msg); log_msg_ref (msg); break; } } log_msg_ack (msg, &path_options, AT_PROCESSED); log_msg_unref (msg); } }
static void afmongodb_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data) { MongoDBDestDriver *self = (MongoDBDestDriver *)s; LogPathOptions local_options; if (!path_options->flow_control_requested) path_options = log_msg_break_ack(msg, path_options, &local_options); self->last_msg_stamp = cached_g_current_time_sec (); log_msg_add_ack(msg, path_options); log_queue_push_tail(self->queue, log_msg_ref(msg), path_options); log_dest_driver_queue_method(s, msg, path_options, user_data); }
static void log_threaded_dest_driver_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data) { LogThrDestDriver *self = (LogThrDestDriver *)s; LogPathOptions local_options; if (!path_options->flow_control_requested) path_options = log_msg_break_ack(msg, path_options, &local_options); if (self->queue_method) self->queue_method(self); log_msg_add_ack(msg, path_options); log_queue_push_tail(self->queue, log_msg_ref(msg), path_options); log_dest_driver_queue_method(s, msg, path_options, user_data); }
/* This function is called to populate the emitted_messages array in * process_params. It only manipulates per-thread data structure so it does * not require locks but does not mind them being locked either. */ static void _emit_message(PatternDB *self, PDBProcessParams *process_params, gboolean synthetic, LogMessage *msg) { if (!self->emit) return; if (process_params->num_emitted_messages < EXPECTED_NUMBER_OF_MESSAGES_EMITTED) { process_params->emitted_messages[process_params->num_emitted_messages++] = _piggy_back_log_message_pointer_with_synthetic_value(msg, synthetic); } else { if (!process_params->emitted_messages_overflow) process_params->emitted_messages_overflow = g_ptr_array_new(); g_ptr_array_add(process_params->emitted_messages_overflow, _piggy_back_log_message_pointer_with_synthetic_value(msg, synthetic)); } log_msg_ref(msg); }
static void log_multiplexer_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data) { LogMultiplexer *self = (LogMultiplexer *) s; gint i; LogPathOptions local_options = *path_options; gboolean matched; gboolean delivered = FALSE; gint fallback; local_options.matched = &matched; for (fallback = 0; (fallback == 0) || (fallback == 1 && self->fallback_exists && !delivered); fallback++) { for (i = 0; i < self->next_hops->len; i++) { LogPipe *next_hop = g_ptr_array_index(self->next_hops, i); if (G_UNLIKELY(fallback == 0 && (next_hop->flags & PIF_BRANCH_FALLBACK) != 0)) { continue; } else if (G_UNLIKELY(fallback && (next_hop->flags & PIF_BRANCH_FALLBACK) == 0)) { continue; } matched = TRUE; log_msg_add_ack(msg, &local_options); log_pipe_queue(next_hop, log_msg_ref(msg), &local_options); if (matched) { delivered = TRUE; if (G_UNLIKELY(next_hop->flags & PIF_BRANCH_FINAL)) break; } } } log_pipe_forward_msg(s, msg, path_options); }
static LogMessage * _pop_head (LogQueueDisk *s, LogPathOptions *path_options) { LogQueueDiskNonReliable *self = (LogQueueDiskNonReliable *) s; LogMessage *msg = NULL; if (self->qout->length > 0) { msg = g_queue_pop_head (self->qout); POINTER_TO_LOG_PATH_OPTIONS (g_queue_pop_head (self->qout), path_options); } if (msg == NULL) { msg = s->read_message(s, path_options); if (msg) { path_options->ack_needed = FALSE; } } if (msg == NULL) { if (self->qoverflow->length > 0 && qdisk_is_read_only (self->super.qdisk)) { msg = g_queue_pop_head (self->qoverflow); POINTER_TO_LOG_PATH_OPTIONS (g_queue_pop_head (self->qoverflow), path_options); } } if (msg != NULL) { if (self->super.super.use_backlog) { log_msg_ref (msg); g_queue_push_tail (self->qbacklog, msg); g_queue_push_tail (self->qbacklog, LOG_PATH_OPTIONS_TO_POINTER (path_options)); } _move_disk (self); } return msg; }
static void _emit_func(LogMessage *msg, gboolean synthetic, gpointer user_data) { g_ptr_array_add(messages, log_msg_ref(msg)); }
static void log_center_queue(LogPipe *s, LogMessage *msg, gint path_flags) { LogCenter *self = (LogCenter *) s; gboolean match, fallbacks, have_fallbacks = 1; gint ci, fi, di; (*self->received_messages)++; afinter_postpone_mark(self->cfg->mark_freq); log_msg_ref(msg); log_msg_ack_block_start(msg, log_center_ack, NULL); for (match = 0, fallbacks = 0; !match && have_fallbacks && (fallbacks <= 1); fallbacks++) { have_fallbacks = 0; for (ci = 0; ci < self->cfg->connections->len; ci++) { LogConnection *conn = (LogConnection *) g_ptr_array_index(self->cfg->connections, ci); if (!fallbacks && (conn->flags & LC_FALLBACK)) { have_fallbacks = 1; continue; } else if (fallbacks && !(conn->flags & LC_FALLBACK)) { continue; } if (!(conn->flags & LC_CATCHALL)) { /* check source */ if (!g_hash_table_lookup(conn->source_cache, msg->source_group->name->str)) { goto next_connection; } } else { /* catchall, every source matches */ ; } for (fi = 0; fi < conn->filters->len; fi++) { LogEndpoint *ep = (LogEndpoint *) g_ptr_array_index(conn->filters, fi); LogFilterRule *f; f = (LogFilterRule *) ep->ref; if (!log_filter_rule_eval(f, msg)) { goto next_connection; } } match = 1; for (di = 0; di < conn->destinations->len; di++) { LogEndpoint *ep = (LogEndpoint *) g_ptr_array_index(conn->destinations, di); LogDestGroup *dest; if (conn->flags & LC_FLOW_CONTROL) log_msg_ack_block_inc(msg); dest = (LogDestGroup *) ep->ref; log_pipe_queue(&dest->super, log_msg_ref(msg), path_flags | ((conn->flags & LC_FLOW_CONTROL) ? 0 : PF_FLOW_CTL_OFF)); (*self->queued_messages)++; } if (conn->flags & LC_FINAL) { break; } next_connection: ; } } /* our own ack */ log_msg_ack(msg); }
void invoke_rewrite_rule(LogRewrite *pipe, LogMessage *msg) { LogPathOptions po = LOG_PATH_OPTIONS_INIT; log_pipe_queue((LogPipe *) pipe, log_msg_ref(msg), &po); };
int lua_message_create_from_logmsg(lua_State *state, LogMessage *self) { log_msg_ref(self); return lua_create_userdata_from_pointer(state, self, LUA_MESSAGE_TYPE); };
jobject java_log_message_proxy_create_java_object(JavaLogMessageProxy *self, LogMessage *msg) { JNIEnv *java_env = java_machine_get_env(self->java_machine, &java_env); jobject jmsg = CALL_JAVA_FUNCTION(java_env, NewObject, self->loaded_class, self->mi_constructor, log_msg_ref(msg)); if (!jmsg) { msg_error("Can't create object", evt_tag_str("class_name", LOG_MESSAGE), NULL); } return jmsg; }
static void _pattern_db_process_matching_rule(PatternDB *self, PDBProcessParams *process_params) { PDBContext *context = NULL; PDBRule *rule = process_params->rule; LogMessage *msg = process_params->msg; GString *buffer = g_string_sized_new(32); g_static_rw_lock_writer_lock(&self->lock); _advance_time_based_on_message(self, process_params, &msg->timestamps[LM_TS_STAMP]); if (rule->context.id_template) { CorrellationKey key; log_template_format(rule->context.id_template, msg, NULL, LTZ_LOCAL, 0, NULL, buffer); log_msg_set_value(msg, context_id_handle, buffer->str, -1); correllation_key_setup(&key, rule->context.scope, msg, buffer->str); context = g_hash_table_lookup(self->correllation.state, &key); if (!context) { msg_debug("Correllation context lookup failure, starting a new context", evt_tag_str("rule", rule->rule_id), evt_tag_str("context", buffer->str), evt_tag_int("context_timeout", rule->context.timeout), evt_tag_int("context_expiration", timer_wheel_get_time(self->timer_wheel) + rule->context.timeout)); context = pdb_context_new(&key); g_hash_table_insert(self->correllation.state, &context->super.key, context); g_string_steal(buffer); } else { msg_debug("Correllation context lookup successful", evt_tag_str("rule", rule->rule_id), evt_tag_str("context", buffer->str), evt_tag_int("context_timeout", rule->context.timeout), evt_tag_int("context_expiration", timer_wheel_get_time(self->timer_wheel) + rule->context.timeout), evt_tag_int("num_messages", context->super.messages->len)); } g_ptr_array_add(context->super.messages, log_msg_ref(msg)); if (context->super.timer) { timer_wheel_mod_timer(self->timer_wheel, context->super.timer, rule->context.timeout); } else { context->super.timer = timer_wheel_add_timer(self->timer_wheel, rule->context.timeout, pattern_db_expire_entry, correllation_context_ref(&context->super), (GDestroyNotify) correllation_context_unref); } if (context->rule != rule) { if (context->rule) pdb_rule_unref(context->rule); context->rule = pdb_rule_ref(rule); } } else { context = NULL; } process_params->context = context; process_params->buffer = buffer; synthetic_message_apply(&rule->msg, &context->super, msg, buffer); _emit_message(self, process_params, FALSE, msg); _execute_rule_actions(self, process_params, RAT_MATCH); pdb_rule_unref(rule); g_static_rw_lock_writer_unlock(&self->lock); if (context) log_msg_write_protect(msg); g_string_free(buffer, TRUE); }
static void log_source_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data) { LogSource *self = (LogSource *) s; LogPathOptions local_options = *path_options; gint old_window_size; gint i; msg_set_context(msg); if (msg->timestamps[LM_TS_STAMP].tv_sec == -1 || !self->options->keep_timestamp) msg->timestamps[LM_TS_STAMP] = msg->timestamps[LM_TS_RECVD]; g_assert(msg->timestamps[LM_TS_STAMP].zone_offset != -1); ack_tracker_track_msg(self->ack_tracker, msg); /* $HOST setup */ log_source_mangle_hostname(self, msg); /* $PROGRAM override */ if (self->options->program_override) { if (self->options->program_override_len < 0) self->options->program_override_len = strlen(self->options->program_override); log_msg_set_value(msg, LM_V_PROGRAM, self->options->program_override, self->options->program_override_len); } /* $HOST override */ if (self->options->host_override) { if (self->options->host_override_len < 0) self->options->host_override_len = strlen(self->options->host_override); log_msg_set_value(msg, LM_V_HOST, self->options->host_override, self->options->host_override_len); } /* source specific tags */ if (self->options->tags) { for (i = 0; i < self->options->tags->len; i++) { log_msg_set_tag_by_id(msg, g_array_index(self->options->tags, LogTagId, i)); } } log_msg_set_tag_by_id(msg, self->options->source_group_tag); /* stats counters */ if (stats_check_level(2)) { stats_lock(); stats_register_and_increment_dynamic_counter(2, SCS_HOST | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_HOST, NULL), msg->timestamps[LM_TS_RECVD].tv_sec); if (stats_check_level(3)) { stats_register_and_increment_dynamic_counter(3, SCS_SENDER | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_HOST_FROM, NULL), msg->timestamps[LM_TS_RECVD].tv_sec); stats_register_and_increment_dynamic_counter(3, SCS_PROGRAM | SCS_SOURCE, NULL, log_msg_get_value(msg, LM_V_PROGRAM, NULL), msg->timestamps[LM_TS_RECVD].tv_sec); } stats_unlock(); } stats_syslog_process_message_pri(msg->pri); /* message setup finished, send it out */ /* NOTE: we start by enabling flow-control, thus we need an acknowledgement */ local_options.ack_needed = TRUE; log_msg_ref(msg); log_msg_add_ack(msg, &local_options); msg->ack_func = log_source_msg_ack; old_window_size = g_atomic_counter_exchange_and_add(&self->window_size, -1); /* * NOTE: this assertion validates that the source is not overflowing its * own flow-control window size, decreased above, by the atomic statement. * * If the _old_ value is zero, that means that the decrement operation * above has decreased the value to -1. */ g_assert(old_window_size > 0); stats_counter_inc(self->recvd_messages); stats_counter_set(self->last_message_seen, msg->timestamps[LM_TS_RECVD].tv_sec); log_pipe_forward_msg(s, msg, &local_options); msg_set_context(NULL); if (accurate_nanosleep && self->threaded && self->window_full_sleep_nsec > 0 && !log_source_free_to_send(self)) { struct timespec ts; /* wait one 0.1msec in the hope that the buffer clears up */ ts.tv_sec = 0; ts.tv_nsec = self->window_full_sleep_nsec; nanosleep(&ts, NULL); } }