// Low level input function
int os_inchar(uint8_t *buf, int maxlen, int ms, int tb_change_cnt)
{
if (maxlen && rbuffer_size(input_buffer)) {
return (int)rbuffer_read(input_buffer, (char *)buf, (size_t)maxlen);
}
InbufPollResult result;
if (ms >= 0) {
if ((result = inbuf_poll(ms)) == kInputNone) {
return 0;
}
} else {
if ((result = inbuf_poll((int)p_ut)) == kInputNone) {
if (read_stream.closed && silent_mode) {
// Drained eventloop & initial input; exit silent/batch-mode (-es/-Es).
read_error_exit();
}
if (trigger_cursorhold() && !typebuf_changed(tb_change_cnt)) {
create_cursorhold_event();
} else {
before_blocking();
result = inbuf_poll(-1);
}
}
}
// If input was put directly in typeahead buffer bail out here.
if (typebuf_changed(tb_change_cnt)) {
return 0;
}
if (maxlen && rbuffer_size(input_buffer)) {
// Safe to convert rbuffer_read to int, it will never overflow since we use
// relatively small buffers.
return (int)rbuffer_read(input_buffer, (char *)buf, (size_t)maxlen);
}
// If there are events, return the keys directly
if (maxlen && pending_events()) {
return push_event_key(buf, maxlen);
}
if (result == kInputEof) {
read_error_exit();
}
return 0;
}
// Low level input function
int os_inchar(uint8_t *buf, int maxlen, int ms, int tb_change_cnt)
{
if (rbuffer_pending(input_buffer)) {
return (int)rbuffer_read(input_buffer, (char *)buf, (size_t)maxlen);
}
InbufPollResult result;
if (ms >= 0) {
if ((result = inbuf_poll(ms)) == kInputNone) {
return 0;
}
} else {
if ((result = inbuf_poll((int)p_ut)) == kInputNone) {
if (trigger_cursorhold() && maxlen >= 3
&& !typebuf_changed(tb_change_cnt)) {
buf[0] = K_SPECIAL;
buf[1] = KS_EXTRA;
buf[2] = KE_CURSORHOLD;
return 3;
}
before_blocking();
result = inbuf_poll(-1);
}
}
// If input was put directly in typeahead buffer bail out here.
if (typebuf_changed(tb_change_cnt)) {
return 0;
}
if (rbuffer_pending(input_buffer)) {
// Safe to convert rbuffer_read to int, it will never overflow since we use
// relatively small buffers.
return (int)rbuffer_read(input_buffer, (char *)buf, (size_t)maxlen);
}
// If there are deferred events, return the keys directly
if (event_has_deferred()) {
return push_event_key(buf, maxlen);
}
if (result == kInputEof) {
read_error_exit();
}
return 0;
}
int32_t bus_terminal_recv_all(bus_terminal_t* bt, char* buf,
size_t* buf_size, bus_addr_t* from)
{
static int32_t index = 0;
int32_t ret;
struct idtable_iterator_t* it;
struct bus_terminal_channel_t* btc;
struct bus_channel_t* bc;
if (!bt || !buf || !buf_size || !from) {
return bus_err_fail;
}
it = idtable_iterator_init(bt->recv_channels, (index ++) % BUS_MAX_TERMINAL_COUNT);
while (it) {
btc = (struct bus_terminal_channel_t*)idtable_iterator_value(it);
assert(btc);
bc = bus_terminal_channel(btc);
assert(bc);
ret = rbuffer_read(bus_terminal_channel_rbuffer(btc), buf, buf_size);
if (ret == 0) {
*from = bc->from;
if (it) {
idtable_iterator_release(it);
}
return 0;
}
if (idtable_iterator_next(it) < 0) {
idtable_iterator_release(it);
break;
}
}
return bus_err_empty;
}
int32_t bus_terminal_recv(bus_terminal_t* bt, char* buf,
size_t* buf_size, bus_addr_t from)
{
struct bus_terminal_channel_t* btc;
int32_t ret;
if (!bt) return bus_err_fail;
btc = (struct bus_terminal_channel_t*)idtable_get(bt->recv_channels, from);
if (!btc) return bus_err_peer_not_found;
ret = rbuffer_read(bus_terminal_channel_rbuffer(btc), buf, buf_size);
return ret == 0 ? bus_err_empty : bus_ok;
}
/**
* \brief Wait for data from input queue in loop.
*
* This function runs in separated thread.
*
* \param[in] config configuration structure
* \return NULL
*/
void *ip_loop(void *config)
{
struct intermediate *conf = (struct intermediate *) config;
struct ipfix_message *msg;
unsigned int index;
prctl(PR_SET_NAME, conf->thread_name, 0, 0, 0);
/* wait for messages and process them */
while (1) {
index = -1;
/* get message from input buffer */
msg = rbuffer_read(conf->in_queue, &index);
if (!msg) {
rbuffer_remove_reference(conf->in_queue, index, 1);
if (conf->new_in) {
/* Set new input queue */
conf->in_queue = conf->new_in;
conf->new_in = NULL;
pthread_cond_signal(&conf->in_q_cond);
continue;
}
/* terminating mediator */
MSG_DEBUG(msg_module, "NULL message; terminating intermediate process %s...", conf->thread_name);
break;
}
conf->index = index;
conf->dropped = false;
/* process message */
conf->intermediate_process_message(conf->plugin_config, msg);
if (!conf->dropped) {
/* remove message from input queue, but do not free memory (it must be done later in output manager) */
rbuffer_remove_reference(conf->in_queue, index, 0);
}
}
return NULL;
}
// recv buffer
static int net_socket_recv_buffer(struct net_pool* np, struct net_socket* ns)
{
log_debug("net_socket_recv_buffer, fd : %d, rdsz : %d \n", ns->fd, ns->rdsz);
int n = rbuffer_read(ns->rbuff, ns->fd, ns->rdsz);
log_debug("net_socket_recv_buffer : data : %s\n", ns->rbuff);
log_debug("recv size %d\n", n);
if(n < 0)
{
if(n == -2)
{
log_debug("client leave %d\n", ns->fd);
net_force_close(np, ns);
return -1;
}
else
{
log_error("recv error : %d", ns->id);
np->onerror(np, ns->id, 6);
return -1;
}
}
// increase or reduce the recv size
if(n == ns->rdsz)
{
ns->rdsz = ns->rdsz * 2;
}
else if((ns->rdsz > MIN_READ_BUFFER) && (ns->rdsz > n * 2))
{
ns->rdsz = ns->rdsz / 2;
}
log_debug("net_socket_recv_buffer rbuffer_read : %d\n", n);
np->onrecv(np, ns->id, n);
return n;
}
/**
* \brief Wait for data from input queue in loop.
*
* This function runs in separated thread.
*
* \param[in] config configuration structure
* \return NULL
*/
void *ip_loop(void *config)
{
struct ip_config *conf;
struct ipfix_message *message;
unsigned int index;
conf = (struct ip_config *) config;
prctl(PR_SET_NAME, conf->thread_name, 0, 0, 0);
index = conf->in_queue->read_offset;
/* wait for messages and process them */
while (1) {
/* get message from input buffer */
message = rbuffer_read(conf->in_queue, &index);
if (!message) {
/* terminating mediator */
MSG_DEBUG(msg_module, "NULL message, terminating intermediate process.");
break;
}
conf->index = index;
conf->dropped = false;
/* process message */
conf->intermediate_process_message(conf->plugin_config, message);
if (!conf->dropped) {
/* remove message from input queue, but do not free memory (it must be done later in output manager) */
rbuffer_remove_reference(conf->in_queue, index, 0);
}
/* update index */
index = (index + 1) % conf->in_queue->size;
}
return (void *) 0;
}
/**
* \brief Output Managers thread
*
* @param[in] config configuration structure
* @return NULL
*/
static void *output_manager_plugin_thread(void* config)
{
struct data_manager_config *data_config = NULL;
struct ipfix_message* msg = NULL;
unsigned int index;
conf = (struct output_manager_config *) config;
index = conf->in_queue->read_offset;
/* set the thread name to reflect the configuration */
prctl(PR_SET_NAME, "ipfixcol OM", 0, 0, 0);
/* loop will break upon receiving NULL from buffer */
while (1) {
/* get next data */
index = -1;
msg = rbuffer_read(conf->in_queue, &index);
if (!msg) {
rbuffer_remove_reference(conf->in_queue, index, 1);
if (conf->new_in) {
/* Set new input queue */
conf->in_queue = (struct ring_buffer *) conf->new_in;
conf->new_in = NULL;
pthread_cond_signal(&conf->in_q_cond);
continue;
}
/* End manager */
break;
}
/* get appropriate data manager's config according to Observation domain ID */
data_config = get_data_mngmt_config (msg->input_info->odid, conf->data_managers);
if (data_config == NULL) {
/*
* no data manager config for this observation domain ID found -
* we have a new observation domain ID, so create new data manager for
* it
*/
data_config = data_manager_create(msg->input_info->odid, conf->storage_plugins);
if (data_config == NULL) {
MSG_WARNING(msg_module, "[%u] Unable to create Data Manager; skipping data...",
msg->input_info->odid);
rbuffer_remove_reference(conf->in_queue, index, 1);
continue;
}
/* add config to data_mngmts structure */
output_manager_insert(conf, data_config);
MSG_NOTICE(msg_module, "[%u] Data Manager created", msg->input_info->odid);
}
if (msg->source_status == SOURCE_STATUS_NEW) {
/* New source, increment reference counter */
MSG_DEBUG(msg_module, "[%u] New source", data_config->observation_domain_id);
data_config->references++;
} else if (msg->source_status == SOURCE_STATUS_CLOSED) {
/* Source closed, decrement reference counter */
MSG_DEBUG(msg_module, "[%u] Closed source", data_config->observation_domain_id);
data_config->references--;
if (data_config->references == 0) {
/* No reference for this ODID, close DM */
MSG_DEBUG(msg_module, "[%u] No source; releasing templates...", data_config->observation_domain_id);
output_manager_remove(conf, data_config);
}
rbuffer_remove_reference(conf->in_queue, index, 1);
continue;
}
__sync_fetch_and_add(&(conf->packets), 1);
__sync_fetch_and_add(&(conf->data_records), msg->data_records_count);
/* Check for lost data records */
uint32_t seq_number = ntohl(msg->pkt_header->sequence_number);
// Set sequence number during first iteration
if (conf->first_seq == 0 && conf->last_seq == 0) {
conf->first_seq = seq_number;
} else if (seq_number < conf->first_seq) {
// Sequence number resetted (modulo 2^32 = 4294967296)
conf->first_seq = seq_number;
uint8_t delta_seq = 4294967296 - conf->last_seq + seq_number;
// Check for sequence number gap
if (delta_seq > msg->data_records_count) {
__sync_fetch_and_add(&(conf->lost_data_records), delta_seq - msg->data_records_count);
}
} else if (seq_number > conf->first_seq) {
// Check for sequence number gap
if (seq_number - msg->data_records_count > conf->last_seq) {
__sync_fetch_and_add(&(conf->lost_data_records), seq_number - msg->data_records_count - conf->last_seq);
}
} else {
// Do nothing
}
conf->last_seq = seq_number;
/* Write data into input queue of Storage Plugins */
if (rbuffer_write(data_config->store_queue, msg, data_config->plugins_count) != 0) {
MSG_WARNING(msg_module, "[%u] Unable to write into Data Manager's input queue; skipping data...", data_config->observation_domain_id);
rbuffer_remove_reference(conf->in_queue, index, 1);
free(msg);
continue;
}
/* Remove data from queue (without deallocating) */
rbuffer_remove_reference(conf->in_queue, index, 0);
}
MSG_NOTICE(msg_module, "Closing Output Manager thread");
return (void *) 0;
}
/// Reads data from a `RStream` instance into a buffer.
///
/// @param rstream The `RStream` instance
/// @param buffer The buffer which will receive the data
/// @param count Number of bytes that `buffer` can accept
/// @return The number of bytes copied into `buffer`
size_t rstream_read(RStream *rstream, char *buffer, size_t count)
{
return rbuffer_read(rstream->buffer, buffer, count);
}
/**
* \brief Output Manager thread
*
* @param[in] config configuration structure
* @return NULL
*/
static void *output_manager_plugin_thread(void* config)
{
struct data_manager_config *data_config = NULL;
struct ipfix_message* msg = NULL;
unsigned int index;
uint32_t odid;
conf = (struct output_manager_config *) config;
index = conf->in_queue->read_offset;
/* Set thread name to reflect the configuration */
prctl(PR_SET_NAME, "ipfixcol OM", 0, 0, 0);
/* loop will break upon receiving NULL from buffer */
while (1) {
/* get next data */
index = -1;
msg = rbuffer_read(conf->in_queue, &index);
if (!msg) {
rbuffer_remove_reference(conf->in_queue, index, 1);
if (conf->new_in) {
/* Set new input queue */
conf->in_queue = (struct ring_buffer *) conf->new_in;
conf->new_in = NULL;
pthread_cond_signal(&conf->in_q_cond);
continue;
}
/* Stop manager */
break;
}
odid = (conf->perman_odid_merge || conf->manager_mode == OM_SINGLE)
? 0 : msg->input_info->odid;
/* Get appropriate data Manager config according to ODID */
data_config = get_data_mngmt_config(odid, conf->data_managers);
if (data_config == NULL) {
/*
* No data manager config for this observation domain ID found -
* we have a new observation domain ID, so create new data manager for
* it
*/
data_config = data_manager_create(odid, conf->storage_plugins);
if (data_config == NULL) {
MSG_WARNING(msg_module, "[%u] Unable to create Data Manager; skipping data...",
odid);
rbuffer_remove_reference(conf->in_queue, index, 1);
continue;
}
/* Add config to data_mngmts structure */
output_manager_insert(conf, data_config);
MSG_INFO(msg_module, "[%u] Data Manager created", odid);
}
if (msg->source_status == SOURCE_STATUS_NEW) {
/* New source, increment reference counter */
MSG_DEBUG(msg_module, "[%u] New source", data_config->observation_domain_id);
data_config->references++;
/* Add input info for the statistics thread to read */
add_input_info(msg->input_info);
} else if (msg->source_status == SOURCE_STATUS_CLOSED) {
/* Source closed, decrement reference counter */
MSG_DEBUG(msg_module, "[%u] Closed source", data_config->observation_domain_id);
data_config->references--;
/* Remove a reference to a template record */
uint32_t original_odid = msg->input_info->odid;
uint32_t source_crc = preprocessor_compute_crc(msg->input_info);
if (tm_source_unregister(template_mgr, original_odid, source_crc)) {
MSG_ERROR(msg_module, "[%u] Unable to unregister the source from the main template manager!", data_config->observation_domain_id);
}
/* Remove input_info from statistics */
remove_input_info(msg->input_info);
if (data_config->references == 0) {
/* No reference for this ODID, close DM */
MSG_DEBUG(msg_module, "[%u] No source; releasing templates...", data_config->observation_domain_id);
output_manager_remove(conf, data_config);
}
rbuffer_remove_reference(conf->in_queue, index, 1);
continue;
}
/* Write data into input queue of Storage Plugins */
if (rbuffer_write(data_config->store_queue, msg, data_config->plugins_count) != 0) {
MSG_WARNING(msg_module, "[%u] Unable to write into Data Manager input queue; skipping data...", data_config->observation_domain_id);
rbuffer_remove_reference(conf->in_queue, index, 1);
free(msg);
continue;
}
/* Remove data from queue (without memory deallocation) */
rbuffer_remove_reference(conf->in_queue, index, 0);
}
MSG_INFO(msg_module, "Closing Output Manager thread");
return (void *) 0;
}
