/** * \brief Output Managers thread * * @param[in] config configuration structure * @return NULL */ static void *output_manager_plugin_thread(void* config) { struct data_manager_config *data_config = NULL; struct ipfix_message* msg = NULL; unsigned int index; conf = (struct output_manager_config *) config; index = conf->in_queue->read_offset; /* set the thread name to reflect the configuration */ prctl(PR_SET_NAME, "ipfixcol OM", 0, 0, 0); /* loop will break upon receiving NULL from buffer */ while (1) { /* get next data */ index = -1; msg = rbuffer_read(conf->in_queue, &index); if (!msg) { rbuffer_remove_reference(conf->in_queue, index, 1); if (conf->new_in) { /* Set new input queue */ conf->in_queue = (struct ring_buffer *) conf->new_in; conf->new_in = NULL; pthread_cond_signal(&conf->in_q_cond); continue; } /* End manager */ break; } /* get appropriate data manager's config according to Observation domain ID */ data_config = get_data_mngmt_config (msg->input_info->odid, conf->data_managers); if (data_config == NULL) { /* * no data manager config for this observation domain ID found - * we have a new observation domain ID, so create new data manager for * it */ data_config = data_manager_create(msg->input_info->odid, conf->storage_plugins); if (data_config == NULL) { MSG_WARNING(msg_module, "[%u] Unable to create Data Manager; skipping data...", msg->input_info->odid); rbuffer_remove_reference(conf->in_queue, index, 1); continue; } /* add config to data_mngmts structure */ output_manager_insert(conf, data_config); MSG_NOTICE(msg_module, "[%u] Data Manager created", msg->input_info->odid); } if (msg->source_status == SOURCE_STATUS_NEW) { /* New source, increment reference counter */ MSG_DEBUG(msg_module, "[%u] New source", data_config->observation_domain_id); data_config->references++; } else if (msg->source_status == SOURCE_STATUS_CLOSED) { /* Source closed, decrement reference counter */ MSG_DEBUG(msg_module, "[%u] Closed source", data_config->observation_domain_id); data_config->references--; if (data_config->references == 0) { /* No reference for this ODID, close DM */ MSG_DEBUG(msg_module, "[%u] No source; releasing templates...", data_config->observation_domain_id); output_manager_remove(conf, data_config); } rbuffer_remove_reference(conf->in_queue, index, 1); continue; } __sync_fetch_and_add(&(conf->packets), 1); __sync_fetch_and_add(&(conf->data_records), msg->data_records_count); /* Check for lost data records */ uint32_t seq_number = ntohl(msg->pkt_header->sequence_number); // Set sequence number during first iteration if (conf->first_seq == 0 && conf->last_seq == 0) { conf->first_seq = seq_number; } else if (seq_number < conf->first_seq) { // Sequence number resetted (modulo 2^32 = 4294967296) conf->first_seq = seq_number; uint8_t delta_seq = 4294967296 - conf->last_seq + seq_number; // Check for sequence number gap if (delta_seq > msg->data_records_count) { __sync_fetch_and_add(&(conf->lost_data_records), delta_seq - msg->data_records_count); } } else if (seq_number > conf->first_seq) { // Check for sequence number gap if (seq_number - msg->data_records_count > conf->last_seq) { __sync_fetch_and_add(&(conf->lost_data_records), seq_number - msg->data_records_count - conf->last_seq); } } else { // Do nothing } conf->last_seq = seq_number; /* Write data into input queue of Storage Plugins */ if (rbuffer_write(data_config->store_queue, msg, data_config->plugins_count) != 0) { MSG_WARNING(msg_module, "[%u] Unable to write into Data Manager's input queue; skipping data...", data_config->observation_domain_id); rbuffer_remove_reference(conf->in_queue, index, 1); free(msg); continue; } /* Remove data from queue (without deallocating) */ rbuffer_remove_reference(conf->in_queue, index, 0); } MSG_NOTICE(msg_module, "Closing Output Manager thread"); return (void *) 0; }
/** * \brief Output Manager thread * * @param[in] config configuration structure * @return NULL */ static void *output_manager_plugin_thread(void* config) { struct data_manager_config *data_config = NULL; struct ipfix_message* msg = NULL; unsigned int index; uint32_t odid; conf = (struct output_manager_config *) config; index = conf->in_queue->read_offset; /* Set thread name to reflect the configuration */ prctl(PR_SET_NAME, "ipfixcol OM", 0, 0, 0); /* loop will break upon receiving NULL from buffer */ while (1) { /* get next data */ index = -1; msg = rbuffer_read(conf->in_queue, &index); if (!msg) { rbuffer_remove_reference(conf->in_queue, index, 1); if (conf->new_in) { /* Set new input queue */ conf->in_queue = (struct ring_buffer *) conf->new_in; conf->new_in = NULL; pthread_cond_signal(&conf->in_q_cond); continue; } /* Stop manager */ break; } odid = (conf->perman_odid_merge || conf->manager_mode == OM_SINGLE) ? 0 : msg->input_info->odid; /* Get appropriate data Manager config according to ODID */ data_config = get_data_mngmt_config(odid, conf->data_managers); if (data_config == NULL) { /* * No data manager config for this observation domain ID found - * we have a new observation domain ID, so create new data manager for * it */ data_config = data_manager_create(odid, conf->storage_plugins); if (data_config == NULL) { MSG_WARNING(msg_module, "[%u] Unable to create Data Manager; skipping data...", odid); rbuffer_remove_reference(conf->in_queue, index, 1); continue; } /* Add config to data_mngmts structure */ output_manager_insert(conf, data_config); MSG_INFO(msg_module, "[%u] Data Manager created", odid); } if (msg->source_status == SOURCE_STATUS_NEW) { /* New source, increment reference counter */ MSG_DEBUG(msg_module, "[%u] New source", data_config->observation_domain_id); data_config->references++; /* Add input info for the statistics thread to read */ add_input_info(msg->input_info); } else if (msg->source_status == SOURCE_STATUS_CLOSED) { /* Source closed, decrement reference counter */ MSG_DEBUG(msg_module, "[%u] Closed source", data_config->observation_domain_id); data_config->references--; /* Remove a reference to a template record */ uint32_t original_odid = msg->input_info->odid; uint32_t source_crc = preprocessor_compute_crc(msg->input_info); if (tm_source_unregister(template_mgr, original_odid, source_crc)) { MSG_ERROR(msg_module, "[%u] Unable to unregister the source from the main template manager!", data_config->observation_domain_id); } /* Remove input_info from statistics */ remove_input_info(msg->input_info); if (data_config->references == 0) { /* No reference for this ODID, close DM */ MSG_DEBUG(msg_module, "[%u] No source; releasing templates...", data_config->observation_domain_id); output_manager_remove(conf, data_config); } rbuffer_remove_reference(conf->in_queue, index, 1); continue; } /* Write data into input queue of Storage Plugins */ if (rbuffer_write(data_config->store_queue, msg, data_config->plugins_count) != 0) { MSG_WARNING(msg_module, "[%u] Unable to write into Data Manager input queue; skipping data...", data_config->observation_domain_id); rbuffer_remove_reference(conf->in_queue, index, 1); free(msg); continue; } /* Remove data from queue (without memory deallocation) */ rbuffer_remove_reference(conf->in_queue, index, 0); } MSG_INFO(msg_module, "Closing Output Manager thread"); return (void *) 0; }