void fill_pipe_buffer() { struct channels_list_entry *chptr; int index; for (index = 0; channels_list[index].aggregation || channels_list[index].aggregation_2; index++) { chptr = &channels_list[index]; chptr->hdr.seq++; chptr->hdr.seq %= MAX_SEQNUM; ((struct ch_buf_hdr *)chptr->rg.ptr)->seq = chptr->hdr.seq; ((struct ch_buf_hdr *)chptr->rg.ptr)->num = chptr->hdr.num; ((struct ch_buf_hdr *)chptr->rg.ptr)->core_pid = chptr->core_pid; if (chptr->plugin->cfg.pipe_amqp) { #ifdef WITH_RABBITMQ p_amqp_publish_binary(&chptr->amqp_host, chptr->rg.ptr, chptr->bufsize); #endif } else if (chptr->plugin->cfg.pipe_kafka) { #ifdef WITH_KAFKA p_kafka_produce_data(&chptr->kafka_host, chptr->rg.ptr, chptr->bufsize); #endif } else { if (chptr->status->wakeup) { chptr->status->wakeup = chptr->request; if (write(chptr->pipe, &chptr->rg.ptr, CharPtrSz) != CharPtrSz) Log(LOG_WARNING, "WARN ( %s/%s ): Failed during write: %s\n", chptr->plugin->cfg.name, chptr->plugin->cfg.type, strerror(errno)); } } } }
void exec_plugins(struct packet_ptrs *pptrs, struct plugin_requests *req) { int saved_have_tag = FALSE, saved_have_tag2 = FALSE, saved_have_label = FALSE; pm_id_t saved_tag = 0, saved_tag2 = 0; pt_label_t saved_label; int num, ret, fixed_size, already_reprocessed = 0; u_int32_t savedptr; char *bptr; int index, got_tags = FALSE; pretag_init_label(&saved_label); #if defined WITH_GEOIPV2 if (reload_geoipv2_file && config.geoipv2_file) { pm_geoipv2_close(); pm_geoipv2_init(); reload_geoipv2_file = FALSE; } #endif for (index = 0; channels_list[index].aggregation || channels_list[index].aggregation_2; index++) { struct plugins_list_entry *p = channels_list[index].plugin; if (p->cfg.pre_tag_map && find_id_func) { if (p->cfg.ptm_global && got_tags) { pptrs->tag = saved_tag; pptrs->tag2 = saved_tag2; pretag_copy_label(&pptrs->label, &saved_label); pptrs->have_tag = saved_have_tag; pptrs->have_tag2 = saved_have_tag2; pptrs->have_label = saved_have_label; } else { find_id_func(&p->cfg.ptm, pptrs, &pptrs->tag, &pptrs->tag2); if (p->cfg.ptm_global) { saved_tag = pptrs->tag; saved_tag2 = pptrs->tag2; pretag_copy_label(&saved_label, &pptrs->label); saved_have_tag = pptrs->have_tag; saved_have_tag2 = pptrs->have_tag2; saved_have_label = pptrs->have_label; got_tags = TRUE; } } } if (evaluate_filters(&channels_list[index].agg_filter, pptrs->packet_ptr, pptrs->pkthdr) && !evaluate_tags(&channels_list[index].tag_filter, pptrs->tag) && !evaluate_tags(&channels_list[index].tag2_filter, pptrs->tag2) && !evaluate_labels(&channels_list[index].label_filter, &pptrs->label) && !check_shadow_status(pptrs, &channels_list[index])) { /* arranging buffer: supported primitives + packet total length */ reprocess: channels_list[index].reprocess = FALSE; num = 0; /* rg.ptr points to slot's base address into the ring (shared memory); bufptr works as a displacement into the slot to place sequentially packets */ bptr = channels_list[index].rg.ptr+ChBufHdrSz+channels_list[index].bufptr; fixed_size = (*channels_list[index].clean_func)(bptr, channels_list[index].datasize); channels_list[index].var_size = 0; savedptr = channels_list[index].bufptr; reset_fallback_status(pptrs); while (channels_list[index].phandler[num]) { (*channels_list[index].phandler[num])(&channels_list[index], pptrs, &bptr); num++; } if (channels_list[index].s.rate && !channels_list[index].s.sampled_pkts) { channels_list[index].reprocess = FALSE; channels_list[index].bufptr = savedptr; channels_list[index].hdr.num--; /* let's cheat this value as it will get increased later */ fixed_size = 0; channels_list[index].var_size = 0; } if (channels_list[index].reprocess) { /* Let's check if we have an issue with the buffer size */ if (already_reprocessed) { struct plugins_list_entry *list = channels_list[index].plugin; Log(LOG_ERR, "ERROR ( %s/%s ): plugin_buffer_size is too short.\n", list->name, list->type.string); exit_all(1); } already_reprocessed = TRUE; /* Let's cheat the size in order to send out the current buffer */ fixed_size = channels_list[index].plugin->cfg.pipe_size; } else { channels_list[index].hdr.num++; channels_list[index].bufptr += (fixed_size + channels_list[index].var_size); } if ((channels_list[index].bufptr+fixed_size) > channels_list[index].bufend || channels_list[index].hdr.num == INT_MAX) { channels_list[index].hdr.seq++; channels_list[index].hdr.seq %= MAX_SEQNUM; /* let's commit the buffer we just finished writing */ ((struct ch_buf_hdr *)channels_list[index].rg.ptr)->seq = channels_list[index].hdr.seq; ((struct ch_buf_hdr *)channels_list[index].rg.ptr)->num = channels_list[index].hdr.num; ((struct ch_buf_hdr *)channels_list[index].rg.ptr)->core_pid = channels_list[index].core_pid; if (config.debug_internal_msg) { struct plugins_list_entry *list = channels_list[index].plugin; Log(LOG_DEBUG, "DEBUG ( %s/%s ): buffer released cpid=%u seq=%u num_entries=%u\n", list->name, list->type.string, channels_list[index].core_pid, channels_list[index].hdr.seq, channels_list[index].hdr.num); } /* sending the buffer to the AMQP broker */ if (channels_list[index].plugin->cfg.pipe_amqp) { #ifdef WITH_RABBITMQ struct channels_list_entry *chptr = &channels_list[index]; plugin_pipe_amqp_sleeper_stop(chptr); if (!chptr->amqp_host_sleep) ret = p_amqp_publish_binary(&chptr->amqp_host, chptr->rg.ptr, chptr->bufsize); else ret = FALSE; if (ret) plugin_pipe_amqp_sleeper_start(chptr); #endif } /* sending the buffer to the Kafka broker */ else if (channels_list[index].plugin->cfg.pipe_kafka) { #ifdef WITH_KAFKA struct channels_list_entry *chptr = &channels_list[index]; /* XXX: no sleeper thread, trusting librdkafka */ ret = p_kafka_produce_data(&chptr->kafka_host, chptr->rg.ptr, chptr->bufsize); #endif } else { if (channels_list[index].status->wakeup) { channels_list[index].status->backlog++; if (channels_list[index].status->backlog > ((channels_list[index].plugin->cfg.pipe_size/channels_list[index].plugin->cfg.buffer_size) *channels_list[index].plugin->cfg.pipe_backlog)/100) { channels_list[index].status->wakeup = channels_list[index].request; if (write(channels_list[index].pipe, &channels_list[index].rg.ptr, CharPtrSz) != CharPtrSz) { struct plugins_list_entry *list = channels_list[index].plugin; Log(LOG_WARNING, "WARN ( %s/%s ): Failed during write: %s\n", list->name, list->type.string, strerror(errno)); } channels_list[index].status->backlog = 0; } } } channels_list[index].rg.ptr += channels_list[index].bufsize; if ((channels_list[index].rg.ptr+channels_list[index].bufsize) > channels_list[index].rg.end) channels_list[index].rg.ptr = channels_list[index].rg.base; /* let's protect the buffer we are going to write */ ((struct ch_buf_hdr *)channels_list[index].rg.ptr)->seq = -1; ((struct ch_buf_hdr *)channels_list[index].rg.ptr)->num = 0; ((struct ch_buf_hdr *)channels_list[index].rg.ptr)->core_pid = 0; /* rewind pointer */ channels_list[index].bufptr = channels_list[index].buf; channels_list[index].hdr.num = 0; if (channels_list[index].reprocess) goto reprocess; /* if reading from a savefile, let's sleep a bit after having sent over a buffer worth of data */ if (channels_list[index].plugin->cfg.pcap_savefile) usleep(1000); /* 1 msec */ } } pptrs->tag = 0; pptrs->tag2 = 0; pretag_free_label(&pptrs->label); } /* check if we have to reload the map: new loop is to ensure we reload it for all plugins and prevent any timing issues with pointers to labels */ if (reload_map_exec_plugins) { for (index = 0; channels_list[index].aggregation || channels_list[index].aggregation_2; index++) { struct plugins_list_entry *p = channels_list[index].plugin; if (p->cfg.pre_tag_map && find_id_func) { load_pre_tag_map(config.acct_type, p->cfg.pre_tag_map, &p->cfg.ptm, req, &p->cfg.ptm_alloc, p->cfg.maps_entries, p->cfg.maps_row_len); } } } /* cleanups */ reload_map_exec_plugins = FALSE; pretag_free_label(&saved_label); }
void amqp_cache_purge(struct chained_cache *queue[], int index, int safe_action) { struct pkt_primitives *data = NULL; struct pkt_bgp_primitives *pbgp = NULL; struct pkt_nat_primitives *pnat = NULL; struct pkt_mpls_primitives *pmpls = NULL; struct pkt_tunnel_primitives *ptun = NULL; char *pcust = NULL; struct pkt_vlen_hdr_primitives *pvlen = NULL; struct pkt_bgp_primitives empty_pbgp; struct pkt_nat_primitives empty_pnat; struct pkt_mpls_primitives empty_pmpls; struct pkt_tunnel_primitives empty_ptun; char *empty_pcust = NULL; char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN], dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN]; char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], dyn_amqp_routing_key[SRVBUFLEN], *orig_amqp_routing_key = NULL; int i, j, stop, batch_idx, is_routing_key_dyn = FALSE, qn = 0, ret, saved_index = index; int mv_num = 0, mv_num_save = 0; time_t start, duration; pid_t writer_pid = getpid(); char *json_buf = NULL; int json_buf_off = 0; #ifdef WITH_AVRO avro_writer_t avro_writer; char *avro_buf = NULL; int avro_buffer_full = FALSE; #endif /* setting some defaults */ if (!config.sql_host) config.sql_host = default_amqp_host; if (!config.sql_db) config.sql_db = default_amqp_exchange; if (!config.amqp_exchange_type) config.amqp_exchange_type = default_amqp_exchange_type; if (!config.amqp_vhost) config.amqp_vhost = default_amqp_vhost; if (!config.sql_table) config.sql_table = default_amqp_routing_key; else { if (strchr(config.sql_table, '$')) { is_routing_key_dyn = TRUE; orig_amqp_routing_key = config.sql_table; config.sql_table = dyn_amqp_routing_key; } } if (config.amqp_routing_key_rr) { orig_amqp_routing_key = config.sql_table; config.sql_table = dyn_amqp_routing_key; } p_amqp_set_exchange(&amqpp_amqp_host, config.sql_db); p_amqp_set_routing_key(&amqpp_amqp_host, config.sql_table); p_amqp_set_exchange_type(&amqpp_amqp_host, config.amqp_exchange_type); p_amqp_set_host(&amqpp_amqp_host, config.sql_host); p_amqp_set_vhost(&amqpp_amqp_host, config.amqp_vhost); p_amqp_set_persistent_msg(&amqpp_amqp_host, config.amqp_persistent_msg); p_amqp_set_frame_max(&amqpp_amqp_host, config.amqp_frame_max); if (config.message_broker_output & PRINT_OUTPUT_JSON) p_amqp_set_content_type_json(&amqpp_amqp_host); else if (config.message_broker_output & PRINT_OUTPUT_AVRO) p_amqp_set_content_type_binary(&amqpp_amqp_host); else { Log(LOG_ERR, "ERROR ( %s/%s ): Unsupported amqp_output value specified. Exiting.\n", config.name, config.type); exit_plugin(1); } p_amqp_init_routing_key_rr(&amqpp_amqp_host); p_amqp_set_routing_key_rr(&amqpp_amqp_host, config.amqp_routing_key_rr); empty_pcust = malloc(config.cpptrs.len); if (!empty_pcust) { Log(LOG_ERR, "ERROR ( %s/%s ): Unable to malloc() empty_pcust. Exiting.\n", config.name, config.type); exit_plugin(1); } memset(&empty_pbgp, 0, sizeof(struct pkt_bgp_primitives)); memset(&empty_pnat, 0, sizeof(struct pkt_nat_primitives)); memset(&empty_pmpls, 0, sizeof(struct pkt_mpls_primitives)); memset(&empty_ptun, 0, sizeof(struct pkt_tunnel_primitives)); memset(empty_pcust, 0, config.cpptrs.len); ret = p_amqp_connect_to_publish(&amqpp_amqp_host); if (ret) return; for (j = 0, stop = 0; (!stop) && P_preprocess_funcs[j]; j++) stop = P_preprocess_funcs[j](queue, &index, j); Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - START (PID: %u) ***\n", config.name, config.type, writer_pid); start = time(NULL); if (config.print_markers) { if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) { void *json_obj; char *json_str; json_obj = compose_purge_init_json(config.name, writer_pid); if (json_obj) json_str = compose_json_str(json_obj); if (json_str) { Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str); ret = p_amqp_publish_string(&amqpp_amqp_host, json_str); free(json_str); json_str = NULL; } } } if (config.message_broker_output & PRINT_OUTPUT_JSON) { if (config.sql_multi_values) { json_buf = malloc(config.sql_multi_values); if (!json_buf) { Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (json_buf). Exiting ..\n", config.name, config.type); exit_plugin(1); } else memset(json_buf, 0, config.sql_multi_values); } } else if (config.message_broker_output & PRINT_OUTPUT_AVRO) { #ifdef WITH_AVRO if (!config.avro_buffer_size) config.avro_buffer_size = LARGEBUFLEN; avro_buf = malloc(config.avro_buffer_size); if (!avro_buf) { Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (avro_buf). Exiting ..\n", config.name, config.type); exit_plugin(1); } avro_writer = avro_writer_memory(avro_buf, config.avro_buffer_size); #endif } for (j = 0; j < index; j++) { void *json_obj; char *json_str; if (queue[j]->valid != PRINT_CACHE_COMMITTED) continue; data = &queue[j]->primitives; if (queue[j]->pbgp) pbgp = queue[j]->pbgp; else pbgp = &empty_pbgp; if (queue[j]->pnat) pnat = queue[j]->pnat; else pnat = &empty_pnat; if (queue[j]->pmpls) pmpls = queue[j]->pmpls; else pmpls = &empty_pmpls; if (queue[j]->ptun) ptun = queue[j]->ptun; else ptun = &empty_ptun; if (queue[j]->pcust) pcust = queue[j]->pcust; else pcust = empty_pcust; if (queue[j]->pvlen) pvlen = queue[j]->pvlen; else pvlen = NULL; if (queue[j]->valid == PRINT_CACHE_FREE) continue; if (config.message_broker_output & PRINT_OUTPUT_JSON) { #ifdef WITH_JANSSON json_t *json_obj = json_object(); int idx; for (idx = 0; idx < N_PRIMITIVES && cjhandler[idx]; idx++) cjhandler[idx](json_obj, queue[j]); add_writer_name_and_pid_json(json_obj, config.name, writer_pid); json_str = compose_json_str(json_obj); #endif } else if (config.message_broker_output & PRINT_OUTPUT_AVRO) { #ifdef WITH_AVRO avro_value_iface_t *avro_iface = avro_generic_class_from_schema(avro_acct_schema); avro_value_t avro_value = compose_avro(config.what_to_count, config.what_to_count_2, queue[j]->flow_type, &queue[j]->primitives, pbgp, pnat, pmpls, ptun, pcust, pvlen, queue[j]->bytes_counter, queue[j]->packet_counter, queue[j]->flow_counter, queue[j]->tcp_flags, &queue[j]->basetime, queue[j]->stitch, avro_iface); size_t avro_value_size; add_writer_name_and_pid_avro(avro_value, config.name, writer_pid); avro_value_sizeof(&avro_value, &avro_value_size); if (avro_value_size > config.avro_buffer_size) { Log(LOG_ERR, "ERROR ( %s/%s ): AVRO; insufficient buffer size (avro_buffer_size=%u)\n", config.name, config.type, config.avro_buffer_size); Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: increase value or look for avro_buffer_size in CONFIG-KEYS document.\n\n", config.name, config.type); exit_plugin(1); } else if (avro_value_size >= (config.avro_buffer_size - avro_writer_tell(avro_writer))) { avro_buffer_full = TRUE; j--; } else if (avro_value_write(avro_writer, &avro_value)) { Log(LOG_ERR, "ERROR ( %s/%s ): ARVO: unable to write value: %s\n", config.name, config.type, avro_strerror()); exit_plugin(1); } else { mv_num++; } avro_value_decref(&avro_value); avro_value_iface_decref(avro_iface); #else if (config.debug) Log(LOG_DEBUG, "DEBUG ( %s/%s ): compose_avro(): AVRO object not created due to missing --enable-avro\n", config.name, config.type); #endif } if (config.message_broker_output & PRINT_OUTPUT_JSON) { char *tmp_str = NULL; if (json_str && config.sql_multi_values) { int json_strlen = (strlen(json_str) ? (strlen(json_str) + 1) : 0); if (json_strlen >= (config.sql_multi_values - json_buf_off)) { if (json_strlen >= config.sql_multi_values) { Log(LOG_ERR, "ERROR ( %s/%s ): amqp_multi_values not large enough to store JSON elements. Exiting ..\n", config.name, config.type); exit(1); } tmp_str = json_str; json_str = json_buf; } else { strcat(json_buf, json_str); mv_num++; string_add_newline(json_buf); json_buf_off = strlen(json_buf); free(json_str); json_str = NULL; } } if (json_str) { if (is_routing_key_dyn) { P_handle_table_dyn_strings(dyn_amqp_routing_key, SRVBUFLEN, orig_amqp_routing_key, queue[j]); p_amqp_set_routing_key(&amqpp_amqp_host, dyn_amqp_routing_key); } if (config.amqp_routing_key_rr) { P_handle_table_dyn_rr(dyn_amqp_routing_key, SRVBUFLEN, orig_amqp_routing_key, &amqpp_amqp_host.rk_rr); p_amqp_set_routing_key(&amqpp_amqp_host, dyn_amqp_routing_key); } Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str); ret = p_amqp_publish_string(&amqpp_amqp_host, json_str); if (config.sql_multi_values) { json_str = tmp_str; strcpy(json_buf, json_str); mv_num_save = mv_num; mv_num = 1; string_add_newline(json_buf); json_buf_off = strlen(json_buf); } free(json_str); json_str = NULL; if (!ret) { if (!config.sql_multi_values) qn++; else qn += mv_num_save; } else break; } } else if (config.message_broker_output & PRINT_OUTPUT_AVRO) { #ifdef WITH_AVRO if (!config.sql_multi_values || (mv_num >= config.sql_multi_values) || avro_buffer_full) { if (is_routing_key_dyn) { P_handle_table_dyn_strings(dyn_amqp_routing_key, SRVBUFLEN, orig_amqp_routing_key, queue[j]); p_amqp_set_routing_key(&amqpp_amqp_host, dyn_amqp_routing_key); } if (config.amqp_routing_key_rr) { P_handle_table_dyn_rr(dyn_amqp_routing_key, SRVBUFLEN, orig_amqp_routing_key, &amqpp_amqp_host.rk_rr); p_amqp_set_routing_key(&amqpp_amqp_host, dyn_amqp_routing_key); } ret = p_amqp_publish_binary(&amqpp_amqp_host, avro_buf, avro_writer_tell(avro_writer)); avro_writer_reset(avro_writer); avro_buffer_full = FALSE; mv_num_save = mv_num; mv_num = 0; if (!ret) qn += mv_num_save; else break; } #endif } } if (config.sql_multi_values) { if (config.message_broker_output & PRINT_OUTPUT_JSON) { if (json_buf && json_buf_off) { /* no handling of dyn routing keys here: not compatible */ Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_buf); ret = p_amqp_publish_string(&amqpp_amqp_host, json_buf); if (!ret) qn += mv_num; } } else if (config.message_broker_output & PRINT_OUTPUT_AVRO) { #ifdef WITH_AVRO if (avro_writer_tell(avro_writer)) { ret = p_amqp_publish_binary(&amqpp_amqp_host, avro_buf, avro_writer_tell(avro_writer)); avro_writer_free(avro_writer); if (!ret) qn += mv_num; } #endif } } duration = time(NULL)-start; if (config.print_markers) { if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) { void *json_obj; char *json_str; json_obj = compose_purge_close_json(config.name, writer_pid, qn, saved_index, duration); if (json_obj) json_str = compose_json_str(json_obj); if (json_str) { Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str); ret = p_amqp_publish_string(&amqpp_amqp_host, json_str); free(json_str); json_str = NULL; } } } p_amqp_close(&amqpp_amqp_host, FALSE); Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - END (PID: %u, QN: %u/%u, ET: %u) ***\n", config.name, config.type, writer_pid, qn, saved_index, duration); if (config.sql_trigger_exec && !safe_action) P_trigger_exec(config.sql_trigger_exec); if (empty_pcust) free(empty_pcust); if (json_buf) free(json_buf); #ifdef WITH_AVRO if (avro_buf) free(avro_buf); #endif }