void amqp_cache_purge(struct chained_cache *queue[], int index) { struct pkt_primitives *data = NULL; struct pkt_bgp_primitives *pbgp = NULL; struct pkt_nat_primitives *pnat = NULL; struct pkt_mpls_primitives *pmpls = NULL; char *pcust = NULL; struct pkt_bgp_primitives empty_pbgp; struct pkt_nat_primitives empty_pnat; struct pkt_mpls_primitives empty_pmpls; char *empty_pcust = NULL; struct amqp_basic_properties_t_ amqp_msg_props; char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN], dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN]; char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], dyn_amqp_routing_key[SRVBUFLEN], *orig_amqp_routing_key = NULL; char default_amqp_exchange[] = "pmacct", default_amqp_exchange_type[] = "direct"; char default_amqp_routing_key[] = "acct"; int i, j, amqp_status, batch_idx, is_routing_key_dyn = FALSE; time_t start, duration; pid_t writer_pid = getpid(); amqp_connection_state_t amqp_conn; amqp_socket_t *amqp_socket = NULL; amqp_rpc_reply_t amqp_ret; /* setting some defaults */ if (!config.sql_db) config.sql_db = default_amqp_exchange; if (!config.sql_table) config.sql_table = default_amqp_routing_key; else { if (strchr(config.sql_table, '$')) { is_routing_key_dyn = TRUE; orig_amqp_routing_key = config.sql_table; config.sql_table = dyn_amqp_routing_key; } } if (!config.amqp_exchange_type) config.amqp_exchange_type = default_amqp_exchange_type; empty_pcust = malloc(config.cpptrs.len); if (!empty_pcust) { Log(LOG_ERR, "ERROR ( %s/%s ): Unable to malloc() empty_pcust. Exiting.\n", config.name, config.type); exit_plugin(1); } memset(&empty_pbgp, 0, sizeof(struct pkt_bgp_primitives)); memset(&empty_pnat, 0, sizeof(struct pkt_nat_primitives)); memset(&empty_pmpls, 0, sizeof(struct pkt_mpls_primitives)); memset(empty_pcust, 0, config.cpptrs.len); memset(&amqp_msg_props, 0, sizeof(amqp_msg_props)); amqp_conn = amqp_new_connection(); amqp_socket = amqp_tcp_socket_new(amqp_conn); if (!socket) { Log(LOG_ERR, "ERROR ( %s/%s ): Connection failed to RabbitMQ: no socket\n", config.name, config.type); return; } if (config.sql_host) amqp_status = amqp_socket_open(amqp_socket, config.sql_host, 5672/* default port */); else amqp_status = amqp_socket_open(amqp_socket, "127.0.0.1", 5672 /* default port */); if (amqp_status) { Log(LOG_ERR, "ERROR ( %s/%s ): Connection failed to RabbitMQ: unable to open socket\n", config.name, config.type); return; } amqp_ret = amqp_login(amqp_conn, "/", 0, 131072, 0, AMQP_SASL_METHOD_PLAIN, config.sql_user, config.sql_passwd); if (amqp_ret.reply_type != AMQP_RESPONSE_NORMAL) { Log(LOG_ERR, "ERROR ( %s/%s ): Connection failed to RabbitMQ: login\n", config.name, config.type); return; } amqp_channel_open(amqp_conn, 1); amqp_ret = amqp_get_rpc_reply(amqp_conn); if (amqp_ret.reply_type != AMQP_RESPONSE_NORMAL) { Log(LOG_ERR, "ERROR ( %s/%s ): Connection failed to RabbitMQ: unable to open channel\n", config.name, config.type); return; } amqp_exchange_declare(amqp_conn, 1, amqp_cstring_bytes(config.sql_db), amqp_cstring_bytes(config.amqp_exchange_type), 0, 0, amqp_empty_table); amqp_ret = amqp_get_rpc_reply(amqp_conn); if (amqp_ret.reply_type != AMQP_RESPONSE_NORMAL) { Log(LOG_ERR, "ERROR ( %s/%s ): Connection failed to RabbitMQ: exchange declare\n", config.name, config.type); return; } if (config.amqp_persistent_msg) { amqp_msg_props._flags = (AMQP_BASIC_CONTENT_TYPE_FLAG | AMQP_BASIC_DELIVERY_MODE_FLAG); amqp_msg_props.content_type = amqp_cstring_bytes("text/json"); amqp_msg_props.delivery_mode = 2; /* persistent delivery */ } Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - START (PID: %u) ***\n", config.name, config.type, writer_pid); start = time(NULL); for (j = 0; j < index; j++) { char *json_str; data = &queue[j]->primitives; if (queue[j]->pbgp) pbgp = queue[j]->pbgp; else pbgp = &empty_pbgp; if (queue[j]->pnat) pnat = queue[j]->pnat; else pnat = &empty_pnat; if (queue[j]->pmpls) pmpls = queue[j]->pmpls; else pmpls = &empty_pmpls; if (queue[j]->pcust) pcust = queue[j]->pcust; else pcust = empty_pcust; if (queue[j]->valid == PRINT_CACHE_FREE) continue; json_str = compose_json(config.what_to_count, config.what_to_count_2, queue[j]->flow_type, &queue[j]->primitives, pbgp, pnat, pmpls, pcust, queue[j]->bytes_counter, queue[j]->packet_counter, queue[j]->flow_counter, queue[j]->tcp_flags, &queue[j]->basetime); if (json_str) { if (is_routing_key_dyn) amqp_handle_routing_key_dyn_strings(config.sql_table, SRVBUFLEN, orig_amqp_routing_key, queue[j]); if (config.debug) Log(LOG_DEBUG, "DEBUG ( %s/%s ): publishing [E=%s RK=%s DM=%u]: %s\n", config.name, config.type, config.sql_db, config.sql_table, amqp_msg_props.delivery_mode, json_str); amqp_basic_publish(amqp_conn, 1, amqp_cstring_bytes(config.sql_db), amqp_cstring_bytes(config.sql_table), 0, 0, &amqp_msg_props, amqp_cstring_bytes(json_str)); amqp_ret = amqp_get_rpc_reply(amqp_conn); if (amqp_ret.reply_type != AMQP_RESPONSE_NORMAL) { Log(LOG_ERR, "ERROR ( %s/%s ): Connection failed to RabbitMQ: publishing\n", config.name, config.type); return; } free(json_str); } } amqp_channel_close(amqp_conn, 1, AMQP_REPLY_SUCCESS); amqp_connection_close(amqp_conn, AMQP_REPLY_SUCCESS); amqp_destroy_connection(amqp_conn); duration = time(NULL)-start; Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - END (PID: %u, QN: %u, ET: %u) ***\n", config.name, config.type, writer_pid, index, duration); if (config.sql_trigger_exec) P_trigger_exec(config.sql_trigger_exec); }
void amqp_cache_purge(struct chained_cache *queue[], int index) { struct pkt_primitives *data = NULL; struct pkt_bgp_primitives *pbgp = NULL; struct pkt_nat_primitives *pnat = NULL; struct pkt_mpls_primitives *pmpls = NULL; char *pcust = NULL; struct pkt_vlen_hdr_primitives *pvlen = NULL; struct pkt_bgp_primitives empty_pbgp; struct pkt_nat_primitives empty_pnat; struct pkt_mpls_primitives empty_pmpls; char *empty_pcust = NULL; char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN], dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN]; char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], dyn_amqp_routing_key[SRVBUFLEN], *orig_amqp_routing_key = NULL; int i, j, stop, batch_idx, is_routing_key_dyn = FALSE, qn = 0, ret, saved_index = index; int mv_num = 0, mv_num_save = 0; time_t start, duration; pid_t writer_pid = getpid(); #ifdef WITH_JANSSON json_t *array = json_array(); #endif /* setting some defaults */ if (!config.sql_host) config.sql_host = default_amqp_host; if (!config.sql_db) config.sql_db = default_amqp_exchange; if (!config.amqp_exchange_type) config.amqp_exchange_type = default_amqp_exchange_type; if (!config.amqp_vhost) config.amqp_vhost = default_amqp_vhost; if (!config.sql_table) config.sql_table = default_amqp_routing_key; else { if (strchr(config.sql_table, '$')) { is_routing_key_dyn = TRUE; orig_amqp_routing_key = config.sql_table; config.sql_table = dyn_amqp_routing_key; } } if (config.amqp_routing_key_rr) { orig_amqp_routing_key = config.sql_table; config.sql_table = dyn_amqp_routing_key; } p_amqp_set_exchange(&amqpp_amqp_host, config.sql_db); p_amqp_set_routing_key(&amqpp_amqp_host, config.sql_table); p_amqp_set_exchange_type(&amqpp_amqp_host, config.amqp_exchange_type); p_amqp_set_host(&amqpp_amqp_host, config.sql_host); p_amqp_set_vhost(&amqpp_amqp_host, config.amqp_vhost); p_amqp_set_persistent_msg(&amqpp_amqp_host, config.amqp_persistent_msg); p_amqp_set_frame_max(&amqpp_amqp_host, config.amqp_frame_max); p_amqp_set_content_type_json(&amqpp_amqp_host); p_amqp_init_routing_key_rr(&amqpp_amqp_host); p_amqp_set_routing_key_rr(&amqpp_amqp_host, config.amqp_routing_key_rr); empty_pcust = malloc(config.cpptrs.len); if (!empty_pcust) { Log(LOG_ERR, "ERROR ( %s/%s ): Unable to malloc() empty_pcust. Exiting.\n", config.name, config.type); exit_plugin(1); } memset(&empty_pbgp, 0, sizeof(struct pkt_bgp_primitives)); memset(&empty_pnat, 0, sizeof(struct pkt_nat_primitives)); memset(&empty_pmpls, 0, sizeof(struct pkt_mpls_primitives)); memset(empty_pcust, 0, config.cpptrs.len); ret = p_amqp_connect_to_publish(&amqpp_amqp_host); if (ret) return; for (j = 0, stop = 0; (!stop) && P_preprocess_funcs[j]; j++) stop = P_preprocess_funcs[j](queue, &index, j); Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - START (PID: %u) ***\n", config.name, config.type, writer_pid); start = time(NULL); for (j = 0; j < index; j++) { char *json_str; if (queue[j]->valid != PRINT_CACHE_COMMITTED) continue; data = &queue[j]->primitives; if (queue[j]->pbgp) pbgp = queue[j]->pbgp; else pbgp = &empty_pbgp; if (queue[j]->pnat) pnat = queue[j]->pnat; else pnat = &empty_pnat; if (queue[j]->pmpls) pmpls = queue[j]->pmpls; else pmpls = &empty_pmpls; if (queue[j]->pcust) pcust = queue[j]->pcust; else pcust = empty_pcust; if (queue[j]->pvlen) pvlen = queue[j]->pvlen; else pvlen = NULL; if (queue[j]->valid == PRINT_CACHE_FREE) continue; json_str = compose_json(config.what_to_count, config.what_to_count_2, queue[j]->flow_type, &queue[j]->primitives, pbgp, pnat, pmpls, pcust, pvlen, queue[j]->bytes_counter, queue[j]->packet_counter, queue[j]->flow_counter, queue[j]->tcp_flags, &queue[j]->basetime, queue[j]->stitch); #ifdef WITH_JANSSON if (json_str && config.sql_multi_values) { json_t *elem = NULL; char *tmp_str = json_str; int do_free = FALSE; if (json_array_size(array) >= config.sql_multi_values) { json_str = json_dumps(array, 0); json_array_clear(array); mv_num_save = mv_num; mv_num = 0; } else do_free = TRUE; elem = json_loads(tmp_str, 0, NULL); json_array_append_new(array, elem); mv_num++; if (do_free) { free(json_str); json_str = NULL; } } #endif if (json_str) { if (is_routing_key_dyn) { amqp_handle_routing_key_dyn_strings(dyn_amqp_routing_key, SRVBUFLEN, orig_amqp_routing_key, queue[j]); p_amqp_set_routing_key(&amqpp_amqp_host, dyn_amqp_routing_key); } if (config.amqp_routing_key_rr) { p_amqp_handle_routing_key_dyn_rr(dyn_amqp_routing_key, SRVBUFLEN, orig_amqp_routing_key, &amqpp_amqp_host.rk_rr); p_amqp_set_routing_key(&amqpp_amqp_host, dyn_amqp_routing_key); } ret = p_amqp_publish_string(&amqpp_amqp_host, json_str); free(json_str); json_str = NULL; if (!ret) { if (!config.sql_multi_values) qn++; else qn += mv_num_save; } else break; } } #ifdef WITH_JANSSON if (config.sql_multi_values && json_array_size(array)) { char *json_str; json_str = json_dumps(array, 0); json_array_clear(array); json_decref(array); if (json_str) { /* no handling of dyn routing keys here: not compatible */ ret = p_amqp_publish_string(&amqpp_amqp_host, json_str); free(json_str); json_str = NULL; if (!ret) qn += mv_num; } } #endif p_amqp_close(&amqpp_amqp_host, FALSE); duration = time(NULL)-start; Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - END (PID: %u, QN: %u/%u, ET: %u) ***\n", config.name, config.type, writer_pid, qn, saved_index, duration); if (config.sql_trigger_exec) P_trigger_exec(config.sql_trigger_exec); if (empty_pcust) free(empty_pcust); }