/* Functions */
void kafka_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)
{
struct pkt_data *data;
struct ports_table pt;
unsigned char *pipebuf;
struct pollfd pfd;
struct insert_data idata;
time_t t, avro_schema_deadline = 0;
int timeout, refresh_timeout, avro_schema_timeout = 0;
int ret, num;
struct ring *rg = &((struct channels_list_entry *)ptr)->rg;
struct ch_status *status = ((struct channels_list_entry *)ptr)->status;
struct plugins_list_entry *plugin_data = ((struct channels_list_entry *)ptr)->plugin;
int datasize = ((struct channels_list_entry *)ptr)->datasize;
u_int32_t bufsz = ((struct channels_list_entry *)ptr)->bufsize;
pid_t core_pid = ((struct channels_list_entry *)ptr)->core_pid;
struct networks_file_data nfd;
unsigned char *rgptr;
int pollagain = TRUE;
u_int32_t seq = 1, rg_err_count = 0;
struct extra_primitives extras;
struct primitives_ptrs prim_ptrs;
char *dataptr;
#ifdef WITH_AVRO
char *avro_acct_schema_str;
#endif
#ifdef WITH_ZMQ
struct p_zmq_host *zmq_host = &((struct channels_list_entry *)ptr)->zmq_host;
#endif
memcpy(&config, cfgptr, sizeof(struct configuration));
memcpy(&extras, &((struct channels_list_entry *)ptr)->extras, sizeof(struct extra_primitives));
recollect_pipe_memory(ptr);
pm_setproctitle("%s [%s]", "Kafka Plugin", config.name);
P_set_signals();
P_init_default_values();
P_config_checks();
pipebuf = (unsigned char *) pm_malloc(config.buffer_size);
memset(pipebuf, 0, config.buffer_size);
timeout = config.sql_refresh_time*1000;
if (!config.message_broker_output) config.message_broker_output = PRINT_OUTPUT_JSON;
if (config.message_broker_output & PRINT_OUTPUT_JSON) {
compose_json(config.what_to_count, config.what_to_count_2);
}
else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
avro_acct_schema = build_avro_schema(config.what_to_count, config.what_to_count_2);
avro_schema_add_writer_id(avro_acct_schema);
if (config.avro_schema_output_file) write_avro_schema_to_file(config.avro_schema_output_file, avro_acct_schema);
if (config.kafka_avro_schema_topic) {
if (!config.kafka_avro_schema_refresh_time)
config.kafka_avro_schema_refresh_time = DEFAULT_AVRO_SCHEMA_REFRESH_TIME;
avro_schema_deadline = time(NULL);
P_init_refresh_deadline(&avro_schema_deadline, config.kafka_avro_schema_refresh_time, 0, "m");
avro_acct_schema_str = compose_avro_purge_schema(avro_acct_schema, config.name);
}
else {
config.kafka_avro_schema_refresh_time = 0;
avro_schema_deadline = 0;
avro_schema_timeout = 0;
avro_acct_schema_str = NULL;
}
#endif
}
if ((config.sql_table && strchr(config.sql_table, '$')) && config.sql_multi_values) {
Log(LOG_ERR, "ERROR ( %s/%s ): dynamic 'kafka_topic' is not compatible with 'kafka_multi_values'. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
if ((config.sql_table && strchr(config.sql_table, '$')) && config.amqp_routing_key_rr) {
Log(LOG_ERR, "ERROR ( %s/%s ): dynamic 'kafka_topic' is not compatible with 'kafka_topic_rr'. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
/* setting function pointers */
if (config.what_to_count & (COUNT_SUM_HOST|COUNT_SUM_NET))
insert_func = P_sum_host_insert;
else if (config.what_to_count & COUNT_SUM_PORT) insert_func = P_sum_port_insert;
else if (config.what_to_count & COUNT_SUM_AS) insert_func = P_sum_as_insert;
#if defined (HAVE_L2)
else if (config.what_to_count & COUNT_SUM_MAC) insert_func = P_sum_mac_insert;
#endif
else insert_func = P_cache_insert;
purge_func = kafka_cache_purge;
memset(&nt, 0, sizeof(nt));
memset(&nc, 0, sizeof(nc));
memset(&pt, 0, sizeof(pt));
load_networks(config.networks_file, &nt, &nc);
set_net_funcs(&nt);
if (config.ports_file) load_ports(config.ports_file, &pt);
if (config.pkt_len_distrib_bins_str) load_pkt_len_distrib_bins();
else {
if (config.what_to_count_2 & COUNT_PKT_LEN_DISTRIB) {
Log(LOG_ERR, "ERROR ( %s/%s ): 'aggregate' contains pkt_len_distrib but no 'pkt_len_distrib_bins' defined. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
}
memset(&idata, 0, sizeof(idata));
memset(&prim_ptrs, 0, sizeof(prim_ptrs));
set_primptrs_funcs(&extras);
if (config.pipe_zmq) {
plugin_pipe_zmq_compile_check();
#ifdef WITH_ZMQ
p_zmq_plugin_pipe_init_plugin(zmq_host);
p_zmq_plugin_pipe_consume(zmq_host);
p_zmq_set_retry_timeout(zmq_host, config.pipe_zmq_retry);
pipe_fd = p_zmq_get_fd(zmq_host);
seq = 0;
#endif
}
else setnonblocking(pipe_fd);
idata.now = time(NULL);
/* print_refresh time init: deadline */
refresh_deadline = idata.now;
P_init_refresh_deadline(&refresh_deadline, config.sql_refresh_time, config.sql_startup_delay, config.sql_history_roundoff);
if (config.sql_history) {
basetime_init = P_init_historical_acct;
basetime_eval = P_eval_historical_acct;
basetime_cmp = P_cmp_historical_acct;
(*basetime_init)(idata.now);
}
/* setting number of entries in _protocols structure */
while (_protocols[protocols_number].number != -1) protocols_number++;
/* plugin main loop */
for(;;) {
poll_again:
status->wakeup = TRUE;
calc_refresh_timeout(refresh_deadline, idata.now, &refresh_timeout);
if (config.kafka_avro_schema_topic) calc_refresh_timeout(avro_schema_deadline, idata.now, &avro_schema_timeout);
pfd.fd = pipe_fd;
pfd.events = POLLIN;
timeout = MIN(refresh_timeout, (avro_schema_timeout ? avro_schema_timeout : INT_MAX));
ret = poll(&pfd, (pfd.fd == ERR ? 0 : 1), timeout);
if (ret <= 0) {
if (getppid() == 1) {
Log(LOG_ERR, "ERROR ( %s/%s ): Core process *seems* gone. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
if (ret < 0) goto poll_again;
}
idata.now = time(NULL);
if (config.sql_history) {
while (idata.now > (basetime.tv_sec + timeslot)) {
new_basetime.tv_sec = basetime.tv_sec;
basetime.tv_sec += timeslot;
if (config.sql_history == COUNT_MONTHLY)
timeslot = calc_monthly_timeslot(basetime.tv_sec, config.sql_history_howmany, ADD);
}
}
#ifdef WITH_AVRO
if (idata.now > avro_schema_deadline) {
kafka_avro_schema_purge(avro_acct_schema_str);
avro_schema_deadline += config.kafka_avro_schema_refresh_time;
}
#endif
switch (ret) {
case 0: /* timeout */
if (idata.now > refresh_deadline) P_cache_handle_flush_event(&pt);
break;
default: /* we received data */
read_data:
if (config.pipe_homegrown) {
if (!pollagain) {
seq++;
seq %= MAX_SEQNUM;
if (seq == 0) rg_err_count = FALSE;
}
else {
if ((ret = read(pipe_fd, &rgptr, sizeof(rgptr))) == 0)
exit_plugin(1); /* we exit silently; something happened at the write end */
}
if ((rg->ptr + bufsz) > rg->end) rg->ptr = rg->base;
if (((struct ch_buf_hdr *)rg->ptr)->seq != seq) {
if (!pollagain) {
pollagain = TRUE;
goto poll_again;
}
else {
rg_err_count++;
if (config.debug || (rg_err_count > MAX_RG_COUNT_ERR)) {
Log(LOG_WARNING, "WARN ( %s/%s ): Missing data detected (plugin_buffer_size=%llu plugin_pipe_size=%llu).\n",
config.name, config.type, config.buffer_size, config.pipe_size);
Log(LOG_WARNING, "WARN ( %s/%s ): Increase values or look for plugin_buffer_size, plugin_pipe_size in CONFIG-KEYS document.\n\n",
config.name, config.type);
}
rg->ptr = (rg->base + status->last_buf_off);
seq = ((struct ch_buf_hdr *)rg->ptr)->seq;
}
}
pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
rg->ptr += bufsz;
}
#ifdef WITH_ZMQ
else if (config.pipe_zmq) {
ret = p_zmq_plugin_pipe_recv(zmq_host, pipebuf, config.buffer_size);
if (ret > 0) {
if (seq && (((struct ch_buf_hdr *)pipebuf)->seq != ((seq + 1) % MAX_SEQNUM))) {
Log(LOG_WARNING, "WARN ( %s/%s ): Missing data detected. Sequence received=%u expected=%u\n",
config.name, config.type, ((struct ch_buf_hdr *)pipebuf)->seq, ((seq + 1) % MAX_SEQNUM));
}
seq = ((struct ch_buf_hdr *)pipebuf)->seq;
}
else goto poll_again;
}
#endif
/* lazy refresh time handling */
if (idata.now > refresh_deadline) P_cache_handle_flush_event(&pt);
data = (struct pkt_data *) (pipebuf+sizeof(struct ch_buf_hdr));
if (config.debug_internal_msg)
Log(LOG_DEBUG, "DEBUG ( %s/%s ): buffer received cpid=%u len=%llu seq=%u num_entries=%u\n",
config.name, config.type, core_pid, ((struct ch_buf_hdr *)pipebuf)->len,
seq, ((struct ch_buf_hdr *)pipebuf)->num);
if (!config.pipe_check_core_pid || ((struct ch_buf_hdr *)pipebuf)->core_pid == core_pid) {
while (((struct ch_buf_hdr *)pipebuf)->num > 0) {
for (num = 0; primptrs_funcs[num]; num++)
(*primptrs_funcs[num])((u_char *)data, &extras, &prim_ptrs);
for (num = 0; net_funcs[num]; num++)
(*net_funcs[num])(&nt, &nc, &data->primitives, prim_ptrs.pbgp, &nfd);
if (config.ports_file) {
if (!pt.table[data->primitives.src_port]) data->primitives.src_port = 0;
if (!pt.table[data->primitives.dst_port]) data->primitives.dst_port = 0;
}
if (config.pkt_len_distrib_bins_str &&
config.what_to_count_2 & COUNT_PKT_LEN_DISTRIB)
evaluate_pkt_len_distrib(data);
prim_ptrs.data = data;
(*insert_func)(&prim_ptrs, &idata);
((struct ch_buf_hdr *)pipebuf)->num--;
if (((struct ch_buf_hdr *)pipebuf)->num) {
dataptr = (unsigned char *) data;
if (!prim_ptrs.vlen_next_off) dataptr += datasize;
else dataptr += prim_ptrs.vlen_next_off;
data = (struct pkt_data *) dataptr;
}
}
}
goto read_data;
}
}
}
/* Functions */
void imt_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)
{
int maxqsize = (MAX_QUERIES*sizeof(struct pkt_primitives))+sizeof(struct query_header)+2;
struct sockaddr cAddr;
struct pkt_data *data;
struct ports_table pt;
unsigned char srvbuf[maxqsize];
unsigned char *srvbufptr;
struct query_header *qh;
unsigned char *pipebuf;
char path[] = "/tmp/collect.pipe";
short int go_to_clear = FALSE;
u_int32_t request, sz;
struct ch_status *status = ((struct channels_list_entry *)ptr)->status;
int datasize = ((struct channels_list_entry *)ptr)->datasize;
struct extra_primitives extras;
unsigned char *rgptr;
int pollagain = 0;
u_int32_t seq = 0;
int rg_err_count = 0;
int ret;
struct pkt_bgp_primitives *pbgp, empty_pbgp;
struct pkt_nat_primitives *pnat, empty_pnat;
struct pkt_mpls_primitives *pmpls, empty_pmpls;
char *pcust, empty_pcust[] = "";
struct pkt_vlen_hdr_primitives *pvlen, empty_pvlen;
struct networks_file_data nfd;
struct timeval select_timeout;
struct primitives_ptrs prim_ptrs;
fd_set read_descs, bkp_read_descs; /* select() stuff */
int select_fd, lock = FALSE;
int cLen, num, sd, sd2;
char *dataptr;
memcpy(&config, cfgptr, sizeof(struct configuration));
memcpy(&extras, &((struct channels_list_entry *)ptr)->extras, sizeof(struct extra_primitives));
recollect_pipe_memory(ptr);
pm_setproctitle("%s [%s]", "IMT Plugin", config.name);
if (config.pidfile) write_pid_file_plugin(config.pidfile, config.type, config.name);
if (config.logfile) {
fclose(config.logfile_fd);
config.logfile_fd = open_logfile(config.logfile, "a");
}
if (extras.off_pkt_vlen_hdr_primitives) {
Log(LOG_ERR, "ERROR ( %s/%s ): variable-length primitives, ie. label, are not supported in IMT plugin. Exiting ..\n", config.name, config.type);
exit_plugin(1);
}
reload_map = FALSE;
status->wakeup = TRUE;
/* a bunch of default definitions and post-checks */
pipebuf = (unsigned char *) malloc(config.buffer_size);
if (!pipebuf) {
Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (pipebuf). Exiting ..\n", config.name, config.type);
exit_plugin(1);
}
setnonblocking(pipe_fd);
memset(pipebuf, 0, config.buffer_size);
no_more_space = FALSE;
if (config.what_to_count & (COUNT_SUM_HOST|COUNT_SUM_NET))
insert_func = sum_host_insert;
else if (config.what_to_count & COUNT_SUM_PORT) insert_func = sum_port_insert;
else if (config.what_to_count & COUNT_SUM_AS) insert_func = sum_as_insert;
#if defined (HAVE_L2)
else if (config.what_to_count & COUNT_SUM_MAC) insert_func = sum_mac_insert;
#endif
else insert_func = insert_accounting_structure;
memset(&nt, 0, sizeof(nt));
memset(&nc, 0, sizeof(nc));
memset(&pt, 0, sizeof(pt));
load_networks(config.networks_file, &nt, &nc);
set_net_funcs(&nt);
if (config.ports_file) load_ports(config.ports_file, &pt);
if (config.pkt_len_distrib_bins_str) load_pkt_len_distrib_bins();
else {
if (config.what_to_count_2 & COUNT_PKT_LEN_DISTRIB) {
Log(LOG_ERR, "ERROR ( %s/%s ): 'aggregate' contains pkt_len_distrib but no 'pkt_len_distrib_bins' defined. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
}
if ((!config.num_memory_pools) && (!have_num_memory_pools))
config.num_memory_pools = NUM_MEMORY_POOLS;
if (!config.memory_pool_size) config.memory_pool_size = MEMORY_POOL_SIZE;
else {
if (config.memory_pool_size < sizeof(struct acc)) {
Log(LOG_WARNING, "WARN ( %s/%s ): enforcing memory pool's minimum size, %d bytes.\n", config.name, config.type, sizeof(struct acc));
config.memory_pool_size = MEMORY_POOL_SIZE;
}
}
if (!config.imt_plugin_path) config.imt_plugin_path = path;
if (!config.buckets) config.buckets = MAX_HOSTS;
init_memory_pool_table(config);
if (mpd == NULL) {
Log(LOG_ERR, "ERROR ( %s/%s ): unable to allocate memory pools table\n", config.name, config.type);
exit_plugin(1);
}
current_pool = request_memory_pool(config.buckets*sizeof(struct acc));
if (current_pool == NULL) {
Log(LOG_ERR, "ERROR ( %s/%s ): unable to allocate first memory pool, try with larger value.\n", config.name, config.type);
exit_plugin(1);
}
a = current_pool->base_ptr;
lru_elem_ptr = malloc(config.buckets*sizeof(struct acc *));
if (lru_elem_ptr == NULL) {
Log(LOG_ERR, "ERROR ( %s/%s ): unable to allocate LRU element pointers.\n", config.name, config.type);
exit_plugin(1);
}
else memset(lru_elem_ptr, 0, config.buckets*sizeof(struct acc *));
current_pool = request_memory_pool(config.memory_pool_size);
if (current_pool == NULL) {
Log(LOG_ERR, "ERROR ( %s/%s ): unable to allocate more memory pools, try with larger value.\n", config.name, config.type);
exit_plugin(1);
}
signal(SIGHUP, reload); /* handles reopening of syslog channel */
signal(SIGINT, exit_now); /* exit lane */
signal(SIGUSR1, SIG_IGN);
signal(SIGUSR2, reload_maps);
signal(SIGPIPE, SIG_IGN);
signal(SIGCHLD, SIG_IGN);
memset(&empty_pbgp, 0, sizeof(empty_pbgp));
memset(&empty_pnat, 0, sizeof(empty_pnat));
memset(&empty_pmpls, 0, sizeof(empty_pmpls));
memset(&empty_pvlen, 0, sizeof(empty_pvlen));
/* building a server for interrogations by clients */
sd = build_query_server(config.imt_plugin_path);
cLen = sizeof(cAddr);
/* preparing for synchronous I/O multiplexing */
select_fd = 0;
FD_ZERO(&read_descs);
FD_SET(sd, &read_descs);
if (sd > select_fd) select_fd = sd;
FD_SET(pipe_fd, &read_descs);
if (pipe_fd > select_fd) select_fd = pipe_fd;
select_fd++;
memcpy(&bkp_read_descs, &read_descs, sizeof(read_descs));
qh = (struct query_header *) srvbuf;
/* plugin main loop */
for(;;) {
select_again:
select_timeout.tv_sec = DEFAULT_IMT_PLUGIN_SELECT_TIMEOUT;
select_timeout.tv_usec = 0;
memcpy(&read_descs, &bkp_read_descs, sizeof(bkp_read_descs));
num = select(select_fd, &read_descs, NULL, NULL, &select_timeout);
if (num <= 0) {
if (getppid() == 1) {
Log(LOG_ERR, "ERROR ( %s/%s ): Core process *seems* gone. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
goto select_again;
}
gettimeofday(&cycle_stamp, NULL);
/* doing server tasks */
if (FD_ISSET(sd, &read_descs)) {
struct pollfd pfd;
int ret;
sd2 = accept(sd, &cAddr, &cLen);
setblocking(sd2);
srvbufptr = srvbuf;
sz = maxqsize;
pfd.fd = sd2;
pfd.events = POLLIN;
recv_again:
ret = poll(&pfd, 1, 1000);
if (ret == 0) {
Log(LOG_WARNING, "WARN ( %s/%s ): Timed out while processing fragmented query.\n", config.name, config.type);
close(sd2);
goto select_again;
}
else {
num = recv(sd2, srvbufptr, sz, 0);
if (srvbufptr[num-1] != '\x4') {
srvbufptr += num;
sz -= num;
goto recv_again; /* fragmented query */
}
}
num = num+(maxqsize-sz);
if (qh->num > MAX_QUERIES) {
Log(LOG_DEBUG, "DEBUG ( %s/%s ): request discarded. Too much queries.\n", config.name, config.type);
close(sd2);
continue;
}
request = qh->type;
if (request & WANT_RESET) request ^= WANT_RESET;
if (request & WANT_LOCK_OP) {
lock = TRUE;
request ^= WANT_LOCK_OP;
}
/*
- if explicitely required, we do not fork: query obtains exclusive
control - lock - over the memory table;
- operations that may cause inconsistencies (full erasure, counter
reset for individual entries, etc.) are entitled of an exclusive
lock.
- if query is matter of just a single short-lived walk through the
table, we avoid fork(): the plugin will serve the request;
- in all other cases, we fork; the newly created child will serve
queries asyncronously.
*/
if (request & WANT_ERASE) {
request ^= WANT_ERASE;
if (request) {
if (num > 0) process_query_data(sd2, srvbuf, num, &extras, datasize, FALSE);
else Log(LOG_DEBUG, "DEBUG ( %s/%s ): %d incoming bytes. Errno: %d\n", config.name, config.type, num, errno);
}
Log(LOG_DEBUG, "DEBUG ( %s/%s ): Closing connection with client ...\n", config.name, config.type);
go_to_clear = TRUE;
}
else if (((request == WANT_COUNTER) || (request == WANT_MATCH)) &&
(qh->num == 1) && (qh->what_to_count == config.what_to_count)) {
if (num > 0) process_query_data(sd2, srvbuf, num, &extras, datasize, FALSE);
else Log(LOG_DEBUG, "DEBUG ( %s/%s ): %d incoming bytes. ERRNO: %d\n", config.name, config.type, num, errno);
Log(LOG_DEBUG, "DEBUG ( %s/%s ): Closing connection with client ...\n", config.name, config.type);
}
else if (request == WANT_CLASS_TABLE) {
if (num > 0) process_query_data(sd2, srvbuf, num, &extras, datasize, FALSE);
else Log(LOG_DEBUG, "DEBUG ( %s/%s ): %d incoming bytes. ERRNO: %d\n", config.name, config.type, num, errno);
Log(LOG_DEBUG, "DEBUG ( %s/%s ): Closing connection with client ...\n", config.name, config.type);
}
else if (request == WANT_PKT_LEN_DISTRIB_TABLE) {
if (num > 0) process_query_data(sd2, srvbuf, num, &extras, datasize, FALSE);
else Log(LOG_DEBUG, "DEBUG ( %s/%s ): %d incoming bytes. ERRNO: %d\n", config.name, config.type, num, errno);
Log(LOG_DEBUG, "DEBUG ( %s/%s ): Closing connection with client ...\n", config.name, config.type);
}
else {
if (lock) {
if (num > 0) process_query_data(sd2, srvbuf, num, &extras, datasize, FALSE);
else Log(LOG_DEBUG, "DEBUG ( %s/%s ): %d incoming bytes. Errno: %d\n", config.name, config.type, num, errno);
Log(LOG_DEBUG, "DEBUG ( %s/%s ): Closing connection with client ...\n", config.name, config.type);
}
else {
switch (fork()) {
case -1: /* Something went wrong */
Log(LOG_WARNING, "WARN ( %s/%s ): Unable to serve client query: %s\n", config.name, config.type, strerror(errno));
break;
case 0: /* Child */
close(sd);
pm_setproctitle("%s [%s]", "IMT Plugin -- serving client", config.name);
if (num > 0) process_query_data(sd2, srvbuf, num, &extras, datasize, TRUE);
else Log(LOG_DEBUG, "DEBUG ( %s/%s ): %d incoming bytes. Errno: %d\n", config.name, config.type, num, errno);
Log(LOG_DEBUG, "DEBUG ( %s/%s ): Closing connection with client ...\n", config.name, config.type);
close(sd2);
exit(0);
default: /* Parent */
break;
}
}
}
close(sd2);
}
/* clearing stats if requested */
if (go_to_clear) {
/* When using extended BGP features we need to
free() up memory allocations before erasing */
/* XXX: given the current use of empty_* vars we have always to
free_extra_allocs() in order to prevent memory leaks */
/*
if (extras.off_pkt_bgp_primitives || extras.off_pkt_nat_primitives ||
extras.off_pkt_mpls_primitives || extras.off_custom_primitives)
*/
free_extra_allocs();
clear_memory_pool_table();
current_pool = request_memory_pool(config.buckets*sizeof(struct acc));
if (current_pool == NULL) {
Log(LOG_ERR, "ERROR ( %s/%s ): Cannot allocate my first memory pool, try with larger value.\n", config.name, config.type);
exit_plugin(1);
}
a = current_pool->base_ptr;
current_pool = request_memory_pool(config.memory_pool_size);
if (current_pool == NULL) {
Log(LOG_ERR, "ERROR ( %s/%s ): Cannot allocate more memory pools, try with larger value.\n", config.name, config.type);
exit_plugin(1);
}
go_to_clear = FALSE;
no_more_space = FALSE;
memcpy(&table_reset_stamp, &cycle_stamp, sizeof(struct timeval));
}
if (FD_ISSET(pipe_fd, &read_descs)) {
if (!pollagain) {
seq++;
seq %= MAX_SEQNUM;
}
pollagain = FALSE;
if ((num = read(pipe_fd, &rgptr, sizeof(rgptr))) == 0)
exit_plugin(1); /* we exit silently; something happened at the write end */
if (num < 0) {
pollagain = TRUE;
goto select_again;
}
memcpy(pipebuf, rgptr, config.buffer_size);
if (((struct ch_buf_hdr *)pipebuf)->seq != seq) {
rg_err_count++;
if (config.debug || (rg_err_count > MAX_RG_COUNT_ERR)) {
Log(LOG_ERR, "ERROR ( %s/%s ): We are missing data.\n", config.name, config.type);
Log(LOG_ERR, "If you see this message once in a while, discard it. Otherwise some solutions follow:\n");
Log(LOG_ERR, "- increase shared memory size, 'plugin_pipe_size'; now: '%d'.\n", config.pipe_size);
Log(LOG_ERR, "- increase buffer size, 'plugin_buffer_size'; now: '%d'.\n", config.buffer_size);
Log(LOG_ERR, "- increase system maximum socket size.\n\n");
seq = ((struct ch_buf_hdr *)pipebuf)->seq;
}
}
if (num > 0) {
data = (struct pkt_data *) (pipebuf+sizeof(struct ch_buf_hdr));
while (((struct ch_buf_hdr *)pipebuf)->num > 0) {
// XXX: to be optimized: remove empty_* vars
if (extras.off_pkt_bgp_primitives)
pbgp = (struct pkt_bgp_primitives *) ((u_char *)data + extras.off_pkt_bgp_primitives);
else pbgp = &empty_pbgp;
if (extras.off_pkt_nat_primitives)
pnat = (struct pkt_nat_primitives *) ((u_char *)data + extras.off_pkt_nat_primitives);
else pnat = &empty_pnat;
if (extras.off_pkt_mpls_primitives)
pmpls = (struct pkt_mpls_primitives *) ((u_char *)data + extras.off_pkt_mpls_primitives);
else pmpls = &empty_pmpls;
if (extras.off_custom_primitives)
pcust = ((u_char *)data + extras.off_custom_primitives);
else pcust = empty_pcust;
if (extras.off_pkt_vlen_hdr_primitives)
pvlen = (struct pkt_vlen_hdr_primitives *) ((u_char *)data + extras.off_pkt_vlen_hdr_primitives);
else pvlen = &empty_pvlen;
for (num = 0; net_funcs[num]; num++)
(*net_funcs[num])(&nt, &nc, &data->primitives, pbgp, &nfd);
if (config.ports_file) {
if (!pt.table[data->primitives.src_port]) data->primitives.src_port = 0;
if (!pt.table[data->primitives.dst_port]) data->primitives.dst_port = 0;
}
if (config.pkt_len_distrib_bins_str &&
config.what_to_count_2 & COUNT_PKT_LEN_DISTRIB)
evaluate_pkt_len_distrib(data);
prim_ptrs.data = data;
prim_ptrs.pbgp = pbgp;
prim_ptrs.pnat = pnat;
prim_ptrs.pmpls = pmpls;
prim_ptrs.pcust = pcust;
prim_ptrs.pvlen = pvlen;
(*insert_func)(&prim_ptrs);
((struct ch_buf_hdr *)pipebuf)->num--;
if (((struct ch_buf_hdr *)pipebuf)->num) {
dataptr = (unsigned char *) data;
dataptr += datasize;
data = (struct pkt_data *) dataptr;
}
}
}
}
if (reload_map) {
load_networks(config.networks_file, &nt, &nc);
load_ports(config.ports_file, &pt);
reload_map = FALSE;
}
}
}
/* Functions */
void pgsql_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)
{
struct pkt_data *data;
struct ports_table pt;
struct pollfd pfd;
struct insert_data idata;
time_t refresh_deadline;
int timeout, refresh_timeout, amqp_timeout;
int ret, num;
struct ring *rg = &((struct channels_list_entry *)ptr)->rg;
struct ch_status *status = ((struct channels_list_entry *)ptr)->status;
struct plugins_list_entry *plugin_data = ((struct channels_list_entry *)ptr)->plugin;
int datasize = ((struct channels_list_entry *)ptr)->datasize;
u_int32_t bufsz = ((struct channels_list_entry *)ptr)->bufsize;
struct networks_file_data nfd;
char *dataptr;
unsigned char *rgptr;
int pollagain = TRUE;
u_int32_t seq = 1, rg_err_count = 0;
struct extra_primitives extras;
struct primitives_ptrs prim_ptrs;
#ifdef WITH_RABBITMQ
struct p_amqp_host *amqp_host = &((struct channels_list_entry *)ptr)->amqp_host;
#endif
memcpy(&config, cfgptr, sizeof(struct configuration));
memcpy(&extras, &((struct channels_list_entry *)ptr)->extras, sizeof(struct extra_primitives));
recollect_pipe_memory(ptr);
pm_setproctitle("%s [%s]", "PostgreSQL Plugin", config.name);
memset(&idata, 0, sizeof(idata));
if (config.pidfile) write_pid_file_plugin(config.pidfile, config.type, config.name);
if (config.logfile) {
fclose(config.logfile_fd);
config.logfile_fd = open_logfile(config.logfile, "a");
}
sql_set_signals();
sql_init_default_values(&extras);
PG_init_default_values(&idata);
PG_set_callbacks(&sqlfunc_cbr);
sql_set_insert_func();
/* some LOCAL initialization AFTER setting some default values */
reload_map = FALSE;
idata.now = time(NULL);
refresh_deadline = idata.now;
idata.cfg = &config;
sql_init_maps(&extras, &prim_ptrs, &nt, &nc, &pt);
sql_init_global_buffers();
sql_init_historical_acct(idata.now, &idata);
sql_init_triggers(idata.now, &idata);
sql_init_refresh_deadline(&refresh_deadline);
if (config.pipe_amqp) {
plugin_pipe_amqp_compile_check();
#ifdef WITH_RABBITMQ
pipe_fd = plugin_pipe_amqp_connect_to_consume(amqp_host, plugin_data);
amqp_timeout = plugin_pipe_amqp_set_poll_timeout(amqp_host, pipe_fd);
#endif
}
else setnonblocking(pipe_fd);
/* building up static SQL clauses */
idata.num_primitives = PG_compose_static_queries();
glob_num_primitives = idata.num_primitives;
/* handling logfile template stuff */
te = sql_init_logfile_template(&th);
INIT_BUF(logbuf);
/* setting up environment variables */
SQL_SetENV();
sql_link_backend_descriptors(&bed, &p, &b);
/* plugin main loop */
for(;;) {
poll_again:
status->wakeup = TRUE;
calc_refresh_timeout(refresh_deadline, idata.now, &refresh_timeout);
pfd.fd = pipe_fd;
pfd.events = POLLIN;
timeout = MIN(refresh_timeout, (amqp_timeout ? amqp_timeout : INT_MAX));
ret = poll(&pfd, (pfd.fd == ERR ? 0 : 1), timeout);
if (ret <= 0) {
if (getppid() == 1) {
Log(LOG_ERR, "ERROR ( %s/%s ): Core process *seems* gone. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
if (ret < 0) goto poll_again;
}
idata.now = time(NULL);
now = idata.now;
if (config.sql_history) {
while (idata.now > (idata.basetime + idata.timeslot)) {
time_t saved_basetime = idata.basetime;
idata.basetime += idata.timeslot;
if (config.sql_history == COUNT_MONTHLY)
idata.timeslot = calc_monthly_timeslot(idata.basetime, config.sql_history_howmany, ADD);
glob_basetime = idata.basetime;
idata.new_basetime = saved_basetime;
glob_new_basetime = saved_basetime;
}
}
#ifdef WITH_RABBITMQ
if (config.pipe_amqp && pipe_fd == ERR) {
if (timeout == amqp_timeout) {
pipe_fd = plugin_pipe_amqp_connect_to_consume(amqp_host, plugin_data);
amqp_timeout = plugin_pipe_amqp_set_poll_timeout(amqp_host, pipe_fd);
}
else amqp_timeout = plugin_pipe_amqp_calc_poll_timeout_diff(amqp_host, idata.now);
}
#endif
switch (ret) {
case 0: /* poll(): timeout */
if (qq_ptr) sql_cache_flush(queries_queue, qq_ptr, &idata, FALSE);
sql_cache_handle_flush_event(&idata, &refresh_deadline, &pt);
break;
default: /* poll(): received data */
read_data:
if (!config.pipe_amqp) {
if (!pollagain) {
seq++;
seq %= MAX_SEQNUM;
if (seq == 0) rg_err_count = FALSE;
idata.now = time(NULL);
now = idata.now;
}
else {
if ((ret = read(pipe_fd, &rgptr, sizeof(rgptr))) == 0)
exit_plugin(1); /* we exit silently; something happened at the write end */
}
if ((rg->ptr + bufsz) > rg->end) rg->ptr = rg->base;
if (((struct ch_buf_hdr *)rg->ptr)->seq != seq) {
if (!pollagain) {
pollagain = TRUE;
goto poll_again;
}
else {
rg_err_count++;
if (config.debug || (rg_err_count > MAX_RG_COUNT_ERR)) {
Log(LOG_ERR, "ERROR ( %s/%s ): We are missing data.\n", config.name, config.type);
Log(LOG_ERR, "If you see this message once in a while, discard it. Otherwise some solutions follow:\n");
Log(LOG_ERR, "- increase shared memory size, 'plugin_pipe_size'; now: '%u'.\n", config.pipe_size);
Log(LOG_ERR, "- increase buffer size, 'plugin_buffer_size'; now: '%u'.\n", config.buffer_size);
Log(LOG_ERR, "- increase system maximum socket size.\n\n");
}
seq = ((struct ch_buf_hdr *)rg->ptr)->seq;
}
}
pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
else {
ret = p_amqp_consume_binary(amqp_host, pipebuf, config.buffer_size);
if (ret) pipe_fd = ERR;
seq = ((struct ch_buf_hdr *)pipebuf)->seq;
amqp_timeout = plugin_pipe_amqp_set_poll_timeout(amqp_host, pipe_fd);
}
#endif
/* lazy sql refresh handling */
if (idata.now > refresh_deadline) {
if (qq_ptr) sql_cache_flush(queries_queue, qq_ptr, &idata, FALSE);
sql_cache_handle_flush_event(&idata, &refresh_deadline, &pt);
}
else {
if (config.sql_trigger_exec) {
while (idata.now > idata.triggertime && idata.t_timeslot > 0) {
sql_trigger_exec(config.sql_trigger_exec);
idata.triggertime += idata.t_timeslot;
if (config.sql_trigger_time == COUNT_MONTHLY)
idata.t_timeslot = calc_monthly_timeslot(idata.triggertime, config.sql_trigger_time_howmany, ADD);
}
}
}
data = (struct pkt_data *) (pipebuf+sizeof(struct ch_buf_hdr));
Log(LOG_DEBUG, "DEBUG ( %s/%s ): buffer received seq=%u num_entries=%u\n", config.name, config.type, seq, ((struct ch_buf_hdr *)pipebuf)->num);
while (((struct ch_buf_hdr *)pipebuf)->num > 0) {
for (num = 0; primptrs_funcs[num]; num++)
(*primptrs_funcs[num])((u_char *)data, &extras, &prim_ptrs);
for (num = 0; net_funcs[num]; num++)
(*net_funcs[num])(&nt, &nc, &data->primitives, prim_ptrs.pbgp, &nfd);
if (config.ports_file) {
if (!pt.table[data->primitives.src_port]) data->primitives.src_port = 0;
if (!pt.table[data->primitives.dst_port]) data->primitives.dst_port = 0;
}
if (config.pkt_len_distrib_bins_str &&
config.what_to_count_2 & COUNT_PKT_LEN_DISTRIB)
evaluate_pkt_len_distrib(data);
prim_ptrs.data = data;
(*insert_func)(&prim_ptrs, &idata);
((struct ch_buf_hdr *)pipebuf)->num--;
if (((struct ch_buf_hdr *)pipebuf)->num) {
dataptr = (unsigned char *) data;
if (!prim_ptrs.vlen_next_off) dataptr += datasize;
else dataptr += prim_ptrs.vlen_next_off;
data = (struct pkt_data *) dataptr;
}
}
if (!config.pipe_amqp) goto read_data;
}
}
}
/* Functions */
void amqp_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)
{
struct pkt_data *data;
struct ports_table pt;
unsigned char *pipebuf;
struct pollfd pfd;
time_t t, now;
int timeout, ret, num;
struct ring *rg = &((struct channels_list_entry *)ptr)->rg;
struct ch_status *status = ((struct channels_list_entry *)ptr)->status;
int datasize = ((struct channels_list_entry *)ptr)->datasize;
u_int32_t bufsz = ((struct channels_list_entry *)ptr)->bufsize;
struct networks_file_data nfd;
unsigned char *rgptr;
int pollagain = TRUE;
u_int32_t seq = 1, rg_err_count = 0;
struct extra_primitives extras;
struct primitives_ptrs prim_ptrs;
char *dataptr;
memcpy(&config, cfgptr, sizeof(struct configuration));
memcpy(&extras, &((struct channels_list_entry *)ptr)->extras, sizeof(struct extra_primitives));
recollect_pipe_memory(ptr);
pm_setproctitle("%s [%s]", "RabbitMQ/AMQP Plugin", config.name);
P_set_signals();
P_init_default_values();
pipebuf = (unsigned char *) Malloc(config.buffer_size);
memset(pipebuf, 0, config.buffer_size);
timeout = config.sql_refresh_time*1000;
if (!config.sql_user) config.sql_user = rabbitmq_user;
if (!config.sql_passwd) config.sql_passwd = rabbitmq_pwd;
/* setting function pointers */
if (config.what_to_count & (COUNT_SUM_HOST|COUNT_SUM_NET))
insert_func = P_sum_host_insert;
else if (config.what_to_count & COUNT_SUM_PORT) insert_func = P_sum_port_insert;
else if (config.what_to_count & COUNT_SUM_AS) insert_func = P_sum_as_insert;
#if defined (HAVE_L2)
else if (config.what_to_count & COUNT_SUM_MAC) insert_func = P_sum_mac_insert;
#endif
else insert_func = P_cache_insert;
purge_func = amqp_cache_purge;
memset(&nt, 0, sizeof(nt));
memset(&nc, 0, sizeof(nc));
memset(&pt, 0, sizeof(pt));
load_networks(config.networks_file, &nt, &nc);
set_net_funcs(&nt);
if (config.ports_file) load_ports(config.ports_file, &pt);
if (config.pkt_len_distrib_bins_str) load_pkt_len_distrib_bins();
else {
if (config.what_to_count_2 & COUNT_PKT_LEN_DISTRIB) {
Log(LOG_ERR, "ERROR ( %s/%s ): 'aggregate' contains pkt_len_distrib but no 'pkt_len_distrib_bins' defined. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
}
memset(&prim_ptrs, 0, sizeof(prim_ptrs));
set_primptrs_funcs(&extras);
pfd.fd = pipe_fd;
pfd.events = POLLIN;
setnonblocking(pipe_fd);
now = time(NULL);
/* print_refresh time init: deadline */
refresh_deadline = now;
t = roundoff_time(refresh_deadline, config.sql_history_roundoff);
while ((t+config.sql_refresh_time) < refresh_deadline) t += config.sql_refresh_time;
refresh_deadline = t;
refresh_deadline += config.sql_refresh_time; /* it's a deadline not a basetime */
if (config.sql_history) {
basetime_init = P_init_historical_acct;
basetime_eval = P_eval_historical_acct;
basetime_cmp = P_cmp_historical_acct;
(*basetime_init)(now);
}
/* setting number of entries in _protocols structure */
while (_protocols[protocols_number].number != -1) protocols_number++;
/* plugin main loop */
for(;;) {
poll_again:
status->wakeup = TRUE;
ret = poll(&pfd, 1, timeout);
if (ret <= 0) {
if (getppid() == 1) {
Log(LOG_ERR, "ERROR ( %s/%s ): Core process *seems* gone. Exiting.\n", config.name, config.type);
exit_plugin(1);
}
if (ret < 0) goto poll_again;
}
now = time(NULL);
if (config.sql_history) {
while (now > (basetime.tv_sec + timeslot)) {
new_basetime.tv_sec = basetime.tv_sec;
basetime.tv_sec += timeslot;
if (config.sql_history == COUNT_MONTHLY)
timeslot = calc_monthly_timeslot(basetime.tv_sec, config.sql_history_howmany, ADD);
}
}
switch (ret) {
case 0: /* timeout */
P_cache_handle_flush_event(&pt);
break;
default: /* we received data */
read_data:
if (!pollagain) {
seq++;
seq %= MAX_SEQNUM;
if (seq == 0) rg_err_count = FALSE;
}
else {
if ((ret = read(pipe_fd, &rgptr, sizeof(rgptr))) == 0)
exit_plugin(1); /* we exit silently; something happened at the write end */
}
if (((struct ch_buf_hdr *)rg->ptr)->seq != seq) {
if (!pollagain) {
pollagain = TRUE;
goto poll_again;
}
else {
rg_err_count++;
if (config.debug || (rg_err_count > MAX_RG_COUNT_ERR)) {
Log(LOG_ERR, "ERROR ( %s/%s ): We are missing data.\n", config.name, config.type);
Log(LOG_ERR, "If you see this message once in a while, discard it. Otherwise some solutions follow:\n");
Log(LOG_ERR, "- increase shared memory size, 'plugin_pipe_size'; now: '%u'.\n", config.pipe_size);
Log(LOG_ERR, "- increase buffer size, 'plugin_buffer_size'; now: '%u'.\n", config.buffer_size);
Log(LOG_ERR, "- increase system maximum socket size.\n\n");
}
seq = ((struct ch_buf_hdr *)rg->ptr)->seq;
}
}
pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
if ((rg->ptr+bufsz) >= rg->end) rg->ptr = rg->base;
else rg->ptr += bufsz;
/* lazy refresh time handling */
if (now > refresh_deadline) P_cache_handle_flush_event(&pt);
data = (struct pkt_data *) (pipebuf+sizeof(struct ch_buf_hdr));
while (((struct ch_buf_hdr *)pipebuf)->num > 0) {
for (num = 0; primptrs_funcs[num]; num++)
(*primptrs_funcs[num])((u_char *)data, &extras, &prim_ptrs);
for (num = 0; net_funcs[num]; num++)
(*net_funcs[num])(&nt, &nc, &data->primitives, prim_ptrs.pbgp, &nfd);
if (config.ports_file) {
if (!pt.table[data->primitives.src_port]) data->primitives.src_port = 0;
if (!pt.table[data->primitives.dst_port]) data->primitives.dst_port = 0;
}
if (config.pkt_len_distrib_bins_str &&
config.what_to_count_2 & COUNT_PKT_LEN_DISTRIB)
evaluate_pkt_len_distrib(data);
prim_ptrs.data = data;
(*insert_func)(&prim_ptrs);
((struct ch_buf_hdr *)pipebuf)->num--;
if (((struct ch_buf_hdr *)pipebuf)->num) {
dataptr = (unsigned char *) data;
dataptr += datasize;
data = (struct pkt_data *) dataptr;
}
}
goto read_data;
}
}
}