static void run(packet_function_t receive_packet, int socket, send_peer_t peer, time_t twin, time_t t_begin, int report_seq, int use_subdirs, int compress, int do_xstat) { common_flow_header_t *nf_header; FlowSource_t *fs; struct sockaddr_storage nf_sender; socklen_t nf_sender_size = sizeof(nf_sender); time_t t_start, t_now; uint64_t export_packets; uint32_t blast_cnt, blast_failures, ignored_packets; uint16_t version; ssize_t cnt; void *in_buff; int err; char *string; srecord_t *commbuff; if ( !Init_v1() || !Init_v5_v7_input() || !Init_v9() || !Init_IPFIX() ) return; in_buff = malloc(NETWORK_INPUT_BUFF_SIZE); if ( !in_buff ) { LogError("malloc() allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); return; } // init vars commbuff = (srecord_t *)shmem; nf_header = (common_flow_header_t *)in_buff; // Init each netflow source output data buffer fs = FlowSource; while ( fs ) { // prepare file fs->nffile = OpenNewFile(fs->current, NULL, compress, 0, NULL); if ( !fs->nffile ) { return; } if ( do_xstat ) { fs->xstat = InitXStat(fs->nffile); if ( !fs->xstat ) return; } // init vars fs->bad_packets = 0; fs->first_seen = 0xffffffffffffLL; fs->last_seen = 0; // next source fs = fs->next; } export_packets = blast_cnt = blast_failures = 0; t_start = t_begin; cnt = 0; periodic_trigger = 0; ignored_packets = 0; // wake up at least at next time slot (twin) + some Overdue time alarm(t_start + twin + OVERDUE_TIME - time(NULL)); /* * Main processing loop: * this loop, continues until done = 1, set by the signal handler * The while loop will be breaked by the periodic file renaming code * for proper cleanup */ while ( 1 ) { struct timeval tv; /* read next bunch of data into beginn of input buffer */ if ( !done) { #ifdef PCAP // Debug code to read from pcap file, or from socket cnt = receive_packet(socket, in_buff, NETWORK_INPUT_BUFF_SIZE , 0, (struct sockaddr *)&nf_sender, &nf_sender_size); // in case of reading from file EOF => -2 if ( cnt == -2 ) done = 1; #else cnt = recvfrom (socket, in_buff, NETWORK_INPUT_BUFF_SIZE , 0, (struct sockaddr *)&nf_sender, &nf_sender_size); #endif if ( cnt == -1 && errno != EINTR ) { LogError("ERROR: recvfrom: %s", strerror(errno)); continue; } if ( peer.hostname ) { ssize_t len; len = sendto(peer.sockfd, in_buff, cnt, 0, (struct sockaddr *)&(peer.addr), peer.addrlen); if ( len < 0 ) { LogError("ERROR: sendto(): %s", strerror(errno)); } } } /* Periodic file renaming, if time limit reached or if we are done. */ // t_now = time(NULL); gettimeofday(&tv, NULL); t_now = tv.tv_sec; if ( ((t_now - t_start) >= twin) || done ) { char subfilename[64]; struct tm *now; char *subdir; alarm(0); now = localtime(&t_start); // prepare sub dir hierarchy if ( use_subdirs ) { subdir = GetSubDir(now); if ( !subdir ) { // failed to generate subdir path - put flows into base directory LogError("Failed to create subdir path!"); // failed to generate subdir path - put flows into base directory subdir = NULL; snprintf(subfilename, 63, "nfcapd.%i%02i%02i%02i%02i%02i", now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec); } else { snprintf(subfilename, 63, "%s/nfcapd.%i%02i%02i%02i%02i%02i", subdir, now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec); } } else { subdir = NULL; snprintf(subfilename, 63, "nfcapd.%i%02i%02i%02i%02i%02i", now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec); } subfilename[63] = '\0'; // for each flow source update the stats, close the file and re-initialize the new file fs = FlowSource; while ( fs ) { char nfcapd_filename[MAXPATHLEN]; char error[255]; nffile_t *nffile = fs->nffile; if ( verbose ) { // Dump to stdout format_file_block_header(nffile->block_header, &string, 0); printf("%s\n", string); } if ( nffile->block_header->NumRecords ) { // flush current buffer to disc if ( WriteBlock(nffile) <= 0 ) LogError("Ident: %s, failed to write output buffer to disk: '%s'" , fs->Ident, strerror(errno)); } // else - no new records in current block // prepare filename snprintf(nfcapd_filename, MAXPATHLEN-1, "%s/%s", fs->datadir, subfilename); nfcapd_filename[MAXPATHLEN-1] = '\0'; // update stat record // if no flows were collected, fs->last_seen is still 0 // set first_seen to start of this time slot, with twin window size. if ( fs->last_seen == 0 ) { fs->first_seen = (uint64_t)1000 * (uint64_t)t_start; fs->last_seen = (uint64_t)1000 * (uint64_t)(t_start + twin); } nffile->stat_record->first_seen = fs->first_seen/1000; nffile->stat_record->msec_first = fs->first_seen - nffile->stat_record->first_seen*1000; nffile->stat_record->last_seen = fs->last_seen/1000; nffile->stat_record->msec_last = fs->last_seen - nffile->stat_record->last_seen*1000; if ( fs->xstat ) { if ( WriteExtraBlock(nffile, fs->xstat->block_header ) <= 0 ) LogError("Ident: %s, failed to write xstat buffer to disk: '%s'" , fs->Ident, strerror(errno)); ResetPortHistogram(fs->xstat->port_histogram); ResetBppHistogram(fs->xstat->bpp_histogram); } // Flush Exporter Stat to file FlushExporterStats(fs); // Close file CloseUpdateFile(nffile, fs->Ident); if ( subdir && !SetupSubDir(fs->datadir, subdir, error, 255) ) { // in this case the flows get lost! - the rename will fail // but this should not happen anyway, unless i/o problems, inode problems etc. LogError("Ident: %s, Failed to create sub hier directories: %s", fs->Ident, error ); } // if rename fails, we are in big trouble, as we need to get rid of the old .current file // otherwise, we will loose flows and can not continue collecting new flows err = rename(fs->current, nfcapd_filename); if ( err ) { LogError("Ident: %s, Can't rename dump file: %s", fs->Ident, strerror(errno)); LogError("Ident: %s, Serious Problem! Fix manually", fs->Ident); if ( launcher_pid ) commbuff->failed = 1; // we do not update the books here, as the file failed to rename properly // otherwise the books may be wrong } else { struct stat fstat; if ( launcher_pid ) commbuff->failed = 0; // Update books stat(nfcapd_filename, &fstat); UpdateBooks(fs->bookkeeper, t_start, 512*fstat.st_blocks); } // log stats LogInfo("Ident: '%s' Flows: %llu, Packets: %llu, Bytes: %llu, Sequence Errors: %u, Bad Packets: %u", fs->Ident, (unsigned long long)nffile->stat_record->numflows, (unsigned long long)nffile->stat_record->numpackets, (unsigned long long)nffile->stat_record->numbytes, nffile->stat_record->sequence_failure, fs->bad_packets); // reset stats fs->bad_packets = 0; fs->first_seen = 0xffffffffffffLL; fs->last_seen = 0; if ( !done ) { nffile = OpenNewFile(fs->current, nffile, compress, 0, NULL); if ( !nffile ) { LogError("killed due to fatal error: ident: %s", fs->Ident); break; } /* XXX needs fixing */ if ( fs->xstat ) { // to be implemented } } // Dump all extension maps and exporters to the buffer FlushStdRecords(fs); // next flow source fs = fs->next; } // end of while (fs) // All flow sources updated - signal launcher if required if ( launcher_pid ) { // Signal launcher // prepare filename for %f expansion strncpy(commbuff->fname, subfilename, FNAME_SIZE-1); commbuff->fname[FNAME_SIZE-1] = 0; snprintf(commbuff->tstring, 16, "%i%02i%02i%02i%02i", now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min); commbuff->tstring[15] = 0; commbuff->tstamp = t_start; if ( subdir ) strncpy(commbuff->subdir, subdir, FNAME_SIZE); else commbuff->subdir[0] = '\0'; if ( launcher_alive ) { LogInfo("Signal launcher"); kill(launcher_pid, SIGHUP); } else LogError("ERROR: Launcher did unexpectedly!"); } LogInfo("Total ignored packets: %u", ignored_packets); ignored_packets = 0; if ( done ) break; // update alarm for next cycle t_start += twin; /* t_start = filename time stamp: begin of slot * + twin = end of next time interval * + OVERDUE_TIME = if no data is collected, this is at latest to act * - , now->tm_sect_now = difference value to now */ alarm(t_start + twin + OVERDUE_TIME - t_now); } /* check for error condition or done . errno may only be EINTR */ if ( cnt < 0 ) { if ( periodic_trigger ) { // alarm triggered, no new flow data periodic_trigger = 0; continue; } if ( done ) // signaled to terminate - exit from loop break; else { /* this should never be executed as it should be caught in other places */ LogError("error condition in '%s', line '%d', cnt: %i", __FILE__, __LINE__ ,(int)cnt); continue; } } /* enough data? */ if ( cnt == 0 ) continue; // get flow source record for current packet, identified by sender IP address fs = GetFlowSource(&nf_sender); if ( fs == NULL ) { fs = AddDynamicSource(&FlowSource, &nf_sender); if ( fs == NULL ) { LogError("Skip UDP packet. Ignored packets so far %u packets", ignored_packets); ignored_packets++; continue; } if ( InitBookkeeper(&fs->bookkeeper, fs->datadir, getpid(), launcher_pid) != BOOKKEEPER_OK ) { LogError("Failed to initialise bookkeeper for new source"); // fatal error return; } fs->nffile = OpenNewFile(fs->current, NULL, compress, 0, NULL); if ( !fs->nffile ) { LogError("Failed to open new collector file"); return; } } /* check for too little data - cnt must be > 0 at this point */ if ( cnt < sizeof(common_flow_header_t) ) { LogError("Ident: %s, Data length error: too little data for common netflow header. cnt: %i",fs->Ident, (int)cnt); fs->bad_packets++; continue; } fs->received = tv; /* Process data - have a look at the common header */ version = ntohs(nf_header->version); switch (version) { case 1: Process_v1(in_buff, cnt, fs); break; case 5: // fall through case 7: Process_v5_v7(in_buff, cnt, fs); break; case 9: Process_v9(in_buff, cnt, fs); break; case 10: Process_IPFIX(in_buff, cnt, fs); break; case 255: // blast test header if ( verbose ) { uint16_t count = ntohs(nf_header->count); if ( blast_cnt != count ) { // LogError("Missmatch blast check: Expected %u got %u\n", blast_cnt, count); blast_cnt = count; blast_failures++; } else { blast_cnt++; } if ( blast_cnt == 65535 ) { fprintf(stderr, "Total missed packets: %u\n", blast_failures); done = 1; } break; } default: // data error, while reading data from socket LogError("Ident: %s, Error reading netflow header: Unexpected netflow version %i", fs->Ident, version); fs->bad_packets++; continue; // not reached break; } // each Process_xx function has to process the entire input buffer, therefore it's empty now. export_packets++; // flush current buffer to disc if ( fs->nffile->block_header->size > BUFFSIZE ) { // fishy! - we already wrote into someone elses memory! - I'm sorry // reset output buffer - data may be lost, as we don not know, where it happen fs->nffile->block_header->size = 0; fs->nffile->block_header->NumRecords = 0; fs->nffile->buff_ptr = (void *)((pointer_addr_t)fs->nffile->block_header + sizeof(data_block_header_t) ); LogError("### Software bug ### Ident: %s, output buffer overflow: expect memory inconsitency", fs->Ident); } } if ( verbose && blast_failures ) { fprintf(stderr, "Total missed packets: %u\n", blast_failures); } free(in_buff); fs = FlowSource; while ( fs ) { DisposeFile(fs->nffile); fs = fs->next; } } /* End of run */
__attribute__((noreturn)) static void *p_pcap_flush_thread(void *thread_data) { // argument dispatching p_pcap_flush_thread_args_t *args = (p_pcap_flush_thread_args_t *)thread_data; char *pcap_datadir = args->pcap_datadir; pcap_dev_t *pcap_dev = args->pcap_dev; pcapfile_t *pcapfile = args->pcapfile; char pcap_dumpfile[MAXPATHLEN]; int err; int runs = 0; dbg_printf("New flush thread[%lu]\n", (long unsigned)args->tid); args->done = 0; args->exit = 0; err = pthread_setspecific( buffer_key, (void *)args ); if ( err ) { LogError("[%lu] pthread_setspecific() error in %s line %d: %s\n", (long unsigned)args->tid, __FILE__, __LINE__, strerror(errno) ); args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } snprintf(pcap_dumpfile, MAXPATHLEN-1, "%s/%s.%lu", pcap_datadir , PCAP_DUMPFILE, (unsigned long)getpid() ); pcapfile = OpenNewPcapFile(pcap_dev->handle, pcap_dumpfile, pcapfile); if ( !pcapfile ) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); /* NOTREACHED */ } // wait for alternate buffer to be ready to flush while ( 1 ) { pthread_mutex_lock(&pcapfile->m_pbuff); while ( pcapfile->alternate_size == 0 && !args->done ) { pthread_cond_wait(&pcapfile->c_pbuff, &pcapfile->m_pbuff); } dbg_printf("Flush cycle\n"); runs++; // try to flush alternate buffer if ( pcapfile->alternate_size ) { // flush alternate buffer dbg_printf("Flush alternate\n"); if ( write(pcapfile->pfd, (void *)pcapfile->alternate_buffer, pcapfile->alternate_size) <= 0 ) { LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); } pcapfile->alternate_size = 0; } // if we are done, try to flsuh main data buffer if ( args->done && pcapfile->data_size ) { dbg_printf("Done: Flush all buffers\n"); // flush alternate buffer if ( write(pcapfile->pfd, (void *)pcapfile->data_buffer, pcapfile->data_size) <= 0 ) { LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); } pcapfile->data_size = 0; pcapfile->data_ptr = pcapfile->data_buffer; } // check if we need to rotate/close the file if ( args->done || pcapfile->t_CloseRename ) { /* rotate file */ struct tm *when; char FullName[MAXPATHLEN]; char pcapFname[128]; char error[256]; char *subdir; int err; dbg_printf("Flush rotate file\n"); when = localtime(&pcapfile->t_CloseRename); pcapfile->t_CloseRename = 0; // prepare sub dir hierarchy if ( args->subdir_index ) { subdir = GetSubDir(when); if ( !subdir ) { // failed to generate subdir path - put flows into base directory LogError("Failed to create subdir path!"); // failed to generate subdir path - put flows into base directory subdir = NULL; snprintf(pcapFname, 127, "pcapd.%i%02i%02i%02i%02i", when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } else { snprintf(pcapFname, 127, "%s/pcapd.%i%02i%02i%02i%02i", subdir, when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } } else { subdir = NULL; snprintf(pcapFname, 127, "pcapd.%i%02i%02i%02i%02i", when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } pcapFname[127] = '\0'; if ( subdir && !SetupSubDir(pcap_datadir, subdir, error, 255) ) { // in this case the flows get lost! - the rename will fail // but this should not happen anyway, unless i/o problems, inode problems etc. LogError("p_packet_thread() Failed to create sub hier directories: %s", error ); } // prepare full filename snprintf(FullName, MAXPATHLEN-1, "%s/%s", pcap_datadir, pcapFname); FullName[MAXPATHLEN-1] = '\0'; ClosePcapFile(pcapfile); err = rename(pcap_dumpfile, FullName); if (err) { LogError("rename() pcap failed in %s line %d: %s", __FILE__, __LINE__, strerror(errno) ); } dbg_printf("Rotate file: %s -> %s\n", pcap_dumpfile, FullName); if ( args->done ) { pthread_mutex_unlock(&pcapfile->m_pbuff); pthread_cond_signal(&pcapfile->c_pbuff); break; } // open new files pcapfile = OpenNewPcapFile(pcap_dev->handle, pcap_dumpfile, pcapfile); if (!pcapfile) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_mutex_unlock(&pcapfile->m_pbuff); pthread_cond_signal(&pcapfile->c_pbuff); break; } } dbg_printf("Flush cycle done\n"); pthread_mutex_unlock(&pcapfile->m_pbuff); pthread_cond_signal(&pcapfile->c_pbuff); } dbg_printf("End flush thread[%lu]: %i runs\n", (long unsigned)args->tid, runs); pthread_exit((void *)args); /* NOTREACHED */ } // End of p_pcap_flush_thread
static void SetupProfileChannels(char *profile_datadir, char *profile_statdir, profile_param_info_t *profile_param, int subdir_index, char *filterfile, char *filename, int verify_only, int compress ) { FilterEngine_data_t *engine; struct stat stat_buf; char *p, *filter, *subdir, *wfile, *ofile, *rrdfile, *source_filter; char path[MAXPATHLEN]; int ffd, ret; size_t filter_size; nffile_t *nffile; ofile = wfile = NULL; nffile = NULL; /* * Compile the complete filter: * this consists of the source list and the filter stored in the file */ snprintf(path, MAXPATHLEN-1, "%s/%s/%s/%s-%s", profile_statdir, profile_param->profilegroup, profile_param->profilename, profile_param->channelname, filterfile); path[MAXPATHLEN-1] = '\0'; if ( stat(path, &stat_buf) || !S_ISREG(stat_buf.st_mode) ) { LogError("Skipping channel %s in profile '%s' group '%s'. No profile filter found.\n", profile_param->channelname, profile_param->profilename, profile_param->profilegroup); return; } // prepare source filter for this channel if ( profile_param->channel_sourcelist ) { // we have a channel_sourcelist: channel1|channel2|channel3 // source filter - therefore pattern is '( sourcefilter ) and ( filter )' // where sourcefilter is 'ident source1 or ident source2 ... ' char *q; size_t len = strlen(profile_param->channel_sourcelist); int num_sources = 1; // at least one source, otherwise we would not be in this code q = profile_param->channel_sourcelist; while ( (p = strchr(q, '|')) != NULL ) { num_sources++; q = p; q++; } // allocate a temp buffer for the source filter. // for each source add 'ident <source> or ', which makes 10 char per sources, including '()\0 and ' = 8 len += 10 * num_sources + 8; source_filter = (char *)malloc(len); if ( !source_filter ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } source_filter[0] = '('; source_filter[1] = '\0'; len--; q = profile_param->channel_sourcelist; do { p = strchr(q, '|'); if (p) *p = '\0'; if ( !AppendString(source_filter, "ident ", &len) ) return; if ( !AppendString(source_filter, q, &len) ) return; if ( p ) { // there is another source waiting behind *p if ( !AppendString(source_filter, " or ", &len) ) return; q = p; q++; } } while (p); if ( !AppendString(source_filter, ") and (", &len) ) return; } else // no source filter - therefore pattern is '(' filter ')' source_filter = "("; filter_size = stat_buf.st_size + strlen(source_filter) + 2; // +2 : ')\0' at the end of the filter filter = (char *)malloc(filter_size); if ( !filter ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } ffd = open(path, O_RDONLY); if ( ffd < 0 ) { LogError("Can't open file '%s' for reading: %s\n",path, strerror(errno) ); return; } strncpy(filter, source_filter, strlen(source_filter)); p = filter + strlen(source_filter); ret = read(ffd, (void *)p, stat_buf.st_size); if ( ret < 0 ) { LogError("Can't read from file '%s': %s\n",path, strerror(errno) ); close(ffd); return; } close(ffd); p[stat_buf.st_size] = ')'; p[stat_buf.st_size+1] = '\0'; // compile profile filter if ( verify_only ) printf("Check filter for channel %s in profile '%s' in group '%s': ", profile_param->channelname, profile_param->profilename, profile_param->profilegroup); engine = CompileFilter(filter); if ( !engine ) { printf("\n"); LogError("*** Compiling filter failed for channel %s in profile '%s' in group '%s'.", profile_param->channelname, profile_param->profilename, profile_param->profilegroup); LogError("*** File: %s", path); LogError("*** Error: %s\n", yyerror_buff); LogError("*** Failed Filter: %s", filter); free(filter); return; } free(filter); if ( verify_only ) { printf("ok.\n"); return; } // path to the channel // channel exists and is a directory - checked in ParseParams snprintf(path, MAXPATHLEN-1, "%s/%s/%s/%s", profile_datadir, profile_param->profilegroup, profile_param->profilename, profile_param->channelname); path[MAXPATHLEN-1] = '\0'; if ( chdir(path)) { LogError("Error can't chdir to '%s': %s", path, strerror(errno)); exit(255); } // check for subdir hierarchy subdir = NULL; if ( (profile_param->profiletype & 4) == 0 ) { // no shadow profile int is_alert = (profile_param->profiletype & 8) == 8; if ( !is_alert && subdir_index && strlen(filename) == 19 && (strncmp(filename, "nfcapd.", 7) == 0) ) { char *p = &filename[7]; // points to ISO timstamp in filename time_t t = ISO2UNIX(p); struct tm *t_tm = localtime(&t); char error[255]; subdir = GetSubDir(t_tm); if ( !subdir ) { // failed to generate subdir path - put flows into base directory LogError( "Failed to create subdir path!"); } if ( !SetupSubDir(path, subdir, error, 255) ) { LogError( "Failed to create subdir path: '%s'" , error); // nothing else need to be done, as subdir == NULL means put files into channel directory } } if ( is_alert ) { // alert snprintf(path, MAXPATHLEN, "%s/%s/%s/%s/%s", profile_datadir, profile_param->profilegroup, profile_param->profilename, profile_param->channelname, filename); } else { // prepare output file for profile types != shadow if ( subdir ) snprintf(path, MAXPATHLEN, "%s/%s/%s/%s/%s/%s", profile_datadir, profile_param->profilegroup, profile_param->profilename, profile_param->channelname, subdir, filename); else snprintf(path, MAXPATHLEN, "%s/%s/%s/%s/%s", profile_datadir, profile_param->profilegroup, profile_param->profilename, profile_param->channelname, filename); } path[MAXPATHLEN-1] = '\0'; wfile = strdup(path); // ofile: file while profiling snprintf(path, MAXPATHLEN, "%s/%s/%s/%s/nfprofile.%llu", profile_datadir, profile_param->profilegroup, profile_param->profilename, profile_param->channelname, (unsigned long long)getpid()); path[MAXPATHLEN-1] = '\0'; ofile = strdup(path); nffile = OpenNewFile(path, NULL, compress, 0, NULL); if ( !nffile ) { return; } } snprintf(path, MAXPATHLEN-1, "%s/%s/%s/%s.rrd", profile_statdir, profile_param->profilegroup, profile_param->profilename, profile_param->channelname); path[MAXPATHLEN-1] = '\0'; rrdfile = strdup(path); snprintf(path, MAXPATHLEN, "%s/%s/%s/%s", profile_datadir, profile_param->profilegroup, profile_param->profilename, profile_param->channelname ); path[MAXPATHLEN-1] = '\0'; // collect all channel info profile_channels = realloc(profile_channels, (num_channels+1) * sizeof(profile_channel_info_t) ); if ( !profile_channels ) { LogError("Memory allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } memset(&profile_channels[num_channels], 0, sizeof(profile_channel_info_t)); profile_channels[num_channels].engine = engine; profile_channels[num_channels].group = profile_param->profilegroup; profile_channels[num_channels].profile = profile_param->profilename; profile_channels[num_channels].channel = profile_param->channelname; profile_channels[num_channels].wfile = wfile; profile_channels[num_channels].ofile = ofile; profile_channels[num_channels].rrdfile = rrdfile; profile_channels[num_channels].dirstat_path = strdup(path); profile_channels[num_channels].type = profile_param->profiletype; profile_channels[num_channels].nffile = nffile; memset((void *)&profile_channels[num_channels].stat_record, 0, sizeof(stat_record_t)); profile_channels[num_channels].stat_record.first_seen = 0x7fffffff; profile_channels[num_channels].stat_record.last_seen = 0; num_channels++; return; } // End of SetupProfileChannels
__attribute__((noreturn)) static void *p_flow_thread(void *thread_data) { // argument dispatching p_flow_thread_args_t *args = (p_flow_thread_args_t *)thread_data; time_t t_win = args->t_win; int subdir_index = args->subdir_index; int compress = args->compress; FlowSource_t *fs = args->fs; // locals time_t t_start, t_clock, t_udp_flush; int err, done; done = 0; args->done = 0; args->exit = 0; err = pthread_setspecific( buffer_key, (void *)args ); if ( err ) { LogError("[%lu] pthread_setspecific() error in %s line %d: %s\n", (long unsigned)args->tid, __FILE__, __LINE__, strerror(errno) ); args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } if ( !Init_pcap2nf() ) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } // prepare file fs->nffile = OpenNewFile(fs->current, NULL, compress, 0, NULL); if ( !fs->nffile ) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } fs->xstat = NULL; // init vars fs->bad_packets = 0; fs->first_seen = 0xffffffffffffLL; fs->last_seen = 0; t_start = 0; t_clock = 0; t_udp_flush = 0; while ( 1 ) { struct FlowNode *Node; Node = Pop_Node(args->NodeList, &args->done); if ( Node ) { t_clock = Node->t_last.tv_sec; dbg_printf("p_flow_thread() Next Node\n"); } else { done = args->done; dbg_printf("p_flow_thread() NULL Node\n"); } if ( t_start == 0 ) { t_udp_flush = t_start = t_clock - (t_clock % t_win); } if (((t_clock - t_start) >= t_win) || done) { /* rotate file */ struct tm *when; nffile_t *nffile; char FullName[MAXPATHLEN]; char netflowFname[128]; char error[256]; char *subdir; // flush all flows to disk DumpNodeStat(); uint32_t NumFlows = Flush_FlowTree(fs); when = localtime(&t_start); nffile = fs->nffile; // prepare sub dir hierarchy if ( subdir_index ) { subdir = GetSubDir(when); if ( !subdir ) { // failed to generate subdir path - put flows into base directory LogError("Failed to create subdir path!"); // failed to generate subdir path - put flows into base directory subdir = NULL; snprintf(netflowFname, 127, "nfcapd.%i%02i%02i%02i%02i", when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } else { snprintf(netflowFname, 127, "%s/nfcapd.%i%02i%02i%02i%02i", subdir, when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } } else { subdir = NULL; snprintf(netflowFname, 127, "nfcapd.%i%02i%02i%02i%02i", when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } netflowFname[127] = '\0'; if ( subdir && !SetupSubDir(fs->datadir, subdir, error, 255) ) { // in this case the flows get lost! - the rename will fail // but this should not happen anyway, unless i/o problems, inode problems etc. LogError("Ident: %s, Failed to create sub hier directories: %s", fs->Ident, error ); } if ( nffile->block_header->NumRecords ) { // flush current buffer to disc if ( WriteBlock(nffile) <= 0 ) LogError("Ident: %s, failed to write output buffer to disk: '%s'" , fs->Ident, strerror(errno)); } // else - no new records in current block // prepare full filename snprintf(FullName, MAXPATHLEN-1, "%s/%s", fs->datadir, netflowFname); FullName[MAXPATHLEN-1] = '\0'; // update stat record // if no flows were collected, fs->last_seen is still 0 // set first_seen to start of this time slot, with twin window size. if ( fs->last_seen == 0 ) { fs->first_seen = (uint64_t)1000 * (uint64_t)t_start; fs->last_seen = (uint64_t)1000 * (uint64_t)(t_start + t_win); } nffile->stat_record->first_seen = fs->first_seen/1000; nffile->stat_record->msec_first = fs->first_seen - nffile->stat_record->first_seen*1000; nffile->stat_record->last_seen = fs->last_seen/1000; nffile->stat_record->msec_last = fs->last_seen - nffile->stat_record->last_seen*1000; // Flush Exporter Stat to file FlushExporterStats(fs); // Close file CloseUpdateFile(nffile, fs->Ident); // if rename fails, we are in big trouble, as we need to get rid of the old .current file // otherwise, we will loose flows and can not continue collecting new flows if ( !RenameAppend(fs->current, FullName) ) { LogError("Ident: %s, Can't rename dump file: %s", fs->Ident, strerror(errno)); LogError("Ident: %s, Serious Problem! Fix manually", fs->Ident); /* XXX if ( launcher_pid ) commbuff->failed = 1; */ // we do not update the books here, as the file failed to rename properly // otherwise the books may be wrong } else { struct stat fstat; /* XXX if ( launcher_pid ) commbuff->failed = 0; */ // Update books stat(FullName, &fstat); UpdateBooks(fs->bookkeeper, t_start, 512*fstat.st_blocks); } LogInfo("Ident: '%s' Flows: %llu, Packets: %llu, Bytes: %llu, Max Flows: %u", fs->Ident, (unsigned long long)nffile->stat_record->numflows, (unsigned long long)nffile->stat_record->numpackets, (unsigned long long)nffile->stat_record->numbytes, NumFlows); // reset stats fs->bad_packets = 0; fs->first_seen = 0xffffffffffffLL; fs->last_seen = 0; // Dump all extension maps and exporters to the buffer FlushStdRecords(fs); if ( done ) break; t_start = t_clock - (t_clock % t_win); nffile = OpenNewFile(fs->current, nffile, compress, 0, NULL); if ( !nffile ) { LogError("Fatal: OpenNewFile() failed for ident: %s", fs->Ident); args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); break; } } if (((t_clock - t_udp_flush) >= 10) || !done) { /* flush inactive UDP list */ UDPexpire(fs, t_clock - 10 ); t_udp_flush = t_clock; } if ( Node->fin != SIGNAL_NODE ) // Process the Node ProcessFlowNode(fs, Node); } while ( fs ) { DisposeFile(fs->nffile); fs = fs->next; } LogInfo("Terminating flow processng: exit: %i", args->exit); dbg_printf("End flow thread[%lu]\n", (long unsigned)args->tid); pthread_exit((void *)args); /* NOTREACHED */ } // End of p_flow_thread