stat_record_t process_data(char *wfile, int element_stat, int flow_stat, int sort_flows,
printer_t print_header, printer_t print_record, time_t twin_start, time_t twin_end,
uint64_t limitflows, int tag, int compress, int do_xstat) {
common_record_t *flow_record;
master_record_t *master_record;
nffile_t *nffile_w, *nffile_r;
xstat_t *xstat;
stat_record_t stat_record;
int done, write_file;
#ifdef COMPAT15
int v1_map_done = 0;
#endif
// time window of all matched flows
memset((void *)&stat_record, 0, sizeof(stat_record_t));
stat_record.first_seen = 0x7fffffff;
stat_record.msec_first = 999;
// Do the logic first
// do not print flows when doing any stats are sorting
if ( sort_flows || flow_stat || element_stat ) {
print_record = NULL;
}
// do not write flows to file, when doing any stats
// -w may apply for flow_stats later
write_file = !(sort_flows || flow_stat || element_stat) && wfile;
nffile_r = NULL;
nffile_w = NULL;
xstat = NULL;
// Get the first file handle
nffile_r = GetNextFile(NULL, twin_start, twin_end);
if ( !nffile_r ) {
LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
return stat_record;
}
if ( nffile_r == EMPTY_LIST ) {
LogError("Empty file list. No files to process\n");
return stat_record;
}
// preset time window of all processed flows to the stat record in first flow file
t_first_flow = nffile_r->stat_record->first_seen;
t_last_flow = nffile_r->stat_record->last_seen;
// store infos away for later use
// although multiple files may be processed, it is assumed that all
// have the same settings
is_anonymized = IP_ANONYMIZED(nffile_r);
strncpy(Ident, nffile_r->file_header->ident, IDENTLEN);
Ident[IDENTLEN-1] = '\0';
// prepare output file if requested
if ( write_file ) {
nffile_w = OpenNewFile(wfile, NULL, compress, IP_ANONYMIZED(nffile_r), NULL );
if ( !nffile_w ) {
if ( nffile_r ) {
CloseFile(nffile_r);
DisposeFile(nffile_r);
}
return stat_record;
}
if ( do_xstat ) {
xstat = InitXStat(nffile_w);
if ( !xstat ) {
if ( nffile_r ) {
CloseFile(nffile_r);
DisposeFile(nffile_r);
}
return stat_record;
}
}
}
// setup Filter Engine to point to master_record, as any record read from file
// is expanded into this record
// Engine->nfrecord = (uint64_t *)master_record;
done = 0;
while ( !done ) {
int i, ret;
// get next data block from file
ret = ReadBlock(nffile_r);
switch (ret) {
case NF_CORRUPT:
case NF_ERROR:
if ( ret == NF_CORRUPT )
LogError("Skip corrupt data file '%s'\n",GetCurrentFilename());
else
LogError("Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) );
// fall through - get next file in chain
case NF_EOF: {
nffile_t *next = GetNextFile(nffile_r, twin_start, twin_end);
if ( next == EMPTY_LIST ) {
done = 1;
} else if ( next == NULL ) {
done = 1;
LogError("Unexpected end of file list\n");
} else {
// Update global time span window
if ( next->stat_record->first_seen < t_first_flow )
t_first_flow = next->stat_record->first_seen;
if ( next->stat_record->last_seen > t_last_flow )
t_last_flow = next->stat_record->last_seen;
// continue with next file
}
continue;
} break; // not really needed
default:
// successfully read block
total_bytes += ret;
}
#ifdef COMPAT15
if ( nffile_r->block_header->id == DATA_BLOCK_TYPE_1 ) {
common_record_v1_t *v1_record = (common_record_v1_t *)nffile_r->buff_ptr;
// create an extension map for v1 blocks
if ( v1_map_done == 0 ) {
extension_map_t *map = malloc(sizeof(extension_map_t) + 2 * sizeof(uint16_t) );
if ( ! map ) {
LogError("malloc() allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
exit(255);
}
map->type = ExtensionMapType;
map->size = sizeof(extension_map_t) + 2 * sizeof(uint16_t);
if (( map->size & 0x3 ) != 0 ) {
map->size += 4 - ( map->size & 0x3 );
}
map->map_id = INIT_ID;
map->ex_id[0] = EX_IO_SNMP_2;
map->ex_id[1] = EX_AS_2;
map->ex_id[2] = 0;
map->extension_size = 0;
map->extension_size += extension_descriptor[EX_IO_SNMP_2].size;
map->extension_size += extension_descriptor[EX_AS_2].size;
if ( Insert_Extension_Map(extension_map_list,map) && write_file ) {
// flush new map
AppendToBuffer(nffile_w, (void *)map, map->size);
} // else map already known and flushed
v1_map_done = 1;
}
// convert the records to v2
for ( i=0; i < nffile_r->block_header->NumRecords; i++ ) {
common_record_t *v2_record = (common_record_t *)v1_record;
Convert_v1_to_v2((void *)v1_record);
// now we have a v2 record -> use size of v2_record->size
v1_record = (common_record_v1_t *)((pointer_addr_t)v1_record + v2_record->size);
}
nffile_r->block_header->id = DATA_BLOCK_TYPE_2;
}
#endif
if ( nffile_r->block_header->id == Large_BLOCK_Type ) {
// skip
printf("Xstat block skipped ...\n");
continue;
}
if ( nffile_r->block_header->id != DATA_BLOCK_TYPE_2 ) {
if ( nffile_r->block_header->id == DATA_BLOCK_TYPE_1 ) {
LogError("Can't process nfdump 1.5.x block type 1. Add --enable-compat15 to compile compatibility code. Skip block.\n");
} else {
LogError("Can't process block type %u. Skip block.\n", nffile_r->block_header->id);
}
skipped_blocks++;
continue;
}
flow_record = nffile_r->buff_ptr;
for ( i=0; i < nffile_r->block_header->NumRecords; i++ ) {
switch ( flow_record->type ) {
case CommonRecordV0Type:
case CommonRecordType: {
int match;
uint32_t map_id = flow_record->ext_map;
generic_exporter_t *exp_info = exporter_list[flow_record->exporter_sysid];
if ( map_id >= MAX_EXTENSION_MAPS ) {
LogError("Corrupt data file. Extension map id %u too big.\n", flow_record->ext_map);
exit(255);
}
if ( extension_map_list->slot[map_id] == NULL ) {
LogError("Corrupt data file. Missing extension map %u. Skip record.\n", flow_record->ext_map);
flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
continue;
}
total_flows++;
master_record = &(extension_map_list->slot[map_id]->master_record);
Engine->nfrecord = (uint64_t *)master_record;
ExpandRecord_v2( flow_record, extension_map_list->slot[map_id],
exp_info ? &(exp_info->info) : NULL, master_record);
// Time based filter
// if no time filter is given, the result is always true
match = twin_start && (master_record->first < twin_start || master_record->last > twin_end) ? 0 : 1;
match &= limitflows ? stat_record.numflows < limitflows : 1;
// filter netflow record with user supplied filter
if ( match )
match = (*Engine->FilterEngine)(Engine);
if ( match == 0 ) { // record failed to pass all filters
// increment pointer by number of bytes for netflow record
flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
// go to next record
continue;
}
// Records passed filter -> continue record processing
// Update statistics
UpdateStat(&stat_record, master_record);
// update number of flows matching a given map
extension_map_list->slot[map_id]->ref_count++;
if ( flow_stat ) {
AddFlow(flow_record, master_record, extension_map_list->slot[map_id]);
if ( element_stat ) {
AddStat(flow_record, master_record);
}
} else if ( element_stat ) {
AddStat(flow_record, master_record);
} else if ( sort_flows ) {
InsertFlow(flow_record, master_record, extension_map_list->slot[map_id]);
} else {
if ( write_file ) {
AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size);
if ( xstat )
UpdateXStat(xstat, master_record);
} else if ( print_record ) {
char *string;
// if we need to print out this record
print_record(master_record, &string, tag);
if ( string ) {
if ( limitflows ) {
if ( (stat_record.numflows <= limitflows) )
printf("%s\n", string);
} else
printf("%s\n", string);
}
} else {
// mutually exclusive conditions should prevent executing this code
// this is buggy!
printf("Bug! - this code should never get executed in file %s line %d\n", __FILE__, __LINE__);
}
} // sort_flows - else
} break;
case ExtensionMapType: {
extension_map_t *map = (extension_map_t *)flow_record;
if ( Insert_Extension_Map(extension_map_list, map) && write_file ) {
// flush new map
AppendToBuffer(nffile_w, (void *)map, map->size);
} // else map already known and flushed
} break;
case ExporterRecordType:
case SamplerRecordype:
// Silently skip exporter records
break;
case ExporterInfoRecordType: {
int ret = AddExporterInfo((exporter_info_record_t *)flow_record);
if ( ret != 0 ) {
if ( write_file && ret == 1 )
AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size);
} else {
LogError("Failed to add Exporter Record\n");
}
} break;
case ExporterStatRecordType:
AddExporterStat((exporter_stats_record_t *)flow_record);
break;
case SamplerInfoRecordype: {
int ret = AddSamplerInfo((sampler_info_record_t *)flow_record);
if ( ret != 0 ) {
if ( write_file && ret == 1 )
AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size);
} else {
LogError("Failed to add Sampler Record\n");
}
} break;
default: {
LogError("Skip unknown record type %i\n", flow_record->type);
}
}
// Advance pointer by number of bytes for netflow record
flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
} // for all records
// check if we are done, due to -c option
if ( limitflows )
done = stat_record.numflows >= limitflows;
} // while
CloseFile(nffile_r);
// flush output file
if ( write_file ) {
// flush current buffer to disc
if ( nffile_w->block_header->NumRecords ) {
if ( WriteBlock(nffile_w) <= 0 ) {
LogError("Failed to write output buffer to disk: '%s'" , strerror(errno));
}
}
if ( xstat ) {
if ( WriteExtraBlock(nffile_w, xstat->block_header ) <= 0 ) {
LogError("Failed to write xstat buffer to disk: '%s'" , strerror(errno));
}
}
/* Stat info */
if ( write_file ) {
/* Copy stat info and close file */
memcpy((void *)nffile_w->stat_record, (void *)&stat_record, sizeof(stat_record_t));
CloseUpdateFile(nffile_w, nffile_r->file_header->ident );
nffile_w = DisposeFile(nffile_w);
} // else stdout
}
PackExtensionMapList(extension_map_list);
DisposeFile(nffile_r);
return stat_record;
} // End of process_data
static void run(packet_function_t receive_packet, int socket, send_peer_t peer,
time_t twin, time_t t_begin, int report_seq, int use_subdirs, int compress, int do_xstat) {
common_flow_header_t *nf_header;
FlowSource_t *fs;
struct sockaddr_storage nf_sender;
socklen_t nf_sender_size = sizeof(nf_sender);
time_t t_start, t_now;
uint64_t export_packets;
uint32_t blast_cnt, blast_failures, ignored_packets;
uint16_t version;
ssize_t cnt;
void *in_buff;
int err;
char *string;
srecord_t *commbuff;
if ( !Init_v1() || !Init_v5_v7_input() || !Init_v9() || !Init_IPFIX() )
return;
in_buff = malloc(NETWORK_INPUT_BUFF_SIZE);
if ( !in_buff ) {
LogError("malloc() allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
return;
}
// init vars
commbuff = (srecord_t *)shmem;
nf_header = (common_flow_header_t *)in_buff;
// Init each netflow source output data buffer
fs = FlowSource;
while ( fs ) {
// prepare file
fs->nffile = OpenNewFile(fs->current, NULL, compress, 0, NULL);
if ( !fs->nffile ) {
return;
}
if ( do_xstat ) {
fs->xstat = InitXStat(fs->nffile);
if ( !fs->xstat )
return;
}
// init vars
fs->bad_packets = 0;
fs->first_seen = 0xffffffffffffLL;
fs->last_seen = 0;
// next source
fs = fs->next;
}
export_packets = blast_cnt = blast_failures = 0;
t_start = t_begin;
cnt = 0;
periodic_trigger = 0;
ignored_packets = 0;
// wake up at least at next time slot (twin) + some Overdue time
alarm(t_start + twin + OVERDUE_TIME - time(NULL));
/*
* Main processing loop:
* this loop, continues until done = 1, set by the signal handler
* The while loop will be breaked by the periodic file renaming code
* for proper cleanup
*/
while ( 1 ) {
struct timeval tv;
/* read next bunch of data into beginn of input buffer */
if ( !done) {
#ifdef PCAP
// Debug code to read from pcap file, or from socket
cnt = receive_packet(socket, in_buff, NETWORK_INPUT_BUFF_SIZE , 0,
(struct sockaddr *)&nf_sender, &nf_sender_size);
// in case of reading from file EOF => -2
if ( cnt == -2 )
done = 1;
#else
cnt = recvfrom (socket, in_buff, NETWORK_INPUT_BUFF_SIZE , 0,
(struct sockaddr *)&nf_sender, &nf_sender_size);
#endif
if ( cnt == -1 && errno != EINTR ) {
LogError("ERROR: recvfrom: %s", strerror(errno));
continue;
}
if ( peer.hostname ) {
ssize_t len;
len = sendto(peer.sockfd, in_buff, cnt, 0, (struct sockaddr *)&(peer.addr), peer.addrlen);
if ( len < 0 ) {
LogError("ERROR: sendto(): %s", strerror(errno));
}
}
}
/* Periodic file renaming, if time limit reached or if we are done. */
// t_now = time(NULL);
gettimeofday(&tv, NULL);
t_now = tv.tv_sec;
if ( ((t_now - t_start) >= twin) || done ) {
char subfilename[64];
struct tm *now;
char *subdir;
alarm(0);
now = localtime(&t_start);
// prepare sub dir hierarchy
if ( use_subdirs ) {
subdir = GetSubDir(now);
if ( !subdir ) {
// failed to generate subdir path - put flows into base directory
LogError("Failed to create subdir path!");
// failed to generate subdir path - put flows into base directory
subdir = NULL;
snprintf(subfilename, 63, "nfcapd.%i%02i%02i%02i%02i%02i",
now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec);
} else {
snprintf(subfilename, 63, "%s/nfcapd.%i%02i%02i%02i%02i%02i", subdir,
now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec);
}
} else {
subdir = NULL;
snprintf(subfilename, 63, "nfcapd.%i%02i%02i%02i%02i%02i",
now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec);
}
subfilename[63] = '\0';
// for each flow source update the stats, close the file and re-initialize the new file
fs = FlowSource;
while ( fs ) {
char nfcapd_filename[MAXPATHLEN];
char error[255];
nffile_t *nffile = fs->nffile;
if ( verbose ) {
// Dump to stdout
format_file_block_header(nffile->block_header, &string, 0);
printf("%s\n", string);
}
if ( nffile->block_header->NumRecords ) {
// flush current buffer to disc
if ( WriteBlock(nffile) <= 0 )
LogError("Ident: %s, failed to write output buffer to disk: '%s'" , fs->Ident, strerror(errno));
} // else - no new records in current block
// prepare filename
snprintf(nfcapd_filename, MAXPATHLEN-1, "%s/%s", fs->datadir, subfilename);
nfcapd_filename[MAXPATHLEN-1] = '\0';
// update stat record
// if no flows were collected, fs->last_seen is still 0
// set first_seen to start of this time slot, with twin window size.
if ( fs->last_seen == 0 ) {
fs->first_seen = (uint64_t)1000 * (uint64_t)t_start;
fs->last_seen = (uint64_t)1000 * (uint64_t)(t_start + twin);
}
nffile->stat_record->first_seen = fs->first_seen/1000;
nffile->stat_record->msec_first = fs->first_seen - nffile->stat_record->first_seen*1000;
nffile->stat_record->last_seen = fs->last_seen/1000;
nffile->stat_record->msec_last = fs->last_seen - nffile->stat_record->last_seen*1000;
if ( fs->xstat ) {
if ( WriteExtraBlock(nffile, fs->xstat->block_header ) <= 0 )
LogError("Ident: %s, failed to write xstat buffer to disk: '%s'" , fs->Ident, strerror(errno));
ResetPortHistogram(fs->xstat->port_histogram);
ResetBppHistogram(fs->xstat->bpp_histogram);
}
// Flush Exporter Stat to file
FlushExporterStats(fs);
// Close file
CloseUpdateFile(nffile, fs->Ident);
if ( subdir && !SetupSubDir(fs->datadir, subdir, error, 255) ) {
// in this case the flows get lost! - the rename will fail
// but this should not happen anyway, unless i/o problems, inode problems etc.
LogError("Ident: %s, Failed to create sub hier directories: %s", fs->Ident, error );
}
// if rename fails, we are in big trouble, as we need to get rid of the old .current file
// otherwise, we will loose flows and can not continue collecting new flows
err = rename(fs->current, nfcapd_filename);
if ( err ) {
LogError("Ident: %s, Can't rename dump file: %s", fs->Ident, strerror(errno));
LogError("Ident: %s, Serious Problem! Fix manually", fs->Ident);
if ( launcher_pid )
commbuff->failed = 1;
// we do not update the books here, as the file failed to rename properly
// otherwise the books may be wrong
} else {
struct stat fstat;
if ( launcher_pid )
commbuff->failed = 0;
// Update books
stat(nfcapd_filename, &fstat);
UpdateBooks(fs->bookkeeper, t_start, 512*fstat.st_blocks);
}
// log stats
LogInfo("Ident: '%s' Flows: %llu, Packets: %llu, Bytes: %llu, Sequence Errors: %u, Bad Packets: %u",
fs->Ident, (unsigned long long)nffile->stat_record->numflows, (unsigned long long)nffile->stat_record->numpackets,
(unsigned long long)nffile->stat_record->numbytes, nffile->stat_record->sequence_failure, fs->bad_packets);
// reset stats
fs->bad_packets = 0;
fs->first_seen = 0xffffffffffffLL;
fs->last_seen = 0;
if ( !done ) {
nffile = OpenNewFile(fs->current, nffile, compress, 0, NULL);
if ( !nffile ) {
LogError("killed due to fatal error: ident: %s", fs->Ident);
break;
}
/* XXX needs fixing */
if ( fs->xstat ) {
// to be implemented
}
}
// Dump all extension maps and exporters to the buffer
FlushStdRecords(fs);
// next flow source
fs = fs->next;
} // end of while (fs)
// All flow sources updated - signal launcher if required
if ( launcher_pid ) {
// Signal launcher
// prepare filename for %f expansion
strncpy(commbuff->fname, subfilename, FNAME_SIZE-1);
commbuff->fname[FNAME_SIZE-1] = 0;
snprintf(commbuff->tstring, 16, "%i%02i%02i%02i%02i",
now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min);
commbuff->tstring[15] = 0;
commbuff->tstamp = t_start;
if ( subdir )
strncpy(commbuff->subdir, subdir, FNAME_SIZE);
else
commbuff->subdir[0] = '\0';
if ( launcher_alive ) {
LogInfo("Signal launcher");
kill(launcher_pid, SIGHUP);
} else
LogError("ERROR: Launcher did unexpectedly!");
}
LogInfo("Total ignored packets: %u", ignored_packets);
ignored_packets = 0;
if ( done )
break;
// update alarm for next cycle
t_start += twin;
/* t_start = filename time stamp: begin of slot
* + twin = end of next time interval
* + OVERDUE_TIME = if no data is collected, this is at latest to act
* - , now->tm_sect_now = difference value to now
*/
alarm(t_start + twin + OVERDUE_TIME - t_now);
}
/* check for error condition or done . errno may only be EINTR */
if ( cnt < 0 ) {
if ( periodic_trigger ) {
// alarm triggered, no new flow data
periodic_trigger = 0;
continue;
}
if ( done )
// signaled to terminate - exit from loop
break;
else {
/* this should never be executed as it should be caught in other places */
LogError("error condition in '%s', line '%d', cnt: %i", __FILE__, __LINE__ ,(int)cnt);
continue;
}
}
/* enough data? */
if ( cnt == 0 )
continue;
// get flow source record for current packet, identified by sender IP address
fs = GetFlowSource(&nf_sender);
if ( fs == NULL ) {
fs = AddDynamicSource(&FlowSource, &nf_sender);
if ( fs == NULL ) {
LogError("Skip UDP packet. Ignored packets so far %u packets", ignored_packets);
ignored_packets++;
continue;
}
if ( InitBookkeeper(&fs->bookkeeper, fs->datadir, getpid(), launcher_pid) != BOOKKEEPER_OK ) {
LogError("Failed to initialise bookkeeper for new source");
// fatal error
return;
}
fs->nffile = OpenNewFile(fs->current, NULL, compress, 0, NULL);
if ( !fs->nffile ) {
LogError("Failed to open new collector file");
return;
}
}
/* check for too little data - cnt must be > 0 at this point */
if ( cnt < sizeof(common_flow_header_t) ) {
LogError("Ident: %s, Data length error: too little data for common netflow header. cnt: %i",fs->Ident, (int)cnt);
fs->bad_packets++;
continue;
}
fs->received = tv;
/* Process data - have a look at the common header */
version = ntohs(nf_header->version);
switch (version) {
case 1:
Process_v1(in_buff, cnt, fs);
break;
case 5: // fall through
case 7:
Process_v5_v7(in_buff, cnt, fs);
break;
case 9:
Process_v9(in_buff, cnt, fs);
break;
case 10:
Process_IPFIX(in_buff, cnt, fs);
break;
case 255:
// blast test header
if ( verbose ) {
uint16_t count = ntohs(nf_header->count);
if ( blast_cnt != count ) {
// LogError("Missmatch blast check: Expected %u got %u\n", blast_cnt, count);
blast_cnt = count;
blast_failures++;
} else {
blast_cnt++;
}
if ( blast_cnt == 65535 ) {
fprintf(stderr, "Total missed packets: %u\n", blast_failures);
done = 1;
}
break;
}
default:
// data error, while reading data from socket
LogError("Ident: %s, Error reading netflow header: Unexpected netflow version %i", fs->Ident, version);
fs->bad_packets++;
continue;
// not reached
break;
}
// each Process_xx function has to process the entire input buffer, therefore it's empty now.
export_packets++;
// flush current buffer to disc
if ( fs->nffile->block_header->size > BUFFSIZE ) {
// fishy! - we already wrote into someone elses memory! - I'm sorry
// reset output buffer - data may be lost, as we don not know, where it happen
fs->nffile->block_header->size = 0;
fs->nffile->block_header->NumRecords = 0;
fs->nffile->buff_ptr = (void *)((pointer_addr_t)fs->nffile->block_header + sizeof(data_block_header_t) );
LogError("### Software bug ### Ident: %s, output buffer overflow: expect memory inconsitency", fs->Ident);
}
}
if ( verbose && blast_failures ) {
fprintf(stderr, "Total missed packets: %u\n", blast_failures);
}
free(in_buff);
fs = FlowSource;
while ( fs ) {
DisposeFile(fs->nffile);
fs = fs->next;
}
} /* End of run */
