static void send_data(char *rfile, time_t twin_start, time_t twin_end, uint32_t count, unsigned int delay, int confirm, int netflow_version) { master_record_t master_record; common_record_t *flow_record; nffile_t *nffile; int i, done, ret, again; uint32_t numflows, cnt; #ifdef COMPAT15 int v1_map_done = 0; #endif // Get the first file handle nffile = GetNextFile(NULL, twin_start, twin_end); if ( !nffile ) { LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); return; } if ( nffile == EMPTY_LIST ) { LogError("Empty file list. No files to process\n"); return; } peer.send_buffer = malloc(UDP_PACKET_SIZE); peer.flush = 0; if ( !peer.send_buffer ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); CloseFile(nffile); DisposeFile(nffile); return; } peer.buff_ptr = peer.send_buffer; peer.endp = (void *)((pointer_addr_t)peer.send_buffer + UDP_PACKET_SIZE - 1); if ( netflow_version == 5 ) Init_v5_v7_output(&peer); else Init_v9_output(&peer); numflows = 0; done = 0; // setup Filter Engine to point to master_record, as any record read from file // is expanded into this record Engine->nfrecord = (uint64_t *)&master_record; cnt = 0; while ( !done ) { // get next data block from file ret = ReadBlock(nffile); switch (ret) { case NF_CORRUPT: case NF_ERROR: if ( ret == NF_CORRUPT ) LogError("Skip corrupt data file '%s'\n",GetCurrentFilename()); else LogError("Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) ); // fall through - get next file in chain case NF_EOF: { nffile_t *next = GetNextFile(nffile, twin_start, twin_end); if ( next == EMPTY_LIST ) { done = 1; } if ( next == NULL ) { done = 1; LogError("Unexpected end of file list\n"); } // else continue with next file continue; } break; // not really needed } #ifdef COMPAT15 if ( nffile->block_header->id == DATA_BLOCK_TYPE_1 ) { common_record_v1_t *v1_record = (common_record_v1_t *)nffile->buff_ptr; // create an extension map for v1 blocks if ( v1_map_done == 0 ) { extension_map_t *map = malloc(sizeof(extension_map_t) + 2 * sizeof(uint16_t) ); if ( ! map ) { perror("Memory allocation error"); exit(255); } map->type = ExtensionMapType; map->size = sizeof(extension_map_t) + 2 * sizeof(uint16_t); if (( map->size & 0x3 ) != 0 ) { map->size += 4 - ( map->size & 0x3 ); } map->map_id = INIT_ID; map->ex_id[0] = EX_IO_SNMP_2; map->ex_id[1] = EX_AS_2; map->ex_id[2] = 0; map->extension_size = 0; map->extension_size += extension_descriptor[EX_IO_SNMP_2].size; map->extension_size += extension_descriptor[EX_AS_2].size; Insert_Extension_Map(extension_map_list, map); v1_map_done = 1; } // convert the records to v2 for ( i=0; i < nffile->block_header->NumRecords; i++ ) { common_record_t *v2_record = (common_record_t *)v1_record; Convert_v1_to_v2((void *)v1_record); // now we have a v2 record -> use size of v2_record->size v1_record = (common_record_v1_t *)((pointer_addr_t)v1_record + v2_record->size); } nffile->block_header->id = DATA_BLOCK_TYPE_2; } #endif if ( nffile->block_header->id != DATA_BLOCK_TYPE_2 ) { LogError("Can't process block type %u. Skip block.\n", nffile->block_header->id); continue; } // cnt is the number of blocks, which survived the filter // and added to the output buffer flow_record = nffile->buff_ptr; for ( i=0; i < nffile->block_header->NumRecords; i++ ) { int match; switch ( flow_record->type ) { case CommonRecordType: { if ( extension_map_list->slot[flow_record->ext_map] == NULL ) { LogError("Corrupt data file. Missing extension map %u. Skip record.\n", flow_record->ext_map); flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); continue; } // if no filter is given, the result is always true ExpandRecord_v2( flow_record, extension_map_list->slot[flow_record->ext_map], NULL, &master_record); match = twin_start && (master_record.first < twin_start || master_record.last > twin_end) ? 0 : 1; // filter netflow record with user supplied filter if ( match ) match = (*Engine->FilterEngine)(Engine); if ( match == 0 ) { // record failed to pass all filters // increment pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); // go to next record continue; } // Records passed filter -> continue record processing if ( netflow_version == 5 ) again = Add_v5_output_record(&master_record, &peer); else again = Add_v9_output_record(&master_record, &peer); cnt++; numflows++; if ( peer.flush ) { ret = FlushBuffer(confirm); if ( ret < 0 ) { perror("Error sending data"); CloseFile(nffile); DisposeFile(nffile); return; } if ( delay ) { // sleep as specified usleep(delay); } cnt = 0; } if ( again ) { if ( netflow_version == 5 ) Add_v5_output_record(&master_record, &peer); else Add_v9_output_record(&master_record, &peer); cnt++; } } break; case ExtensionMapType: { extension_map_t *map = (extension_map_t *)flow_record; if ( Insert_Extension_Map(extension_map_list, map) ) { // flush new map } // else map already known and flushed } break; case ExporterRecordType: case SamplerRecordype: case ExporterInfoRecordType: case ExporterStatRecordType: case SamplerInfoRecordype: // Silently skip exporter/sampler records break; default: { LogError("Skip unknown record type %i\n", flow_record->type); } } // Advance pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); } } // while // flush still remaining records if ( cnt ) { ret = FlushBuffer(confirm); if ( ret < 0 ) { perror("Error sending data"); } } // if cnt if (nffile) { CloseFile(nffile); DisposeFile(nffile); } close(peer.sockfd); return; } // End of send_data
stat_record_t process_data(char *wfile, int element_stat, int flow_stat, int sort_flows, printer_t print_header, printer_t print_record, time_t twin_start, time_t twin_end, uint64_t limitflows, int tag, int compress, int do_xstat) { common_record_t *flow_record; master_record_t *master_record; nffile_t *nffile_w, *nffile_r; xstat_t *xstat; stat_record_t stat_record; int done, write_file; #ifdef COMPAT15 int v1_map_done = 0; #endif // time window of all matched flows memset((void *)&stat_record, 0, sizeof(stat_record_t)); stat_record.first_seen = 0x7fffffff; stat_record.msec_first = 999; // Do the logic first // do not print flows when doing any stats are sorting if ( sort_flows || flow_stat || element_stat ) { print_record = NULL; } // do not write flows to file, when doing any stats // -w may apply for flow_stats later write_file = !(sort_flows || flow_stat || element_stat) && wfile; nffile_r = NULL; nffile_w = NULL; xstat = NULL; // Get the first file handle nffile_r = GetNextFile(NULL, twin_start, twin_end); if ( !nffile_r ) { LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); return stat_record; } if ( nffile_r == EMPTY_LIST ) { LogError("Empty file list. No files to process\n"); return stat_record; } // preset time window of all processed flows to the stat record in first flow file t_first_flow = nffile_r->stat_record->first_seen; t_last_flow = nffile_r->stat_record->last_seen; // store infos away for later use // although multiple files may be processed, it is assumed that all // have the same settings is_anonymized = IP_ANONYMIZED(nffile_r); strncpy(Ident, nffile_r->file_header->ident, IDENTLEN); Ident[IDENTLEN-1] = '\0'; // prepare output file if requested if ( write_file ) { nffile_w = OpenNewFile(wfile, NULL, compress, IP_ANONYMIZED(nffile_r), NULL ); if ( !nffile_w ) { if ( nffile_r ) { CloseFile(nffile_r); DisposeFile(nffile_r); } return stat_record; } if ( do_xstat ) { xstat = InitXStat(nffile_w); if ( !xstat ) { if ( nffile_r ) { CloseFile(nffile_r); DisposeFile(nffile_r); } return stat_record; } } } // setup Filter Engine to point to master_record, as any record read from file // is expanded into this record // Engine->nfrecord = (uint64_t *)master_record; done = 0; while ( !done ) { int i, ret; // get next data block from file ret = ReadBlock(nffile_r); switch (ret) { case NF_CORRUPT: case NF_ERROR: if ( ret == NF_CORRUPT ) LogError("Skip corrupt data file '%s'\n",GetCurrentFilename()); else LogError("Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) ); // fall through - get next file in chain case NF_EOF: { nffile_t *next = GetNextFile(nffile_r, twin_start, twin_end); if ( next == EMPTY_LIST ) { done = 1; } else if ( next == NULL ) { done = 1; LogError("Unexpected end of file list\n"); } else { // Update global time span window if ( next->stat_record->first_seen < t_first_flow ) t_first_flow = next->stat_record->first_seen; if ( next->stat_record->last_seen > t_last_flow ) t_last_flow = next->stat_record->last_seen; // continue with next file } continue; } break; // not really needed default: // successfully read block total_bytes += ret; } #ifdef COMPAT15 if ( nffile_r->block_header->id == DATA_BLOCK_TYPE_1 ) { common_record_v1_t *v1_record = (common_record_v1_t *)nffile_r->buff_ptr; // create an extension map for v1 blocks if ( v1_map_done == 0 ) { extension_map_t *map = malloc(sizeof(extension_map_t) + 2 * sizeof(uint16_t) ); if ( ! map ) { LogError("malloc() allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } map->type = ExtensionMapType; map->size = sizeof(extension_map_t) + 2 * sizeof(uint16_t); if (( map->size & 0x3 ) != 0 ) { map->size += 4 - ( map->size & 0x3 ); } map->map_id = INIT_ID; map->ex_id[0] = EX_IO_SNMP_2; map->ex_id[1] = EX_AS_2; map->ex_id[2] = 0; map->extension_size = 0; map->extension_size += extension_descriptor[EX_IO_SNMP_2].size; map->extension_size += extension_descriptor[EX_AS_2].size; if ( Insert_Extension_Map(extension_map_list,map) && write_file ) { // flush new map AppendToBuffer(nffile_w, (void *)map, map->size); } // else map already known and flushed v1_map_done = 1; } // convert the records to v2 for ( i=0; i < nffile_r->block_header->NumRecords; i++ ) { common_record_t *v2_record = (common_record_t *)v1_record; Convert_v1_to_v2((void *)v1_record); // now we have a v2 record -> use size of v2_record->size v1_record = (common_record_v1_t *)((pointer_addr_t)v1_record + v2_record->size); } nffile_r->block_header->id = DATA_BLOCK_TYPE_2; } #endif if ( nffile_r->block_header->id == Large_BLOCK_Type ) { // skip printf("Xstat block skipped ...\n"); continue; } if ( nffile_r->block_header->id != DATA_BLOCK_TYPE_2 ) { if ( nffile_r->block_header->id == DATA_BLOCK_TYPE_1 ) { LogError("Can't process nfdump 1.5.x block type 1. Add --enable-compat15 to compile compatibility code. Skip block.\n"); } else { LogError("Can't process block type %u. Skip block.\n", nffile_r->block_header->id); } skipped_blocks++; continue; } flow_record = nffile_r->buff_ptr; for ( i=0; i < nffile_r->block_header->NumRecords; i++ ) { switch ( flow_record->type ) { case CommonRecordV0Type: case CommonRecordType: { int match; uint32_t map_id = flow_record->ext_map; generic_exporter_t *exp_info = exporter_list[flow_record->exporter_sysid]; if ( map_id >= MAX_EXTENSION_MAPS ) { LogError("Corrupt data file. Extension map id %u too big.\n", flow_record->ext_map); exit(255); } if ( extension_map_list->slot[map_id] == NULL ) { LogError("Corrupt data file. Missing extension map %u. Skip record.\n", flow_record->ext_map); flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); continue; } total_flows++; master_record = &(extension_map_list->slot[map_id]->master_record); Engine->nfrecord = (uint64_t *)master_record; ExpandRecord_v2( flow_record, extension_map_list->slot[map_id], exp_info ? &(exp_info->info) : NULL, master_record); // Time based filter // if no time filter is given, the result is always true match = twin_start && (master_record->first < twin_start || master_record->last > twin_end) ? 0 : 1; match &= limitflows ? stat_record.numflows < limitflows : 1; // filter netflow record with user supplied filter if ( match ) match = (*Engine->FilterEngine)(Engine); if ( match == 0 ) { // record failed to pass all filters // increment pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); // go to next record continue; } // Records passed filter -> continue record processing // Update statistics UpdateStat(&stat_record, master_record); // update number of flows matching a given map extension_map_list->slot[map_id]->ref_count++; if ( flow_stat ) { AddFlow(flow_record, master_record, extension_map_list->slot[map_id]); if ( element_stat ) { AddStat(flow_record, master_record); } } else if ( element_stat ) { AddStat(flow_record, master_record); } else if ( sort_flows ) { InsertFlow(flow_record, master_record, extension_map_list->slot[map_id]); } else { if ( write_file ) { AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size); if ( xstat ) UpdateXStat(xstat, master_record); } else if ( print_record ) { char *string; // if we need to print out this record print_record(master_record, &string, tag); if ( string ) { if ( limitflows ) { if ( (stat_record.numflows <= limitflows) ) printf("%s\n", string); } else printf("%s\n", string); } } else { // mutually exclusive conditions should prevent executing this code // this is buggy! printf("Bug! - this code should never get executed in file %s line %d\n", __FILE__, __LINE__); } } // sort_flows - else } break; case ExtensionMapType: { extension_map_t *map = (extension_map_t *)flow_record; if ( Insert_Extension_Map(extension_map_list, map) && write_file ) { // flush new map AppendToBuffer(nffile_w, (void *)map, map->size); } // else map already known and flushed } break; case ExporterRecordType: case SamplerRecordype: // Silently skip exporter records break; case ExporterInfoRecordType: { int ret = AddExporterInfo((exporter_info_record_t *)flow_record); if ( ret != 0 ) { if ( write_file && ret == 1 ) AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size); } else { LogError("Failed to add Exporter Record\n"); } } break; case ExporterStatRecordType: AddExporterStat((exporter_stats_record_t *)flow_record); break; case SamplerInfoRecordype: { int ret = AddSamplerInfo((sampler_info_record_t *)flow_record); if ( ret != 0 ) { if ( write_file && ret == 1 ) AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size); } else { LogError("Failed to add Sampler Record\n"); } } break; default: { LogError("Skip unknown record type %i\n", flow_record->type); } } // Advance pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); } // for all records // check if we are done, due to -c option if ( limitflows ) done = stat_record.numflows >= limitflows; } // while CloseFile(nffile_r); // flush output file if ( write_file ) { // flush current buffer to disc if ( nffile_w->block_header->NumRecords ) { if ( WriteBlock(nffile_w) <= 0 ) { LogError("Failed to write output buffer to disk: '%s'" , strerror(errno)); } } if ( xstat ) { if ( WriteExtraBlock(nffile_w, xstat->block_header ) <= 0 ) { LogError("Failed to write xstat buffer to disk: '%s'" , strerror(errno)); } } /* Stat info */ if ( write_file ) { /* Copy stat info and close file */ memcpy((void *)nffile_w->stat_record, (void *)&stat_record, sizeof(stat_record_t)); CloseUpdateFile(nffile_w, nffile_r->file_header->ident ); nffile_w = DisposeFile(nffile_w); } // else stdout } PackExtensionMapList(extension_map_list); DisposeFile(nffile_r); return stat_record; } // End of process_data
/* status of read and fill pre-prepared structure lnf_rec */ int lnf_read(lnf_file_t *lnf_file, lnf_rec_t *lnf_rec) { //master_record_t *master_record; int ret; uint32_t map_id; extension_map_t *map; int i; #ifdef COMPAT15 int v1_map_done = 0; #endif begin: if (lnf_file->blk_record_remains == 0) { /* all records in block have been processed, we are going to load nex block */ // get next data block from file if (lnf_file->nffile) { ret = ReadBlock(lnf_file->nffile); lnf_file->processed_blocks++; } else { ret = NF_EOF; /* the firt file in the list */ } switch (ret) { case NF_CORRUPT: return LNF_ERR_CORRUPT; case NF_ERROR: return LNF_ERR_READ; case NF_EOF: return LNF_EOF; default: // successfully read block lnf_file->processed_bytes += ret; } /* block types to be skipped -> goto begin */ /* block types that are unknown -> return */ switch (lnf_file->nffile->block_header->id) { case DATA_BLOCK_TYPE_1: /* old record type - nfdump 1.5 */ lnf_file->skipped_blocks++; goto begin; return LNF_ERR_COMPAT15; break; case DATA_BLOCK_TYPE_2: /* common record type - normally processed */ break; case Large_BLOCK_Type: lnf_file->skipped_blocks++; goto begin; break; default: lnf_file->skipped_blocks++; return LNF_ERR_UNKBLOCK; } lnf_file->flow_record = lnf_file->nffile->buff_ptr; lnf_file->blk_record_remains = lnf_file->nffile->block_header->NumRecords; } /* reading block */ /* there are some records to process - we are going continue reading next record */ lnf_file->blk_record_remains--; switch (lnf_file->flow_record->type) { case ExporterRecordType: case SamplerRecordype: case ExporterInfoRecordType: case ExporterStatRecordType: case SamplerInfoRecordype: /* just skip */ FLOW_RECORD_NEXT(lnf_file->flow_record); goto begin; break; case ExtensionMapType: map = (extension_map_t *)lnf_file->flow_record; //Insert_Extension_Map(&instance->extension_map_list, map); Insert_Extension_Map(lnf_file->extension_map_list, map); FLOW_RECORD_NEXT(lnf_file->flow_record); goto begin; break; case CommonRecordV0Type: case CommonRecordType: /* data record type - go ahead */ break; default: FLOW_RECORD_NEXT(lnf_file->flow_record); return LNF_ERR_UNKREC; } /* we are sure that record is CommonRecordType */ map_id = lnf_file->flow_record->ext_map; if ( map_id >= MAX_EXTENSION_MAPS ) { FLOW_RECORD_NEXT(lnf_file->flow_record); return LNF_ERR_EXTMAPB; } if ( lnf_file->extension_map_list->slot[map_id] == NULL ) { FLOW_RECORD_NEXT(lnf_file->flow_record); return LNF_ERR_EXTMAPM; } // changed in 1.6.8 - added exporter info // ExpandRecord_v2( flow_record, extension_map_list.slot[map_id], master_record); ExpandRecord_v2(lnf_file->flow_record, lnf_file->extension_map_list->slot[map_id], NULL, lnf_rec->master_record); // update number of flows matching a given map lnf_file->extension_map_list->slot[map_id]->ref_count++; // Move pointer by number of bytes for netflow record FLOW_RECORD_NEXT(lnf_file->flow_record); /* { char *s; PrintExtensionMap(instance->extension_map_list.slot[map_id]->map); format_file_block_record(master_record, &s, 0); printf("READ: %s\n", s); } */ // processing map //bit_array_clear(&lnf_file->extensions_arr); bit_array_clear(lnf_rec->extensions_arr); i = 0; while (lnf_rec->master_record->map_ref->ex_id[i]) { __bit_array_set(lnf_rec->extensions_arr, lnf_rec->master_record->map_ref->ex_id[i], 1); i++; } // lnf_rec->extensions_arr = &(lnf_file->extensions_arr); /* the record seems OK. We prepare hash reference with items */ // lnf_file->lnf_rec = lnf_rec; /* XXX temporary */ return LNF_OK; } /* end of _readfnction */
static void send_data(char *rfile, time_t twin_start, time_t twin_end, uint32_t count, unsigned int delay, int confirm, int anon, int netflow_version) { data_block_header_t in_block_header; master_record_t master_record; common_record_t *flow_record, *in_buff; stat_record_t *stat_record; int i, rfd, done, ret, again; uint32_t numflows, cnt; char *string; #ifdef COMPAT15 int v1_map_done = 0; #endif rfd = GetNextFile(0, twin_start, twin_end, &stat_record); if ( rfd < 0 ) { if ( rfd == FILE_ERROR ) fprintf(stderr, "Can't open file for reading: %s\n", strerror(errno)); return; } // prepare read and send buffer in_buff = (common_record_t *) malloc(BUFFSIZE); peer.send_buffer = malloc(UDP_PACKET_SIZE); peer.flush = 0; if ( !in_buff || !peer.send_buffer ) { perror("Memory allocation error"); close(rfd); return; } peer.writeto = peer.send_buffer; peer.endp = (void *)((pointer_addr_t)peer.send_buffer + UDP_PACKET_SIZE - 1); if ( netflow_version == 5 ) Init_v5_v7_output(&peer); else Init_v9_output(&peer); numflows = 0; done = 0; // setup Filter Engine to point to master_record, as any record read from file // is expanded into this record Engine->nfrecord = (uint64_t *)&master_record; cnt = 0; while ( !done ) { // get next data block from file ret = ReadBlock(rfd, &in_block_header, (void *)in_buff, &string); switch (ret) { case NF_CORRUPT: case NF_ERROR: if ( ret == NF_CORRUPT ) fprintf(stderr, "Skip corrupt data file '%s': '%s'\n",GetCurrentFilename(), string); else fprintf(stderr, "Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) ); // fall through - get next file in chain case NF_EOF: rfd = GetNextFile(rfd, twin_start, twin_end, NULL); if ( rfd < 0 ) { if ( rfd == NF_ERROR ) fprintf(stderr, "Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) ); // rfd == EMPTY_LIST done = 1; } // else continue with next file continue; break; // not really needed } #ifdef COMPAT15 if ( in_block_header.id == DATA_BLOCK_TYPE_1 ) { common_record_v1_t *v1_record = (common_record_v1_t *)in_buff; // create an extension map for v1 blocks if ( v1_map_done == 0 ) { extension_map_t *map = malloc(sizeof(extension_map_t) + 2 * sizeof(uint16_t) ); if ( ! map ) { perror("Memory allocation error"); exit(255); } map->type = ExtensionMapType; map->size = sizeof(extension_map_t) + 2 * sizeof(uint16_t); map->map_id = INIT_ID; map->ex_id[0] = EX_IO_SNMP_2; map->ex_id[1] = EX_AS_2; map->ex_id[2] = 0; Insert_Extension_Map(&extension_map_list, map); v1_map_done = 1; } // convert the records to v2 for ( i=0; i < in_block_header.NumRecords; i++ ) { common_record_t *v2_record = (common_record_t *)v1_record; Convert_v1_to_v2((void *)v1_record); // now we have a v2 record -> use size of v2_record->size v1_record = (common_record_v1_t *)((pointer_addr_t)v1_record + v2_record->size); } in_block_header.id = DATA_BLOCK_TYPE_2; } #endif if ( in_block_header.id != DATA_BLOCK_TYPE_2 ) { fprintf(stderr, "Can't process block type %u. Skip block.\n", in_block_header.id); continue; } // cnt is the number of blocks, which survived the filter // and added to the output buffer flow_record = in_buff; for ( i=0; i < in_block_header.NumRecords; i++ ) { int match; if ( flow_record->type == CommonRecordType ) { if ( extension_map_list.slot[flow_record->ext_map] == NULL ) { fprintf(stderr, "Corrupt data file. Missing extension map %u. Skip record.\n", flow_record->ext_map); flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); continue; } // if no filter is given, the result is always true ExpandRecord_v2( flow_record, extension_map_list.slot[flow_record->ext_map], &master_record); match = twin_start && (master_record.first < twin_start || master_record.last > twin_end) ? 0 : 1; // filter netflow record with user supplied filter if ( match ) match = (*Engine->FilterEngine)(Engine); if ( match == 0 ) { // record failed to pass all filters // increment pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); // go to next record continue; } // Records passed filter -> continue record processing if ( anon ) { if ( (flow_record->flags & FLAG_IPV6_ADDR ) == 0 ) { master_record.v4.srcaddr = anonymize(master_record.v4.srcaddr); master_record.v4.dstaddr = anonymize(master_record.v4.dstaddr); } else { uint64_t anon_ip[2]; anonymize_v6(master_record.v6.srcaddr, anon_ip); master_record.v6.srcaddr[0] = anon_ip[0]; master_record.v6.srcaddr[1] = anon_ip[1]; anonymize_v6(master_record.v6.dstaddr, anon_ip); master_record.v6.dstaddr[0] = anon_ip[0]; master_record.v6.dstaddr[1] = anon_ip[1]; } } if ( netflow_version == 5 ) again = Add_v5_output_record(&master_record, &peer); else again = Add_v9_output_record(&master_record, &peer); cnt++; numflows++; if ( peer.flush ) { ret = FlushBuffer(confirm); if ( ret < 0 ) { perror("Error sending data"); close(rfd); return; } if ( delay ) { // sleep as specified usleep(delay); } cnt = 0; } if ( again ) { if ( netflow_version == 5 ) Add_v5_output_record(&master_record, &peer); else Add_v9_output_record(&master_record, &peer); cnt++; } } else if ( flow_record->type == ExtensionMapType ) { extension_map_t *map = (extension_map_t *)flow_record; if ( Insert_Extension_Map(&extension_map_list, map) ) { // flush new map } // else map already known and flushed } else { fprintf(stderr, "Skip unknown record type %i\n", flow_record->type); } // Advance pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); } } // while // flush still remaining records if ( cnt ) { ret = FlushBuffer(confirm); if ( ret < 0 ) { perror("Error sending data"); } } // if cnt if ( rfd ) close(rfd); close(peer.sockfd); return; } // End of send_data
void readNFCap(int rfd,int file_index,uint32_t start,uint32_t end) { data_block_header_t block_header; master_record_t master_record; master_record_t *p_mr = NULL; common_record_t *flow_record; // *in_buff; char *string; char toString[1024]; int ret,done =0,i=0; memset(in_buff,0,BUFFSIZE); p_mr = (master_record_t *)flow_records[file_index].master; while (!done) { // get next data block from file ret = ReadBlock(rfd, &block_header, (void *) &in_buff, &string); switch (ret) { case NF_CORRUPT: case NF_ERROR: if (ret == NF_CORRUPT) fprintf(stderr, "Skip corrupt data file '%s': '%s'\n", GetCurrentFilename(), string); else fprintf(stderr, "Read error in file '%s': %s\n", GetCurrentFilename(), strerror(errno)); // fall through - get next file in chain case NF_EOF: goto out ; } if (block_header.id != DATA_BLOCK_TYPE_2) { fprintf(stderr, "Can't process block type %u. Skip block.\n", block_header.id); continue; } flow_record = in_buff; for (i = 0; i < block_header.NumRecords; i++) { memset(toString,0,1024); if(unlikely(!p_mr)) { printf("**** p_mr is NULL!"); goto out; } if (likely(flow_record->type == CommonRecordType)) { uint32_t map_id = flow_record->ext_map; if (unlikely(extension_map_list.slot[map_id] == NULL)) { snprintf( toString, 1024, "Corrupt data file! No such extension map id: %u. Skip record", flow_record->ext_map); toString[1023] = '\0'; } else { ExpandRecord_v2(flow_record, extension_map_list.slot[flow_record->ext_map], &master_record); // update number of flows matching a given map extension_map_list.slot[map_id]->ref_count++; if(!IS_OUT_TIME_INTERVAL(master_record.first,start,master_record.last,end)) { //aggiungo il record perche' cade nell'intervallo che mi interessa memcpy(p_mr,&master_record,sizeof(master_record_t)); //incremento il contatore flow_records[file_index].total++; //sposto il puntatore p_mr++; } } } else if (flow_record->type == ExtensionMapType) { extension_map_t *map = (extension_map_t *) flow_record; Insert_Extension_Map(&extension_map_list, map); } else { fprintf(stderr, "Skip unknown record type %i\n", flow_record->type); } // Advance pointer by number of bytes for netflow record flow_record = (common_record_t *) ((pointer_addr_t) flow_record + flow_record->size); } // for all records } // while out: return ; }
int StorePcapFlow(FlowSource_t *fs, struct FlowNode *Node) { common_record_t *common_record; uint32_t packets, bytes, pcap_output_record_size; uint64_t start_time, end_time; int j, id; char *string; void *data_ptr; if ( !pcap_extension_map ) { pcap_extension_map = (extension_map_t *)malloc(pcap_extension_info.map->size); if ( !pcap_extension_map ) { LogError("Process_pcap: malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror (errno)); return 0; } memcpy((void *)pcap_extension_map, (void *)pcap_extension_info.map, pcap_extension_info.map->size); if ( !AddExtensionMap(fs, pcap_extension_map) ) { LogError("Process_pcap: Fatal: AddExtensionMap() failed in %s line %d\n", __FILE__, __LINE__); return 0; } } if ( Node->version == AF_INET6 ) { pcap_output_record_size = pcap_output_record_size_v6; dbg_printf("Store Flow v6 node: size: %u\n", pcap_output_record_size); } else if ( Node->version == AF_INET ) { pcap_output_record_size = pcap_output_record_size_v4; dbg_printf("Store Flow v4 node: size: %u\n", pcap_output_record_size); } else { LogError("Process_pcap: Unexpected version in %s line %d: %u\n", __FILE__, __LINE__, Node->version); return 0; } // output buffer size check for all expected records if ( !CheckBufferSpace(fs->nffile, pcap_output_record_size) ) { // fishy! - should never happen. maybe disk full? LogError("Process_pcap: output buffer size error. Abort pcap record processing"); return 0; } // map output record to memory buffer common_record = (common_record_t *)fs->nffile->buff_ptr; // header data common_record->flags = 0; common_record->type = CommonRecordType; common_record->exporter_sysid = 0; common_record->ext_map = pcap_extension_map->map_id; common_record->size = pcap_output_record_size; // pcap common fields common_record->srcport = Node->src_port; common_record->dstport = Node->dst_port; common_record->tcp_flags = Node->flags; common_record->prot = Node->proto; common_record->tos = 0; common_record->fwd_status = 0; if ( Node->version == AF_INET6 ) { SetFlag(common_record->flags, FLAG_IPV6_ADDR); pcap_v6_block_t *pcap_v6_block = (pcap_v6_block_t *)common_record->data; pcap_v6_block->srcaddr[0] = Node->src_addr.v6[0]; pcap_v6_block->srcaddr[1] = Node->src_addr.v6[1]; pcap_v6_block->dstaddr[0] = Node->dst_addr.v6[0]; pcap_v6_block->dstaddr[1] = Node->dst_addr.v6[1]; pcap_v6_block->dPkts = packets = Node->packets; pcap_v6_block->dOctets = bytes = Node->bytes; data_ptr = (void *)pcap_v6_block->data; } else { pcap_v4_block_t *pcap_v4_block = (pcap_v4_block_t *)common_record->data; pcap_v4_block->srcaddr = Node->src_addr.v4; pcap_v4_block->dstaddr = Node->dst_addr.v4; pcap_v4_block->dPkts = packets = Node->packets; pcap_v4_block->dOctets = bytes = Node->bytes; data_ptr = (void *)pcap_v4_block->data; } // process optional extensions j = 0; while ( (id = pcap_extension_map->ex_id[j]) != 0 ) { switch (id) { case EX_IO_SNMP_2: { // 2 byte input/output interface index tpl_ext_4_t *tpl = (tpl_ext_4_t *)data_ptr; tpl->input = 0; tpl->output = 0; data_ptr = (void *)tpl->data; } break; default: // this should never happen, as pcap has no other extensions LogError("Process_pcap: Unexpected extension %i for pcap record. Skip extension", id); } j++; } common_record->first = Node->t_first.tv_sec; common_record->msec_first = Node->t_first.tv_usec / 1000; common_record->last = Node->t_last.tv_sec; common_record->msec_last = Node->t_last.tv_usec / 1000; start_time = (1000LL * (uint64_t)common_record->first) + (uint64_t)common_record->msec_first; end_time = (1000LL * (uint64_t)common_record->last) + (uint64_t)common_record->msec_last; // update first_seen, last_seen if ( start_time < fs->first_seen ) fs->first_seen = start_time; if ( end_time > fs->last_seen ) fs->last_seen = end_time; // Update stats switch (common_record->prot) { case IPPROTO_ICMP: case IPPROTO_ICMPV6: fs->nffile->stat_record->numflows_icmp++; fs->nffile->stat_record->numpackets_icmp += packets; fs->nffile->stat_record->numbytes_icmp += bytes; // fix odd CISCO behaviour for ICMP port/type in src port if ( common_record->srcport != 0 ) { uint8_t *s1, *s2; s1 = (uint8_t *)&(common_record->srcport); s2 = (uint8_t *)&(common_record->dstport); s2[0] = s1[1]; s2[1] = s1[0]; common_record->srcport = 0; } break; case IPPROTO_TCP: fs->nffile->stat_record->numflows_tcp++; fs->nffile->stat_record->numpackets_tcp += packets; fs->nffile->stat_record->numbytes_tcp += bytes; break; case IPPROTO_UDP: fs->nffile->stat_record->numflows_udp++; fs->nffile->stat_record->numpackets_udp += packets; fs->nffile->stat_record->numbytes_udp += bytes; break; default: fs->nffile->stat_record->numflows_other++; fs->nffile->stat_record->numpackets_other += packets; fs->nffile->stat_record->numbytes_other += bytes; } fs->nffile->stat_record->numflows++; fs->nffile->stat_record->numpackets += packets; fs->nffile->stat_record->numbytes += bytes; if ( fs->xstat ) { uint32_t bpp = packets ? bytes/packets : 0; if ( bpp > MAX_BPP ) bpp = MAX_BPP; if ( common_record->prot == IPPROTO_TCP ) { fs->xstat->bpp_histogram->tcp.bpp[bpp]++; fs->xstat->bpp_histogram->tcp.count++; fs->xstat->port_histogram->src_tcp.port[common_record->srcport]++; fs->xstat->port_histogram->dst_tcp.port[common_record->dstport]++; fs->xstat->port_histogram->src_tcp.count++; fs->xstat->port_histogram->dst_tcp.count++; } else if ( common_record->prot == IPPROTO_UDP ) { fs->xstat->bpp_histogram->udp.bpp[bpp]++; fs->xstat->bpp_histogram->udp.count++; fs->xstat->port_histogram->src_udp.port[common_record->srcport]++; fs->xstat->port_histogram->dst_udp.port[common_record->dstport]++; fs->xstat->port_histogram->src_udp.count++; fs->xstat->port_histogram->dst_udp.count++; } } if ( verbose ) { master_record_t master_record; ExpandRecord_v2((common_record_t *)common_record, &pcap_extension_info, NULL, &master_record); format_file_block_record(&master_record, &string, 0); printf("%s\n", string); } // update file record size ( -> output buffer size ) fs->nffile->block_header->NumRecords += 1; fs->nffile->block_header->size += pcap_output_record_size; fs->nffile->buff_ptr = data_ptr; return 1; } /* End of StorePcapFlow */
void Process_v1(void *in_buff, ssize_t in_buff_cnt, FlowSource_t *fs) { netflow_v1_header_t *v1_header; netflow_v1_record_t *v1_record; exporter_v1_t *exporter; extension_map_t *extension_map; common_record_t *common_record; uint64_t start_time, end_time, boot_time; uint32_t First, Last; uint16_t count; uint8_t flags; int i, done, flow_record_length; ssize_t size_left; char *string; // map v1 data structure to input buffer v1_header = (netflow_v1_header_t *)in_buff; exporter = GetExporter(fs, v1_header); if ( !exporter ) { syslog(LOG_ERR,"Process_v1: Exporter NULL: Abort v1 record processing"); return; } flags = 0; exporter->packets++; extension_map = exporter->extension_map; flow_record_length = NETFLOW_V1_RECORD_LENGTH; // this many data to process size_left = in_buff_cnt; common_record = fs->nffile->buff_ptr; done = 0; while ( !done ) { v1_block_t *v1_block; /* Process header */ // count check count = ntohs(v1_header->count); if ( count > NETFLOW_V1_MAX_RECORDS ) { syslog(LOG_ERR,"Process_v1: Unexpected record count in header: %i. Abort v1 record processing", count); fs->nffile->buff_ptr = (void *)common_record; return; } // input buffer size check for all expected records if ( size_left < ( NETFLOW_V1_HEADER_LENGTH + count * flow_record_length) ) { syslog(LOG_ERR,"Process_v1: Not enough data to process v1 record. Abort v1 record processing"); fs->nffile->buff_ptr = (void *)common_record; return; } // output buffer size check for all expected records if ( !CheckBufferSpace(fs->nffile, count * v1_output_record_size) ) { // fishy! - should never happen. maybe disk full? syslog(LOG_ERR,"Process_v1: output buffer size error. Abort v1 record processing"); return; } // map output record to memory buffer common_record = (common_record_t *)fs->nffile->buff_ptr; v1_block = (v1_block_t *)common_record->data; v1_header->SysUptime = ntohl(v1_header->SysUptime); v1_header->unix_secs = ntohl(v1_header->unix_secs); v1_header->unix_nsecs = ntohl(v1_header->unix_nsecs); /* calculate boot time in msec */ boot_time = ((uint64_t)(v1_header->unix_secs)*1000 + ((uint64_t)(v1_header->unix_nsecs) / 1000000) ) - (uint64_t)(v1_header->SysUptime); // process all records v1_record = (netflow_v1_record_t *)((pointer_addr_t)v1_header + NETFLOW_V1_HEADER_LENGTH); /* loop over each records associated with this header */ for (i = 0; i < count; i++) { pointer_addr_t bsize; void *data_ptr; uint8_t *s1, *s2; int j, id; // header data common_record->flags = flags; common_record->type = CommonRecordType; common_record->exporter_sysid = exporter->info.sysid; common_record->ext_map = extension_map->map_id; common_record->size = v1_output_record_size; // v1 common fields common_record->srcport = ntohs(v1_record->srcport); common_record->dstport = ntohs(v1_record->dstport); common_record->tcp_flags = v1_record->tcp_flags; common_record->prot = v1_record->prot; common_record->tos = v1_record->tos; common_record->fwd_status = 0; common_record->reserved = 0; // v1 typed data as fixed struct v1_block v1_block->srcaddr = ntohl(v1_record->srcaddr); v1_block->dstaddr = ntohl(v1_record->dstaddr); v1_block->dPkts = ntohl(v1_record->dPkts); v1_block->dOctets = ntohl(v1_record->dOctets); // process optional extensions data_ptr = (void *)v1_block->data; j = 0; while ( (id = extension_map->ex_id[j]) != 0 ) { switch (id) { case EX_IO_SNMP_2: { // 2 byte input/output interface index tpl_ext_4_t *tpl = (tpl_ext_4_t *)data_ptr; tpl->input = ntohs(v1_record->input); tpl->output = ntohs(v1_record->output); data_ptr = (void *)tpl->data; } break; case EX_NEXT_HOP_v4: { // IPv4 next hop tpl_ext_9_t *tpl = (tpl_ext_9_t *)data_ptr; tpl->nexthop = ntohl(v1_record->nexthop); data_ptr = (void *)tpl->data; } break; case EX_ROUTER_IP_v4: { // IPv4 router address tpl_ext_23_t *tpl = (tpl_ext_23_t *)data_ptr; tpl->router_ip = fs->ip.v4; data_ptr = (void *)tpl->data; ClearFlag(common_record->flags, FLAG_IPV6_EXP); } break; case EX_RECEIVED: { tpl_ext_27_t *tpl = (tpl_ext_27_t *)data_ptr; tpl->received = (uint64_t)((uint64_t)fs->received.tv_sec * 1000LL) + (uint64_t)((uint64_t)fs->received.tv_usec / 1000LL); data_ptr = (void *)tpl->data; } break; default: // this should never happen, as v1 has no other extensions syslog(LOG_ERR,"Process_v1: Unexpected extension %i for v1 record. Skip extension", id); } j++; } // Time issues First = ntohl(v1_record->First); Last = ntohl(v1_record->Last); if ( First > Last ) { /* First in msec, in case of msec overflow, between start and end */ start_time = boot_time - 0x100000000LL + (uint64_t)First; } else { start_time = boot_time + (uint64_t)First; } /* end time in msecs */ end_time = (uint64_t)Last + boot_time; // if overflow happened after flow ended but before got exported if ( Last > v1_header->SysUptime ) { start_time -= 0x100000000LL; end_time -= 0x100000000LL; } common_record->first = start_time/1000; common_record->msec_first = start_time - common_record->first*1000; common_record->last = end_time/1000; common_record->msec_last = end_time - common_record->last*1000; // update first_seen, last_seen if ( start_time < fs->first_seen ) fs->first_seen = start_time; if ( end_time > fs->last_seen ) fs->last_seen = end_time; // Update stats switch (common_record->prot) { case IPPROTO_ICMP: fs->nffile->stat_record->numflows_icmp++; fs->nffile->stat_record->numpackets_icmp += v1_block->dPkts; fs->nffile->stat_record->numbytes_icmp += v1_block->dOctets; // fix odd CISCO behaviour for ICMP port/type in src port if ( common_record->srcport != 0 ) { s1 = (uint8_t *)&(common_record->srcport); s2 = (uint8_t *)&(common_record->dstport); s2[0] = s1[1]; s2[1] = s1[0]; common_record->srcport = 0; } break; case IPPROTO_TCP: fs->nffile->stat_record->numflows_tcp++; fs->nffile->stat_record->numpackets_tcp += v1_block->dPkts; fs->nffile->stat_record->numbytes_tcp += v1_block->dOctets; break; case IPPROTO_UDP: fs->nffile->stat_record->numflows_udp++; fs->nffile->stat_record->numpackets_udp += v1_block->dPkts; fs->nffile->stat_record->numbytes_udp += v1_block->dOctets; break; default: fs->nffile->stat_record->numflows_other++; fs->nffile->stat_record->numpackets_other += v1_block->dPkts; fs->nffile->stat_record->numbytes_other += v1_block->dOctets; } exporter->flows++; fs->nffile->stat_record->numflows++; fs->nffile->stat_record->numpackets += v1_block->dPkts; fs->nffile->stat_record->numbytes += v1_block->dOctets; if ( fs->xstat ) { uint32_t bpp = v1_block->dPkts ? v1_block->dOctets/v1_block->dPkts : 0; if ( bpp > MAX_BPP ) bpp = MAX_BPP; if ( common_record->prot == IPPROTO_TCP ) { fs->xstat->bpp_histogram->tcp.bpp[bpp]++; fs->xstat->bpp_histogram->tcp.count++; fs->xstat->port_histogram->src_tcp.port[common_record->srcport]++; fs->xstat->port_histogram->dst_tcp.port[common_record->dstport]++; fs->xstat->port_histogram->src_tcp.count++; fs->xstat->port_histogram->dst_tcp.count++; } else if ( common_record->prot == IPPROTO_UDP ) { fs->xstat->bpp_histogram->udp.bpp[bpp]++; fs->xstat->bpp_histogram->udp.count++; fs->xstat->port_histogram->src_udp.port[common_record->srcport]++; fs->xstat->port_histogram->dst_udp.port[common_record->dstport]++; fs->xstat->port_histogram->src_udp.count++; fs->xstat->port_histogram->dst_udp.count++; } } if ( verbose ) { master_record_t master_record; ExpandRecord_v2((common_record_t *)common_record, &v1_extension_info, &(exporter->info), &master_record); format_file_block_record(&master_record, &string, 0); printf("%s\n", string); } // advance to next input flow record v1_record = (netflow_v1_record_t *)((pointer_addr_t)v1_record + flow_record_length); if ( ((pointer_addr_t)data_ptr - (pointer_addr_t)common_record) != v1_output_record_size ) { printf("Panic size check: ptr diff: %llu, record size: %u\n", (unsigned long long)((pointer_addr_t)data_ptr - (pointer_addr_t)common_record), v1_output_record_size ); abort(); } // advance to next output record common_record = (common_record_t *)data_ptr; v1_block = (v1_block_t *)common_record->data; // buffer size sanity check - should never happen, but check it anyway bsize = (pointer_addr_t)common_record - (pointer_addr_t)fs->nffile->block_header - sizeof(data_block_header_t); if ( bsize > BUFFSIZE ) { syslog(LOG_ERR,"### Software error ###: %s line %d", __FILE__, __LINE__); syslog(LOG_ERR,"Process_v1: Output buffer overflow! Flush buffer and skip records."); syslog(LOG_ERR,"Buffer size: size: %u, bsize: %llu > %u", fs->nffile->block_header->size, (unsigned long long)bsize, BUFFSIZE); // reset buffer fs->nffile->block_header->size = 0; fs->nffile->block_header->NumRecords = 0; fs->nffile->buff_ptr = (void *)((pointer_addr_t)fs->nffile->block_header + sizeof(data_block_header_t) ); return; } } // End of foreach v1 record // update file record size ( -> output buffer size ) fs->nffile->block_header->NumRecords += count; fs->nffile->block_header->size += count * v1_output_record_size; fs->nffile->buff_ptr = (void *)common_record; // still to go for this many input bytes size_left -= NETFLOW_V1_HEADER_LENGTH + count * flow_record_length; // next header v1_header = (netflow_v1_header_t *)v1_record; // should never be < 0 done = size_left <= 0; } // End of while !done return; } /* End of Process_v1 */
int ExportFlowTable(nffile_t *nffile, int aggregate, int bidir, int date_sorted, extension_map_list_t *extension_map_list) { hash_FlowTable *FlowTable; FlowTableRecord_t *r; SortElement_t *SortList; master_record_t *aggr_record_mask; uint32_t i; uint32_t maxindex, c; #ifdef DEVEL char *string; #endif ExportExtensionMaps(aggregate, bidir, nffile, extension_map_list); ExportExporterList(nffile); aggr_record_mask = GetMasterAggregateMask(); FlowTable = GetFlowTable(); c = 0; maxindex = FlowTable->NumRecords; if ( date_sorted ) { // Sort records according the date SortList = (SortElement_t *)calloc(maxindex, sizeof(SortElement_t)); if ( !SortList ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror (errno)); return 0; } // preset SortList table - still unsorted for ( i=0; i<=FlowTable->IndexMask; i++ ) { r = FlowTable->bucket[i]; if ( !r ) continue; // foreach elem in this bucket while ( r ) { SortList[c].count = 1000LL * r->flowrecord.first + r->flowrecord.msec_first; // sort according the date SortList[c].record = (void *)r; c++; r = r->next; } } if ( c != maxindex ) { LogError("Abort: Mismatch %s line %d: %s\n", __FILE__, __LINE__, strerror (errno)); return 0; } if ( c >= 2 ) heapSort(SortList, c, 0); for ( i = 0; i < c; i++ ) { master_record_t *flow_record; common_record_t *raw_record; extension_info_t *extension_info; r = (FlowTableRecord_t *)(SortList[i].record); raw_record = &(r->flowrecord); extension_info = r->map_info_ref; flow_record = &(extension_info->master_record); ExpandRecord_v2( raw_record, extension_info, r->exp_ref, flow_record); flow_record->dPkts = r->counter[INPACKETS]; flow_record->dOctets = r->counter[INBYTES]; flow_record->out_pkts = r->counter[OUTPACKETS]; flow_record->out_bytes = r->counter[OUTBYTES]; flow_record->aggr_flows = r->counter[FLOWS]; // apply IP mask from aggregation, to provide a pretty output if ( FlowTable->has_masks ) { flow_record->V6.srcaddr[0] &= FlowTable->IPmask[0]; flow_record->V6.srcaddr[1] &= FlowTable->IPmask[1]; flow_record->V6.dstaddr[0] &= FlowTable->IPmask[2]; flow_record->V6.dstaddr[1] &= FlowTable->IPmask[3]; } if ( FlowTable->apply_netbits ) ApplyNetMaskBits(flow_record, FlowTable->apply_netbits); if ( aggr_record_mask ) { ApplyAggrMask(flow_record, aggr_record_mask); } // switch to output extension map flow_record->map_ref = extension_info->map; flow_record->ext_map = extension_info->map->map_id; PackRecord(flow_record, nffile); #ifdef DEVEL format_file_block_record((void *)flow_record, &string, 0); printf("%s\n", string); #endif // Update statistics UpdateStat(nffile->stat_record, flow_record); } } else { // print them as they came for ( i=0; i<=FlowTable->IndexMask; i++ ) { r = FlowTable->bucket[i]; while ( r ) { master_record_t *flow_record; common_record_t *raw_record; extension_info_t *extension_info; raw_record = &(r->flowrecord); extension_info = r->map_info_ref; flow_record = &(extension_info->master_record); ExpandRecord_v2( raw_record, extension_info, r->exp_ref, flow_record); flow_record->dPkts = r->counter[INPACKETS]; flow_record->dOctets = r->counter[INBYTES]; flow_record->out_pkts = r->counter[OUTPACKETS]; flow_record->out_bytes = r->counter[OUTBYTES]; flow_record->aggr_flows = r->counter[FLOWS]; // apply IP mask from aggregation, to provide a pretty output if ( FlowTable->has_masks ) { flow_record->V6.srcaddr[0] &= FlowTable->IPmask[0]; flow_record->V6.srcaddr[1] &= FlowTable->IPmask[1]; flow_record->V6.dstaddr[0] &= FlowTable->IPmask[2]; flow_record->V6.dstaddr[1] &= FlowTable->IPmask[3]; } if ( FlowTable->apply_netbits ) ApplyNetMaskBits(flow_record, FlowTable->apply_netbits); if ( aggr_record_mask ) { ApplyAggrMask(flow_record, aggr_record_mask); } // switch to output extension map flow_record->map_ref = extension_info->map; flow_record->ext_map = extension_info->map->map_id; PackRecord(flow_record, nffile); #ifdef DEVEL format_file_block_record((void *)flow_record, &string, 0); printf("%s\n", string); #endif // Update statistics UpdateStat(nffile->stat_record, flow_record); r = r->next; } } } if ( nffile->block_header->NumRecords ) { if ( WriteBlock(nffile) <= 0 ) { LogError("Failed to write output buffer to disk: '%s'" , strerror(errno)); return 0; } } return 1; } // End of ExportFlowTable
static data_row *process(char *filter) { data_block_header_t block_header; master_record_t master_record; common_record_t *flow_record, *in_buff; int i, rfd, done, ret; uint32_t buffer_size; data_row * port_table; char *string; uint64_t total_bytes; rfd = GetNextFile(0, 0, 0, NULL); if ( rfd < 0 ) { if ( errno ) perror("Can't open file for reading"); return NULL; } // prepare read and send buffer buffer_size = BUFFSIZE; in_buff = (common_record_t *) malloc(buffer_size); if ( !in_buff ) { perror("Memory allocation error"); close(rfd); return NULL; } port_table = (data_row *)calloc(65536, sizeof(data_row)); if ( !port_table) { perror("Memory allocation error"); close(rfd); return NULL; } memset((void *)port_table, 0, 65536 * sizeof(data_row)); // setup Filter Engine to point to master_record, as any record read from file // is expanded into this record Engine->nfrecord = (uint64_t *)&master_record; done = 0; while ( !done ) { // get next data block from file ret = ReadBlock(rfd, &block_header, (void *)in_buff, &string); switch (ret) { case NF_CORRUPT: case NF_ERROR: if ( ret == NF_CORRUPT ) fprintf(stderr, "Skip corrupt data file '%s': '%s'\n",GetCurrentFilename(), string); else fprintf(stderr, "Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) ); // fall through - get next file in chain case NF_EOF: rfd = GetNextFile(rfd, 0, 0, NULL); if ( rfd < 0 ) { if ( rfd == NF_ERROR ) fprintf(stderr, "Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) ); // rfd == EMPTY_LIST done = 1; } // else continue with next file continue; break; // not really needed default: // successfully read block total_bytes += ret; } if ( block_header.id != DATA_BLOCK_TYPE_2 ) { fprintf(stderr, "Can't process block type %u\n", block_header.id); continue; } flow_record = in_buff; for ( i=0; i < block_header.NumRecords; i++ ) { char string[1024]; int ret; if ( flow_record->type == CommonRecordType ) { if ( extension_map_list.slot[flow_record->ext_map] == NULL ) { snprintf(string, 1024, "Corrupt data file! No such extension map id: %u. Skip record", flow_record->ext_map ); string[1023] = '\0'; } else { ExpandRecord_v2( flow_record, extension_map_list.slot[flow_record->ext_map], &master_record); ret = (*Engine->FilterEngine)(Engine); if ( ret == 0 ) { // record failed to pass the filter // increment pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); // go to next record continue; } // Add to stat record if ( master_record.prot == 6 ) { port_table[master_record.dstport].proto[tcp].type[flows]++; port_table[master_record.dstport].proto[tcp].type[packets] += master_record.dPkts; port_table[master_record.dstport].proto[tcp].type[bytes] += master_record.dOctets; } else if ( master_record.prot == 17 ) { port_table[master_record.dstport].proto[udp].type[flows]++; port_table[master_record.dstport].proto[udp].type[packets] += master_record.dPkts; port_table[master_record.dstport].proto[udp].type[bytes] += master_record.dOctets; } } } else if ( flow_record->type == ExtensionMapType ) { extension_map_t *map = (extension_map_t *)flow_record; if ( Insert_Extension_Map(&extension_map_list, map) ) { // flush new map } // else map already known and flushed } else { fprintf(stderr, "Skip unknown record type %i\n", flow_record->type); } // Advance pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); } } // while return port_table; } // End of process
static void process_data(void) { master_record_t master_record; common_record_t *flow_record; nffile_t *nffile; int i, done, ret; #ifdef COMPAT15 int v1_map_done = 0; #endif // Get the first file handle nffile = GetNextFile(NULL, 0, 0); if ( !nffile ) { LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); return; } if ( nffile == EMPTY_LIST ) { LogError("Empty file list. No files to process\n"); return; } done = 0; while ( !done ) { // get next data block from file ret = ReadBlock(nffile); switch (ret) { case NF_CORRUPT: case NF_ERROR: if ( ret == NF_CORRUPT ) fprintf(stderr, "Skip corrupt data file '%s'\n",GetCurrentFilename()); else fprintf(stderr, "Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) ); // fall through - get next file in chain case NF_EOF: { nffile_t *next = GetNextFile(nffile, 0, 0); if ( next == EMPTY_LIST ) { done = 1; } if ( next == NULL ) { done = 1; LogError("Unexpected end of file list\n"); } // else continue with next file continue; } break; // not really needed } #ifdef COMPAT15 if ( nffile->block_header->id == DATA_BLOCK_TYPE_1 ) { common_record_v1_t *v1_record = (common_record_v1_t *)nffile->buff_ptr; // create an extension map for v1 blocks if ( v1_map_done == 0 ) { extension_map_t *map = malloc(sizeof(extension_map_t) + 2 * sizeof(uint16_t) ); if ( ! map ) { LogError("malloc() allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } map->type = ExtensionMapType; map->size = sizeof(extension_map_t) + 2 * sizeof(uint16_t); if (( map->size & 0x3 ) != 0 ) { map->size += 4 - ( map->size & 0x3 ); } map->map_id = INIT_ID; map->ex_id[0] = EX_IO_SNMP_2; map->ex_id[1] = EX_AS_2; map->ex_id[2] = 0; map->extension_size = 0; Insert_Extension_Map(&extension_map_list, map); v1_map_done = 1; } // convert the records to v2 for ( i=0; i < nffile->block_header->NumRecords; i++ ) { common_record_t *v2_record = (common_record_t *)v1_record; Convert_v1_to_v2((void *)v1_record); // now we have a v2 record -> use size of v2_record->size v1_record = (common_record_v1_t *)((pointer_addr_t)v1_record + v2_record->size); } nffile->block_header->id = DATA_BLOCK_TYPE_2; } #endif if ( nffile->block_header->id == Large_BLOCK_Type ) { // skip continue; } if ( nffile->block_header->id != DATA_BLOCK_TYPE_2 ) { fprintf(stderr, "Can't process block type %u. Skip block.\n", nffile->block_header->id); continue; } flow_record = nffile->buff_ptr; for ( i=0; i < nffile->block_header->NumRecords; i++ ) { char string[1024]; switch ( flow_record->type ) { case CommonRecordType: { uint32_t map_id = flow_record->ext_map; generic_exporter_t *exp_info = exporter_list[flow_record->exporter_sysid]; if ( extension_map_list.slot[map_id] == NULL ) { snprintf(string, 1024, "Corrupt data file! No such extension map id: %u. Skip record", flow_record->ext_map ); string[1023] = '\0'; } else { ExpandRecord_v2( flow_record, extension_map_list.slot[flow_record->ext_map], exp_info ? &(exp_info->info) : NULL, &master_record); // update number of flows matching a given map extension_map_list.slot[map_id]->ref_count++; /* * insert hier your calls to your processing routine * master_record now contains the next flow record as specified in nffile.c * for example you can print each record: * */ print_record(&master_record, string); } printf("%s\n", string); } break; case ExtensionMapType: { extension_map_t *map = (extension_map_t *)flow_record; if ( Insert_Extension_Map(&extension_map_list, map) ) { // flush new map } // else map already known and flushed } break; case ExporterRecordType: case SamplerRecordype: // Silently skip exporter records break; default: { fprintf(stderr, "Skip unknown record type %i\n", flow_record->type); } } // Advance pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); } // for all records } // while CloseFile(nffile); DisposeFile(nffile); PackExtensionMapList(&extension_map_list); } // End of process_data
static void process_data(profile_channel_info_t *channels, unsigned int num_channels, time_t tslot, int do_xstat) { common_record_t *flow_record; nffile_t *nffile; FilterEngine_data_t *engine; int i, j, done, ret ; #ifdef COMPAT15 int v1_map_done = 0; #endif nffile = GetNextFile(NULL, 0, 0); if ( !nffile ) { LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); return; } if ( nffile == EMPTY_LIST ) { LogError("Empty file list. No files to process\n"); return; } // store infos away for later use // although multiple files may be processed, it is assumed that all // have the same settings is_anonymized = IP_ANONYMIZED(nffile); strncpy(Ident, nffile->file_header->ident, IDENTLEN); Ident[IDENTLEN-1] = '\0'; done = 0; while ( !done ) { // get next data block from file ret = ReadBlock(nffile); switch (ret) { case NF_CORRUPT: case NF_ERROR: if ( ret == NF_CORRUPT ) LogError("Skip corrupt data file '%s'\n",GetCurrentFilename()); else LogError("Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) ); // fall through - get next file in chain case NF_EOF: { nffile_t *next = GetNextFile(nffile, 0, 0); if ( next == EMPTY_LIST ) { done = 1; } if ( next == NULL ) { done = 1; LogError("Unexpected end of file list\n"); } continue; } break; // not really needed } #ifdef COMPAT15 if ( nffile->block_header->id == DATA_BLOCK_TYPE_1 ) { common_record_v1_t *v1_record = (common_record_v1_t *)nffile->buff_ptr; // create an extension map for v1 blocks if ( v1_map_done == 0 ) { extension_map_t *map = malloc(sizeof(extension_map_t) + 2 * sizeof(uint16_t) ); if ( ! map ) { LogError("malloc() allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } map->type = ExtensionMapType; map->size = sizeof(extension_map_t) + 2 * sizeof(uint16_t); if (( map->size & 0x3 ) != 0 ) { map->size += 4 - ( map->size & 0x3 ); } map->map_id = INIT_ID; map->ex_id[0] = EX_IO_SNMP_2; map->ex_id[1] = EX_AS_2; map->ex_id[2] = 0; map->extension_size = 0; map->extension_size += extension_descriptor[EX_IO_SNMP_2].size; map->extension_size += extension_descriptor[EX_AS_2].size; if ( Insert_Extension_Map(extension_map_list, map) ) { int j; for ( j=0; j < num_channels; j++ ) { if ( channels[j].nffile != NULL) { // flush new map AppendToBuffer(channels[j].nffile, (void *)map, map->size); } } } // else map already known and flushed v1_map_done = 1; } // convert the records to v2 for ( i=0; i < nffile->block_header->NumRecords; i++ ) { common_record_t *v2_record = (common_record_t *)v1_record; Convert_v1_to_v2((void *)v1_record); // now we have a v2 record -> use size of v2_record->size v1_record = (common_record_v1_t *)((pointer_addr_t)v1_record + v2_record->size); } nffile->block_header->id = DATA_BLOCK_TYPE_2; } #endif if ( nffile->block_header->id == Large_BLOCK_Type ) { // skip continue; } if ( nffile->block_header->id != DATA_BLOCK_TYPE_2 ) { LogError("Can't process block type %u. Skip block.\n", nffile->block_header->id); continue; } flow_record = nffile->buff_ptr; for ( i=0; i < nffile->block_header->NumRecords; i++ ) { switch ( flow_record->type ) { case CommonRecordType: { generic_exporter_t *exp_info = exporter_list[flow_record->exporter_sysid]; uint32_t map_id = flow_record->ext_map; master_record_t *master_record; if ( extension_map_list->slot[map_id] == NULL ) { LogError("Corrupt data file. Missing extension map %u. Skip record.\n", flow_record->ext_map); flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); continue; } master_record = &(extension_map_list->slot[map_id]->master_record); ExpandRecord_v2( flow_record, extension_map_list->slot[flow_record->ext_map], exp_info ? &(exp_info->info) : NULL, master_record); for ( j=0; j < num_channels; j++ ) { int match; // apply profile filter (channels[j].engine)->nfrecord = (uint64_t *)master_record; engine = channels[j].engine; match = (*engine->FilterEngine)(engine); // if profile filter failed -> next profile if ( !match ) continue; // filter was successful -> continue record processing // update statistics UpdateStat(&channels[j].stat_record, master_record); if ( channels[j].nffile ) UpdateStat(channels[j].nffile->stat_record, master_record); if ( channels[j].xstat ) UpdateXStat(channels[j].xstat, master_record); // do we need to write data to new file - shadow profiles do not have files. // check if we need to flush the output buffer if ( channels[j].nffile != NULL ) { // write record to output buffer AppendToBuffer(channels[j].nffile, (void *)flow_record, flow_record->size); } } // End of for all channels } break; case ExtensionMapType: { extension_map_t *map = (extension_map_t *)flow_record; if ( Insert_Extension_Map(extension_map_list, map) ) { int j; for ( j=0; j < num_channels; j++ ) { if ( channels[j].nffile != NULL ) { // flush new map AppendToBuffer(channels[j].nffile, (void *)map, map->size); } } } // else map already known and flushed } break; case ExporterInfoRecordType: { int ret = AddExporterInfo((exporter_info_record_t *)flow_record); if ( ret != 0 ) { int j; for ( j=0; j < num_channels; j++ ) { if ( channels[j].nffile != NULL && ret == 1) { // flush new exporter AppendToBuffer(channels[j].nffile, (void *)flow_record, flow_record->size); } } } else { LogError("Failed to add Exporter Record\n"); } } break; case SamplerInfoRecordype: { int ret = AddSamplerInfo((sampler_info_record_t *)flow_record); if ( ret != 0 ) { int j; for ( j=0; j < num_channels; j++ ) { if ( channels[j].nffile != NULL && ret == 1 ) { // flush new map AppendToBuffer(channels[j].nffile, (void *)flow_record, flow_record->size); } } } else { LogError("Failed to add Sampler Record\n"); } } break; case ExporterRecordType: case SamplerRecordype: case ExporterStatRecordType: // Silently skip exporter records break; default: { LogError("Skip unknown record type %i\n", flow_record->type); } } // Advance pointer by number of bytes for netflow record flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size); } // End of for all umRecords } // End of while !done // do we need to write data to new file - shadow profiles do not have files. for ( j=0; j < num_channels; j++ ) { if ( channels[j].nffile != NULL ) { // flush output buffer if ( channels[j].nffile->block_header->NumRecords ) { if ( WriteBlock(channels[j].nffile) <= 0 ) { LogError("Failed to write output buffer to disk: '%s'" , strerror(errno)); } } } } CloseFile(nffile); DisposeFile(nffile); } // End of process_data