int AddExtensionMap(FlowSource_t *fs, extension_map_t *map) {
int next_slot = fs->extension_map_list.next_free;
// is it a new map, we have not yet in the list
if ( map->map_id == INIT_ID ) {
if ( next_slot >= fs->extension_map_list.max_maps ) {
// extend map list
extension_map_t **p = realloc((void *)fs->extension_map_list.maps,
(fs->extension_map_list.max_maps + BLOCK_SIZE ) * sizeof(extension_map_t *));
if ( !p ) {
LogError("realloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
return 0;
}
fs->extension_map_list.maps = p;
fs->extension_map_list.max_maps += BLOCK_SIZE;
}
fs->extension_map_list.maps[next_slot] = map;
map->map_id = next_slot;
fs->extension_map_list.next_free++;
}
AppendToBuffer(fs->nffile, (void *)map, map->size);
return 1;
} // End of AddExtensionMap
int FlushInfoExporter(FlowSource_t *fs, exporter_info_record_t *exporter) {
exporter->sysid = AssignExporterID();
fs->exporter_count++;
AppendToBuffer(fs->nffile, (void *)exporter, exporter->header.size);
#ifdef DEVEL
{
#define IP_STRING_LEN 40
char ipstr[IP_STRING_LEN];
printf("Flush Exporter: ");
if ( exporter->sa_family == AF_INET ) {
uint32_t _ip = htonl(exporter->ip.v4);
inet_ntop(AF_INET, &_ip, ipstr, sizeof(ipstr));
printf("SysID: %u, IP: %16s, version: %u, ID: %2u\n", exporter->sysid,
ipstr, exporter->version, exporter->id);
} else if ( exporter->sa_family == AF_INET6 ) {
uint64_t _ip[2];
_ip[0] = htonll(exporter->ip.v6[0]);
_ip[1] = htonll(exporter->ip.v6[1]);
inet_ntop(AF_INET6, &_ip, ipstr, sizeof(ipstr));
printf("SysID: %u, IP: %40s, version: %u, ID: %2u\n", exporter->sysid,
ipstr, exporter->version, exporter->id);
} else {
strncpy(ipstr, "<unknown>", IP_STRING_LEN);
printf("**** Exporter IP version unknown ****\n");
}
}
#endif
return 1;
} // End of FlushInfoExporter
void ReadBuffer::Unset()
{
// ::printf(" Unset()\n");
AppendToBuffer(readRangeLength_);
readRange_ = nullptr;
readRangeLength_ = 0;
}
void FlushExporterStats(FlowSource_t *fs) {
generic_exporter_t *e = fs->exporter_data;
exporter_stats_record_t *exporter_stats;
uint32_t i, size;
// idle collector ..
if ( !fs->exporter_count )
return;
size = sizeof(exporter_stats_record_t) + (fs->exporter_count -1) * sizeof(struct exporter_stat_s);
exporter_stats = ( exporter_stats_record_t *)malloc(size);
if ( !exporter_stats ) {
LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
return;
}
exporter_stats->header.type = ExporterStatRecordType;
exporter_stats->header.size = size;
exporter_stats->stat_count = fs->exporter_count;
#ifdef DEVEL
printf("Flush Exporter Stats: %u exporters, size: %u\n", fs->exporter_count, size);
#endif
i = 0;
while ( e ) {
// prevent memory corruption - just in case .. should not happen anyway
// continue loop for error reporting after while
if ( i >= fs->exporter_count ) {
i++;
e = e->next;
continue;
}
exporter_stats->stat[i].sysid = e->info.sysid;
exporter_stats->stat[i].sequence_failure = e->sequence_failure;
exporter_stats->stat[i].packets = e->packets;
exporter_stats->stat[i].flows = e->flows;
#ifdef DEVEL
printf("Stat: SysID: %u, version: %u, ID: %2u, Packets: %llu, Flows: %llu, Sequence Failures: %u\n", e->info.sysid,
e->info.version, e->info.id, e->packets, e->flows, e->sequence_failure);
#endif
// reset counters
e->sequence_failure = 0;
e->packets = 0;
e->flows = 0;
i++;
e = e->next;
}
AppendToBuffer(fs->nffile, (void *)exporter_stats, size);
free(exporter_stats);
if ( i != fs->exporter_count ) {
LogError("ERROR: exporter stats: Expected %u records, but found %u in %s line %d: %s\n",
fs->exporter_count, i, __FILE__, __LINE__, strerror(errno) );
}
} // End of FlushExporterStats
void FlushStdRecords(FlowSource_t *fs) {
generic_exporter_t *e = fs->exporter_data;
int i;
while ( e ) {
generic_sampler_t *sampler = e->sampler;
AppendToBuffer(fs->nffile, (void *)&(e->info), e->info.header.size);
while ( sampler ) {
AppendToBuffer(fs->nffile, (void *)&(sampler->info), sampler->info.header.size);
sampler = sampler->next;
}
e = e->next;
}
for ( i=0; i<fs->extension_map_list.next_free; i++ ) {
extension_map_t *map = fs->extension_map_list.maps[i];
AppendToBuffer(fs->nffile, (void *)map, map->size);
}
} // End of FlushStdRecords
void nsScanner::AppendASCIItoBuffer(const char* aData, PRUint32 aLen,
nsIRequest *aRequest)
{
nsScannerString::Buffer* buf = nsScannerString::AllocBuffer(aLen);
if (buf)
{
LossyConvertEncoding<char, PRUnichar> converter(buf->DataStart());
converter.write(aData, aLen);
converter.write_terminator();
AppendToBuffer(buf, aRequest);
}
}
int AddExtensionMap(FlowSource_t *fs, extension_map_t *map) {
int next_slot = fs->extension_map_list.next_free;
dbg_printf("Add extension map\n");
// is it a new map, we have not yet in the list
if ( map->map_id == INIT_ID ) {
if ( next_slot >= fs->extension_map_list.max_maps ) {
// extend map list
dbg_printf("List full - extend extension list to %d slots\n", fs->extension_map_list.max_maps + BLOCK_SIZE);
extension_map_t **p = realloc((void *)fs->extension_map_list.maps,
(fs->extension_map_list.max_maps + BLOCK_SIZE ) * sizeof(extension_map_t *));
if ( !p ) {
LogError("realloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
return 0;
}
fs->extension_map_list.maps = p;
fs->extension_map_list.max_maps += BLOCK_SIZE;
}
dbg_printf("Add map to slot %d\n", next_slot);
fs->extension_map_list.maps[next_slot] = map;
map->map_id = next_slot;
fs->extension_map_list.num_maps++;
if ( (next_slot + 1) == fs->extension_map_list.num_maps ) {
// sequencially filled slots
// next free is next slot
fs->extension_map_list.next_free++;
dbg_printf("Next slot is sequential: %d\n", fs->extension_map_list.next_free);
} else {
// fill gap in list - search for next free
int i;
dbg_printf("Search next slot ... \n");
for ( i = (next_slot + 1); i < fs->extension_map_list.max_maps; i++ ) {
if ( fs->extension_map_list.maps[i] == NULL ) {
// empty slot found
dbg_printf("Empty slot found at %d\n", i);
break;
}
}
// assign next_free - if none found up to max, the list will get extended
// in the next round
dbg_printf("Set next free to %d\n", i);
fs->extension_map_list.next_free = i;
}
}
AppendToBuffer(fs->nffile, (void *)map, map->size);
return 1;
} // End of AddExtensionMap
static inline void GetLayouter(Layouter::LineCacheItem &line, const char *&str, FontState &state)
{
if (line.buffer != NULL) free(line.buffer);
typename T::CharType *buff_begin = MallocT<typename T::CharType>(DRAW_STRING_BUFFER);
const typename T::CharType *buffer_last = buff_begin + DRAW_STRING_BUFFER;
typename T::CharType *buff = buff_begin;
FontMap &fontMapping = line.runs;
Font *f = Layouter::GetFont(state.fontsize, state.cur_colour);
line.buffer = buff_begin;
/*
* Go through the whole string while adding Font instances to the font map
* whenever the font changes, and convert the wide characters into a format
* usable by ParagraphLayout.
*/
for (; buff < buffer_last;) {
WChar c = Utf8Consume(const_cast<const char **>(&str));
if (c == '\0' || c == '\n') {
break;
} else if (c >= SCC_BLUE && c <= SCC_BLACK) {
state.SetColour((TextColour)(c - SCC_BLUE));
} else if (c == SCC_PREVIOUS_COLOUR) { // Revert to the previous colour.
state.SetPreviousColour();
} else if (c == SCC_TINYFONT) {
state.SetFontSize(FS_SMALL);
} else if (c == SCC_BIGFONT) {
state.SetFontSize(FS_LARGE);
} else {
/* Filter out text direction characters that shouldn't be drawn, and
* will not be handled in the fallback non ICU case because they are
* mostly needed for RTL languages which need more ICU support. */
if (!T::SUPPORTS_RTL && IsTextDirectionChar(c)) continue;
buff += AppendToBuffer(buff, buffer_last, c);
continue;
}
if (!fontMapping.Contains(buff - buff_begin)) {
fontMapping.Insert(buff - buff_begin, f);
}
f = Layouter::GetFont(state.fontsize, state.cur_colour);
}
/* Better safe than sorry. */
*buff = '\0';
if (!fontMapping.Contains(buff - buff_begin)) {
fontMapping.Insert(buff - buff_begin, f);
}
line.layout = GetParagraphLayout(buff_begin, buff, fontMapping);
line.state_after = state;
}
/**
* Use this constructor if you want i/o to be based on
* a single string you hand in during construction.
* This short cut was added for Javascript.
*
* @update gess 5/12/98
* @param aMode represents the parser mode (nav, other)
* @return
*/
nsScanner::nsScanner(const nsAString& anHTMLString, const nsACString& aCharset,
PRInt32 aSource)
: mParser(nsnull)
{
MOZ_COUNT_CTOR(nsScanner);
mTotalRead = anHTMLString.Length();
mSlidingBuffer = nsnull;
mCountRemaining = 0;
mFirstNonWhitespacePosition = -1;
AppendToBuffer(anHTMLString);
mSlidingBuffer->BeginReading(mCurrentPosition);
mMarkPosition = mCurrentPosition;
mIncremental = PR_FALSE;
mUnicodeDecoder = 0;
mCharsetSource = kCharsetUninitialized;
SetDocumentCharset(aCharset, aSource);
}
int FlushInfoSampler(FlowSource_t *fs, sampler_info_record_t *sampler) {
AppendToBuffer(fs->nffile, (void *)sampler, sampler->header.size);
#ifdef DEVEL
{
printf("Flush Sampler: ");
if ( sampler->id < 0 ) {
printf("Exporter SysID: %u, Generic Sampler: mode: %u, interval: %u\n",
sampler->exporter_sysid, sampler->mode, sampler->interval);
} else {
printf("Exporter SysID: %u, Sampler: id: %i, mode: %u, interval: %u\n",
sampler->exporter_sysid, sampler->id, sampler->mode, sampler->interval);
}
}
#endif
return 1;
} // End of FlushInfoSampler
/**
* Use this constructor if you want i/o to be based on
* a single string you hand in during construction.
* This short cut was added for Javascript.
*
* @update gess 5/12/98
* @param aMode represents the parser mode (nav, other)
* @return
*/
nsScanner::nsScanner(const nsAString& anHTMLString, const nsACString& aCharset,
PRInt32 aSource)
: mParser(nsnull)
{
MOZ_COUNT_CTOR(nsScanner);
mSlidingBuffer = nsnull;
mCountRemaining = 0;
mFirstNonWhitespacePosition = -1;
if (AppendToBuffer(anHTMLString)) {
mSlidingBuffer->BeginReading(mCurrentPosition);
} else {
/* XXX see hack below, re: bug 182067 */
memset(&mCurrentPosition, 0, sizeof(mCurrentPosition));
mEndPosition = mCurrentPosition;
}
mMarkPosition = mCurrentPosition;
mIncremental = PR_FALSE;
mUnicodeDecoder = 0;
mCharsetSource = kCharsetUninitialized;
}
stat_record_t process_data(char *wfile, int element_stat, int flow_stat, int sort_flows,
printer_t print_header, printer_t print_record, time_t twin_start, time_t twin_end,
uint64_t limitflows, int tag, int compress, int do_xstat) {
common_record_t *flow_record;
master_record_t *master_record;
nffile_t *nffile_w, *nffile_r;
xstat_t *xstat;
stat_record_t stat_record;
int done, write_file;
#ifdef COMPAT15
int v1_map_done = 0;
#endif
// time window of all matched flows
memset((void *)&stat_record, 0, sizeof(stat_record_t));
stat_record.first_seen = 0x7fffffff;
stat_record.msec_first = 999;
// Do the logic first
// do not print flows when doing any stats are sorting
if ( sort_flows || flow_stat || element_stat ) {
print_record = NULL;
}
// do not write flows to file, when doing any stats
// -w may apply for flow_stats later
write_file = !(sort_flows || flow_stat || element_stat) && wfile;
nffile_r = NULL;
nffile_w = NULL;
xstat = NULL;
// Get the first file handle
nffile_r = GetNextFile(NULL, twin_start, twin_end);
if ( !nffile_r ) {
LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
return stat_record;
}
if ( nffile_r == EMPTY_LIST ) {
LogError("Empty file list. No files to process\n");
return stat_record;
}
// preset time window of all processed flows to the stat record in first flow file
t_first_flow = nffile_r->stat_record->first_seen;
t_last_flow = nffile_r->stat_record->last_seen;
// store infos away for later use
// although multiple files may be processed, it is assumed that all
// have the same settings
is_anonymized = IP_ANONYMIZED(nffile_r);
strncpy(Ident, nffile_r->file_header->ident, IDENTLEN);
Ident[IDENTLEN-1] = '\0';
// prepare output file if requested
if ( write_file ) {
nffile_w = OpenNewFile(wfile, NULL, compress, IP_ANONYMIZED(nffile_r), NULL );
if ( !nffile_w ) {
if ( nffile_r ) {
CloseFile(nffile_r);
DisposeFile(nffile_r);
}
return stat_record;
}
if ( do_xstat ) {
xstat = InitXStat(nffile_w);
if ( !xstat ) {
if ( nffile_r ) {
CloseFile(nffile_r);
DisposeFile(nffile_r);
}
return stat_record;
}
}
}
// setup Filter Engine to point to master_record, as any record read from file
// is expanded into this record
// Engine->nfrecord = (uint64_t *)master_record;
done = 0;
while ( !done ) {
int i, ret;
// get next data block from file
ret = ReadBlock(nffile_r);
switch (ret) {
case NF_CORRUPT:
case NF_ERROR:
if ( ret == NF_CORRUPT )
LogError("Skip corrupt data file '%s'\n",GetCurrentFilename());
else
LogError("Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) );
// fall through - get next file in chain
case NF_EOF: {
nffile_t *next = GetNextFile(nffile_r, twin_start, twin_end);
if ( next == EMPTY_LIST ) {
done = 1;
} else if ( next == NULL ) {
done = 1;
LogError("Unexpected end of file list\n");
} else {
// Update global time span window
if ( next->stat_record->first_seen < t_first_flow )
t_first_flow = next->stat_record->first_seen;
if ( next->stat_record->last_seen > t_last_flow )
t_last_flow = next->stat_record->last_seen;
// continue with next file
}
continue;
} break; // not really needed
default:
// successfully read block
total_bytes += ret;
}
#ifdef COMPAT15
if ( nffile_r->block_header->id == DATA_BLOCK_TYPE_1 ) {
common_record_v1_t *v1_record = (common_record_v1_t *)nffile_r->buff_ptr;
// create an extension map for v1 blocks
if ( v1_map_done == 0 ) {
extension_map_t *map = malloc(sizeof(extension_map_t) + 2 * sizeof(uint16_t) );
if ( ! map ) {
LogError("malloc() allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
exit(255);
}
map->type = ExtensionMapType;
map->size = sizeof(extension_map_t) + 2 * sizeof(uint16_t);
if (( map->size & 0x3 ) != 0 ) {
map->size += 4 - ( map->size & 0x3 );
}
map->map_id = INIT_ID;
map->ex_id[0] = EX_IO_SNMP_2;
map->ex_id[1] = EX_AS_2;
map->ex_id[2] = 0;
map->extension_size = 0;
map->extension_size += extension_descriptor[EX_IO_SNMP_2].size;
map->extension_size += extension_descriptor[EX_AS_2].size;
if ( Insert_Extension_Map(extension_map_list,map) && write_file ) {
// flush new map
AppendToBuffer(nffile_w, (void *)map, map->size);
} // else map already known and flushed
v1_map_done = 1;
}
// convert the records to v2
for ( i=0; i < nffile_r->block_header->NumRecords; i++ ) {
common_record_t *v2_record = (common_record_t *)v1_record;
Convert_v1_to_v2((void *)v1_record);
// now we have a v2 record -> use size of v2_record->size
v1_record = (common_record_v1_t *)((pointer_addr_t)v1_record + v2_record->size);
}
nffile_r->block_header->id = DATA_BLOCK_TYPE_2;
}
#endif
if ( nffile_r->block_header->id == Large_BLOCK_Type ) {
// skip
printf("Xstat block skipped ...\n");
continue;
}
if ( nffile_r->block_header->id != DATA_BLOCK_TYPE_2 ) {
if ( nffile_r->block_header->id == DATA_BLOCK_TYPE_1 ) {
LogError("Can't process nfdump 1.5.x block type 1. Add --enable-compat15 to compile compatibility code. Skip block.\n");
} else {
LogError("Can't process block type %u. Skip block.\n", nffile_r->block_header->id);
}
skipped_blocks++;
continue;
}
flow_record = nffile_r->buff_ptr;
for ( i=0; i < nffile_r->block_header->NumRecords; i++ ) {
switch ( flow_record->type ) {
case CommonRecordV0Type:
case CommonRecordType: {
int match;
uint32_t map_id = flow_record->ext_map;
generic_exporter_t *exp_info = exporter_list[flow_record->exporter_sysid];
if ( map_id >= MAX_EXTENSION_MAPS ) {
LogError("Corrupt data file. Extension map id %u too big.\n", flow_record->ext_map);
exit(255);
}
if ( extension_map_list->slot[map_id] == NULL ) {
LogError("Corrupt data file. Missing extension map %u. Skip record.\n", flow_record->ext_map);
flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
continue;
}
total_flows++;
master_record = &(extension_map_list->slot[map_id]->master_record);
Engine->nfrecord = (uint64_t *)master_record;
ExpandRecord_v2( flow_record, extension_map_list->slot[map_id],
exp_info ? &(exp_info->info) : NULL, master_record);
// Time based filter
// if no time filter is given, the result is always true
match = twin_start && (master_record->first < twin_start || master_record->last > twin_end) ? 0 : 1;
match &= limitflows ? stat_record.numflows < limitflows : 1;
// filter netflow record with user supplied filter
if ( match )
match = (*Engine->FilterEngine)(Engine);
if ( match == 0 ) { // record failed to pass all filters
// increment pointer by number of bytes for netflow record
flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
// go to next record
continue;
}
// Records passed filter -> continue record processing
// Update statistics
UpdateStat(&stat_record, master_record);
// update number of flows matching a given map
extension_map_list->slot[map_id]->ref_count++;
if ( flow_stat ) {
AddFlow(flow_record, master_record, extension_map_list->slot[map_id]);
if ( element_stat ) {
AddStat(flow_record, master_record);
}
} else if ( element_stat ) {
AddStat(flow_record, master_record);
} else if ( sort_flows ) {
InsertFlow(flow_record, master_record, extension_map_list->slot[map_id]);
} else {
if ( write_file ) {
AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size);
if ( xstat )
UpdateXStat(xstat, master_record);
} else if ( print_record ) {
char *string;
// if we need to print out this record
print_record(master_record, &string, tag);
if ( string ) {
if ( limitflows ) {
if ( (stat_record.numflows <= limitflows) )
printf("%s\n", string);
} else
printf("%s\n", string);
}
} else {
// mutually exclusive conditions should prevent executing this code
// this is buggy!
printf("Bug! - this code should never get executed in file %s line %d\n", __FILE__, __LINE__);
}
} // sort_flows - else
} break;
case ExtensionMapType: {
extension_map_t *map = (extension_map_t *)flow_record;
if ( Insert_Extension_Map(extension_map_list, map) && write_file ) {
// flush new map
AppendToBuffer(nffile_w, (void *)map, map->size);
} // else map already known and flushed
} break;
case ExporterRecordType:
case SamplerRecordype:
// Silently skip exporter records
break;
case ExporterInfoRecordType: {
int ret = AddExporterInfo((exporter_info_record_t *)flow_record);
if ( ret != 0 ) {
if ( write_file && ret == 1 )
AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size);
} else {
LogError("Failed to add Exporter Record\n");
}
} break;
case ExporterStatRecordType:
AddExporterStat((exporter_stats_record_t *)flow_record);
break;
case SamplerInfoRecordype: {
int ret = AddSamplerInfo((sampler_info_record_t *)flow_record);
if ( ret != 0 ) {
if ( write_file && ret == 1 )
AppendToBuffer(nffile_w, (void *)flow_record, flow_record->size);
} else {
LogError("Failed to add Sampler Record\n");
}
} break;
default: {
LogError("Skip unknown record type %i\n", flow_record->type);
}
}
// Advance pointer by number of bytes for netflow record
flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
} // for all records
// check if we are done, due to -c option
if ( limitflows )
done = stat_record.numflows >= limitflows;
} // while
CloseFile(nffile_r);
// flush output file
if ( write_file ) {
// flush current buffer to disc
if ( nffile_w->block_header->NumRecords ) {
if ( WriteBlock(nffile_w) <= 0 ) {
LogError("Failed to write output buffer to disk: '%s'" , strerror(errno));
}
}
if ( xstat ) {
if ( WriteExtraBlock(nffile_w, xstat->block_header ) <= 0 ) {
LogError("Failed to write xstat buffer to disk: '%s'" , strerror(errno));
}
}
/* Stat info */
if ( write_file ) {
/* Copy stat info and close file */
memcpy((void *)nffile_w->stat_record, (void *)&stat_record, sizeof(stat_record_t));
CloseUpdateFile(nffile_w, nffile_r->file_header->ident );
nffile_w = DisposeFile(nffile_w);
} // else stdout
}
PackExtensionMapList(extension_map_list);
DisposeFile(nffile_r);
return stat_record;
} // End of process_data
int main( int argc, char **argv ) {
int i, c;
master_record_t record;
nffile_t *nffile;
when = ISO2UNIX(strdup("200407111030"));
while ((c = getopt(argc, argv, "h")) != EOF) {
switch(c) {
case 'h':
break;
default:
fprintf(stderr, "ERROR: Unsupported option: '%c'\n", c);
exit(255);
}
}
extension_info.map = (extension_map_t *)malloc(sizeof(extension_map_t) + 32 * sizeof(uint16_t));
if ( !extension_info.map ) {
fprintf(stderr, "malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror (errno));
exit(255);
}
extension_info.map->type = ExtensionMapType;
extension_info.map->map_id = 0;
i = 0;
extension_info.map->ex_id[i++] = EX_IO_SNMP_2;
extension_info.map->ex_id[i++] = EX_AS_2;
extension_info.map->ex_id[i++] = EX_MULIPLE;
extension_info.map->ex_id[i++] = EX_NEXT_HOP_v4;
extension_info.map->ex_id[i++] = EX_NEXT_HOP_BGP_v4;
extension_info.map->ex_id[i++] = EX_VLAN;
extension_info.map->ex_id[i++] = EX_OUT_PKG_4;
extension_info.map->ex_id[i++] = EX_OUT_BYTES_4;
extension_info.map->ex_id[i++] = EX_AGGR_FLOWS_4;
extension_info.map->ex_id[i++] = EX_MAC_1;
extension_info.map->ex_id[i++] = EX_MAC_2;
extension_info.map->ex_id[i++] = EX_MPLS;
extension_info.map->ex_id[i++] = EX_ROUTER_IP_v4;
extension_info.map->ex_id[i++] = EX_ROUTER_ID;
extension_info.map->ex_id[i++] = EX_BGPADJ;
extension_info.map->ex_id[i] = 0;
extension_info.map->size = sizeof(extension_map_t) + i * sizeof(uint16_t);
// align 32bits
if (( extension_info.map->size & 0x3 ) != 0 ) {
extension_info.map->size += 4 - ( extension_info.map->size & 0x3 );
}
extension_info.map->extension_size = 0;
i=0;
while (extension_info.map->ex_id[i]) {
int id = extension_info.map->ex_id[i];
extension_info.map->extension_size += extension_descriptor[id].size;
i++;
}
memset((void *)&record, 0, sizeof(record));
nffile = OpenNewFile("-", NULL, 0, 0, NULL);
if ( !nffile ) {
exit(255);
}
AppendToBuffer(nffile, (void *)extension_info.map, extension_info.map->size);
record.map_ref = extension_info.map;
record.type = CommonRecordType;
record.flags = 0;
record.exporter_sysid = 1;
record.tcp_flags = 1;
record.tos = 2;
record.fwd_status = 0;
record.srcport = 1024;
record.dstport = 25;
record.prot = IPPROTO_TCP;
record.input = 12;
record.output = 14;
record.srcas = 775;
record.dstas = 8404;
SetIPaddress(&record, PF_INET, "172.16.1.66", "192.168.170.100");
SetNextIPaddress(&record, PF_INET, "172.72.1.2");
SetBGPNextIPaddress(&record, PF_INET, "172.73.2.3");
SetRouterIPaddress(&record, PF_INET, "127.0.0.1");
record.engine_type = 5;
record.engine_id = 6;
record.dPkts = 202;
record.dOctets = 303;
record.dst_tos = 128;
record.dir = 1;
record.src_mask = 16;
record.dst_mask = 24;
record.src_vlan = 82;
record.dst_vlan = 93;
record.out_pkts = 212;
record.out_bytes = 3234;
record.aggr_flows = 3;
record.in_src_mac = 0x0234567890aaLL;
record.out_dst_mac = 0xffeeddccbbaaLL;
record.out_src_mac = 0xaa3456789002LL;
record.in_dst_mac = 0xaaeeddccbbffLL;
record.mpls_label[0] = 1010 << 4;
record.mpls_label[1] = 2020 << 4;
record.mpls_label[2] = 3030 << 4;
record.mpls_label[3] = 4040 << 4;
record.mpls_label[4] = 5050 << 4;
record.mpls_label[5] = 6060 << 4;
record.mpls_label[6] = 7070 << 4;
record.mpls_label[7] = 8080 << 4;
record.mpls_label[8] = 9090 << 4;
record.mpls_label[9] = (100100 << 4) + 1;
record.client_nw_delay_usec = 2;
record.server_nw_delay_usec = 22;
record.appl_latency_usec = 222;
record.bgpNextAdjacentAS = 45804;
record.bgpPrevAdjacentAS = 32775;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.2.66", "192.168.170.101");
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
record.dPkts = 101;
record.dOctets = 102;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.3.66", "192.168.170.102");
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.4.66", "192.168.170.103");
record.srcport = 2024;
record.prot = IPPROTO_UDP;
record.tcp_flags = 1;
record.tos = 1;
record.dPkts = 1001;
record.dOctets = 1002;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.5.66", "192.168.170.104");
record.srcport = 3024;
record.prot = 51;
record.tcp_flags = 2;
record.tos = 2;
record.dPkts = 10001;
record.dOctets = 10002;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.6.66", "192.168.170.105");
record.srcport = 4024;
record.prot = IPPROTO_TCP;
record.tcp_flags = 4;
record.tos = 3;
record.dPkts = 100001;
record.dOctets = 100002;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.7.66", "192.168.170.106");
record.srcport = 5024;
record.tcp_flags = 8;
record.tos = 4;
record.dPkts = 1000001;
record.dOctets = 1000002;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.8.66", "192.168.170.107");
record.tcp_flags = 1;
record.tos = 4;
record.dPkts = 10000001;
record.dOctets = 1001;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.9.66", "192.168.170.108");
record.srcport = 6024;
record.tcp_flags = 16;
record.tos = 5;
record.dPkts = 500;
record.dOctets = 10000001;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.10.66", "192.168.170.109");
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.11.66", "192.168.170.110");
record.srcport = 7024;
record.tcp_flags = 32;
record.tos = 255;
record.dPkts = 5000;
record.dOctets = 100000001;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.12.66", "192.168.170.111");
record.srcport = 8024;
record.tcp_flags = 63;
record.tos = 0;
record.dOctets = 1000000001;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.13.66", "192.168.170.112");
record.srcport = 0;
record.dstport = 8;
record.prot = 1;
record.tcp_flags = 0;
record.tos = 0;
record.dPkts = 50002;
record.dOctets = 50000;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.160.160.166", "172.160.160.180");
record.srcport = 10024;
record.dstport = 25000;
record.prot = IPPROTO_TCP;
record.dPkts = 500001;
record.dOctets = 500000;
fprintf(stderr, "IPv4 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET6, "fe80::2110:abcd:1234:0", "fe80::2110:abcd:1235:4321");
// SetNextIPaddress(&record, PF_INET6, "2003:234:aabb::211:24ff:fe80:d01e");
// SetBGPNextIPaddress(&record, PF_INET6, "2004:234:aabb::211:24ff:fe80:d01e");
record.srcport = 1024;
record.dstport = 25;
record.tcp_flags = 27;
record.dPkts = 10;
record.dOctets = 15100;
fprintf(stderr, "IPv6 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET6, "2001:234:aabb::211:24ff:fe80:d01e", "2001:620::8:203:baff:fe52:38e5");
record.srcport = 10240;
record.dstport = 52345;
record.dPkts = 10100;
record.dOctets = 15000000;
fprintf(stderr, "IPv6 32bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
record.dPkts = 10100000;
record.dOctets = 0x100000000LL;
fprintf(stderr, "IPv6 32bit packets 64bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
record.dPkts = 0x100000000LL;
record.dOctets = 15000000;
fprintf(stderr, "IPv6 64bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
record.dOctets = 0x200000000LL;
fprintf(stderr, "IPv6 64bit packets 64bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.14.18", "192.168.170.113");
// SetNextIPaddress(&record, PF_INET, "172.72.1.2");
// SetBGPNextIPaddress(&record, PF_INET, "172.73.2.3");
record.srcport = 10240;
record.dstport = 52345;
record.dPkts = 10100000;
record.dOctets = 0x100000000LL;
fprintf(stderr, "IPv4 32bit packets 64bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.15.18", "192.168.170.114");
record.dPkts = 0x100000000LL;
record.dOctets = 15000000;
fprintf(stderr, "IPv4 64bit packets 32bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
SetIPaddress(&record, PF_INET, "172.16.16.18", "192.168.170.115");
record.dOctets = 0x200000000LL;
fprintf(stderr, "IPv4 64bit packets 64bit bytes\n");
UpdateRecord(&record);
PackRecord(&record, nffile);
extension_info.map->ex_id[0] = EX_IO_SNMP_4;
extension_info.map->extension_size = 0;
i=0;
while (extension_info.map->ex_id[i]) {
int id = extension_info.map->ex_id[i];
extension_info.map->extension_size += extension_descriptor[id].size;
i++;
}
memcpy(nffile->buff_ptr, (void *)extension_info.map, extension_info.map->size);
nffile->buff_ptr += extension_info.map->size;
nffile->block_header->NumRecords++;
nffile->block_header->size += extension_info.map->size;
UpdateRecord(&record);
fprintf(stderr, "4 bytes interfaces, 2 bytes AS numbers %d %d\n", record.fwd_status, nffile->block_header->NumRecords);
PackRecord(&record, nffile);
extension_info.map->ex_id[0] = EX_IO_SNMP_2;
extension_info.map->ex_id[1] = EX_AS_4;
extension_info.map->extension_size = 0;
i=0;
while (extension_info.map->ex_id[i]) {
int id = extension_info.map->ex_id[i];
extension_info.map->extension_size += extension_descriptor[id].size;
i++;
}
memcpy(nffile->buff_ptr, (void *)extension_info.map, extension_info.map->size);
nffile->buff_ptr += extension_info.map->size;
nffile->block_header->NumRecords++;
nffile->block_header->size += extension_info.map->size;
UpdateRecord(&record);
fprintf(stderr, "2 bytes interfaces, 4 bytes AS numbers %d %d\n", record.fwd_status, nffile->block_header->NumRecords);
PackRecord(&record, nffile);
extension_info.map->ex_id[0] = EX_IO_SNMP_4;
extension_info.map->extension_size = 0;
i=0;
while (extension_info.map->ex_id[i]) {
int id = extension_info.map->ex_id[i];
extension_info.map->extension_size += extension_descriptor[id].size;
i++;
}
memcpy(nffile->buff_ptr, (void *)extension_info.map, extension_info.map->size);
nffile->buff_ptr += extension_info.map->size;
nffile->block_header->NumRecords++;
nffile->block_header->size += extension_info.map->size;
UpdateRecord(&record);
fprintf(stderr, "4 bytes interfaces, 4 bytes AS numbers %d %d\n", record.fwd_status, nffile->block_header->NumRecords);
PackRecord(&record, nffile);
if ( nffile->block_header->NumRecords ) {
if ( WriteBlock(nffile) <= 0 ) {
fprintf(stderr, "Failed to write output buffer to disk: '%s'" , strerror(errno));
}
}
return 0;
}
/**
*
*
* @update gess 5/21/98
* @param
* @return
*/
nsresult nsScanner::Append(const char* aBuffer, PRUint32 aLen,
nsIRequest *aRequest)
{
nsresult res=NS_OK;
PRUnichar *unichars, *start;
if(mUnicodeDecoder) {
PRInt32 unicharBufLen = 0;
mUnicodeDecoder->GetMaxLength(aBuffer, aLen, &unicharBufLen);
nsScannerString::Buffer* buffer = nsScannerString::AllocBuffer(unicharBufLen + 1);
NS_ENSURE_TRUE(buffer,NS_ERROR_OUT_OF_MEMORY);
start = unichars = buffer->DataStart();
PRInt32 totalChars = 0;
PRInt32 unicharLength = unicharBufLen;
do {
PRInt32 srcLength = aLen;
res = mUnicodeDecoder->Convert(aBuffer, &srcLength, unichars, &unicharLength);
totalChars += unicharLength;
// Continuation of failure case
if(NS_FAILED(res)) {
// if we failed, we consume one byte, replace it with U+FFFD
// and try the conversion again.
// This is only needed because some decoders don't follow the
// nsIUnicodeDecoder contract: they return a failure when *aDestLength
// is 0 rather than the correct NS_OK_UDEC_MOREOUTPUT. See bug 244177
if ((unichars + unicharLength) >= buffer->DataEnd()) {
NS_ERROR("Unexpected end of destination buffer");
break;
}
unichars[unicharLength++] = (PRUnichar)0xFFFD;
unichars = unichars + unicharLength;
unicharLength = unicharBufLen - (++totalChars);
mUnicodeDecoder->Reset();
if(((PRUint32) (srcLength + 1)) > aLen) {
srcLength = aLen;
}
else {
++srcLength;
}
aBuffer += srcLength;
aLen -= srcLength;
}
} while (NS_FAILED(res) && (aLen > 0));
buffer->SetDataLength(totalChars);
// Don't propagate return code of unicode decoder
// since it doesn't reflect on our success or failure
// - Ref. bug 87110
res = NS_OK;
if (!AppendToBuffer(buffer, aRequest))
res = NS_ERROR_OUT_OF_MEMORY;
else
mTotalRead += totalChars;
}
else {
AppendASCIItoBuffer(aBuffer, aLen, aRequest);
mTotalRead+=aLen;
}
return res;
}
/**
* Append data to our underlying input buffer as
* if it were read from an input stream.
*
* @update gess4/3/98
* @return error code
*/
nsresult nsScanner::Append(const nsAString& aBuffer) {
if (!AppendToBuffer(aBuffer))
return NS_ERROR_OUT_OF_MEMORY;
mTotalRead += aBuffer.Length();
return NS_OK;
}
int flows2nfdump(struct ftio *ftio, char *wfile, int compress, extension_info_t *extension_info, int extended, uint32_t limitflows) {
// required flow tools variables
struct fttime ftt;
struct fts3rec_offsets fo;
struct ftver ftv;
char *rec;
// nfdump variables
nffile_t *nffile;
master_record_t record;
char *s;
uint32_t cnt;
s = "flow-tools";
nffile = OpenNewFile( wfile , NULL, compress, 0, s);
if ( !nffile ) {
fprintf(stderr, "%s\n", s);
return 1;
}
AppendToBuffer(nffile, (void *)extension_info->map, extension_info->map->size);
ftio_get_ver(ftio, &ftv);
fts3rec_compute_offsets(&fo, &ftv);
memset((void *)&record, 0, sizeof(record));
record.map_ref = extension_info->map;
record.type = CommonRecordType;
record.exporter_sysid = 0;
// only v4 addresses
ClearFlag(record.flags, FLAG_IPV6_ADDR);
cnt = 0;
while ((rec = ftio_read(ftio))) {
uint32_t when, unix_secs, unix_nsecs, sysUpTime;
int i, id;
unix_secs = *((uint32_t*)(rec+fo.unix_secs));
unix_nsecs = *((uint32_t*)(rec+fo.unix_nsecs));
sysUpTime = *((uint32_t*)(rec+fo.sysUpTime));
when = *((uint32_t*)(rec+fo.First));
ftt = ftltime(sysUpTime, unix_secs, unix_nsecs, when);
record.first = ftt.secs;
record.msec_first = ftt.msecs;
when = *((uint32_t*)(rec+fo.Last));
ftt = ftltime(sysUpTime, unix_secs, unix_nsecs, when);
record.last = ftt.secs;
record.msec_last = ftt.msecs;
record.v4.srcaddr = *((uint32_t*)(rec+fo.srcaddr));
record.v4.dstaddr = *((uint32_t*)(rec+fo.dstaddr));
record.srcport = *((uint16_t*)(rec+fo.srcport));
record.dstport = *((uint16_t*)(rec+fo.dstport));
record.prot = *((uint8_t*)(rec+fo.prot));
record.tcp_flags = *((uint8_t*)(rec+fo.tcp_flags));
record.tos = *((uint8_t*)(rec+fo.tos));
record.dOctets = *((uint32_t*)(rec+fo.dOctets));
record.dPkts = *((uint32_t*)(rec+fo.dPkts));
i = 0;
while ( (id = extension_info->map->ex_id[i]) != 0 ) {
switch (id) {
case EX_IO_SNMP_2:
record.input = *((uint16_t*)(rec+fo.input));
record.output = *((uint16_t*)(rec+fo.output));
break;
case EX_AS_2:
record.srcas = *((uint16_t*)(rec+fo.src_as));
record.dstas = *((uint16_t*)(rec+fo.dst_as));
break;
case EX_MULIPLE:
record.src_mask = *((uint8_t*)(rec+fo.src_mask));
record.dst_mask = *((uint8_t*)(rec+fo.dst_mask));
record.dir = 0;
record.dst_tos = 0;
break;
case EX_ROUTER_IP_v4:
record.ip_nexthop.v4 = *((uint32_t*)(rec+fo.peer_nexthop));
break;
case EX_NEXT_HOP_v4:
record.ip_router.v4 = *((uint32_t*)(rec+fo.router_sc));
break;
case EX_ROUTER_ID:
record.engine_type = *((uint8_t*)(rec+fo.engine_type));
record.engine_id = *((uint8_t*)(rec+fo.engine_id));
break;
// default: Other extensions can not be sent with v5
}
i++;
}
PackRecord(&record, nffile);
if ( extended ) {
char *string;
format_file_block_record(&record, &string, 0);
fprintf(stderr, "%s\n", string);
}
cnt++;
if ( cnt == limitflows )
break;
} /* while */
// write the last records in buffer
if ( nffile->block_header->NumRecords ) {
if ( WriteBlock(nffile) <= 0 ) {
fprintf(stderr, "Failed to write output buffer: '%s'" , strerror(errno));
}
}
free((void *)extension_info->map);
free((void *)extension_info);
DisposeFile(nffile);
return 0;
} // End of flows2nfdump
/**
Adds a variable to the variable serialization instance
@param[in] Handle - Handle for a variable serialization instance
@param[in] VariableName - Refer to RuntimeServices GetVariable
@param[in] VendorGuid - Refer to RuntimeServices GetVariable
@param[in] Attributes - Refer to RuntimeServices GetVariable
@param[in] DataSize - Refer to RuntimeServices GetVariable
@param[in] Data - Refer to RuntimeServices GetVariable
@retval RETURN_SUCCESS - All variables were set successfully
@retval RETURN_OUT_OF_RESOURCES - There we not enough resources to
add the variable
@retval RETURN_INVALID_PARAMETER - Handle was not a valid
variable serialization instance or
VariableName, VariableGuid or Data are NULL.
**/
RETURN_STATUS
EFIAPI
SerializeVariablesAddVariable (
IN EFI_HANDLE Handle,
IN CHAR16 *VariableName,
IN EFI_GUID *VendorGuid,
IN UINT32 Attributes,
IN UINTN DataSize,
IN VOID *Data
)
{
RETURN_STATUS Status;
SV_INSTANCE *Instance;
UINT32 SerializedNameSize;
UINT32 SerializedDataSize;
UINTN SerializedSize;
Instance = SV_FROM_HANDLE (Handle);
if ((Instance->Signature != SV_SIGNATURE) ||
(VariableName == NULL) || (VendorGuid == NULL) || (Data == NULL)) {
}
SerializedNameSize = (UINT32) StrSize (VariableName);
SerializedSize =
sizeof (SerializedNameSize) +
SerializedNameSize +
sizeof (*VendorGuid) +
sizeof (Attributes) +
sizeof (SerializedDataSize) +
DataSize;
Status = EnsureExtraBufferSpace (
Instance,
SerializedSize
);
if (RETURN_ERROR (Status)) {
return Status;
}
//
// Add name size (UINT32)
//
AppendToBuffer (Instance, (VOID*) &SerializedNameSize, sizeof (SerializedNameSize));
//
// Add variable unicode name string
//
AppendToBuffer (Instance, (VOID*) VariableName, SerializedNameSize);
//
// Add variable GUID
//
AppendToBuffer (Instance, (VOID*) VendorGuid, sizeof (*VendorGuid));
//
// Add variable attributes
//
AppendToBuffer (Instance, (VOID*) &Attributes, sizeof (Attributes));
//
// Add variable data size (UINT32)
//
SerializedDataSize = (UINT32) DataSize;
AppendToBuffer (Instance, (VOID*) &SerializedDataSize, sizeof (SerializedDataSize));
//
// Add variable data
//
AppendToBuffer (Instance, Data, DataSize);
return RETURN_SUCCESS;
}
static void process_data(profile_channel_info_t *channels, unsigned int num_channels, time_t tslot, int do_xstat) {
common_record_t *flow_record;
nffile_t *nffile;
FilterEngine_data_t *engine;
int i, j, done, ret ;
#ifdef COMPAT15
int v1_map_done = 0;
#endif
nffile = GetNextFile(NULL, 0, 0);
if ( !nffile ) {
LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
return;
}
if ( nffile == EMPTY_LIST ) {
LogError("Empty file list. No files to process\n");
return;
}
// store infos away for later use
// although multiple files may be processed, it is assumed that all
// have the same settings
is_anonymized = IP_ANONYMIZED(nffile);
strncpy(Ident, nffile->file_header->ident, IDENTLEN);
Ident[IDENTLEN-1] = '\0';
done = 0;
while ( !done ) {
// get next data block from file
ret = ReadBlock(nffile);
switch (ret) {
case NF_CORRUPT:
case NF_ERROR:
if ( ret == NF_CORRUPT )
LogError("Skip corrupt data file '%s'\n",GetCurrentFilename());
else
LogError("Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) );
// fall through - get next file in chain
case NF_EOF: {
nffile_t *next = GetNextFile(nffile, 0, 0);
if ( next == EMPTY_LIST ) {
done = 1;
}
if ( next == NULL ) {
done = 1;
LogError("Unexpected end of file list\n");
}
continue;
} break; // not really needed
}
#ifdef COMPAT15
if ( nffile->block_header->id == DATA_BLOCK_TYPE_1 ) {
common_record_v1_t *v1_record = (common_record_v1_t *)nffile->buff_ptr;
// create an extension map for v1 blocks
if ( v1_map_done == 0 ) {
extension_map_t *map = malloc(sizeof(extension_map_t) + 2 * sizeof(uint16_t) );
if ( ! map ) {
LogError("malloc() allocation error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
exit(255);
}
map->type = ExtensionMapType;
map->size = sizeof(extension_map_t) + 2 * sizeof(uint16_t);
if (( map->size & 0x3 ) != 0 ) {
map->size += 4 - ( map->size & 0x3 );
}
map->map_id = INIT_ID;
map->ex_id[0] = EX_IO_SNMP_2;
map->ex_id[1] = EX_AS_2;
map->ex_id[2] = 0;
map->extension_size = 0;
map->extension_size += extension_descriptor[EX_IO_SNMP_2].size;
map->extension_size += extension_descriptor[EX_AS_2].size;
if ( Insert_Extension_Map(extension_map_list, map) ) {
int j;
for ( j=0; j < num_channels; j++ ) {
if ( channels[j].nffile != NULL) {
// flush new map
AppendToBuffer(channels[j].nffile, (void *)map, map->size);
}
}
} // else map already known and flushed
v1_map_done = 1;
}
// convert the records to v2
for ( i=0; i < nffile->block_header->NumRecords; i++ ) {
common_record_t *v2_record = (common_record_t *)v1_record;
Convert_v1_to_v2((void *)v1_record);
// now we have a v2 record -> use size of v2_record->size
v1_record = (common_record_v1_t *)((pointer_addr_t)v1_record + v2_record->size);
}
nffile->block_header->id = DATA_BLOCK_TYPE_2;
}
#endif
if ( nffile->block_header->id == Large_BLOCK_Type ) {
// skip
continue;
}
if ( nffile->block_header->id != DATA_BLOCK_TYPE_2 ) {
LogError("Can't process block type %u. Skip block.\n", nffile->block_header->id);
continue;
}
flow_record = nffile->buff_ptr;
for ( i=0; i < nffile->block_header->NumRecords; i++ ) {
switch ( flow_record->type ) {
case CommonRecordType: {
generic_exporter_t *exp_info = exporter_list[flow_record->exporter_sysid];
uint32_t map_id = flow_record->ext_map;
master_record_t *master_record;
if ( extension_map_list->slot[map_id] == NULL ) {
LogError("Corrupt data file. Missing extension map %u. Skip record.\n", flow_record->ext_map);
flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
continue;
}
master_record = &(extension_map_list->slot[map_id]->master_record);
ExpandRecord_v2( flow_record, extension_map_list->slot[flow_record->ext_map],
exp_info ? &(exp_info->info) : NULL, master_record);
for ( j=0; j < num_channels; j++ ) {
int match;
// apply profile filter
(channels[j].engine)->nfrecord = (uint64_t *)master_record;
engine = channels[j].engine;
match = (*engine->FilterEngine)(engine);
// if profile filter failed -> next profile
if ( !match )
continue;
// filter was successful -> continue record processing
// update statistics
UpdateStat(&channels[j].stat_record, master_record);
if ( channels[j].nffile )
UpdateStat(channels[j].nffile->stat_record, master_record);
if ( channels[j].xstat )
UpdateXStat(channels[j].xstat, master_record);
// do we need to write data to new file - shadow profiles do not have files.
// check if we need to flush the output buffer
if ( channels[j].nffile != NULL ) {
// write record to output buffer
AppendToBuffer(channels[j].nffile, (void *)flow_record, flow_record->size);
}
} // End of for all channels
} break;
case ExtensionMapType: {
extension_map_t *map = (extension_map_t *)flow_record;
if ( Insert_Extension_Map(extension_map_list, map) ) {
int j;
for ( j=0; j < num_channels; j++ ) {
if ( channels[j].nffile != NULL ) {
// flush new map
AppendToBuffer(channels[j].nffile, (void *)map, map->size);
}
}
} // else map already known and flushed
} break;
case ExporterInfoRecordType: {
int ret = AddExporterInfo((exporter_info_record_t *)flow_record);
if ( ret != 0 ) {
int j;
for ( j=0; j < num_channels; j++ ) {
if ( channels[j].nffile != NULL && ret == 1) {
// flush new exporter
AppendToBuffer(channels[j].nffile, (void *)flow_record, flow_record->size);
}
}
} else {
LogError("Failed to add Exporter Record\n");
}
} break;
case SamplerInfoRecordype: {
int ret = AddSamplerInfo((sampler_info_record_t *)flow_record);
if ( ret != 0 ) {
int j;
for ( j=0; j < num_channels; j++ ) {
if ( channels[j].nffile != NULL && ret == 1 ) {
// flush new map
AppendToBuffer(channels[j].nffile, (void *)flow_record, flow_record->size);
}
}
} else {
LogError("Failed to add Sampler Record\n");
}
} break;
case ExporterRecordType:
case SamplerRecordype:
case ExporterStatRecordType:
// Silently skip exporter records
break;
default: {
LogError("Skip unknown record type %i\n", flow_record->type);
}
}
// Advance pointer by number of bytes for netflow record
flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
} // End of for all umRecords
} // End of while !done
// do we need to write data to new file - shadow profiles do not have files.
for ( j=0; j < num_channels; j++ ) {
if ( channels[j].nffile != NULL ) {
// flush output buffer
if ( channels[j].nffile->block_header->NumRecords ) {
if ( WriteBlock(channels[j].nffile) <= 0 ) {
LogError("Failed to write output buffer to disk: '%s'" , strerror(errno));
}
}
}
}
CloseFile(nffile);
DisposeFile(nffile);
} // End of process_data
Byte const *ReadBuffer::Read(int min, /*out */ int &length)
{
// ::printf("ReadRange(%d)\n", min);
// min must be > 0
if (!buffer_.size() && readRange_ && readRangeLength_ >= min)
{
// most common (fast) case: buffer is empty, readRange_ big enough.
// ::printf(" from range\n");
// buffer is empty, but readRange_ is set and is long enough.
length = readRangeLength_;
return readRange_;
}
int bufferFullness = buffer_.size() - bufferPos_;
// Try to avoid using the buffer:
if (bufferFullness &&
readRange_ &&
bufferFullness + readRangeLength_ <= readRangeOrigLength_)
{
// ::printf(" Avoid buffer\n");
readRangeLength_ += bufferFullness;
readRange_ -= bufferFullness;
buffer_.resize(0);
bufferPos_ = 0;
bufferFullness = 0;
}
if (bufferFullness >= min)
{
// ::printf(" from buffer\n");
// serve request from buffer.
length = bufferFullness;
return &buffer_[bufferPos_];
}
else if (bufferFullness)
{
// buffer is not empty, but has not enoug data.
if (bufferFullness + readRangeLength_ < min)
{
// not enough total data.
// ::printf(" not enough total\n");
return nullptr;
}
int appSize = min - bufferFullness + 16;
if (appSize > readRangeLength_)
{
appSize = readRangeLength_;
}
AppendToBuffer(appSize);
// ::printf(" from buffer (app)\n");
// now buffer has at least min bytes of data
length = buffer_.size() - bufferPos_;
return &buffer_[bufferPos_];
}
else if (readRange_ && readRangeLength_ >= min)
{
// ::printf(" from range\n");
// buffer is empty, but readRange_ is set and is long enough.
length = readRangeLength_;
return readRange_;
}
// ::printf(" fail\n");
return nullptr;
}
static void ExportExtensionMaps( int aggregate, int bidir, nffile_t *nffile, extension_map_list_t *extension_map_list ) {
int map_id, opt_extensions, num_extensions, new_map_size, opt_align;
extension_map_t *new_map;
// no extension maps to export - nothing to do
if ( extension_map_list->map_list == NULL )
return;
new_map = NULL;
for ( map_id = 0; map_id <= extension_map_list->max_used; map_id++ ) {
extension_map_t *SourceMap = extension_map_list->slot[map_id]->map;
int i, has_aggr_flows, has_out_bytes, has_out_packets, has_nat;
// skip maps, never referenced
#ifdef DEVEL
printf("Process map id: %i\n", map_id);
printf("Ref count: %i\n", extension_map_list->slot[map_id]->ref_count);
#endif
if ( extension_map_list->slot[map_id]->ref_count == 0 ) {
#ifdef DEVEL
printf("Ref count = 0 => Skip map\n");
#endif
continue;
}
// parse Source map if it contains all required fields:
// for aggregation EX_AGGR_FLOWS_4 or _8 is required
// for bidir flows EX_OUT_PKG_4 or _8 and EX_OUT_BYTES_4 or_8 are required
has_aggr_flows = 0;
has_out_bytes = 0;
has_out_packets = 0;
// parse map for older NEL nat extension
has_nat = 0;
num_extensions = 0;
i = 0;
while ( SourceMap->ex_id[i] ) {
switch (SourceMap->ex_id[i]) {
case EX_AGGR_FLOWS_4:
case EX_AGGR_FLOWS_8:
has_aggr_flows = 1;
break;
case EX_OUT_BYTES_4:
case EX_OUT_BYTES_8:
has_out_bytes = 1;
break;
case EX_OUT_PKG_4:
case EX_OUT_PKG_8:
has_out_packets = 1;
break;
case EX_NEL_GLOBAL_IP_v4:
// Map old nat extension to common NSEL extension
SourceMap->ex_id[i] = EX_NSEL_XLATE_IP_v4;
has_nat = 1;
// default: nothing to do
}
i++;
num_extensions++;
}
#ifdef DEVEL
printf("map: num_extensions: %i, has_aggr_flows: %i, has_out_bytes: %i, has_out_packets: %i, has_nat: %i\n",
num_extensions, has_aggr_flows, has_out_bytes, has_out_packets, has_nat);
#endif
// count missing extensions
opt_extensions = 0;
if ( aggregate && !has_aggr_flows )
opt_extensions++;
if ( bidir && !has_out_bytes )
opt_extensions++;
if ( bidir && !has_out_packets )
opt_extensions++;
opt_extensions += has_nat;
// calculate new map size
new_map_size = sizeof(extension_map_t) + ( num_extensions + opt_extensions) * sizeof(uint16_t);
#ifdef DEVEL
printf("opt_extensions: %i, new_map_size: %i\n", opt_extensions,new_map_size );
PrintExtensionMap(SourceMap);
#endif
if ( opt_extensions ) {
// align 32bits
if (( new_map_size & 0x3 ) != 0 ) {
new_map_size += 4 - ( new_map_size & 0x3 );
opt_align = 1;
} else {
opt_align = 0;
}
} else {
// no missing elements in extension map - we can used the original one
// and we are done
#ifdef DEVEL
printf("New map identical => use this map:\n");
PrintExtensionMap(SourceMap);
#endif
// Flush the map to disk
AppendToBuffer(nffile, (void *)SourceMap, SourceMap->size);
continue;
}
#ifdef DEVEL
printf("Create new map:\n");
#endif
// new map is different - create the new map
new_map = (extension_map_t *)malloc((ssize_t)new_map_size);
if ( !new_map ) {
LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
exit(255);
}
// Panic check - should never happen, but we are going to copy memory
if ( new_map_size < SourceMap->size ) {
LogError("PANIC! new_map_size(%i) < SourceMap->size(%i) in %s line %d\n",
new_map_size, SourceMap->size, __FILE__, __LINE__);
exit(255);
}
// copy existing map
memcpy((void *)new_map, (void *)SourceMap, SourceMap->size);
new_map->size = new_map_size;
// add the missing extensions to the output map
// skip to end of current map
while ( new_map->ex_id[i] )
i++;
if ( has_nat ) {
new_map->ex_id[i++] = EX_NSEL_XLATE_PORTS;
new_map->extension_size += extension_descriptor[EX_NSEL_XLATE_PORTS].size;
}
// add missing map elements
if ( aggregate && !has_aggr_flows ) {
new_map->ex_id[i++] = EX_AGGR_FLOWS_4;
new_map->extension_size += extension_descriptor[EX_AGGR_FLOWS_4].size;
}
if ( bidir && !has_out_bytes ) {
new_map->ex_id[i++] = EX_OUT_BYTES_8;
new_map->extension_size += extension_descriptor[EX_OUT_BYTES_8].size;
}
if ( bidir && !has_out_packets ) {
new_map->ex_id[i++] = EX_OUT_PKG_8;
new_map->extension_size += extension_descriptor[EX_OUT_PKG_8].size;
}
// end of map tag
new_map->ex_id[i++] = 0;
if ( opt_align )
new_map->ex_id[i] = 0;
#ifdef DEVEL
PrintExtensionMap(new_map);
#endif
free(extension_map_list->slot[map_id]->map);
extension_map_list->slot[map_id]->map = new_map;
// Flush the map to disk
AppendToBuffer(nffile, (void *)new_map, new_map->size);
}
} // End of ExportExtensionMaps
void CMapGenerator::GenerateSMF(CVirtualArchive* archive)
{
CVirtualFile* fileSMF = archive->AddFile("maps/generated.smf");
//FileBuffer b;
SMFHeader smfHeader;
MapTileHeader smfTile;
MapFeatureHeader smfFeature;
//--- Make SMFHeader ---
strcpy(smfHeader.magic, "spring map file");
smfHeader.version = 1;
smfHeader.mapid = 0x524d4746 ^ (int)setup->mapSeed;
//Set settings
smfHeader.mapx = GetGridSize().x;
smfHeader.mapy = GetGridSize().y;
smfHeader.squareSize = 8;
smfHeader.texelPerSquare = 8;
smfHeader.tilesize = 32;
smfHeader.minHeight = -100;
smfHeader.maxHeight = 0x1000;
const int numSmallTiles = 1; //2087; //32 * 32 * (mapSize.x / 2) * (mapSize.y / 2);
const char smtFileName[] = "generated.smt";
//--- Extra headers ---
ExtraHeader vegHeader;
vegHeader.type = MEH_Vegetation;
vegHeader.size = sizeof(int);
smfHeader.numExtraHeaders = 1;
//Make buffers for each map
int heightmapDimensions = (smfHeader.mapx + 1) * (smfHeader.mapy + 1);
int typemapDimensions = (smfHeader.mapx / 2) * (smfHeader.mapy / 2);
int metalmapDimensions = (smfHeader.mapx / 2) * (smfHeader.mapy / 2);
int tilemapDimensions = (smfHeader.mapx * smfHeader.mapy) / 16;
int vegmapDimensions = (smfHeader.mapx / 4) * (smfHeader.mapy / 4);
int heightmapSize = heightmapDimensions * sizeof(short);
int typemapSize = typemapDimensions * sizeof(unsigned char);
int metalmapSize = metalmapDimensions * sizeof(unsigned char);
int tilemapSize = tilemapDimensions * sizeof(int);
int tilemapTotalSize = sizeof(MapTileHeader) + sizeof(numSmallTiles) + sizeof(smtFileName) + tilemapSize;
int vegmapSize = vegmapDimensions * sizeof(unsigned char);
short* heightmapPtr = new short[heightmapDimensions];
unsigned char* typemapPtr = new unsigned char[typemapDimensions];
unsigned char* metalmapPtr = new unsigned char[metalmapDimensions];
int* tilemapPtr = new int[tilemapDimensions];
unsigned char* vegmapPtr = new unsigned char[vegmapDimensions];
//--- Set offsets, increment each member with the previous one ---
int vegmapOffset = sizeof(smfHeader) + sizeof(vegHeader) + sizeof(int);
smfHeader.heightmapPtr = vegmapOffset + vegmapSize;
smfHeader.typeMapPtr = smfHeader.heightmapPtr + heightmapSize;
smfHeader.tilesPtr = smfHeader.typeMapPtr + typemapSize;
smfHeader.minimapPtr = 0; //smfHeader.tilesPtr + sizeof(MapTileHeader);
smfHeader.metalmapPtr = smfHeader.tilesPtr + tilemapTotalSize; //smfHeader.minimapPtr + minimapSize;
smfHeader.featurePtr = smfHeader.metalmapPtr + metalmapSize;
//--- Make MapTileHeader ---
smfTile.numTileFiles = 1;
smfTile.numTiles = numSmallTiles;
//--- Make MapFeatureHeader ---
smfFeature.numFeatures = 0;
smfFeature.numFeatureType = 0;
//--- Update Ptrs and write to buffer ---
memset(vegmapPtr, 0, vegmapSize);
float heightMin = smfHeader.minHeight;
float heightMax = smfHeader.maxHeight;
float heightMul = (float)0xFFFF / (smfHeader.maxHeight - smfHeader.minHeight);
for(int x = 0; x < heightmapDimensions; x++)
{
float h = heightMap[x];
if(h < heightMin) h = heightMin;
if(h > heightMax) h = heightMax;
h -= heightMin;
h *= heightMul;
heightmapPtr[x] = (short)h;
}
memset(typemapPtr, 0, typemapSize);
/*for(u32 x = 0; x < smfHeader.mapx; x++)
{
for(u32 y = 0; y < smfHeader.mapy; y++)
{
u32 index =
tilemapPtr[]
}
}*/
memset(tilemapPtr, 0, tilemapSize);
memset(metalmapPtr, 0, metalmapSize);
//--- Write to final buffer ---
//std::vector<boost::uint8_t>& smb = fileSMF->buffer;
AppendToBuffer(fileSMF, smfHeader);
AppendToBuffer(fileSMF, vegHeader);
AppendToBuffer(fileSMF, vegmapOffset);
AppendToBuffer(fileSMF, vegmapPtr, vegmapSize);
AppendToBuffer(fileSMF, heightmapPtr, heightmapSize);
AppendToBuffer(fileSMF, typemapPtr, typemapSize);
AppendToBuffer(fileSMF, smfTile);
AppendToBuffer(fileSMF, numSmallTiles);
AppendToBuffer(fileSMF, smtFileName, sizeof(smtFileName));
AppendToBuffer(fileSMF, tilemapPtr, tilemapSize);
AppendToBuffer(fileSMF, metalmapPtr, metalmapSize);
AppendToBuffer(fileSMF, smfFeature);
delete[] heightmapPtr;
delete[] typemapPtr;
delete[] metalmapPtr;
delete[] tilemapPtr;
delete[] vegmapPtr;
}
extension_map_t * lnf_lookup_map(lnf_file_t *lnf_file, bit_array_t *ext ) {
extension_map_t *map;
lnf_map_list_t *map_list;
int i = 0;
int is_set = 0;
int id = 0;
int map_id = 0;
// find whether the template already exist
map_id = 0;
map_list = lnf_file->lnf_map_list;
if (map_list == NULL) {
// first map
map_list = malloc(sizeof(lnf_map_list_t));
if (map_list == NULL) {
return NULL;
}
lnf_file->lnf_map_list = map_list;
} else {
if (bit_array_cmp(&(map_list->bit_array), ext) == 0) {
return map_list->map;
}
map_id++;
while (map_list->next != NULL ) {
if (bit_array_cmp(&(map_list->bit_array), ext) == 0) {
return map_list->map;
} else {
map_id++;
map_list = map_list->next;
}
}
map_list->next = malloc(sizeof(lnf_map_list_t));
if (map_list->next == NULL) {
return NULL;
}
map_list = map_list->next;
}
// allocate memory potentially for all extensions
map = malloc(sizeof(extension_map_t) + (lnf_file->max_num_extensions + 1) * sizeof(uint16_t));
if (map == NULL) {
return NULL;
}
map_list->map = map;
map_list->next = NULL;
bit_array_init(&map_list->bit_array, lnf_file->max_num_extensions + 1);
bit_array_copy(&map_list->bit_array, ext);
map->type = ExtensionMapType;
map->map_id = map_id;
// set extension map according the bits set in ext structure
id = 0;
i = 0;
while ( (is_set = bit_array_get(ext, id)) != -1 ) {
// fprintf(stderr, "i: %d, bit %d, val: %d\n", i, id, is_set);
if (is_set)
map->ex_id[i++] = id;
id++;
}
map->ex_id[i++] = 0;
// determine size and align 32bits
map->size = sizeof(extension_map_t) + ( i - 1 ) * sizeof(uint16_t);
if (( map->size & 0x3 ) != 0 ) {
map->size += (4 - ( map->size & 0x3 ));
}
map->extension_size = 0;
i=0;
while (map->ex_id[i]) {
int id = map->ex_id[i];
map->extension_size += extension_descriptor[id].size;
i++;
}
//Insert_Extension_Map(&instance->extension_map_list, map);
Insert_Extension_Map(lnf_file->extension_map_list, map);
AppendToBuffer(lnf_file->nffile, (void *)map, map->size);
return map;
}
// -------------------------------------------------------------------------------------------------
static void PerformScan
(
char* bufferPtr, ///< [OUT] On success or failure, a message is written to this buffer.
size_t bufferSz, ///< [IN] Size of the output buffer.
const char* requesterPtr ///< [IN] If not NULL, then any response text is SMSed to this target.
)
{
int bufferIdx = 0;
le_mrc_ScanInformationListRef_t scanInformationList = NULL;
fprintf(OutputFilePtr, "Scan was asked");
scanInformationList = le_mrc_PerformCellularNetworkScan(LE_MRC_BITMASK_RAT_ALL);
if (!scanInformationList)
{
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "Could not perform scan\n");
return;
}
le_mrc_ScanInformationRef_t cellRef;
for (cellRef=le_mrc_GetFirstCellularNetworkScan(scanInformationList);
cellRef!=NULL;
cellRef=le_mrc_GetNextCellularNetworkScan(scanInformationList))
{
char mcc[4], mnc[4];
char name[100];
le_mrc_Rat_t rat;
bufferIdx = 0;
rat = le_mrc_GetCellularNetworkRat(cellRef);
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), " %s ", PrintNetworkName(rat));
if (le_mrc_GetCellularNetworkMccMnc(cellRef, mcc, sizeof(mcc), mnc, sizeof(mnc)) != LE_OK)
{
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "Failed to get operator code.\n");
}
else
{
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "%s-%s ", mcc, mnc);
}
if (le_mrc_GetCellularNetworkName(cellRef, name, sizeof(name)) != LE_OK)
{
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "Failed to get operator name.\n");
}
else
{
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "%s", name);
}
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), " - %s ", PrintNetworkName(rat));
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "%s,",
le_mrc_IsCellularNetworkInUse(cellRef)?"In use":"Unused");
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "%s,",
le_mrc_IsCellularNetworkAvailable(cellRef)?"Available":"Unavailable");
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "%s,",
le_mrc_IsCellularNetworkHome(cellRef)?"Home":"Roaming");
bufferIdx += AppendToBuffer(&bufferPtr[bufferIdx], (bufferSz - bufferIdx), "%s\n",
le_mrc_IsCellularNetworkForbidden(cellRef)?"Forbidden":"Allowed");
// Check to see if any processing has occurred. If so, check to see if the request came from a
// local or remote requester.
// If the requester was local, (requesterPtr == NULL) then simply log our response to the SMS log.
// Otherwise, attempt to SMS the response string to the original caller.
if (requesterPtr != NULL)
{
SendMessage(requesterPtr, bufferPtr);
}
else if (OutputFilePtr != NULL)
{
fprintf(OutputFilePtr, "## %s ##\n", bufferPtr);
fflush(OutputFilePtr);
}
}
le_mrc_DeleteCellularNetworkScan(scanInformationList);
bufferIdx += snprintf(&bufferPtr[0], bufferSz, "Scan was Performed");
}
/**
* Append data to our underlying input buffer as
* if it were read from an input stream.
*
* @update gess4/3/98
* @return error code
*/
nsresult nsScanner::Append(const nsAString& aBuffer) {
mTotalRead += aBuffer.Length();
AppendToBuffer(aBuffer);
return NS_OK;
}
/**
* Append data to our underlying input buffer as
* if it were read from an input stream.
*
* @update gess4/3/98
* @return error code
*/
nsresult nsScanner::Append(const nsAString& aBuffer) {
if (!AppendToBuffer(aBuffer))
return NS_ERROR_OUT_OF_MEMORY;
return NS_OK;
}