/*
* function: format2
*
* export flows in ASCII CSV Format
*/
int format2(struct ftio *ftio, struct options *opt)
{
struct fts3rec_offsets fo;
struct ftver ftv;
char fmt_buf[1024];
char *rec;
int len;
ftio_get_ver(ftio, &ftv);
/* remove invalid fields */
opt->ft_mask &= ftrec_xfield(&ftv);
fts3rec_compute_offsets(&fo, &ftv);
fmt_xfields_type(fmt_buf, opt->ft_mask);
printf("#:%s\n", fmt_buf);
while ((rec = ftio_read(ftio))) {
len = fmt_xfields_val(fmt_buf, rec, &fo, opt->ft_mask, 0);
if (len)
printf("%s\n", fmt_buf);
++opt->records;
} /* while */
return 0;
} /* format2 */
struct ft_data *ft_open(char *filename)
{
int ret;
struct ft_data *data;
char *record;
data = (struct ft_data *)calloc(1, sizeof(struct ft_data));
data->fd = STDIN_FILENO;
if (filename && strcmp(filename, "-") != 0) {
data->fd = open(filename, O_RDONLY);
if (data->fd == -1) {
perror("could not open file");
return NULL;
}
}
ret = ftio_init(&data->io, data->fd, FT_IO_FLAG_READ);
if (ret < 0) {
perror("ftio_init failed");
return NULL;
}
ftio_get_ver(&data->io, &data->version);
data->xfield = ftio_xfield(&data->io);
fts3rec_compute_offsets(&data->offsets, &data->version);
data->rec_size = ftio_rec_size(&data->io);
/*
* TODO: optimize the reallocs here (eg by doubling the space every time
* one runs out of it)
*
* TODO: maybe allocate everything in one big chunk for faster iteration
*
*/
while ((record = ftio_read(&data->io)) != NULL) {
data->numrecords++;
data->records = (char **)realloc(data->records, sizeof(char *)*data->numrecords);
data->records[data->numrecords-1] = (char *)malloc(data->rec_size);
memcpy(data->records[data->numrecords-1], record, data->rec_size);
}
return data;
}
int main(int argc, char **argv)
{
struct ftio ftio;
struct ftprof ftp;
struct ftstat ftstat;
struct ftstat_def *ftsd;
struct ftver ftv;
struct ftvar ftvar;
struct ftset ftset;
struct fts3rec_offsets fo;
char *rec;
const char *fname, *dname;
uint32_t total_flows;
int i, split, done;
int usage_call;
/* init fterr */
fterr_setid(argv[0]);
bzero(&ftv, sizeof ftv);
bzero(&ftvar, sizeof ftvar);
total_flows = 0;
usage_call = 0;
/* init var binding */
if (ftvar_new(&ftvar) < 0)
fterr_errx(1, "ftvar_new(): failed");
fname = FT_PATH_CFG_STAT;
dname = "default";
/* configure signal handler */
if (mysignal(SIGPIPE, sig_pipe) == SIG_ERR)
fterr_err(1, "signal(SIGPIPE)");
/* defaults + no compression */
ftset_init(&ftset, 0);
while ((i = getopt(argc, argv, "b:C:d:h?s:S:kz:v:")) != -1)
switch (i) {
case 'd': /* debug */
debug = atoi(optarg);
break;
case 's': /* stat file name */
fname = optarg;
break;
case 'S': /* stat definition name */
dname = optarg;
break;
case 'v': /* variable */
if (ftvar_pset(&ftvar, optarg) < 0)
fterr_errx(1, "ftvar_pset(%s): failed", optarg);
break;
case 'h': /* help */
case '?':
usage();
++usage_call;
break;
default:
usage();
exit(1);
break;
} /* switch */
if (argc - optind)
fterr_errx(1, "Extra arguments starting with %s.", argv[optind]);
if (usage_call)
exit(0);
/* initialize and load stats config */
if (ftstat_load(&ftstat, &ftvar, fname))
fterr_errx(1, "ftstat_load(): failed");
if (!(ftsd = ftstat_def_find(&ftstat, dname)))
fterr_errx(1, "ftstat_find_def(%s): failed", dname);
/* input is stdin */
if (ftio_init(&ftio, 0, FT_IO_FLAG_READ) < 0)
fterr_errx(1, "ftio_init(): failed");
ftio_get_ver(&ftio, &ftv);
if (ftstat_def_test_xfields(ftsd, ftrec_xfield(&ftv)))
fterr_errx(1, "Report definition references a field not in flow.");
fts3rec_compute_offsets(&fo, &ftv);
/* profile */
ftprof_start (&ftp);
if (ftstat_def_new(ftsd)) {
fterr_errx(1, "ftstat_new(%s): failed.",ftsd->name);
}
while ((rec = ftio_read(&ftio))) {
++total_flows;
done = 0;
if ((split = ftstat_def_accum(ftsd, rec, &fo)) < 0) {
fterr_errx(1, "ftstat_eval(%s): failed.",ftsd->name);
}
if (split) {
if (ftstat_def_calc(ftsd)) {
fterr_errx(1, "ftstat_dump(%s): failed.",ftsd->name);
}
if (ftstat_def_dump(&ftio, ftsd)) {
fterr_errx(1, "ftstat_dump(%s): failed.",ftsd->name);
}
if (ftstat_def_reset(ftsd)) {
fterr_errx(1, "ftstat_def_reset(%s): failed.",ftsd->name);
}
if ((split = ftstat_def_accum(ftsd, rec, &fo)) < 0) {
fterr_errx(1, "ftstat_eval(%s): failed.",ftsd->name);
}
if (split == 1)
fterr_errx(1, "ftstat_def_accum(): looping on split");
} /* split */
} /* while more flows */
if (ftstat_def_calc(ftsd)) {
fterr_errx(1, "ftstat_dump(%s): failed.",ftsd->name);
}
if (ftstat_def_dump(&ftio, ftsd)) {
fterr_errx(1, "ftstat_dump(%s): failed.",ftsd->name);
}
if (ftstat_def_free(ftsd)) {
fterr_errx(1, "ftstat_def_free(%s): failed.",ftsd->name);
}
if (ftio_close(&ftio) < 0)
fterr_errx(1, "ftio_close(): failed");
if (debug > 0) {
ftprof_end (&ftp, total_flows);
ftprof_print(&ftp, argv[0], stderr);
}
ftstat_free(&ftstat);
ftvar_free(&ftvar);
return 0;
} /* main */
int main(int argc, char **argv)
{
struct fts3rec_all cur;
struct fts3rec_offsets fo;
struct ftio ftio_in, ftio_out;
struct ftset ftset;
struct ftver ftv;
struct ftprof ftp;
uint32_t time_start, time_end, exaddr;
int i, ret;
char *acl_fname, *acl_std_src_name, *acl_std_dst_name;
char *acl_std_nexthop_name;
char *acl_ext_name, *str, *strm;
int acl_std_src_index, acl_std_src_index2;
int acl_std_dst_index, acl_std_dst_index2;
int acl_std_nexthop_index, acl_std_nexthop_index2;
int acl_ext_index, acl_ext_index2;
struct acl_ip_ext_entry tmp_ext;
int keep_input_time;
int filter_input, filter_output, filter_srcport, filter_dstport;
int filter_prot, filter_srcas, filter_dstas, filter_tos, filter_tcp_flags;
int filter_exaddr;
char in_tbl[65536], out_tbl[65536], src_tbl[65536], dst_tbl[65536];
char srcas_tbl[65536], dstas_tbl[65536], tos_tbl[65536];
char tcp_flags_tbl[65536];
char prot_tbl[256];
u_char tos_mask, tos, tcp_flags_mask, tcp_flags;
uint64_t total_flows, xflag;
char *rec;
int first_match = 0;
/* init fterr */
fterr_setid(argv[0]);
bzero(&ftv, sizeof ftv);
/* defaults + no compression */
ftset_init(&ftset, 0);
/* init */
bzero(&acl_list, sizeof acl_list);
acl_fname = acl_std_src_name = acl_std_dst_name = (char*)0L;
acl_std_nexthop_name = (char*)0L;
acl_ext_name = (char*)0L;
acl_std_src_index = acl_std_dst_index = -1;
acl_std_nexthop_index = -1;
acl_ext_index = -1;
bzero(&tmp_ext, sizeof tmp_ext);
total_flows = 0;
tos_mask = 0xff;
tcp_flags_mask = 0xff;
keep_input_time = 0;
filter_input = filter_output = filter_srcport = filter_dstport = 0;
filter_prot = filter_srcas = filter_dstas = filter_tos = filter_exaddr = 0;
filter_tcp_flags = 0;
while ((i = getopt(argc, argv, "a:A:b:C:d:D:e:E:f:i:I:kop:P:r:S:t:T:x:z:"))
!= -1)
switch (i) {
case 'a': /* src AS filter list */
if (load_lookup(optarg, 65536, srcas_tbl))
fterr_errx(1, "load_lookup(): failed");
filter_srcas = 1;
break;
case 'A': /* dst AS filter list */
if (load_lookup(optarg, 65536, dstas_tbl))
fterr_errx(1, "load_lookup(): failed");
filter_dstas = 1;
break;
case 'b': /* output byte order */
if (!strcasecmp(optarg, "little"))
ftset.byte_order = FT_HEADER_LITTLE_ENDIAN;
else if (!strcasecmp(optarg, "big"))
ftset.byte_order = FT_HEADER_BIG_ENDIAN;
else
fterr_errx(1, "expecting \"big\" or \"little\"");
break;
case 'C': /* comment field */
ftset.comments = optarg;
break;
case 'D': /* dst ip standard access list filter */
acl_std_dst_name = optarg;
break;
case 'd': /* debug */
debug = atoi(optarg);
break;
case 'e': /* export IP address */
filter_exaddr = 1;
exaddr = scan_ip(optarg);
break;
case 'E': /* extended access list filter */
acl_ext_name = optarg;
break;
case 'f': /* acl file name */
acl_fname = optarg;
break;
case 'i': /* input filter interface list */
if (load_lookup(optarg, 65536, in_tbl))
fterr_errx(1, "load_lookup(): failed");
filter_input = 1;
break;
case 'I': /* output filter interface list */
if (load_lookup(optarg, 65536, out_tbl))
fterr_errx(1, "load_lookup(): failed");
filter_output = 1;
break;
case 'k': /* keep the start/end time from the input */
keep_input_time = 1;
break;
case 'o': /* do logical OR between different statements (first match) */
first_match = 1;
break;
case 'P': /* filter dstport */
if (load_lookup(optarg, 65536, dst_tbl))
fterr_errx(1, "load_lookup(): failed");
filter_dstport = 1;
break;
case 'p': /* filter srcport */
if (load_lookup(optarg, 65536, src_tbl))
fterr_errx(1, "load_lookup(): failed");
filter_srcport = 1;
break;
case 'r': /* filter protocol */
if (load_lookup(optarg, 256, prot_tbl))
fterr_errx(1, "load_lookup(): failed");
filter_prot = 1;
break;
case 'S': /* src ip standard access list filter */
acl_std_src_name = optarg;
break;
case 't': /* ToS Filter */
if (!(str = malloc(strlen(optarg+1))))
fterr_err(1, "malloc()");
strcpy(str, optarg);
/* search for mask option */
if ((strm = index(str, '/'))) {
++strm;
tos_mask = (u_char)strtol(strm, (char**)0L, 0);
--strm;
*strm = 0;
}
if (load_lookup(str, 65536, tos_tbl)) {
free(str);
fterr_errx(1, "load_lookup(): failed");
}
free(str);
filter_tos = 1;
break;
case 'T': /* tcp flags filter */
if (!(str = malloc(strlen(optarg+1))))
fterr_err(1, "malloc()");
strcpy(str, optarg);
/* search for mask option */
if ((strm = index(str, '/'))) {
++strm;
tcp_flags_mask = (u_char)strtol(strm, (char**)0L, 0);
--strm;
*strm = 0;
}
if (load_lookup(str, 65536, tcp_flags_tbl)) {
free(str);
fterr_errx(1, "load_lookup(): failed");
}
free(str);
filter_tcp_flags = 1;
break;
case 'x': /* nexthop ip standard access list filter */
acl_std_nexthop_name = optarg;
break;
case 'h': /* help */
case '?':
usage();
exit (0);
break;
case 'z': /* compress level */
ftset.z_level = atoi(optarg);
if ((ftset.z_level < 0) || (ftset.z_level > 9))
fterr_errx(1, "Compression level must be between 0 and 9");
break;
default:
usage();
exit (1);
break;
} /* switch */
if (argc - optind)
fterr_errx(1, "Extra arguments starting with %s.", argv[optind]);
/* input from stdin */
if (ftio_init(&ftio_in, 0, FT_IO_FLAG_READ) < 0)
fterr_errx(1, "ftio_init(): failed");
ftio_get_ver(&ftio_in, &ftv);
ftv.s_version = FT_IO_SVERSION;
xflag = 0;
if (filter_input) xflag |= FT_XFIELD_INPUT;
if (filter_output) xflag |= FT_XFIELD_OUTPUT;
if (filter_srcport) xflag |= FT_XFIELD_SRCPORT;
if (filter_dstport) xflag |= FT_XFIELD_DSTPORT;
if (filter_exaddr) xflag |= FT_XFIELD_EXADDR;
if (filter_prot) xflag |= FT_XFIELD_PROT;
if (filter_srcas) xflag |= FT_XFIELD_SRC_AS;
if (filter_dstas) xflag |= FT_XFIELD_DST_AS;
if (filter_tos) xflag |= FT_XFIELD_TOS;
if (filter_tcp_flags) xflag |= FT_XFIELD_TCP_FLAGS;
if (acl_std_nexthop_name) xflag |= FT_XFIELD_NEXTHOP;
if (acl_std_src_name) xflag |= FT_XFIELD_SRCADDR;
if (acl_std_dst_name) xflag |= FT_XFIELD_DSTADDR;
if (acl_ext_name) xflag |= (FT_XFIELD_SRCADDR | FT_XFIELD_DSTADDR);
if (ftio_check_xfield(&ftio_in, xflag)) {
fterr_warnx("Flow record missing required field for format.");
exit (1);
}
fts3rec_compute_offsets(&fo, &ftv);
/* output to stdout */
if (ftio_init(&ftio_out, 1, FT_IO_FLAG_WRITE |
((ftset.z_level) ? FT_IO_FLAG_ZINIT : 0) ) < 0)
fterr_errx(1, "ftio_init(): failed");
/* preserve start/end time from input stream? */
if (keep_input_time) {
time_start = ftio_get_cap_start(&ftio_in);
time_end = ftio_get_cap_end(&ftio_in);
if (time_start && time_end) {
ftio_set_preloaded(&ftio_out, 1);
ftio_set_cap_time(&ftio_out, time_start, time_end);
}
} /* keep_input_time */
ftio_set_comment(&ftio_out, ftset.comments);
ftio_set_byte_order(&ftio_out, ftset.byte_order);
ftio_set_z_level(&ftio_out, ftset.z_level);
ftio_set_streaming(&ftio_out, 1);
ftio_set_debug(&ftio_out, debug);
if (ftio_set_ver(&ftio_out, &ftv) < 0)
fterr_errx(1, "ftio_set_ver(): failed");
/*
* load acl's
* XXX add fname check and close
*/
if ((yyin = fopen(acl_fname ? acl_fname : "flow.acl", "r")))
while (!feof(yyin))
yyparse();
/*
* normalize masks
*/
/* XXX TODO */
if (debug > 5)
acl_dump(acl_list);
if (acl_std_src_name) {
if ((acl_std_src_index = acl_find(acl_list, acl_std_src_name)) == -1)
fterr_errx(1, "Couldn't find list %s\n", acl_std_src_name);
acl_std_src_index2 = acl_list.names[acl_std_src_index].num;
}
if (acl_std_dst_name) {
if ((acl_std_dst_index = acl_find(acl_list, acl_std_dst_name)) == -1)
fterr_errx(1, "Couldn't find list %s\n", acl_std_dst_name);
acl_std_dst_index2 = acl_list.names[acl_std_dst_index].num;
}
if (acl_ext_name) {
if ((acl_ext_index = acl_find(acl_list, acl_ext_name)) == -1)
fterr_errx(1, "Couldn't find list %s\n", acl_ext_name);
acl_ext_index2 = acl_list.names[acl_ext_index].num;
}
if (acl_std_nexthop_name) {
if ((acl_std_nexthop_index = acl_find(acl_list, acl_std_nexthop_name))
== -1)
fterr_errx(1, "Couldn't find list %s\n", acl_std_nexthop_name);
acl_std_nexthop_index2 = acl_list.names[acl_std_nexthop_index].num;
}
/* header first */
if (ftio_write_header(&ftio_out) < 0)
fterr_errx(1, "ftio_write_header(): failed");
/* profile */
ftprof_start (&ftp);
/* grab 1 flow */
while ((rec = ftio_read(&ftio_in))) {
cur.srcaddr = ((uint32_t*)(rec+fo.srcaddr));
cur.dstaddr = ((uint32_t*)(rec+fo.dstaddr));
cur.exaddr = ((uint32_t*)(rec+fo.exaddr));
cur.nexthop = ((uint32_t*)(rec+fo.nexthop));
cur.input = ((uint16_t*)(rec+fo.input));
cur.output = ((uint16_t*)(rec+fo.output));
cur.srcport = ((uint16_t*)(rec+fo.srcport));
cur.dstport = ((uint16_t*)(rec+fo.dstport));
cur.src_as = ((uint16_t*)(rec+fo.src_as));
cur.dst_as = ((uint16_t*)(rec+fo.dst_as));
cur.prot = ((uint8_t*)(rec+fo.prot));
cur.tcp_flags = ((uint8_t*)(rec+fo.tcp_flags));
cur.tos = ((uint8_t*)(rec+fo.tos));
++ total_flows;
/* filter on input interface */
if (filter_input) {
if (!in_tbl[*cur.input]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* filter on output interface */
if (filter_output) {
if (!out_tbl[*cur.output]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* filter on src port */
if (filter_srcport) {
if (!src_tbl[*cur.srcport]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* filter on dst port */
if (filter_dstport) {
if (!dst_tbl[*cur.dstport]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* filter on exporter addr */
if (filter_exaddr) {
if (*cur.exaddr != exaddr) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* filter on protocol */
if (filter_prot) {
if (!prot_tbl[*cur.prot]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* filter on ToS */
if (filter_tos) {
tos = *cur.tos & tos_mask;
if (!tos_tbl[tos]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* filter on tcp_flags */
if (filter_tcp_flags && (*cur.prot == IPPROTO_TCP)) {
tcp_flags = *cur.tcp_flags & tcp_flags_mask;
if (!tcp_flags_tbl[tcp_flags]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
if (filter_srcas) {
if (!srcas_tbl[*cur.src_as]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* filter on src AS */
if (filter_dstas) {
if (!dstas_tbl[*cur.dst_as]) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* eval flow nexthop addr and nexthop standard acl */
if (acl_std_nexthop_index != -1) {
ret = acl_eval_std(acl_list, acl_std_nexthop_index2, *cur.nexthop);
if (ret == 1) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* eval flow src addr and source standard acl */
if (acl_std_src_index != -1) {
ret = acl_eval_std(acl_list, acl_std_src_index2, *cur.srcaddr);
if (ret == 1) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* eval flow dst addr and destination standard acl */
if (acl_std_dst_index != -1) {
ret = acl_eval_std(acl_list, acl_std_dst_index2, *cur.dstaddr);
if (ret == 1) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
/* eval flow and extended acl */
if (acl_ext_index != -1) {
tmp_ext.protocol = *cur.prot;
tmp_ext.tos = *cur.tos;
/* XXX */
tmp_ext.type = 0;
tmp_ext.type_code = 0;
tmp_ext.message = 0;
tmp_ext.src_addr = *cur.srcaddr;
tmp_ext.src_port = *cur.srcport;
tmp_ext.dst_addr = *cur.dstaddr;
tmp_ext.dst_port = *cur.dstport;
ret = acl_eval_ext(acl_list, acl_ext_index2, tmp_ext);
if (ret == 1) {
if (!first_match)
continue;
} else if (first_match) {
goto found;
}
}
if (first_match) /* No matches yet? next try.. */
continue;
/*
* made it by the filters, write it
*/
found:
if (ftio_write(&ftio_out, rec) < 0)
fterr_errx(1, "ftio_write(): failed");
} /* while more flows to read */
if (ftio_close(&ftio_in) < 0)
fterr_errx(1, "ftio_close(): failed");
if (ftio_close(&ftio_out) < 0)
fterr_errx(1, "ftio_close(): failed");
if (debug > 0) {
ftprof_end (&ftp, total_flows);
ftprof_print(&ftp, argv[0], stderr);
}
if (debug > 1) {
acl_dump_std(acl_list, acl_std_src_index);
acl_dump_std(acl_list, acl_std_dst_index);
acl_dump_std(acl_list, acl_std_nexthop_index);
}
return 0;
} /* main */
int flows2nfdump(struct ftio *ftio, char *wfile, int compress, extension_info_t *extension_info, int extended, uint32_t limitflows) {
// required flow tools variables
struct fttime ftt;
struct fts3rec_offsets fo;
struct ftver ftv;
char *rec;
// nfdump variables
nffile_t *nffile;
master_record_t record;
char *s;
uint32_t cnt;
s = "flow-tools";
nffile = OpenNewFile( wfile , NULL, compress, 0, s);
if ( !nffile ) {
fprintf(stderr, "%s\n", s);
return 1;
}
AppendToBuffer(nffile, (void *)extension_info->map, extension_info->map->size);
ftio_get_ver(ftio, &ftv);
fts3rec_compute_offsets(&fo, &ftv);
memset((void *)&record, 0, sizeof(record));
record.map_ref = extension_info->map;
record.type = CommonRecordType;
record.exporter_sysid = 0;
// only v4 addresses
ClearFlag(record.flags, FLAG_IPV6_ADDR);
cnt = 0;
while ((rec = ftio_read(ftio))) {
uint32_t when, unix_secs, unix_nsecs, sysUpTime;
int i, id;
unix_secs = *((uint32_t*)(rec+fo.unix_secs));
unix_nsecs = *((uint32_t*)(rec+fo.unix_nsecs));
sysUpTime = *((uint32_t*)(rec+fo.sysUpTime));
when = *((uint32_t*)(rec+fo.First));
ftt = ftltime(sysUpTime, unix_secs, unix_nsecs, when);
record.first = ftt.secs;
record.msec_first = ftt.msecs;
when = *((uint32_t*)(rec+fo.Last));
ftt = ftltime(sysUpTime, unix_secs, unix_nsecs, when);
record.last = ftt.secs;
record.msec_last = ftt.msecs;
record.v4.srcaddr = *((uint32_t*)(rec+fo.srcaddr));
record.v4.dstaddr = *((uint32_t*)(rec+fo.dstaddr));
record.srcport = *((uint16_t*)(rec+fo.srcport));
record.dstport = *((uint16_t*)(rec+fo.dstport));
record.prot = *((uint8_t*)(rec+fo.prot));
record.tcp_flags = *((uint8_t*)(rec+fo.tcp_flags));
record.tos = *((uint8_t*)(rec+fo.tos));
record.dOctets = *((uint32_t*)(rec+fo.dOctets));
record.dPkts = *((uint32_t*)(rec+fo.dPkts));
i = 0;
while ( (id = extension_info->map->ex_id[i]) != 0 ) {
switch (id) {
case EX_IO_SNMP_2:
record.input = *((uint16_t*)(rec+fo.input));
record.output = *((uint16_t*)(rec+fo.output));
break;
case EX_AS_2:
record.srcas = *((uint16_t*)(rec+fo.src_as));
record.dstas = *((uint16_t*)(rec+fo.dst_as));
break;
case EX_MULIPLE:
record.src_mask = *((uint8_t*)(rec+fo.src_mask));
record.dst_mask = *((uint8_t*)(rec+fo.dst_mask));
record.dir = 0;
record.dst_tos = 0;
break;
case EX_ROUTER_IP_v4:
record.ip_nexthop.v4 = *((uint32_t*)(rec+fo.peer_nexthop));
break;
case EX_NEXT_HOP_v4:
record.ip_router.v4 = *((uint32_t*)(rec+fo.router_sc));
break;
case EX_ROUTER_ID:
record.engine_type = *((uint8_t*)(rec+fo.engine_type));
record.engine_id = *((uint8_t*)(rec+fo.engine_id));
break;
// default: Other extensions can not be sent with v5
}
i++;
}
PackRecord(&record, nffile);
if ( extended ) {
char *string;
format_file_block_record(&record, &string, 0);
fprintf(stderr, "%s\n", string);
}
cnt++;
if ( cnt == limitflows )
break;
} /* while */
// write the last records in buffer
if ( nffile->block_header->NumRecords ) {
if ( WriteBlock(nffile) <= 0 ) {
fprintf(stderr, "Failed to write output buffer: '%s'" , strerror(errno));
}
}
free((void *)extension_info->map);
free((void *)extension_info);
DisposeFile(nffile);
return 0;
} // End of flows2nfdump
/*
* function: format5
*
* export flows into PostgreSQL Database
*/
int format5(struct ftio *ftio, struct options *opt)
{
#ifdef HAVE_POSTGRESQL
struct fts3rec_offsets fo;
struct ftver ftv;
char fields[1024], values[1024], query[3*1024];
char *rec;
char *db_host, *db_name, *db_table, *db_user, *db_pwd, *db_tmp, *tmp;
char *db_port;
int len;
PGconn *conn;
PGresult *res;
db_host = POSTGRESQL_DEFAULT_DBHOST;
db_name = POSTGRESQL_DEFAULT_DBNAME;
db_port = POSTGRESQL_DEFAULT_DBPORT;
db_user = POSTGRESQL_DEFAULT_DBUSER;
db_table = POSTGRESQL_DEFAULT_DBTABLE;
db_pwd = POSTGRESQL_DEFAULT_DBPWD;
/* parse URI string */
if (strlen(opt->dbaseURI)) {
tmp = opt->dbaseURI;
db_user = strsep(&tmp, ":");
db_pwd = strsep(&tmp, ":");
db_host = strsep(&tmp, ":");
db_port = strsep(&tmp, ":");
db_name = strsep(&tmp, ":");
db_table = strsep(&tmp, ":");
if (!db_user || !db_pwd || !db_host || !db_port || !db_name || !db_table) {
fterr_warnx("Missing field in dbaseURI, expecting user:pwd:host:port:name:table.");
return -1;
}
} /* dbaseURI */
ftio_get_ver(ftio, &ftv);
fts3rec_compute_offsets(&fo, &ftv);
/* remove invalid fields */
opt->ft_mask &= ftrec_xfield(&ftv);
/* generate the field names once */
fmt_xfields_type(fields, opt->ft_mask);
/* open PostgreSQL database */
conn = PQsetdbLogin(db_host, db_port, (char *) NULL, (char *) NULL, db_name, db_user, db_pwd);
if (PQstatus(conn) == CONNECTION_BAD)
fterr_errx(1,"PQsetdbLogin(): %s\n", PQerrorMessage(conn));
/* foreach flow */
while ((rec = ftio_read(ftio))) {
len = fmt_xfields_val(values, rec, &fo, opt->ft_mask, 1);
/* form SQL query and execute it */
if (len) {
strcpy (query, "INSERT INTO ");
strcat (query, db_table);
strcat (query, "(");
strcat (query, fields);
strcat (query, ") VALUES (");
strcat (query, values);
strcat (query, ")");
if (debug)
fprintf(stderr, "field=%s\n val=%s\n query=%s\n", fields, values,
query);
res = PQexec(conn, query);
if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) {
PQclear(res);
fterr_errx(1,"PQexec(): %s\n", PQerrorMessage(conn));
} else if (res) {
PQclear(res);
}
}
++opt->records;
} /* while */
/* close database */
PQfinish(conn);
#else /* PGSQL */
fterr_warnx("Format not supported");
#endif /* PGSQL */
return 0;
} /* format5 */
/*
* function: format4
*
* export flows in wire format
*/
int format4(struct ftio *ftio, struct options *opt)
{
struct ftver ftv;
struct ftencode fte;
char *rec;
int ret;
/* initialize encode struct */
ftencode_init(&fte, 0);
/* copy version from io stream */
ftio_get_ver(ftio, &ftv);
bcopy(&ftv, &fte.ver, sizeof ftv);
/* foreach flow */
while ((rec = ftio_read(ftio))) {
retry:
ret = fts3rec_pdu_encode(&fte, rec);
/* ret == 0 then send and clear out buffer
* ret > 0 then encode another
* ret < 0 then this encoding failed, send and clear out buffer
*/
if (ret <= 0) {
/* convert pdu to network byte order */
#if BYTE_ORDER == LITTLE_ENDIAN
ftpdu_swap(fte.buf_enc, BYTE_ORDER);
#endif /* BYTE_ORDER == LITTLE_ENDIAN */
if (fwrite(&fte.buf, fte.buf_size, 1, stdout) != 1)
fterr_err(1, "fwrite()");
/* reset encode buffer */
ftencode_reset(&fte);
/* if ret < 0 then the current record was not encoded */
if (ret < 0)
goto retry;
}
++opt->records;
}
/* any left over? */
if (fte.buf_size) {
/* convert pdu to network byte order */
ftpdu_swap(fte.buf_enc, BYTE_ORDER);
if (fwrite(&fte.buf, fte.buf_size, 1, stdout) != 1)
fterr_err(1, "fwrite()");
} /* fte.buf_size */
return 0;
} /* format4 */
/*
* function: format3
*
* export flows into MySQL Database
*/
int format3(struct ftio *ftio, struct options *opt)
{
#ifdef HAVE_MYSQL
struct fts3rec_offsets fo;
struct ftver ftv;
char fields[1024], values[1024], query[3*1024];
char *rec;
char *db_host, *db_name, *db_table, *db_user, *db_pwd, *db_tmp, *tmp;
int db_port;
int len;
MYSQL mysql;
db_host = MYSQL_DEFAULT_DBHOST;
db_name = MYSQL_DEFAULT_DBNAME;
db_port = MYSQL_DEFAULT_DBPORT;
db_user = MYSQL_DEFAULT_DBUSER;
db_table = MYSQL_DEFAULT_DBTABLE;
db_pwd = MYSQL_DEFAULT_DBPWD;
/* parse URI string */
if (strlen(opt->dbaseURI)) {
tmp = opt->dbaseURI;
db_user = strsep(&tmp, ":");
db_pwd = strsep(&tmp, ":");
db_host = strsep(&tmp, ":");
db_tmp = strsep(&tmp, ":");
db_name = strsep(&tmp, ":");
db_table = strsep(&tmp, ":");
db_port = atoi(db_tmp);
if (!db_user || !db_pwd || !db_host || !db_tmp || !db_name || !db_table) {
fterr_warnx("Missing field in dbaseURI, expecting user:pwd:host:port:name:table.");
return -1;
}
} /* dbaseURI */
ftio_get_ver(ftio, &ftv);
fts3rec_compute_offsets(&fo, &ftv);
/* remove invalid fields */
opt->ft_mask &= ftrec_xfield(&ftv);
/* generate the field names once */
fmt_xfields_type(fields, opt->ft_mask);
/* open MySQL database */
if (!(mysql_init(&mysql)))
fterr_errx(1, "mysql_init(): failed");
if (mysql_options(&mysql, MYSQL_READ_DEFAULT_GROUP, "simple"))
fterr_errx(1, "mysql_options(): %s", mysql_error(&mysql));
if (mysql_real_connect(&mysql, db_host, db_user, db_pwd,
db_name, db_port, NULL, 0) == NULL)
fterr_errx(1,"mysql_real_connect(): %s\n", mysql_error(&mysql));
/* foreach flow */
while ((rec = ftio_read(ftio))) {
len = fmt_xfields_val(values, rec, &fo, opt->ft_mask, 1);
/* form SQL query and execute it */
if (len) {
strcpy (query, "INSERT INTO ");
strcat (query, db_table);
strcat (query, "(");
strcat (query, fields);
strcat (query, ") VALUES (");
strcat (query, values);
strcat (query, ")");
if (debug)
fprintf(stderr, "field=%s\n val=%s\n query=%s\n", fields, values,
query);
if (mysql_real_query(&mysql, query, strlen(query)) != 0)
fterr_warnx("mysql_real_query(): %s", mysql_error(&mysql));
}
++opt->records;
} /* while */
/* close database */
mysql_close(&mysql);
#else /* MYSQL */
fterr_warnx("Format not supported");
#endif /* MYSQL */
return 0;
} /* format3 */
/*
* function: format1
*
* export flows in pcap format. Hack to use tcpdump's packet matcher
*/
int format1(struct ftio *ftio, struct options *opt)
{
struct timeval now;
struct timezone tz;
struct fts3rec_all cur;
struct fts3rec_offsets fo;
struct ftver ftv;
struct pcap_file_header pfh;
struct pcap_packet_header pph;
struct pcap_data1 pd1;
struct pcap_data2 pd2;
struct pcap_data3 pd3;
struct pcap_data4 pd4;
int bsize, bsize2;
long thiszone;
char buf[1024];
char *rec;
if (ftio_check_xfield(ftio, FT_XFIELD_TOS | FT_XFIELD_PROT |
FT_XFIELD_SRCADDR | FT_XFIELD_DSTADDR | FT_XFIELD_SRCPORT |
FT_XFIELD_DSTPORT |
FT_XFIELD_UNIX_SECS | FT_XFIELD_UNIX_NSECS |
FT_XFIELD_DPKTS | FT_XFIELD_DOCTETS)) {
fterr_warnx("Flow record missing required field for format.");
return -1;
}
ftio_get_ver(ftio, &ftv);
fts3rec_compute_offsets(&fo, &ftv);
if (gettimeofday(&now, &tz) < 0) {
fterr_warnx("gettimeofday() failed");
return -1;
}
bzero(&pfh, sizeof pfh);
bzero(&pph, sizeof pph);
bzero(&pd1, sizeof pd1); /* ethernet */
bzero(&pd2, sizeof pd2); /* IP */
bzero(&pd3, sizeof pd3); /* TCP */
bzero(&pd4, sizeof pd4); /* UDP */
bsize = 0;
pfh.magic = TCPDUMP_MAGIC;
pfh.version_major = TCPDUMP_VERSION_MAJOR;
pfh.version_minor = TCPDUMP_VERSION_MINOR;
pfh.sigfigs = 6;
pfh.snaplen = 65535;
pfh.linktype = 1;
if (fwrite(&pfh, sizeof pfh, 1, stdout) != 1) {
fterr_warnx("pcap header write failed");
return -1;
}
pd1.eth_prot = 0x0008;
pd2.version = 0x45;
/* note: bcopy of pph to buf is deferred (so lengths can be adjusted below) */
bsize += sizeof pph;
bcopy(&pd1, buf+bsize, sizeof pd1);
bsize += sizeof pd1;
while ((rec = ftio_read(ftio))) {
cur.unix_secs = (uint32_t *)(rec+fo.unix_secs);
cur.unix_nsecs = (uint32_t *)(rec+fo.unix_nsecs);
cur.srcport = ((uint16_t*)(rec+fo.srcport));
cur.dstport = ((uint16_t*)(rec+fo.dstport));
cur.prot = ((uint8_t*)(rec+fo.prot));
cur.tos = ((uint8_t*)(rec+fo.tos));
cur.srcaddr = ((uint32_t*)(rec+fo.srcaddr));
cur.dstaddr = ((uint32_t*)(rec+fo.dstaddr));
pd2.version = 4 << 4 | ((sizeof pd2) / 4);
pd2.tos = *cur.tos;
pd2.prot = *cur.prot;
pd2.srcaddr = *cur.srcaddr;
pd2.dstaddr = *cur.dstaddr;
pd2.len = sizeof pd2; /* this might be adjusted below for TCP, UDP, etc. */
pph.caplen = sizeof pd1 + sizeof pd2; /* ether and IP header */
#if BYTE_ORDER == LITTLE_ENDIAN
SWAPINT32(pd2.srcaddr);
SWAPINT32(pd2.dstaddr);
#endif /* LITTLE_ENDIAN */
#if 1 /* { */
switch (pd2.prot) {
case 6: /* TCP */
pd3.srcport = *cur.srcport;
pd3.dstport = *cur.dstport;
#if BYTE_ORDER == LITTLE_ENDIAN
SWAPINT16(pd3.srcport);
SWAPINT16(pd3.dstport);
#endif /* LITTLE_ENDIAN */
/* set data offset to 6 32-bit words: */
pd3.data_reserved_flags_window = 0x60000000;
#if BYTE_ORDER == LITTLE_ENDIAN
SWAPINT32(pd3.data_reserved_flags_window);
#endif /* LITTLE_ENDIAN */
pph.caplen += sizeof pd3; /* + TCP header */
pd2.len += sizeof pd3;
bcopy(&pd3, buf+bsize+sizeof pd2, sizeof pd3);
bsize2 = bsize + sizeof pd2 + sizeof pd3;
break;
case 17: /* UDP */
pd4.srcport = *cur.srcport;
pd4.dstport = *cur.dstport;
#if BYTE_ORDER == LITTLE_ENDIAN
SWAPINT16(pd4.srcport);
SWAPINT16(pd4.dstport);
#endif /* LITTLE_ENDIAN */
pph.caplen += sizeof pd4; /* + UDP header */
pd2.len += sizeof pd4;
pd4.len = sizeof pd4; /* FIXME */
#if BYTE_ORDER == LITTLE_ENDIAN
SWAPINT16(pd4.len);
#endif /* LITTLE_ENDIAN */
bcopy(&pd4, buf+bsize+sizeof pd2, sizeof pd4);
bsize2 = bsize + sizeof pd2 + sizeof pd4;
break;
case 1: /* FIXME - handle ICMP specially */
default: /* handle others IP protocols */
bsize2 = bsize + sizeof pd2;
break;
} /* switch */
#else /* }{ */
bsize2 = bsize + sizeof pd2;
#endif /* } */
pph.ts.tv_sec = *(uint32_t *)(rec+fo.unix_secs);
pph.ts.tv_usec = *(uint32_t *)(rec+fo.unix_nsecs) / 1000;
pph.len = pph.caplen; /* FIXME */
bcopy(&pph, buf, sizeof pph);
#if BYTE_ORDER == LITTLE_ENDIAN
SWAPINT16(pd2.len);
#endif /* LITTLE_ENDIAN */
bcopy(&pd2, buf+bsize, sizeof pd2);
{
/* FIXME - add a loop here to write one record per packet (rather than
* one per flow). change the packet size to be
* floor(average_pkt_size), then add one byte to the first or
* last flow, if necessary, to make the byte count correct.
*/
/* uint16_t dPkts; */
/* uint16_t dOctets; */
if (fwrite(&buf, bsize2, 1, stdout) != 1) {
fterr_warnx("pcap pkt write failed");
return -1;
}
}
++opt->records;
} /* while */
return 0;
} /* format1 */
/*
* function: format0
*
* export flows in cflowd format
*/
int format0(struct ftio *ftio, struct options *opt)
{
struct fts3rec_offsets fo;
struct ftver ftv;
struct fttime ftt;
char *rec;
uint32_t ui32, index, sysUpTime, unix_secs, unix_nsecs, First, Last;
uint16_t ui16;
uint8_t ui8;
ftio_get_ver(ftio, &ftv);
fts3rec_compute_offsets(&fo, &ftv);
switch (ftv.d_version) {
case 1:
opt->cflowd_mask &= CF_INDEX_V1_MASK;
break;
case 5:
opt->cflowd_mask &= CF_INDEX_V5_MASK;
break;
case 6:
opt->cflowd_mask &= CF_INDEX_V6_MASK;
break;
case 7:
opt->cflowd_mask &= CF_INDEX_V7_MASK;
break;
case 1005:
opt->cflowd_mask &= CF_INDEX_V5_MASK;
break;
case 8:
switch (ftv.agg_method) {
case 1:
opt->cflowd_mask &= CF_INDEX_V8_1_MASK;
break;
case 2:
opt->cflowd_mask &= CF_INDEX_V8_2_MASK;
break;
case 3:
opt->cflowd_mask &= CF_INDEX_V8_3_MASK;
break;
case 4:
opt->cflowd_mask &= CF_INDEX_V8_4_MASK;
break;
case 5:
opt->cflowd_mask &= CF_INDEX_V8_5_MASK;
break;
case 6:
opt->cflowd_mask &= CF_INDEX_V8_6_MASK;
break;
case 7:
opt->cflowd_mask &= CF_INDEX_V8_7_MASK;
break;
case 8:
opt->cflowd_mask &= CF_INDEX_V8_8_MASK;
break;
case 9:
opt->cflowd_mask &= CF_INDEX_V8_9_MASK;
break;
case 10:
opt->cflowd_mask &= CF_INDEX_V8_10_MASK;
break;
case 11:
opt->cflowd_mask &= CF_INDEX_V8_11_MASK;
break;
case 12:
opt->cflowd_mask &= CF_INDEX_V8_12_MASK;
break;
case 13:
opt->cflowd_mask &= CF_INDEX_V8_13_MASK;
break;
case 14:
opt->cflowd_mask &= CF_INDEX_V8_14_MASK;
break;
default:
fterr_warnx("Unsupported export version");
return -1;
} /* switch */
break;
default:
fterr_warnx("Unsupported export version");
return -1;
} /* switch */
/* index */
index = opt->cflowd_mask;
index = htonl(index);
while ((rec = ftio_read(ftio))) {
fwrite(&index, sizeof (index), 1, stdout);
if (opt->cflowd_mask & CF_ROUTERMASK) {
ui32 = *((uint32_t*)(rec+fo.exaddr));
ui32 = htonl(ui32);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_SRCIPADDRMASK) {
ui32 = *((uint32_t*)(rec+fo.srcaddr));
ui32 = htonl(ui32);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_DSTIPADDRMASK) {
ui32 = *((uint32_t*)(rec+fo.dstaddr));
ui32 = htonl(ui32);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_INPUTIFINDEXMASK) {
ui16 = *((uint16_t*)(rec+fo.input));
ui16 = htons(ui16);
fwrite(&ui16, sizeof (ui16), 1, stdout);
}
if (opt->cflowd_mask & CF_OUTPUTIFINDEXMASK) {
ui16 = *((uint16_t*)(rec+fo.output));
ui16 = htons(ui16);
fwrite(&ui16, sizeof (ui16), 1, stdout);
}
if (opt->cflowd_mask & CF_SRCPORTMASK) {
ui16 = *((uint16_t*)(rec+fo.srcport));
ui16 = htons(ui16);
fwrite(&ui16, sizeof (ui16), 1, stdout);
}
if (opt->cflowd_mask & CF_DSTPORTMASK) {
ui16 = *((uint16_t*)(rec+fo.dstport));
ui16 = htons(ui16);
fwrite(&ui16, sizeof (ui16), 1, stdout);
}
if (opt->cflowd_mask & CF_PKTSMASK) {
ui32 = *((uint32_t*)(rec+fo.dPkts));
ui32 = htonl(ui32);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_BYTESMASK) {
ui32 = *((uint32_t*)(rec+fo.dOctets));
ui32 = htonl(ui32);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_IPNEXTHOPMASK) {
ui32 = *((uint32_t*)(rec+fo.nexthop));
ui32 = htonl(ui32);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_STARTTIMEMASK) {
sysUpTime = *((uint32_t*)(rec+fo.sysUpTime));
unix_secs = *((uint32_t*)(rec+fo.unix_secs));
unix_nsecs = *((uint32_t*)(rec+fo.unix_nsecs));
First = *((uint32_t*)(rec+fo.First));
ftt = ftltime(sysUpTime, unix_secs, unix_nsecs, First);
ui32 = htonl(ftt.secs);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_ENDTIMEMASK) {
sysUpTime = *((uint32_t*)(rec+fo.sysUpTime));
unix_secs = *((uint32_t*)(rec+fo.unix_secs));
unix_nsecs = *((uint32_t*)(rec+fo.unix_nsecs));
Last = *((uint32_t*)(rec+fo.Last));
ftt = ftltime(sysUpTime, unix_secs, unix_nsecs, Last);
ui32 = htonl(ftt.secs);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_PROTOCOLMASK) {
ui8 = *((uint8_t*)(rec+fo.prot));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
if (opt->cflowd_mask & CF_TOSMASK) {
ui8 = *((uint8_t*)(rec+fo.tos));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
if (opt->cflowd_mask & CF_SRCASMASK) {
ui16 = *((uint16_t*)(rec+fo.src_as));
ui16 = htons(ui16);
fwrite(&ui16, sizeof (ui16), 1, stdout);
}
if (opt->cflowd_mask & CF_DSTASMASK) {
ui16 = *((uint16_t*)(rec+fo.dst_as));
ui16 = htons(ui16);
fwrite(&ui16, sizeof (ui16), 1, stdout);
}
if (opt->cflowd_mask & CF_SRCMASKLENMASK) {
ui8 = *((uint8_t*)(rec+fo.src_mask));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
if (opt->cflowd_mask & CF_DSTMASKLENMASK) {
ui8 = *((uint8_t*)(rec+fo.dst_mask));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
if (opt->cflowd_mask & CF_TCPFLAGSMASK) {
ui8 = *((uint8_t*)(rec+fo.tcp_flags));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
if (opt->cflowd_mask & CF_INPUTENCAPMASK) {
ui8 = *((uint8_t*)(rec+fo.in_encaps));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
if (opt->cflowd_mask & CF_OUTPUTENCAPMASK) {
ui8 = *((uint8_t*)(rec+fo.out_encaps));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
if (opt->cflowd_mask & CF_PEERNEXTHOPMASK) {
ui32 = *((uint32_t*)(rec+fo.peer_nexthop));
ui32 = htonl(ui32);
fwrite(&ui32, sizeof (ui32), 1, stdout);
}
if (opt->cflowd_mask & CF_ENGINETYPEMASK) {
ui8 = *((uint8_t*)(rec+fo.engine_type));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
if (opt->cflowd_mask & CF_ENGINEIDMASK) {
ui8 = *((uint8_t*)(rec+fo.engine_id));
fwrite(&ui8, sizeof (ui8), 1, stdout);
}
++opt->records;
} /* while */
return 0;
} /* format 0 */