void NSDFWriter::reinit(const Eref& eref, const ProcPtr proc)
{
// write environment
// write model
// write map
if (filehandle_ >0){
close();
}
// TODO: what to do when reinit is called? Close the existing file
// and open a new one in append mode? Or keep adding to the
// current file?
if (filename_.empty()){
filename_ = "moose_data.nsdf.h5";
}
openFile();
writeScalarAttr<string>(filehandle_, "created", iso_time(0));
writeScalarAttr<string>(filehandle_, "tstart", iso_time(0));
writeScalarAttr<string>(filehandle_, "nsdf_version", "1.0");
openUniformData(eref);
for (map < string, hid_t >::iterator it = classFieldToUniform_.begin();
it != classFieldToUniform_.end();
++it){
// tstart is reset to 0 on reinit
writeScalarAttr< double >(it->second, "tstart", 0.0);
// dt is same for all requested data - that of the NSDFWriter
writeScalarAttr< double >(it->second, "dt", proc->dt);
}
openEventData(eref);
writeModelTree();
createUniformMap();
createEventMap();
steps_ = 0;
}
void NSDFWriter::flush()
{
// We need to update the tend on each write since we do not know
// when the simulation is getting over and when it is just paused.
writeScalarAttr<string>(filehandle_, "tend", iso_time(NULL));
// append all uniform data
for (map< string, hid_t>::iterator it = classFieldToUniform_.begin();
it != classFieldToUniform_.end(); ++it){
map< string, vector < unsigned int > >::iterator idxit = classFieldToSrcIndex_.find(it->first);
if (idxit == classFieldToSrcIndex_.end()){
cerr << "Error: NSDFWriter::flush - could not find entry for " << it->first <<endl;
break;
}
if (data_.size() == 0 || data_[0].size() == 0){
break;
}
double * buffer = (double*)calloc(idxit->second.size() * steps_, sizeof(double));
vector< double > values;
for (unsigned int ii = 0; ii < idxit->second.size(); ++ii){
for (unsigned int jj = 0; jj < steps_; ++jj){
buffer[ii * steps_ + jj] = data_[idxit->second[ii]][jj];
}
data_[idxit->second[ii]].clear();
}
hid_t filespace = H5Dget_space(it->second);
if (filespace < 0){
break;
}
hsize_t dims[2];
hsize_t maxdims[2];
// retrieve current datset dimensions
herr_t status = H5Sget_simple_extent_dims(filespace, dims, maxdims);
hsize_t newdims[] = {dims[0], dims[1] + steps_}; // new column count
status = H5Dset_extent(it->second, newdims); // extend dataset to new column count
H5Sclose(filespace);
filespace = H5Dget_space(it->second); // get the updated filespace
hsize_t start[2] = {0, dims[1]};
dims[1] = steps_; // change dims for memspace & hyperslab
hid_t memspace = H5Screate_simple(2, dims, NULL);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, dims, NULL);
status = H5Dwrite(it->second, H5T_NATIVE_DOUBLE, memspace, filespace, H5P_DEFAULT, buffer);
H5Sclose(memspace);
H5Sclose(filespace);
free(buffer);
}
// append all event data
for (unsigned int ii = 0; ii < eventSrc_.size(); ++ii){
appendToDataset(getEventDataset(eventSrc_[ii], eventSrcFields_[ii]),
events_[ii]);
events_[ii].clear();
}
// flush HDF5 nodes.
HDF5DataWriter::flush();
}
static PyObject *
flow_iso_time(PyObject *self, PyObject *args, PyObject *kw_args)
{
static char *keywords[] = { "time", "utc_flag", NULL };
int utc_flag = 0;
long when;
if (!PyArg_ParseTupleAndKeywords(args, kw_args, "l|i:iso_time",
keywords, &when, &utc_flag))
return NULL;
return (PyObject *)PyString_FromString(iso_time(when, utc_flag));
}
/*---------------------------------------------------------------------------*/
static void isotime(char *str)
{
uint8_t timestore_read[7];
uint8_t iso_timestore[16];
read_time(timestore_read);
iso_time(timestore_read, iso_timestore);
/* iso_timestore[0] = 'a' + timestore_read[0];
iso_timestore[1] = 'a' + timestore_read[1];
iso_timestore[2] = 'a' + timestore_read[2];
iso_timestore[3] = 'a' + timestore_read[3];
iso_timestore[4] = '\0';
*/
shell_output(iso_timestore, "");
}
char* event_string() {
char *event_json;
asprintf(&event_json,"{\"Event\":{\"occurrenceTime\":\"%s\",\"hostname\":\"domain.localhost.cz\",\"type\":\"org.linux.cron.Started\",\"application\":\"Cron\",\"process\":\"proc_cron NAme\",\"processId\":\"id005\",\"severity\":5,\"priority\":4,\"Payload\":{\"schema\":null,\"schemaVersion\":null,\"value\":4648,\"value2\":\"aax4x46aeEF\"}}}\n", iso_time());
return event_json;
}
void netflow_format_flow(struct store_flow_complete* flow, char* buf, size_t len,
int utc_flag, u_int32_t display_mask, int hostorder) {
char tmp[256];
u_int32_t fields;
u_int64_t (*fmt_ntoh64)(u_int64_t) = netflow_swp_ntoh64;
u_int32_t (*fmt_ntoh32)(u_int32_t) = netflow_swp_ntoh32;
u_int16_t (*fmt_ntoh16)(u_int16_t) = netflow_swp_ntoh16;
if (hostorder) {
fmt_ntoh64 = netflow_swp_fake64;
fmt_ntoh32 = netflow_swp_fake32;
fmt_ntoh16 = netflow_swp_fake16;
}
*buf = '\0';
fields = fmt_ntoh32(flow->hdr.fields) & display_mask;
strlcat(buf, "FLOW ", len);
if (SHASFIELD(TAG)) {
snprintf(tmp, sizeof(tmp), "tag %u ", fmt_ntoh32(flow->tag.tag));
strlcat(buf, tmp, len);
}
if (SHASFIELD(RECV_TIME)) {
snprintf(tmp, sizeof(tmp), "recv_time %s.%05d ",
iso_time(fmt_ntoh32(flow->recv_time.recv_sec), utc_flag),
fmt_ntoh32(flow->recv_time.recv_usec));
strlcat(buf, tmp, len);
}
if (SHASFIELD(PROTO_FLAGS_TOS)) {
snprintf(tmp, sizeof(tmp), "proto %d ", flow->pft.protocol);
strlcat(buf, tmp, len);
snprintf(tmp, sizeof(tmp), "tcpflags %02x ",
flow->pft.tcp_flags);
strlcat(buf, tmp, len);
snprintf(tmp, sizeof(tmp), "tos %02x " , flow->pft.tos);
strlcat(buf, tmp, len);
}
if (SHASFIELD(AGENT_ADDR4) || SHASFIELD(AGENT_ADDR6)) {
snprintf(tmp, sizeof(tmp), "agent [%s] ",
addr_ntop_buf(&flow->agent_addr));
strlcat(buf, tmp, len);
}
if (SHASFIELD(SRC_ADDR4) || SHASFIELD(SRC_ADDR6)) {
snprintf(tmp, sizeof(tmp), "src [%s]",
addr_ntop_buf(&flow->src_addr));
strlcat(buf, tmp, len);
if (SHASFIELD(SRCDST_PORT)) {
snprintf(tmp, sizeof(tmp), ":%d",
fmt_ntoh16(flow->ports.src_port));
strlcat(buf, tmp, len);
}
strlcat(buf, " ", len);
}
if (SHASFIELD(DST_ADDR4) || SHASFIELD(DST_ADDR6)) {
snprintf(tmp, sizeof(tmp), "dst [%s]",
addr_ntop_buf(&flow->dst_addr));
strlcat(buf, tmp, len);
if (SHASFIELD(SRCDST_PORT)) {
snprintf(tmp, sizeof(tmp), ":%d",
fmt_ntoh16(flow->ports.dst_port));
strlcat(buf, tmp, len);
}
strlcat(buf, " ", len);
}
if (SHASFIELD(GATEWAY_ADDR4) || SHASFIELD(GATEWAY_ADDR6)) {
snprintf(tmp, sizeof(tmp), "gateway [%s] ",
addr_ntop_buf(&flow->gateway_addr));
strlcat(buf, tmp, len);
}
if (SHASFIELD(PACKETS)) {
snprintf(tmp, sizeof(tmp), "packets %lu ",
fmt_ntoh64(flow->packets.flow_packets));
strlcat(buf, tmp, len);
}
if (SHASFIELD(OCTETS)) {
snprintf(tmp, sizeof(tmp), "octets %lu ",
fmt_ntoh64(flow->octets.flow_octets));
strlcat(buf, tmp, len);
}
if (SHASFIELD(IF_INDICES)) {
snprintf(tmp, sizeof(tmp), "in_if %d out_if %d ",
fmt_ntoh32(flow->ifndx.if_index_in),
fmt_ntoh32(flow->ifndx.if_index_out));
strlcat(buf, tmp, len);
}
if (SHASFIELD(AGENT_INFO)) {
snprintf(tmp, sizeof(tmp), "sys_uptime_ms %s.%03u ",
interval_time(fmt_ntoh32(flow->ainfo.sys_uptime_ms) / 1000),
fmt_ntoh32(flow->ainfo.sys_uptime_ms) % 1000);
strlcat(buf, tmp, len);
snprintf(tmp, sizeof(tmp), "time_sec %s ",
iso_time(fmt_ntoh32(flow->ainfo.time_sec), utc_flag));
strlcat(buf, tmp, len);
snprintf(tmp, sizeof(tmp), "time_nanosec %lu netflow ver %u ",
(u_long)fmt_ntoh32(flow->ainfo.time_nanosec),
fmt_ntoh16(flow->ainfo.netflow_version));
strlcat(buf, tmp, len);
}
if (SHASFIELD(FLOW_TIMES)) {
snprintf(tmp, sizeof(tmp), "flow_start %s.%03u ",
interval_time(fmt_ntoh32(flow->ftimes.flow_start) / 1000),
fmt_ntoh32(flow->ftimes.flow_start) % 1000);
strlcat(buf, tmp, len);
snprintf(tmp, sizeof(tmp), "flow_finish %s.%03u ",
interval_time(fmt_ntoh32(flow->ftimes.flow_finish) / 1000),
fmt_ntoh32(flow->ftimes.flow_finish) % 1000);
strlcat(buf, tmp, len);
}
if (SHASFIELD(AS_INFO)) {
snprintf(tmp, sizeof(tmp), "src_AS %u src_masklen %u ",
fmt_ntoh32(flow->asinf.src_as), flow->asinf.src_mask);
strlcat(buf, tmp, len);
snprintf(tmp, sizeof(tmp), "dst_AS %u dst_masklen %u ",
fmt_ntoh32(flow->asinf.dst_as), flow->asinf.dst_mask);
strlcat(buf, tmp, len);
}
if (SHASFIELD(FLOW_ENGINE_INFO)) {
snprintf(tmp, sizeof(tmp),
"engine_type %u engine_id %u seq %lu source %lu ",
fmt_ntoh16(flow->finf.engine_type),
fmt_ntoh16(flow->finf.engine_id),
(u_long)fmt_ntoh32(flow->finf.flow_sequence),
(u_long)fmt_ntoh32(flow->finf.source_id));
strlcat(buf, tmp, len);
}
if (SHASFIELD(CRC32)) {
snprintf(tmp, sizeof(tmp), "crc32 %08x ",
fmt_ntoh32(flow->crc32.crc32));
strlcat(buf, tmp, len);
}
}
uint8_t* ss_metadata_prepare_netflow(
const char* source, nn_queue_t* nn_queue,
struct store_flow_complete* flow,
ss_ioc_entry_t* ioc_entry) {
json_object* jobject = NULL;
json_object* item = NULL;
uint8_t* rv = NULL;
uint8_t* jstring = NULL;
uint32_t fields;
uint64_t (*fmt_ntoh64)(uint64_t) = netflow_swp_ntoh64;
uint32_t (*fmt_ntoh32)(uint32_t) = netflow_swp_ntoh32;
uint16_t (*fmt_ntoh16)(uint16_t) = netflow_swp_ntoh16;
uint32_t display_mask = STORE_DISPLAY_ALL;
fields = fmt_ntoh32(flow->hdr.fields) & display_mask;
int utc_flag = 0;
jobject = json_object_new_object();
if (jobject == NULL) {
RTE_LOG(ERR, EXTRACTOR, "could not allocate netflow json object\n");
goto error_out;
}
if (SHASFIELD(TAG)) {
item = json_object_new_int(fmt_ntoh32(flow->tag.tag));
json_object_object_add(jobject, "tag", item);
}
if (SHASFIELD(RECV_TIME)) {
item = json_object_new_string(iso_time(fmt_ntoh32(flow->recv_time.recv_sec), utc_flag));
json_object_object_add(jobject, "flow_sec", item);
item = json_object_new_int(fmt_ntoh32(flow->recv_time.recv_usec));
json_object_object_add(jobject, "flow_usec", item);
}
if (SHASFIELD(PROTO_FLAGS_TOS)) {
item = json_object_new_int(flow->pft.protocol);
json_object_object_add(jobject, "ip_protocol", item);
item = json_object_new_int(flow->pft.tcp_flags);
json_object_object_add(jobject, "tcp_flags", item);
item = json_object_new_int(flow->pft.tos);
json_object_object_add(jobject, "ip_tos", item);
}
if (SHASFIELD(AGENT_ADDR4) || SHASFIELD(AGENT_ADDR6)) {
item = json_object_new_string(addr_ntop_buf(&flow->agent_addr));
json_object_object_add(jobject, "agentip", item);
}
if (SHASFIELD(SRC_ADDR4) || SHASFIELD(SRC_ADDR6)) {
item = json_object_new_string(addr_ntop_buf(&flow->src_addr));
json_object_object_add(jobject, "sip", item);
item = json_object_new_int(fmt_ntoh16(flow->ports.src_port));
json_object_object_add(jobject, "sport", item);
}
if (SHASFIELD(DST_ADDR4) || SHASFIELD(DST_ADDR6)) {
item = json_object_new_string(addr_ntop_buf(&flow->dst_addr));
json_object_object_add(jobject, "dip", item);
item = json_object_new_int(fmt_ntoh16(flow->ports.dst_port));
json_object_object_add(jobject, "dport", item);
}
if (SHASFIELD(GATEWAY_ADDR4) || SHASFIELD(GATEWAY_ADDR6)) {
item = json_object_new_string(addr_ntop_buf(&flow->gateway_addr));
json_object_object_add(jobject, "gatewayip", item);
}
if (SHASFIELD(PACKETS)) {
item = json_object_new_int(fmt_ntoh64(flow->packets.flow_packets));
json_object_object_add(jobject, "packets", item);
}
if (SHASFIELD(OCTETS)) {
item = json_object_new_int(fmt_ntoh64(flow->octets.flow_octets));
json_object_object_add(jobject, "bytes", item);
}
if (SHASFIELD(IF_INDICES)) {
item = json_object_new_int(fmt_ntoh32(flow->ifndx.if_index_in));
json_object_object_add(jobject, "sifindex", item);
item = json_object_new_int(fmt_ntoh32(flow->ifndx.if_index_out));
json_object_object_add(jobject, "difindex", item);
}
if (SHASFIELD(AGENT_INFO)) {
item = json_object_new_string(interval_time(fmt_ntoh32(flow->ainfo.sys_uptime_ms) / 1000));
json_object_object_add(jobject, "sys_uptime_sec", item);
item = json_object_new_int(fmt_ntoh32(flow->ainfo.sys_uptime_ms) % 1000);
json_object_object_add(jobject, "sys_uptime_msec", item);
item = json_object_new_string(iso_time(fmt_ntoh32(flow->ainfo.time_sec), utc_flag));
json_object_object_add(jobject, "sys_time_sec", item);
item = json_object_new_int((u_long)fmt_ntoh32(flow->ainfo.time_nanosec));
json_object_object_add(jobject, "sys_time_nsec", item);
item = json_object_new_int(fmt_ntoh16(flow->ainfo.netflow_version));
json_object_object_add(jobject, "netflow_version", item);
}
if (SHASFIELD(FLOW_TIMES)) {
item = json_object_new_string(interval_time(fmt_ntoh32(flow->ftimes.flow_start) / 1000));
json_object_object_add(jobject, "flow_start_sec", item);
item = json_object_new_int(fmt_ntoh32(flow->ftimes.flow_start) % 1000);
json_object_object_add(jobject, "flow_start_msec", item);
item = json_object_new_string(interval_time(fmt_ntoh32(flow->ftimes.flow_finish) / 1000));
json_object_object_add(jobject, "flow_stop_sec", item);
item = json_object_new_int(fmt_ntoh32(flow->ftimes.flow_finish) % 1000);
json_object_object_add(jobject, "flow_stop_msec", item);
}
if (SHASFIELD(AS_INFO)) {
item = json_object_new_int(fmt_ntoh32(flow->asinf.src_as));
json_object_object_add(jobject, "src_as", item);
item = json_object_new_int(fmt_ntoh32(flow->asinf.src_mask));
json_object_object_add(jobject, "src_masklen", item);
item = json_object_new_int(fmt_ntoh32(flow->asinf.dst_as));
json_object_object_add(jobject, "dst_as", item);
item = json_object_new_int(fmt_ntoh32(flow->asinf.dst_mask));
json_object_object_add(jobject, "dst_masklen", item);
}
if (SHASFIELD(FLOW_ENGINE_INFO)) {
item = json_object_new_int(fmt_ntoh16(flow->finf.engine_type));
json_object_object_add(jobject, "engine_type", item);
item = json_object_new_int(fmt_ntoh16(flow->finf.engine_id));
json_object_object_add(jobject, "engine_id", item);
item = json_object_new_int((u_long)fmt_ntoh32(flow->finf.flow_sequence));
json_object_object_add(jobject, "seq_num", item);
item = json_object_new_int((u_long)fmt_ntoh32(flow->finf.source_id));
json_object_object_add(jobject, "source_id", item);
}
if (SHASFIELD(CRC32)) {
item = json_object_new_int(fmt_ntoh32(flow->crc32.crc32));
json_object_object_add(jobject, "crc32", item);
}
item = NULL;
rv = 0;
// XXX: NOTE: String pointer is internal to JSON object.
jstring = (uint8_t*) json_object_to_json_string_ext(jobject, JSON_C_TO_STRING_SPACED);
rv = (uint8_t*) je_strdup((char*)jstring);
if (!rv) goto error_out;
json_object_put(jobject); jobject = NULL;
return rv;
error_out:
fprintf(stderr, "could not serialize packet metadata\n");
if (rv) { je_free(rv); rv = NULL; }
if (jobject) { json_object_put(jobject); jobject = NULL; }
if (item) { json_object_put(item); item = NULL; }
return NULL;
}
int
main(int argc, char **argv)
{
int ch, i, fd, utc, r, verbose, debug, csv;
extern char *optarg;
extern int optind;
struct store_flow_complete flow;
struct store_v2_flow_complete flow_v2;
char buf[2048], ebuf[512];
const char *ffile, *ofile;
FILE *ffilef;
int ofd, read_legacy, head, nflows;
u_int32_t disp_mask;
struct flowd_config filter_config;
struct store_v2_header hdr_v2;
utc = verbose = debug = read_legacy = csv = 0;
ofile = ffile = NULL;
ofd = -1;
ffilef = NULL;
head = 0;
bzero(&filter_config, sizeof(filter_config));
while ((ch = getopt(argc, argv, "H:LUdf:ho:qvc")) != -1) {
switch (ch) {
case 'h':
usage();
return (0);
case 'H':
if ((head = atoi(optarg)) <= 0) {
fprintf(stderr, "Invalid -H value.\n");
usage();
exit(1);
}
break;
case 'L':
read_legacy = 1;
break;
case 'U':
utc = 1;
break;
case 'd':
debug = 1;
filter_config.opts |= FLOWD_OPT_VERBOSE;
break;
case 'f':
ffile = optarg;
break;
case 'o':
ofile = optarg;
break;
case 'q':
verbose = -1;
break;
case 'v':
verbose = 1;
break;
case 'c':
csv = 1;
break;
default:
usage();
exit(1);
}
}
loginit(PROGNAME, 1, debug);
if (argc - optind < 1) {
fprintf(stderr, "No logfile specified\n");
usage();
exit(1);
}
if (ffile != NULL) {
if ((ffilef = fopen(ffile, "r")) == NULL)
logerr("fopen(%s)", ffile);
if (parse_config(ffile, ffilef, &filter_config, 1) != 0)
exit(1);
fclose(ffilef);
}
if (ofile != NULL) {
if (strcmp(ofile, "-") == 0) {
if (!debug)
verbose = -1;
ofile = NULL;
if (isatty(STDOUT_FILENO))
logerrx("Refusing to write binary flow data to "
"standard output.");
}
ofd = open_start_log(ofile, debug);
}
if (filter_config.store_mask == 0)
filter_config.store_mask = STORE_FIELD_ALL;
disp_mask = (verbose > 0) ? STORE_DISPLAY_ALL: STORE_DISPLAY_BRIEF;
disp_mask &= filter_config.store_mask;
for (i = optind; i < argc; i++) {
if (strcmp(argv[i], "-") == 0)
fd = STDIN_FILENO;
else if ((fd = open(argv[i], O_RDONLY)) == -1)
logerr("open(%s)", argv[i]);
if (read_legacy && store_v2_get_header(fd, &hdr_v2, ebuf,
sizeof(ebuf)) != STORE_ERR_OK)
logerrx("%s", ebuf);
if (verbose >= 1) {
printf("LOGFILE %s", argv[i]);
if (read_legacy)
printf(" started at %s",
iso_time(ntohl(hdr_v2.start_time), utc));
printf("\n");
fflush(stdout);
}
if (csv == 1) {
csv++;
printf("#:unix_secs,unix_nsecs,sysuptime,exaddr,"
"dpkts,doctets,first,last,engine_type,engine_id,"
"srcaddr,dstaddr,nexthop,input,output,srcport,"
"dstport,prot,tos,tcp_flags,src_mask,dst_mask,"
"src_as,dst_as\n");
}
for (nflows = 0; head == 0 || nflows < head; nflows++) {
bzero(&flow, sizeof(flow));
if (read_legacy)
r = store_v2_get_flow(fd, &flow_v2, ebuf,
sizeof(ebuf));
else
r = store_get_flow(fd, &flow, ebuf,
sizeof(ebuf));
if (r == STORE_ERR_EOF)
break;
else if (r != STORE_ERR_OK)
logerrx("%s", ebuf);
if (read_legacy &&
store_v2_flow_convert(&flow_v2, &flow) == -1)
logerrx("legacy flow conversion failed");
if (ffile != NULL && filter_flow(&flow,
&filter_config.filter_list) == FF_ACTION_DISCARD)
continue;
if (csv) {
store_format_flow_flowtools_csv(&flow, buf,
sizeof(buf), utc, disp_mask, 0);
printf("%s\n", buf);
}
else if (verbose >= 0) {
store_format_flow(&flow, buf, sizeof(buf),
utc, disp_mask, 0);
printf("%s\n", buf);
fflush(stdout);
}
if (ofd != -1 && store_put_flow(ofd, &flow,
filter_config.store_mask, ebuf,
sizeof(ebuf)) == -1)
logerrx("%s", ebuf);
}
if (fd != STDIN_FILENO)
close(fd);
}
if (ofd != -1)
close(ofd);
if (ffile != NULL && debug)
dump_config(&filter_config, "final", 1);
return (0);
}
