void nf9_flowset_to_store(u_int8_t *pkt, size_t len, struct NF9_HEADER *nf9_hdr, netflow9_template_records_map& template_records) {
// Should be done according to https://github.com/FastVPSEestiOu/fastnetmon/issues/147
//if (template->total_len > len)
// return 1;
u_int offset = 0;
simple_packet packet;
// We use shifted values and should process only zeroed values
// because we are working with little and big endian data in same time
packet.number_of_packets = 0;
packet.ts.tv_sec = ntohl(nf9_hdr->time_sec);
packet.sample_ratio = sampling_rate;
// We should iterate over all available template fields
for (netflow9_template_records_map::iterator iter = template_records.begin(); iter != template_records.end(); iter++) {
u_int record_type = iter->type;
u_int record_length = iter->len;
nf9_rec_to_flow(record_type, record_length, pkt + offset, packet);
//logger<< log4cpp::Priority::INFO<<"Read data with type: "<<record_type<<" and length:"<<record_length;
offset += record_length;
}
// decode data in network byte order to host byte order
packet.length = fast_ntoh(packet.length);
packet.number_of_packets = fast_ntoh(packet.number_of_packets);
packet.protocol = fast_ntoh(packet.protocol);
// We should convert ports to host byte order too
packet.source_port = fast_ntoh(packet.source_port);
packet.destination_port = fast_ntoh(packet.destination_port);
// Set protocol
switch (packet.protocol) {
case 1: {
packet.protocol = IPPROTO_ICMP;
packet.source_port = 0;
packet.destination_port = 0;
}
break;
case 6: {
packet.protocol = IPPROTO_TCP;
}
break;
case 17: {
packet.protocol = IPPROTO_UDP;
}
break;
}
// pass data to FastNetMon
netflow_process_func_ptr(packet);
}
void process_netflow_packet_v5(u_int8_t *packet, u_int len) {
//logger<< log4cpp::Priority::INFO<<"We get v5 netflow packet!";
struct NF5_HEADER* nf5_hdr = (struct NF5_HEADER*)packet;
if (len < sizeof(*nf5_hdr)) {
logger<< log4cpp::Priority::ERROR<<"Short netflow v5 packet "<<len;
return;
}
u_int nflows = ntohs(nf5_hdr->c.flows);
if (nflows == 0 || nflows > NF5_MAXFLOWS) {
logger<< log4cpp::Priority::ERROR<<"Invalid number of flows in netflow "<<nflows;
return;
}
for (u_int i = 0; i < nflows; i++) {
size_t offset = NF5_PACKET_SIZE(i);
struct NF5_FLOW* nf5_flow = (struct NF5_FLOW *)(packet + offset);
/* Check packet bounds */
if (offset + sizeof(struct NF5_FLOW) > len) {
logger<< log4cpp::Priority::ERROR<<"Error! You will try to read outside the packet";
}
// convert netflow to simple packet form
simple_packet current_packet;
current_packet.src_ip = nf5_flow->src_ip;
current_packet.dst_ip = nf5_flow->dest_ip;
current_packet.ts.tv_sec = ntohl(nf5_hdr->time_sec);
current_packet.ts.tv_usec = ntohl(nf5_hdr->time_nanosec);
current_packet.flags = 0;
current_packet.source_port = 0;
current_packet.destination_port = 0;
// TODO: we should pass data about "flow" structure of this data
// htobe64 removed
current_packet.length = fast_ntoh(nf5_flow->flow_octets);
current_packet.number_of_packets = fast_ntoh(nf5_flow->flow_packets);
current_packet.sample_ratio = sampling_rate;
current_packet.source_port = fast_ntoh(nf5_flow->src_port);
current_packet.destination_port = fast_ntoh(nf5_flow->dest_port);
switch (nf5_flow->protocol) {
case 1: {
//ICMP
current_packet.protocol = IPPROTO_ICMP;
}
break;
case 6: {
// TCP
current_packet.protocol = IPPROTO_TCP;
// TODO: flags can be in another format!
current_packet.flags = nf5_flow->tcp_flags;
}
break;
case 17: {
// UDP
current_packet.protocol = IPPROTO_UDP;
}
break;
}
// Call processing function for every flow in packet
netflow_process_func_ptr(current_packet);
}
}
void process_netflow_packet_v5(u_int8_t* packet, u_int len, std::string client_addres_in_string_format) {
// logger<< log4cpp::Priority::INFO<<"We get v5 netflow packet!";
struct NF5_HEADER* nf5_hdr = (struct NF5_HEADER*)packet;
if (len < sizeof(*nf5_hdr)) {
logger << log4cpp::Priority::ERROR << "Short netflow v5 packet " << len;
return;
}
u_int nflows = ntohs(nf5_hdr->c.flows);
if (nflows == 0 || nflows > NF5_MAXFLOWS) {
logger << log4cpp::Priority::ERROR << "Invalid number of flows in netflow " << nflows;
return;
}
uint16_t netflow5_sampling_ratio = fast_ntoh(nf5_hdr->sampling_rate);
// In first two bits we store sampling type.
// We are not interested in it and should zeroify it for getting correct value of sampling rate
clear_bit_value(netflow5_sampling_ratio, 15);
clear_bit_value(netflow5_sampling_ratio, 16);
// Sampling not enabled on device
if (netflow5_sampling_ratio == 0) {
netflow5_sampling_ratio = 1;
}
for (u_int i = 0; i < nflows; i++) {
size_t offset = NF5_PACKET_SIZE(i);
struct NF5_FLOW* nf5_flow = (struct NF5_FLOW*)(packet + offset);
/* Check packet bounds */
if (offset + sizeof(struct NF5_FLOW) > len) {
logger << log4cpp::Priority::ERROR << "Error! You will try to read outside the packet";
}
/* Decode to host encoding */
// TODO: move to separate function
nf5_flow->flow_octets = fast_ntoh(nf5_flow->flow_octets);
nf5_flow->flow_packets = fast_ntoh(nf5_flow->flow_packets);
nf5_flow->if_index_in = fast_ntoh(nf5_flow->if_index_in);
nf5_flow->if_index_out = fast_ntoh(nf5_flow->if_index_out);
// convert netflow to simple packet form
simple_packet current_packet;
current_packet.src_ip = nf5_flow->src_ip;
current_packet.dst_ip = nf5_flow->dest_ip;
current_packet.ts.tv_sec = ntohl(nf5_hdr->time_sec);
current_packet.ts.tv_usec = ntohl(nf5_hdr->time_nanosec);
current_packet.flags = 0;
current_packet.source_port = 0;
current_packet.destination_port = 0;
// TODO: we should pass data about "flow" structure of this data
current_packet.length = nf5_flow->flow_octets;
current_packet.number_of_packets = nf5_flow->flow_packets;
if (netflow_divide_counters_on_interval_length) {
// This interval in milliseconds, convert it to seconds
int64_t interval_length = (fast_ntoh(nf5_flow->flow_finish) - fast_ntoh(nf5_flow->flow_start)) / 1000;
/*
if (interval_length > 0) {
logger << log4cpp::Priority::INFO << "NetFlow v5 from: " << client_addres_in_string_format
<< " start: " << fast_ntoh(nf5_flow->flow_start)
<< " finish: " << fast_ntoh(nf5_flow->flow_finish)
<< " interval length:" << interval_length
<< "\n";
}
*/
if (interval_length == 0) {
// it's OK
} else if (interval_length < 0) {
// it's internal error
logger << log4cpp::Priority::ERROR << "We got negative interval length from netflow agent, something goes wrong!";
} else {
// OK, let's divide
// We will get integer result for this operation
current_packet.length = current_packet.length / interval_length;
current_packet.number_of_packets = current_packet.number_of_packets / interval_length;
}
}
// TODO: use sampling data from packet, disable customization here
// Wireshark dump approves this idea
current_packet.sample_ratio = netflow5_sampling_ratio;
current_packet.source_port = fast_ntoh(nf5_flow->src_port);
current_packet.destination_port = fast_ntoh(nf5_flow->dest_port);
// We do not support IPv6 in NetFlow v5 at all
current_packet.ip_protocol_version = 4;
switch (nf5_flow->protocol) {
case 1: {
// ICMP
current_packet.protocol = IPPROTO_ICMP;
} break;
case 6: {
// TCP
current_packet.protocol = IPPROTO_TCP;
// TODO: flags can be in another format!
current_packet.flags = nf5_flow->tcp_flags;
} break;
case 17: {
// UDP
current_packet.protocol = IPPROTO_UDP;
} break;
}
#ifdef ENABLE_LUA_HOOKS
if (lua_hooks_enabled) {
// This code could be used only for tests with pcap_reader
//if (lua_state == NULL) {
// init_lua_jit();
//}
if (call_lua_function("process_netflow", netflow_lua_state, client_addres_in_string_format, (void*)nf5_flow)) {
// We will process this packet
} else {
logger << log4cpp::Priority::INFO << "We will drop this packets because LUA script decided to do it";
return;
}
}
#endif
// Call processing function for every flow in packet
netflow_process_func_ptr(current_packet);
}
}
// We should rewrite nf9_flowset_to_store accroding to fixes here
void nf10_flowset_to_store(u_int8_t *pkt, size_t len, struct NF10_HEADER *nf10_hdr, struct peer_nf9_template* field_template) {
u_int offset = 0;
if (len < field_template->total_len) {
logger<< log4cpp::Priority::ERROR<<"Total len from template bigger than packet len";
return;
}
simple_packet packet;
// We use shifted values and should process only zeroed values
// because we are working with little and big endian data in same time
packet.number_of_packets = 0;
packet.ts.tv_sec = ntohl(nf10_hdr->time_sec);
packet.sample_ratio = sampling_rate;
netflow_ipfix_struct data_in_ipfix_format;
memset(&data_in_ipfix_format, sizeof(netflow_ipfix_struct), 0);
for (netflow9_template_records_map::iterator iter = field_template->records.begin(); iter != field_template->records.end(); iter++) {
u_int record_type = iter->type;
u_int record_length = iter->len;
nf10_rec_to_flow(record_type, record_length, pkt + offset, packet);
// New code
/*
unsigned int field_id = record_type;
std::string field_name = ipfix_db_instance.get_name_by_id(field_id);
if (field_name == "octetDeltaCount") {
unsigned int reference_field_length = sizeof(data_in_ipfix_format.octetDeltaCount);
if (reference_field_length == record_length) {
// We use standard copy
memcpy(&data_in_ipfix_format.octetDeltaCount, pkt + offset, record_length);
// Convert to host byte order
data_in_ipfix_format.octetDeltaCount = fast_ntoh(data_in_ipfix_format.octetDeltaCount);
} else if (record_length < reference_field_length) {
logger<< log4cpp::Priority::ERROR<<"We can't copy data because magic memcpy is not implemented yet";
// We use copy memcpy for netfowrk byte order
} else {
// Holy cow! It's impossible!
logger<< log4cpp::Priority::ERROR<<"We can't copy data because receiver data is bigger than our storage.";
return;
}
logger<< log4cpp::Priority::INFO<<"We received packet size with new parser: "<<data_in_ipfix_format.octetDeltaCount;
}
*/
offset += record_length;
}
// decode data in network byte order to host byte order
packet.length = fast_ntoh(packet.length);
packet.number_of_packets = fast_ntoh(packet.number_of_packets);
packet.protocol = fast_ntoh(packet.protocol);
// We should convert ports to host byte order too
packet.source_port = fast_ntoh(packet.source_port);
packet.destination_port = fast_ntoh(packet.destination_port);
// Set protocol
switch (packet.protocol) {
case 1: {
packet.protocol = IPPROTO_ICMP;
packet.source_port = 0;
packet.destination_port = 0;
}
break;
case 6: {
packet.protocol = IPPROTO_TCP;
}
break;
case 17: {
packet.protocol = IPPROTO_UDP;
}
break;
}
// pass data to FastNetMon
netflow_process_func_ptr(packet);
}
int nf9_rec_to_flow(u_int record_type, u_int record_length, u_int8_t* data, simple_packet& packet) {
/* XXX: use a table-based interpreter */
switch (record_type) {
V9_FIELD(NF9_IN_BYTES, OCTETS, length);
V9_FIELD(NF9_IN_PACKETS, PACKETS, number_of_packets);
V9_FIELD(NF9_IN_PROTOCOL, PROTO_FLAGS_TOS, protocol);
V9_FIELD(NF9_TCP_FLAGS, PROTO_FLAGS_TOS, flags);
V9_FIELD(NF9_L4_SRC_PORT, SRCDST_PORT, source_port);
V9_FIELD(NF9_L4_DST_PORT, SRCDST_PORT, destination_port);
case NF9_IPV4_SRC_ADDR:
memcpy(&packet.src_ip, data, record_length);
break;
case NF9_IPV4_DST_ADDR:
memcpy(&packet.dst_ip, data, record_length);
break;
case NF9_INPUT_SNMP: {
// TODO: port number could be 4 byte (Juniper MX) and we should rewrite BE_COPY for correct handling
uint16_t input_port = 0;
if (record_length > sizeof(input_port)) {
//logger << log4cpp::Priority::ERROR << "Received very big packet for NF9_INPUT_SNMP!";
//return 0;
} else {
BE_COPY(input_port);
input_port = fast_ntoh(input_port);
// logger << log4cpp::Priority::INFO << "NF9_INPUT_SNMP is: " << input_port;
}
}
break;
case NF9_OUTPUT_SNMP: {
uint16_t output_port = 0;
if (record_length > sizeof(output_port)) {
//logger << log4cpp::Priority::ERROR << "Received very big packet for NF9_OUTPUT_SNMP!";
//return 0;
} else {
BE_COPY(output_port);
output_port = fast_ntoh(output_port);
// logger << log4cpp::Priority::INFO << "NF9_OUTPUT_SNMP is: " << output_port;
}
}
break;
//V9_FIELD_ADDR(NF9_IPV4_SRC_ADDR, SRC_ADDR4, src_ip);
//V9_FIELD_ADDR(NF9_IPV4_DST_ADDR, DST_ADDR4, dst_ip);
// Sampling rate
// We use NULL as second argument because it's suelles for us
// It did not help us because looks like sampling rate implemented with OPTIONS flowset
// V9_FIELD(NF9_SAMPLING_INTERVAL, NULL, sample_ratio);
// V9_FIELD(NF9_SRC_TOS, PROTO_FLAGS_TOS, pft.tos);
// V9_FIELD(NF9_SRC_MASK, AS_INFO, asinf.src_mask);
// V9_FIELD(NF9_INPUT_SNMP, IF_INDICES, ifndx.if_index_in);
// V9_FIELD(NF9_DST_MASK, AS_INFO, asinf.dst_mask);
// V9_FIELD(NF9_OUTPUT_SNMP, IF_INDICES, ifndx.if_index_out);
// V9_FIELD(NF9_SRC_AS, AS_INFO, asinf.src_as);
// V9_FIELD(NF9_DST_AS, AS_INFO, asinf.dst_as);
// V9_FIELD(NF9_LAST_SWITCHED, FLOW_TIMES, ftimes.flow_finish);
// V9_FIELD(NF9_FIRST_SWITCHED, FLOW_TIMES, ftimes.flow_start);
// V9_FIELD(NF9_IPV6_SRC_MASK, AS_INFO, asinf.src_mask);
// V9_FIELD(NF9_IPV6_DST_MASK, AS_INFO, asinf.dst_mask);
// V9_FIELD(NF9_ENGINE_TYPE, FLOW_ENGINE_INFO, finf.engine_type);
// V9_FIELD(NF9_ENGINE_ID, FLOW_ENGINE_INFO, finf.engine_id);
// V9_FIELD_ADDR(NF9_IPV4_NEXT_HOP, GATEWAY_ADDR4, gateway_addr, 4, INET);
// V9_FIELD_ADDR(NF9_IPV6_SRC_ADDR, SRC_ADDR6, src_addr, 6, INET6);
// V9_FIELD_ADDR(NF9_IPV6_DST_ADDR, DST_ADDR6, dst_addr, 6, INET6);
// V9_FIELD_ADDR(NF9_IPV6_NEXT_HOP, GATEWAY_ADDR6, gateway_addr, 6, INET6);
//#undef V9_FIELD
//#undef V9_FIELD_ADDR
//#undef BE_COPY
}
return 0;
}
