static ndpi_protocol nDPIPacket(packet *pkt, struct ndpi_flow_struct *l7flow, struct ndpi_id_struct *l7src, struct ndpi_id_struct *l7dst, bool ipv6)
{
void *data;
size_t offset, size;
ftval voffset;
const pstack_f *ip;
unsigned long when;
ndpi_protocol l7prot_id;
if (ipv6) {
ip = ProtStackSearchProt(pkt->stk, ipv6_id);
ProtGetAttr(ip, ipv6_offset_id, &voffset);
offset = voffset.uint32;
data = pkt->raw + offset;
size = pkt->raw_len - offset;
}
else {
ip = ProtStackSearchProt(pkt->stk, ip_id);
ProtGetAttr(ip, ip_offset_id, &voffset);
offset = voffset.uint32;
data = pkt->raw + offset;
size = pkt->raw_len - offset;
}
when = pkt->cap_sec;
when = when * NDPI_TICK_RES;
when += pkt->cap_usec/1000; /* (1000000 / NDPI_TICK_RES) */;
l7prot_id = ndpi_detection_process_packet(ndpi, l7flow, data, size, when, l7src, l7dst);
return l7prot_id;
}
void pkt(struct lfc *lfc, void *pdata,
struct lfc_flow *lf, void *data,
double ts, bool up, bool is_new, libtrace_packet_t *pkt)
{
struct ndpi *nd = pdata;
struct flow *f = data;
struct ndpi_id_struct *srcid, *dstid;
uint8_t *iph;
uint16_t et;
uint32_t rem;
uint64_t time;
if (!f->ndpi_flow)
f->ndpi_flow = mmatic_zalloc(nd->mm, ndpi_detection_get_sizeof_ndpi_flow_struct());
iph = trace_get_layer3(pkt, &et, &rem);
time = ts * 1000;
srcid = getid(nd, &lf->src);
dstid = getid(nd, &lf->dst);
f->proto = ndpi_detection_process_packet(
nd->ndpi, f->ndpi_flow, iph, rem, time, srcid, dstid);
}
static unsigned int packet_processing(const u_int64_t time,
const struct ndpi_iphdr *iph,
struct ndpi_ip6_hdr *iph6,
u_int16_t ip_offset,
u_int16_t ipsize, u_int16_t rawsize)
{
struct ndpi_id_struct *src, *dst;
struct ndpi_flow *flow;
struct ndpi_flow_struct *ndpi_flow = NULL;
u_int32_t protocol = 0;
u_int8_t proto;
if(iph)
flow = get_ndpi_flow(4, iph, ip_offset, ipsize,
ntohs(iph->tot_len) - (iph->ihl * 4),
&src, &dst, &proto);
else
flow = get_ndpi_flow6(iph6, ip_offset, &src, &dst, &proto);
if(flow != NULL) {
ndpi_flow = flow->ndpi_flow;
flow->packets++, flow->bytes += rawsize;
} else
return(0);
ip_packet_count++;
total_bytes += rawsize + 24 /* CRC etc */;
if(flow->detection_completed) return(0);
protocol = (const u_int32_t)ndpi_detection_process_packet(ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
flow->detected_protocol = protocol;
if((flow->detected_protocol != NDPI_PROTOCOL_UNKNOWN)
|| (proto == IPPROTO_UDP)
|| ((proto == IPPROTO_TCP) && (flow->packets > 10))) {
flow->detection_completed = 1;
#if 0
if(flow->ndpi_flow->l4.tcp.host_server_name[0] != '\0')
printf("%s\n", flow->ndpi_flow->l4.tcp.host_server_name);
#endif
if(verbose > 1) {
char buf1[32], buf2[32];
printf("%s %s:%u > %s:%u [proto: %u/%s][%s]\n",
ipProto2Name(flow->protocol),
intoaV4(ntohl(flow->lower_ip), buf1, sizeof(buf1)), ntohs(flow->lower_port),
intoaV4(ntohl(flow->upper_ip), buf2, sizeof(buf2)), ntohs(flow->upper_port),
protocol, ndpi_get_proto_name(ndpi_struct, protocol),
flow->ndpi_flow->host_server_name);
}
snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s", flow->ndpi_flow->host_server_name);
free_ndpi_flow(flow);
}
#if 0
if(ndpi_flow->l4.tcp.host_server_name[0] != '\0')
printf("%s\n", ndpi_flow->l4.tcp.host_server_name);
#endif
return 0;
}
int doingDetection()
{
int status;
struct nlmsghdr *nlh;
struct ndpi_ethher *ethernet;
struct ndpi_iphdr *iph;
ipq_packet_msg_t *ipq_packet;
int ip_len;
struct ndpi_id_struct *src, *dst;
struct ndpi_flow *flow;
struct ndpi_flow_struct *ndpi_flow = NULL;
u_int32_t protocol = 0;
u_int8_t proto;
u_int64_t time;
static u_int64_t lasttime=0;
unsigned char payload[1024*1024];
while(1)
{
status = ipq_read(h, buf, sizeof(buf),0);
if(status==0||status==-1)continue;
memset(payload, 0x00, sizeof(payload));
if(status > sizeof(struct nlmsghdr))
{
nlh = (struct nlmsghdr *)buf;//测试是否和ndpi_ethher一致。
ipq_packet = ipq_get_packet(buf);
ip_len=ipq_packet->data_len;
time = ((uint64_t) ipq_packet->timestamp_sec) * detection_tick_resolution +ipq_packet->timestamp_usec / (1000000 / detection_tick_resolution);
memcpy(payload + ETH_HDRLEN, ipq_packet->payload, ip_len);
// printf("2\n");
if(lasttime > time) {
time = lasttime;
}
lasttime = time;
iph = (struct ndpi_iphdr *)(&(ipq_packet->payload[0]));//需要测试是否和pcap来的一致
if(iph)
{
// printf("before get_ndpi_flow\n");
flow = get_ndpi_flow(iph, ip_len,&src, &dst, &proto);
// printf("after get_ndpi_flow\n");
}
if(flow != NULL)
{
ndpi_flow = flow->ndpi_flow;
flow->packets++, flow->bytes += ip_len;
} else
continue;
// printf("3\n");
ip_packet_count++;
total_bytes+=ip_len+24;
if(flow->detection_completed)
{
/*ipq_set_verdict(h, ipq_packet->packet_id, NF_ACCEPT,ipq_packet->data_len,payload + ETH_HDRLEN);*/
ipq_set_mark(h,ipq_packet->packet_id,1);
continue;
}
protocol = (const u_int32_t)ndpi_detection_process_packet(ndpi_struct, ndpi_flow,(char *)iph,ip_len, time, src, dst);
// printf("4\n");
if((flow->detected_protocol != NDPI_PROTOCOL_UNKNOWN)
|| ((proto == IPPROTO_UDP) && (flow->packets > 8))
|| ((proto == IPPROTO_TCP) && (flow->packets > 10)))
{
if(flow->detected_protocol==NDPI_PROTOCOL_UNKNOWN)
flow->detected_protocol = ndpi_guess_undetected_protocol(ndpi_struct,
flow->protocol,
ntohl(flow->lower_ip),
ntohs(flow->lower_port),
ntohl(flow->upper_ip),
ntohs(flow->upper_port));
flow->detection_completed = 1;
protocol_counter[flow->detected_protocol]+=flow->packets;
protocol_flows[flow->detected_protocol]++;
protocol_counter_bytes[flow->detected_protocol]+=flow->bytes;
snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s", flow->ndpi_flow->host_server_name);
}
ipq_set_verdict(h, ipq_packet->packet_id, NF_ACCEPT,ipq_packet->data_len,payload + ETH_HDRLEN);
snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s", flow->ndpi_flow->host_server_name);
}
}
}
void PktAnalyzer::analyzer(Packet &pkt)
{
Poco::Stopwatch sw;
_logger.debug("Got packet from queue");
unsigned char *full_packet=pkt.get_payload();
int id = pkt.get_id();
uint32_t size = pkt.get_size();
struct nfq_q_handle *qh=pkt.get_qh();
struct ip *iph = (struct ip *)full_packet;
struct ip6_hdr *iph6 = (struct ip6_hdr *)full_packet;
// определяем версию протокола
int ip_version=0;
if(iph->ip_v == 6)
ip_version = 6;
else if (iph->ip_v == 4)
ip_version = 4;
if(!ip_version)
{
_logger.error("Unsupported IP protocol version %d for packet id %d",(int) iph->ip_v, id);
nfq_set_verdict(qh,id,NF_ACCEPT,0,NULL);
dump_file(full_packet,size,id);
return ;
}
unsigned char *pkt_data_ptr = NULL;
struct tcphdr* tcph;
pkt_data_ptr = full_packet + (ip_version == 4 ? sizeof(struct ip) : sizeof(struct ip6_hdr));
tcph = (struct tcphdr *) pkt_data_ptr;
// длина ip заголовка
int iphlen = iphdr(full_packet)->ihl*4; // ipv4
if(ip_version == 6)
iphlen = sizeof(struct ip6_hdr);
// длина tcp заголовка
int tcphlen = tcphdr(full_packet+iphlen)->doff*4;
// общая длина всех заголовков
uint32_t hlen = iphlen + tcphlen;
_parent->inc_total_bytes_packets(size);
// пропускаем пакет без данных
if(hlen == size)
{
nfq_set_verdict(qh,id,NF_ACCEPT,0,NULL);
return ;
}
int tcp_src_port=ntohs(tcph->source);
int tcp_dst_port=ntohs(tcph->dest);
std::unique_ptr<Poco::Net::IPAddress> src_ip;
std::unique_ptr<Poco::Net::IPAddress> dst_ip;
if(ip_version == 4)
{
src_ip.reset(new Poco::Net::IPAddress(&iph->ip_src,sizeof(in_addr)));
dst_ip.reset(new Poco::Net::IPAddress(&iph->ip_dst,sizeof(in_addr)));
} else {
src_ip.reset(new Poco::Net::IPAddress(&iph6->ip6_src,sizeof(in6_addr)));
dst_ip.reset(new Poco::Net::IPAddress(&iph6->ip6_dst,sizeof(in6_addr)));
}
uint8_t ip_protocol=(ip_version == 4 ? iph->ip_p : iph6->ip6_ctlun.ip6_un1.ip6_un1_nxt);
{
Poco::ScopedReadRWLock lock(nfqFilter::_ipportMapMutex);
IPPortMap::iterator it_ip=nfqFilter::_ipportMap->find(*dst_ip.get());
if(it_ip != nfqFilter::_ipportMap->end())
{
unsigned short port=tcp_dst_port;
if (it_ip->second.size() == 0 || it_ip->second.find(port) != it_ip->second.end())
{
_parent->inc_matched_ip_port();
if(_config.send_rst)
{
_logger.debug("HostList: Send RST to the client (%s) and server (%s) (packet no %d)",src_ip->toString(),dst_ip->toString(),id);
std::string empty_str;
SenderTask::queue.enqueueNotification(new RedirectNotification(tcp_src_port, tcp_dst_port,src_ip.get(), dst_ip.get(),/*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq,/* flag psh */ (tcph->psh ? 1 : 0 ),empty_str,true));
_parent->inc_sended_rst();
nfq_set_verdict(qh,id,NF_DROP,0,NULL);
} else {
_logger.debug("HostList: Set mark %d to packet no %d port %hu",_config.mark_value,id,port);
_parent->inc_marked_hosts();
nfq_set_verdict2(qh,id,NF_ACCEPT,_config.mark_value,0,NULL);
}
return ;
}
}
}
// nDPI usage
sw.reset();
sw.start();
nDPIWrapper nw;
struct ndpi_flow_struct *flow=nw.get_flow();
uint32_t current_tickt = 0;
ndpi_protocol protocol = ndpi_detection_process_packet(nfqFilter::my_ndpi_struct, flow, full_packet, size, current_tickt, nw.get_src(), nw.get_dst());
if(protocol.protocol == NDPI_PROTOCOL_UNKNOWN)
{
_logger.debug("Guessing protocol...");
protocol = ndpi_guess_undetected_protocol(nfqFilter::my_ndpi_struct,
ip_protocol,
0,//ip
tcp_src_port, // sport
0,
tcp_dst_port); // dport
}
_logger.debug("Protocol is %hu/%hu ",protocol.master_protocol,protocol.protocol);
sw.stop();
_logger.debug("nDPI protocol detection occupied %ld us",sw.elapsed());
if(protocol.master_protocol == NDPI_PROTOCOL_SSL || protocol.protocol == NDPI_PROTOCOL_SSL || protocol.protocol == NDPI_PROTOCOL_TOR)
{
if(flow->l4.tcp.ssl_seen_client_cert == 1)
{
std::string ssl_client;
_logger.debug("Analysing SSL protocol");
if(flow->protos.ssl.client_certificate[0] != '\0')
{
ssl_client=flow->protos.ssl.client_certificate;
_logger.debug("SSL client is: %s",ssl_client);
}
if(!ssl_client.empty())
{
sw.reset();
sw.start();
if(_config.lower_host)
std::transform(ssl_client.begin(), ssl_client.end(), ssl_client.begin(), ::tolower);
AhoCorasickPlus::Match match;
std::size_t host_len=ssl_client.length();
bool found=false;
{
Poco::Mutex::ScopedLock lock(nfqFilter::_sslMutex);
nfqFilter::atm_ssl->search(ssl_client,false);
while(nfqFilter::atm_ssl->findNext(match) && !found)
{
if(match.pattern.length != host_len)
{
DomainsMatchType::Iterator it=nfqFilter::_SSLdomainsMatchType->find(match.id);
bool exact_match=false;
if(it != nfqFilter::_SSLdomainsMatchType->end())
exact_match = it->second;
if(exact_match)
continue;
if(ssl_client[host_len-match.pattern.length-1] != '.')
continue;
}
found=true;
}
}
sw.stop();
_logger.debug("SSL Host seek occupied %ld us, host: %s",sw.elapsed(),ssl_client);
if(found)
{
_parent->inc_matched_ssl();
if(_config.send_rst)
{
_logger.debug("SSLHostList: Send RST to the client (%s) and server (%s) (packet no %d)",src_ip->toString(),dst_ip->toString(),id);
std::string empty_str;
SenderTask::queue.enqueueNotification(new RedirectNotification(tcp_src_port, tcp_dst_port,src_ip.get(), dst_ip.get(),/*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq,/* flag psh */ (tcph->psh ? 1 : 0 ),empty_str,true));
_parent->inc_sended_rst();
nfq_set_verdict(qh,id,NF_DROP,0,NULL);
} else {
_logger.debug("SSLHostList: Set mark %d to packet no %d, ssl host name: %s",_config.mark_value,id,ssl_client);
_parent->inc_marked_ssl();
nfq_set_verdict2(qh,id,NF_ACCEPT,_config.mark_value,0,NULL);
}
return ;
} else {
nfq_set_verdict(qh,id,NF_ACCEPT,0,NULL);
return ;
}
} else {
if(_config.block_undetected_ssl)
{
Poco::ScopedReadRWLock lock(nfqFilter::_sslIpsSetMutex);
if(nfqFilter::_sslIps->try_search_exact_ip(*dst_ip.get()))
{
_parent->inc_matched_ssl_ip();
_logger.debug("Blocking/Marking SSL client hello packet from %s:%d to %s:%d", src_ip->toString(),tcp_src_port,dst_ip->toString(),tcp_dst_port);
if(_config.send_rst)
{
_logger.debug("SSLClientHello: Send RST to the client (%s) and server (%s) (packet no %d)",src_ip->toString(),dst_ip->toString(),id);
std::string empty_str;
SenderTask::queue.enqueueNotification(new RedirectNotification(tcp_src_port, tcp_dst_port,src_ip.get(), dst_ip.get(),/*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq,/* flag psh */ (tcph->psh ? 1 : 0 ),empty_str,true));
_parent->inc_sended_rst();
nfq_set_verdict(qh,id,NF_DROP,0,NULL);
} else {
_logger.debug("SSLClientHello: Set mark %d to packet no %d",_config.mark_value,id);
_parent->inc_marked_ssl();
nfq_set_verdict2(qh,id,NF_ACCEPT,_config.mark_value,0,NULL);
}
return ;
}
}
_logger.debug("No ssl client certificate found! Accept packet from %s:%d to %s:%d.",src_ip->toString(),tcp_src_port,dst_ip->toString(),tcp_dst_port);
}
}
nfq_set_verdict(qh,id,NF_ACCEPT,0,NULL);
return ;
}
if(protocol.master_protocol != NDPI_PROTOCOL_HTTP && protocol.protocol != NDPI_PROTOCOL_HTTP && protocol.protocol != NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK)
{
_logger.debug("Not http protocol. Protocol is %hu/%hu from %s:%d to %s:%d",protocol.master_protocol,protocol.protocol,src_ip->toString(),tcp_src_port,dst_ip->toString(),tcp_dst_port);
nfq_set_verdict(qh,id,NF_ACCEPT,0,NULL);
return ;
}
_logger.debug("Got HTTP protocol");
std::string host((char *)&flow->host_server_name[0]);
if((flow->http.method == HTTP_METHOD_GET || flow->http.method == HTTP_METHOD_POST || flow->http.method == HTTP_METHOD_HEAD) && !host.empty())
{
int dot_del=0;
if(host[host.length()-1] == '.')
{
dot_del=host.length()-1;
host.erase(dot_del,1);
}
if(_config.lower_host)
std::transform(host.begin(), host.end(), host.begin(), ::tolower);
sw.reset();
sw.start();
AhoCorasickPlus::Match match;
bool found=false;
{
Poco::Mutex::ScopedLock lock(nfqFilter::_domainMapMutex);
nfqFilter::atm_domains->search(host,false);
std::size_t host_len=host.length();
while(nfqFilter::atm_domains->findNext(match) && !found)
{
if(match.pattern.length != host_len)
{
DomainsMatchType::Iterator it=nfqFilter::_domainsMatchType->find(match.id);
bool exact_match=false;
if(it != nfqFilter::_domainsMatchType->end())
exact_match = it->second;
if(exact_match)
continue;
if(host[host_len-match.pattern.length-1] != '.')
continue;
}
found=true;
}
}
sw.stop();
_logger.debug("Host seek occupied %ld us",sw.elapsed());
if(found)
{
_logger.debug("Host %s present in domain (file line %d) list from ip %s", host, match.id, src_ip->toString());
std::string add_param;
switch (_config.add_p_type)
{
case A_TYPE_ID: add_param="id="+std::to_string(match.id);
break;
case A_TYPE_URL: add_param="url="+host;
break;
default: break;
}
SenderTask::queue.enqueueNotification(new RedirectNotification(tcp_src_port, tcp_dst_port, src_ip.get(), dst_ip.get(),/*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq,/* flag psh */ (tcph->psh ? 1 : 0 ),add_param));
_parent->inc_redirected_domains();
nfq_set_verdict(qh,id,NF_DROP,0,NULL);
return ;
}
sw.reset();
sw.start();
found=false;
std::string uri_o(flow->http.url ? flow->http.url : "");
if(flow->http.url)
{
std::string uri;
if(dot_del)
uri_o.erase(dot_del+7,1);
try
{
Poco::URI uri_p(uri_o);
uri_p.normalize();
uri.assign(uri_p.toString());
if(_config.url_decode)
{
#ifdef __USE_POCO_URI_DECODE
Poco::URI::decode(uri_p.toString(),uri);
#else
uri=url_decode(uri);
#endif
}
} catch (Poco::SyntaxException &ex)
{
_logger.debug("An SyntaxException occured: '%s' on URI: '%s'",ex.displayText(), uri_o);
uri.assign(flow->http.url);
}
{
Poco::Mutex::ScopedLock lock(nfqFilter::_urlMapMutex);
nfqFilter::atm->search(uri,false);
while(nfqFilter::atm->findNext(match) && !found)
{
if(_config.match_url_exactly && uri.length() != match.pattern.length)
continue;
found=true;
}
}
sw.stop();
_logger.debug("URL seek occupied %ld us for uri %s",sw.elapsed(),uri);
if(found)
{
_logger.debug("URL %s present in url (file pos %u) list from ip %s",uri,match.id,src_ip->toString());
std::string add_param;
switch (_config.add_p_type)
{
case A_TYPE_ID: add_param="id="+std::to_string(match.id);
break;
case A_TYPE_URL: add_param="url="+uri;
break;
default: break;
}
SenderTask::queue.enqueueNotification(new RedirectNotification(tcp_src_port, tcp_dst_port,src_ip.get(),dst_ip.get(),/*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq,/* flag psh */ (tcph->psh ? 1 : 0 ),add_param));
_parent->inc_redirected_urls();
nfq_set_verdict(qh,id,NF_DROP,0,NULL);
return ;
}
}
}
nfq_set_verdict(qh,id,NF_ACCEPT,0,NULL);
}
void firehose_packet(const char *pciaddr, char *data, int length) {
// Garbadge collection code
double current_timestamp = (double)rte_rdtsc() / system_tsc_resolution_hz;
if (current_timestamp - last_timestamp > gc_call_timeout) {
std::vector<conntrack_hash_struct_for_simple_packet_t> keys_to_remove;
for (auto& itr : my_connection_tracking_storage) {
// Remove all records who older than X seconds
if (current_timestamp - itr.second.last_timestamp > gc_clean_how_old_records) {
keys_to_remove.push_back(itr.first);
}
}
//if (!keys_to_remove.empty()) {
// std::cout << "We will remove " << keys_to_remove.size() << " keys" << std::endl;
//}
for (auto key_to_remove : keys_to_remove) {
my_connection_tracking_storage.erase(key_to_remove);
}
last_timestamp = current_timestamp;
}
// GC code ends
__sync_fetch_and_add(&received_packets, 1);
__sync_fetch_and_add(&received_bytes, length);
struct pfring_pkthdr packet_header;
memset(&packet_header, 0, sizeof(packet_header));
packet_header.len = length;
packet_header.caplen = length;
// We do not calculate timestamps because timestamping is very CPU intensive operation:
// https://github.com/ntop/PF_RING/issues/9
u_int8_t timestamp = 0;
u_int8_t add_hash = 0;
fastnetmon_parse_pkt((u_char*)data, &packet_header, 4, timestamp, add_hash);
simple_packet current_packet;
parse_raw_packet_to_simple_packet((u_char*)data, length, current_packet);
conntrack_hash_struct_for_simple_packet_t conntrack_structure;
convert_simple_packet_toconntrack_hash_struct(current_packet, conntrack_structure);
ndpi_tracking_flow_t& dpi_tracking_structure = my_connection_tracking_storage[ conntrack_structure ];
// Protocol already detected
/*
if (dpi_tracking_structure.protocol_detected && dpi_tracking_structure.detected_protocol.protocol == NDPI_PROTOCOL_IRC) {
char print_buffer[512];
fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)data, &packet_header);
printf("packet: %s\n", print_buffer);
for (unsigned int index = packet_header.extended_hdr.parsed_pkt.offset.payload_offset; index < packet_header.len; index++) {
printf("%c", data[index]);
}
printf("\n");
return;
}
*/
dpi_tracking_structure.update_timestamp();
uint32_t current_tickt = 0 ;
uint8_t* iph = (uint8_t*)(&data[packet_header.extended_hdr.parsed_pkt.offset.l3_offset]);
// printf("vlan: %d\n", packet_header.extended_hdr.parsed_pkt.vlan_id);
struct ndpi_iphdr* ndpi_ip_header = (struct ndpi_iphdr*)iph;
unsigned int ipsize = packet_header.len;
ndpi_protocol detected_protocol = ndpi_detection_process_packet(my_ndpi_struct, dpi_tracking_structure.flow, iph, ipsize, current_tickt, dpi_tracking_structure.src, dpi_tracking_structure.dst);
if (detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN && detected_protocol.master_protocol == NDPI_PROTOCOL_UNKNOWN) {
// printf("Can't detect protocol\n");
} else {
dpi_tracking_structure.detected_protocol = detected_protocol;
dpi_tracking_structure.protocol_detected = true;
//printf("Master protocol: %d protocol: %d\n", detected_protocol.master_protocol, detected_protocol.protocol);
char* protocol_name = ndpi_get_proto_name(my_ndpi_struct, detected_protocol.protocol);
char* master_protocol_name = ndpi_get_proto_name(my_ndpi_struct, detected_protocol.master_protocol);
if (detected_protocol.protocol == NDPI_PROTOCOL_HTTP) {
std::string host_name = std::string((const char*)dpi_tracking_structure.flow->host_server_name);
//printf("server name: %s\n", dpi_tracking_structure.flow->host_server_name);
if (redis_context != NULL) {
known_http_hosts_t::iterator itr = known_http_hosts.find(host_name);
if (itr == known_http_hosts.end()) {
// Not defined in internal cache
// Add in local cache:
known_http_hosts[ host_name ] = 1;
// Add to Redis
store_data_in_redis(host_name, "1");
} else {
// Already stored
}
}
}
//printf("Protocol: %s master protocol: %s\n", protocol_name, master_protocol_name);
bool its_bad_protocol = false;
//if(ndpi_is_proto(detected_protocol, NDPI_PROTOCOL_TOR)) {
// its_bad_protocol = true;
//}
if (detected_protocol.protocol == NDPI_PROTOCOL_IRC or detected_protocol.master_protocol == NDPI_PROTOCOL_IRC) {
its_bad_protocol = true;
}
if (its_bad_protocol) {
printf("Bad protocol %s master protocol %s found\n", protocol_name, master_protocol_name);
char print_buffer[512];
fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)data, &packet_header);
printf("packet: %s\n", print_buffer);
for (unsigned int index = packet_header.extended_hdr.parsed_pkt.offset.payload_offset; index < packet_header.len; index++) {
printf("%c", data[index]);
}
printf("\n");
}
}
}
void pcap_parse_packet(char* buffer, uint32_t len) {
struct pfring_pkthdr packet_header;
memset(&packet_header, 0, sizeof(packet_header));
packet_header.len = len;
packet_header.caplen = len;
// We do not calculate timestamps because timestamping is very CPU intensive operation:
// https://github.com/ntop/PF_RING/issues/9
u_int8_t timestamp = 0;
u_int8_t add_hash = 0;
fastnetmon_parse_pkt((u_char*)buffer, &packet_header, 4, timestamp, add_hash);
struct ndpi_id_struct *src = NULL;
struct ndpi_id_struct *dst = NULL;
struct ndpi_flow_struct *flow = NULL;
// So, we will init nDPI flow here
if (flow == NULL) {
src = (struct ndpi_id_struct*)malloc(size_id_struct);
memset(src, 0, size_id_struct);
dst = (struct ndpi_id_struct*)malloc(size_id_struct);
memset(dst, 0, size_id_struct);
flow = (struct ndpi_flow_struct *)malloc(size_flow_struct);
memset(flow, 0, size_flow_struct);
/*
struct ndpi_flow *newflow = (struct ndpi_flow*)malloc(sizeof(struct ndpi_flow));
memset(newflow, 0, sizeof(struct ndpi_flow));
newflow->protocol = packet_header.extended_hdr.parsed_pkt.l3_proto;
newflow->vlan_id = packet_header.extended_hdr.parsed_pkt.vlan_id;
uint32_t ip_src = packet_header.extended_hdr.parsed_pkt.ip_src.v4;
uint32_t ip_dst = packet_header.extended_hdr.parsed_pkt.ip_dst.v4;
uint16_t src_port = packet_header.extended_hdr.parsed_pkt.l4_src_port;
uint16_t dst_port = packet_header.extended_hdr.parsed_pkt.l4_dst_port;
if (ip_src < ip_dst) {
newflow->lower_ip = ip_src
newflow->upper_ip = ip_dst;
newflow->lower_port = src_port;
newflow->upper_port = dst_port;
} else {
newflow->lower_ip = ip_dst;
newflow->upper_ip = ip_src;
newflow->lower_port = dst_port;
newflow->upper_port = src_port;
}
newflow->src_id = malloc(size_id_struct);
memset(newflow->src_id, 0, size_id_struct);
newflow->dst_id = malloc(size_id_struct);
memset(newflow->dst_id, 0, size_id_struct);
*src = newflow->src_id, *dst = newflow->dst_id;
flow = newflow;
*/
} else {
//printf("We process only single packet\n");
//exit(0);
return;
}
uint32_t current_tickt = 0 ;
uint8_t* iph = (uint8_t*)(&buffer[packet_header.extended_hdr.parsed_pkt.offset.l3_offset]);
struct ndpi_iphdr* ndpi_ip_header = (struct ndpi_iphdr*)iph;
unsigned int ipsize = packet_header.len;
ndpi_protocol detected_protocol = ndpi_detection_process_packet(my_ndpi_struct, flow, iph, ipsize, current_tickt, src, dst);
if (detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN &&
detected_protocol.master_protocol == NDPI_PROTOCOL_UNKNOWN) {
printf("Can't detect protocol\n");
} else {
//printf("Master protocol: %d protocol: %d\n", detected_protocol.master_protocol, detected_protocol.protocol);
char* protocol_name = ndpi_get_proto_name(my_ndpi_struct, detected_protocol.protocol);
char* master_protocol_name = ndpi_get_proto_name(my_ndpi_struct, detected_protocol.master_protocol);
printf("Protocol: %s master protocol: %s\n", protocol_name, master_protocol_name);
// It's DNS request or answer
if (detected_protocol.protocol == NDPI_PROTOCOL_DNS) {
}
if (strstr(master_protocol_name, "Tor") == master_protocol_name or strstr(protocol_name, "IRC") != NULL) {
printf("Bad protocol %s master protocol %s found\n", protocol_name, master_protocol_name);
char print_buffer[512];
fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)buffer, &packet_header);
printf("packet: %s\n", print_buffer);
}
}
// We need use custom function because standard free could not free all memory here
ndpi_free_flow(flow);
free(dst);
free(src);
flow = NULL;
dst = NULL;
src = NULL;
}
static unsigned int packet_processing(const u_int64_t time, const struct pcap_pkthdr *header,
const struct ndpi_iphdr *iph, u_int16_t ipsize, u_int16_t rawsize)
{
struct ndpi_id_struct *src, *dst;
struct ndpi_flow *flow;
struct ndpi_flow_struct *ndpi_flow = NULL;
u_int16_t protocol = 0;
u_int16_t frag_off = ntohs(iph->frag_off);
flow = get_ndpi_flow(header, iph, ipsize);
if (flow != NULL) {
ndpi_flow = flow->ndpi_flow;
flow->packets++, flow->bytes += rawsize;
src = flow->src_id, dst = flow->dst_id;
} else
return;
ip_packet_count++;
total_bytes += rawsize;
if(flow->detection_completed) return;
// only handle unfragmented packets
if ((frag_off & 0x3FFF) == 0) {
// here the actual detection is performed
ndpi_protocol detected = ndpi_detection_process_packet(ndpi_struct, ndpi_flow, (uint8_t *) iph, ipsize, time, src, dst);
protocol = detected.master_protocol;
} else {
static u_int8_t frag_warning_used = 0;
if (frag_warning_used == 0) {
printf("\n\nWARNING: fragmented ip packets are not supported and will be skipped \n\n");
frag_warning_used = 1;
}
return 0;
}
#if 0
if(verbose && (protocol == 0)) {
char buf1[32], buf2[32];
printf("%s %s:%u > %s:%u [proto: %u/%s]\n",
ipProto2Name(flow->protocol),
intoaV4(ntohl(flow->lower_ip), buf1, sizeof(buf1)), ntohs(flow->lower_port),
intoaV4(ntohl(flow->upper_ip), buf2, sizeof(buf2)), ntohs(flow->upper_port),
protocol, ndpi_get_proto_name(ndpi_struct, protocol));
}
#endif
flow->detected_protocol = protocol;
if((flow->detected_protocol != NDPI_PROTOCOL_UNKNOWN)
|| (iph->protocol == IPPROTO_UDP)
|| ((iph->protocol == IPPROTO_TCP) && (flow->packets > 10))) {
flow->detection_completed = 1;
#if 0
if(flow->ndpi_flow->l4.tcp.host_server_name[0] != '\0')
printf("%s\n", flow->ndpi_flow->l4.tcp.host_server_name);
#endif
free_ndpi_flow(flow);
}
#if 0
if(ndpi_flow->l4.tcp.host_server_name[0] != '\0')
printf("%s\n", ndpi_flow->l4.tcp.host_server_name);
#endif
return 0;
}
void NetworkInterface::packet_processing(const u_int32_t when,
const u_int64_t time,
struct ndpi_ethhdr *eth,
u_int16_t vlan_id,
struct ndpi_iphdr *iph,
struct ndpi_ip6_hdr *ip6,
u_int16_t ipsize, u_int16_t rawsize)
{
bool src2dst_direction;
u_int8_t l4_proto;
Flow *flow;
u_int8_t *eth_src = eth->h_source, *eth_dst = eth->h_dest;
IpAddress src_ip, dst_ip;
u_int16_t src_port, dst_port;
struct ndpi_tcphdr *tcph = NULL;
struct ndpi_udphdr *udph = NULL;
u_int16_t l4_packet_len;
u_int8_t *l4, tcp_flags = 0;
u_int8_t *ip;
bool is_fragment = false;
if(iph != NULL) {
/* IPv4 */
if(ipsize < 20) {
incStats(ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, rawsize, 1, 24 /* 8 Preamble + 4 CRC + 12 IFG */);
return;
}
if((iph->ihl * 4) > ipsize || ipsize < ntohs(iph->tot_len)
|| (iph->frag_off & htons(0x1FFF /* IP_OFFSET */)) != 0) {
is_fragment = true;
}
l4_packet_len = ntohs(iph->tot_len) - (iph->ihl * 4);
l4_proto = iph->protocol;
l4 = ((u_int8_t *) iph + iph->ihl * 4);
ip = (u_int8_t*)iph;
} else {
/* IPv6 */
if(ipsize < sizeof(const struct ndpi_ip6_hdr)) {
incStats(ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, rawsize, 1, 24 /* 8 Preamble + 4 CRC + 12 IFG */);
return;
}
l4_packet_len = ntohs(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen)-sizeof(const struct ndpi_ip6_hdr);
l4_proto = ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt;
l4 = (u_int8_t*)ip6 + sizeof(const struct ndpi_ip6_hdr);
ip = (u_int8_t*)ip6;
}
if((l4_proto == IPPROTO_TCP) && (l4_packet_len >= 20)) {
/* tcp */
tcph = (struct ndpi_tcphdr *)l4;
src_port = tcph->source, dst_port = tcph->dest;
tcp_flags = l4[13];
} else if((l4_proto == IPPROTO_UDP) && (l4_packet_len >= 8)) {
/* udp */
udph = (struct ndpi_udphdr *)l4;
src_port = udph->source, dst_port = udph->dest;
} else {
/* non tcp/udp protocols */
src_port = dst_port = 0;
}
if(iph != NULL) {
src_ip.set_ipv4(iph->saddr);
dst_ip.set_ipv4(iph->daddr);
} else {
src_ip.set_ipv6(&ip6->ip6_src);
dst_ip.set_ipv6(&ip6->ip6_dst);
}
#if defined(WIN32) && defined(DEMO_WIN32)
if(this->ethStats.getNumPackets() > MAX_NUM_PACKETS) {
static bool showMsg = false;
if(!showMsg) {
ntop->getTrace()->traceEvent(TRACE_NORMAL, "-----------------------------------------------------------");
ntop->getTrace()->traceEvent(TRACE_NORMAL, "WARNING: this demo application is a limited ntopng version able to");
ntop->getTrace()->traceEvent(TRACE_NORMAL, "capture up to %d packets. If you are interested", MAX_NUM_PACKETS);
ntop->getTrace()->traceEvent(TRACE_NORMAL, "in the full version please have a look at the ntop");
ntop->getTrace()->traceEvent(TRACE_NORMAL, "home page http://www.ntop.org/.");
ntop->getTrace()->traceEvent(TRACE_NORMAL, "-----------------------------------------------------------");
ntop->getTrace()->traceEvent(TRACE_NORMAL, "");
showMsg = true;
}
return;
}
#endif
/* Updating Flow */
flow = getFlow(eth_src, eth_dst, vlan_id, &src_ip, &dst_ip, src_port, dst_port,
l4_proto, &src2dst_direction, last_pkt_rcvd, last_pkt_rcvd);
if(flow == NULL) {
incStats(iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, rawsize, 1, 24 /* 8 Preamble + 4 CRC + 12 IFG */);
return;
} else {
flow->incStats(src2dst_direction, rawsize);
if(l4_proto == IPPROTO_TCP) flow->updateTcpFlags(tcp_flags);
}
/* Protocol Detection */
flow->updateActivities();
if(flow->isDetectionCompleted()) {
/* Handle aggregations here */
switch(flow->get_detected_protocol()) {
case NDPI_PROTOCOL_DNS:
struct ndpi_flow_struct *ndpi_flow = flow->get_ndpi_flow();
struct ndpi_id_struct *cli = (struct ndpi_id_struct*)flow->get_cli_id();
struct ndpi_id_struct *srv = (struct ndpi_id_struct*)flow->get_srv_id();
if(ndpi_flow) {
memset(&ndpi_flow->detected_protocol_stack,
0, sizeof(ndpi_flow->detected_protocol_stack));
ndpi_detection_process_packet(ndpi_struct, ndpi_flow,
ip, ipsize, (u_int32_t)time,
cli, srv);
if(ndpi_flow->protos.dns.ret_code != 0) {
/*
This is a negative reply thus we notify the system that
this aggregation must not be tracked
*/
flow->aggregateInfo((char*)ndpi_flow->host_server_name, l4_proto,
NDPI_PROTOCOL_DNS, false);
}
}
break;
}
flow->processDetectedProtocol();
flow->deleteFlowMemory();
incStats(iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, flow->get_detected_protocol(), rawsize, 1, 24 /* 8 Preamble + 4 CRC + 12 IFG */);
return;
} else
incStats(iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, flow->get_detected_protocol(), rawsize, 1, 24 /* 8 Preamble + 4 CRC + 12 IFG */);
if(!is_fragment) {
struct ndpi_flow_struct *ndpi_flow = flow->get_ndpi_flow();
struct ndpi_id_struct *cli = (struct ndpi_id_struct*)flow->get_cli_id();
struct ndpi_id_struct *srv = (struct ndpi_id_struct*)flow->get_srv_id();
flow->setDetectedProtocol(ndpi_detection_process_packet(ndpi_struct, ndpi_flow,
ip, ipsize, (u_int32_t)time,
cli, srv));
} else {
// FIX - only handle unfragmented packets
// ntop->getTrace()->traceEvent(TRACE_WARNING, "IP fragments are not handled yet!");
}
}
