void compose_json_ndpi_class(json_t *obj, struct chained_cache *cc) { char ndpi_class[SUPERSHORTBUFLEN]; struct pkt_primitives *pbase = &cc->primitives; snprintf(ndpi_class, SUPERSHORTBUFLEN, "%s/%s", ndpi_get_proto_name(pm_ndpi_wfl->ndpi_struct, pbase->ndpi_class.master_protocol), ndpi_get_proto_name(pm_ndpi_wfl->ndpi_struct, pbase->ndpi_class.app_protocol)); json_object_set_new_nocheck(obj, "class", json_string(ndpi_class)); }
/** * @brief return the protocol name for the protocol given by id * @param pp_ctx packet processor context that holds the ndpi context * @param protocol_id id of the protocol to get the name for * @retval name of the protocol if ndpi enabled * @retval <ndpi disabled> if ndpi is not used */ inline const char* pp_ndpi_get_protocol_name(struct pp_context *pp_ctx, uint32_t protocol_id) { assert(pp_ctx); if (pp_ctx->ndpi_ctx) { return ndpi_get_proto_name(pp_ctx->ndpi_ctx, protocol_id); } else { return "<ndpi disabled>"; } }
static void printFlow(struct ndpi_flow *flow) { char buf1[32], buf2[32]; printf("\t%s %s:%u > %s:%u [proto: %u/%s][%u pkts/%u bytes]\n", ipProto2Name(flow->protocol), intoaV4(ntohl(flow->lower_ip), buf1, sizeof(buf1)), ntohs(flow->lower_port), intoaV4(ntohl(flow->upper_ip), buf2, sizeof(buf2)), ntohs(flow->upper_port), flow->detected_protocol, ndpi_get_proto_name(ndpi_struct, flow->detected_protocol), flow->packets, flow->bytes); }
static void printResults(u_int64_t tot_usec) { u_int32_t i; printf("\x1b[2K\n"); printf("pcap file contains\n"); printf("\tIP packets: \x1b[33m%-13llu\x1b[0m of %llu packets total\n", (long long unsigned int)ip_packet_count, (long long unsigned int)raw_packet_count); printf("\tIP bytes: \x1b[34m%-13llu\x1b[0m\n", (long long unsigned int)total_bytes); printf("\tUnique flows: \x1b[36m%-13u\x1b[0m\n", ndpi_flow_count); if(tot_usec > 0) { char buf[32], buf1[32]; float t = (float)(ip_packet_count*1000000)/(float)tot_usec; float b = (float)(total_bytes * 8 *1000000)/(float)tot_usec; printf("\tnDPI throughout: \x1b[36m%s pps / %s/sec\x1b[0m\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); } for(i=0; i<NUM_ROOTS; i++) ndpi_twalk(ndpi_flows_root[i], node_proto_guess_walker, NULL); if(enable_protocol_guess) printf("\tGuessed flow protocols: \x1b[35m%-13u\x1b[0m\n", guessed_flow_protocols); printf("\n\nDetected protocols:\n"); for (i = 0; i <= ndpi_get_num_supported_protocols(ndpi_struct); i++) { if(protocol_counter[i] > 0) { printf("\t\x1b[31m%-20s\x1b[0m packets: \x1b[33m%-13llu\x1b[0m bytes: \x1b[34m%-13llu\x1b[0m " "flows: \x1b[36m%-13u\x1b[0m\n", ndpi_get_proto_name(ndpi_struct, i), (long long unsigned int)protocol_counter[i], (long long unsigned int)protocol_counter_bytes[i], protocol_flows[i]); } } if(verbose && (protocol_counter[0] > 0)) { printf("\n"); for(i=0; i<NUM_ROOTS; i++) ndpi_twalk(ndpi_flows_root[i], node_print_known_proto_walker, NULL); printf("\n\nUndetected flows:\n"); for(i=0; i<NUM_ROOTS; i++) ndpi_twalk(ndpi_flows_root[i], node_print_unknown_proto_walker, NULL); } printf("\n\n"); }
static void fPrintFlow(FILE *stream, struct ndpi_flow *flow) { char buf1[32], buf2[32]; fprintf(stream, "%s %u %s %u %.6f %s %u/%s %u %u\n", intoaV4(ntohl(flow->lower_ip), buf1, sizeof(buf1)), ntohs(flow->lower_port), intoaV4(ntohl(flow->upper_ip), buf2, sizeof(buf2)), ntohs(flow->upper_port), flow->first_packet_time_sec + flow->first_packet_time_usec/1000000.0, ipProto2Name(flow->protocol), flow->detected_protocol, ndpi_get_proto_name(ndpi_struct, flow->detected_protocol), flow->packets, flow->bytes); }
char* Flow::print(char *buf, u_int buf_len) { char buf1[32], buf2[32]; buf[0] = '\0'; if((cli_host == NULL) || (srv_host == NULL)) return(buf); snprintf(buf, buf_len, "%s %s:%u > %s:%u [proto: %u/%s][%u/%u pkts][%llu/%llu bytes]\n", get_protocol_name(), cli_host->get_ip()->print(buf1, sizeof(buf1)), ntohs(cli_port), srv_host->get_ip()->print(buf2, sizeof(buf2)), ntohs(srv_port), ndpi_detected_protocol, ndpi_get_proto_name(iface->get_ndpi_struct(), ndpi_detected_protocol), cli2srv_packets, srv2cli_packets, (long long unsigned) cli2srv_bytes, (long long unsigned) srv2cli_bytes); return(buf); }
static void printResults(void) { u_int32_t i, j; printf("\x1b[2K\n"); printf("pcap file contains\n"); printf("\tip packets: \x1b[33m%-13llu\x1b[0m of %llu packets total\n", (long long unsigned int)ip_packet_count, (long long unsigned int)raw_packet_count); printf("\tip bytes: \x1b[34m%-13llu\x1b[0m\n", (long long unsigned int)total_bytes); printf("\tunique flows: \x1b[36m%-13u\x1b[0m\n", ndpi_flow_count); ndpi_twalk(ndpi_flows_root, node_proto_guess_walker, NULL); if(enable_protocol_guess) printf("\tguessed flow protocols: \x1b[35m%-13u\x1b[0m\n", guessed_flow_protocols); printf("\n\ndetected protocols:\n"); for (i = 0; i <= ndpi_get_num_supported_protocols(ndpi_struct); i++) { if (protocol_counter[i] > 0) { printf("\t\x1b[31m%-20s\x1b[0m packets: \x1b[33m%-13llu\x1b[0m bytes: \x1b[34m%-13llu\x1b[0m " "flows: \x1b[36m%-13u\x1b[0m\n", ndpi_get_proto_name(ndpi_struct, i), (long long unsigned int)protocol_counter[i], (long long unsigned int)protocol_counter_bytes[i], protocol_flows[i]); } } if(verbose && (protocol_counter[0] > 0)) { printf("\n\nundetected flows:\n"); ndpi_twalk(ndpi_flows_root, node_print_unknown_proto_walker, NULL); } if (1) { flow_info_file = fopen(flow_info_file_name, "wb"); fputs("source_ip source_port dest_ip dest_port first_packet_time l4_proto detect_proto packets bytes\n", flow_info_file); ndpi_twalk(ndpi_flows_root, node_output_flow_info_walker, NULL); fclose(flow_info_file); } printf("\n\n"); }
static unsigned int packet_processing(const u_int64_t time, const struct ndpi_iphdr *iph, struct ndpi_ip6_hdr *iph6, u_int16_t ip_offset, u_int16_t ipsize, u_int16_t rawsize) { struct ndpi_id_struct *src, *dst; struct ndpi_flow *flow; struct ndpi_flow_struct *ndpi_flow = NULL; u_int32_t protocol = 0; u_int8_t proto; if(iph) flow = get_ndpi_flow(4, iph, ip_offset, ipsize, ntohs(iph->tot_len) - (iph->ihl * 4), &src, &dst, &proto); else flow = get_ndpi_flow6(iph6, ip_offset, &src, &dst, &proto); if(flow != NULL) { ndpi_flow = flow->ndpi_flow; flow->packets++, flow->bytes += rawsize; } else return(0); ip_packet_count++; total_bytes += rawsize + 24 /* CRC etc */; if(flow->detection_completed) return(0); protocol = (const u_int32_t)ndpi_detection_process_packet(ndpi_struct, ndpi_flow, iph ? (uint8_t *)iph : (uint8_t *)iph6, ipsize, time, src, dst); flow->detected_protocol = protocol; if((flow->detected_protocol != NDPI_PROTOCOL_UNKNOWN) || (proto == IPPROTO_UDP) || ((proto == IPPROTO_TCP) && (flow->packets > 10))) { flow->detection_completed = 1; #if 0 if(flow->ndpi_flow->l4.tcp.host_server_name[0] != '\0') printf("%s\n", flow->ndpi_flow->l4.tcp.host_server_name); #endif if(verbose > 1) { char buf1[32], buf2[32]; printf("%s %s:%u > %s:%u [proto: %u/%s][%s]\n", ipProto2Name(flow->protocol), intoaV4(ntohl(flow->lower_ip), buf1, sizeof(buf1)), ntohs(flow->lower_port), intoaV4(ntohl(flow->upper_ip), buf2, sizeof(buf2)), ntohs(flow->upper_port), protocol, ndpi_get_proto_name(ndpi_struct, protocol), flow->ndpi_flow->host_server_name); } snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s", flow->ndpi_flow->host_server_name); free_ndpi_flow(flow); } #if 0 if(ndpi_flow->l4.tcp.host_server_name[0] != '\0') printf("%s\n", ndpi_flow->l4.tcp.host_server_name); #endif return 0; }
void firehose_packet(const char *pciaddr, char *data, int length) { // Garbadge collection code double current_timestamp = (double)rte_rdtsc() / system_tsc_resolution_hz; if (current_timestamp - last_timestamp > gc_call_timeout) { std::vector<conntrack_hash_struct_for_simple_packet_t> keys_to_remove; for (auto& itr : my_connection_tracking_storage) { // Remove all records who older than X seconds if (current_timestamp - itr.second.last_timestamp > gc_clean_how_old_records) { keys_to_remove.push_back(itr.first); } } //if (!keys_to_remove.empty()) { // std::cout << "We will remove " << keys_to_remove.size() << " keys" << std::endl; //} for (auto key_to_remove : keys_to_remove) { my_connection_tracking_storage.erase(key_to_remove); } last_timestamp = current_timestamp; } // GC code ends __sync_fetch_and_add(&received_packets, 1); __sync_fetch_and_add(&received_bytes, length); struct pfring_pkthdr packet_header; memset(&packet_header, 0, sizeof(packet_header)); packet_header.len = length; packet_header.caplen = length; // We do not calculate timestamps because timestamping is very CPU intensive operation: // https://github.com/ntop/PF_RING/issues/9 u_int8_t timestamp = 0; u_int8_t add_hash = 0; fastnetmon_parse_pkt((u_char*)data, &packet_header, 4, timestamp, add_hash); simple_packet current_packet; parse_raw_packet_to_simple_packet((u_char*)data, length, current_packet); conntrack_hash_struct_for_simple_packet_t conntrack_structure; convert_simple_packet_toconntrack_hash_struct(current_packet, conntrack_structure); ndpi_tracking_flow_t& dpi_tracking_structure = my_connection_tracking_storage[ conntrack_structure ]; // Protocol already detected /* if (dpi_tracking_structure.protocol_detected && dpi_tracking_structure.detected_protocol.protocol == NDPI_PROTOCOL_IRC) { char print_buffer[512]; fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)data, &packet_header); printf("packet: %s\n", print_buffer); for (unsigned int index = packet_header.extended_hdr.parsed_pkt.offset.payload_offset; index < packet_header.len; index++) { printf("%c", data[index]); } printf("\n"); return; } */ dpi_tracking_structure.update_timestamp(); uint32_t current_tickt = 0 ; uint8_t* iph = (uint8_t*)(&data[packet_header.extended_hdr.parsed_pkt.offset.l3_offset]); // printf("vlan: %d\n", packet_header.extended_hdr.parsed_pkt.vlan_id); struct ndpi_iphdr* ndpi_ip_header = (struct ndpi_iphdr*)iph; unsigned int ipsize = packet_header.len; ndpi_protocol detected_protocol = ndpi_detection_process_packet(my_ndpi_struct, dpi_tracking_structure.flow, iph, ipsize, current_tickt, dpi_tracking_structure.src, dpi_tracking_structure.dst); if (detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN && detected_protocol.master_protocol == NDPI_PROTOCOL_UNKNOWN) { // printf("Can't detect protocol\n"); } else { dpi_tracking_structure.detected_protocol = detected_protocol; dpi_tracking_structure.protocol_detected = true; //printf("Master protocol: %d protocol: %d\n", detected_protocol.master_protocol, detected_protocol.protocol); char* protocol_name = ndpi_get_proto_name(my_ndpi_struct, detected_protocol.protocol); char* master_protocol_name = ndpi_get_proto_name(my_ndpi_struct, detected_protocol.master_protocol); if (detected_protocol.protocol == NDPI_PROTOCOL_HTTP) { std::string host_name = std::string((const char*)dpi_tracking_structure.flow->host_server_name); //printf("server name: %s\n", dpi_tracking_structure.flow->host_server_name); if (redis_context != NULL) { known_http_hosts_t::iterator itr = known_http_hosts.find(host_name); if (itr == known_http_hosts.end()) { // Not defined in internal cache // Add in local cache: known_http_hosts[ host_name ] = 1; // Add to Redis store_data_in_redis(host_name, "1"); } else { // Already stored } } } //printf("Protocol: %s master protocol: %s\n", protocol_name, master_protocol_name); bool its_bad_protocol = false; //if(ndpi_is_proto(detected_protocol, NDPI_PROTOCOL_TOR)) { // its_bad_protocol = true; //} if (detected_protocol.protocol == NDPI_PROTOCOL_IRC or detected_protocol.master_protocol == NDPI_PROTOCOL_IRC) { its_bad_protocol = true; } if (its_bad_protocol) { printf("Bad protocol %s master protocol %s found\n", protocol_name, master_protocol_name); char print_buffer[512]; fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)data, &packet_header); printf("packet: %s\n", print_buffer); for (unsigned int index = packet_header.extended_hdr.parsed_pkt.offset.payload_offset; index < packet_header.len; index++) { printf("%c", data[index]); } printf("\n"); } } }
void pcap_parse_packet(char* buffer, uint32_t len) { struct pfring_pkthdr packet_header; memset(&packet_header, 0, sizeof(packet_header)); packet_header.len = len; packet_header.caplen = len; // We do not calculate timestamps because timestamping is very CPU intensive operation: // https://github.com/ntop/PF_RING/issues/9 u_int8_t timestamp = 0; u_int8_t add_hash = 0; fastnetmon_parse_pkt((u_char*)buffer, &packet_header, 4, timestamp, add_hash); struct ndpi_id_struct *src = NULL; struct ndpi_id_struct *dst = NULL; struct ndpi_flow_struct *flow = NULL; // So, we will init nDPI flow here if (flow == NULL) { src = (struct ndpi_id_struct*)malloc(size_id_struct); memset(src, 0, size_id_struct); dst = (struct ndpi_id_struct*)malloc(size_id_struct); memset(dst, 0, size_id_struct); flow = (struct ndpi_flow_struct *)malloc(size_flow_struct); memset(flow, 0, size_flow_struct); /* struct ndpi_flow *newflow = (struct ndpi_flow*)malloc(sizeof(struct ndpi_flow)); memset(newflow, 0, sizeof(struct ndpi_flow)); newflow->protocol = packet_header.extended_hdr.parsed_pkt.l3_proto; newflow->vlan_id = packet_header.extended_hdr.parsed_pkt.vlan_id; uint32_t ip_src = packet_header.extended_hdr.parsed_pkt.ip_src.v4; uint32_t ip_dst = packet_header.extended_hdr.parsed_pkt.ip_dst.v4; uint16_t src_port = packet_header.extended_hdr.parsed_pkt.l4_src_port; uint16_t dst_port = packet_header.extended_hdr.parsed_pkt.l4_dst_port; if (ip_src < ip_dst) { newflow->lower_ip = ip_src newflow->upper_ip = ip_dst; newflow->lower_port = src_port; newflow->upper_port = dst_port; } else { newflow->lower_ip = ip_dst; newflow->upper_ip = ip_src; newflow->lower_port = dst_port; newflow->upper_port = src_port; } newflow->src_id = malloc(size_id_struct); memset(newflow->src_id, 0, size_id_struct); newflow->dst_id = malloc(size_id_struct); memset(newflow->dst_id, 0, size_id_struct); *src = newflow->src_id, *dst = newflow->dst_id; flow = newflow; */ } else { //printf("We process only single packet\n"); //exit(0); return; } uint32_t current_tickt = 0 ; uint8_t* iph = (uint8_t*)(&buffer[packet_header.extended_hdr.parsed_pkt.offset.l3_offset]); struct ndpi_iphdr* ndpi_ip_header = (struct ndpi_iphdr*)iph; unsigned int ipsize = packet_header.len; ndpi_protocol detected_protocol = ndpi_detection_process_packet(my_ndpi_struct, flow, iph, ipsize, current_tickt, src, dst); if (detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN && detected_protocol.master_protocol == NDPI_PROTOCOL_UNKNOWN) { printf("Can't detect protocol\n"); } else { //printf("Master protocol: %d protocol: %d\n", detected_protocol.master_protocol, detected_protocol.protocol); char* protocol_name = ndpi_get_proto_name(my_ndpi_struct, detected_protocol.protocol); char* master_protocol_name = ndpi_get_proto_name(my_ndpi_struct, detected_protocol.master_protocol); printf("Protocol: %s master protocol: %s\n", protocol_name, master_protocol_name); // It's DNS request or answer if (detected_protocol.protocol == NDPI_PROTOCOL_DNS) { } if (strstr(master_protocol_name, "Tor") == master_protocol_name or strstr(protocol_name, "IRC") != NULL) { printf("Bad protocol %s master protocol %s found\n", protocol_name, master_protocol_name); char print_buffer[512]; fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)buffer, &packet_header); printf("packet: %s\n", print_buffer); } } // We need use custom function because standard free could not free all memory here ndpi_free_flow(flow); free(dst); free(src); flow = NULL; dst = NULL; src = NULL; }
// 在 winrow 上用黑体重写 static void prepareResults(/*u_int64_t tot_usec*/) { u_int32_t i; int row=0; int m = 0; /* Default output mode: color (0) */ memset(results,0,sizeof(results)); if (m) { printf("\n"); } else { printf("\x1b[2K\n"); } if (m) { sprintf(results[row++],"\tIP packets: %-13llu of %llu packets total\n", (long long unsigned int)ip_packet_count, (long long unsigned int)raw_packet_count); if(total_bytes > 0) sprintf(results[row++],"\tIP bytes: %-13llu (avg pkt size %u bytes)\n", (long long unsigned int)total_bytes,raw_packet_count>0?0: (unsigned int)(total_bytes/raw_packet_count)); sprintf(results[row++],"\tUnique flows: %-13u\n", ndpi_flow_count); } else { sprintf(results[row++],"\tIP packets: %-13llu of %llu packets total\n", (long long unsigned int)ip_packet_count, (long long unsigned int)raw_packet_count); sprintf(results[row++],"\tIP bytes: %-13llu (avg pkt size %u bytes)\n", (long long unsigned int)total_bytes,/*raw_packet_count>0?0:(unsigned int)(total_bytes/ip_packet_count)*/0); sprintf(results[row++],"\tUnique flows: %-13u\n", ndpi_flow_count); } /* if(tot_usec > 0) { char buf[32], buf1[32]; float t = (float)(ip_packet_count*1000000)/(float)tot_usec; float b = (float)(total_bytes * 8 *1000000)/(float)tot_usec; if (m) { printf("\tnDPI throughout: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); } else { //printf("\tGuessed flow protocols: \x1b[35m%-13u\x1b[0m\n", guessed_flow_protocols); } } */ sprintf(results[row++],"\n"); sprintf(results[row++],"\n"); sprintf(results[row++],"\tDetected protocols:"); sprintf(results[row++],"\n"); sprintf(results[row++],"\n"); for (i = 0; i <= ndpi_get_num_supported_protocols(ndpi_struct) /*&& row < NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 10*/; i++,row++) { if(protocol_counter[i] > 0) { if (m) { sprintf(results[row],"\t\%-20s packets: %-13llu bytes: %-13llu " "flows: %-13u\n", ndpi_get_proto_name(ndpi_struct, i), (long long unsigned int)protocol_counter[i], (long long unsigned int)protocol_counter_bytes[i], protocol_flows[i]); } else { printf("%d\n",row); sprintf(results[row],"\t%-20s packets: %-13llu bytes: %-13llu " "flows: %-13u\n", ndpi_get_proto_name(ndpi_struct, i), (long long unsigned int)protocol_counter[i], (long long unsigned int)protocol_counter_bytes[i], protocol_flows[i]); } } }
static unsigned int packet_processing(const u_int64_t time, const struct pcap_pkthdr *header, const struct ndpi_iphdr *iph, u_int16_t ipsize, u_int16_t rawsize) { struct ndpi_id_struct *src, *dst; struct ndpi_flow *flow; struct ndpi_flow_struct *ndpi_flow = NULL; u_int16_t protocol = 0; u_int16_t frag_off = ntohs(iph->frag_off); flow = get_ndpi_flow(header, iph, ipsize); if (flow != NULL) { ndpi_flow = flow->ndpi_flow; flow->packets++, flow->bytes += rawsize; src = flow->src_id, dst = flow->dst_id; } else return; ip_packet_count++; total_bytes += rawsize; if(flow->detection_completed) return; // only handle unfragmented packets if ((frag_off & 0x3FFF) == 0) { // here the actual detection is performed ndpi_protocol detected = ndpi_detection_process_packet(ndpi_struct, ndpi_flow, (uint8_t *) iph, ipsize, time, src, dst); protocol = detected.master_protocol; } else { static u_int8_t frag_warning_used = 0; if (frag_warning_used == 0) { printf("\n\nWARNING: fragmented ip packets are not supported and will be skipped \n\n"); frag_warning_used = 1; } return 0; } #if 0 if(verbose && (protocol == 0)) { char buf1[32], buf2[32]; printf("%s %s:%u > %s:%u [proto: %u/%s]\n", ipProto2Name(flow->protocol), intoaV4(ntohl(flow->lower_ip), buf1, sizeof(buf1)), ntohs(flow->lower_port), intoaV4(ntohl(flow->upper_ip), buf2, sizeof(buf2)), ntohs(flow->upper_port), protocol, ndpi_get_proto_name(ndpi_struct, protocol)); } #endif flow->detected_protocol = protocol; if((flow->detected_protocol != NDPI_PROTOCOL_UNKNOWN) || (iph->protocol == IPPROTO_UDP) || ((iph->protocol == IPPROTO_TCP) && (flow->packets > 10))) { flow->detection_completed = 1; #if 0 if(flow->ndpi_flow->l4.tcp.host_server_name[0] != '\0') printf("%s\n", flow->ndpi_flow->l4.tcp.host_server_name); #endif free_ndpi_flow(flow); } #if 0 if(ndpi_flow->l4.tcp.host_server_name[0] != '\0') printf("%s\n", ndpi_flow->l4.tcp.host_server_name); #endif return 0; }