void reader_libpcapfile_init(char *UNUSED(name)) { moloch_reader_start = reader_libpcapfile_start; moloch_reader_stats = reader_libpcapfile_stats; if (config.pcapMonitor) reader_libpcapfile_init_monitor(); DLL_INIT(s_, &monitorQ); moloch_packet_batch_init(&batch); }
gboolean reader_libpcapfile_read() { // pause reading if too many waiting disk operations if (moloch_writer_queue_length() > 10) { return TRUE; } // pause reading if too many waiting ES operations if (moloch_http_queue_length(esServer) > 100) { return TRUE; } // pause reading if too many packets are waiting to be processed if (moloch_packet_outstanding() > (int32_t)(config.maxPacketsInQueue/2)) { return TRUE; } moloch_packet_batch_init(&batch); int r = pcap_dispatch(pcap, 10000, reader_libpcapfile_pcap_cb, NULL); moloch_packet_batch_flush(&batch); // Some kind of failure, move to the next file or quit if (r <= 0) { if (config.pcapDelete && r == 0) { if (config.debug) LOG("Deleting %s", offlinePcapFilename); int rc = unlink(offlinePcapFilename); if (rc != 0) LOG("Failed to delete file %s %s (%d)", offlinePcapFilename, strerror(errno), errno); } pcap_close(pcap); if (reader_libpcapfile_next()) { return FALSE; } if (config.pcapMonitor) g_timeout_add(100, reader_libpcapfile_monitor_gfunc, 0); else moloch_quit(); return FALSE; } return TRUE; }
LOCAL void *reader_tpacketv3_thread(gpointer infov) { long info = (long)infov; struct pollfd pfd; int pos = -1; memset(&pfd, 0, sizeof(pfd)); pfd.fd = infos[info].fd; pfd.events = POLLIN | POLLERR; pfd.revents = 0; MolochPacketBatch_t batch; moloch_packet_batch_init(&batch); while (!config.quitting) { if (pos == -1) { MOLOCH_LOCK(infos[info].lock); pos = infos[info].nextPos; infos[info].nextPos = (infos[info].nextPos + 1) % infos[info].req.tp_block_nr; MOLOCH_UNLOCK(infos[info].lock); } struct tpacket_block_desc *tbd = infos[info].rd[pos].iov_base; if (config.debug > 2) { int i; int cnt = 0; int waiting = 0; for (i = 0; i < (int)infos[info].req.tp_block_nr; i++) { struct tpacket_block_desc *stbd = infos[info].rd[i].iov_base; if (stbd->hdr.bh1.block_status & TP_STATUS_USER) { cnt++; waiting += stbd->hdr.bh1.num_pkts; } } LOG("Stats pos:%d info:%ld status:%x waiting:%d total cnt:%d total waiting:%d", pos, info, tbd->hdr.bh1.block_status, tbd->hdr.bh1.num_pkts, cnt, waiting); } // Wait until the block is owned by moloch if ((tbd->hdr.bh1.block_status & TP_STATUS_USER) == 0) { poll(&pfd, 1, -1); continue; } struct tpacket3_hdr *th; th = (struct tpacket3_hdr *) ((uint8_t *) tbd + tbd->hdr.bh1.offset_to_first_pkt); uint16_t p; for (p = 0; p < tbd->hdr.bh1.num_pkts; p++) { if (unlikely(th->tp_snaplen != th->tp_len)) { LOGEXIT("ERROR - Moloch requires full packet captures caplen: %d pktlen: %d\n" "See https://github.com/aol/moloch/wiki/FAQ#Moloch_requires_full_packet_captures_error", th->tp_snaplen, th->tp_len); } MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t); packet->pkt = (u_char *)th + th->tp_mac; packet->pktlen = th->tp_len; packet->ts.tv_sec = th->tp_sec; packet->ts.tv_usec = th->tp_nsec/1000; packet->readerPos = info; moloch_packet_batch(&batch, packet); th = (struct tpacket3_hdr *) ((uint8_t *) th + th->tp_next_offset); } moloch_packet_batch_flush(&batch); tbd->hdr.bh1.block_status = TP_STATUS_KERNEL; pos = -1; } return NULL; }