LOCAL gboolean reader_libpcapfile_read()
{
// pause reading if too many waiting disk operations
if (moloch_writer_queue_length() > 10) {
return TRUE;
}
// pause reading if too many waiting ES operations
if (moloch_http_queue_length(esServer) > 50) {
return TRUE;
}
// pause reading if too many packets are waiting to be processed
if (moloch_packet_outstanding() > (int32_t)(config.maxPacketsInQueue/3)) {
return TRUE;
}
int r;
if (pktsToRead > 0) {
r = pcap_dispatch(pcap, MIN(pktsToRead, 5000), reader_libpcapfile_pcap_cb, NULL);
if (r > 0)
pktsToRead -= r;
if (pktsToRead == 0)
r = 0;
} else {
r = pcap_dispatch(pcap, 5000, reader_libpcapfile_pcap_cb, NULL);
}
moloch_packet_batch_flush(&batch);
// Some kind of failure, move to the next file or quit
if (r <= 0) {
if (config.pcapDelete && r == 0) {
if (config.debug)
LOG("Deleting %s", offlinePcapFilename);
int rc = unlink(offlinePcapFilename);
if (rc != 0)
LOG("Failed to delete file %s %s (%d)", offlinePcapFilename, strerror(errno), errno);
}
pcap_close(pcap);
if (reader_libpcapfile_next()) {
return FALSE;
}
if (config.pcapMonitor)
g_timeout_add(100, reader_libpcapfile_monitor_gfunc, 0);
else
moloch_quit();
return FALSE;
}
return TRUE;
}
LOCAL void *reader_tpacketv3_thread(gpointer infov)
{
long info = (long)infov;
struct pollfd pfd;
int pos = -1;
memset(&pfd, 0, sizeof(pfd));
pfd.fd = infos[info].fd;
pfd.events = POLLIN | POLLERR;
pfd.revents = 0;
MolochPacketBatch_t batch;
moloch_packet_batch_init(&batch);
while (!config.quitting) {
if (pos == -1) {
MOLOCH_LOCK(infos[info].lock);
pos = infos[info].nextPos;
infos[info].nextPos = (infos[info].nextPos + 1) % infos[info].req.tp_block_nr;
MOLOCH_UNLOCK(infos[info].lock);
}
struct tpacket_block_desc *tbd = infos[info].rd[pos].iov_base;
if (config.debug > 2) {
int i;
int cnt = 0;
int waiting = 0;
for (i = 0; i < (int)infos[info].req.tp_block_nr; i++) {
struct tpacket_block_desc *stbd = infos[info].rd[i].iov_base;
if (stbd->hdr.bh1.block_status & TP_STATUS_USER) {
cnt++;
waiting += stbd->hdr.bh1.num_pkts;
}
}
LOG("Stats pos:%d info:%ld status:%x waiting:%d total cnt:%d total waiting:%d", pos, info, tbd->hdr.bh1.block_status, tbd->hdr.bh1.num_pkts, cnt, waiting);
}
// Wait until the block is owned by moloch
if ((tbd->hdr.bh1.block_status & TP_STATUS_USER) == 0) {
poll(&pfd, 1, -1);
continue;
}
struct tpacket3_hdr *th;
th = (struct tpacket3_hdr *) ((uint8_t *) tbd + tbd->hdr.bh1.offset_to_first_pkt);
uint16_t p;
for (p = 0; p < tbd->hdr.bh1.num_pkts; p++) {
if (unlikely(th->tp_snaplen != th->tp_len)) {
LOGEXIT("ERROR - Moloch requires full packet captures caplen: %d pktlen: %d\n"
"See https://github.com/aol/moloch/wiki/FAQ#Moloch_requires_full_packet_captures_error",
th->tp_snaplen, th->tp_len);
}
MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t);
packet->pkt = (u_char *)th + th->tp_mac;
packet->pktlen = th->tp_len;
packet->ts.tv_sec = th->tp_sec;
packet->ts.tv_usec = th->tp_nsec/1000;
packet->readerPos = info;
moloch_packet_batch(&batch, packet);
th = (struct tpacket3_hdr *) ((uint8_t *) th + th->tp_next_offset);
}
moloch_packet_batch_flush(&batch);
tbd->hdr.bh1.block_status = TP_STATUS_KERNEL;
pos = -1;
}
return NULL;
}