void* PfRingDevice::captureThreadMain(void *ptr)
{
PfRingDevice* device = (PfRingDevice*)ptr;
int coreId = device->getCurrentCoreId().Id;
pfring* ring = NULL;
LOG_DEBUG("Starting capture thread %d", coreId);
ring = device->m_CoreConfiguration[coreId].Channel;
if (ring == NULL)
{
LOG_ERROR("Couldn't find ring for core %d. Exiting capture thread", coreId);
return (void*)NULL;
}
while (!device->m_StopThread)
{
// if buffer is NULL PF_RING avoids copy of the data
uint8_t* buffer = NULL;
uint32_t bufferLen = 0;
// in multi-threaded mode flag PF_RING_REENTRANT is set, and this flag doesn't work with zero copy
// so I need to allocate a buffer and set buffer to point to it
if (device->m_ReentrantMode)
{
uint8_t tempBuffer[MAX_PACKET_SIZE];
buffer = tempBuffer;
bufferLen = MAX_PACKET_SIZE;
}
struct pfring_pkthdr pktHdr;
int recvRes = pfring_recv(ring, &buffer, bufferLen, &pktHdr, 0);
if (recvRes > 0)
{
// if caplen < len it means we don't have the whole packet. Treat this case as packet drop
// TODO: add this packet to dropped packet stats
// if (pktHdr.caplen != pktHdr.len)
// {
// LOG_ERROR("Packet dropped due to len != caplen");
// continue;
// }
RawPacket rawPacket(buffer, pktHdr.caplen, pktHdr.ts, false);
device->m_OnPacketsArriveCallback(&rawPacket, 1, coreId, device, device->m_OnPacketsArriveUserCookie);
}
else if (recvRes < 0)
{
LOG_ERROR("pfring_recv returned an error: [Err=%d]", recvRes);
}
}
LOG_DEBUG("Exiting capture thread %d", coreId);
return (void*)NULL;
}
int main(int argc, char* argv[])
{
PfRingDevice* dev = NULL;
int totalNumOfCores = getNumOfCores();
int numOfCaptureThreads = totalNumOfCores-1;
PfRingDevice* sendPacketsToIface = NULL;
std::string packetFilePath = "";
bool writePacketsToDisk = true;
IPv4Address srcIPToMatch = IPv4Address::Zero;
IPv4Address dstIPToMatch = IPv4Address::Zero;
uint16_t srcPortToMatch = 0;
uint16_t dstPortToMatch = 0;
ProtocolType protocolToMatch = Unknown;
int optionIndex = 0;
char opt = 0;
while((opt = getopt_long (argc, argv, "n:s:t:f:i:I:p:P:r:hl", PfFilterTrafficOptions, &optionIndex)) != -1)
{
switch (opt)
{
case 0:
{
break;
}
case 'n':
{
std::string ifaceName = std::string(optarg);
dev = PfRingDeviceList::getInstance().getPfRingDeviceByName(ifaceName);
if (dev == NULL)
EXIT_WITH_ERROR("Could not find PF_RING device '%s'", ifaceName.c_str());
break;
}
case 's':
{
std::string sendPacketsToIfaceName = std::string(optarg);
sendPacketsToIface = PfRingDeviceList::getInstance().getPfRingDeviceByName(sendPacketsToIfaceName);
if (sendPacketsToIface == NULL)
EXIT_WITH_ERROR("Could not find PF_RING device '%s'", sendPacketsToIfaceName.c_str());
break;
}
case 't':
{
numOfCaptureThreads = atoi(optarg);
if (numOfCaptureThreads < 1 || numOfCaptureThreads > totalNumOfCores-1)
EXIT_WITH_ERROR("Number of capture threads must be in the range of 1 to %d", totalNumOfCores-1);
break;
}
case 'f':
{
packetFilePath = string(optarg);
// make sure the path ends with '/'
if (packetFilePath.length() > 1 && (0 != packetFilePath.compare(packetFilePath.length()-1, 1, "/")))
packetFilePath += "/";
writePacketsToDisk = true;
break;
}
case 'i':
{
srcIPToMatch = IPv4Address(optarg);
if (!srcIPToMatch.isValid())
{
EXIT_WITH_ERROR_AND_PRINT_USAGE("Source IP to match isn't a valid IP address");
}
break;
}
case 'I':
{
dstIPToMatch = IPv4Address(optarg);
if (!dstIPToMatch.isValid())
{
EXIT_WITH_ERROR_AND_PRINT_USAGE("Destination IP to match isn't a valid IP address");
}
break;
}
case 'p':
{
srcPortToMatch = atoi(optarg);
if (srcPortToMatch <= 0)
{
EXIT_WITH_ERROR_AND_PRINT_USAGE("Source port to match isn't a valid TCP/UDP port");
}
break;
}
case 'P':
{
dstPortToMatch = atoi(optarg);
if (dstPortToMatch <= 0)
{
EXIT_WITH_ERROR_AND_PRINT_USAGE("Destination port to match isn't a valid TCP/UDP port");
}
break;
}
case 'r':
{
string protocol = string(optarg);
if (protocol == "TCP")
protocolToMatch = TCP;
else if (protocol == "UDP")
protocolToMatch = UDP;
else
{
EXIT_WITH_ERROR_AND_PRINT_USAGE("Protocol to match isn't TCP or UDP");
}
break;
}
case 'h':
{
printUsage();
exit(0);
}
case 'l':
{
listPfRingDevices();
exit(0);
}
default:
{
printUsage();
exit(0);
}
}
}
if (dev == NULL)
EXIT_WITH_ERROR_AND_PRINT_USAGE("Interface name was not provided");
// open the PF_RING device in multi-thread mode. Distribution of packets between threads will be done per-flow (as opposed to
// round-robin)
if (!dev->openMultiRxChannels(numOfCaptureThreads, PfRingDevice::PerFlow))
EXIT_WITH_ERROR("Couldn't open %d RX channels on interface '%s'", numOfCaptureThreads, dev->getDeviceName().c_str());
if (sendPacketsToIface != NULL && !sendPacketsToIface->open())
EXIT_WITH_ERROR("Couldn't open PF_RING device '%s' for sending matched packets", sendPacketsToIface->getDeviceName().c_str());
CoreMask coreMask = 0;
int threadId = 0;
int threadCount = 0;
// create an array of packet stats with the size of all machine cores
PacketStats packetStatsArr[totalNumOfCores];
// init each packet stats instance with an illegal core ID
for (int coreId = 0; coreId < totalNumOfCores; coreId++)
packetStatsArr[coreId].ThreadId = MAX_NUM_OF_CORES+1;
// mark only relevant cores by adding them to core mask
// mark only relevant packet stats instances by setting their core ID
while (threadCount < numOfCaptureThreads)
{
if (SystemCores::IdToSystemCore[threadId].Id != dev->getCurrentCoreId().Id)
{
coreMask |= SystemCores::IdToSystemCore[threadId].Mask;
packetStatsArr[threadId].ThreadId = SystemCores::IdToSystemCore[threadId].Id;
threadCount++;
}
threadId++;
}
// create the matching engine instance
PacketMatchingEngine matchingEngine(srcIPToMatch, dstIPToMatch, srcPortToMatch, dstPortToMatch, protocolToMatch);
// create a flow table for each core
map<uint32_t, bool> flowTables[totalNumOfCores];
PcapFileWriterDevice** pcapWriters = NULL;
// if needed, prepare pcap writers for all capturing threads
if (writePacketsToDisk)
{
pcapWriters = new PcapFileWriterDevice*[totalNumOfCores];
for (int coreId = 0; coreId < totalNumOfCores; coreId++)
{
// if core doesn't participate in capturing, skip it
if ((coreMask & SystemCores::IdToSystemCore[coreId].Mask) == 0)
{
pcapWriters[coreId] = NULL;
continue;
}
std::stringstream packetFileName;
packetFileName << packetFilePath << "Thread" << coreId << ".pcap";
pcapWriters[coreId] = new PcapFileWriterDevice(packetFileName.str().c_str());
if (!pcapWriters[coreId]->open())
{
EXIT_WITH_ERROR("Couldn't open pcap writer device for core %d", coreId);
}
}
}
printf("Start capturing on %d threads core mask = 0x%X\n", numOfCaptureThreads, coreMask);
// prepare packet capture configuration
CaptureThreadArgs args;
args.packetStatArr = packetStatsArr;
args.matchingEngine = &matchingEngine;
args.flowTables = flowTables;
args.sendPacketsTo = sendPacketsToIface;
args.pcapWriters = pcapWriters;
// start capturing packets on all threads
if (!dev->startCaptureMultiThread(packetArrived, &args, coreMask))
EXIT_WITH_ERROR("Couldn't start capturing on core mask %X on interface '%s'", coreMask, dev->getDeviceName().c_str());
bool shouldStop = false;
// register the on app close event to print summary stats on app termination
ApplicationEventHandler::getInstance().onApplicationInterrupted(onApplicationInterrupted, &shouldStop);
// infinite loop (until program is terminated)
while (!shouldStop)
{
sleep(5);
}
// stop capturing packets, close the device
dev->stopCapture();
dev->close();
// close and delete pcap writers
if (writePacketsToDisk)
{
for (int coreId = 0; coreId < totalNumOfCores; coreId++)
{
if ((coreMask & SystemCores::IdToSystemCore[coreId].Mask) == 0)
continue;
pcapWriters[coreId]->close();
delete pcapWriters[coreId];
}
}
printf("\n\nApplication stopped\n");
// print final stats for every capture thread plus sum of all threads and free worker threads memory
PacketStats aggregatedStats;
bool printedStatsHeadline = false;
for (int i = 0; i < totalNumOfCores; i++)
{
if (packetStatsArr[i].ThreadId == MAX_NUM_OF_CORES+1)
continue;
aggregatedStats.collectStats(packetStatsArr[i]);
if (!printedStatsHeadline)
{
packetStatsArr[i].printStatsHeadline();
printedStatsHeadline = true;
}
packetStatsArr[i].printStats();
}
aggregatedStats.printStats();
}