void PacketHashtable::copyDataNanoseconds(IpfixRecord::Data* bucket, const IpfixRecord::Data* src, ExpFieldData* efd)
{
uint64_t ntptime;
ntptime = ntp64timegcc(*reinterpret_cast<const struct timeval*>(src));
uint64_t ntp2 = htonll(ntptime);
DPRINTFL(MSG_VDEBUG, "ntp2: %llu, ntptime/ntp2 %llX/%llX", ntp2, ntptime, ntp2);
memcpy(bucket+efd->dstIndex, &ntp2, sizeof(ntp2));
#ifdef DEBUG
if (ntohll(*(uint64_t*)(bucket+efd->dstIndex))<(1000000000ULL+(2208988800ULL<<32)) || ntohll(*(uint64_t*)(bucket+efd->dstIndex))>(1300000000ULL+(2208988800ULL<<32))) {
DPRINTFL(MSG_VDEBUG, "time before: %ds", reinterpret_cast<const struct timeval*>(src)->tv_sec);
DPRINTFL(MSG_VDEBUG, "copy invalid end nano seconds: %lld s (%llX)", (ntohll(*(uint64_t*)(bucket+efd->dstIndex))>>32)-2208988800U, *(uint64_t*)(bucket+efd->dstIndex));
}
/*
* this is the main filter function: each packet is run thru a series of
* PacketProcessors and then pushed to the next PacketSource
*/
void FilterModule::receive(Packet* p)
{
vector<PacketProcessor *>::iterator it;
// set to true so packet is always pushed to receiver
// even if the processors-iterator below is NULL
bool keepPacket=true;
DPRINTFL(MSG_VDEBUG, "FilterModule: got packet");
// run packet through all packetProcessors
for (it = processors.begin();
it != processors.end() && keepPacket; ++it)
{
keepPacket = (*it)->processPacket(p);
}
// check if we passed all filters
if (keepPacket) {
// push packet to the receiver
DPRINTF("FilterModule: pushing packet %d", p);
while (!exitFlag && !send(p));
return;
}
// immediately drop the packet
DPRINTF("FilterModule: releasing packet");
p->removeReference();
}
/**
* Receives a packet and tries to write the packet into the ringbuffer
*/
void PCAPExporterMem::receive(Packet* packet)
{
DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive() called");
if (onRestart) {
DPRINTF("Dropping incoming packet, as attached process is not ready");
DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive() ended");
packet->removeReference();
return;
}
if (fifoReaderPid == 0) {
msg(MSG_VDEBUG, "fifoReaderPid = 0...this might happen during reconfiguration");
DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive() ended");
packet->removeReference();
return;
}
if (restartInterval) {
if (nextRestart.tv_sec==0) {
DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive(): updating nextRestart");
nextRestart = packet->timestamp;
nextRestart.tv_sec += restartInterval;
} else if (compareTime(nextRestart, packet->timestamp)<0) {
DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive(): restarting process");
// we need to unregister our signal handlers, as we get race conditions with the signal handler for restarting the process
unregisterSignalHandlers();
stopProcess();
startProcess();
registerSignalHandlers();
DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive(): updating nextRestart");
nextRestart.tv_sec += ((packet->timestamp.tv_sec-nextRestart.tv_sec)/restartInterval+1)*restartInterval;
}
}
if (writeIntoMemory(packet)) {
statPktsForwarded++;
statBytesForwarded += packet->data_length;
DPRINTFL(MSG_VDEBUG, "Wrote packet at pos %u", *nextWrite);
} else {
batchUpdate();
statPktsDropped++;
statBytesDropped += packet->data_length;
DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive(): dropped packet");
}
packet->removeReference();
DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive() ended");
}
/**
* Buffer passed flow in Hashtable @c ht
*/
void FlowHashtable::aggregateTemplateData(IpfixRecord::TemplateInfo* ti, IpfixRecord::Data* data)
{
// the following lock should almost never fail (only during reconfiguration)
while (atomic_lock(&aggInProgress)) {
timespec req;
req.tv_sec = 0;
req.tv_nsec = 50000000;
nanosleep(&req, &req);
}
int i;
/* Create data block to be inserted into buffer... */
boost::shared_array<IpfixRecord::Data> htdata(new IpfixRecord::Data[fieldLength+privDataLength]);
// set private data fields to zero
memset(htdata.get()+fieldLength, 0, privDataLength);
for (i = 0; i < dataTemplate->fieldCount; i++) {
IpfixRecord::FieldInfo* hfi = &dataTemplate->fieldInfo[i];
IpfixRecord::FieldInfo* tfi = ti->getFieldInfo(&hfi->type);
if (!tfi) {
DPRINTF("Flow to be buffered did not contain %s field\n", typeid2string(hfi->type.id));
// if field was not copied, fill it with 0
memset(htdata.get() + hfi->offset, 0, hfi->type.length);
continue;
}
DPRINTFL(MSG_VDEBUG, "copyData for type %d, offset %x, starting from pointer %X", tfi->type.id, tfi->offset, data+tfi->offset);
DPRINTFL(MSG_VDEBUG, "copyData to offset %X", hfi->offset);
copyData(hfi, htdata.get(), tfi, data, fieldModifier[i]);
/* copy associated mask, should there be one */
switch (hfi->type.id) {
case IPFIX_TYPEID_sourceIPv4Address:
tfi = ti->getFieldInfo(IPFIX_TYPEID_sourceIPv4Mask, 0);
if(tfi) {
if(hfi->type.length != 5) {
DPRINTF("Tried to set mask of length %d IP address\n", hfi->type.length);
} else {
if(tfi->type.length == 1) {
*(uint8_t*)(htdata.get() + hfi->offset + 4) = *(uint8_t*)(data + tfi->offset);
} else {
DPRINTF("Cannot process associated mask with invalid length %d\n", tfi->type.length);
}
}
}
break;
case IPFIX_TYPEID_destinationIPv4Address:
tfi = ti->getFieldInfo(IPFIX_TYPEID_destinationIPv4Mask, 0);
if(tfi) {
if(hfi->type.length != 5) {
DPRINTF("Tried to set mask of length %d IP address\n", hfi->type.length);
} else {
if(tfi->type.length == 1) {
*(uint8_t*)(htdata.get() + hfi->offset + 4) = *(uint8_t*)(data + tfi->offset);
} else {
DPRINTF("Cannot process associated mask with invalid length %d\n", tfi->type.length);
}
}
}
break;
default:
break;
}
}
/* ...then buffer it */
bufferDataBlock(htdata);
atomic_release(&aggInProgress);
}
/**
* Inserts a data block into the hashtable
*/
void FlowHashtable::bufferDataBlock(boost::shared_array<IpfixRecord::Data> data)
{
statRecordsReceived++;
uint32_t nhash = getHash(data.get(), false);
DPRINTFL(MSG_VDEBUG, "nhash=%u", nhash);
HashtableBucket* prevbucket;
HashtableBucket* bucket = lookupBucket(nhash, data.get(), false, &prevbucket);
if (bucket != NULL) {
DPRINTFL(MSG_VDEBUG, "aggregating flow");
aggregateFlow(bucket->data.get(), data.get(), false);
bucket->expireTime = time(0) + minBufferTime;
if (bucket->forceExpireTime>bucket->expireTime) {
exportList.remove(bucket->listNode);
exportList.push(bucket->listNode);
}
} else {
if (biflowAggregation) {
// try reverse flow
uint32_t rhash = getHash(data.get(), true);
DPRINTFL(MSG_VDEBUG, "rhash=%u", rhash);
bucket = lookupBucket(rhash, data.get(), true, &prevbucket);
if (bucket != NULL) {
DPRINTFL(MSG_VDEBUG, "aggregating reverse flow");
if (aggregateFlow(bucket->data.get(), data.get(), true)==1) {
DPRINTFL(MSG_VDEBUG, "reversing whole flow");
// reverse flow
reverseFlowBucket(bucket);
// delete reference from hash table
if (prevbucket==NULL)
buckets[rhash] = bucket->next;
else {
prevbucket->next = bucket->next;
}
if (bucket->next!=NULL)
bucket->next->prev = prevbucket;
// insert into hash table again
nhash = getHash(bucket->data.get(), false);
DPRINTFL(MSG_VDEBUG, "nhash=%u", nhash);
bucket->next = buckets[nhash];
bucket->hash = nhash;
buckets[nhash] = bucket;
bucket->prev = 0;
if (bucket->next != NULL) bucket->next->prev = bucket;
bucket->expireTime = time(0) + minBufferTime;
if (bucket->forceExpireTime>bucket->expireTime) {
exportList.remove(bucket->listNode);
exportList.push(bucket->listNode);
}
}
}
}
if (bucket == NULL) {
DPRINTFL(MSG_VDEBUG, "creating new bucket");
HashtableBucket* n = buckets[nhash];
buckets[nhash] = createBucket(data, 0, n, 0, nhash); // FIXME: insert observationDomainID!
if (n != NULL) n->prev = buckets[nhash];
BucketListElement* node = hbucketIM.getNewInstance();
node->reset();
buckets[nhash]->listNode = node;
node->bucket = buckets[nhash];
exportList.push(node);
}
}
atomic_release(&aggInProgress);
}
/**
* Writes a packet into the pipe
*/
void PCAPExporterPipe::receive(Packet* packet)
{
DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive() called");
if (onRestart){
DPRINTF("Dropping incoming packet, as attached process is not ready");
DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive() ended");
packet->removeReference();
return;
}
if (fifoReaderPid == 0){
msg(MSG_VDEBUG, "fifoReaderPid = 0...this might happen during reconfiguration");
DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive() ended");
packet->removeReference();
return;
}
if (restartInterval) {
if (nextRestart.tv_sec==0) {
DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive(): updating nextRestart");
nextRestart = packet->timestamp;
struct timeval tv = { restartInterval/1000, (restartInterval % 1000)*1000 };
timeval_add(&nextRestart, &tv);
} else if (compareTime(nextRestart, packet->timestamp)<0) {
DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive(): restarting process");
// we need to unregister our signal handlers, as we get race conditions with the signal handler for restarting the process
unregisterSignalHandlers();
stopProcess();
startProcess();
registerSignalHandlers();
DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive(): updating nextRestart");
timeval tvdiff;
timeval_subtract(&tvdiff, &packet->timestamp, &nextRestart);
uint32_t msdiff = tvdiff.tv_sec*1000+tvdiff.tv_usec/1000;
uint32_t mswait = (msdiff/restartInterval+1)*restartInterval;
tvdiff.tv_sec = mswait/1000;
tvdiff.tv_usec = (mswait%1000)*1000;
timeval_add(&nextRestart, &tvdiff);
}
}
// write packet
static struct pcap_pkthdr packetHeader;
packetHeader.ts = packet->timestamp;
packetHeader.caplen = packet->data_length;
packetHeader.len = packet->pcapPacketLength;
struct iovec wvec[2];
wvec[0].iov_base = &packetHeader;
wvec[0].iov_len = sizeof(packetHeader);
wvec[1].iov_base = packet->data;
wvec[1].iov_len = packetHeader.caplen;
if (writev(pcapFile, wvec, 2)!=(ssize_t)(sizeof(packetHeader)+packetHeader.caplen)) {
if (errno==EAGAIN) {
// pipe is full, drop packet
statBytesDropped += packet->data_length;
statPktsDropped++;
} else
THROWEXCEPTION("PCAPExporterPipe: failed to write, error %u (%s)", errno, strerror(errno));
} else {
statBytesForwarded += packet->data_length;
statPktsForwarded++;
}
packet->removeReference();
DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive() ended");
}
/**
* @returns an index for the given field type which is relative to the start of the Packet's netheader
*/
static const uint16_t getRawPacketFieldIndex(uint16_t typeId, const Packet* p)
{
switch (typeId) {
case IPFIX_TYPEID_packetDeltaCount:
return 10;
break;
case IPFIX_TYPEID_flowStartSeconds:
case IPFIX_TYPEID_flowEndSeconds:
return reinterpret_cast<const unsigned char*>(&p->time_sec_nbo) - p->netHeader;
break;
case IPFIX_TYPEID_flowStartMilliSeconds:
case IPFIX_TYPEID_flowEndMilliSeconds:
case IPFIX_ETYPEID_maxPacketGap:
return reinterpret_cast<const unsigned char*>(&p->time_msec_nbo) - p->netHeader;
break;
case IPFIX_TYPEID_flowStartNanoSeconds:
case IPFIX_TYPEID_flowEndNanoSeconds:
return reinterpret_cast<const unsigned char*>(&p->timestamp) - p->netHeader;
break;
case IPFIX_TYPEID_octetDeltaCount:
return 2;
break;
case IPFIX_TYPEID_protocolIdentifier:
return 9;
break;
case IPFIX_TYPEID_sourceIPv4Address:
return 12;
break;
case IPFIX_TYPEID_destinationIPv4Address:
return 16;
break;
case IPFIX_TYPEID_icmpTypeCode:
if(p->ipProtocolType == Packet::ICMP) {
return p->transportHeader + 0 - p->netHeader;
} else {
DPRINTFL(MSG_VDEBUG, "given typeid is %d, protocol type is %d, but expected was %d or %d", typeId, p->ipProtocolType, Packet::UDP, Packet::TCP);
}
break;
case IPFIX_TYPEID_sourceTransportPort:
if((p->ipProtocolType == Packet::TCP) || (p->ipProtocolType == Packet::UDP)) {
return p->transportHeader + 0 - p->netHeader;
} else {
DPRINTFL(MSG_VDEBUG, "given typeid is %d, protocol type is %d, but expected was %d or %d", typeId, p->ipProtocolType, Packet::UDP, Packet::TCP);
}
break;
case IPFIX_TYPEID_destinationTransportPort:
if((p->ipProtocolType == Packet::TCP) || (p->ipProtocolType == Packet::UDP)) {
return p->transportHeader + 2 - p->netHeader;
} else {
DPRINTFL(MSG_VDEBUG, "given typeid is %d, protocol type is %d, but expected was %d or %d", typeId, p->ipProtocolType, Packet::UDP, Packet::TCP);
}
break;
case IPFIX_TYPEID_tcpControlBits:
if(p->ipProtocolType == Packet::TCP) {
return p->transportHeader + 13 - p->netHeader;
} else {
DPRINTFL(MSG_VDEBUG, "given typeid is %d, protocol type is %d, but expected was %d", typeId, p->ipProtocolType, Packet::TCP);
}
break;
}
// return just pointer to zero bytes as result
return reinterpret_cast<const unsigned char*>(&p->zeroBytes) - p->netHeader;
}
