void PacketHashtable::copyDataNanoseconds(IpfixRecord::Data* bucket, const IpfixRecord::Data* src, ExpFieldData* efd) { uint64_t ntptime; ntptime = ntp64timegcc(*reinterpret_cast<const struct timeval*>(src)); uint64_t ntp2 = htonll(ntptime); DPRINTFL(MSG_VDEBUG, "ntp2: %llu, ntptime/ntp2 %llX/%llX", ntp2, ntptime, ntp2); memcpy(bucket+efd->dstIndex, &ntp2, sizeof(ntp2)); #ifdef DEBUG if (ntohll(*(uint64_t*)(bucket+efd->dstIndex))<(1000000000ULL+(2208988800ULL<<32)) || ntohll(*(uint64_t*)(bucket+efd->dstIndex))>(1300000000ULL+(2208988800ULL<<32))) { DPRINTFL(MSG_VDEBUG, "time before: %ds", reinterpret_cast<const struct timeval*>(src)->tv_sec); DPRINTFL(MSG_VDEBUG, "copy invalid end nano seconds: %lld s (%llX)", (ntohll(*(uint64_t*)(bucket+efd->dstIndex))>>32)-2208988800U, *(uint64_t*)(bucket+efd->dstIndex)); }
/* * this is the main filter function: each packet is run thru a series of * PacketProcessors and then pushed to the next PacketSource */ void FilterModule::receive(Packet* p) { vector<PacketProcessor *>::iterator it; // set to true so packet is always pushed to receiver // even if the processors-iterator below is NULL bool keepPacket=true; DPRINTFL(MSG_VDEBUG, "FilterModule: got packet"); // run packet through all packetProcessors for (it = processors.begin(); it != processors.end() && keepPacket; ++it) { keepPacket = (*it)->processPacket(p); } // check if we passed all filters if (keepPacket) { // push packet to the receiver DPRINTF("FilterModule: pushing packet %d", p); while (!exitFlag && !send(p)); return; } // immediately drop the packet DPRINTF("FilterModule: releasing packet"); p->removeReference(); }
/** * Receives a packet and tries to write the packet into the ringbuffer */ void PCAPExporterMem::receive(Packet* packet) { DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive() called"); if (onRestart) { DPRINTF("Dropping incoming packet, as attached process is not ready"); DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive() ended"); packet->removeReference(); return; } if (fifoReaderPid == 0) { msg(MSG_VDEBUG, "fifoReaderPid = 0...this might happen during reconfiguration"); DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive() ended"); packet->removeReference(); return; } if (restartInterval) { if (nextRestart.tv_sec==0) { DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive(): updating nextRestart"); nextRestart = packet->timestamp; nextRestart.tv_sec += restartInterval; } else if (compareTime(nextRestart, packet->timestamp)<0) { DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive(): restarting process"); // we need to unregister our signal handlers, as we get race conditions with the signal handler for restarting the process unregisterSignalHandlers(); stopProcess(); startProcess(); registerSignalHandlers(); DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive(): updating nextRestart"); nextRestart.tv_sec += ((packet->timestamp.tv_sec-nextRestart.tv_sec)/restartInterval+1)*restartInterval; } } if (writeIntoMemory(packet)) { statPktsForwarded++; statBytesForwarded += packet->data_length; DPRINTFL(MSG_VDEBUG, "Wrote packet at pos %u", *nextWrite); } else { batchUpdate(); statPktsDropped++; statBytesDropped += packet->data_length; DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive(): dropped packet"); } packet->removeReference(); DPRINTFL(MSG_VDEBUG, "PCAPExporterMem::receive() ended"); }
/** * Buffer passed flow in Hashtable @c ht */ void FlowHashtable::aggregateTemplateData(IpfixRecord::TemplateInfo* ti, IpfixRecord::Data* data) { // the following lock should almost never fail (only during reconfiguration) while (atomic_lock(&aggInProgress)) { timespec req; req.tv_sec = 0; req.tv_nsec = 50000000; nanosleep(&req, &req); } int i; /* Create data block to be inserted into buffer... */ boost::shared_array<IpfixRecord::Data> htdata(new IpfixRecord::Data[fieldLength+privDataLength]); // set private data fields to zero memset(htdata.get()+fieldLength, 0, privDataLength); for (i = 0; i < dataTemplate->fieldCount; i++) { IpfixRecord::FieldInfo* hfi = &dataTemplate->fieldInfo[i]; IpfixRecord::FieldInfo* tfi = ti->getFieldInfo(&hfi->type); if (!tfi) { DPRINTF("Flow to be buffered did not contain %s field\n", typeid2string(hfi->type.id)); // if field was not copied, fill it with 0 memset(htdata.get() + hfi->offset, 0, hfi->type.length); continue; } DPRINTFL(MSG_VDEBUG, "copyData for type %d, offset %x, starting from pointer %X", tfi->type.id, tfi->offset, data+tfi->offset); DPRINTFL(MSG_VDEBUG, "copyData to offset %X", hfi->offset); copyData(hfi, htdata.get(), tfi, data, fieldModifier[i]); /* copy associated mask, should there be one */ switch (hfi->type.id) { case IPFIX_TYPEID_sourceIPv4Address: tfi = ti->getFieldInfo(IPFIX_TYPEID_sourceIPv4Mask, 0); if(tfi) { if(hfi->type.length != 5) { DPRINTF("Tried to set mask of length %d IP address\n", hfi->type.length); } else { if(tfi->type.length == 1) { *(uint8_t*)(htdata.get() + hfi->offset + 4) = *(uint8_t*)(data + tfi->offset); } else { DPRINTF("Cannot process associated mask with invalid length %d\n", tfi->type.length); } } } break; case IPFIX_TYPEID_destinationIPv4Address: tfi = ti->getFieldInfo(IPFIX_TYPEID_destinationIPv4Mask, 0); if(tfi) { if(hfi->type.length != 5) { DPRINTF("Tried to set mask of length %d IP address\n", hfi->type.length); } else { if(tfi->type.length == 1) { *(uint8_t*)(htdata.get() + hfi->offset + 4) = *(uint8_t*)(data + tfi->offset); } else { DPRINTF("Cannot process associated mask with invalid length %d\n", tfi->type.length); } } } break; default: break; } } /* ...then buffer it */ bufferDataBlock(htdata); atomic_release(&aggInProgress); }
/** * Inserts a data block into the hashtable */ void FlowHashtable::bufferDataBlock(boost::shared_array<IpfixRecord::Data> data) { statRecordsReceived++; uint32_t nhash = getHash(data.get(), false); DPRINTFL(MSG_VDEBUG, "nhash=%u", nhash); HashtableBucket* prevbucket; HashtableBucket* bucket = lookupBucket(nhash, data.get(), false, &prevbucket); if (bucket != NULL) { DPRINTFL(MSG_VDEBUG, "aggregating flow"); aggregateFlow(bucket->data.get(), data.get(), false); bucket->expireTime = time(0) + minBufferTime; if (bucket->forceExpireTime>bucket->expireTime) { exportList.remove(bucket->listNode); exportList.push(bucket->listNode); } } else { if (biflowAggregation) { // try reverse flow uint32_t rhash = getHash(data.get(), true); DPRINTFL(MSG_VDEBUG, "rhash=%u", rhash); bucket = lookupBucket(rhash, data.get(), true, &prevbucket); if (bucket != NULL) { DPRINTFL(MSG_VDEBUG, "aggregating reverse flow"); if (aggregateFlow(bucket->data.get(), data.get(), true)==1) { DPRINTFL(MSG_VDEBUG, "reversing whole flow"); // reverse flow reverseFlowBucket(bucket); // delete reference from hash table if (prevbucket==NULL) buckets[rhash] = bucket->next; else { prevbucket->next = bucket->next; } if (bucket->next!=NULL) bucket->next->prev = prevbucket; // insert into hash table again nhash = getHash(bucket->data.get(), false); DPRINTFL(MSG_VDEBUG, "nhash=%u", nhash); bucket->next = buckets[nhash]; bucket->hash = nhash; buckets[nhash] = bucket; bucket->prev = 0; if (bucket->next != NULL) bucket->next->prev = bucket; bucket->expireTime = time(0) + minBufferTime; if (bucket->forceExpireTime>bucket->expireTime) { exportList.remove(bucket->listNode); exportList.push(bucket->listNode); } } } } if (bucket == NULL) { DPRINTFL(MSG_VDEBUG, "creating new bucket"); HashtableBucket* n = buckets[nhash]; buckets[nhash] = createBucket(data, 0, n, 0, nhash); // FIXME: insert observationDomainID! if (n != NULL) n->prev = buckets[nhash]; BucketListElement* node = hbucketIM.getNewInstance(); node->reset(); buckets[nhash]->listNode = node; node->bucket = buckets[nhash]; exportList.push(node); } } atomic_release(&aggInProgress); }
/** * Writes a packet into the pipe */ void PCAPExporterPipe::receive(Packet* packet) { DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive() called"); if (onRestart){ DPRINTF("Dropping incoming packet, as attached process is not ready"); DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive() ended"); packet->removeReference(); return; } if (fifoReaderPid == 0){ msg(MSG_VDEBUG, "fifoReaderPid = 0...this might happen during reconfiguration"); DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive() ended"); packet->removeReference(); return; } if (restartInterval) { if (nextRestart.tv_sec==0) { DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive(): updating nextRestart"); nextRestart = packet->timestamp; struct timeval tv = { restartInterval/1000, (restartInterval % 1000)*1000 }; timeval_add(&nextRestart, &tv); } else if (compareTime(nextRestart, packet->timestamp)<0) { DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive(): restarting process"); // we need to unregister our signal handlers, as we get race conditions with the signal handler for restarting the process unregisterSignalHandlers(); stopProcess(); startProcess(); registerSignalHandlers(); DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive(): updating nextRestart"); timeval tvdiff; timeval_subtract(&tvdiff, &packet->timestamp, &nextRestart); uint32_t msdiff = tvdiff.tv_sec*1000+tvdiff.tv_usec/1000; uint32_t mswait = (msdiff/restartInterval+1)*restartInterval; tvdiff.tv_sec = mswait/1000; tvdiff.tv_usec = (mswait%1000)*1000; timeval_add(&nextRestart, &tvdiff); } } // write packet static struct pcap_pkthdr packetHeader; packetHeader.ts = packet->timestamp; packetHeader.caplen = packet->data_length; packetHeader.len = packet->pcapPacketLength; struct iovec wvec[2]; wvec[0].iov_base = &packetHeader; wvec[0].iov_len = sizeof(packetHeader); wvec[1].iov_base = packet->data; wvec[1].iov_len = packetHeader.caplen; if (writev(pcapFile, wvec, 2)!=(ssize_t)(sizeof(packetHeader)+packetHeader.caplen)) { if (errno==EAGAIN) { // pipe is full, drop packet statBytesDropped += packet->data_length; statPktsDropped++; } else THROWEXCEPTION("PCAPExporterPipe: failed to write, error %u (%s)", errno, strerror(errno)); } else { statBytesForwarded += packet->data_length; statPktsForwarded++; } packet->removeReference(); DPRINTFL(MSG_VDEBUG, "PCAPExporterPipe::receive() ended"); }
/** * @returns an index for the given field type which is relative to the start of the Packet's netheader */ static const uint16_t getRawPacketFieldIndex(uint16_t typeId, const Packet* p) { switch (typeId) { case IPFIX_TYPEID_packetDeltaCount: return 10; break; case IPFIX_TYPEID_flowStartSeconds: case IPFIX_TYPEID_flowEndSeconds: return reinterpret_cast<const unsigned char*>(&p->time_sec_nbo) - p->netHeader; break; case IPFIX_TYPEID_flowStartMilliSeconds: case IPFIX_TYPEID_flowEndMilliSeconds: case IPFIX_ETYPEID_maxPacketGap: return reinterpret_cast<const unsigned char*>(&p->time_msec_nbo) - p->netHeader; break; case IPFIX_TYPEID_flowStartNanoSeconds: case IPFIX_TYPEID_flowEndNanoSeconds: return reinterpret_cast<const unsigned char*>(&p->timestamp) - p->netHeader; break; case IPFIX_TYPEID_octetDeltaCount: return 2; break; case IPFIX_TYPEID_protocolIdentifier: return 9; break; case IPFIX_TYPEID_sourceIPv4Address: return 12; break; case IPFIX_TYPEID_destinationIPv4Address: return 16; break; case IPFIX_TYPEID_icmpTypeCode: if(p->ipProtocolType == Packet::ICMP) { return p->transportHeader + 0 - p->netHeader; } else { DPRINTFL(MSG_VDEBUG, "given typeid is %d, protocol type is %d, but expected was %d or %d", typeId, p->ipProtocolType, Packet::UDP, Packet::TCP); } break; case IPFIX_TYPEID_sourceTransportPort: if((p->ipProtocolType == Packet::TCP) || (p->ipProtocolType == Packet::UDP)) { return p->transportHeader + 0 - p->netHeader; } else { DPRINTFL(MSG_VDEBUG, "given typeid is %d, protocol type is %d, but expected was %d or %d", typeId, p->ipProtocolType, Packet::UDP, Packet::TCP); } break; case IPFIX_TYPEID_destinationTransportPort: if((p->ipProtocolType == Packet::TCP) || (p->ipProtocolType == Packet::UDP)) { return p->transportHeader + 2 - p->netHeader; } else { DPRINTFL(MSG_VDEBUG, "given typeid is %d, protocol type is %d, but expected was %d or %d", typeId, p->ipProtocolType, Packet::UDP, Packet::TCP); } break; case IPFIX_TYPEID_tcpControlBits: if(p->ipProtocolType == Packet::TCP) { return p->transportHeader + 13 - p->netHeader; } else { DPRINTFL(MSG_VDEBUG, "given typeid is %d, protocol type is %d, but expected was %d", typeId, p->ipProtocolType, Packet::TCP); } break; } // return just pointer to zero bytes as result return reinterpret_cast<const unsigned char*>(&p->zeroBytes) - p->netHeader; }