// Called whenever there is data available on the device. void SignalChunker::next(QIODevice* device) { // Get a pointer to the UDP socket. QUdpSocket* udpSocket = static_cast<QUdpSocket*>(device); _bytesRead = 0; // Get writable buffer space for the chunk. WritableData writableData = getDataStorage(_chunkSize); if (writableData.isValid()) { // Get pointer to start of writable memory. char* ptr = (char*) (writableData.ptr()); // Read datagrams for chunk from the UDP socket. while (isActive() && _bytesRead < _chunkSize) { // Read the datagram, but avoid using pendingDatagramSize(). if (!udpSocket->hasPendingDatagrams()) { // MUST WAIT for the next datagram. udpSocket->waitForReadyRead(100); continue; } qint64 maxlen = _chunkSize - _bytesRead; qint64 length = udpSocket->readDatagram(ptr + _bytesRead, maxlen); if (length > 0) _bytesRead += length; } } // Must discard the datagram if there is no available space. else { udpSocket->readDatagram(0, 0); } }
/** * @details * Gets the next chunk of data from the device (if it exists). */ void TestChunker::next(QIODevice* device) { // Read data off the device. device->open(QIODevice::ReadOnly); QByteArray array = device->readAll(); device->close(); if (array.size() == 0) { std::cout << "Nothing to read!" << std::endl; return; } // Get writable data object. WritableData writableData(0); try { writableData = getDataStorage(_size); } catch (const QString& e) { std::cout << "Unexpected exception: " << e.toStdString() << std::endl; return; } // If the writable data object is not valid, then return. if (!writableData.isValid()) return; // ... or, block until writable data is valid. while (!writableData.isValid()) { writableData = getDataStorage(_size); usleep(1); } // Write some test data. writableData.write((void*)array.data(), _size, 0); }
void EmbraceChunker::next(QIODevice* device) { QAbstractSocket* socket = static_cast<QAbstractSocket*>(device); WritableData writableData = getDataStorage(_sizeOfFrame); int readTotal = 0; if (writableData.isValid() ) { do { int read = socket->read((char*)writableData.ptr() + readTotal*sizeof(char) , _sizeOfFrame); if( read == -1 ) { std::cerr << "read error"; // TODO - clean up and annul writableData } readTotal += read; } while ( readTotal < _sizeOfFrame ); } else { // TODO dump the data here std::cerr << "EmbraceChunker: " "WritableData is not valid!" << std::endl; } }
/** * @details * Gets the next chunk of data from the UDP socket (if it exists). */ void LofarDataSplittingChunker::next(QIODevice* device) { QUdpSocket* socket = static_cast<QUdpSocket*>(device); unsigned offsetStream1 = 0; unsigned offsetStream2 = 0; unsigned prevSeqid = _startTime; unsigned prevBlockid = _startBlockid; UDPPacket currPacket; UDPPacket outputPacket1; UDPPacket outputPacket2; UDPPacket _emptyPacket1; UDPPacket _emptyPacket2; WritableData writableData1 = getDataStorage(_nPackets * _packetSizeStream1, chunkTypes().at(0)); WritableData writableData2 = getDataStorage(_nPackets * _packetSizeStream2, chunkTypes().at(1)); unsigned seqid, blockid; unsigned totBlocks, lostPackets, diff; unsigned packetCounter; if (writableData1.isValid() && writableData2.isValid()) { // Loop over the number of UDP packets to put in a chunk. for (unsigned i = 0; i < _nPackets; ++i) { // Chunker sanity check. if (!isActive()) return; // Wait for datagram to be available. while (!socket->hasPendingDatagrams()) socket->waitForReadyRead(100); // Read the current packet from the socket. if (socket->readDatagram(reinterpret_cast<char*>(&currPacket), _packetSize) <= 0) { cerr << "LofarDataSplittingChunker::next(): " "Error while receiving UDP Packet!" << endl; i--; continue; } // Check for endianness (Packet data is in little endian format). #if Q_BYTE_ORDER == Q_BIG_ENDIAN // TODO: Convert from little endian to big endian. throw QString("LofarDataSplittingChunker: Endianness not supported."); seqid = currPacket.header.timestamp; std::cout << seqid << std::endl; blockid = currPacket.header.blockSequenceNumber; #elif Q_BYTE_ORDER == Q_LITTLE_ENDIAN seqid = currPacket.header.timestamp; blockid = currPacket.header.blockSequenceNumber; #endif // First time next has been run, initialise startTime and startBlockId. if (i == 0 && _startTime == 0) { prevSeqid = _startTime = _startTime == 0 ? seqid : _startTime; prevBlockid = _startBlockid = _startBlockid == 0 ? blockid : _startBlockid; } // Sanity check in seqid. If the seconds counter is 0xFFFFFFFF, // the data cannot be trusted (ignore). if (seqid == ~0U || prevSeqid + 10 < seqid) { _packetsRejected++; i--; continue; } // Check that the packets are contiguous. // Block id increments by nrblocks which is defined in the header. // Blockid is reset every interval (although it might not start // from 0 as the previous frame might contain data from this one). totBlocks = (_clock == 160) ? 156250 : (prevSeqid % 2 == 0 ? 195313 : 195312); lostPackets = 0; diff = (blockid >= prevBlockid) ? (blockid - prevBlockid) : (blockid + totBlocks - prevBlockid); // Duplicated packets... ignore if (diff < _nSamples) { ++_packetsRejected; i -= 1; continue; } // Missing packets else if (diff > _nSamples) { // -1 since it includes this includes the received packet as well lostPackets = (diff / _nSamples) - 1; } if (lostPackets > 0) { printf("Generate %u empty packets, prevSeq: %u, new Seq: %u, prevBlock: %u, newBlock: %u\n", lostPackets, prevSeqid, seqid, prevBlockid, blockid); } // TODO // BELOW HERE WRITE INTO WRITABLE DATA 1 and 2 correctly. // = Missing packets -> write pair of empty data packets. // = Full packets -> split the packet into the two buffers. // ================================================================= // Generate lostPackets (empty packets) if needed. packetCounter = 0; for (packetCounter = 0; packetCounter < lostPackets && i + packetCounter < _nPackets; ++packetCounter) { // Generate empty packet with correct seqid and blockid prevSeqid = (prevBlockid + _nSamples < totBlocks) ? prevSeqid : prevSeqid + 1; prevBlockid = (prevBlockid + _nSamples) % totBlocks; // TODO: probably a cost saving here. updateEmptyPacket(_emptyPacket1, prevSeqid, prevBlockid); updateEmptyPacket(_emptyPacket2, prevSeqid, prevBlockid); offsetStream1 = writePacket(&writableData1, _emptyPacket1, _packetSizeStream1, offsetStream1); offsetStream2 = writePacket(&writableData2, _emptyPacket2, _packetSizeStream2, offsetStream2); // Check if the number of required packets is reached // TODO writePacket(&writableData2, emptyPacket, offset); } i += packetCounter; // Write received packet to 2 streams after updating header and data if (i != _nPackets) { ++_packetsAccepted; // Generate Stream 1 packet outputPacket1.header = currPacket.header; outputPacket1.header.nrBeamlets = _stream1Subbands; memcpy((void*)outputPacket1.data, &currPacket.data[_byte1OfStream1], _bytesStream1); offsetStream1 = writePacket(&writableData1, outputPacket1, _packetSizeStream1, offsetStream1); // Generate Stream 2 packet //-------------------------------------------------------------- outputPacket2.header = currPacket.header; outputPacket2.header.nrBeamlets = _stream2Subbands; memcpy((void*)outputPacket2.data, &currPacket.data[_byte1OfStream2], _bytesStream2); offsetStream2 = writePacket(&writableData2, outputPacket2, _packetSizeStream2, offsetStream2); prevSeqid = seqid; prevBlockid = blockid; } } } else { // Must discard the datagram if there is no available space. if (!isActive()) return; socket->readDatagram(0, 0); cout << "LofarDataSplittingChunker::next(): " "Writable data not valid, discarding packets." << endl; } // Update _startTime _startTime = prevSeqid; _startBlockid = prevBlockid; }
// Gets the next chunk of data from the UDP socket (if it exists). void K7Chunker::next(QIODevice* device) { unsigned int i; unsigned int lostPackets, difference; unsigned int offsetStream; unsigned int lostPacketCounter; unsigned long int previousTimestamp; unsigned int previousAccumulation; unsigned long int timestamp; unsigned int accumulation; unsigned int rate; QUdpSocket* udpSocket = static_cast<QUdpSocket*>(device); K7Packet currentPacket; K7Packet outputPacket; K7Packet _emptyPacket; difference = 0; offsetStream = 0; timestamp = 0; accumulation = 0; rate = 0; previousTimestamp = _startTimestamp; previousAccumulation = _startAccumulation; // Get writable buffer space for the chunk. WritableData writableData; if ( ! _writable.isValid() ) { writableData = getDataStorage(_nPackets * _packetSizeStream, chunkTypes().at(0)); } else { writableData = _writable; } if (writableData.isValid()) { // Loop over the number of UDP packets to put in a chunk. for (i = 0; i < _nPackets; ++i) { // Chunker sanity check. if (!isActive()) return; // Wait for datagram to be available. while ( !udpSocket->hasPendingDatagrams() ) { _writable = writableData; continue; } // Read the current UDP packet from the socket. if (udpSocket->readDatagram(reinterpret_cast<char*>(¤tPacket), _packetSize) <= 0) { std::cerr << "K7Chunker::next(): Error while receiving UDP Packet!" << std::endl; i--; continue; } // Get the UDP packet header. timestamp = currentPacket.header.UTCtimestamp; accumulation = currentPacket.header.accumulationNumber; rate = currentPacket.header.accumulationRate; //std::cout << "K7Chunker::next(): timestamp " << timestamp << " accumulation " << accumulation << " rate " << rate << std::endl; // First time next() has been run, initialise _startTimestamp and _startAccumulation. if (i == 0 && _startTimestamp == 0) { previousTimestamp = _startTimestamp = _startTimestamp == 0 ? timestamp : _startTimestamp; previousAccumulation = _startAccumulation = _startAccumulation == 0 ? accumulation : _startAccumulation; //std::cout << "K7Chunker::next(): timestamp " << timestamp << " accumulation " << accumulation << " rate " << rate << std::endl; } // Sanity check in UTCtimestamp. If the seconds counter is 0xFFFFFFFFFFFFFFFF, the data cannot be trusted (ignore). if (timestamp == ~0UL || previousTimestamp + 10 < timestamp) { std::cerr << "K7Chunker::next(): Data cannot be trusted! Timestamp is " << timestamp << " or previousTimestamp is " << previousTimestamp << std::endl; exit(-1); } // Check that the packets are contiguous. accumulationNumber increments by // accumulationRate which is defined in the header of UDP packet. // accumulationNumber is reset every interval (although it might not start from 0 // as the previous frame might contain data from this one). lostPackets = 0; difference = (accumulation >= previousAccumulation) ? (accumulation - previousAccumulation) : (accumulation + _packetsPerSecond - previousAccumulation); #if 0 // Duplicated packets. Need to address this. ICBF does not duplicate packets though. if (difference < rate) { std::cout << "Duplicated packets, difference " << difference << std::endl; ++_packetsRejected; i -= 1; continue; } else #endif // Missing packets. if (difference > rate) { // -1 since it includes the received packet as well. lostPackets = (difference / rate) - 1; } if (lostPackets > 0) { printf("K7Chunker::next(): generate %u empty packets, previousTimestamp: %lu, new timestamp: %lu, prevAccumulation: %u, newAccumulation: %u\n", lostPackets, previousTimestamp, timestamp, previousAccumulation, accumulation); } // Generate lostPackets (empty packets) if needed. lostPacketCounter = 0; for (lostPacketCounter = 0; lostPacketCounter < lostPackets && i + lostPacketCounter < _nPackets; ++lostPacketCounter) { // Generate empty packet with correct timestamp and accumulation number. previousTimestamp = (previousAccumulation < _packetsPerSecond) ? previousTimestamp : previousTimestamp + 1; previousAccumulation = previousAccumulation % _packetsPerSecond; updateEmptyPacket(_emptyPacket, previousTimestamp, previousAccumulation, rate); offsetStream = writePacket(&writableData, _emptyPacket, _packetSizeStream, offsetStream); } i += lostPacketCounter; // Write received packet to stream after updating header and data. if (i != _nPackets) { ++_packetsAccepted; // Generate stream packet. outputPacket.header = currentPacket.header; memcpy((void*)outputPacket.data, ¤tPacket.data[_byte1OfStream], _bytesStream); offsetStream = writePacket(&writableData, outputPacket, _packetSizeStream, offsetStream); previousTimestamp = timestamp; previousAccumulation = accumulation; } } _chunksProcessed++; _chunkerCounter++; // Clear any data locks. _writable = WritableData(); if (_chunkerCounter % 100 == 0) { std::cout << "K7Chunker::next(): " << _chunksProcessed << " chunks processed. " << "UTC timestamp " << timestamp << " accumulationNumber " << accumulation << " accumulationRate " << rate << std::endl; } } else { // Must discard the datagram if there is no available space. if (!isActive()) return; udpSocket->readDatagram(0, 0); std::cout << "K7Chunker::next(): Writable data not valid, discarding packets." << std::endl; } // Update _startTime. _startTimestamp = previousTimestamp; _startAccumulation = previousAccumulation; }