/**
 * @details
 * Gets the next chunk of data from the UDP socket (if it exists).
 */
void LofarDataSplittingChunker::next(QIODevice* device)
{
    QUdpSocket* socket = static_cast<QUdpSocket*>(device);

    unsigned offsetStream1 = 0;
    unsigned offsetStream2 = 0;
    unsigned prevSeqid = _startTime;
    unsigned prevBlockid = _startBlockid;
    UDPPacket currPacket;
    UDPPacket outputPacket1;
    UDPPacket outputPacket2;
    UDPPacket _emptyPacket1;
    UDPPacket _emptyPacket2;

    WritableData writableData1 = getDataStorage(_nPackets * _packetSizeStream1,
            chunkTypes().at(0));
    WritableData writableData2 = getDataStorage(_nPackets * _packetSizeStream2,
            chunkTypes().at(1));

    unsigned seqid, blockid;
    unsigned totBlocks, lostPackets, diff;
    unsigned packetCounter;

    if (writableData1.isValid() && writableData2.isValid())
    {
        // Loop over the number of UDP packets to put in a chunk.
        for (unsigned i = 0; i < _nPackets; ++i)
        {
            // Chunker sanity check.
            if (!isActive()) return;

            // Wait for datagram to be available.
            while (!socket->hasPendingDatagrams())
                socket->waitForReadyRead(100);

            // Read the current packet from the socket.
            if (socket->readDatagram(reinterpret_cast<char*>(&currPacket), _packetSize) <= 0)
            {
                cerr << "LofarDataSplittingChunker::next(): "
                        "Error while receiving UDP Packet!" << endl;
                i--;
                continue;
            }

            // Check for endianness (Packet data is in little endian format).
#if Q_BYTE_ORDER == Q_BIG_ENDIAN
            // TODO: Convert from little endian to big endian.
            throw QString("LofarDataSplittingChunker: Endianness not supported.");
            seqid   = currPacket.header.timestamp;
            std::cout << seqid << std::endl;
            blockid = currPacket.header.blockSequenceNumber;
#elif Q_BYTE_ORDER == Q_LITTLE_ENDIAN
            seqid   = currPacket.header.timestamp;
            blockid = currPacket.header.blockSequenceNumber;
#endif

            // First time next has been run, initialise startTime and startBlockId.
            if (i == 0 && _startTime == 0) {
                prevSeqid = _startTime = _startTime == 0 ? seqid : _startTime;
                prevBlockid = _startBlockid = _startBlockid == 0 ? blockid : _startBlockid;
            }

            // Sanity check in seqid. If the seconds counter is 0xFFFFFFFF,
            // the data cannot be trusted (ignore).
            if (seqid == ~0U || prevSeqid + 10 < seqid)
            {
                _packetsRejected++;
                i--;
                continue;
            }

            // Check that the packets are contiguous.
            // Block id increments by nrblocks which is defined in the header.
            // Blockid is reset every interval (although it might not start
            // from 0 as the previous frame might contain data from this one).
            totBlocks = (_clock == 160) ?
                    156250 : (prevSeqid % 2 == 0 ? 195313 : 195312);
            lostPackets = 0;
            diff = (blockid >= prevBlockid) ?
                    (blockid - prevBlockid) : (blockid + totBlocks - prevBlockid);

            // Duplicated packets... ignore
            if (diff < _nSamples)
            {
                ++_packetsRejected;
                i -= 1;
                continue;
            }
            // Missing packets
            else if (diff > _nSamples)
            {
                // -1 since it includes this includes the received packet as well
                lostPackets = (diff / _nSamples) - 1;
            }


            if (lostPackets > 0)
            {
                printf("Generate %u empty packets, prevSeq: %u, new Seq: %u, prevBlock: %u, newBlock: %u\n",
                        lostPackets, prevSeqid, seqid, prevBlockid, blockid);
            }

            // TODO
            // BELOW HERE WRITE INTO WRITABLE DATA 1 and 2 correctly.
            //   = Missing packets -> write pair of empty data packets.
            //   = Full packets -> split the packet into the two buffers.
            // =================================================================

            // Generate lostPackets (empty packets) if needed.
            packetCounter = 0;
            for (packetCounter = 0; packetCounter < lostPackets && i + packetCounter < _nPackets; ++packetCounter)
            {
                // Generate empty packet with correct seqid and blockid
                prevSeqid = (prevBlockid + _nSamples < totBlocks) ?
                        prevSeqid : prevSeqid + 1;
                prevBlockid = (prevBlockid + _nSamples) % totBlocks;
                // TODO: probably a cost saving here.
                updateEmptyPacket(_emptyPacket1, prevSeqid, prevBlockid);
                updateEmptyPacket(_emptyPacket2, prevSeqid, prevBlockid);
                offsetStream1 = writePacket(&writableData1, _emptyPacket1, _packetSizeStream1, offsetStream1);
                offsetStream2 = writePacket(&writableData2, _emptyPacket2, _packetSizeStream2, offsetStream2);

                // Check if the number of required packets is reached

                // TODO writePacket(&writableData2, emptyPacket, offset);
            }

            i += packetCounter;

            // Write received packet to 2 streams after updating header and data
            if (i != _nPackets)
            {
                ++_packetsAccepted;

                // Generate Stream 1 packet
                outputPacket1.header = currPacket.header;
                outputPacket1.header.nrBeamlets = _stream1Subbands;
                memcpy((void*)outputPacket1.data, &currPacket.data[_byte1OfStream1], _bytesStream1);
                offsetStream1 = writePacket(&writableData1, outputPacket1, _packetSizeStream1, offsetStream1);

                // Generate Stream 2 packet
                //--------------------------------------------------------------
                outputPacket2.header = currPacket.header;
                outputPacket2.header.nrBeamlets = _stream2Subbands;
                memcpy((void*)outputPacket2.data, &currPacket.data[_byte1OfStream2], _bytesStream2);
                offsetStream2 = writePacket(&writableData2, outputPacket2, _packetSizeStream2, offsetStream2);

                prevSeqid = seqid;
                prevBlockid = blockid;
            }
        }
    }

    else {
        // Must discard the datagram if there is no available space.
        if (!isActive()) return;
        socket->readDatagram(0, 0);
        cout << "LofarDataSplittingChunker::next(): "
                "Writable data not valid, discarding packets." << endl;
    }

    // Update _startTime
    _startTime = prevSeqid;
    _startBlockid = prevBlockid;
}
/**
 * @details
 * Constructor
 *
 * XML options.
 * ================================
 *
 * Read from base class:
 *  - data (type)  LIST.
 *  - connection (host, port)
 *
 * Class Specific options:
 *  - samplesPerPacket (value)
 *  - subbandsPerPacket (value)
 *  - nRawPolarisations (value)
 *
 *  - udpPacketsPerIteration (value)
 *  - clock (value)
 *  - dataBitSize (value)
 *
 */
LofarDataSplittingChunker::LofarDataSplittingChunker(const ConfigNode& config)
: AbstractChunker(config)
{
    // Check the configuration type matches the class name.
    if (config.type() != "LofarDataSplittingChunker")
        throw _err("LofarDataSplittingChunker(): "
                "Invalid or missing XML configuration.");

    // Retrieve configuration options.
    // -------------------------------------------------------------------------
    // Packet dimensions.
    _nSamples = config.getOption("samplesPerPacket", "value").toUInt();
    // Total number of subbands per incoming packet
    _nSubbands = config.getOption("subbandsPerPacket", "value").toUInt();
    // Details of two streams to process, split into subband ranges
    _stream1SubbandStart = config.getOption("Stream1", "subbandStart").toUInt();
    _stream1SubbandEnd = config.getOption("Stream1", "subbandEnd").toUInt();
    _stream2SubbandStart = config.getOption("Stream2", "subbandStart").toUInt();
    _stream2SubbandEnd = config.getOption("Stream2", "subbandEnd").toUInt();

    _nPolarisations = config.getOption("nRawPolarisations", "value").toUInt();
    // Number of UDP packets collected into one chunk (iteration of the pipeline).
    _nPackets = config.getOption("udpPacketsPerIteration", "value").toUInt();
    // Clock => sample rate.
    _clock = config.getOption("clock", "value").toUInt();

    // Calculate the packet data size and the size of the packets for
    // two output streams
    _stream1Subbands = _stream1SubbandEnd - _stream1SubbandStart + 1;
    _stream2Subbands = _stream2SubbandEnd - _stream2SubbandStart + 1;
    if (_stream1SubbandEnd > _nSubbands || _stream2SubbandEnd > _nSubbands)
        throw _err("Subband ranges exceed number of subbands");


    size_t headerSize = sizeof(struct UDPPacket::Header);
    _packetSize = _nSubbands * _nSamples * _nPolarisations;
    // And the output streams
    _packetSizeStream1 = _stream1Subbands * _nSamples * _nPolarisations;
    _packetSizeStream2 = _stream2Subbands * _nSamples * _nPolarisations;
    unsigned sampleBits = config.getOption("dataBitSize", "value").toUInt();
    switch (sampleBits)
    {
        case 8:
            _packetSize = _packetSize * sizeof(TYPES::i8complex) + headerSize;
            _packetSizeStream1 = _packetSizeStream1 * sizeof(TYPES::i8complex) + headerSize;
            _packetSizeStream2 = _packetSizeStream2 * sizeof(TYPES::i8complex) + headerSize;
            _bytesStream1 = _packetSizeStream1 - headerSize;
            _bytesStream2 = _packetSizeStream2 - headerSize;
            _byte1OfStream1 = _stream1SubbandStart * _nSamples * _nPolarisations * sizeof(TYPES::i8complex);
            _byte1OfStream2 = _stream2SubbandStart * _nSamples * _nPolarisations * sizeof(TYPES::i8complex);
            break;
        case 16:
            _packetSize = _packetSize * sizeof(TYPES::i16complex) + headerSize;
            _packetSizeStream1 = _packetSizeStream1 * sizeof(TYPES::i16complex) + headerSize;
            _packetSizeStream2 = _packetSizeStream2 * sizeof(TYPES::i16complex) + headerSize;
            _bytesStream1 = _packetSizeStream1 - headerSize;
            _bytesStream2 = _packetSizeStream2 - headerSize;
            _byte1OfStream1 = _stream1SubbandStart * _nSamples * _nPolarisations * sizeof(TYPES::i16complex);
            _byte1OfStream2 = _stream2SubbandStart * _nSamples * _nPolarisations * sizeof(TYPES::i16complex);
            break;
        default:
            throw _err("LofarDataSplittingChunker(): "
                    "Unsupported number of data bits.");
    }

    // Initialise class variables.
    _startTime = _startBlockid = 0;
    _packetsAccepted = 0;
    _packetsRejected = 0;

    // Check a number of data chunk types are registered to be written.
    // These are set in the XML.
    if (type().isEmpty())
        throw _err("LofarDataSplittingChunker(): Data type unspecified.");

    if (chunkTypes().size() != 2)
        throw _err("LofarDataSplittingChunker(): "
                "Chunk types missing, expecting 2.");

    // Set the empty packet data for stream 1
    memset((void*)_emptyPacket1.data, 0, _bytesStream1);
    _emptyPacket1.header.nrBeamlets = _stream1Subbands;
    _emptyPacket1.header.nrBlocks = _nSamples;

    // Set the empty packet data for stream 2
    memset((void*)_emptyPacket2.data, 0, _bytesStream2);
    _emptyPacket2.header.nrBeamlets = _stream2Subbands;
    _emptyPacket2.header.nrBlocks = _nSamples;
}
Exemplo n.º 3
0
// Gets the next chunk of data from the UDP socket (if it exists).
void K7Chunker::next(QIODevice* device)
{
    unsigned int i;
    unsigned int lostPackets, difference;
    unsigned int offsetStream;
    unsigned int lostPacketCounter;
    unsigned long int previousTimestamp;
    unsigned int previousAccumulation;
    unsigned long int timestamp;
    unsigned int accumulation;
    unsigned int rate;
    QUdpSocket* udpSocket = static_cast<QUdpSocket*>(device);
    K7Packet currentPacket;
    K7Packet outputPacket;
    K7Packet _emptyPacket;

    difference = 0;
    offsetStream = 0;
    timestamp = 0;
    accumulation = 0;
    rate = 0;
    previousTimestamp = _startTimestamp;
    previousAccumulation = _startAccumulation;

    // Get writable buffer space for the chunk.
    WritableData writableData;
    if ( ! _writable.isValid() )
    {
        writableData = getDataStorage(_nPackets * _packetSizeStream, chunkTypes().at(0));
    }
    else
    {
        writableData = _writable;
    }

    if (writableData.isValid())
    {
        // Loop over the number of UDP packets to put in a chunk.
        for (i = 0; i < _nPackets; ++i)
        {
            // Chunker sanity check.
            if (!isActive())
                return;

            // Wait for datagram to be available.
            while ( !udpSocket->hasPendingDatagrams() )
            {
                _writable = writableData;
                continue;
            }

            // Read the current UDP packet from the socket.
            if (udpSocket->readDatagram(reinterpret_cast<char*>(&currentPacket), _packetSize) <= 0)
            {
                std::cerr << "K7Chunker::next(): Error while receiving UDP Packet!" << std::endl;
                i--;
                continue;
            }

            // Get the UDP packet header.
            timestamp    = currentPacket.header.UTCtimestamp;
            accumulation = currentPacket.header.accumulationNumber;
            rate         = currentPacket.header.accumulationRate;
            //std::cout << "K7Chunker::next(): timestamp " << timestamp << " accumulation " << accumulation << " rate " << rate << std::endl;
            // First time next() has been run, initialise _startTimestamp and _startAccumulation.
            if (i == 0 && _startTimestamp == 0)
            {
                previousTimestamp = _startTimestamp = _startTimestamp == 0 ? timestamp : _startTimestamp;
                previousAccumulation = _startAccumulation = _startAccumulation == 0 ? accumulation : _startAccumulation;
                //std::cout << "K7Chunker::next(): timestamp " << timestamp << " accumulation " << accumulation << " rate " << rate << std::endl;
            }

            // Sanity check in UTCtimestamp. If the seconds counter is 0xFFFFFFFFFFFFFFFF, the data cannot be trusted (ignore).
            if (timestamp == ~0UL || previousTimestamp + 10 < timestamp)
            {
                std::cerr << "K7Chunker::next(): Data cannot be trusted! Timestamp is " << timestamp << " or previousTimestamp is " << previousTimestamp << std::endl;
                exit(-1);
            }

            // Check that the packets are contiguous. accumulationNumber increments by
            // accumulationRate which is defined in the header of UDP packet.
            // accumulationNumber is reset every interval (although it might not start from 0
            // as the previous frame might contain data from this one).
            lostPackets = 0;
            difference = (accumulation >= previousAccumulation) ? (accumulation - previousAccumulation) : (accumulation + _packetsPerSecond - previousAccumulation);
#if 0
            // Duplicated packets. Need to address this. ICBF does not duplicate packets though.
            if (difference < rate)
            {
                std::cout << "Duplicated packets, difference " << difference << std::endl;
                ++_packetsRejected;
                i -= 1;
                continue;
            }
            else
#endif
            // Missing packets.
            if (difference > rate)
            {
                // -1 since it includes the received packet as well.
                lostPackets = (difference / rate) - 1;
            }

            if (lostPackets > 0)
            {
                printf("K7Chunker::next(): generate %u empty packets, previousTimestamp: %lu, new timestamp: %lu, prevAccumulation: %u, newAccumulation: %u\n",
                        lostPackets, previousTimestamp, timestamp, previousAccumulation, accumulation);
            }

            // Generate lostPackets (empty packets) if needed.
            lostPacketCounter = 0;
            for (lostPacketCounter = 0; lostPacketCounter < lostPackets && i + lostPacketCounter < _nPackets; ++lostPacketCounter)
            {
                // Generate empty packet with correct timestamp and accumulation number.
                previousTimestamp = (previousAccumulation < _packetsPerSecond) ? previousTimestamp : previousTimestamp + 1;
                previousAccumulation = previousAccumulation % _packetsPerSecond;
                updateEmptyPacket(_emptyPacket, previousTimestamp, previousAccumulation, rate);
                offsetStream = writePacket(&writableData, _emptyPacket, _packetSizeStream, offsetStream);
            }
            i += lostPacketCounter;

            // Write received packet to stream after updating header and data.
            if (i != _nPackets)
            {
                ++_packetsAccepted;
                // Generate stream packet.
                outputPacket.header = currentPacket.header;
                memcpy((void*)outputPacket.data, &currentPacket.data[_byte1OfStream], _bytesStream);
                offsetStream = writePacket(&writableData, outputPacket, _packetSizeStream, offsetStream);
                previousTimestamp = timestamp;
                previousAccumulation = accumulation;
            }
        }
        _chunksProcessed++;
        _chunkerCounter++;
        // Clear any data locks.
        _writable = WritableData();
        if (_chunkerCounter % 100 == 0)
        {
            std::cout << "K7Chunker::next(): " << _chunksProcessed << " chunks processed. " << "UTC timestamp " << timestamp << " accumulationNumber " << accumulation << " accumulationRate " << rate << std::endl;
        }
    }
    else
    {
        // Must discard the datagram if there is no available space.
        if (!isActive()) return;
        udpSocket->readDatagram(0, 0);
        std::cout << "K7Chunker::next(): Writable data not valid, discarding packets." << std::endl;
    }

    // Update _startTime.
    _startTimestamp = previousTimestamp;
    _startAccumulation = previousAccumulation;
}