int InboundAudioStream::parseData(ReceivedMessage& message) { // parse sequence number and track it quint16 sequence; message.readPrimitive(&sequence); SequenceNumberStats::ArrivalInfo arrivalInfo = _incomingSequenceNumberStats.sequenceNumberReceived(sequence, message.getSourceID()); QString codecInPacket = message.readString(); packetReceivedUpdateTimingStats(); int networkSamples; // parse the info after the seq number and before the audio data (the stream properties) int prePropertyPosition = message.getPosition(); int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkSamples); message.seek(prePropertyPosition + propertyBytes); // handle this packet based on its arrival status. switch (arrivalInfo._status) { case SequenceNumberStats::Early: { // Packet is early; write droppable silent samples for each of the skipped packets. // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; writeSamplesForDroppedPackets(packetsDropped * networkSamples); // fall through to OnTime case } case SequenceNumberStats::OnTime: { // Packet is on time; parse its data to the ringbuffer if (message.getType() == PacketType::SilentAudioFrame) { // FIXME - Some codecs need to know about these silent frames... and can produce better output writeDroppableSilentSamples(networkSamples); } else { // note: PCM and no codec are identical bool selectedPCM = _selectedCodecName == "pcm" || _selectedCodecName == ""; bool packetPCM = codecInPacket == "pcm" || codecInPacket == ""; if (codecInPacket == _selectedCodecName || (packetPCM && selectedPCM)) { auto afterProperties = message.readWithoutCopy(message.getBytesLeftToRead()); parseAudioData(message.getType(), afterProperties); } else { qDebug() << "Codec mismatch: expected" << _selectedCodecName << "got" << codecInPacket << "writing silence"; writeDroppableSilentSamples(networkSamples); // inform others of the mismatch auto sendingNode = DependencyManager::get<NodeList>()->nodeWithUUID(message.getSourceID()); emit mismatchedAudioCodec(sendingNode, _selectedCodecName, codecInPacket); } } break; } default: { // For now, late packets are ignored. It may be good in the future to insert the late audio packet data // into the ring buffer to fill in the missing frame if it hasn't been mixed yet. break; } } int framesAvailable = _ringBuffer.framesAvailable(); // if this stream was starved, check if we're still starved. if (_isStarved && framesAvailable >= _desiredJitterBufferFrames) { _isStarved = false; } // if the ringbuffer exceeds the desired size by more than the threshold specified, // drop the oldest frames so the ringbuffer is down to the desired size. if (framesAvailable > _desiredJitterBufferFrames + _maxFramesOverDesired) { int framesToDrop = framesAvailable - (_desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING); _ringBuffer.shiftReadPosition(framesToDrop * _ringBuffer.getNumFrameSamples()); _framesAvailableStat.reset(); _currentJitterBufferFrames = 0; _oldFramesDropped += framesToDrop; qCDebug(audiostream, "Dropped %d frames", framesToDrop); qCDebug(audiostream, "Resetted current jitter frames"); } framesAvailableChanged(); return message.getPosition(); }
int InboundAudioStream::parseData(ReceivedMessage& message) { // parse sequence number and track it quint16 sequence; message.readPrimitive(&sequence); SequenceNumberStats::ArrivalInfo arrivalInfo = _incomingSequenceNumberStats.sequenceNumberReceived(sequence, message.getSourceID()); QString codecInPacket = message.readString(); packetReceivedUpdateTimingStats(); int networkFrames; // parse the info after the seq number and before the audio data (the stream properties) int prePropertyPosition = message.getPosition(); int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); message.seek(prePropertyPosition + propertyBytes); // handle this packet based on its arrival status. switch (arrivalInfo._status) { case SequenceNumberStats::Unreasonable: { lostAudioData(1); break; } case SequenceNumberStats::Early: { // Packet is early. Treat the packets as if all the packets between the last // OnTime packet and this packet were lost. If we're using a codec this will // also result in allowing the codec to interpolate lost data. Then // fall through to the "on time" logic to actually handle this packet int packetsDropped = arrivalInfo._seqDiffFromExpected; lostAudioData(packetsDropped); // fall through to OnTime case } case SequenceNumberStats::OnTime: { // Packet is on time; parse its data to the ringbuffer if (message.getType() == PacketType::SilentAudioFrame || message.getType() == PacketType::ReplicatedSilentAudioFrame) { // If we recieved a SilentAudioFrame from our sender, we might want to drop // some of the samples in order to catch up to our desired jitter buffer size. writeDroppableSilentFrames(networkFrames); } else { // note: PCM and no codec are identical bool selectedPCM = _selectedCodecName == "pcm" || _selectedCodecName == ""; bool packetPCM = codecInPacket == "pcm" || codecInPacket == ""; if (codecInPacket == _selectedCodecName || (packetPCM && selectedPCM)) { auto afterProperties = message.readWithoutCopy(message.getBytesLeftToRead()); parseAudioData(message.getType(), afterProperties); _mismatchedAudioCodecCount = 0; } else { _mismatchedAudioCodecCount++; qDebug(audio) << "Codec mismatch: expected" << _selectedCodecName << "got" << codecInPacket; if (packetPCM) { // If there are PCM packets in-flight after the codec is changed, use them. auto afterProperties = message.readWithoutCopy(message.getBytesLeftToRead()); _ringBuffer.writeData(afterProperties.data(), afterProperties.size()); } else { // Since the data in the stream is using a codec that we aren't prepared for, // we need to let the codec know that we don't have data for it, this will // allow the codec to interpolate missing data and produce a fade to silence. lostAudioData(1); } if (_mismatchedAudioCodecCount > MAX_MISMATCHED_AUDIO_CODEC_COUNT) { _mismatchedAudioCodecCount = 0; // inform others of the mismatch auto sendingNode = DependencyManager::get<NodeList>()->nodeWithLocalID(message.getSourceID()); if (sendingNode) { emit mismatchedAudioCodec(sendingNode, _selectedCodecName, codecInPacket); qDebug(audio) << "Codec mismatch threshold exceeded, SelectedAudioFormat(" << _selectedCodecName << " ) sent"; } } } } break; } default: { // For now, late packets are ignored. It may be good in the future to insert the late audio packet data // into the ring buffer to fill in the missing frame if it hasn't been mixed yet. break; } } int framesAvailable = _ringBuffer.framesAvailable(); // if this stream was starved, check if we're still starved. if (_isStarved && framesAvailable >= _desiredJitterBufferFrames) { qCInfo(audiostream, "Starve ended"); _isStarved = false; } // if the ringbuffer exceeds the desired size by more than the threshold specified, // drop the oldest frames so the ringbuffer is down to the desired size. if (framesAvailable > _desiredJitterBufferFrames + MAX_FRAMES_OVER_DESIRED) { int framesToDrop = framesAvailable - (_desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING); _ringBuffer.shiftReadPosition(framesToDrop * _ringBuffer.getNumFrameSamples()); _framesAvailableStat.reset(); _currentJitterBufferFrames = 0; _oldFramesDropped += framesToDrop; qCInfo(audiostream, "Dropped %d frames", framesToDrop); qCInfo(audiostream, "Reset current jitter frames"); } framesAvailableChanged(); return message.getPosition(); }