// Transport Layer Callbacks
MediaConduitErrorCode
WebrtcAudioConduit::ReceivedRTPPacket(const void *data, int len)
{
CSFLogDebug(logTag, "%s : channel %d", __FUNCTION__, mChannel);
if(mEngineReceiving)
{
#ifdef MOZILLA_INTERNAL_API
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG)) {
// timestamp is at 32 bits in ([1])
struct Processing insert = { TimeStamp::Now(),
ntohl(static_cast<const uint32_t *>(data)[1]) };
mProcessing.AppendElement(insert);
}
#endif
if(mPtrVoENetwork->ReceivedRTPPacket(mChannel,data,len) == -1)
{
int error = mPtrVoEBase->LastError();
CSFLogError(logTag, "%s RTP Processing Error %d", __FUNCTION__, error);
if(error == VE_RTP_RTCP_MODULE_ERROR)
{
return kMediaConduitRTPRTCPModuleError;
}
return kMediaConduitUnknownError;
}
} else {
CSFLogError(logTag, "Error: %s when not receiving", __FUNCTION__);
return kMediaConduitSessionNotInited;
}
return kMediaConduitNoError;
}
//WebRTC::RTP Callback Implementation
// Called on AudioGUM or MSG thread
int WebrtcAudioConduit::SendPacket(int channel, const void* data, int len)
{
CSFLogDebug(logTag, "%s : channel %d", __FUNCTION__, channel);
#if !defined(MOZILLA_EXTERNAL_LINKAGE)
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG)) {
if (mProcessing.Length() > 0) {
TimeStamp started = mProcessing[0].mTimeStamp;
mProcessing.RemoveElementAt(0);
mProcessing.RemoveElementAt(0); // 20ms packetization! Could automate this by watching sizes
TimeDuration t = TimeStamp::Now() - started;
int64_t delta = t.ToMilliseconds();
LogTime(AsyncLatencyLogger::AudioSendRTP, ((uint64_t) this), delta);
}
}
#endif
ReentrantMonitorAutoEnter enter(mTransportMonitor);
if(mTransmitterTransport &&
(mTransmitterTransport->SendRtpPacket(data, len) == NS_OK))
{
CSFLogDebug(logTag, "%s Sent RTP Packet ", __FUNCTION__);
return len;
} else {
CSFLogError(logTag, "%s RTP Packet Send Failed ", __FUNCTION__);
return -1;
}
}
//WebRTC::RTP Callback Implementation
int WebrtcAudioConduit::SendPacket(int channel, const void* data, int len)
{
CSFLogDebug(logTag, "%s : channel %d %s", __FUNCTION__, channel,
(mEngineReceiving && mOtherDirection) ? "(using mOtherDirection)" : "");
if (mEngineReceiving)
{
if (mOtherDirection)
{
return mOtherDirection->SendPacket(channel, data, len);
}
CSFLogDebug(logTag, "%s : Asked to send RTP without an RTP sender on channel %d",
__FUNCTION__, channel);
return -1;
} else {
#ifdef MOZILLA_INTERNAL_API
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG)) {
if (mProcessing.Length() > 0) {
TimeStamp started = mProcessing[0].mTimeStamp;
mProcessing.RemoveElementAt(0);
mProcessing.RemoveElementAt(0); // 20ms packetization! Could automate this by watching sizes
TimeDuration t = TimeStamp::Now() - started;
int64_t delta = t.ToMilliseconds();
LogTime(AsyncLatencyLogger::AudioSendRTP, ((uint64_t) this), delta);
}
}
#endif
if(mTransport && (mTransport->SendRtpPacket(data, len) == NS_OK))
{
CSFLogDebug(logTag, "%s Sent RTP Packet ", __FUNCTION__);
return len;
} else {
CSFLogError(logTag, "%s RTP Packet Send Failed ", __FUNCTION__);
return -1;
}
}
}
MediaConduitErrorCode
WebrtcAudioConduit::GetAudioFrame(int16_t speechData[],
int32_t samplingFreqHz,
int32_t capture_delay,
int& lengthSamples)
{
CSFLogDebug(logTag, "%s ", __FUNCTION__);
unsigned int numSamples = 0;
//validate params
if(!speechData )
{
CSFLogError(logTag,"%s Null Audio Buffer Pointer", __FUNCTION__);
MOZ_ASSERT(PR_FALSE);
return kMediaConduitMalformedArgument;
}
// Validate sample length
if((numSamples = GetNum10msSamplesForFrequency(samplingFreqHz)) == 0 )
{
CSFLogError(logTag,"%s Invalid Sampling Frequency ", __FUNCTION__);
MOZ_ASSERT(PR_FALSE);
return kMediaConduitMalformedArgument;
}
//validate capture time
if(capture_delay < 0 )
{
CSFLogError(logTag,"%s Invalid Capture Delay ", __FUNCTION__);
MOZ_ASSERT(PR_FALSE);
return kMediaConduitMalformedArgument;
}
//Conduit should have reception enabled before we ask for decoded
// samples
if(!mEngineReceiving)
{
CSFLogError(logTag, "%s Engine not Receiving ", __FUNCTION__);
return kMediaConduitSessionNotInited;
}
lengthSamples = 0; //output paramter
if(mPtrVoEXmedia->ExternalPlayoutGetData( speechData,
samplingFreqHz,
capture_delay,
lengthSamples) == -1)
{
int error = mPtrVoEBase->LastError();
CSFLogError(logTag, "%s Getting audio data Failed %d", __FUNCTION__, error);
if(error == VE_RUNTIME_PLAY_ERROR)
{
return kMediaConduitPlayoutError;
}
return kMediaConduitUnknownError;
}
// Not #ifdef DEBUG or on a log module so we can use it for about:webrtc/etc
mSamples += lengthSamples;
if (mSamples >= mLastSyncLog + samplingFreqHz) {
int jitter_buffer_delay_ms;
int playout_buffer_delay_ms;
int avsync_offset_ms;
if (GetAVStats(&jitter_buffer_delay_ms,
&playout_buffer_delay_ms,
&avsync_offset_ms)) {
#ifdef MOZILLA_INTERNAL_API
if (avsync_offset_ms < 0) {
Telemetry::Accumulate(Telemetry::WEBRTC_AVSYNC_WHEN_VIDEO_LAGS_AUDIO_MS,
-avsync_offset_ms);
} else {
Telemetry::Accumulate(Telemetry::WEBRTC_AVSYNC_WHEN_AUDIO_LAGS_VIDEO_MS,
avsync_offset_ms);
}
#endif
CSFLogError(logTag,
"A/V sync: sync delta: %dms, audio jitter delay %dms, playout delay %dms",
avsync_offset_ms, jitter_buffer_delay_ms, playout_buffer_delay_ms);
} else {
CSFLogError(logTag, "A/V sync: GetAVStats failed");
}
mLastSyncLog = mSamples;
}
#ifdef MOZILLA_INTERNAL_API
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG)) {
if (mProcessing.Length() > 0) {
unsigned int now;
mPtrVoEVideoSync->GetPlayoutTimestamp(mChannel, now);
if (static_cast<uint32_t>(now) != mLastTimestamp) {
mLastTimestamp = static_cast<uint32_t>(now);
// Find the block that includes this timestamp in the network input
while (mProcessing.Length() > 0) {
// FIX! assumes 20ms @ 48000Hz
// FIX handle wrap-around
if (mProcessing[0].mRTPTimeStamp + 20*(48000/1000) >= now) {
TimeDuration t = TimeStamp::Now() - mProcessing[0].mTimeStamp;
// Wrap-around?
int64_t delta = t.ToMilliseconds() + (now - mProcessing[0].mRTPTimeStamp)/(48000/1000);
LogTime(AsyncLatencyLogger::AudioRecvRTP, ((uint64_t) this), delta);
break;
}
mProcessing.RemoveElementAt(0);
}
}
}
}
#endif
CSFLogDebug(logTag,"%s GetAudioFrame:Got samples: length %d ",__FUNCTION__,
lengthSamples);
return kMediaConduitNoError;
}
MediaConduitErrorCode
WebrtcAudioConduit::SendAudioFrame(const int16_t audio_data[],
int32_t lengthSamples,
int32_t samplingFreqHz,
int32_t capture_delay)
{
CSFLogDebug(logTag, "%s ", __FUNCTION__);
// Following checks need to be performed
// 1. Non null audio buffer pointer,
// 2. invalid sampling frequency - less than 0 or unsupported ones
// 3. Appropriate Sample Length for 10 ms audio-frame. This represents
// block size the VoiceEngine feeds into encoder for passed in audio-frame
// Ex: for 16000 sampling rate , valid block-length is 160
// Similarly for 32000 sampling rate, valid block length is 320
// We do the check by the verify modular operator below to be zero
if(!audio_data || (lengthSamples <= 0) ||
(IsSamplingFreqSupported(samplingFreqHz) == false) ||
((lengthSamples % (samplingFreqHz / 100) != 0)) )
{
CSFLogError(logTag, "%s Invalid Parameters ",__FUNCTION__);
MOZ_ASSERT(PR_FALSE);
return kMediaConduitMalformedArgument;
}
//validate capture time
if(capture_delay < 0 )
{
CSFLogError(logTag,"%s Invalid Capture Delay ", __FUNCTION__);
MOZ_ASSERT(PR_FALSE);
return kMediaConduitMalformedArgument;
}
// if transmission is not started .. conduit cannot insert frames
if(!mEngineTransmitting)
{
CSFLogError(logTag, "%s Engine not transmitting ", __FUNCTION__);
return kMediaConduitSessionNotInited;
}
#ifdef MOZILLA_INTERNAL_API
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG)) {
struct Processing insert = { TimeStamp::Now(), 0 };
mProcessing.AppendElement(insert);
}
#endif
capture_delay = mCaptureDelay;
//Insert the samples
if(mPtrVoEXmedia->ExternalRecordingInsertData(audio_data,
lengthSamples,
samplingFreqHz,
capture_delay) == -1)
{
int error = mPtrVoEBase->LastError();
CSFLogError(logTag, "%s Inserting audio data Failed %d", __FUNCTION__, error);
if(error == VE_RUNTIME_REC_ERROR)
{
return kMediaConduitRecordingError;
}
return kMediaConduitUnknownError;
}
// we should be good here
return kMediaConduitNoError;
}
// aTime is the time in ms the samples were inserted into MediaStreamGraph
nsresult
AudioStream::Write(const AudioDataValue* aBuf, uint32_t aFrames, TimeStamp *aTime)
{
MonitorAutoLock mon(mMonitor);
if (mState == ERRORED) {
return NS_ERROR_FAILURE;
}
NS_ASSERTION(mState == INITIALIZED || mState == STARTED || mState == RUNNING,
"Stream write in unexpected state.");
// See if we need to start() the stream, since we must do that from this thread
CheckForStart();
// Downmix to Stereo.
if (mChannels > 2 && mChannels <= 8) {
DownmixAudioToStereo(const_cast<AudioDataValue*> (aBuf), mChannels, aFrames);
}
else if (mChannels > 8) {
return NS_ERROR_FAILURE;
}
const uint8_t* src = reinterpret_cast<const uint8_t*>(aBuf);
uint32_t bytesToCopy = FramesToBytes(aFrames);
// XXX this will need to change if we want to enable this on-the-fly!
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG)) {
// Record the position and time this data was inserted
int64_t timeMs;
if (aTime && !aTime->IsNull()) {
if (mStartTime.IsNull()) {
AsyncLatencyLogger::Get(true)->GetStartTime(mStartTime);
}
timeMs = (*aTime - mStartTime).ToMilliseconds();
} else {
timeMs = 0;
}
struct Inserts insert = { timeMs, aFrames};
mInserts.AppendElement(insert);
}
while (bytesToCopy > 0) {
uint32_t available = std::min(bytesToCopy, mBuffer.Available());
NS_ABORT_IF_FALSE(available % mBytesPerFrame == 0,
"Must copy complete frames.");
mBuffer.AppendElements(src, available);
src += available;
bytesToCopy -= available;
if (bytesToCopy > 0) {
// Careful - the CubebInit thread may not have gotten to STARTED yet
if ((mState == INITIALIZED || mState == STARTED) && mLatencyRequest == LowLatency) {
// don't ever block MediaStreamGraph low-latency streams
uint32_t remains = 0; // we presume the buffer is full
if (mBuffer.Length() > bytesToCopy) {
remains = mBuffer.Length() - bytesToCopy; // Free up just enough space
}
// account for dropping samples
PR_LOG(gAudioStreamLog, PR_LOG_WARNING, ("Stream %p dropping %u bytes (%u frames)in Write()",
this, mBuffer.Length() - remains, BytesToFrames(mBuffer.Length() - remains)));
mReadPoint += BytesToFrames(mBuffer.Length() - remains);
mBuffer.ContractTo(remains);
} else { // RUNNING or high latency
// If we are not playing, but our buffer is full, start playing to make
// room for soon-to-be-decoded data.
if (mState != STARTED && mState != RUNNING) {
PR_LOG(gAudioStreamLog, PR_LOG_WARNING, ("Starting stream %p in Write (%u waiting)",
this, bytesToCopy));
StartUnlocked();
if (mState == ERRORED) {
return NS_ERROR_FAILURE;
}
}
PR_LOG(gAudioStreamLog, PR_LOG_WARNING, ("Stream %p waiting in Write() (%u waiting)",
this, bytesToCopy));
mon.Wait();
}
}
}
mWritten += aFrames;
return NS_OK;
}
long
AudioStream::DataCallback(void* aBuffer, long aFrames)
{
MonitorAutoLock mon(mMonitor);
MOZ_ASSERT(mState != SHUTDOWN, "No data callback after shutdown");
uint32_t available = std::min(static_cast<uint32_t>(FramesToBytes(aFrames)), mBuffer.Length());
NS_ABORT_IF_FALSE(available % mBytesPerFrame == 0, "Must copy complete frames");
AudioDataValue* output = reinterpret_cast<AudioDataValue*>(aBuffer);
uint32_t underrunFrames = 0;
uint32_t servicedFrames = 0;
int64_t insertTime;
// NOTE: wasapi (others?) can call us back *after* stop()/Shutdown() (mState == SHUTDOWN)
// Bug 996162
// callback tells us cubeb succeeded initializing
if (mState == STARTED) {
// For low-latency streams, we want to minimize any built-up data when
// we start getting callbacks.
// Simple version - contract on first callback only.
if (mLatencyRequest == LowLatency) {
#ifdef PR_LOGGING
uint32_t old_len = mBuffer.Length();
#endif
available = mBuffer.ContractTo(FramesToBytes(aFrames));
#ifdef PR_LOGGING
TimeStamp now = TimeStamp::Now();
if (!mStartTime.IsNull()) {
int64_t timeMs = (now - mStartTime).ToMilliseconds();
PR_LOG(gAudioStreamLog, PR_LOG_WARNING,
("Stream took %lldms to start after first Write() @ %u", timeMs, mOutRate));
} else {
PR_LOG(gAudioStreamLog, PR_LOG_WARNING,
("Stream started before Write() @ %u", mOutRate));
}
if (old_len != available) {
// Note that we may have dropped samples in Write() as well!
PR_LOG(gAudioStreamLog, PR_LOG_WARNING,
("AudioStream %p dropped %u + %u initial frames @ %u", this,
mReadPoint, BytesToFrames(old_len - available), mOutRate));
mReadPoint += BytesToFrames(old_len - available);
}
#endif
}
mState = RUNNING;
}
if (available) {
// When we are playing a low latency stream, and it is the first time we are
// getting data from the buffer, we prefer to add the silence for an
// underrun at the beginning of the buffer, so the first buffer is not cut
// in half by the silence inserted to compensate for the underrun.
if (mInRate == mOutRate) {
if (mLatencyRequest == LowLatency && !mWritten) {
servicedFrames = GetUnprocessedWithSilencePadding(output, aFrames, insertTime);
} else {
servicedFrames = GetUnprocessed(output, aFrames, insertTime);
}
} else {
servicedFrames = GetTimeStretched(output, aFrames, insertTime);
}
float scaled_volume = float(GetVolumeScale() * mVolume);
ScaleAudioSamples(output, aFrames * mOutChannels, scaled_volume);
NS_ABORT_IF_FALSE(mBuffer.Length() % mBytesPerFrame == 0, "Must copy complete frames");
// Notify any blocked Write() call that more space is available in mBuffer.
mon.NotifyAll();
} else {
GetBufferInsertTime(insertTime);
}
underrunFrames = aFrames - servicedFrames;
// Always send audible frames first, and silent frames later.
// Otherwise it will break the assumption of FrameHistory.
if (mState != DRAINING) {
mAudioClock.UpdateFrameHistory(servicedFrames, underrunFrames);
uint8_t* rpos = static_cast<uint8_t*>(aBuffer) + FramesToBytes(aFrames - underrunFrames);
memset(rpos, 0, FramesToBytes(underrunFrames));
if (underrunFrames) {
PR_LOG(gAudioStreamLog, PR_LOG_WARNING,
("AudioStream %p lost %d frames", this, underrunFrames));
}
servicedFrames += underrunFrames;
} else {
mAudioClock.UpdateFrameHistory(servicedFrames, 0);
}
WriteDumpFile(mDumpFile, this, aFrames, aBuffer);
// Don't log if we're not interested or if the stream is inactive
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG) &&
mState != SHUTDOWN &&
insertTime != INT64_MAX && servicedFrames > underrunFrames) {
uint32_t latency = UINT32_MAX;
if (cubeb_stream_get_latency(mCubebStream, &latency)) {
NS_WARNING("Could not get latency from cubeb.");
}
TimeStamp now = TimeStamp::Now();
mLatencyLog->Log(AsyncLatencyLogger::AudioStream, reinterpret_cast<uint64_t>(this),
insertTime, now);
mLatencyLog->Log(AsyncLatencyLogger::Cubeb, reinterpret_cast<uint64_t>(mCubebStream.get()),
(latency * 1000) / mOutRate, now);
}
return servicedFrames;
}
long
AudioStream::DataCallback(void* aBuffer, long aFrames)
{
MonitorAutoLock mon(mMonitor);
uint32_t available = std::min(static_cast<uint32_t>(FramesToBytes(aFrames)), mBuffer.Length());
NS_ABORT_IF_FALSE(available % mBytesPerFrame == 0, "Must copy complete frames");
AudioDataValue* output = reinterpret_cast<AudioDataValue*>(aBuffer);
uint32_t underrunFrames = 0;
uint32_t servicedFrames = 0;
int64_t insertTime;
if (available) {
// When we are playing a low latency stream, and it is the first time we are
// getting data from the buffer, we prefer to add the silence for an
// underrun at the beginning of the buffer, so the first buffer is not cut
// in half by the silence inserted to compensate for the underrun.
if (mInRate == mOutRate) {
if (mLatencyRequest == LowLatency && !mWritten) {
servicedFrames = GetUnprocessedWithSilencePadding(output, aFrames, insertTime);
} else {
servicedFrames = GetUnprocessed(output, aFrames, insertTime);
}
} else {
servicedFrames = GetTimeStretched(output, aFrames, insertTime);
}
float scaled_volume = float(GetVolumeScale() * mVolume);
ScaleAudioSamples(output, aFrames * mOutChannels, scaled_volume);
NS_ABORT_IF_FALSE(mBuffer.Length() % mBytesPerFrame == 0, "Must copy complete frames");
// Notify any blocked Write() call that more space is available in mBuffer.
mon.NotifyAll();
} else {
GetBufferInsertTime(insertTime);
}
underrunFrames = aFrames - servicedFrames;
if (mState != DRAINING) {
uint8_t* rpos = static_cast<uint8_t*>(aBuffer) + FramesToBytes(aFrames - underrunFrames);
memset(rpos, 0, FramesToBytes(underrunFrames));
if (underrunFrames) {
PR_LOG(gAudioStreamLog, PR_LOG_WARNING,
("AudioStream %p lost %d frames", this, underrunFrames));
}
mLostFrames += underrunFrames;
servicedFrames += underrunFrames;
}
WriteDumpFile(mDumpFile, this, aFrames, aBuffer);
// Don't log if we're not interested or if the stream is inactive
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG) &&
insertTime != INT64_MAX && servicedFrames > underrunFrames) {
uint32_t latency = UINT32_MAX;
if (cubeb_stream_get_latency(mCubebStream, &latency)) {
NS_WARNING("Could not get latency from cubeb.");
}
TimeStamp now = TimeStamp::Now();
mLatencyLog->Log(AsyncLatencyLogger::AudioStream, reinterpret_cast<uint64_t>(this),
insertTime, now);
mLatencyLog->Log(AsyncLatencyLogger::Cubeb, reinterpret_cast<uint64_t>(mCubebStream.get()),
(latency * 1000) / mOutRate, now);
}
mAudioClock.UpdateWritePosition(servicedFrames);
return servicedFrames;
}
// aTime is the time in ms the samples were inserted into MediaStreamGraph
nsresult
AudioStream::Write(const AudioDataValue* aBuf, uint32_t aFrames, TimeStamp *aTime)
{
MonitorAutoLock mon(mMonitor);
if (!mCubebStream || mState == ERRORED) {
return NS_ERROR_FAILURE;
}
NS_ASSERTION(mState == INITIALIZED || mState == STARTED,
"Stream write in unexpected state.");
// Downmix to Stereo.
if (mChannels > 2 && mChannels <= 8) {
DownmixAudioToStereo(const_cast<AudioDataValue*> (aBuf), mChannels, aFrames);
}
else if (mChannels > 8) {
return NS_ERROR_FAILURE;
}
const uint8_t* src = reinterpret_cast<const uint8_t*>(aBuf);
uint32_t bytesToCopy = FramesToBytes(aFrames);
// XXX this will need to change if we want to enable this on-the-fly!
if (PR_LOG_TEST(GetLatencyLog(), PR_LOG_DEBUG)) {
// Record the position and time this data was inserted
int64_t timeMs;
if (aTime && !aTime->IsNull()) {
if (mStartTime.IsNull()) {
AsyncLatencyLogger::Get(true)->GetStartTime(mStartTime);
}
timeMs = (*aTime - mStartTime).ToMilliseconds();
} else {
timeMs = 0;
}
struct Inserts insert = { timeMs, aFrames};
mInserts.AppendElement(insert);
}
while (bytesToCopy > 0) {
uint32_t available = std::min(bytesToCopy, mBuffer.Available());
NS_ABORT_IF_FALSE(available % mBytesPerFrame == 0,
"Must copy complete frames.");
mBuffer.AppendElements(src, available);
src += available;
bytesToCopy -= available;
if (bytesToCopy > 0) {
// If we are not playing, but our buffer is full, start playing to make
// room for soon-to-be-decoded data.
if (mState != STARTED) {
StartUnlocked();
if (mState != STARTED) {
return NS_ERROR_FAILURE;
}
}
mon.Wait();
}
}
mWritten += aFrames;
return NS_OK;
}