void WebRtcAec_ProcessFrame(aec_t *aec, const short *nearend, const short *nearendH, int knownDelay) { // For each frame the process is as follows: // 1) If the system_delay indicates on being too small for processing a // frame we stuff the buffer with enough data for 10 ms. // 2) Adjust the buffer to the system delay, by moving the read pointer. // 3) If we can't move read pointer due to buffer size limitations we // flush/stuff the buffer. // 4) Process as many partitions as possible. // 5) Update the |system_delay| with respect to a full frame of FRAME_LEN // samples. Even though we will have data left to process (we work with // partitions) we consider updating a whole frame, since that's the // amount of data we input and output in audio_processing. // TODO(bjornv): Investigate how we should round the delay difference; right // now we know that incoming |knownDelay| is underestimated when it's less // than |aec->knownDelay|. We therefore, round (-32) in that direction. In // the other direction, we don't have this situation, but might flush one // partition too little. This can cause non-causality, which should be // investigated. Maybe, allow for a non-symmetric rounding, like -16. int move_elements = (aec->knownDelay - knownDelay - 32) / PART_LEN; int moved_elements = 0; // TODO(bjornv): Change the near-end buffer handling to be the same as for // far-end, that is, with a near_pre_buf. // Buffer the near-end frame. WebRtc_WriteBuffer(aec->nearFrBuf, nearend, FRAME_LEN); // For H band if (aec->sampFreq == 32000) { WebRtc_WriteBuffer(aec->nearFrBufH, nearendH, FRAME_LEN); } // 1) At most we process |aec->mult|+1 partitions in 10 ms. Make sure we // have enough far-end data for that by stuffing the buffer if the // |system_delay| indicates others. if (aec->system_delay < FRAME_LEN) { // We don't have enough data so we rewind 10 ms. WebRtcAec_MoveFarReadPtr(aec, -(aec->mult + 1)); } // 2) Compensate for a possible change in the system delay. WebRtc_MoveReadPtr(aec->far_buf_windowed, move_elements); moved_elements = WebRtc_MoveReadPtr(aec->far_buf, move_elements); aec->knownDelay -= moved_elements * PART_LEN; #ifdef WEBRTC_AEC_DEBUG_DUMP WebRtc_MoveReadPtr(aec->far_time_buf, move_elements); #endif // 4) Process as many blocks as possible. while (WebRtc_available_read(aec->nearFrBuf) >= PART_LEN) { ProcessBlock(aec); } // 5) Update system delay with respect to the entire frame. aec->system_delay -= FRAME_LEN; }
static void EstBufDelayExtended(aecpc_t* self) { int reported_delay = self->msInSndCardBuf * sampMsNb * self->rate_factor; int current_delay = reported_delay - WebRtcAec_system_delay(self->aec); int delay_difference = 0; // Before we proceed with the delay estimate filtering we: // 1) Compensate for the frame that will be read. // 2) Compensate for drift resampling. // 3) Compensate for non-causality if needed, since the estimated delay can't // be negative. // 1) Compensating for the frame(s) that will be read/processed. current_delay += FRAME_LEN * self->rate_factor; // 2) Account for resampling frame delay. if (self->skewMode == kAecTrue && self->resample == kAecTrue) { current_delay -= kResamplingDelay; } // 3) Compensate for non-causality, if needed, by flushing two blocks. if (current_delay < PART_LEN) { current_delay += WebRtcAec_MoveFarReadPtr(self->aec, 2) * PART_LEN; } if (self->filtDelay == -1) { self->filtDelay = WEBRTC_SPL_MAX(0, 0.5 * current_delay); } else { self->filtDelay = WEBRTC_SPL_MAX( 0, (short)(0.95 * self->filtDelay + 0.05 * current_delay)); } delay_difference = self->filtDelay - self->knownDelay; if (delay_difference > 384) { if (self->lastDelayDiff < 128) { self->timeForDelayChange = 0; } else { self->timeForDelayChange++; } } else if (delay_difference < 128 && self->knownDelay > 0) { if (self->lastDelayDiff > 384) { self->timeForDelayChange = 0; } else { self->timeForDelayChange++; } } else { self->timeForDelayChange = 0; } self->lastDelayDiff = delay_difference; if (self->timeForDelayChange > 25) { self->knownDelay = WEBRTC_SPL_MAX((int)self->filtDelay - 256, 0); } }
static void EstBufDelayNormal(aecpc_t* aecpc) { int nSampSndCard = aecpc->msInSndCardBuf * sampMsNb * aecpc->rate_factor; int current_delay = nSampSndCard - WebRtcAec_system_delay(aecpc->aec); int delay_difference = 0; // Before we proceed with the delay estimate filtering we: // 1) Compensate for the frame that will be read. // 2) Compensate for drift resampling. // 3) Compensate for non-causality if needed, since the estimated delay can't // be negative. // 1) Compensating for the frame(s) that will be read/processed. current_delay += FRAME_LEN * aecpc->rate_factor; // 2) Account for resampling frame delay. if (aecpc->skewMode == kAecTrue && aecpc->resample == kAecTrue) { current_delay -= kResamplingDelay; } // 3) Compensate for non-causality, if needed, by flushing one block. if (current_delay < PART_LEN) { current_delay += WebRtcAec_MoveFarReadPtr(aecpc->aec, 1) * PART_LEN; } // We use -1 to signal an initialized state in the "extended" implementation; // compensate for that. aecpc->filtDelay = aecpc->filtDelay < 0 ? 0 : aecpc->filtDelay; aecpc->filtDelay = WEBRTC_SPL_MAX(0, (short)(0.8 * aecpc->filtDelay + 0.2 * current_delay)); delay_difference = aecpc->filtDelay - aecpc->knownDelay; if (delay_difference > 224) { if (aecpc->lastDelayDiff < 96) { aecpc->timeForDelayChange = 0; } else { aecpc->timeForDelayChange++; } } else if (delay_difference < 96 && aecpc->knownDelay > 0) { if (aecpc->lastDelayDiff > 224) { aecpc->timeForDelayChange = 0; } else { aecpc->timeForDelayChange++; } } else { aecpc->timeForDelayChange = 0; } aecpc->lastDelayDiff = delay_difference; if (aecpc->timeForDelayChange > 25) { aecpc->knownDelay = WEBRTC_SPL_MAX((int)aecpc->filtDelay - 160, 0); } }
void WebRtcAec_BufferFarendPartition(aec_t *aec, const float* farend) { float fft[PART_LEN2]; float xf[2][PART_LEN1]; // Check if the buffer is full, and in that case flush the oldest data. if (WebRtc_available_write(aec->far_buf) < 1) { WebRtcAec_MoveFarReadPtr(aec, 1); } // Convert far-end partition to the frequency domain without windowing. memcpy(fft, farend, sizeof(float) * PART_LEN2); TimeToFrequency(fft, xf, 0); WebRtc_WriteBuffer(aec->far_buf, &xf[0][0], 1); // Convert far-end partition to the frequency domain with windowing. memcpy(fft, farend, sizeof(float) * PART_LEN2); TimeToFrequency(fft, xf, 1); WebRtc_WriteBuffer(aec->far_buf_windowed, &xf[0][0], 1); }
int32_t WebRtcAec_Process(void *aecInst, const int16_t *nearend, const int16_t *nearendH, int16_t *out, int16_t *outH, int16_t nrOfSamples, int16_t msInSndCardBuf, int32_t skew) { aecpc_t *aecpc = aecInst; int32_t retVal = 0; short i; short nBlocks10ms; short nFrames; // Limit resampling to doubling/halving of signal const float minSkewEst = -0.5f; const float maxSkewEst = 1.0f; if (aecpc == NULL) { return -1; } if (nearend == NULL) { aecpc->lastError = AEC_NULL_POINTER_ERROR; return -1; } if (out == NULL) { aecpc->lastError = AEC_NULL_POINTER_ERROR; return -1; } if (aecpc->initFlag != initCheck) { aecpc->lastError = AEC_UNINITIALIZED_ERROR; return -1; } // number of samples == 160 for SWB input if (nrOfSamples != 80 && nrOfSamples != 160) { aecpc->lastError = AEC_BAD_PARAMETER_ERROR; return -1; } // Check for valid pointers based on sampling rate if (aecpc->sampFreq == 32000 && nearendH == NULL) { aecpc->lastError = AEC_NULL_POINTER_ERROR; return -1; } if (msInSndCardBuf < 0) { msInSndCardBuf = 0; aecpc->lastError = AEC_BAD_PARAMETER_WARNING; retVal = -1; } else if (msInSndCardBuf > 500) { msInSndCardBuf = 500; aecpc->lastError = AEC_BAD_PARAMETER_WARNING; retVal = -1; } // TODO(andrew): we need to investigate if this +10 is really wanted. msInSndCardBuf += 10; aecpc->msInSndCardBuf = msInSndCardBuf; if (aecpc->skewMode == kAecTrue) { if (aecpc->skewFrCtr < 25) { aecpc->skewFrCtr++; } else { retVal = WebRtcAec_GetSkew(aecpc->resampler, skew, &aecpc->skew); if (retVal == -1) { aecpc->skew = 0; aecpc->lastError = AEC_BAD_PARAMETER_WARNING; } aecpc->skew /= aecpc->sampFactor*nrOfSamples; if (aecpc->skew < 1.0e-3 && aecpc->skew > -1.0e-3) { aecpc->resample = kAecFalse; } else { aecpc->resample = kAecTrue; } if (aecpc->skew < minSkewEst) { aecpc->skew = minSkewEst; } else if (aecpc->skew > maxSkewEst) { aecpc->skew = maxSkewEst; } #ifdef WEBRTC_AEC_DEBUG_DUMP (void)fwrite(&aecpc->skew, sizeof(aecpc->skew), 1, aecpc->skewFile); #endif } } nFrames = nrOfSamples / FRAME_LEN; nBlocks10ms = nFrames / aecpc->rate_factor; if (aecpc->ECstartup) { if (nearend != out) { // Only needed if they don't already point to the same place. memcpy(out, nearend, sizeof(short) * nrOfSamples); } // The AEC is in the start up mode // AEC is disabled until the system delay is OK // Mechanism to ensure that the system delay is reasonably stable. if (aecpc->checkBuffSize) { aecpc->checkBufSizeCtr++; // Before we fill up the far-end buffer we require the system delay // to be stable (+/-8 ms) compared to the first value. This // comparison is made during the following 6 consecutive 10 ms // blocks. If it seems to be stable then we start to fill up the // far-end buffer. if (aecpc->counter == 0) { aecpc->firstVal = aecpc->msInSndCardBuf; aecpc->sum = 0; } if (abs(aecpc->firstVal - aecpc->msInSndCardBuf) < WEBRTC_SPL_MAX(0.2 * aecpc->msInSndCardBuf, sampMsNb)) { aecpc->sum += aecpc->msInSndCardBuf; aecpc->counter++; } else { aecpc->counter = 0; } if (aecpc->counter * nBlocks10ms >= 6) { // The far-end buffer size is determined in partitions of // PART_LEN samples. Use 75% of the average value of the system // delay as buffer size to start with. aecpc->bufSizeStart = WEBRTC_SPL_MIN((3 * aecpc->sum * aecpc->rate_factor * 8) / (4 * aecpc->counter * PART_LEN), kMaxBufSizeStart); // Buffer size has now been determined. aecpc->checkBuffSize = 0; } if (aecpc->checkBufSizeCtr * nBlocks10ms > 50) { // For really bad systems, don't disable the echo canceller for // more than 0.5 sec. aecpc->bufSizeStart = WEBRTC_SPL_MIN((aecpc->msInSndCardBuf * aecpc->rate_factor * 3) / 40, kMaxBufSizeStart); aecpc->checkBuffSize = 0; } } // If |checkBuffSize| changed in the if-statement above. if (!aecpc->checkBuffSize) { // The system delay is now reasonably stable (or has been unstable // for too long). When the far-end buffer is filled with // approximately the same amount of data as reported by the system // we end the startup phase. int overhead_elements = WebRtcAec_system_delay(aecpc->aec) / PART_LEN - aecpc->bufSizeStart; if (overhead_elements == 0) { // Enable the AEC aecpc->ECstartup = 0; } else if (overhead_elements > 0) { // TODO(bjornv): Do we need a check on how much we actually // moved the read pointer? It should always be possible to move // the pointer |overhead_elements| since we have only added data // to the buffer and no delay compensation nor AEC processing // has been done. WebRtcAec_MoveFarReadPtr(aecpc->aec, overhead_elements); // Enable the AEC aecpc->ECstartup = 0; } } } else { // AEC is enabled. EstBufDelay(aecpc); // Note that 1 frame is supported for NB and 2 frames for WB. for (i = 0; i < nFrames; i++) { // Call the AEC. WebRtcAec_ProcessFrame(aecpc->aec, &nearend[FRAME_LEN * i], &nearendH[FRAME_LEN * i], aecpc->knownDelay, &out[FRAME_LEN * i], &outH[FRAME_LEN * i]); // TODO(bjornv): Re-structure such that we don't have to pass // |aecpc->knownDelay| as input. Change name to something like // |system_buffer_diff|. } } #ifdef WEBRTC_AEC_DEBUG_DUMP { int16_t far_buf_size_ms = (int16_t)(WebRtcAec_system_delay(aecpc->aec) / (sampMsNb * aecpc->rate_factor)); (void)fwrite(&far_buf_size_ms, 2, 1, aecpc->bufFile); (void)fwrite(&aecpc->knownDelay, sizeof(aecpc->knownDelay), 1, aecpc->delayFile); } #endif return retVal; }
static void ProcessExtended(aecpc_t* self, const int16_t* near, const int16_t* near_high, int16_t* out, int16_t* out_high, int16_t num_samples, int16_t reported_delay_ms, int32_t skew) { int i; const int num_frames = num_samples / FRAME_LEN; #if defined(WEBRTC_UNTRUSTED_DELAY) const int delay_diff_offset = kDelayDiffOffsetSamples; reported_delay_ms = kFixedDelayMs; #else // This is the usual mode where we trust the reported system delay values. const int delay_diff_offset = 0; // Due to the longer filter, we no longer add 10 ms to the reported delay // to reduce chance of non-causality. Instead we apply a minimum here to avoid // issues with the read pointer jumping around needlessly. reported_delay_ms = reported_delay_ms < kMinTrustedDelayMs ? kMinTrustedDelayMs : reported_delay_ms; // If the reported delay appears to be bogus, we attempt to recover by using // the measured fixed delay values. We use >= here because higher layers // may already clamp to this maximum value, and we would otherwise not // detect it here. reported_delay_ms = reported_delay_ms >= kMaxTrustedDelayMs ? kFixedDelayMs : reported_delay_ms; #endif self->msInSndCardBuf = reported_delay_ms; if (!self->farend_started) { // Only needed if they don't already point to the same place. if (near != out) { memcpy(out, near, sizeof(short) * num_samples); } if (near_high != out_high) { memcpy(out_high, near_high, sizeof(short) * num_samples); } return; } if (self->startup_phase) { // In the extended mode, there isn't a startup "phase", just a special // action on the first frame. In the trusted delay case, we'll take the // current reported delay, unless it's less then our conservative // measurement. int startup_size_ms = reported_delay_ms < kFixedDelayMs ? kFixedDelayMs : reported_delay_ms; int overhead_elements = (WebRtcAec_system_delay(self->aec) - startup_size_ms / 2 * self->rate_factor * 8) / PART_LEN; WebRtcAec_MoveFarReadPtr(self->aec, overhead_elements); self->startup_phase = 0; } EstBufDelayExtended(self); { // |delay_diff_offset| gives us the option to manually rewind the delay on // very low delay platforms which can't be expressed purely through // |reported_delay_ms|. const int adjusted_known_delay = WEBRTC_SPL_MAX(0, self->knownDelay + delay_diff_offset); for (i = 0; i < num_frames; ++i) { WebRtcAec_ProcessFrame(self->aec, &near[FRAME_LEN * i], &near_high[FRAME_LEN * i], adjusted_known_delay, &out[FRAME_LEN * i], &out_high[FRAME_LEN * i]); } } }
static int ProcessNormal(Aec* aecpc, const float* const* nearend, int num_bands, float* const* out, int16_t nrOfSamples, int16_t msInSndCardBuf, int32_t skew) { int retVal = 0; short i; short nBlocks10ms; // Limit resampling to doubling/halving of signal const float minSkewEst = -0.5f; const float maxSkewEst = 1.0f; msInSndCardBuf = msInSndCardBuf > kMaxTrustedDelayMs ? kMaxTrustedDelayMs : msInSndCardBuf; // TODO(andrew): we need to investigate if this +10 is really wanted. msInSndCardBuf += 10; aecpc->msInSndCardBuf = msInSndCardBuf; if (aecpc->skewMode == kAecTrue) { if (aecpc->skewFrCtr < 25) { aecpc->skewFrCtr++; } else { retVal = WebRtcAec_GetSkew(aecpc->resampler, skew, &aecpc->skew); if (retVal == -1) { aecpc->skew = 0; aecpc->lastError = AEC_BAD_PARAMETER_WARNING; } aecpc->skew /= aecpc->sampFactor * nrOfSamples; if (aecpc->skew < 1.0e-3 && aecpc->skew > -1.0e-3) { aecpc->resample = kAecFalse; } else { aecpc->resample = kAecTrue; } if (aecpc->skew < minSkewEst) { aecpc->skew = minSkewEst; } else if (aecpc->skew > maxSkewEst) { aecpc->skew = maxSkewEst; } #ifdef WEBRTC_AEC_DEBUG_DUMP (void)fwrite(&aecpc->skew, sizeof(aecpc->skew), 1, aecpc->skewFile); #endif } } nBlocks10ms = nrOfSamples / (FRAME_LEN * aecpc->rate_factor); if (aecpc->startup_phase) { for (i = 0; i < num_bands; ++i) { // Only needed if they don't already point to the same place. if (nearend[i] != out[i]) { memcpy(out[i], nearend[i], sizeof(nearend[i][0]) * nrOfSamples); } } // The AEC is in the start up mode // AEC is disabled until the system delay is OK // Mechanism to ensure that the system delay is reasonably stable. if (aecpc->checkBuffSize) { aecpc->checkBufSizeCtr++; // Before we fill up the far-end buffer we require the system delay // to be stable (+/-8 ms) compared to the first value. This // comparison is made during the following 6 consecutive 10 ms // blocks. If it seems to be stable then we start to fill up the // far-end buffer. if (aecpc->counter == 0) { aecpc->firstVal = aecpc->msInSndCardBuf; aecpc->sum = 0; } if (abs(aecpc->firstVal - aecpc->msInSndCardBuf) < WEBRTC_SPL_MAX(0.2 * aecpc->msInSndCardBuf, sampMsNb)) { aecpc->sum += aecpc->msInSndCardBuf; aecpc->counter++; } else { aecpc->counter = 0; } if (aecpc->counter * nBlocks10ms >= 6) { // The far-end buffer size is determined in partitions of // PART_LEN samples. Use 75% of the average value of the system // delay as buffer size to start with. aecpc->bufSizeStart = WEBRTC_SPL_MIN((3 * aecpc->sum * aecpc->rate_factor * 8) / (4 * aecpc->counter * PART_LEN), kMaxBufSizeStart); // Buffer size has now been determined. aecpc->checkBuffSize = 0; } if (aecpc->checkBufSizeCtr * nBlocks10ms > 50) { // For really bad systems, don't disable the echo canceller for // more than 0.5 sec. aecpc->bufSizeStart = WEBRTC_SPL_MIN( (aecpc->msInSndCardBuf * aecpc->rate_factor * 3) / 40, kMaxBufSizeStart); aecpc->checkBuffSize = 0; } } // If |checkBuffSize| changed in the if-statement above. if (!aecpc->checkBuffSize) { // The system delay is now reasonably stable (or has been unstable // for too long). When the far-end buffer is filled with // approximately the same amount of data as reported by the system // we end the startup phase. int overhead_elements = WebRtcAec_system_delay(aecpc->aec) / PART_LEN - aecpc->bufSizeStart; if (overhead_elements == 0) { // Enable the AEC aecpc->startup_phase = 0; } else if (overhead_elements > 0) { // TODO(bjornv): Do we need a check on how much we actually // moved the read pointer? It should always be possible to move // the pointer |overhead_elements| since we have only added data // to the buffer and no delay compensation nor AEC processing // has been done. WebRtcAec_MoveFarReadPtr(aecpc->aec, overhead_elements); // Enable the AEC aecpc->startup_phase = 0; } } } else { // AEC is enabled. if (WebRtcAec_reported_delay_enabled(aecpc->aec)) { EstBufDelayNormal(aecpc); } // Call the AEC. // TODO(bjornv): Re-structure such that we don't have to pass // |aecpc->knownDelay| as input. Change name to something like // |system_buffer_diff|. WebRtcAec_ProcessFrames(aecpc->aec, nearend, num_bands, nrOfSamples, aecpc->knownDelay, out); } return retVal; }
static void ProcessExtended(Aec* self, const float* const* near, size_t num_bands, float* const* out, size_t num_samples, int16_t reported_delay_ms, int32_t skew) { size_t i; const int delay_diff_offset = kDelayDiffOffsetSamples; #if defined(WEBRTC_UNTRUSTED_DELAY) reported_delay_ms = kFixedDelayMs; #else // This is the usual mode where we trust the reported system delay values. // Due to the longer filter, we no longer add 10 ms to the reported delay // to reduce chance of non-causality. Instead we apply a minimum here to avoid // issues with the read pointer jumping around needlessly. reported_delay_ms = reported_delay_ms < kMinTrustedDelayMs ? kMinTrustedDelayMs : reported_delay_ms; // If the reported delay appears to be bogus, we attempt to recover by using // the measured fixed delay values. We use >= here because higher layers // may already clamp to this maximum value, and we would otherwise not // detect it here. reported_delay_ms = reported_delay_ms >= kMaxTrustedDelayMs ? kFixedDelayMs : reported_delay_ms; #endif self->msInSndCardBuf = reported_delay_ms; if (!self->farend_started) { for (i = 0; i < num_bands; ++i) { // Only needed if they don't already point to the same place. if (near[i] != out[i]) { memcpy(out[i], near[i], sizeof(near[i][0]) * num_samples); } } return; } if (self->startup_phase) { // In the extended mode, there isn't a startup "phase", just a special // action on the first frame. In the trusted delay case, we'll take the // current reported delay, unless it's less then our conservative // measurement. int startup_size_ms = reported_delay_ms < kFixedDelayMs ? kFixedDelayMs : reported_delay_ms; #if defined(WEBRTC_ANDROID) || defined(WEBRTC_GONK) int target_delay = startup_size_ms * self->rate_factor * 8; #else // To avoid putting the AEC in a non-causal state we're being slightly // conservative and scale by 2. On Android we use a fixed delay and // therefore there is no need to scale the target_delay. int target_delay = startup_size_ms * self->rate_factor * 8 / 2; #endif int overhead_elements = (WebRtcAec_system_delay(self->aec) - target_delay) / PART_LEN; WebRtcAec_MoveFarReadPtr(self->aec, overhead_elements); self->startup_phase = 0; } EstBufDelayExtended(self); { // |delay_diff_offset| gives us the option to manually rewind the delay on // very low delay platforms which can't be expressed purely through // |reported_delay_ms|. const int adjusted_known_delay = WEBRTC_SPL_MAX(0, self->knownDelay + delay_diff_offset); WebRtcAec_ProcessFrames(self->aec, near, num_bands, num_samples, adjusted_known_delay, out); } }