/* Compute the amplitude (sqrt energy) in each of the bands */ void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bandE, int end, int C, int M) { int i, c, N; const opus_int16 *eBands = m->eBands; N = M*m->shortMdctSize; c=0; do { for (i=0;i<end;i++) { int j; opus_val32 maxval=0; opus_val32 sum = 0; j=M*eBands[i]; do { maxval = MAX32(maxval, X[j+c*N]); maxval = MAX32(maxval, -X[j+c*N]); } while (++j<M*eBands[i+1]); if (maxval > 0) { int shift = celt_ilog2(maxval)-10; j=M*eBands[i]; do { sum = MAC16_16(sum, EXTRACT16(VSHR32(X[j+c*N],shift)), EXTRACT16(VSHR32(X[j+c*N],shift))); } while (++j<M*eBands[i+1]); /* We're adding one here to make damn sure we never end up with a pitch vector that's larger than unity norm */ bandE[i+c*m->nbEBands] = EPSILON+VSHR32(EXTEND32(celt_sqrt(sum)),-shift); } else { bandE[i+c*m->nbEBands] = EPSILON; } /*printf ("%f ", bandE[i+c*m->nbEBands]);*/ } } while (++c<C); /*printf ("\n");*/ }
void tonality_get_info(TonalityAnalysisState *tonal, AnalysisInfo *info_out, int len) { int pos; int curr_lookahead; float psum; int i; pos = tonal->read_pos; curr_lookahead = tonal->write_pos-tonal->read_pos; if (curr_lookahead<0) curr_lookahead += DETECT_SIZE; /* On long frames, look at the second analysis window rather than the first. */ if (len > tonal->Fs/50 && pos != tonal->write_pos) { pos++; if (pos==DETECT_SIZE) pos=0; } if (pos == tonal->write_pos) pos--; if (pos<0) pos = DETECT_SIZE-1; OPUS_COPY(info_out, &tonal->info[pos], 1); /* If possible, look ahead for a tone to compensate for the delay in the tone detector. */ for (i=0;i<3;i++) { pos++; if (pos==DETECT_SIZE) pos = 0; if (pos == tonal->write_pos) break; info_out->tonality = MAX32(0, -.03f + MAX32(info_out->tonality, tonal->info[pos].tonality-.05f)); } tonal->read_subframe += len/(tonal->Fs/400); while (tonal->read_subframe>=8) { tonal->read_subframe -= 8; tonal->read_pos++; } if (tonal->read_pos>=DETECT_SIZE) tonal->read_pos-=DETECT_SIZE; /* The -1 is to compensate for the delay in the features themselves. */ curr_lookahead = IMAX(curr_lookahead-1, 0); psum=0; /* Summing the probability of transition patterns that involve music at time (DETECT_SIZE-curr_lookahead-1) */ for (i=0;i<DETECT_SIZE-curr_lookahead;i++) psum += tonal->pmusic[i]; for (;i<DETECT_SIZE;i++) psum += tonal->pspeech[i]; psum = psum*tonal->music_confidence + (1-psum)*tonal->speech_confidence; /*printf("%f %f %f %f %f\n", psum, info_out->music_prob, info_out->vad_prob, info_out->activity_probability, info_out->tonality);*/ info_out->music_prob = psum; }
static __inline celt_int16_t FLOAT2INT16(float x) { x = x*CELT_SIG_SCALE; x = MAX32(x, -32768); x = MIN32(x, 32767); return (celt_int16_t)float2int(x); }
static void find_best_pitch(opus_val32 *xcorr, opus_val16 *y, int len, int max_pitch, int *best_pitch #ifdef FIXED_POINT , int yshift, opus_val32 maxcorr #endif ) { int i, j; opus_val32 Syy=1; opus_val16 best_num[2]; opus_val32 best_den[2]; #ifdef FIXED_POINT int xshift; xshift = celt_ilog2(maxcorr)-14; #endif best_num[0] = -1; best_num[1] = -1; best_den[0] = 0; best_den[1] = 0; best_pitch[0] = 0; best_pitch[1] = 1; for (j=0;j<len;j++) Syy = ADD32(Syy, SHR32(MULT16_16(y[j],y[j]), yshift)); for (i=0;i<max_pitch;i++) { if (xcorr[i]>0) { opus_val16 num; opus_val32 xcorr16; xcorr16 = EXTRACT16(VSHR32(xcorr[i], xshift)); #ifndef FIXED_POINT /* Considering the range of xcorr16, this should avoid both underflows and overflows (inf) when squaring xcorr16 */ xcorr16 *= 1e-12f; #endif num = MULT16_16_Q15(xcorr16,xcorr16); if (MULT16_32_Q15(num,best_den[1]) > MULT16_32_Q15(best_num[1],Syy)) { if (MULT16_32_Q15(num,best_den[0]) > MULT16_32_Q15(best_num[0],Syy)) { best_num[1] = best_num[0]; best_den[1] = best_den[0]; best_pitch[1] = best_pitch[0]; best_num[0] = num; best_den[0] = Syy; best_pitch[0] = i; } else { best_num[1] = num; best_den[1] = Syy; best_pitch[1] = i; } } } Syy += SHR32(MULT16_16(y[i+len],y[i+len]),yshift) - SHR32(MULT16_16(y[i],y[i]),yshift); Syy = MAX32(1, Syy); } }
static int transient_analysis(celt_word32_t *in, int len, int C, int *transient_time, int *transient_shift) { int c, i, n; celt_word32_t ratio; VARDECL(celt_word32_t, begin); SAVE_STACK; ALLOC(begin, len, celt_word32_t); for (i=0;i<len;i++) begin[i] = ABS32(SHR32(in[C*i],SIG_SHIFT)); for (c=1;c<C;c++) { for (i=0;i<len;i++) begin[i] = MAX32(begin[i], ABS32(SHR32(in[C*i+c],SIG_SHIFT))); } for (i=1;i<len;i++) begin[i] = MAX32(begin[i-1],begin[i]); n = -1; for (i=8;i<len-8;i++) { if (begin[i] < MULT16_32_Q15(QCONST16(.2f,15),begin[len-1])) n=i; } if (n<32) { n = -1; ratio = 0; } else { ratio = DIV32(begin[len-1],1+begin[n-16]); } if (ratio < 0) ratio = 0; if (ratio > 1000) ratio = 1000; ratio *= ratio; if (ratio > 2048) *transient_shift = 3; else *transient_shift = 0; *transient_time = n; RESTORE_STACK; return ratio > 20; }
static inline opus_val16 SIG2WORD16(celt_sig x) { #ifdef FIXED_POINT x = PSHR32(x, SIG_SHIFT); x = MAX32(x, -32768); x = MIN32(x, 32767); return EXTRACT16(x); #else return (opus_val16)x; #endif }
static __inline celt_word16_t SIG2WORD16(celt_sig_t x) { #ifdef FIXED_POINT x = PSHR32(x, SIG_SHIFT); x = MAX32(x, -32768); x = MIN32(x, 32767); return EXTRACT16(x); #else return (celt_word16_t)x; #endif }
static void find_best_pitch(opus_val32 *xcorr, opus_val16 *y, int len, int max_pitch, int *best_pitch #ifdef FIXED_POINT , int yshift, opus_val32 maxcorr #endif ) { int i, j; opus_val32 Syy=1; opus_val16 best_num[2]; opus_val32 best_den[2]; #ifdef FIXED_POINT int xshift; xshift = celt_ilog2(maxcorr)-14; #endif best_num[0] = -1; best_num[1] = -1; best_den[0] = 0; best_den[1] = 0; best_pitch[0] = 0; best_pitch[1] = 1; for (j=0;j<len;j++) Syy = MAC16_16(Syy, y[j],y[j]); for (i=0;i<max_pitch;i++) { if (xcorr[i]>0) { opus_val16 num; opus_val32 xcorr16; xcorr16 = EXTRACT16(VSHR32(xcorr[i], xshift)); num = MULT16_16_Q15(xcorr16,xcorr16); if (MULT16_32_Q15(num,best_den[1]) > MULT16_32_Q15(best_num[1],Syy)) { if (MULT16_32_Q15(num,best_den[0]) > MULT16_32_Q15(best_num[0],Syy)) { best_num[1] = best_num[0]; best_den[1] = best_den[0]; best_pitch[1] = best_pitch[0]; best_num[0] = num; best_den[0] = Syy; best_pitch[0] = i; } else { best_num[1] = num; best_den[1] = Syy; best_pitch[1] = i; } } } Syy += SHR32(MULT16_16(y[i+len],y[i+len]),yshift) - SHR32(MULT16_16(y[i],y[i]),yshift); Syy = MAX32(1, Syy); } }
/* Compute the amplitude (sqrt energy) in each of the bands */ void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bank, int _C) { int i, c, N; const celt_int16 *eBands = m->eBands; const int C = CHANNELS(_C); N = FRAMESIZE(m); for (c=0;c<C;c++) { for (i=0;i<m->nbEBands;i++) { int j; celt_word32 maxval=0; celt_word32 sum = 0; j=eBands[i]; do { maxval = MAX32(maxval, X[j+c*N]); maxval = MAX32(maxval, -X[j+c*N]); } while (++j<eBands[i+1]); if (maxval > 0) { int shift = celt_ilog2(maxval)-10; j=eBands[i]; do { sum = MAC16_16(sum, EXTRACT16(VSHR32(X[j+c*N],shift)), EXTRACT16(VSHR32(X[j+c*N],shift))); } while (++j<eBands[i+1]); /* We're adding one here to make damn sure we never end up with a pitch vector that's larger than unity norm */ bank[i+c*m->nbEBands] = EPSILON+VSHR32(EXTEND32(celt_sqrt(sum)),-shift); } else { bank[i+c*m->nbEBands] = EPSILON; } /*printf ("%f ", bank[i+c*m->nbEBands]);*/ } } /*printf ("\n");*/ }
/** Finds the best quantized 3-tap pitch predictor by analysis by synthesis */ int pitch_search_3tap( spx_word16_t target[], /* Target vector */ spx_word16_t* sw, spx_coef_t ak[], /* LPCs for this subframe */ spx_coef_t awk1[], /* Weighted LPCs #1 for this subframe */ spx_coef_t awk2[], /* Weighted LPCs #2 for this subframe */ spx_sig_t exc[], /* Excitation */ const void* par, int start, /* Smallest pitch value allowed */ int end, /* Largest pitch value allowed */ spx_word16_t pitch_coef, /* Voicing (pitch) coefficient */ int p, /* Number of LPC coeffs */ int nsf, /* Number of samples in subframe */ SpeexBits* bits, char* stack, spx_word16_t* exc2, spx_word16_t* r, int complexity, int cdbk_offset, int plc_tuning, spx_word32_t* cumul_gain ) { int i; int cdbk_index, pitch = 0, best_gain_index = 0; VARDECL(spx_sig_t * best_exc); VARDECL(spx_word16_t * new_target); VARDECL(spx_word16_t * best_target); int best_pitch = 0; spx_word32_t err, best_err = -1; int N; const ltp_params* params; const signed char* gain_cdbk; int gain_cdbk_size; int scaledown = 0; VARDECL(int * nbest); params = (const ltp_params*) par; gain_cdbk_size = 1 << params->gain_bits; gain_cdbk = params->gain_cdbk + 4 * gain_cdbk_size * cdbk_offset; N = complexity; if (N > 10) N = 10; if (N < 1) N = 1; ALLOC(nbest, N, int); params = (const ltp_params*) par; if (end < start) { speex_bits_pack(bits, 0, params->pitch_bits); speex_bits_pack(bits, 0, params->gain_bits); SPEEX_MEMSET(exc, 0, nsf); return start; } #ifdef FIXED_POINT /* Check if we need to scale everything down in the pitch search to avoid overflows */ for (i = 0; i < nsf; i++) { if (ABS16(target[i]) > 16383) { scaledown = 1; break; } } for (i = -end; i < nsf; i++) { if (ABS16(exc2[i]) > 16383) { scaledown = 1; break; } } #endif if (N > end - start + 1) N = end - start + 1; if (end != start) open_loop_nbest_pitch(sw, start, end, nsf, nbest, NULL, N, stack); else nbest[0] = start; ALLOC(best_exc, nsf, spx_sig_t); ALLOC(new_target, nsf, spx_word16_t); ALLOC(best_target, nsf, spx_word16_t); for (i = 0; i < N; i++) { pitch = nbest[i]; SPEEX_MEMSET(exc, 0, nsf); err = pitch_gain_search_3tap(target, ak, awk1, awk2, exc, gain_cdbk, gain_cdbk_size, pitch, p, nsf, bits, stack, exc2, r, new_target, &cdbk_index, plc_tuning, *cumul_gain, scaledown); if (err < best_err || best_err < 0) { SPEEX_COPY(best_exc, exc, nsf); SPEEX_COPY(best_target, new_target, nsf); best_err = err; best_pitch = pitch; best_gain_index = cdbk_index; } } /*printf ("pitch: %d %d\n", best_pitch, best_gain_index);*/ speex_bits_pack(bits, best_pitch - start, params->pitch_bits); speex_bits_pack(bits, best_gain_index, params->gain_bits); #ifdef FIXED_POINT *cumul_gain = MULT16_32_Q13(SHL16(params->gain_cdbk[4 * best_gain_index + 3], 8), MAX32(1024, *cumul_gain)); #else *cumul_gain = 0.03125 * MAX32(1024, *cumul_gain) * params->gain_cdbk[4 * best_gain_index + 3]; #endif /*printf ("%f\n", cumul_gain);*/ /*printf ("encode pitch: %d %d\n", best_pitch, best_gain_index);*/ SPEEX_COPY(exc, best_exc, nsf); SPEEX_COPY(target, best_target, nsf); #ifdef FIXED_POINT /* Scale target back up if needed */ if (scaledown) { for (i = 0; i < nsf; i++) target[i] = SHL16(target[i], 1); } #endif return pitch; }
void tonality_analysis(TonalityAnalysisState *tonal, AnalysisInfo *info_out, const CELTMode *celt_mode, const void *x, int len, int offset, int c1, int c2, int C, int lsb_depth, downmix_func downmix) { int i, b; const kiss_fft_state *kfft; VARDECL(kiss_fft_cpx, in); VARDECL(kiss_fft_cpx, out); int N = 480, N2=240; float * OPUS_RESTRICT A = tonal->angle; float * OPUS_RESTRICT dA = tonal->d_angle; float * OPUS_RESTRICT d2A = tonal->d2_angle; VARDECL(float, tonality); VARDECL(float, noisiness); float band_tonality[NB_TBANDS]; float logE[NB_TBANDS]; float BFCC[8]; float features[25]; float frame_tonality; float max_frame_tonality; /*float tw_sum=0;*/ float frame_noisiness; const float pi4 = (float)(M_PI*M_PI*M_PI*M_PI); float slope=0; float frame_stationarity; float relativeE; float frame_probs[2]; float alpha, alphaE, alphaE2; float frame_loudness; float bandwidth_mask; int bandwidth=0; float maxE = 0; float noise_floor; int remaining; AnalysisInfo *info; SAVE_STACK; tonal->last_transition++; alpha = 1.f/IMIN(20, 1+tonal->count); alphaE = 1.f/IMIN(50, 1+tonal->count); alphaE2 = 1.f/IMIN(1000, 1+tonal->count); if (tonal->count<4) tonal->music_prob = .5; kfft = celt_mode->mdct.kfft[0]; if (tonal->count==0) tonal->mem_fill = 240; downmix(x, &tonal->inmem[tonal->mem_fill], IMIN(len, ANALYSIS_BUF_SIZE-tonal->mem_fill), offset, c1, c2, C); if (tonal->mem_fill+len < ANALYSIS_BUF_SIZE) { tonal->mem_fill += len; /* Don't have enough to update the analysis */ RESTORE_STACK; return; } info = &tonal->info[tonal->write_pos++]; if (tonal->write_pos>=DETECT_SIZE) tonal->write_pos-=DETECT_SIZE; ALLOC(in, 480, kiss_fft_cpx); ALLOC(out, 480, kiss_fft_cpx); ALLOC(tonality, 240, float); ALLOC(noisiness, 240, float); for (i=0;i<N2;i++) { float w = analysis_window[i]; in[i].r = (kiss_fft_scalar)(w*tonal->inmem[i]); in[i].i = (kiss_fft_scalar)(w*tonal->inmem[N2+i]); in[N-i-1].r = (kiss_fft_scalar)(w*tonal->inmem[N-i-1]); in[N-i-1].i = (kiss_fft_scalar)(w*tonal->inmem[N+N2-i-1]); } OPUS_MOVE(tonal->inmem, tonal->inmem+ANALYSIS_BUF_SIZE-240, 240); remaining = len - (ANALYSIS_BUF_SIZE-tonal->mem_fill); downmix(x, &tonal->inmem[240], remaining, offset+ANALYSIS_BUF_SIZE-tonal->mem_fill, c1, c2, C); tonal->mem_fill = 240 + remaining; opus_fft(kfft, in, out); for (i=1;i<N2;i++) { float X1r, X2r, X1i, X2i; float angle, d_angle, d2_angle; float angle2, d_angle2, d2_angle2; float mod1, mod2, avg_mod; X1r = (float)out[i].r+out[N-i].r; X1i = (float)out[i].i-out[N-i].i; X2r = (float)out[i].i+out[N-i].i; X2i = (float)out[N-i].r-out[i].r; angle = (float)(.5f/M_PI)*fast_atan2f(X1i, X1r); d_angle = angle - A[i]; d2_angle = d_angle - dA[i]; angle2 = (float)(.5f/M_PI)*fast_atan2f(X2i, X2r); d_angle2 = angle2 - angle; d2_angle2 = d_angle2 - d_angle; mod1 = d2_angle - (float)floor(.5+d2_angle); noisiness[i] = ABS16(mod1); mod1 *= mod1; mod1 *= mod1; mod2 = d2_angle2 - (float)floor(.5+d2_angle2); noisiness[i] += ABS16(mod2); mod2 *= mod2; mod2 *= mod2; avg_mod = .25f*(d2A[i]+2.f*mod1+mod2); tonality[i] = 1.f/(1.f+40.f*16.f*pi4*avg_mod)-.015f; A[i] = angle2; dA[i] = d_angle2; d2A[i] = mod2; } frame_tonality = 0; max_frame_tonality = 0; /*tw_sum = 0;*/ info->activity = 0; frame_noisiness = 0; frame_stationarity = 0; if (!tonal->count) { for (b=0;b<NB_TBANDS;b++) { tonal->lowE[b] = 1e10; tonal->highE[b] = -1e10; } } relativeE = 0; frame_loudness = 0; for (b=0;b<NB_TBANDS;b++) { float E=0, tE=0, nE=0; float L1, L2; float stationarity; for (i=tbands[b];i<tbands[b+1];i++) { float binE = out[i].r*(float)out[i].r + out[N-i].r*(float)out[N-i].r + out[i].i*(float)out[i].i + out[N-i].i*(float)out[N-i].i; #ifdef FIXED_POINT /* FIXME: It's probably best to change the BFCC filter initial state instead */ binE *= 5.55e-17f; #endif E += binE; tE += binE*tonality[i]; nE += binE*2.f*(.5f-noisiness[i]); } tonal->E[tonal->E_count][b] = E; frame_noisiness += nE/(1e-15f+E); frame_loudness += (float)sqrt(E+1e-10f); logE[b] = (float)log(E+1e-10f); tonal->lowE[b] = MIN32(logE[b], tonal->lowE[b]+.01f); tonal->highE[b] = MAX32(logE[b], tonal->highE[b]-.1f); if (tonal->highE[b] < tonal->lowE[b]+1.f) { tonal->highE[b]+=.5f; tonal->lowE[b]-=.5f; } relativeE += (logE[b]-tonal->lowE[b])/(1e-15f+tonal->highE[b]-tonal->lowE[b]); L1=L2=0; for (i=0;i<NB_FRAMES;i++) { L1 += (float)sqrt(tonal->E[i][b]); L2 += tonal->E[i][b]; } stationarity = MIN16(0.99f,L1/(float)sqrt(1e-15+NB_FRAMES*L2)); stationarity *= stationarity; stationarity *= stationarity; frame_stationarity += stationarity; /*band_tonality[b] = tE/(1e-15+E)*/; band_tonality[b] = MAX16(tE/(1e-15f+E), stationarity*tonal->prev_band_tonality[b]); #if 0 if (b>=NB_TONAL_SKIP_BANDS) { frame_tonality += tweight[b]*band_tonality[b]; tw_sum += tweight[b]; } #else frame_tonality += band_tonality[b]; if (b>=NB_TBANDS-NB_TONAL_SKIP_BANDS) frame_tonality -= band_tonality[b-NB_TBANDS+NB_TONAL_SKIP_BANDS]; #endif max_frame_tonality = MAX16(max_frame_tonality, (1.f+.03f*(b-NB_TBANDS))*frame_tonality); slope += band_tonality[b]*(b-8); /*printf("%f %f ", band_tonality[b], stationarity);*/ tonal->prev_band_tonality[b] = band_tonality[b]; } bandwidth_mask = 0; bandwidth = 0; maxE = 0; noise_floor = 5.7e-4f/(1<<(IMAX(0,lsb_depth-8))); #ifdef FIXED_POINT noise_floor *= 1<<(15+SIG_SHIFT); #endif noise_floor *= noise_floor; for (b=0;b<NB_TOT_BANDS;b++) { float E=0; int band_start, band_end; /* Keep a margin of 300 Hz for aliasing */ band_start = extra_bands[b]; band_end = extra_bands[b+1]; for (i=band_start;i<band_end;i++) { float binE = out[i].r*(float)out[i].r + out[N-i].r*(float)out[N-i].r + out[i].i*(float)out[i].i + out[N-i].i*(float)out[N-i].i; E += binE; } maxE = MAX32(maxE, E); tonal->meanE[b] = MAX32((1-alphaE2)*tonal->meanE[b], E); E = MAX32(E, tonal->meanE[b]); /* Use a simple follower with 13 dB/Bark slope for spreading function */ bandwidth_mask = MAX32(.05f*bandwidth_mask, E); /* Consider the band "active" only if all these conditions are met: 1) less than 10 dB below the simple follower 2) less than 90 dB below the peak band (maximal masking possible considering both the ATH and the loudness-dependent slope of the spreading function) 3) above the PCM quantization noise floor */ if (E>.1*bandwidth_mask && E*1e9f > maxE && E > noise_floor*(band_end-band_start)) bandwidth = b; } if (tonal->count<=2) bandwidth = 20; frame_loudness = 20*(float)log10(frame_loudness); tonal->Etracker = MAX32(tonal->Etracker-.03f, frame_loudness); tonal->lowECount *= (1-alphaE); if (frame_loudness < tonal->Etracker-30) tonal->lowECount += alphaE; for (i=0;i<8;i++) { float sum=0; for (b=0;b<16;b++) sum += dct_table[i*16+b]*logE[b]; BFCC[i] = sum; } frame_stationarity /= NB_TBANDS; relativeE /= NB_TBANDS; if (tonal->count<10) relativeE = .5; frame_noisiness /= NB_TBANDS; #if 1 info->activity = frame_noisiness + (1-frame_noisiness)*relativeE; #else info->activity = .5*(1+frame_noisiness-frame_stationarity); #endif frame_tonality = (max_frame_tonality/(NB_TBANDS-NB_TONAL_SKIP_BANDS)); frame_tonality = MAX16(frame_tonality, tonal->prev_tonality*.8f); tonal->prev_tonality = frame_tonality; slope /= 8*8; info->tonality_slope = slope; tonal->E_count = (tonal->E_count+1)%NB_FRAMES; tonal->count++; info->tonality = frame_tonality; for (i=0;i<4;i++) features[i] = -0.12299f*(BFCC[i]+tonal->mem[i+24]) + 0.49195f*(tonal->mem[i]+tonal->mem[i+16]) + 0.69693f*tonal->mem[i+8] - 1.4349f*tonal->cmean[i]; for (i=0;i<4;i++) tonal->cmean[i] = (1-alpha)*tonal->cmean[i] + alpha*BFCC[i]; for (i=0;i<4;i++) features[4+i] = 0.63246f*(BFCC[i]-tonal->mem[i+24]) + 0.31623f*(tonal->mem[i]-tonal->mem[i+16]); for (i=0;i<3;i++) features[8+i] = 0.53452f*(BFCC[i]+tonal->mem[i+24]) - 0.26726f*(tonal->mem[i]+tonal->mem[i+16]) -0.53452f*tonal->mem[i+8]; if (tonal->count > 5) { for (i=0;i<9;i++) tonal->std[i] = (1-alpha)*tonal->std[i] + alpha*features[i]*features[i]; } for (i=0;i<8;i++) { tonal->mem[i+24] = tonal->mem[i+16]; tonal->mem[i+16] = tonal->mem[i+8]; tonal->mem[i+8] = tonal->mem[i]; tonal->mem[i] = BFCC[i]; } for (i=0;i<9;i++) features[11+i] = (float)sqrt(tonal->std[i]); features[20] = info->tonality; features[21] = info->activity; features[22] = frame_stationarity; features[23] = info->tonality_slope; features[24] = tonal->lowECount; #ifndef DISABLE_FLOAT_API mlp_process(&net, features, frame_probs); frame_probs[0] = .5f*(frame_probs[0]+1); /* Curve fitting between the MLP probability and the actual probability */ frame_probs[0] = .01f + 1.21f*frame_probs[0]*frame_probs[0] - .23f*(float)pow(frame_probs[0], 10); /* Probability of active audio (as opposed to silence) */ frame_probs[1] = .5f*frame_probs[1]+.5f; /* Consider that silence has a 50-50 probability. */ frame_probs[0] = frame_probs[1]*frame_probs[0] + (1-frame_probs[1])*.5f; /*printf("%f %f ", frame_probs[0], frame_probs[1]);*/ { /* Probability of state transition */ float tau; /* Represents independence of the MLP probabilities, where beta=1 means fully independent. */ float beta; /* Denormalized probability of speech (p0) and music (p1) after update */ float p0, p1; /* Probabilities for "all speech" and "all music" */ float s0, m0; /* Probability sum for renormalisation */ float psum; /* Instantaneous probability of speech and music, with beta pre-applied. */ float speech0; float music0; /* One transition every 3 minutes of active audio */ tau = .00005f*frame_probs[1]; beta = .05f; if (1) { /* Adapt beta based on how "unexpected" the new prob is */ float p, q; p = MAX16(.05f,MIN16(.95f,frame_probs[0])); q = MAX16(.05f,MIN16(.95f,tonal->music_prob)); beta = .01f+.05f*ABS16(p-q)/(p*(1-q)+q*(1-p)); } /* p0 and p1 are the probabilities of speech and music at this frame using only information from previous frame and applying the state transition model */ p0 = (1-tonal->music_prob)*(1-tau) + tonal->music_prob *tau; p1 = tonal->music_prob *(1-tau) + (1-tonal->music_prob)*tau; /* We apply the current probability with exponent beta to work around the fact that the probability estimates aren't independent. */ p0 *= (float)pow(1-frame_probs[0], beta); p1 *= (float)pow(frame_probs[0], beta); /* Normalise the probabilities to get the Marokv probability of music. */ tonal->music_prob = p1/(p0+p1); info->music_prob = tonal->music_prob; /* This chunk of code deals with delayed decision. */ psum=1e-20f; /* Instantaneous probability of speech and music, with beta pre-applied. */ speech0 = (float)pow(1-frame_probs[0], beta); music0 = (float)pow(frame_probs[0], beta); if (tonal->count==1) { tonal->pspeech[0]=.5; tonal->pmusic [0]=.5; } /* Updated probability of having only speech (s0) or only music (m0), before considering the new observation. */ s0 = tonal->pspeech[0] + tonal->pspeech[1]; m0 = tonal->pmusic [0] + tonal->pmusic [1]; /* Updates s0 and m0 with instantaneous probability. */ tonal->pspeech[0] = s0*(1-tau)*speech0; tonal->pmusic [0] = m0*(1-tau)*music0; /* Propagate the transition probabilities */ for (i=1;i<DETECT_SIZE-1;i++) { tonal->pspeech[i] = tonal->pspeech[i+1]*speech0; tonal->pmusic [i] = tonal->pmusic [i+1]*music0; } /* Probability that the latest frame is speech, when all the previous ones were music. */ tonal->pspeech[DETECT_SIZE-1] = m0*tau*speech0; /* Probability that the latest frame is music, when all the previous ones were speech. */ tonal->pmusic [DETECT_SIZE-1] = s0*tau*music0; /* Renormalise probabilities to 1 */ for (i=0;i<DETECT_SIZE;i++) psum += tonal->pspeech[i] + tonal->pmusic[i]; psum = 1.f/psum; for (i=0;i<DETECT_SIZE;i++) { tonal->pspeech[i] *= psum; tonal->pmusic [i] *= psum; } psum = tonal->pmusic[0]; for (i=1;i<DETECT_SIZE;i++) psum += tonal->pspeech[i]; /* Estimate our confidence in the speech/music decisions */ if (frame_probs[1]>.75) { if (tonal->music_prob>.9) { float adapt; adapt = 1.f/(++tonal->music_confidence_count); tonal->music_confidence_count = IMIN(tonal->music_confidence_count, 500); tonal->music_confidence += adapt*MAX16(-.2f,frame_probs[0]-tonal->music_confidence); } if (tonal->music_prob<.1) { float adapt; adapt = 1.f/(++tonal->speech_confidence_count); tonal->speech_confidence_count = IMIN(tonal->speech_confidence_count, 500); tonal->speech_confidence += adapt*MIN16(.2f,frame_probs[0]-tonal->speech_confidence); } } else { if (tonal->music_confidence_count==0) tonal->music_confidence = .9f; if (tonal->speech_confidence_count==0) tonal->speech_confidence = .1f; } } if (tonal->last_music != (tonal->music_prob>.5f)) tonal->last_transition=0; tonal->last_music = tonal->music_prob>.5f; #else info->music_prob = 0; #endif /*for (i=0;i<25;i++) printf("%f ", features[i]); printf("\n");*/ info->bandwidth = bandwidth; /*printf("%d %d\n", info->bandwidth, info->opus_bandwidth);*/ info->noisiness = frame_noisiness; info->valid = 1; if (info_out!=NULL) OPUS_COPY(info_out, info, 1); RESTORE_STACK; }
static void tonality_analysis(TonalityAnalysisState *tonal, const CELTMode *celt_mode, const void *x, int len, int offset, int c1, int c2, int C, int lsb_depth, downmix_func downmix) { int i, b; const kiss_fft_state *kfft; VARDECL(kiss_fft_cpx, in); VARDECL(kiss_fft_cpx, out); int N = 480, N2=240; float * OPUS_RESTRICT A = tonal->angle; float * OPUS_RESTRICT dA = tonal->d_angle; float * OPUS_RESTRICT d2A = tonal->d2_angle; VARDECL(float, tonality); VARDECL(float, noisiness); float band_tonality[NB_TBANDS]; float logE[NB_TBANDS]; float BFCC[8]; float features[25]; float frame_tonality; float max_frame_tonality; /*float tw_sum=0;*/ float frame_noisiness; const float pi4 = (float)(M_PI*M_PI*M_PI*M_PI); float slope=0; float frame_stationarity; float relativeE; float frame_probs[2]; float alpha, alphaE, alphaE2; float frame_loudness; float bandwidth_mask; int bandwidth=0; float maxE = 0; float noise_floor; int remaining; AnalysisInfo *info; float hp_ener; float tonality2[240]; float midE[8]; float spec_variability=0; float band_log2[NB_TBANDS+1]; float leakage_from[NB_TBANDS+1]; float leakage_to[NB_TBANDS+1]; SAVE_STACK; alpha = 1.f/IMIN(10, 1+tonal->count); alphaE = 1.f/IMIN(25, 1+tonal->count); alphaE2 = 1.f/IMIN(500, 1+tonal->count); if (tonal->Fs == 48000) { /* len and offset are now at 24 kHz. */ len/= 2; offset /= 2; } else if (tonal->Fs == 16000) { len = 3*len/2; offset = 3*offset/2; } if (tonal->count<4) { if (tonal->application == OPUS_APPLICATION_VOIP) tonal->music_prob = .1f; else tonal->music_prob = .625f; } kfft = celt_mode->mdct.kfft[0]; if (tonal->count==0) tonal->mem_fill = 240; tonal->hp_ener_accum += (float)downmix_and_resample(downmix, x, &tonal->inmem[tonal->mem_fill], tonal->downmix_state, IMIN(len, ANALYSIS_BUF_SIZE-tonal->mem_fill), offset, c1, c2, C, tonal->Fs); if (tonal->mem_fill+len < ANALYSIS_BUF_SIZE) { tonal->mem_fill += len; /* Don't have enough to update the analysis */ RESTORE_STACK; return; } hp_ener = tonal->hp_ener_accum; info = &tonal->info[tonal->write_pos++]; if (tonal->write_pos>=DETECT_SIZE) tonal->write_pos-=DETECT_SIZE; ALLOC(in, 480, kiss_fft_cpx); ALLOC(out, 480, kiss_fft_cpx); ALLOC(tonality, 240, float); ALLOC(noisiness, 240, float); for (i=0;i<N2;i++) { float w = analysis_window[i]; in[i].r = (kiss_fft_scalar)(w*tonal->inmem[i]); in[i].i = (kiss_fft_scalar)(w*tonal->inmem[N2+i]); in[N-i-1].r = (kiss_fft_scalar)(w*tonal->inmem[N-i-1]); in[N-i-1].i = (kiss_fft_scalar)(w*tonal->inmem[N+N2-i-1]); } OPUS_MOVE(tonal->inmem, tonal->inmem+ANALYSIS_BUF_SIZE-240, 240); remaining = len - (ANALYSIS_BUF_SIZE-tonal->mem_fill); tonal->hp_ener_accum = (float)downmix_and_resample(downmix, x, &tonal->inmem[240], tonal->downmix_state, remaining, offset+ANALYSIS_BUF_SIZE-tonal->mem_fill, c1, c2, C, tonal->Fs); tonal->mem_fill = 240 + remaining; opus_fft(kfft, in, out, tonal->arch); #ifndef FIXED_POINT /* If there's any NaN on the input, the entire output will be NaN, so we only need to check one value. */ if (celt_isnan(out[0].r)) { info->valid = 0; RESTORE_STACK; return; } #endif for (i=1;i<N2;i++) { float X1r, X2r, X1i, X2i; float angle, d_angle, d2_angle; float angle2, d_angle2, d2_angle2; float mod1, mod2, avg_mod; X1r = (float)out[i].r+out[N-i].r; X1i = (float)out[i].i-out[N-i].i; X2r = (float)out[i].i+out[N-i].i; X2i = (float)out[N-i].r-out[i].r; angle = (float)(.5f/M_PI)*fast_atan2f(X1i, X1r); d_angle = angle - A[i]; d2_angle = d_angle - dA[i]; angle2 = (float)(.5f/M_PI)*fast_atan2f(X2i, X2r); d_angle2 = angle2 - angle; d2_angle2 = d_angle2 - d_angle; mod1 = d2_angle - (float)float2int(d2_angle); noisiness[i] = ABS16(mod1); mod1 *= mod1; mod1 *= mod1; mod2 = d2_angle2 - (float)float2int(d2_angle2); noisiness[i] += ABS16(mod2); mod2 *= mod2; mod2 *= mod2; avg_mod = .25f*(d2A[i]+mod1+2*mod2); /* This introduces an extra delay of 2 frames in the detection. */ tonality[i] = 1.f/(1.f+40.f*16.f*pi4*avg_mod)-.015f; /* No delay on this detection, but it's less reliable. */ tonality2[i] = 1.f/(1.f+40.f*16.f*pi4*mod2)-.015f; A[i] = angle2; dA[i] = d_angle2; d2A[i] = mod2; } for (i=2;i<N2-1;i++) { float tt = MIN32(tonality2[i], MAX32(tonality2[i-1], tonality2[i+1])); tonality[i] = .9f*MAX32(tonality[i], tt-.1f); } frame_tonality = 0; max_frame_tonality = 0; /*tw_sum = 0;*/ info->activity = 0; frame_noisiness = 0; frame_stationarity = 0; if (!tonal->count) { for (b=0;b<NB_TBANDS;b++) { tonal->lowE[b] = 1e10; tonal->highE[b] = -1e10; } } relativeE = 0; frame_loudness = 0; /* The energy of the very first band is special because of DC. */ { float E = 0; float X1r, X2r; X1r = 2*(float)out[0].r; X2r = 2*(float)out[0].i; E = X1r*X1r + X2r*X2r; for (i=1;i<4;i++) { float binE = out[i].r*(float)out[i].r + out[N-i].r*(float)out[N-i].r + out[i].i*(float)out[i].i + out[N-i].i*(float)out[N-i].i; E += binE; } E = SCALE_ENER(E); band_log2[0] = .5f*1.442695f*(float)log(E+1e-10f); } for (b=0;b<NB_TBANDS;b++) { float E=0, tE=0, nE=0; float L1, L2; float stationarity; for (i=tbands[b];i<tbands[b+1];i++) { float binE = out[i].r*(float)out[i].r + out[N-i].r*(float)out[N-i].r + out[i].i*(float)out[i].i + out[N-i].i*(float)out[N-i].i; binE = SCALE_ENER(binE); E += binE; tE += binE*MAX32(0, tonality[i]); nE += binE*2.f*(.5f-noisiness[i]); } #ifndef FIXED_POINT /* Check for extreme band energies that could cause NaNs later. */ if (!(E<1e9f) || celt_isnan(E)) { info->valid = 0; RESTORE_STACK; return; } #endif tonal->E[tonal->E_count][b] = E; frame_noisiness += nE/(1e-15f+E); frame_loudness += (float)sqrt(E+1e-10f); logE[b] = (float)log(E+1e-10f); band_log2[b+1] = .5f*1.442695f*(float)log(E+1e-10f); tonal->logE[tonal->E_count][b] = logE[b]; if (tonal->count==0) tonal->highE[b] = tonal->lowE[b] = logE[b]; if (tonal->highE[b] > tonal->lowE[b] + 7.5) { if (tonal->highE[b] - logE[b] > logE[b] - tonal->lowE[b]) tonal->highE[b] -= .01f; else tonal->lowE[b] += .01f; } if (logE[b] > tonal->highE[b]) { tonal->highE[b] = logE[b]; tonal->lowE[b] = MAX32(tonal->highE[b]-15, tonal->lowE[b]); } else if (logE[b] < tonal->lowE[b]) { tonal->lowE[b] = logE[b]; tonal->highE[b] = MIN32(tonal->lowE[b]+15, tonal->highE[b]); } relativeE += (logE[b]-tonal->lowE[b])/(1e-15f + (tonal->highE[b]-tonal->lowE[b])); L1=L2=0; for (i=0;i<NB_FRAMES;i++) { L1 += (float)sqrt(tonal->E[i][b]); L2 += tonal->E[i][b]; } stationarity = MIN16(0.99f,L1/(float)sqrt(1e-15+NB_FRAMES*L2)); stationarity *= stationarity; stationarity *= stationarity; frame_stationarity += stationarity; /*band_tonality[b] = tE/(1e-15+E)*/; band_tonality[b] = MAX16(tE/(1e-15f+E), stationarity*tonal->prev_band_tonality[b]); #if 0 if (b>=NB_TONAL_SKIP_BANDS) { frame_tonality += tweight[b]*band_tonality[b]; tw_sum += tweight[b]; } #else frame_tonality += band_tonality[b]; if (b>=NB_TBANDS-NB_TONAL_SKIP_BANDS) frame_tonality -= band_tonality[b-NB_TBANDS+NB_TONAL_SKIP_BANDS]; #endif max_frame_tonality = MAX16(max_frame_tonality, (1.f+.03f*(b-NB_TBANDS))*frame_tonality); slope += band_tonality[b]*(b-8); /*printf("%f %f ", band_tonality[b], stationarity);*/ tonal->prev_band_tonality[b] = band_tonality[b]; } leakage_from[0] = band_log2[0]; leakage_to[0] = band_log2[0] - LEAKAGE_OFFSET; for (b=1;b<NB_TBANDS+1;b++) { float leak_slope = LEAKAGE_SLOPE*(tbands[b]-tbands[b-1])/4; leakage_from[b] = MIN16(leakage_from[b-1]+leak_slope, band_log2[b]); leakage_to[b] = MAX16(leakage_to[b-1]-leak_slope, band_log2[b]-LEAKAGE_OFFSET); } for (b=NB_TBANDS-2;b>=0;b--) { float leak_slope = LEAKAGE_SLOPE*(tbands[b+1]-tbands[b])/4; leakage_from[b] = MIN16(leakage_from[b+1]+leak_slope, leakage_from[b]); leakage_to[b] = MAX16(leakage_to[b+1]-leak_slope, leakage_to[b]); } celt_assert(NB_TBANDS+1 <= LEAK_BANDS); for (b=0;b<NB_TBANDS+1;b++) { /* leak_boost[] is made up of two terms. The first, based on leakage_to[], represents the boost needed to overcome the amount of analysis leakage cause in a weaker band b by louder neighbouring bands. The second, based on leakage_from[], applies to a loud band b for which the quantization noise causes synthesis leakage to the weaker neighbouring bands. */ float boost = MAX16(0, leakage_to[b] - band_log2[b]) + MAX16(0, band_log2[b] - (leakage_from[b]+LEAKAGE_OFFSET)); info->leak_boost[b] = IMIN(255, (int)floor(.5 + 64.f*boost)); } for (;b<LEAK_BANDS;b++) info->leak_boost[b] = 0; for (i=0;i<NB_FRAMES;i++) { int j; float mindist = 1e15f; for (j=0;j<NB_FRAMES;j++) { int k; float dist=0; for (k=0;k<NB_TBANDS;k++) { float tmp; tmp = tonal->logE[i][k] - tonal->logE[j][k]; dist += tmp*tmp; } if (j!=i) mindist = MIN32(mindist, dist); } spec_variability += mindist; } spec_variability = (float)sqrt(spec_variability/NB_FRAMES/NB_TBANDS); bandwidth_mask = 0; bandwidth = 0; maxE = 0; noise_floor = 5.7e-4f/(1<<(IMAX(0,lsb_depth-8))); noise_floor *= noise_floor; for (b=0;b<NB_TBANDS;b++) { float E=0; int band_start, band_end; /* Keep a margin of 300 Hz for aliasing */ band_start = tbands[b]; band_end = tbands[b+1]; for (i=band_start;i<band_end;i++) { float binE = out[i].r*(float)out[i].r + out[N-i].r*(float)out[N-i].r + out[i].i*(float)out[i].i + out[N-i].i*(float)out[N-i].i; E += binE; } E = SCALE_ENER(E); maxE = MAX32(maxE, E); tonal->meanE[b] = MAX32((1-alphaE2)*tonal->meanE[b], E); E = MAX32(E, tonal->meanE[b]); /* Use a simple follower with 13 dB/Bark slope for spreading function */ bandwidth_mask = MAX32(.05f*bandwidth_mask, E); /* Consider the band "active" only if all these conditions are met: 1) less than 10 dB below the simple follower 2) less than 90 dB below the peak band (maximal masking possible considering both the ATH and the loudness-dependent slope of the spreading function) 3) above the PCM quantization noise floor We use b+1 because the first CELT band isn't included in tbands[] */ if (E>.1*bandwidth_mask && E*1e9f > maxE && E > noise_floor*(band_end-band_start)) bandwidth = b+1; } /* Special case for the last two bands, for which we don't have spectrum but only the energy above 12 kHz. */ if (tonal->Fs == 48000) { float ratio; float E = hp_ener*(1.f/(240*240)); ratio = tonal->prev_bandwidth==20 ? 0.03f : 0.07f; #ifdef FIXED_POINT /* silk_resampler_down2_hp() shifted right by an extra 8 bits. */ E *= 256.f*(1.f/Q15ONE)*(1.f/Q15ONE); #endif maxE = MAX32(maxE, E); tonal->meanE[b] = MAX32((1-alphaE2)*tonal->meanE[b], E); E = MAX32(E, tonal->meanE[b]); /* Use a simple follower with 13 dB/Bark slope for spreading function */ bandwidth_mask = MAX32(.05f*bandwidth_mask, E); if (E>ratio*bandwidth_mask && E*1e9f > maxE && E > noise_floor*160) bandwidth = 20; /* This detector is unreliable, so if the bandwidth is close to SWB, assume it's FB. */ if (bandwidth >= 17) bandwidth = 20; } if (tonal->count<=2) bandwidth = 20; frame_loudness = 20*(float)log10(frame_loudness); tonal->Etracker = MAX32(tonal->Etracker-.003f, frame_loudness); tonal->lowECount *= (1-alphaE); if (frame_loudness < tonal->Etracker-30) tonal->lowECount += alphaE; for (i=0;i<8;i++) { float sum=0; for (b=0;b<16;b++) sum += dct_table[i*16+b]*logE[b]; BFCC[i] = sum; } for (i=0;i<8;i++) { float sum=0; for (b=0;b<16;b++) sum += dct_table[i*16+b]*.5f*(tonal->highE[b]+tonal->lowE[b]); midE[i] = sum; } frame_stationarity /= NB_TBANDS; relativeE /= NB_TBANDS; if (tonal->count<10) relativeE = .5f; frame_noisiness /= NB_TBANDS; #if 1 info->activity = frame_noisiness + (1-frame_noisiness)*relativeE; #else info->activity = .5*(1+frame_noisiness-frame_stationarity); #endif frame_tonality = (max_frame_tonality/(NB_TBANDS-NB_TONAL_SKIP_BANDS)); frame_tonality = MAX16(frame_tonality, tonal->prev_tonality*.8f); tonal->prev_tonality = frame_tonality; slope /= 8*8; info->tonality_slope = slope; tonal->E_count = (tonal->E_count+1)%NB_FRAMES; tonal->count = IMIN(tonal->count+1, ANALYSIS_COUNT_MAX); info->tonality = frame_tonality; for (i=0;i<4;i++) features[i] = -0.12299f*(BFCC[i]+tonal->mem[i+24]) + 0.49195f*(tonal->mem[i]+tonal->mem[i+16]) + 0.69693f*tonal->mem[i+8] - 1.4349f*tonal->cmean[i]; for (i=0;i<4;i++) tonal->cmean[i] = (1-alpha)*tonal->cmean[i] + alpha*BFCC[i]; for (i=0;i<4;i++) features[4+i] = 0.63246f*(BFCC[i]-tonal->mem[i+24]) + 0.31623f*(tonal->mem[i]-tonal->mem[i+16]); for (i=0;i<3;i++) features[8+i] = 0.53452f*(BFCC[i]+tonal->mem[i+24]) - 0.26726f*(tonal->mem[i]+tonal->mem[i+16]) -0.53452f*tonal->mem[i+8]; if (tonal->count > 5) { for (i=0;i<9;i++) tonal->std[i] = (1-alpha)*tonal->std[i] + alpha*features[i]*features[i]; } for (i=0;i<4;i++) features[i] = BFCC[i]-midE[i]; for (i=0;i<8;i++) { tonal->mem[i+24] = tonal->mem[i+16]; tonal->mem[i+16] = tonal->mem[i+8]; tonal->mem[i+8] = tonal->mem[i]; tonal->mem[i] = BFCC[i]; } for (i=0;i<9;i++) features[11+i] = (float)sqrt(tonal->std[i]) - std_feature_bias[i]; features[18] = spec_variability - 0.78f; features[20] = info->tonality - 0.154723f; features[21] = info->activity - 0.724643f; features[22] = frame_stationarity - 0.743717f; features[23] = info->tonality_slope + 0.069216f; features[24] = tonal->lowECount - 0.067930f; mlp_process(&net, features, frame_probs); frame_probs[0] = .5f*(frame_probs[0]+1); /* Curve fitting between the MLP probability and the actual probability */ /*frame_probs[0] = .01f + 1.21f*frame_probs[0]*frame_probs[0] - .23f*(float)pow(frame_probs[0], 10);*/ /* Probability of active audio (as opposed to silence) */ frame_probs[1] = .5f*frame_probs[1]+.5f; frame_probs[1] *= frame_probs[1]; /* Probability of speech or music vs noise */ info->activity_probability = frame_probs[1]; /*printf("%f %f\n", frame_probs[0], frame_probs[1]);*/ { /* Probability of state transition */ float tau; /* Represents independence of the MLP probabilities, where beta=1 means fully independent. */ float beta; /* Denormalized probability of speech (p0) and music (p1) after update */ float p0, p1; /* Probabilities for "all speech" and "all music" */ float s0, m0; /* Probability sum for renormalisation */ float psum; /* Instantaneous probability of speech and music, with beta pre-applied. */ float speech0; float music0; float p, q; /* More silence transitions for speech than for music. */ tau = .001f*tonal->music_prob + .01f*(1-tonal->music_prob); p = MAX16(.05f,MIN16(.95f,frame_probs[1])); q = MAX16(.05f,MIN16(.95f,tonal->vad_prob)); beta = .02f+.05f*ABS16(p-q)/(p*(1-q)+q*(1-p)); /* p0 and p1 are the probabilities of speech and music at this frame using only information from previous frame and applying the state transition model */ p0 = (1-tonal->vad_prob)*(1-tau) + tonal->vad_prob *tau; p1 = tonal->vad_prob *(1-tau) + (1-tonal->vad_prob)*tau; /* We apply the current probability with exponent beta to work around the fact that the probability estimates aren't independent. */ p0 *= (float)pow(1-frame_probs[1], beta); p1 *= (float)pow(frame_probs[1], beta); /* Normalise the probabilities to get the Marokv probability of music. */ tonal->vad_prob = p1/(p0+p1); info->vad_prob = tonal->vad_prob; /* Consider that silence has a 50-50 probability of being speech or music. */ frame_probs[0] = tonal->vad_prob*frame_probs[0] + (1-tonal->vad_prob)*.5f; /* One transition every 3 minutes of active audio */ tau = .0001f; /* Adapt beta based on how "unexpected" the new prob is */ p = MAX16(.05f,MIN16(.95f,frame_probs[0])); q = MAX16(.05f,MIN16(.95f,tonal->music_prob)); beta = .02f+.05f*ABS16(p-q)/(p*(1-q)+q*(1-p)); /* p0 and p1 are the probabilities of speech and music at this frame using only information from previous frame and applying the state transition model */ p0 = (1-tonal->music_prob)*(1-tau) + tonal->music_prob *tau; p1 = tonal->music_prob *(1-tau) + (1-tonal->music_prob)*tau; /* We apply the current probability with exponent beta to work around the fact that the probability estimates aren't independent. */ p0 *= (float)pow(1-frame_probs[0], beta); p1 *= (float)pow(frame_probs[0], beta); /* Normalise the probabilities to get the Marokv probability of music. */ tonal->music_prob = p1/(p0+p1); info->music_prob = tonal->music_prob; /*printf("%f %f %f %f\n", frame_probs[0], frame_probs[1], tonal->music_prob, tonal->vad_prob);*/ /* This chunk of code deals with delayed decision. */ psum=1e-20f; /* Instantaneous probability of speech and music, with beta pre-applied. */ speech0 = (float)pow(1-frame_probs[0], beta); music0 = (float)pow(frame_probs[0], beta); if (tonal->count==1) { if (tonal->application == OPUS_APPLICATION_VOIP) tonal->pmusic[0] = .1f; else tonal->pmusic[0] = .625f; tonal->pspeech[0] = 1-tonal->pmusic[0]; } /* Updated probability of having only speech (s0) or only music (m0), before considering the new observation. */ s0 = tonal->pspeech[0] + tonal->pspeech[1]; m0 = tonal->pmusic [0] + tonal->pmusic [1]; /* Updates s0 and m0 with instantaneous probability. */ tonal->pspeech[0] = s0*(1-tau)*speech0; tonal->pmusic [0] = m0*(1-tau)*music0; /* Propagate the transition probabilities */ for (i=1;i<DETECT_SIZE-1;i++) { tonal->pspeech[i] = tonal->pspeech[i+1]*speech0; tonal->pmusic [i] = tonal->pmusic [i+1]*music0; } /* Probability that the latest frame is speech, when all the previous ones were music. */ tonal->pspeech[DETECT_SIZE-1] = m0*tau*speech0; /* Probability that the latest frame is music, when all the previous ones were speech. */ tonal->pmusic [DETECT_SIZE-1] = s0*tau*music0; /* Renormalise probabilities to 1 */ for (i=0;i<DETECT_SIZE;i++) psum += tonal->pspeech[i] + tonal->pmusic[i]; psum = 1.f/psum; for (i=0;i<DETECT_SIZE;i++) { tonal->pspeech[i] *= psum; tonal->pmusic [i] *= psum; } psum = tonal->pmusic[0]; for (i=1;i<DETECT_SIZE;i++) psum += tonal->pspeech[i]; /* Estimate our confidence in the speech/music decisions */ if (frame_probs[1]>.75) { if (tonal->music_prob>.9) { float adapt; adapt = 1.f/(++tonal->music_confidence_count); tonal->music_confidence_count = IMIN(tonal->music_confidence_count, 500); tonal->music_confidence += adapt*MAX16(-.2f,frame_probs[0]-tonal->music_confidence); } if (tonal->music_prob<.1) { float adapt; adapt = 1.f/(++tonal->speech_confidence_count); tonal->speech_confidence_count = IMIN(tonal->speech_confidence_count, 500); tonal->speech_confidence += adapt*MIN16(.2f,frame_probs[0]-tonal->speech_confidence); } } } tonal->last_music = tonal->music_prob>.5f; #ifdef MLP_TRAINING for (i=0;i<25;i++) printf("%f ", features[i]); printf("\n"); #endif info->bandwidth = bandwidth; tonal->prev_bandwidth = bandwidth; /*printf("%d %d\n", info->bandwidth, info->opus_bandwidth);*/ info->noisiness = frame_noisiness; info->valid = 1; RESTORE_STACK; }
static int quant_coarse_energy_impl(const CELTMode *m, int start, int end, const opus_val16 *eBands, opus_val16 *oldEBands, opus_int32 budget, opus_int32 tell, const unsigned char *prob_model, opus_val16 *error, ec_enc *enc, int C, int LM, int intra, opus_val16 max_decay) { int i, c; int badness = 0; opus_val32 prev[2] = {0,0}; opus_val16 coef; opus_val16 beta; if (tell+3 <= budget) ec_enc_bit_logp(enc, intra, 3); if (intra) { coef = 0; beta = beta_intra; } else { beta = beta_coef[LM]; coef = pred_coef[LM]; } /* Encode at a fixed coarse resolution */ for (i=start;i<end;i++) { c=0; do { int bits_left; int qi, qi0; opus_val32 q; opus_val16 x; opus_val32 f, tmp; opus_val16 oldE; opus_val16 decay_bound; x = eBands[i+c*m->nbEBands]; oldE = MAX16(-QCONST16(9.f,DB_SHIFT), oldEBands[i+c*m->nbEBands]); #ifdef FIXED_POINT f = SHL32(EXTEND32(x),7) - PSHR32(MULT16_16(coef,oldE), 8) - prev[c]; /* Rounding to nearest integer here is really important! */ qi = (f+QCONST32(.5f,DB_SHIFT+7))>>(DB_SHIFT+7); decay_bound = EXTRACT16(MAX32(-QCONST16(28.f,DB_SHIFT), SUB32((opus_val32)oldEBands[i+c*m->nbEBands],max_decay))); #else f = x-coef*oldE-prev[c]; /* Rounding to nearest integer here is really important! */ qi = (int)floor(.5f+f); decay_bound = MAX16(-QCONST16(28.f,DB_SHIFT), oldEBands[i+c*m->nbEBands]) - max_decay; #endif /* Prevent the energy from going down too quickly (e.g. for bands that have just one bin) */ if (qi < 0 && x < decay_bound) { qi += (int)SHR16(SUB16(decay_bound,x), DB_SHIFT); if (qi > 0) qi = 0; } qi0 = qi; /* If we don't have enough bits to encode all the energy, just assume something safe. */ tell = ec_tell(enc); bits_left = budget-tell-3*C*(end-i); if (i!=start && bits_left < 30) { if (bits_left < 24) qi = IMIN(1, qi); if (bits_left < 16) qi = IMAX(-1, qi); } if (budget-tell >= 15) { int pi; pi = 2*IMIN(i,20); ec_laplace_encode(enc, &qi, prob_model[pi]<<7, prob_model[pi+1]<<6); } else if(budget-tell >= 2) { qi = IMAX(-1, IMIN(qi, 1)); ec_enc_icdf(enc, 2*qi^-(qi<0), small_energy_icdf, 2); } else if(budget-tell >= 1) { qi = IMIN(0, qi); ec_enc_bit_logp(enc, -qi, 1); } else qi = -1; error[i+c*m->nbEBands] = PSHR32(f,7) - SHL16(qi,DB_SHIFT); badness += abs(qi0-qi); q = (opus_val32)SHL32(EXTEND32(qi),DB_SHIFT); tmp = PSHR32(MULT16_16(coef,oldE),8) + prev[c] + SHL32(q,7); #ifdef FIXED_POINT tmp = MAX32(-QCONST32(28.f, DB_SHIFT+7), tmp); #endif oldEBands[i+c*m->nbEBands] = PSHR32(tmp, 7); prev[c] = prev[c] + SHL32(q,7) - MULT16_16(beta,PSHR32(q,8)); } while (++c < C); } return badness; }
void surround_analysis(const CELTMode *celt_mode, const void *pcm, opus_val16 *bandLogE, opus_val32 *mem, opus_val32 *preemph_mem, int len, int overlap, int channels, int rate, opus_copy_channel_in_func copy_channel_in, int arch ) { int c; int i; int LM; int pos[8] = {0}; int upsample; int frame_size; int freq_size; opus_val16 channel_offset; opus_val32 bandE[21]; opus_val16 maskLogE[3][21]; VARDECL(opus_val32, in); VARDECL(opus_val16, x); VARDECL(opus_val32, freq); SAVE_STACK; upsample = resampling_factor(rate); frame_size = len*upsample; freq_size = IMIN(960, frame_size); /* LM = log2(frame_size / 120) */ for (LM=0;LM<celt_mode->maxLM;LM++) if (celt_mode->shortMdctSize<<LM==frame_size) break; ALLOC(in, frame_size+overlap, opus_val32); ALLOC(x, len, opus_val16); ALLOC(freq, freq_size, opus_val32); channel_pos(channels, pos); for (c=0;c<3;c++) for (i=0;i<21;i++) maskLogE[c][i] = -QCONST16(28.f, DB_SHIFT); for (c=0;c<channels;c++) { int frame; int nb_frames = frame_size/freq_size; celt_assert(nb_frames*freq_size == frame_size); OPUS_COPY(in, mem+c*overlap, overlap); (*copy_channel_in)(x, 1, pcm, channels, c, len); celt_preemphasis(x, in+overlap, frame_size, 1, upsample, celt_mode->preemph, preemph_mem+c, 0); #ifndef FIXED_POINT { opus_val32 sum; sum = celt_inner_prod(in, in, frame_size+overlap, 0); /* This should filter out both NaNs and ridiculous signals that could cause NaNs further down. */ if (!(sum < 1e9f) || celt_isnan(sum)) { OPUS_CLEAR(in, frame_size+overlap); preemph_mem[c] = 0; } } #endif OPUS_CLEAR(bandE, 21); for (frame=0;frame<nb_frames;frame++) { opus_val32 tmpE[21]; clt_mdct_forward(&celt_mode->mdct, in+960*frame, freq, celt_mode->window, overlap, celt_mode->maxLM-LM, 1, arch); if (upsample != 1) { int bound = freq_size/upsample; for (i=0;i<bound;i++) freq[i] *= upsample; for (;i<freq_size;i++) freq[i] = 0; } compute_band_energies(celt_mode, freq, tmpE, 21, 1, LM); /* If we have multiple frames, take the max energy. */ for (i=0;i<21;i++) bandE[i] = MAX32(bandE[i], tmpE[i]); } amp2Log2(celt_mode, 21, 21, bandE, bandLogE+21*c, 1); /* Apply spreading function with -6 dB/band going up and -12 dB/band going down. */ for (i=1;i<21;i++) bandLogE[21*c+i] = MAX16(bandLogE[21*c+i], bandLogE[21*c+i-1]-QCONST16(1.f, DB_SHIFT)); for (i=19;i>=0;i--) bandLogE[21*c+i] = MAX16(bandLogE[21*c+i], bandLogE[21*c+i+1]-QCONST16(2.f, DB_SHIFT)); if (pos[c]==1) { for (i=0;i<21;i++) maskLogE[0][i] = logSum(maskLogE[0][i], bandLogE[21*c+i]); } else if (pos[c]==3) { for (i=0;i<21;i++) maskLogE[2][i] = logSum(maskLogE[2][i], bandLogE[21*c+i]); } else if (pos[c]==2) { for (i=0;i<21;i++) { maskLogE[0][i] = logSum(maskLogE[0][i], bandLogE[21*c+i]-QCONST16(.5f, DB_SHIFT)); maskLogE[2][i] = logSum(maskLogE[2][i], bandLogE[21*c+i]-QCONST16(.5f, DB_SHIFT)); } } #if 0 for (i=0;i<21;i++) printf("%f ", bandLogE[21*c+i]); float sum=0; for (i=0;i<21;i++) sum += bandLogE[21*c+i]; printf("%f ", sum/21); #endif OPUS_COPY(mem+c*overlap, in+frame_size, overlap); } for (i=0;i<21;i++) maskLogE[1][i] = MIN32(maskLogE[0][i],maskLogE[2][i]); channel_offset = HALF16(celt_log2(QCONST32(2.f,14)/(channels-1))); for (c=0;c<3;c++) for (i=0;i<21;i++) maskLogE[c][i] += channel_offset; #if 0 for (c=0;c<3;c++) { for (i=0;i<21;i++) printf("%f ", maskLogE[c][i]); } #endif for (c=0;c<channels;c++) { opus_val16 *mask; if (pos[c]!=0) { mask = &maskLogE[pos[c]-1][0]; for (i=0;i<21;i++) bandLogE[21*c+i] = bandLogE[21*c+i] - mask[i]; } else { for (i=0;i<21;i++) bandLogE[21*c+i] = 0; } #if 0 for (i=0;i<21;i++) printf("%f ", bandLogE[21*c+i]); printf("\n"); #endif #if 0 float sum=0; for (i=0;i<21;i++) sum += bandLogE[21*c+i]; printf("%f ", sum/(float)QCONST32(21.f, DB_SHIFT)); printf("\n"); #endif } RESTORE_STACK; }
/** Performs echo cancellation on a frame */ EXPORT void speex_echo_cancellation(SpeexEchoState *st, const spx_int16_t *in, const spx_int16_t *far_end, spx_int16_t *out) { int i,j, chan, speak; int N,M, C, K; spx_word32_t Syy,See,Sxx,Sdd, Sff; #ifdef TWO_PATH spx_word32_t Dbf; int update_foreground; #endif spx_word32_t Sey; spx_word16_t ss, ss_1; spx_float_t Pey = FLOAT_ONE, Pyy=FLOAT_ONE; spx_float_t alpha, alpha_1; spx_word16_t RER; spx_word32_t tmp32; N = st->window_size; M = st->M; C = st->C; K = st->K; st->cancel_count++; #ifdef FIXED_POINT ss=DIV32_16(11469,M); ss_1 = SUB16(32767,ss); #else ss=.35/M; ss_1 = 1-ss; #endif for (chan = 0; chan < C; chan++) { /* Apply a notch filter to make sure DC doesn't end up causing problems */ filter_dc_notch16(in+chan, st->notch_radius, st->input+chan*st->frame_size, st->frame_size, st->notch_mem+2*chan, C); /* Copy input data to buffer and apply pre-emphasis */ /* Copy input data to buffer */ for (i=0;i<st->frame_size;i++) { spx_word32_t tmp32; /* FIXME: This core has changed a bit, need to merge properly */ tmp32 = SUB32(EXTEND32(st->input[chan*st->frame_size+i]), EXTEND32(MULT16_16_P15(st->preemph, st->memD[chan]))); #ifdef FIXED_POINT if (tmp32 > 32767) { tmp32 = 32767; if (st->saturated == 0) st->saturated = 1; } if (tmp32 < -32767) { tmp32 = -32767; if (st->saturated == 0) st->saturated = 1; } #endif st->memD[chan] = st->input[chan*st->frame_size+i]; st->input[chan*st->frame_size+i] = EXTRACT16(tmp32); } } for (speak = 0; speak < K; speak++) { for (i=0;i<st->frame_size;i++) { spx_word32_t tmp32; st->x[speak*N+i] = st->x[speak*N+i+st->frame_size]; tmp32 = SUB32(EXTEND32(far_end[i*K+speak]), EXTEND32(MULT16_16_P15(st->preemph, st->memX[speak]))); #ifdef FIXED_POINT /*FIXME: If saturation occurs here, we need to freeze adaptation for M frames (not just one) */ if (tmp32 > 32767) { tmp32 = 32767; st->saturated = M+1; } if (tmp32 < -32767) { tmp32 = -32767; st->saturated = M+1; } #endif st->x[speak*N+i+st->frame_size] = EXTRACT16(tmp32); st->memX[speak] = far_end[i*K+speak]; } } for (speak = 0; speak < K; speak++) { /* Shift memory: this could be optimized eventually*/ for (j=M-1;j>=0;j--) { for (i=0;i<N;i++) st->X[(j+1)*N*K+speak*N+i] = st->X[j*N*K+speak*N+i]; } /* Convert x (echo input) to frequency domain */ spx_fft(st->fft_table, st->x+speak*N, &st->X[speak*N]); } Sxx = 0; for (speak = 0; speak < K; speak++) { Sxx += mdf_inner_prod(st->x+speak*N+st->frame_size, st->x+speak*N+st->frame_size, st->frame_size); power_spectrum_accum(st->X+speak*N, st->Xf, N); } Sff = 0; for (chan = 0; chan < C; chan++) { #ifdef TWO_PATH /* Compute foreground filter */ spectral_mul_accum16(st->X, st->foreground+chan*N*K*M, st->Y+chan*N, N, M*K); spx_ifft(st->fft_table, st->Y+chan*N, st->e+chan*N); for (i=0;i<st->frame_size;i++) st->e[chan*N+i] = SUB16(st->input[chan*st->frame_size+i], st->e[chan*N+i+st->frame_size]); Sff += mdf_inner_prod(st->e+chan*N, st->e+chan*N, st->frame_size); #endif } /* Adjust proportional adaption rate */ /* FIXME: Adjust that for C, K*/ if (st->adapted) mdf_adjust_prop (st->W, N, M, C*K, st->prop); /* Compute weight gradient */ if (st->saturated == 0) { for (chan = 0; chan < C; chan++) { for (speak = 0; speak < K; speak++) { for (j=M-1;j>=0;j--) { weighted_spectral_mul_conj(st->power_1, FLOAT_SHL(PSEUDOFLOAT(st->prop[j]),-15), &st->X[(j+1)*N*K+speak*N], st->E+chan*N, st->PHI, N); for (i=0;i<N;i++) st->W[chan*N*K*M + j*N*K + speak*N + i] += st->PHI[i]; } } } } else { st->saturated--; } /* FIXME: MC conversion required */ /* Update weight to prevent circular convolution (MDF / AUMDF) */ for (chan = 0; chan < C; chan++) { for (speak = 0; speak < K; speak++) { for (j=0;j<M;j++) { /* This is a variant of the Alternatively Updated MDF (AUMDF) */ /* Remove the "if" to make this an MDF filter */ if (j==0 || st->cancel_count%(M-1) == j-1) { #ifdef FIXED_POINT for (i=0;i<N;i++) st->wtmp2[i] = EXTRACT16(PSHR32(st->W[chan*N*K*M + j*N*K + speak*N + i],NORMALIZE_SCALEDOWN+16)); spx_ifft(st->fft_table, st->wtmp2, st->wtmp); for (i=0;i<st->frame_size;i++) { st->wtmp[i]=0; } for (i=st->frame_size;i<N;i++) { st->wtmp[i]=SHL16(st->wtmp[i],NORMALIZE_SCALEUP); } spx_fft(st->fft_table, st->wtmp, st->wtmp2); /* The "-1" in the shift is a sort of kludge that trades less efficient update speed for decrease noise */ for (i=0;i<N;i++) st->W[chan*N*K*M + j*N*K + speak*N + i] -= SHL32(EXTEND32(st->wtmp2[i]),16+NORMALIZE_SCALEDOWN-NORMALIZE_SCALEUP-1); #else spx_ifft(st->fft_table, &st->W[chan*N*K*M + j*N*K + speak*N], st->wtmp); for (i=st->frame_size;i<N;i++) { st->wtmp[i]=0; } spx_fft(st->fft_table, st->wtmp, &st->W[chan*N*K*M + j*N*K + speak*N]); #endif } } } } /* So we can use power_spectrum_accum */ for (i=0;i<=st->frame_size;i++) st->Rf[i] = st->Yf[i] = st->Xf[i] = 0; Dbf = 0; See = 0; #ifdef TWO_PATH /* Difference in response, this is used to estimate the variance of our residual power estimate */ for (chan = 0; chan < C; chan++) { spectral_mul_accum(st->X, st->W+chan*N*K*M, st->Y+chan*N, N, M*K); spx_ifft(st->fft_table, st->Y+chan*N, st->y+chan*N); for (i=0;i<st->frame_size;i++) st->e[chan*N+i] = SUB16(st->e[chan*N+i+st->frame_size], st->y[chan*N+i+st->frame_size]); Dbf += 10+mdf_inner_prod(st->e+chan*N, st->e+chan*N, st->frame_size); for (i=0;i<st->frame_size;i++) st->e[chan*N+i] = SUB16(st->input[chan*st->frame_size+i], st->y[chan*N+i+st->frame_size]); See += mdf_inner_prod(st->e+chan*N, st->e+chan*N, st->frame_size); } #endif #ifndef TWO_PATH Sff = See; #endif #ifdef TWO_PATH /* Logic for updating the foreground filter */ /* For two time windows, compute the mean of the energy difference, as well as the variance */ st->Davg1 = ADD32(MULT16_32_Q15(QCONST16(.6f,15),st->Davg1), MULT16_32_Q15(QCONST16(.4f,15),SUB32(Sff,See))); st->Davg2 = ADD32(MULT16_32_Q15(QCONST16(.85f,15),st->Davg2), MULT16_32_Q15(QCONST16(.15f,15),SUB32(Sff,See))); st->Dvar1 = FLOAT_ADD(FLOAT_MULT(VAR1_SMOOTH, st->Dvar1), FLOAT_MUL32U(MULT16_32_Q15(QCONST16(.4f,15),Sff), MULT16_32_Q15(QCONST16(.4f,15),Dbf))); st->Dvar2 = FLOAT_ADD(FLOAT_MULT(VAR2_SMOOTH, st->Dvar2), FLOAT_MUL32U(MULT16_32_Q15(QCONST16(.15f,15),Sff), MULT16_32_Q15(QCONST16(.15f,15),Dbf))); /* Equivalent float code: st->Davg1 = .6*st->Davg1 + .4*(Sff-See); st->Davg2 = .85*st->Davg2 + .15*(Sff-See); st->Dvar1 = .36*st->Dvar1 + .16*Sff*Dbf; st->Dvar2 = .7225*st->Dvar2 + .0225*Sff*Dbf; */ update_foreground = 0; /* Check if we have a statistically significant reduction in the residual echo */ /* Note that this is *not* Gaussian, so we need to be careful about the longer tail */ if (FLOAT_GT(FLOAT_MUL32U(SUB32(Sff,See),ABS32(SUB32(Sff,See))), FLOAT_MUL32U(Sff,Dbf))) update_foreground = 1; else if (FLOAT_GT(FLOAT_MUL32U(st->Davg1, ABS32(st->Davg1)), FLOAT_MULT(VAR1_UPDATE,(st->Dvar1)))) update_foreground = 1; else if (FLOAT_GT(FLOAT_MUL32U(st->Davg2, ABS32(st->Davg2)), FLOAT_MULT(VAR2_UPDATE,(st->Dvar2)))) update_foreground = 1; /* Do we update? */ if (update_foreground) { st->Davg1 = st->Davg2 = 0; st->Dvar1 = st->Dvar2 = FLOAT_ZERO; /* Copy background filter to foreground filter */ for (i=0;i<N*M*C*K;i++) st->foreground[i] = EXTRACT16(PSHR32(st->W[i],16)); /* Apply a smooth transition so as to not introduce blocking artifacts */ for (chan = 0; chan < C; chan++) for (i=0;i<st->frame_size;i++) st->e[chan*N+i+st->frame_size] = MULT16_16_Q15(st->window[i+st->frame_size],st->e[chan*N+i+st->frame_size]) + MULT16_16_Q15(st->window[i],st->y[chan*N+i+st->frame_size]); } else { int reset_background=0; /* Otherwise, check if the background filter is significantly worse */ if (FLOAT_GT(FLOAT_MUL32U(NEG32(SUB32(Sff,See)),ABS32(SUB32(Sff,See))), FLOAT_MULT(VAR_BACKTRACK,FLOAT_MUL32U(Sff,Dbf)))) reset_background = 1; if (FLOAT_GT(FLOAT_MUL32U(NEG32(st->Davg1), ABS32(st->Davg1)), FLOAT_MULT(VAR_BACKTRACK,st->Dvar1))) reset_background = 1; if (FLOAT_GT(FLOAT_MUL32U(NEG32(st->Davg2), ABS32(st->Davg2)), FLOAT_MULT(VAR_BACKTRACK,st->Dvar2))) reset_background = 1; if (reset_background) { /* Copy foreground filter to background filter */ for (i=0;i<N*M*C*K;i++) st->W[i] = SHL32(EXTEND32(st->foreground[i]),16); /* We also need to copy the output so as to get correct adaptation */ for (chan = 0; chan < C; chan++) { for (i=0;i<st->frame_size;i++) st->y[chan*N+i+st->frame_size] = st->e[chan*N+i+st->frame_size]; for (i=0;i<st->frame_size;i++) st->e[chan*N+i] = SUB16(st->input[chan*st->frame_size+i], st->y[chan*N+i+st->frame_size]); } See = Sff; st->Davg1 = st->Davg2 = 0; st->Dvar1 = st->Dvar2 = FLOAT_ZERO; } } #endif Sey = Syy = Sdd = 0; for (chan = 0; chan < C; chan++) { /* Compute error signal (for the output with de-emphasis) */ for (i=0;i<st->frame_size;i++) { spx_word32_t tmp_out; #ifdef TWO_PATH tmp_out = SUB32(EXTEND32(st->input[chan*st->frame_size+i]), EXTEND32(st->e[chan*N+i+st->frame_size])); #else tmp_out = SUB32(EXTEND32(st->input[chan*st->frame_size+i]), EXTEND32(st->y[chan*N+i+st->frame_size])); #endif tmp_out = ADD32(tmp_out, EXTEND32(MULT16_16_P15(st->preemph, st->memE[chan]))); /* This is an arbitrary test for saturation in the microphone signal */ if (in[i*C+chan] <= -32000 || in[i*C+chan] >= 32000) { if (st->saturated == 0) st->saturated = 1; } out[i*C+chan] = WORD2INT(tmp_out); st->memE[chan] = tmp_out; } #ifdef DUMP_ECHO_CANCEL_DATA dump_audio(in, far_end, out, st->frame_size); #endif /* Compute error signal (filter update version) */ for (i=0;i<st->frame_size;i++) { st->e[chan*N+i+st->frame_size] = st->e[chan*N+i]; st->e[chan*N+i] = 0; } /* Compute a bunch of correlations */ /* FIXME: bad merge */ Sey += mdf_inner_prod(st->e+chan*N+st->frame_size, st->y+chan*N+st->frame_size, st->frame_size); Syy += mdf_inner_prod(st->y+chan*N+st->frame_size, st->y+chan*N+st->frame_size, st->frame_size); Sdd += mdf_inner_prod(st->input+chan*st->frame_size, st->input+chan*st->frame_size, st->frame_size); /* Convert error to frequency domain */ spx_fft(st->fft_table, st->e+chan*N, st->E+chan*N); for (i=0;i<st->frame_size;i++) st->y[i+chan*N] = 0; spx_fft(st->fft_table, st->y+chan*N, st->Y+chan*N); /* Compute power spectrum of echo (X), error (E) and filter response (Y) */ power_spectrum_accum(st->E+chan*N, st->Rf, N); power_spectrum_accum(st->Y+chan*N, st->Yf, N); } /*printf ("%f %f %f %f\n", Sff, See, Syy, Sdd, st->update_cond);*/ /* Do some sanity check */ if (!(Syy>=0 && Sxx>=0 && See >= 0) #ifndef FIXED_POINT || !(Sff < N*1e9 && Syy < N*1e9 && Sxx < N*1e9) #endif ) { /* Things have gone really bad */ st->screwed_up += 50; for (i=0;i<st->frame_size*C;i++) out[i] = 0; } else if (SHR32(Sff, 2) > ADD32(Sdd, SHR32(MULT16_16(N, 10000),6))) { /* AEC seems to add lots of echo instead of removing it, let's see if it will improve */ st->screwed_up++; } else { /* Everything's fine */ st->screwed_up=0; } if (st->screwed_up>=50) { speex_warning("The echo canceller started acting funny and got slapped (reset). It swears it will behave now."); speex_echo_state_reset(st); return; } /* Add a small noise floor to make sure not to have problems when dividing */ See = MAX32(See, SHR32(MULT16_16(N, 100),6)); for (speak = 0; speak < K; speak++) { Sxx += mdf_inner_prod(st->x+speak*N+st->frame_size, st->x+speak*N+st->frame_size, st->frame_size); power_spectrum_accum(st->X+speak*N, st->Xf, N); } /* Smooth far end energy estimate over time */ for (j=0;j<=st->frame_size;j++) st->power[j] = MULT16_32_Q15(ss_1,st->power[j]) + 1 + MULT16_32_Q15(ss,st->Xf[j]); /* Compute filtered spectra and (cross-)correlations */ for (j=st->frame_size;j>=0;j--) { spx_float_t Eh, Yh; Eh = PSEUDOFLOAT(st->Rf[j] - st->Eh[j]); Yh = PSEUDOFLOAT(st->Yf[j] - st->Yh[j]); Pey = FLOAT_ADD(Pey,FLOAT_MULT(Eh,Yh)); Pyy = FLOAT_ADD(Pyy,FLOAT_MULT(Yh,Yh)); #ifdef FIXED_POINT st->Eh[j] = MAC16_32_Q15(MULT16_32_Q15(SUB16(32767,st->spec_average),st->Eh[j]), st->spec_average, st->Rf[j]); st->Yh[j] = MAC16_32_Q15(MULT16_32_Q15(SUB16(32767,st->spec_average),st->Yh[j]), st->spec_average, st->Yf[j]); #else st->Eh[j] = (1-st->spec_average)*st->Eh[j] + st->spec_average*st->Rf[j]; st->Yh[j] = (1-st->spec_average)*st->Yh[j] + st->spec_average*st->Yf[j]; #endif } Pyy = FLOAT_SQRT(Pyy); Pey = FLOAT_DIVU(Pey,Pyy); /* Compute correlation updatete rate */ tmp32 = MULT16_32_Q15(st->beta0,Syy); if (tmp32 > MULT16_32_Q15(st->beta_max,See)) tmp32 = MULT16_32_Q15(st->beta_max,See); alpha = FLOAT_DIV32(tmp32, See); alpha_1 = FLOAT_SUB(FLOAT_ONE, alpha); /* Update correlations (recursive average) */ st->Pey = FLOAT_ADD(FLOAT_MULT(alpha_1,st->Pey) , FLOAT_MULT(alpha,Pey)); st->Pyy = FLOAT_ADD(FLOAT_MULT(alpha_1,st->Pyy) , FLOAT_MULT(alpha,Pyy)); if (FLOAT_LT(st->Pyy, FLOAT_ONE)) st->Pyy = FLOAT_ONE; /* We don't really hope to get better than 33 dB (MIN_LEAK-3dB) attenuation anyway */ if (FLOAT_LT(st->Pey, FLOAT_MULT(MIN_LEAK,st->Pyy))) st->Pey = FLOAT_MULT(MIN_LEAK,st->Pyy); if (FLOAT_GT(st->Pey, st->Pyy)) st->Pey = st->Pyy; /* leak_estimate is the linear regression result */ st->leak_estimate = FLOAT_EXTRACT16(FLOAT_SHL(FLOAT_DIVU(st->Pey, st->Pyy),14)); /* This looks like a stupid bug, but it's right (because we convert from Q14 to Q15) */ if (st->leak_estimate > 16383) st->leak_estimate = 32767; else st->leak_estimate = SHL16(st->leak_estimate,1); /*printf ("%f\n", st->leak_estimate);*/ /* Compute Residual to Error Ratio */ #ifdef FIXED_POINT tmp32 = MULT16_32_Q15(st->leak_estimate,Syy); tmp32 = ADD32(SHR32(Sxx,13), ADD32(tmp32, SHL32(tmp32,1))); /* Check for y in e (lower bound on RER) */ { spx_float_t bound = PSEUDOFLOAT(Sey); bound = FLOAT_DIVU(FLOAT_MULT(bound, bound), PSEUDOFLOAT(ADD32(1,Syy))); if (FLOAT_GT(bound, PSEUDOFLOAT(See))) tmp32 = See; else if (tmp32 < FLOAT_EXTRACT32(bound)) tmp32 = FLOAT_EXTRACT32(bound); } if (tmp32 > SHR32(See,1)) tmp32 = SHR32(See,1); RER = FLOAT_EXTRACT16(FLOAT_SHL(FLOAT_DIV32(tmp32,See),15)); #else RER = (.0001*Sxx + 3.*MULT16_32_Q15(st->leak_estimate,Syy)) / See; /* Check for y in e (lower bound on RER) */ if (RER < Sey*Sey/(1+See*Syy)) RER = Sey*Sey/(1+See*Syy); if (RER > .5) RER = .5; #endif /* We consider that the filter has had minimal adaptation if the following is true*/ if (!st->adapted && st->sum_adapt > SHL32(EXTEND32(M),15) && MULT16_32_Q15(st->leak_estimate,Syy) > MULT16_32_Q15(QCONST16(.03f,15),Syy)) { st->adapted = 1; } if (st->adapted) { /* Normal learning rate calculation once we're past the minimal adaptation phase */ for (i=0;i<=st->frame_size;i++) { spx_word32_t r, e; /* Compute frequency-domain adaptation mask */ r = MULT16_32_Q15(st->leak_estimate,SHL32(st->Yf[i],3)); e = SHL32(st->Rf[i],3)+1; #ifdef FIXED_POINT if (r>SHR32(e,1)) r = SHR32(e,1); #else if (r>.5*e) r = .5*e; #endif r = MULT16_32_Q15(QCONST16(.7,15),r) + MULT16_32_Q15(QCONST16(.3,15),(spx_word32_t)(MULT16_32_Q15(RER,e))); /*st->power_1[i] = adapt_rate*r/(e*(1+st->power[i]));*/ st->power_1[i] = FLOAT_SHL(FLOAT_DIV32_FLOAT(r,FLOAT_MUL32U(e,st->power[i]+10)),WEIGHT_SHIFT+16); } } else { /* Temporary adaption rate if filter is not yet adapted enough */ spx_word16_t adapt_rate=0; if (Sxx > SHR32(MULT16_16(N, 1000),6)) { tmp32 = MULT16_32_Q15(QCONST16(.25f, 15), Sxx); #ifdef FIXED_POINT if (tmp32 > SHR32(See,2)) tmp32 = SHR32(See,2); #else if (tmp32 > .25*See) tmp32 = .25*See; #endif adapt_rate = FLOAT_EXTRACT16(FLOAT_SHL(FLOAT_DIV32(tmp32, See),15)); } for (i=0;i<=st->frame_size;i++) st->power_1[i] = FLOAT_SHL(FLOAT_DIV32(EXTEND32(adapt_rate),ADD32(st->power[i],10)),WEIGHT_SHIFT+1); /* How much have we adapted so far? */ st->sum_adapt = ADD32(st->sum_adapt,adapt_rate); } /* FIXME: MC conversion required */ for (i=0;i<st->frame_size;i++) st->last_y[i] = st->last_y[st->frame_size+i]; if (st->adapted) { /* If the filter is adapted, take the filtered echo */ for (i=0;i<st->frame_size;i++) st->last_y[st->frame_size+i] = in[i]-out[i]; } else { /* If filter isn't adapted yet, all we can do is take the far end signal directly */ /* moved earlier: for (i=0;i<N;i++) st->last_y[i] = st->x[i];*/ } }