static int encode(struct aufilt_enc_st *st, int16_t *sampv, size_t *sampc)
{
struct enc_st *est = (struct enc_st *)st;
struct speex_st *sp = est->st;
if (*sampc) {
speex_echo_capture(sp->state, sampv, sp->out);
memcpy(sampv, sp->out, *sampc * 2);
}
return 0;
}
static int tdav_speex_denoise_process_record(tmedia_denoise_t* self, void* audio_frame, tsk_bool_t* silence_or_noise)
{
tdav_speex_denoise_t *denoiser = (tdav_speex_denoise_t *)self;
int vad;
if(denoiser->preprocess_state_record){
if(denoiser->echo_state && denoiser->echo_output_frame){
speex_echo_capture(denoiser->echo_state, audio_frame, denoiser->echo_output_frame);
memcpy(audio_frame, denoiser->echo_output_frame, denoiser->frame_size*sizeof(spx_int16_t));
}
vad = speex_preprocess_run(denoiser->preprocess_state_record, audio_frame);
if(!vad && TMEDIA_DENOISE(denoiser)->vad_enabled){
*silence_or_noise = tsk_true;
}
}
return 0;
}
/*
* Perform echo cancellation to captured frame.
*/
PJ_DEF(pj_status_t) speex_aec_capture( void *state,
pj_int16_t *rec_frm,
unsigned options )
{
speex_ec *echo = (speex_ec*) state;
/* Sanity checks */
PJ_ASSERT_RETURN(echo && rec_frm, PJ_EINVAL);
PJ_UNUSED_ARG(options);
/* Cancel echo */
pjmedia_copy_samples(echo->tmp_frame, rec_frm, echo->samples_per_frame);
speex_echo_capture(echo->state,
(spx_int16_t*)echo->tmp_frame,
(spx_int16_t*)rec_frm);
/* Apply preprocessing */
speex_preprocess_run(echo->preprocess, (spx_int16_t*)rec_frm);
return PJ_SUCCESS;
}
int main(int argc, char *argv[])
{
int sd, rc, n;
int i;
struct sockaddr_in cliAddr, remoteAddr;
char msg[MAX_MSG];
struct hostent *h;
int local_port, remote_port;
int nfds;
struct pollfd *pfds;
SpeexPreprocessState *preprocess;
AlsaDevice *audio_dev;
int tmp;
if (argc != 5)
{
fprintf(stderr, "wrong options\n");
exit(1);
}
h = gethostbyname(argv[2]);
if(h==NULL) {
fprintf(stderr, "%s: unknown host '%s' \n", argv[0], argv[1]);
exit(1);
}
local_port = atoi(argv[3]);
remote_port = atoi(argv[4]);
printf("%s: sending data to '%s' (IP : %s) \n", argv[0], h->h_name,
inet_ntoa(*(struct in_addr *)h->h_addr_list[0]));
{
remoteAddr.sin_family = h->h_addrtype;
memcpy((char *) &remoteAddr.sin_addr.s_addr,
h->h_addr_list[0], h->h_length);
remoteAddr.sin_port = htons(remote_port);
}
/* socket creation */
sd=socket(AF_INET, SOCK_DGRAM, 0);
if(sd<0) {
printf("%s: cannot open socket \n",argv[0]);
exit(1);
}
/* bind any port */
cliAddr.sin_family = AF_INET;
cliAddr.sin_addr.s_addr = htonl(INADDR_ANY);
cliAddr.sin_port = htons(local_port);
rc = bind(sd, (struct sockaddr *) &cliAddr, sizeof(cliAddr));
if(rc<0) {
printf("%s: cannot bind port\n", argv[0]);
exit(1);
}
/* Setup audio device */
audio_dev = alsa_device_open(argv[1], SAMPLING_RATE, 1, FRAME_SIZE);
/* Setup the encoder and decoder in wideband */
void *enc_state, *dec_state;
enc_state = speex_encoder_init(&speex_wb_mode);
tmp = 8;
speex_encoder_ctl(enc_state, SPEEX_SET_QUALITY, &tmp);
tmp = 2;
speex_encoder_ctl(enc_state, SPEEX_SET_COMPLEXITY, &tmp);
dec_state = speex_decoder_init(&speex_wb_mode);
tmp = 1;
speex_decoder_ctl(dec_state, SPEEX_SET_ENH, &tmp);
SpeexBits enc_bits, dec_bits;
speex_bits_init(&enc_bits);
speex_bits_init(&dec_bits);
struct sched_param param;
/*param.sched_priority = 40; */
param.sched_priority = sched_get_priority_min(SCHED_FIFO);
if (sched_setscheduler(0,SCHED_FIFO,¶m))
perror("sched_setscheduler");
int send_timestamp = 0;
int recv_started=0;
/* Setup all file descriptors for poll()ing */
nfds = alsa_device_nfds(audio_dev);
pfds = malloc(sizeof(*pfds)*(nfds+1));
alsa_device_getfds(audio_dev, pfds, nfds);
pfds[nfds].fd = sd;
pfds[nfds].events = POLLIN;
/* Setup jitter buffer using decoder */
SpeexJitter jitter;
speex_jitter_init(&jitter, dec_state, SAMPLING_RATE);
/* Echo canceller with 200 ms tail length */
SpeexEchoState *echo_state = speex_echo_state_init(FRAME_SIZE, 10*FRAME_SIZE);
tmp = SAMPLING_RATE;
speex_echo_ctl(echo_state, SPEEX_ECHO_SET_SAMPLING_RATE, &tmp);
/* Setup preprocessor and associate with echo canceller for residual echo suppression */
preprocess = speex_preprocess_state_init(FRAME_SIZE, SAMPLING_RATE);
speex_preprocess_ctl(preprocess, SPEEX_PREPROCESS_SET_ECHO_STATE, echo_state);
alsa_device_start(audio_dev);
/* Infinite loop on capture, playback and receiving packets */
while (1)
{
/* Wait for either 1) capture 2) playback 3) socket data */
poll(pfds, nfds+1, -1);
/* Received packets */
if (pfds[nfds].revents & POLLIN)
{
/*fprintf (stderr, "x");*/
n = recv(sd, msg, MAX_MSG, 0);
int recv_timestamp = ((int*)msg)[1];
int payload = ((int*)msg)[0];
if ((payload & 0x80000000) == 0)
{
/* Put content of the packet into the jitter buffer, except for the pseudo-header */
speex_jitter_put(&jitter, msg+8, n-8, recv_timestamp);
recv_started = 1;
}
}
/* Ready to play a frame (playback) */
if (alsa_device_playback_ready(audio_dev, pfds, nfds))
{
short pcm[FRAME_SIZE];
if (recv_started)
{
/* Get audio from the jitter buffer */
speex_jitter_get(&jitter, pcm, NULL);
} else {
for (i=0; i<FRAME_SIZE; i++)
pcm[i] = 0;
}
/* Playback the audio and reset the echo canceller if we got an underrun */
if (alsa_device_write(audio_dev, pcm, FRAME_SIZE))
speex_echo_state_reset(echo_state);
/* Put frame into playback buffer */
speex_echo_playback(echo_state, pcm);
}
/* Audio available from the soundcard (capture) */
if (alsa_device_capture_ready(audio_dev, pfds, nfds))
{
short pcm[FRAME_SIZE], pcm2[FRAME_SIZE];
char outpacket[MAX_MSG];
/* Get audio from the soundcard */
alsa_device_read(audio_dev, pcm, FRAME_SIZE);
/* Perform echo cancellation */
speex_echo_capture(echo_state, pcm, pcm2);
for (i=0; i<FRAME_SIZE; i++)
pcm[i] = pcm2[i];
speex_bits_reset(&enc_bits);
/* Apply noise/echo suppression */
speex_preprocess_run(preprocess, pcm);
/* Encode */
speex_encode_int(enc_state, pcm, &enc_bits);
int packetSize = speex_bits_write(&enc_bits, outpacket+8, MAX_MSG);
/* Pseudo header: four null bytes and a 32-bit timestamp */
((int*)outpacket)[0] = htonl(0);
((int*)outpacket)[1] = send_timestamp;
send_timestamp += FRAME_SIZE;
rc = sendto(sd, outpacket, packetSize+8, 0,
(struct sockaddr *) &remoteAddr,
sizeof(remoteAddr));
if(rc<0) {
printf("cannot send audio data\n");
close(sd);
exit(1);
}
}
}
return 0;
}
bool t_audio_rx::get_sound_samples(unsigned short &sound_payload_size, bool &silence) {
int status;
struct timespec sleeptimer;
//struct timeval debug_timer;
silence = false;
mtx_3way.lock();
if (is_3way && !is_main_rx_3way) {
// We are not the main receiver in a 3-way call, so
// get the sound samples from the local media buffer.
// This buffer will be filled by the main receiver.
if (!media_3way_peer_rx->get(input_sample_buf, SAMPLE_BUF_SIZE)) {
// The mutex is unlocked before going to sleep.
// First I had the mutex unlock after the sleep.
// That worked fine with LinuxThreading, but it does
// not work with NPTL. It causes a deadlock when
// the main receiver calls post_media_peer_rx_3way
// as NPTL does not fair scheduling. This thread
// simly gets the lock again and the main receiver
// dies from starvation.
mtx_3way.unlock();
// There is not enough data yet. Sleep for 1 ms.
sleeptimer.tv_sec = 0;
sleeptimer.tv_nsec = 1000000;
nanosleep(&sleeptimer, NULL);
return false;
}
mtx_3way.unlock();
} else {
// Don't keep the 3way mutex locked while waiting for the DSP.
mtx_3way.unlock();
// Get the sound samples from the DSP
status = input_device->read(input_sample_buf, SAMPLE_BUF_SIZE);
if (status != SAMPLE_BUF_SIZE) {
if (!logged_capture_failure) {
// Log this failure only once
log_file->write_header("t_audio_rx::get_sound_samples",
LOG_NORMAL, LOG_WARNING);
log_file->write_raw("Audio rx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": sound capture failed.\n");
log_file->write_raw("Status: ");
log_file->write_raw(status);
log_file->write_endl();
log_file->write_footer();
logged_capture_failure = true;
}
stop_running = true;
return false;
}
// If line is muted, then fill sample buffer with silence.
// Note that we keep reading the dsp, to prevent the DSP buffers
// from filling up.
if (get_line()->get_is_muted()) {
memset(input_sample_buf, 0, SAMPLE_BUF_SIZE);
}
}
// Convert buffer to a buffer of shorts as the samples are 16 bits
short *sb = (short *)input_sample_buf;
mtx_3way.lock();
if (is_3way) {
// Send the sound samples to the other receiver if we
// are the main receiver.
// There may be no other receiver when one of the far-ends
// has put the call on-hold.
if (is_main_rx_3way && peer_rx_3way) {
peer_rx_3way->post_media_peer_rx_3way(input_sample_buf, SAMPLE_BUF_SIZE,
audio_encoder->get_sample_rate());
}
// Mix the sound samples with the 3rd party
if (media_3way_peer_tx->get(mix_buf_3way, SAMPLE_BUF_SIZE)) {
short *mix_sb = (short *)mix_buf_3way;
for (int i = 0; i < SAMPLE_BUF_SIZE / 2; i++) {
sb[i] = mix_linear_pcm(sb[i], mix_sb[i]);
}
}
}
mtx_3way.unlock();
/*** PREPROCESSING & ENCODING ***/
bool preprocessing_silence = false;
#ifdef HAVE_SPEEX
// speex acoustic echo cancellation
if (audio_session->get_do_echo_cancellation() && !audio_session->get_echo_captured_last()) {
spx_int16_t *input_buf = new spx_int16_t[SAMPLE_BUF_SIZE/2];
MEMMAN_NEW_ARRAY(input_buf);
for (int i = 0; i < SAMPLE_BUF_SIZE / 2; i++) {
input_buf[i] = sb[i];
}
speex_echo_capture(audio_session->get_speex_echo_state(), input_buf, sb);
audio_session->set_echo_captured_last(true);
MEMMAN_DELETE_ARRAY(input_buf);
delete [] input_buf;
}
// preprocessing
preprocessing_silence = !speex_preprocess_run(speex_preprocess_state, sb);
// According to the speex API documentation the return value
// from speex_preprocess_run() is only defined when VAD is
// enabled. So to be safe, reset the return value, if VAD is
// disabled.
if (!speex_dsp_vad) preprocessing_silence = false;
#endif
// encoding
sound_payload_size = audio_encoder->encode(sb, nsamples, payload, payload_size, silence);
// recognizing silence (both from preprocessing and encoding)
silence = silence || preprocessing_silence;
return true;
}
void CSpeexEC::DoSpeexEchoCapture(const short* pRecData, short *out)
{
if (!m_bHasInit)
return;
speex_echo_capture(m_pState, pRecData, out);
}
int KotiAEC_process(const int16_t* farend, const int16_t* nearend, int16_t* out)
{
int ret = -1, i = 0, frame_size = 0;
switch(aec_core_used)
{
#ifdef WEBRTC_AEC_CORE_ENABLED
case WEBRTC_AEC:
if(farend)
WebRtcAec_BufferFarend(webrtc_aec_pty.webrtc_aec, farend, webrtc_aec_pty.frame_size);
if(!WebRtcAec_Process(webrtc_aec_pty.webrtc_aec, nearend, NULL, out, NULL, webrtc_aec_pty.frame_size,
webrtc_aec_pty.sndcard_delay_ms, 0))
{
ret = 0;
}
if(webrtc_aec_pty.webrtc_ns)
{
WebRtcNsx_Process((NsxHandle*)webrtc_aec_pty.webrtc_ns, out, NULL, out, NULL);
if(webrtc_aec_pty.frame_size == 160)
WebRtcNsx_Process((NsxHandle*)webrtc_aec_pty.webrtc_ns, out+80, NULL, out+80, NULL);
}
if(webrtc_aec_pty.webrtc_agc)
{
int32_t out_c; uint8_t warn_status;
WebRtcAgc_Process(webrtc_aec_pty.webrtc_agc, out, NULL, webrtc_aec_pty.frame_size, out, NULL, 32,
&out_c, 1, &warn_status);
}
// if(webrtc_aec_pty.webrtc_ns)
// {
// WebRtcNsx_Process((NsxHandle*)webrtc_aec_pty.webrtc_ns, out, NULL, out, NULL);
// if(webrtc_aec_pty.frame_size == 160)
// WebRtcNsx_Process((NsxHandle*)webrtc_aec_pty.webrtc_ns, out+80, NULL, out+80, NULL);
// }
frame_size = webrtc_aec_pty.frame_size;
break;
case WEBRTC_AECM:
/*
if(farend)
WebRtcAecm_BufferFarend(webrtc_aecm_pty.webrtc_aec, farend, webrtc_aecm_pty.frame_size);
if(!WebRtcAecm_Process(webrtc_aecm_pty.webrtc_aec, nearend, NULL, out, webrtc_aecm_pty.frame_size,
webrtc_aecm_pty.sndcard_delay_ms))
{
ret = 0;
}
*/
memcpy(proc_tmp_buf, nearend, webrtc_aecm_pty.frame_size*2);
if(webrtc_aecm_pty.webrtc_ns)
{
WebRtcNsx_Process((NsxHandle*)webrtc_aecm_pty.webrtc_ns, proc_tmp_buf, NULL, proc_tmp_buf, NULL);
if(webrtc_aecm_pty.frame_size == 160)
WebRtcNsx_Process((NsxHandle*)webrtc_aecm_pty.webrtc_ns, proc_tmp_buf+80, NULL, proc_tmp_buf+80, NULL);
}
if(webrtc_aecm_pty.webrtc_agc)
{
int32_t out_c; uint8_t warn_status;
WebRtcAgc_Process(webrtc_aecm_pty.webrtc_agc, proc_tmp_buf, NULL, webrtc_aecm_pty.frame_size, proc_tmp_buf, NULL, 32,
&out_c, 1, &warn_status);
}
// AEC
if(farend)
WebRtcAecm_BufferFarend(webrtc_aecm_pty.webrtc_aec, farend, webrtc_aecm_pty.frame_size);
if(!WebRtcAecm_Process(webrtc_aecm_pty.webrtc_aec, proc_tmp_buf, NULL, out, webrtc_aecm_pty.frame_size,
webrtc_aecm_pty.sndcard_delay_ms))
{
ret = 0;
}
frame_size = webrtc_aecm_pty.frame_size;
break;
#endif
case SPEEX_AEC:
default:
#ifdef OLD_SPEEX_AEC
speex_echo_cancel((SpeexEchoState*)speex_aec_pty.speex_echo_state, nearend, farend, out, speex_aec_pty.nosie);
if(speex_preprocess((SpeexPreprocessState*)speex_aec_pty.speex_preprocess_state, out, speex_aec_pty.nosie) == 1)
ret = 0;
#else
if(farend)
speex_echo_cancellation((SpeexEchoState*)speex_aec_pty.speex_echo_state, nearend, farend, out);
else
speex_echo_capture((SpeexEchoState*)speex_aec_pty.speex_echo_state, nearend, out);
// speex_preprocess_estimate_update((SpeexPreprocessState*)speex_aec_pty.speex_preprocess_state, out);
if(speex_preprocess_run((SpeexPreprocessState*)speex_aec_pty.speex_preprocess_state, out) == 1)
ret = 0;
#endif
frame_size = speex_aec_pty.frame_size;
break;
}
// if the output sound needed amplify
if(output_sound_amplification != 1.0f && output_sound_amplification > 0)
{
for(; i<frame_size; ++i)
out[i] = out[i]*output_sound_amplification;
}
return ret;
}
