int speex_aec_playback_for_async(int16_t* farend, float snd_amplification)
{
int ret_val = -1;
switch(aec_core_used)
{
#ifdef WEBRTC_AEC_CORE_ENABLED
case WEBRTC_AEC:
if(webrtc_aec_pty.webrtc_aec)
ret_val = WebRtcAec_BufferFarend(webrtc_aec_pty.webrtc_aec, farend, webrtc_aec_pty.frame_size);
break;
case WEBRTC_AECM:
if(webrtc_aecm_pty.webrtc_aec)
ret_val = WebRtcAecm_BufferFarend(webrtc_aecm_pty.webrtc_aec, farend, webrtc_aecm_pty.frame_size);
break;
#endif
case SPEEX_AEC:
default:
if(speex_aec_pty.speex_echo_state)
{
speex_echo_playback((SpeexEchoState*)speex_aec_pty.speex_echo_state, farend);
// speex_preprocess_run((SpeexPreprocessState*)speex_aec_pty.speex_preprocess_state_tmp, farend);
ret_val = 0;
}
break;
}
if(snd_amplification != 1.0f && snd_amplification > 0)
{
for(int i=0; i<speex_aec_pty.frame_size; ++i)
farend[i] = farend[i]*snd_amplification;
}
return ret_val;
}
static int tdav_speex_denoise_echo_playback(tmedia_denoise_t* self, const void* echo_frame)
{
tdav_speex_denoise_t *denoiser = (tdav_speex_denoise_t *)self;
if(denoiser->echo_state){
speex_echo_playback(denoiser->echo_state, echo_frame);
}
return 0;
}
static int decode(struct aufilt_dec_st *st, int16_t *sampv, size_t *sampc)
{
struct dec_st *dst = (struct dec_st *)st;
struct speex_st *sp = dst->st;
if (*sampc)
speex_echo_playback(sp->state, sampv);
return 0;
}
/*
* Let AEC know that a frame was queued to be played.
*/
PJ_DEF(pj_status_t) speex_aec_playback( void *state,
pj_int16_t *play_frm )
{
speex_ec *echo = (speex_ec*) state;
/* Sanity checks */
PJ_ASSERT_RETURN(echo && play_frm, PJ_EINVAL);
speex_echo_playback(echo->state, (spx_int16_t*)play_frm);
return PJ_SUCCESS;
}
void t_audio_tx::run(void) {
const AppDataUnit* adu;
struct timespec sleeptimer;
//struct timeval debug_timer, debug_timer_prev;
int last_seqnum = -1; // seqnum of last received RTP packet
// RTP packets with multiple SSRCs may be received. Each SSRC
// represents an audio stream. Twinkle will only play 1 audio stream.
// On a reception of a new SSRC, Twinkle will switch over to play the
// new stream. This supports devices that change SSRC during a call.
uint32 ssrc_current = 0;
bool recvd_dtmf = false; // indicates if last RTP packets is a DTMF event
// The running flag is set already in t_audio_session::run to prevent
// a crash when the thread gets destroyed before it starts running.
// is_running = true;
uint32 rtp_timestamp = 0;
// This thread may not take the lock on the transaction layer to
// prevent dead locks
phone->add_prohibited_thread();
ui->add_prohibited_thread();
while (true) {
do {
adu = NULL;
if (stop_running) break;
rtp_timestamp = rtp_session->getFirstTimestamp();
adu = rtp_session->getData(
rtp_session->getFirstTimestamp());
if (adu == NULL || adu->getSize() <= 0) {
// There is no packet available. This may have
// several reasons:
// - the thread scheduling granularity does
// not match ptime
// - packet lost
// - packet delayed
// Wait another cycle for a packet. The
// jitter buffer will cope with this variation.
if (adu) {
delete adu;
adu = NULL;
}
// If we are the mixer in a 3-way call and there
// is enough media from the other far-end then
// this must be sent to the dsp.
if (is_3way && is_3way_mixer &&
media_3way_peer_tx->size_content() >=
ptime * (audio_sample_rate(codec) / 1000) * 2)
{
// Fill the sample buffer with silence
int len = ptime * (audio_sample_rate(codec) / 1000) * 2;
memset(sample_buf, 0, len);
play_pcm(sample_buf, len, true);
}
// Sleep ptime ms
sleeptimer.tv_sec = 0;
if (ptime >= 20) {
sleeptimer.tv_nsec =
ptime * 1000000 - 10000000;
} else {
// With a thread schedule of 10ms
// granularity, this will schedule the
// thread every 10ms.
sleeptimer.tv_nsec = 5000000;
}
nanosleep(&sleeptimer, NULL);
}
} while (adu == NULL || (adu->getSize() <= 0));
if (stop_running) {
if (adu) delete adu;
break;
}
if (adu) {
// adu is created by ccRTP, but we have to delete it,
// so report it to MEMMAN
MEMMAN_NEW(const_cast<ost::AppDataUnit*>(adu));
}
// Check for a codec change
map<unsigned short, t_audio_codec>::const_iterator it_codec;
it_codec = payload2codec.find(adu->getType());
t_audio_codec recvd_codec = CODEC_NULL;
if (it_codec != payload2codec.end()) {
recvd_codec = it_codec->second;
}
// Switch over to new SSRC
if (last_seqnum == -1 || ssrc_current != adu->getSource().getID()) {
if (recvd_codec != CODEC_NULL) {
ssrc_current = adu->getSource().getID();
// An SSRC defines a sequence number space. So a new
// SSRC starts with a new random sequence number
last_seqnum = -1;
log_file->write_header("t_audio_tx::run",
LOG_NORMAL);
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": play SSRC ");
log_file->write_raw(ssrc_current);
log_file->write_endl();
log_file->write_footer();
} else {
// SSRC received had an unsupported codec
// Discard.
// KLUDGE: for now this supports a scenario where a
// far-end starts ZRTP negotiation by sending CN
// packets with a separate SSRC while ZRTP is disabled
// in Twinkle. Twinkle will then receive the CN packets
// and discard them here as CN is an unsupported codec.
log_file->write_header("t_audio_tx::run",
LOG_NORMAL, LOG_DEBUG);
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": SSRC received (");
log_file->write_raw(adu->getSource().getID());
log_file->write_raw(") has unsupported codec ");
log_file->write_raw(adu->getType());
log_file->write_endl();
log_file->write_footer();
MEMMAN_DELETE(const_cast<ost::AppDataUnit*>(adu));
delete adu;
continue;
}
}
map<t_audio_codec, t_audio_decoder *>::const_iterator it_decoder;
it_decoder = map_audio_decoder.find(recvd_codec);
if (it_decoder != map_audio_decoder.end()) {
if (codec != recvd_codec) {
codec = recvd_codec;
get_line()->ci_set_recv_codec(codec);
ui->cb_async_recv_codec_changed(get_line()->get_line_number(),
codec);
log_file->write_header("t_audio_tx::run",
LOG_NORMAL, LOG_DEBUG);
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": codec change to ");
log_file->write_raw(ui->format_codec(codec));
log_file->write_endl();
log_file->write_footer();
}
} else {
if (adu->getType() == pt_telephone_event ||
adu->getType() == pt_telephone_event_alt)
{
recvd_dtmf = true;
} else {
if (codec != CODEC_UNSUPPORTED) {
codec = CODEC_UNSUPPORTED;
get_line()->ci_set_recv_codec(codec);
ui->cb_async_recv_codec_changed(
get_line()->get_line_number(), codec);
log_file->write_header("t_audio_tx::run",
LOG_NORMAL, LOG_DEBUG);
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": payload type ");
log_file->write_raw(adu->getType());
log_file->write_raw(" not supported\n");
log_file->write_footer();
}
last_seqnum = adu->getSeqNum();
MEMMAN_DELETE(const_cast<ost::AppDataUnit*>(adu));
delete adu;
continue;
}
}
// DTMF event
if (recvd_dtmf) {
// NOTE: the DTMF tone will be detected here
// while there might still be data in the jitter
// buffer. If the jitter buffer was already sent
// to the DSP, then the DSP will continue to play
// out the buffer sound samples.
if (dtmf_previous_timestamp != rtp_timestamp) {
// A new DTMF tone has been received.
dtmf_previous_timestamp = rtp_timestamp;
t_rtp_telephone_event *e =
(t_rtp_telephone_event *)adu->getData();
ui->cb_async_dtmf_detected(get_line()->get_line_number(),
e->get_event());
// Log DTMF event
log_file->write_header("t_audio_tx::run");
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": detected DTMF event - ");
log_file->write_raw(e->get_event());
log_file->write_endl();
log_file->write_footer();
}
recvd_dtmf = false;
last_seqnum = adu->getSeqNum();
MEMMAN_DELETE(const_cast<ost::AppDataUnit*>(adu));
delete adu;
continue;
}
// Discard invalide payload sizes
if (!map_audio_decoder[codec]->valid_payload_size(
adu->getSize(), SAMPLE_BUF_SIZE / 2))
{
log_file->write_header("t_audio_tx::run", LOG_NORMAL, LOG_DEBUG);
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": RTP payload size (");
log_file->write_raw((unsigned long)(adu->getSize()));
log_file->write_raw(" bytes) invalid for \n");
log_file->write_raw(ui->format_codec(codec));
log_file->write_footer();
last_seqnum = adu->getSeqNum();
MEMMAN_DELETE(const_cast<ost::AppDataUnit*>(adu));
delete adu;
continue;
}
unsigned short recvd_ptime;
recvd_ptime = map_audio_decoder[codec]->get_ptime(adu->getSize());
// Log a change of ptime
if (ptime != recvd_ptime) {
log_file->write_header("t_audio_tx::run", LOG_NORMAL, LOG_DEBUG);
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": ptime changed from ");
log_file->write_raw(ptime);
log_file->write_raw(" ms to ");
log_file->write_raw(recvd_ptime);
log_file->write_raw(" ms\n");
log_file->write_footer();
ptime = recvd_ptime;
}
// Check for lost packets
// This must be done before decoding the received samples as the
// speex decoder has its own PLC algorithm for which it needs the decoding
// state before decoding the new samples.
seq16_t seq_recvd(adu->getSeqNum());
seq16_t seq_last(static_cast<uint16>(last_seqnum));
if (last_seqnum != -1 && seq_recvd - seq_last > 1) {
// Packets have been lost
uint16 num_lost = (seq_recvd - seq_last) - 1;
log_file->write_header("t_audio_tx::run", LOG_NORMAL, LOG_DEBUG);
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": ");
log_file->write_raw(num_lost);
log_file->write_raw(" RTP packets lost.\n");
log_file->write_footer();
if (num_lost <= conceal_num) {
// Conceal packet loss
conceal(num_lost);
}
clear_conceal_buf();
}
// Determine if resampling is needed due to dynamic change to
// codec with other sample rate.
short downsample_factor = 1;
short upsample_factor = 1;
if (audio_sample_rate(codec) > sc_sample_rate) {
downsample_factor = audio_sample_rate(codec) / sc_sample_rate;
} else if (audio_sample_rate(codec) < sc_sample_rate) {
upsample_factor = sc_sample_rate / audio_sample_rate(codec);
}
// Create sample buffer. If no resampling is needed, the sample
// buffer from the audio_tx object can be used directly.
// Otherwise a temporary sample buffers is created that will
// be resampled to the object's sample buffer later.
short *sb;
int sb_size;
if (downsample_factor > 1) {
sb_size = SAMPLE_BUF_SIZE / 2 * downsample_factor;
sb = new short[sb_size];
MEMMAN_NEW_ARRAY(sb);
} else if (upsample_factor > 1) {
sb_size = SAMPLE_BUF_SIZE / 2;
sb = new short[SAMPLE_BUF_SIZE / 2];
MEMMAN_NEW_ARRAY(sb);
} else {
sb_size = SAMPLE_BUF_SIZE / 2;
sb = (short *)sample_buf;
}
// Decode the audio
unsigned char *payload = const_cast<uint8 *>(adu->getData());
short sample_size; // size in bytes
sample_size = 2 * map_audio_decoder[codec]->decode(payload, adu->getSize(), sb, sb_size);
// Resample if needed
if (downsample_factor > 1) {
short *p = sb;
sb = (short *)sample_buf;
for (int i = 0; i < sample_size / 2; i += downsample_factor) {
sb[i / downsample_factor] = p[i];
}
MEMMAN_DELETE_ARRAY(p);
delete [] p;
sample_size /= downsample_factor;
} else if (upsample_factor > 1) {
short *p = sb;
sb = (short *)sample_buf;
for (int i = 0; i < sample_size / 2; i++) {
for (int j = 0; j < upsample_factor; j++) {
sb[i * upsample_factor + j] = p[i];
}
}
MEMMAN_DELETE_ARRAY(p);
delete [] p;
sample_size *= upsample_factor;
}
// If the decoder deliverd 0 bytes, then it failed
if (sample_size == 0) {
last_seqnum = adu->getSeqNum();
MEMMAN_DELETE(const_cast<ost::AppDataUnit*>(adu));
delete adu;
continue;
}
// Discard packet if we are lacking behind. This happens if the
// soundcard plays at a rate less than the requested sample rate.
if (rtp_session->isWaiting(&(adu->getSource()))) {
uint32 last_ts = rtp_session->getLastTimestamp(&(adu->getSource()));
uint32 diff;
diff = last_ts - rtp_timestamp;
if (diff > (uint32_t)(JITTER_BUF_SIZE(sc_sample_rate) / AUDIO_SAMPLE_SIZE) * 8)
{
log_file->write_header("t_audio_tx::run", LOG_NORMAL, LOG_DEBUG);
log_file->write_raw("Audio tx line ");
log_file->write_raw(get_line()->get_line_number()+1);
log_file->write_raw(": discard delayed packet.\n");
log_file->write_raw("Timestamp: ");
log_file->write_raw(rtp_timestamp);
log_file->write_raw(", Last timestamp: ");
log_file->write_raw((long unsigned int)last_ts);
log_file->write_endl();
log_file->write_footer();
last_seqnum = adu->getSeqNum();
MEMMAN_DELETE(const_cast<ost::AppDataUnit*>(adu));
delete adu;
continue;
}
}
play_pcm(sample_buf, sample_size);
retain_for_concealment(sample_buf, sample_size);
last_seqnum = adu->getSeqNum();
MEMMAN_DELETE(const_cast<ost::AppDataUnit*>(adu));
delete adu;
// No sleep is done here but in the loop waiting
// for a new packet. If a packet is already available
// it can be send to the sound card immediately so
// the play-out buffer keeps filled.
// If the play-out buffer gets empty you hear a
// crack in the sound.
#ifdef HAVE_SPEEX
// store decoded output for (optional) echo cancellation
if (audio_session->get_do_echo_cancellation()) {
if (audio_session->get_echo_captured_last()) {
speex_echo_playback(audio_session->get_speex_echo_state(), (spx_int16_t *) sb);
audio_session->set_echo_captured_last(false);;
}
}
#endif
}
phone->remove_prohibited_thread();
ui->remove_prohibited_thread();
is_running = false;
}
int main(int argc, char *argv[])
{
int sd, rc, n;
int i;
struct sockaddr_in cliAddr, remoteAddr;
char msg[MAX_MSG];
struct hostent *h;
int local_port, remote_port;
int nfds;
struct pollfd *pfds;
SpeexPreprocessState *preprocess;
AlsaDevice *audio_dev;
int tmp;
if (argc != 5)
{
fprintf(stderr, "wrong options\n");
exit(1);
}
h = gethostbyname(argv[2]);
if(h==NULL) {
fprintf(stderr, "%s: unknown host '%s' \n", argv[0], argv[1]);
exit(1);
}
local_port = atoi(argv[3]);
remote_port = atoi(argv[4]);
printf("%s: sending data to '%s' (IP : %s) \n", argv[0], h->h_name,
inet_ntoa(*(struct in_addr *)h->h_addr_list[0]));
{
remoteAddr.sin_family = h->h_addrtype;
memcpy((char *) &remoteAddr.sin_addr.s_addr,
h->h_addr_list[0], h->h_length);
remoteAddr.sin_port = htons(remote_port);
}
/* socket creation */
sd=socket(AF_INET, SOCK_DGRAM, 0);
if(sd<0) {
printf("%s: cannot open socket \n",argv[0]);
exit(1);
}
/* bind any port */
cliAddr.sin_family = AF_INET;
cliAddr.sin_addr.s_addr = htonl(INADDR_ANY);
cliAddr.sin_port = htons(local_port);
rc = bind(sd, (struct sockaddr *) &cliAddr, sizeof(cliAddr));
if(rc<0) {
printf("%s: cannot bind port\n", argv[0]);
exit(1);
}
/* Setup audio device */
audio_dev = alsa_device_open(argv[1], SAMPLING_RATE, 1, FRAME_SIZE);
/* Setup the encoder and decoder in wideband */
void *enc_state, *dec_state;
enc_state = speex_encoder_init(&speex_wb_mode);
tmp = 8;
speex_encoder_ctl(enc_state, SPEEX_SET_QUALITY, &tmp);
tmp = 2;
speex_encoder_ctl(enc_state, SPEEX_SET_COMPLEXITY, &tmp);
dec_state = speex_decoder_init(&speex_wb_mode);
tmp = 1;
speex_decoder_ctl(dec_state, SPEEX_SET_ENH, &tmp);
SpeexBits enc_bits, dec_bits;
speex_bits_init(&enc_bits);
speex_bits_init(&dec_bits);
struct sched_param param;
/*param.sched_priority = 40; */
param.sched_priority = sched_get_priority_min(SCHED_FIFO);
if (sched_setscheduler(0,SCHED_FIFO,¶m))
perror("sched_setscheduler");
int send_timestamp = 0;
int recv_started=0;
/* Setup all file descriptors for poll()ing */
nfds = alsa_device_nfds(audio_dev);
pfds = malloc(sizeof(*pfds)*(nfds+1));
alsa_device_getfds(audio_dev, pfds, nfds);
pfds[nfds].fd = sd;
pfds[nfds].events = POLLIN;
/* Setup jitter buffer using decoder */
SpeexJitter jitter;
speex_jitter_init(&jitter, dec_state, SAMPLING_RATE);
/* Echo canceller with 200 ms tail length */
SpeexEchoState *echo_state = speex_echo_state_init(FRAME_SIZE, 10*FRAME_SIZE);
tmp = SAMPLING_RATE;
speex_echo_ctl(echo_state, SPEEX_ECHO_SET_SAMPLING_RATE, &tmp);
/* Setup preprocessor and associate with echo canceller for residual echo suppression */
preprocess = speex_preprocess_state_init(FRAME_SIZE, SAMPLING_RATE);
speex_preprocess_ctl(preprocess, SPEEX_PREPROCESS_SET_ECHO_STATE, echo_state);
alsa_device_start(audio_dev);
/* Infinite loop on capture, playback and receiving packets */
while (1)
{
/* Wait for either 1) capture 2) playback 3) socket data */
poll(pfds, nfds+1, -1);
/* Received packets */
if (pfds[nfds].revents & POLLIN)
{
/*fprintf (stderr, "x");*/
n = recv(sd, msg, MAX_MSG, 0);
int recv_timestamp = ((int*)msg)[1];
int payload = ((int*)msg)[0];
if ((payload & 0x80000000) == 0)
{
/* Put content of the packet into the jitter buffer, except for the pseudo-header */
speex_jitter_put(&jitter, msg+8, n-8, recv_timestamp);
recv_started = 1;
}
}
/* Ready to play a frame (playback) */
if (alsa_device_playback_ready(audio_dev, pfds, nfds))
{
short pcm[FRAME_SIZE];
if (recv_started)
{
/* Get audio from the jitter buffer */
speex_jitter_get(&jitter, pcm, NULL);
} else {
for (i=0; i<FRAME_SIZE; i++)
pcm[i] = 0;
}
/* Playback the audio and reset the echo canceller if we got an underrun */
if (alsa_device_write(audio_dev, pcm, FRAME_SIZE))
speex_echo_state_reset(echo_state);
/* Put frame into playback buffer */
speex_echo_playback(echo_state, pcm);
}
/* Audio available from the soundcard (capture) */
if (alsa_device_capture_ready(audio_dev, pfds, nfds))
{
short pcm[FRAME_SIZE], pcm2[FRAME_SIZE];
char outpacket[MAX_MSG];
/* Get audio from the soundcard */
alsa_device_read(audio_dev, pcm, FRAME_SIZE);
/* Perform echo cancellation */
speex_echo_capture(echo_state, pcm, pcm2);
for (i=0; i<FRAME_SIZE; i++)
pcm[i] = pcm2[i];
speex_bits_reset(&enc_bits);
/* Apply noise/echo suppression */
speex_preprocess_run(preprocess, pcm);
/* Encode */
speex_encode_int(enc_state, pcm, &enc_bits);
int packetSize = speex_bits_write(&enc_bits, outpacket+8, MAX_MSG);
/* Pseudo header: four null bytes and a 32-bit timestamp */
((int*)outpacket)[0] = htonl(0);
((int*)outpacket)[1] = send_timestamp;
send_timestamp += FRAME_SIZE;
rc = sendto(sd, outpacket, packetSize+8, 0,
(struct sockaddr *) &remoteAddr,
sizeof(remoteAddr));
if(rc<0) {
printf("cannot send audio data\n");
close(sd);
exit(1);
}
}
}
return 0;
}
void CSpeexEC::DoEchoPlayback(short* pEchoFrame)
{
if (!m_bHasInit)
return;
speex_echo_playback(m_pState, pEchoFrame);
}
