static OSStatus writeRenderProc(void *inRefCon,
AudioUnitRenderActionFlags *inActionFlags,
const AudioTimeStamp *inTimeStamp,
UInt32 inBusNumber,
UInt32 inNumFrames,
AudioBufferList *ioData)
{
AUWrite *d=(AUWrite*)inRefCon;
int read;
if (ioData->mNumberBuffers!=1) ms_warning("writeRenderProc: %"UINT32_PRINTF" buffers",ioData->mNumberBuffers);
ms_mutex_lock(&d->common.mutex);
read=ms_bufferizer_read(d->buffer,ioData->mBuffers[0].mData,ioData->mBuffers[0].mDataByteSize);
if (ms_bufferizer_get_avail(d->buffer) >10*inNumFrames*2) {
ms_message("we are late, bufferizer sise is [%i] bytes in framezize is [%"UINT32_PRINTF"] bytes"
,(int)ms_bufferizer_get_avail(d->buffer)
,inNumFrames*2);
ms_bufferizer_flush(d->buffer);
}
ms_mutex_unlock(&d->common.mutex);
if (read==0){
ms_debug("Silence inserted in audio output unit (%"UINT32_PRINTF" bytes)",ioData->mBuffers[0].mDataByteSize);
memset(ioData->mBuffers[0].mData,0,ioData->mBuffers[0].mDataByteSize);
}
return 0;
}
static void enc_process(MSFilter *f){
EncState *s=(EncState*)f->data;
mblk_t *im;
unsigned int unitary_buff_size = sizeof(int16_t)*160;
unsigned int buff_size = unitary_buff_size*s->ptime/20;
int16_t* buff;
int offset;
while((im=ms_queue_get(f->inputs[0]))!=NULL){
ms_bufferizer_put(s->bufferizer,im);
}
while(ms_bufferizer_get_avail(s->bufferizer) >= buff_size) {
mblk_t *om=allocb(33*s->ptime/20,0);
buff = (int16_t *)alloca(buff_size);
ms_bufferizer_read(s->bufferizer,(uint8_t*)buff,buff_size);
for (offset=0;offset<buff_size;offset+=unitary_buff_size) {
gsm_encode(s->state,(gsm_signal*)&buff[offset/sizeof(int16_t)],(gsm_byte*)om->b_wptr);
om->b_wptr+=33;
}
mblk_set_timestamp_info(om,s->ts);
ms_queue_put(f->outputs[0],om);
s->ts+=buff_size/sizeof(int16_t)/*sizeof(buf)/2*/;
}
}
/***
Encodes 8 kHz-sampled narrowband speech at a bit rate of or 16 kbit/s,
uses 5 ms frames.
The encoder receives 10 ms speech => 160 bytes.
***/
static void enc_process (MSFilter *f){
EncState *s=(EncState*)f->data;
struct BV16_Bit_Stream bs;
short *buf= NULL;
mblk_t *inputMessage = NULL, *outputMessage = NULL;
int frame_per_packet=s->ptime/5;
int in_rcvd_bytes = 0;
in_rcvd_bytes = SIGNAL_FRAME_SIZE * frame_per_packet;
buf=(short*)alloca(in_rcvd_bytes);
memset((void*)buf,0, in_rcvd_bytes );
while((inputMessage=ms_queue_get(f->inputs[0]))!=NULL){
ms_bufferizer_put(s->bufferizer,inputMessage);
}
/* process ptimes ms of data : (ptime in ms)/1000->ptime is seconds * 8000(sample rate) * 2(byte per sample) */
while(ms_bufferizer_get_avail(s->bufferizer)>= in_rcvd_bytes){
int bufferIndex;
outputMessage = allocb(BITSTREAM_FRAME_SIZE*frame_per_packet,0); /* output bitStream is 80 bits long * number of samples */
/* process buffer in 5 ms frames but read everything first*/
ms_bufferizer_read(s->bufferizer,(uint8_t*)buf,in_rcvd_bytes);
for (bufferIndex=0; bufferIndex<frame_per_packet; bufferIndex++) {
BV16_Encode(&bs, &s->state, (short*)&buf[bufferIndex*FRSZ]);
BV16_BitPack( (UWord8*)outputMessage->b_wptr, &bs );
outputMessage->b_wptr+=BITSTREAM_FRAME_SIZE;
}
mblk_set_timestamp_info(outputMessage,s->ts);
ms_bufferizer_fill_current_metas(s->bufferizer, outputMessage);
ms_queue_put(f->outputs[0],outputMessage);
s->ts += FRSZ * frame_per_packet;
}
}
static OSStatus au_render_cb (
void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *inTimeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData
) {
ms_debug("render cb");
AUData *d=(AUData*)inRefCon;
if (d->write_started == TRUE) {
ioData->mBuffers[0].mDataByteSize=inNumberFrames*d->bits/8;
ioData->mNumberBuffers=1;
ms_mutex_lock(&d->mutex);
if(ms_bufferizer_get_avail(d->bufferizer) >= inNumberFrames*d->bits/8) {
ms_bufferizer_read(d->bufferizer, ioData->mBuffers[0].mData, inNumberFrames*d->bits/8);
if (ms_bufferizer_get_avail(d->bufferizer) >10*inNumberFrames*d->bits/8) {
ms_debug("we are late, bufferizer sise is %i bytes in framezize is %i bytes",ms_bufferizer_get_avail(d->bufferizer),inNumberFrames*d->bits/8);
ms_bufferizer_flush(d->bufferizer);
}
ms_mutex_unlock(&d->mutex);
} else {
ms_mutex_unlock(&d->mutex);
memset(ioData->mBuffers[0].mData, 0,ioData->mBuffers[0].mDataByteSize);
ms_debug("nothing to write, pushing silences, bufferizer size is %i bytes in framezize is %i bytes mDataByteSize %i"
,ms_bufferizer_get_avail(d->bufferizer)
,inNumberFrames*d->bits/8
,ioData->mBuffers[0].mDataByteSize);
d->n_lost_frame+=inNumberFrames;
}
}
if (!d->is_ringer) { // no need to read in ringer mode
AudioBufferList readAudioBufferList;
readAudioBufferList.mBuffers[0].mDataByteSize=inNumberFrames*d->bits/8;
readAudioBufferList.mNumberBuffers=1;
readAudioBufferList.mBuffers[0].mData=NULL;
readAudioBufferList.mBuffers[0].mNumberChannels=d->nchannels;
AudioUnitElement inputBus = 1;
au_read_cb(d, ioActionFlags, inTimeStamp, inputBus, inNumberFrames, &readAudioBufferList);
}
return 0;
}
static void filter_process ( MSFilter *f ) {
isac_encoder_struct_t* obj = (isac_encoder_struct_t*)f->data;
mblk_t *im;
mblk_t *om=NULL;
u_int8_t* input_buf = NULL;
WebRtc_Word16 ret;
static int out_count = 0;
// get the input data and put it into our buffered input
while( (im = ms_queue_get( f->inputs[0] ) ) != NULL ) {
ms_bufferizer_put( obj->bufferizer, im );
}
// feed the encoder with 160 16bit samples, until it has reached enough data
// to produce a packet
while( ms_bufferizer_get_avail(obj->bufferizer) > ISAC_SAMPLES_PER_ENCODE*2 ){
om = allocb( WebRtcIsacfix_GetNewFrameLen(obj->isac), 0 );
if(!input_buf) input_buf = ms_malloc( ISAC_SAMPLES_PER_ENCODE*2 );
ms_bufferizer_read(obj->bufferizer, input_buf, ISAC_SAMPLES_PER_ENCODE*2);
ret = WebRtcIsacfix_Encode(obj->isac,
(const WebRtc_Word16*)input_buf,
(WebRtc_Word16*)om->b_wptr);
if( ret < 0) {
ms_error( "WebRtcIsacfix_Encode error: %d", WebRtcIsacfix_GetErrorCode(obj->isac) );
freeb(om);
} else if( ret == 0 ) {
// Encode() buffered the input, not yet able to produce a packet, continue feeding it
// 160 samples per-call
obj->ts += ISAC_SAMPLES_PER_ENCODE;
freeb(om);
} else {
// a new packet has been encoded, send it
obj->ts += ISAC_SAMPLES_PER_ENCODE;
om->b_wptr += ret;
out_count++;
// ms_message("packet %d out, samples %d", out_count, obj->ts);
mblk_set_timestamp_info( om, obj->ts );
ms_queue_put(f->outputs[0], om);
om = NULL;
}
}
if( input_buf ){
ms_free(input_buf);
}
}
static void volume_process(MSFilter *f){
mblk_t *m;
Volume *v=(Volume*)f->data;
float target_gain;
/* Important notice: any processes called herein can modify v->target_gain, at
* end of this function apply_gain() is called, thus: later process calls can
* override this target gain, and order must be well thought out
*/
if (v->agc_enabled || v->peer!=NULL){
mblk_t *om;
int nbytes=v->nsamples*2;
ms_bufferizer_put_from_queue(v->buffer,f->inputs[0]);
while(ms_bufferizer_get_avail(v->buffer)>=nbytes){
om=allocb(nbytes,0);
ms_bufferizer_read(v->buffer,om->b_wptr,nbytes);
om->b_wptr+=nbytes;
update_energy((int16_t*)om->b_rptr, v->nsamples, v);
target_gain = v->static_gain;
if (v->peer) /* this ptr set = echo limiter enable flag */
target_gain = volume_echo_avoider_process(v, om);
/* Multiply with gain from echo limiter, not "choose smallest". Why?
* Remote talks, local echo suppress via mic path, but still audible in
* remote speaker. AGC operates fully, too (local speaker close to local mic!);
* having agc gain reduction also contribute to total reduction makes sense.
*/
if (v->agc_enabled) target_gain/= volume_agc_process(v, om);
if (v->noise_gate_enabled)
volume_noise_gate_process(v, v->level_pk, om);
apply_gain(v, om, target_gain);
ms_queue_put(f->outputs[0],om);
}
}else{
/*light processing: no agc. Work in place in the input buffer*/
while((m=ms_queue_get(f->inputs[0]))!=NULL){
update_energy((int16_t*)m->b_rptr, (m->b_wptr - m->b_rptr) / 2, v);
target_gain = v->static_gain;
if (v->noise_gate_enabled)
volume_noise_gate_process(v, v->level_pk, m);
apply_gain(v, m, target_gain);
ms_queue_put(f->outputs[0],m);
}
}
}
void alsa_read_process(MSFilter *obj){
AlsaReadData *ad=(AlsaReadData*)obj->data;
mblk_t *om=NULL;
int samples=(160*ad->rate)/8000;
ms_mutex_lock(&ad->mutex);
while (ms_bufferizer_get_avail(ad->bufferizer)>=samples*2){
om=allocb(samples*2,0);
ms_bufferizer_read(ad->bufferizer,om->b_wptr,samples*2);
om->b_wptr+=samples*2;
/*ms_message("alsa_read_process: Outputing %i bytes",size);*/
ms_queue_put(obj->outputs[0],om);
}
ms_mutex_unlock(&ad->mutex);
}
static bool_t should_process(MSFilter *f, ConfState *s){
Channel *chan;
int active_channel=0;
int active_buffer=0;
int i;
/* count active channel */
for (i=0;i<CONF_MAX_PINS;++i){
chan=&s->channels[i];
if (chan->is_used == TRUE)
{
active_channel++;
}
if (ms_bufferizer_get_avail(&chan->buff)>=(s->conf_gran*CONF_BUFFER_SIZE))
{
active_buffer++;
}
}
if (active_channel< 0) /* disable mix mode when it's not needed */
s->mix_mode = FALSE;
else
{
s->mix_mode = TRUE;
}
if (s->enable_directmode==FALSE)
{
s->mix_mode = TRUE;
}
if (s->mix_mode == FALSE)
return FALSE;
if (active_buffer>0)
{
return TRUE;
}
return FALSE;
}
static void enc_process(MSFilter *f) {
EncState *s = (EncState*) f->data;
unsigned int unitary_buff_size = sizeof (int16_t)*160;
unsigned int buff_size = unitary_buff_size * s->ptime / 20;
mblk_t *im;
uint8_t tmp[OUT_MAX_SIZE];
int16_t samples[buff_size];
while ((im = ms_queue_get(f->inputs[0])) != NULL) {
ms_bufferizer_put(s->mb, im);
}
while (ms_bufferizer_get_avail(s->mb) >= buff_size) {
mblk_t *om = allocb(OUT_MAX_SIZE * buff_size / unitary_buff_size + 1, 0);
ms_bufferizer_read(s->mb, (uint8_t*) samples, buff_size);
*om->b_wptr = 0xf0;
uint8_t *tocs = om->b_wptr++;
om->b_wptr += buff_size / unitary_buff_size;
int offset;
for (offset = 0; offset < buff_size; offset += unitary_buff_size) {
int ret = Encoder_Interface_Encode(s->enc, s->mode, &samples[offset / sizeof (int16_t)], tmp, s->dtx);
if (ret <= 0 || ret > 32) {
ms_warning("Encoder returned %i", ret);
freemsg(om);
continue;
}
memcpy(om->b_wptr, &tmp[1], ret - 1);
om->b_wptr += ret - 1;
*(++tocs) = tmp[0] | 0x80; // Not last payload
}
*tocs &= 0x7F; // last payload
mblk_set_timestamp_info(om, s->ts);
ms_queue_put(f->outputs[0], om);
s->ts += buff_size / sizeof (int16_t)/*sizeof(buf)/2*/;
}
}
static void sound_read_process(MSFilter *f){
msandroid_sound_read_data *d=(msandroid_sound_read_data*)f->data;
int nbytes=d->framesize*d->nchannels*2;
int avail;
bool_t flush=FALSE;
bool_t can_output=(d->start_time==-1 || ((f->ticker->time-d->start_time)%d->outgran_ms==0));
ms_mutex_lock(&d->mutex);
if (!d->started) {
ms_mutex_unlock(&d->mutex);
return;
}
avail=ms_bufferizer_get_avail(&d->rb);
if (f->ticker->time % 5000==0){
if (d->min_avail>=(sndread_flush_threshold*(float)d->rate*2.0*(float)d->nchannels)){
int excess_ms=(d->min_avail*1000)/(d->rate*2*d->nchannels);
ms_warning("Excess of audio samples in capture side bytes=%i (%i ms)",d->min_avail,excess_ms);
can_output=TRUE;
flush=TRUE;
}
d->min_avail=-1;
}
do{
if (can_output && (avail>=nbytes*2)){//bytes*2 is to insure smooth output, we leave at least one packet in the buffer for next time*/
mblk_t *om=allocb(nbytes,0);
ms_bufferizer_read(&d->rb,om->b_wptr,nbytes);
om->b_wptr+=nbytes;
ms_queue_put(f->outputs[0],om);
//ms_message("Out time=%llu ",f->ticker->time);
if (d->start_time==-1) d->start_time=f->ticker->time;
avail-=nbytes;
}else break;
}while(flush);
ms_mutex_unlock(&d->mutex);
if (d->min_avail==-1 || avail<d->min_avail) d->min_avail=avail;
}
static void ms_opus_enc_process(MSFilter *f) {
OpusEncData *d = (OpusEncData *)f->data;
mblk_t *im;
mblk_t *om = NULL;
int i;
int frameNumber, packet_size;
uint8_t *signalFrameBuffer = NULL;
uint8_t *codedFrameBuffer[MAX_INPUT_FRAMES];
OpusRepacketizer *rp = opus_repacketizer_create();
opus_int32 ret = 0;
opus_int32 totalLength = 0;
int frame_size = d->samplerate * FRAME_LENGTH / 1000; /* in samples */
// lock the access while getting ptime
ms_filter_lock(f);
frameNumber = d->ptime/FRAME_LENGTH; /* encode 20ms frames, ptime is a multiple of 20ms */
packet_size = d->samplerate * d->ptime / 1000; /* in samples */
ms_filter_unlock(f);
while ((im = ms_queue_get(f->inputs[0])) != NULL) {
ms_bufferizer_put(d->bufferizer, im);
}
for (i=0; i<MAX_INPUT_FRAMES; i++) {
codedFrameBuffer[i]=NULL;
}
while (ms_bufferizer_get_avail(d->bufferizer) >= (d->channels * packet_size * SIGNAL_SAMPLE_SIZE)) {
totalLength = 0;
opus_repacketizer_init(rp);
for (i=0; i<frameNumber; i++) { /* encode 20ms by 20ms and repacketize all of them together */
if (!codedFrameBuffer[i]) codedFrameBuffer[i] = ms_malloc(MAX_BYTES_PER_FRAME); /* the repacketizer need the pointer to packet to remain valid, so we shall have a buffer for each coded frame */
if (!signalFrameBuffer) signalFrameBuffer = ms_malloc(frame_size * SIGNAL_SAMPLE_SIZE * d->channels);
ms_bufferizer_read(d->bufferizer, signalFrameBuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels);
ret = opus_encode(d->state, (opus_int16 *)signalFrameBuffer, frame_size, codedFrameBuffer[i], MAX_BYTES_PER_FRAME);
if (ret < 0) {
ms_error("Opus encoder error: %s", opus_strerror(ret));
break;
}
if (ret > 0) {
int err = opus_repacketizer_cat(rp, codedFrameBuffer[i], ret); /* add the encoded frame into the current packet */
if (err != OPUS_OK) {
ms_error("Opus repacketizer error: %s", opus_strerror(err));
break;
}
totalLength += ret;
}
}
if (ret > 0) {
om = allocb(totalLength+frameNumber + 1, 0); /* opus repacktizer API: allocate at leat number of frame + size of all data added before */
ret = opus_repacketizer_out(rp, om->b_wptr, totalLength+frameNumber);
om->b_wptr += ret;
mblk_set_timestamp_info(om, d->ts);
ms_queue_put(f->outputs[0], om);
d->ts += packet_size*48000/d->samplerate; /* RFC payload RTP opus 03 - section 4: RTP timestamp multiplier : WARNING works only with sr at 48000 */
ret = 0;
}
}
opus_repacketizer_destroy(rp);
if (signalFrameBuffer != NULL) {
ms_free(signalFrameBuffer);
}
for (i=0; i<frameNumber; i++) {
if (codedFrameBuffer[i] != NULL) {
ms_free(codedFrameBuffer[i]);
}
}
}
/* inputs[0]= reference signal from far end (sent to soundcard)
* inputs[1]= near speech & echo signal (read from soundcard)
* outputs[0]= is a copy of inputs[0] to be sent to soundcard
* outputs[1]= near end speech, echo removed - towards far end
*/
static void speex_ec_process(MSFilter *f){
SpeexECState *s=(SpeexECState*)f->data;
int nbytes=s->framesize*2;
mblk_t *refm;
uint8_t *ref,*echo;
if (s->bypass_mode) {
while((refm=ms_queue_get(f->inputs[0]))!=NULL){
ms_queue_put(f->outputs[0],refm);
}
while((refm=ms_queue_get(f->inputs[1]))!=NULL){
ms_queue_put(f->outputs[1],refm);
}
return;
}
if (f->inputs[0]!=NULL){
if (s->echostarted){
while((refm=ms_queue_get(f->inputs[0]))!=NULL){
refm=audio_flow_controller_process(&s->afc,refm);
if (refm){
mblk_t *cp=dupmsg(refm);
ms_bufferizer_put(&s->delayed_ref,cp);
ms_bufferizer_put(&s->ref,refm);
}
}
}else{
ms_warning("Getting reference signal but no echo to synchronize on.");
ms_queue_flush(f->inputs[0]);
}
}
ms_bufferizer_put_from_queue(&s->echo,f->inputs[1]);
ref=(uint8_t*)alloca(nbytes);
echo=(uint8_t*)alloca(nbytes);
while (ms_bufferizer_read(&s->echo,echo,nbytes)==nbytes){
mblk_t *oecho=allocb(nbytes,0);
int avail;
int avail_samples;
if (!s->echostarted) s->echostarted=TRUE;
if ((avail=ms_bufferizer_get_avail(&s->delayed_ref))<((s->nominal_ref_samples*2)+nbytes)){
/*we don't have enough to read in a reference signal buffer, inject silence instead*/
avail=nbytes;
refm=allocb(nbytes,0);
memset(refm->b_wptr,0,nbytes);
refm->b_wptr+=nbytes;
ms_bufferizer_put(&s->delayed_ref,refm);
ms_queue_put(f->outputs[0],dupmsg(refm));
if (!s->using_zeroes){
ms_warning("Not enough ref samples, using zeroes");
s->using_zeroes=TRUE;
}
}else{
if (s->using_zeroes){
ms_message("Samples are back.");
s->using_zeroes=FALSE;
}
/* read from our no-delay buffer and output */
refm=allocb(nbytes,0);
if (ms_bufferizer_read(&s->ref,refm->b_wptr,nbytes)==0){
ms_fatal("Should never happen");
}
refm->b_wptr+=nbytes;
ms_queue_put(f->outputs[0],refm);
}
/*now read a valid buffer of delayed ref samples*/
if (ms_bufferizer_read(&s->delayed_ref,ref,nbytes)==0){
ms_fatal("Should never happen");
}
avail-=nbytes;
avail_samples=avail/2;
/*ms_message("avail=%i",avail_samples);*/
if (avail_samples<s->min_ref_samples || s->min_ref_samples==-1){
s->min_ref_samples=avail_samples;
}
#ifdef EC_DUMP
if (s->reffile)
fwrite(ref,nbytes,1,s->reffile);
if (s->echofile)
fwrite(echo,nbytes,1,s->echofile);
#endif
speex_echo_cancellation(s->ecstate,(short*)echo,(short*)ref,(short*)oecho->b_wptr);
speex_preprocess_run(s->den, (short*)oecho->b_wptr);
#ifdef EC_DUMP
if (s->cleanfile)
fwrite(oecho->b_wptr,nbytes,1,s->cleanfile);
#endif
oecho->b_wptr+=nbytes;
ms_queue_put(f->outputs[1],oecho);
}
/*verify our ref buffer does not become too big, meaning that we are receiving more samples than we are sending*/
if ((((uint32_t)(f->ticker->time - s->flow_control_time)) >= flow_control_interval_ms) && (s->min_ref_samples != -1)) {
int diff=s->min_ref_samples-s->nominal_ref_samples;
if (diff>(nbytes/2)){
int purge=diff-(nbytes/2);
ms_warning("echo canceller: we are accumulating too much reference signal, need to throw out %i samples",purge);
audio_flow_controller_set_target(&s->afc,purge,(flow_control_interval_ms*s->samplerate)/1000);
}
s->min_ref_samples=-1;
s->flow_control_time = f->ticker->time;
}
}
static void ms_opus_enc_process(MSFilter *f) {
OpusEncData *d = (OpusEncData *)f->data;
OpusRepacketizer *repacketizer = NULL;
mblk_t *om = NULL;
int packet_size, pcm_buffer_size;
int max_frame_byte_size, ptime = 20;
int frame_count = 0, frame_size = 0;
opus_int32 total_length = 0;
uint8_t *repacketizer_frame_buffer[MAX_INPUT_FRAMES];
int i;
ms_filter_lock(f);
ptime = d->ptime;
packet_size = d->samplerate * ptime / 1000; /* in samples */
ms_filter_unlock(f);
switch (ptime) {
case 10:
frame_size = d->samplerate * 10 / 1000;
frame_count = 1;
break;
case 20:
frame_size = d->samplerate * 20 / 1000;
frame_count = 1;
break;
case 40:
frame_size = d->samplerate * 40 / 1000;
frame_count = 1;
break;
case 60:
frame_size = d->samplerate * 60 / 1000;
frame_count = 1;
break;
case 80:
frame_size = d->samplerate * 40 / 1000;
frame_count = 2;
break;
case 100:
frame_size = d->samplerate * 20 / 1000;
frame_count = 5;
break;
case 120:
frame_size = d->samplerate * 60 / 1000;
frame_count = 2;
break;
default:
frame_size = d->samplerate * 20 / 1000;
frame_count = 1;
}
max_frame_byte_size = MAX_BYTES_PER_MS * ptime/frame_count;
pcm_buffer_size = d->channels * frame_size * SIGNAL_SAMPLE_SIZE;
if (pcm_buffer_size > d->pcmbufsize){
if (d->pcmbuffer) ms_free(d->pcmbuffer);
d->pcmbuffer = ms_malloc(pcm_buffer_size);
d->pcmbufsize = pcm_buffer_size;
}
for (i=0; i<MAX_INPUT_FRAMES; i++) {
repacketizer_frame_buffer[i]=NULL;
}
ms_bufferizer_put_from_queue(d->bufferizer, f->inputs[0]);
while (ms_bufferizer_get_avail(d->bufferizer) >= (d->channels * packet_size * SIGNAL_SAMPLE_SIZE)) {
opus_int32 ret = 0;
if (frame_count == 1) { /* One Opus frame, not using the repacketizer */
om = allocb(max_frame_byte_size, 0);
ms_bufferizer_read(d->bufferizer, d->pcmbuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels);
ret = opus_encode(d->state, (opus_int16 *)d->pcmbuffer, frame_size, om->b_wptr, max_frame_byte_size);
if (ret < 0) {
freemsg(om);
om=NULL;
ms_error("Opus encoder error: %s", opus_strerror(ret));
break;
} else {
total_length = ret;
om->b_wptr += total_length;
}
} else if(frame_count > 1) { /* We have multiple Opus frames we will use the opus repacketizer */
repacketizer = opus_repacketizer_create();
opus_repacketizer_init(repacketizer);
/* Do not include FEC/LBRR in any frame after the first one since it will be sent with the previous one */
ret = opus_encoder_ctl(d->state, OPUS_SET_INBAND_FEC(0));
if (ret != OPUS_OK) {
ms_error("could not set inband FEC to opus encoder: %s", opus_strerror(ret));
}
for (i=0; i<frame_count; i++) {
if(frame_count == i+1){ /* if configured, reactivate FEC on the last frame to tell the encoder he should restart saving LBRR frames */
ret = opus_encoder_ctl(d->state, OPUS_SET_INBAND_FEC(d->useinbandfec));
if (ret != OPUS_OK) {
ms_error("could not set inband FEC to opus encoder: %s", opus_strerror(ret));
}
}
if (!repacketizer_frame_buffer[i]) repacketizer_frame_buffer[i] = ms_malloc(max_frame_byte_size); /* the repacketizer need the pointer to packet to remain valid, so we shall have a buffer for each coded frame */
ms_bufferizer_read(d->bufferizer, d->pcmbuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels);
ret = opus_encode(d->state, (opus_int16 *)d->pcmbuffer, frame_size, repacketizer_frame_buffer[i], max_frame_byte_size);
if (ret < 0) {
ms_error("Opus encoder error: %s", opus_strerror(ret));
break;
} else if (ret > 0) {
int err = opus_repacketizer_cat(repacketizer, repacketizer_frame_buffer[i], ret); /* add the encoded frame into the current packet */
if (err != OPUS_OK) {
ms_error("Opus repacketizer error: %s", opus_strerror(err));
break;
}
total_length += ret;
}
}
om = allocb(total_length + frame_count + 1, 0); /* opus repacketizer API: allocate at least number of frame + size of all data added before */
ret = opus_repacketizer_out(repacketizer, om->b_wptr, total_length+frame_count);
if(ret < 0){
freemsg(om);
om=NULL;
ms_error("Opus repacketizer out error: %s", opus_strerror(ret));
} else {
om->b_wptr += ret;
}
opus_repacketizer_destroy(repacketizer);
for (i=0; i<frame_count; i++) {
if (repacketizer_frame_buffer[i] != NULL) {
ms_free(repacketizer_frame_buffer[i]);
}
}
}
if(om) { /* we have an encoded output message */
mblk_set_timestamp_info(om, d->ts);
ms_bufferizer_fill_current_metas(d->bufferizer, om);
ms_queue_put(f->outputs[0], om);
d->ts += packet_size*48000/d->samplerate; /* RFC payload RTP opus 03 - section 4: RTP timestamp multiplier : WARNING works only with sr at 48000 */
total_length = 0;
}
}
}
static void conf_sum(MSFilter *f, ConfState *s){
int i,j;
Channel *chan;
memset(s->sum,0,s->conf_nsamples*sizeof(int));
for (i=0;i<CONF_MAX_PINS;++i){
chan=&s->channels[i];
/* skip soundread and short buffer entry */
if (ms_bufferizer_get_avail(&chan->buff) > (s->conf_gran*CONF_BUFFER_SIZE) )
{
#if 0
int loudness;
#endif
while (ms_bufferizer_get_avail(&chan->buff)> s->conf_gran)
{
ms_bufferizer_read(&chan->buff,(uint8_t*)chan->input,s->conf_gran);
#if 0
speex_preprocess_ctl(chan->speex_pp, SPEEX_PREPROCESS_GET_AGC_LOUDNESS, &loudness);
#endif
/* we want to remove 4 packets (40ms) in a near future: */
#ifndef DISABLE_SPEEX
if (chan->speex_pp!=NULL && s->enable_vad==TRUE)
{
int vad=0;
vad = speex_preprocess(chan->speex_pp, (short*)chan->input, NULL);
if (vad==1)
break; /* voice detected: process as usual */
if (ms_bufferizer_get_avail(&chan->buff)<s->conf_gran)
break; /* no more data to remove */
ms_message("No voice detected: discarding sample. (idx=%i - bufsize=%i sncardbufsize=%i)",
i, ms_bufferizer_get_avail(&chan->buff), ms_bufferizer_get_avail(&chan->buff));
}
if (ms_bufferizer_get_avail(&chan->buff) == (ms_bufferizer_get_avail(&chan->buff)))
ms_message("same data in soundcard and incoming rtp. (idx=%i - bufsize=%i sncardbufsize=%i)",
i, ms_bufferizer_get_avail(&chan->buff), ms_bufferizer_get_avail(&chan->buff));
#endif
chan->stat_discarded++;
}
for(j=0;j<s->conf_nsamples;++j){
s->sum[j]+=chan->input[j];
}
chan->has_contributed=TRUE;
chan->stat_processed++;
}
else if (ms_bufferizer_get_avail(&chan->buff)>=s->conf_gran)
{
struct channel_volume {
float energy;
int channel;
};
struct channel_volume vol;
float en;
ms_bufferizer_read(&chan->buff,(uint8_t*)chan->input,s->conf_gran);
en=chan->energy;
for(j=0;j<s->conf_nsamples;++j){
float s=chan->input[j];
en=(s*s*coef) + ((float)1.0-coef)*en;
}
chan->energy=en;
vol.energy = chan->energy; //10*log10f(chan->energy/max_e);
vol.channel = i;
ms_filter_notify(f, MS_CONF_CHANNEL_VOLUME, (void*)&vol);
{
if (chan->energy>65)
chan->count_speaking++;
else
chan->count_speaking=0;
}
#ifndef DISABLE_SPEEX
if (chan->speex_pp!=NULL && s->enable_vad==TRUE && i==0)
{
int vad;
vad = speex_preprocess(chan->speex_pp, (short*)chan->input, NULL);
ms_filter_notify(f, MS_CONF_SPEEX_PREPROCESS_MIC, (void*)chan->speex_pp);
if (s->enable_halfduplex>0)
{
powerspectrum_stat_beyond8K(chan);
if (chan->average_psd>s->vad_prob_start)
{
if (chan->is_speaking<0)
ms_message("MIC is turned on");
//ms_message("is_speaking (chan=%i) -> on/stat=%.3lf", i, chan->average_psd);
chan->is_speaking=20; /* keep RTP unmuted for the next few ms */
}
else if (chan->average_psd<s->vad_prob_continue)
{
if (chan->is_speaking==0)
{
ms_message("MIC is turned off");
chan->count_speaking=0;
}
chan->is_speaking--;
}
if (chan->is_speaking>0)
{
chan->count_speaking++;
}
}
}
else if (chan->speex_pp!=NULL && s->enable_vad==TRUE)
{
int vad;
if (s->enable_halfduplex>0)
{
vad = speex_preprocess(chan->speex_pp, (short*)chan->input, NULL);
}
else
{
vad = speex_preprocess(chan->speex_pp, (short*)chan->input, NULL);
//speex_preprocess_estimate_update(chan->speex_pp, (short*)chan->input);
}
}
#endif
for(j=0;j<s->conf_nsamples;++j){
s->sum[j]+=chan->input[j];
}
chan->has_contributed=TRUE;
chan->stat_processed++;
} else {
chan->stat_missed++;
if (i>0 && chan->is_used == TRUE)
{
chan->missed++;
/* delete stream if data is missing since a long time */
if (chan->missed>15)
{
chan->is_used=FALSE;
ms_message("msconf: deleted contributing stream (pin=%i)", i);
}
/* couldn't we add confort noise for those outputs? */
}
chan->has_contributed=FALSE;
}
}
return;
}
static void conf_process(MSFilter *f){
int i;
ConfState *s=(ConfState*)f->data;
Channel *chan;
Channel *chan0;
ms_mutex_lock(&s->lock);
/*read from all inputs and put into bufferizers*/
for (i=0;i<CONF_MAX_PINS;++i){
if (f->inputs[i]!=NULL){
chan=&s->channels[i];
ms_bufferizer_put_from_queue(&chan->buff,f->inputs[i]);
if (ms_bufferizer_get_avail(&chan->buff)>0)
{
chan->missed=0; /* reset counter of missed packet */
if (/*i>0 && */chan->is_used==FALSE)
{
chan->is_used=TRUE;
ms_message("msconf: new contributing stream (chan=%i) %i", ms_bufferizer_get_avail(&chan->buff), i);
}
}
}
}
/*do the job */
while(should_process(f,s)==TRUE){
conf_sum(f, s);
conf_dispatch(f,s);
}
#if 0
/* mixer is disabled! -> copy A->B and B->A*/
if (s->mix_mode == FALSE)
{
/* get the soundread data and copy it to pinX */
for (i=1;i<CONF_MAX_PINS;i=i+2){
if (f->inputs[i]!=NULL){
chan0=&s->channels[0];
chan=&s->channels[i];
if (chan->is_used==TRUE)
{
while (ms_bufferizer_read(&chan->buff,(uint8_t*)chan->input,s->conf_gran)==s->conf_gran)
{
if (f->outputs[0]!=NULL)
{
/* send in pin0 */
mblk_t *m=allocb(s->conf_gran,0);
memcpy(m->b_wptr, chan->input, s->conf_gran);
m->b_wptr+=s->conf_gran;
ms_queue_put(f->outputs[0],m);
}
}
}
if (chan0->is_used==TRUE)
{
while (ms_bufferizer_read(&chan0->buff,(uint8_t*)chan0->input,s->conf_gran)==s->conf_gran)
{
if (f->outputs[i]!=NULL)
{
/* send in pinI */
mblk_t *m=allocb(s->conf_gran,0);
memcpy(m->b_wptr, chan0->input, s->conf_gran);
m->b_wptr+=s->conf_gran;
ms_queue_put(f->outputs[i],m);
}
}
}
break;
}
}
}
#endif // 0
ms_mutex_unlock(&s->lock);
}
static void* msandroid_write_cb(msandroid_sound_write_data* d) {
jbyteArray write_buff;
jmethodID write_id=0;
jmethodID play_id=0;
int min_size=-1;
int count;
int max_size=sndwrite_flush_threshold*(float)d->rate*(float)d->nchannels*2.0;
int check_point_size=3*(float)d->rate*(float)d->nchannels*2.0; /*3 seconds*/
int nwrites=0;
set_high_prio();
int buff_size = d->write_chunk_size;
JNIEnv *jni_env = ms_get_jni_env();
// int write (byte[] audioData, int offsetInBytes, int sizeInBytes)
write_id = jni_env->GetMethodID(d->audio_track_class,"write", "([BII)I");
if(write_id==0) {
ms_error("cannot find AudioTrack.write() method");
goto end;
}
play_id = jni_env->GetMethodID(d->audio_track_class,"play", "()V");
if(play_id==0) {
ms_error("cannot find AudioTrack.play() method");
goto end;
}
write_buff = jni_env->NewByteArray(buff_size);
uint8_t tmpBuff[buff_size];
//start playing
jni_env->CallVoidMethod(d->audio_track,play_id);
ms_mutex_lock(&d->mutex);
ms_bufferizer_flush(d->bufferizer);
ms_mutex_unlock(&d->mutex);
while(d->started) {
int bufferizer_size;
ms_mutex_lock(&d->mutex);
min_size=-1;
count=0;
while((bufferizer_size = ms_bufferizer_get_avail(d->bufferizer)) >= d->write_chunk_size) {
if (min_size==-1) min_size=bufferizer_size;
else if (bufferizer_size<min_size) min_size=bufferizer_size;
ms_bufferizer_read(d->bufferizer, tmpBuff, d->write_chunk_size);
ms_mutex_unlock(&d->mutex);
jni_env->SetByteArrayRegion(write_buff,0,d->write_chunk_size,(jbyte*)tmpBuff);
int result = jni_env->CallIntMethod(d->audio_track,write_id,write_buff,0,d->write_chunk_size);
d->writtenBytes+=result;
if (result <= 0) {
ms_error("write operation has failed [%i]",result);
}
nwrites++;
ms_mutex_lock(&d->mutex);
count+=d->write_chunk_size;
if (count>check_point_size){
if (min_size > max_size) {
ms_warning("we are late, flushing %i bytes",min_size);
ms_bufferizer_skip_bytes(d->bufferizer,min_size);
}
count=0;
}
}
if (d->started) {
d->sleeping=true;
ms_cond_wait(&d->cond,&d->mutex);
d->sleeping=false;
}
ms_mutex_unlock(&d->mutex);
}
goto end;
end: {
ms_thread_exit(NULL);
return NULL;
}
}
static void * oss_thread(void *p){
MSSndCard *card=(MSSndCard*)p;
OssData *d=(OssData*)card->data;
int bsize=0;
uint8_t *rtmpbuff=NULL;
uint8_t *wtmpbuff=NULL;
int err;
mblk_t *rm=NULL;
d->pcmfd=oss_open(d->pcmdev,d->bits,d->stereo,d->rate,&bsize);
if (d->pcmfd>=0){
rtmpbuff=(uint8_t*)malloc(bsize);
wtmpbuff=(uint8_t*)malloc(bsize);
if(rtmpbuff == NULL || wtmpbuff == NULL) {
free(rtmpbuff);
free(wtmpbuff);
return NULL;
}
}
while(d->read_started || d->write_started){
if (d->pcmfd>=0){
if (d->read_started){
struct timeval timeout;
fd_set read_fds;
audio_buf_info info;
if (rm==NULL) rm=allocb(bsize,0);
timeout.tv_sec = 0;
timeout.tv_usec = 0;
FD_ZERO( &read_fds );
FD_SET( d->pcmfd, &read_fds );
if( select( d->pcmfd + 1, &read_fds, NULL, NULL, &timeout ) == -1 ) {
}
if (FD_ISSET( d->pcmfd, &read_fds ) && ioctl( d->pcmfd, SNDCTL_DSP_GETISPACE, &info ) != -1)
{
if (info.bytes>=bsize)
{
err=read(d->pcmfd,rm->b_wptr,bsize);
if (err<0){
ms_warning("Fail to read %i bytes from soundcard: %s",
bsize,strerror(errno));
}else{
rm->b_wptr+=err;
ms_mutex_lock(&d->mutex);
putq(&d->rq,rm);
ms_mutex_unlock(&d->mutex);
rm=NULL;
}
}
else
{
timeout.tv_sec = 0;
timeout.tv_usec = 5000;
select(0, 0, NULL, NULL, &timeout );
}
}
else
{
timeout.tv_sec = 0;
timeout.tv_usec = 5000;
select(0, 0, NULL, NULL, &timeout );
}
}else {
int sz = read(d->pcmfd,rtmpbuff,bsize);
if( sz!=bsize) ms_warning("sound device read returned %i !",sz);
}
if (d->write_started){
audio_buf_info info;
if( ms_bufferizer_get_avail(d->bufferizer)>=bsize && ioctl( d->pcmfd, SNDCTL_DSP_GETOSPACE, &info ) == 0 ) {
if( info.fragstotal - info.fragments > 15 ) {
static int c=0;
/* drop the fragment if the buffer starts to fill up */
/* we got too much data: I prefer to empty the incoming buffer */
while (ms_bufferizer_get_avail(d->bufferizer)>bsize*4){
ms_mutex_lock(&d->mutex);
err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize);
err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize);
err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize);
err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize);
ms_mutex_unlock(&d->mutex);
c=c+err*4;
ms_warning("drop fragment when buffer gets too much data (%i - discarded:%i)", info.fragstotal - info.fragments, c);
if (err==0)
break;
}
}else {
ms_mutex_lock(&d->mutex);
err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize);
ms_mutex_unlock(&d->mutex);
err=write(d->pcmfd,wtmpbuff,bsize);
if (err<0){
ms_warning("Fail to write %i bytes from soundcard: %s",
bsize,strerror(errno));
}
}
}
}else {
int sz;
memset(wtmpbuff,0,bsize);
sz = write(d->pcmfd,wtmpbuff,bsize);
if( sz!=bsize) ms_warning("sound device write returned %i !",sz);
}
}else usleep(20000);
}
if (d->pcmfd>=0) {
close(d->pcmfd);
d->pcmfd=-1;
}
free(rtmpbuff);
free(wtmpbuff);
if (rm!=NULL) freemsg(rm);
/*reset to default parameters */
//d->bits=16;
//d->rate=8000;
//d->stereo=FALSE;
return NULL;
}