static void writeCallback(void *aqData, AudioQueueRef inAQ, AudioQueueBufferRef inBuffer) { AQData *d = (AQData *) aqData; OSStatus err; int len = (d->writeBufferByteSize * d->writeAudioFormat.mSampleRate / 1) / d->devicewriteFormat.mSampleRate / d->devicewriteFormat.mChannelsPerFrame; ms_mutex_lock(&d->mutex); if (d->write_started == FALSE) { ms_mutex_unlock(&d->mutex); return; } if (d->bufferizer->size >= len) { #if 0 UInt32 bsize = d->writeBufferByteSize; uint8_t *pData = ms_malloc(len); ms_bufferizer_read(d->bufferizer, pData, len); err = AudioConverterConvertBuffer(d->writeAudioConverter, len, pData, &bsize, inBuffer->mAudioData); if (err != noErr) { ms_error("writeCallback: AudioConverterConvertBuffer %d", err); } ms_free(pData); if (bsize != d->writeBufferByteSize) ms_warning("d->writeBufferByteSize = %i len = %i bsize = %i", d->writeBufferByteSize, len, bsize); #else ms_bufferizer_read(d->bufferizer, inBuffer->mAudioData, len); #endif } else { memset(inBuffer->mAudioData, 0, d->writeBufferByteSize); } inBuffer->mAudioDataByteSize = d->writeBufferByteSize; if (gain_changed_out == true) { AudioQueueSetParameter (d->writeQueue, kAudioQueueParam_Volume, gain_volume_out); gain_changed_out = false; } err = AudioQueueEnqueueBuffer(d->writeQueue, inBuffer, 0, NULL); if (err != noErr) { ms_error("AudioQueueEnqueueBuffer %d", err); } ms_mutex_unlock(&d->mutex); }
static void aq_put(MSFilter * f, mblk_t * m) { AQData *d = (AQData *) f->data; ms_mutex_lock(&d->mutex); ms_bufferizer_put(d->bufferizer, m); ms_mutex_unlock(&d->mutex); int len = (d->writeBufferByteSize * d->writeAudioFormat.mSampleRate / 1) / d->devicewriteFormat.mSampleRate / d->devicewriteFormat.mChannelsPerFrame; if (d->write_started == FALSE && d->bufferizer->size >= len) { AudioQueueBufferRef curbuf = d->writeBuffers[d->curWriteBuffer]; #if 0 OSStatus err; UInt32 bsize = d->writeBufferByteSize; uint8_t *pData = ms_malloc(len); ms_bufferizer_read(d->bufferizer, pData, len); err = AudioConverterConvertBuffer(d->writeAudioConverter, len, pData, &bsize, curbuf->mAudioData); if (err != noErr) { ms_error("writeCallback: AudioConverterConvertBuffer %d", err); } ms_free(pData); if (bsize != d->writeBufferByteSize) ms_warning("d->writeBufferByteSize = %i len = %i bsize = %i", d->writeBufferByteSize, len, bsize); #else ms_bufferizer_read(d->bufferizer, curbuf->mAudioData, len); #endif curbuf->mAudioDataByteSize = d->writeBufferByteSize; putWriteAQ(d, d->curWriteBuffer); ++d->curWriteBuffer; } if (d->write_started == FALSE && d->curWriteBuffer == kNumberAudioOutDataBuffers - 1) { OSStatus err; err = AudioQueueStart(d->writeQueue, NULL // start time. NULL means ASAP. ); if (err != noErr) { ms_error("AudioQueueStart -write- %d", err); } d->write_started = TRUE; } }
static void enc_process(MSFilter *f){ EncState *s=(EncState*)f->data; mblk_t *im; unsigned int unitary_buff_size = sizeof(int16_t)*160; unsigned int buff_size = unitary_buff_size*s->ptime/20; int16_t* buff; int offset; while((im=ms_queue_get(f->inputs[0]))!=NULL){ ms_bufferizer_put(s->bufferizer,im); } while(ms_bufferizer_get_avail(s->bufferizer) >= buff_size) { mblk_t *om=allocb(33*s->ptime/20,0); buff = (int16_t *)alloca(buff_size); ms_bufferizer_read(s->bufferizer,(uint8_t*)buff,buff_size); for (offset=0;offset<buff_size;offset+=unitary_buff_size) { gsm_encode(s->state,(gsm_signal*)&buff[offset/sizeof(int16_t)],(gsm_byte*)om->b_wptr); om->b_wptr+=33; } mblk_set_timestamp_info(om,s->ts); ms_queue_put(f->outputs[0],om); s->ts+=buff_size/sizeof(int16_t)/*sizeof(buf)/2*/; } }
static OSStatus writeRenderProc(void *inRefCon, AudioUnitRenderActionFlags *inActionFlags, const AudioTimeStamp *inTimeStamp, UInt32 inBusNumber, UInt32 inNumFrames, AudioBufferList *ioData) { AUWrite *d=(AUWrite*)inRefCon; int read; if (ioData->mNumberBuffers!=1) ms_warning("writeRenderProc: %"UINT32_PRINTF" buffers",ioData->mNumberBuffers); ms_mutex_lock(&d->common.mutex); read=ms_bufferizer_read(d->buffer,ioData->mBuffers[0].mData,ioData->mBuffers[0].mDataByteSize); if (ms_bufferizer_get_avail(d->buffer) >10*inNumFrames*2) { ms_message("we are late, bufferizer sise is [%i] bytes in framezize is [%"UINT32_PRINTF"] bytes" ,(int)ms_bufferizer_get_avail(d->buffer) ,inNumFrames*2); ms_bufferizer_flush(d->buffer); } ms_mutex_unlock(&d->common.mutex); if (read==0){ ms_debug("Silence inserted in audio output unit (%"UINT32_PRINTF" bytes)",ioData->mBuffers[0].mDataByteSize); memset(ioData->mBuffers[0].mData,0,ioData->mBuffers[0].mDataByteSize); } return 0; }
static void detector_process(MSFilter *f) { DetectorState *s=(DetectorState *)f->data; mblk_t *m; while ((m=ms_queue_get(f->inputs[0]))!=NULL) { ms_queue_put(f->outputs[0],m); if (s->tone_def->frequency!=0) { ms_bufferizer_put(s->buf,dupmsg(m)); } } if (s->tone_def->frequency!=0) { uint8_t *buf=_alloca(s->framesize); while(ms_bufferizer_read(s->buf,buf,s->framesize)!=0) { float en=compute_energy((int16_t*)buf,s->framesize/2); if (en>energy_min) { float freq_en=goertzel_state_run(&s->tone_gs,(int16_t*)buf,s->framesize/2,en); if (freq_en>=s->tone_def->min_amplitude) { if (s->dur==0) s->starttime=f->ticker->time; s->dur+=s->frame_ms; if (s->dur>s->tone_def->min_duration && !s->event_sent) { MSToneDetectorEvent event; strncpy(event.tone_name,s->tone_def->tone_name,sizeof(event.tone_name)); event.tone_start_time=s->starttime; ms_filter_notify(f,MS_TONE_DETECTOR_EVENT,&event); s->event_sent=TRUE; } } else end_tone(s); } else end_tone(s); } } }
/*** Encodes 8 kHz-sampled narrowband speech at a bit rate of or 16 kbit/s, uses 5 ms frames. The encoder receives 10 ms speech => 160 bytes. ***/ static void enc_process (MSFilter *f){ EncState *s=(EncState*)f->data; struct BV16_Bit_Stream bs; short *buf= NULL; mblk_t *inputMessage = NULL, *outputMessage = NULL; int frame_per_packet=s->ptime/5; int in_rcvd_bytes = 0; in_rcvd_bytes = SIGNAL_FRAME_SIZE * frame_per_packet; buf=(short*)alloca(in_rcvd_bytes); memset((void*)buf,0, in_rcvd_bytes ); while((inputMessage=ms_queue_get(f->inputs[0]))!=NULL){ ms_bufferizer_put(s->bufferizer,inputMessage); } /* process ptimes ms of data : (ptime in ms)/1000->ptime is seconds * 8000(sample rate) * 2(byte per sample) */ while(ms_bufferizer_get_avail(s->bufferizer)>= in_rcvd_bytes){ int bufferIndex; outputMessage = allocb(BITSTREAM_FRAME_SIZE*frame_per_packet,0); /* output bitStream is 80 bits long * number of samples */ /* process buffer in 5 ms frames but read everything first*/ ms_bufferizer_read(s->bufferizer,(uint8_t*)buf,in_rcvd_bytes); for (bufferIndex=0; bufferIndex<frame_per_packet; bufferIndex++) { BV16_Encode(&bs, &s->state, (short*)&buf[bufferIndex*FRSZ]); BV16_BitPack( (UWord8*)outputMessage->b_wptr, &bs ); outputMessage->b_wptr+=BITSTREAM_FRAME_SIZE; } mblk_set_timestamp_info(outputMessage,s->ts); ms_bufferizer_fill_current_metas(s->bufferizer, outputMessage); ms_queue_put(f->outputs[0],outputMessage); s->ts += FRSZ * frame_per_packet; } }
OSStatus writeACInputProc ( AudioConverterRef inAudioConverter, UInt32 *ioNumberDataPackets, AudioBufferList *ioData, AudioStreamPacketDescription **outDataPacketDescription, void* inUserData) { OSStatus err = noErr; CAData *d=(CAData*)inUserData; UInt32 packetSize = (d->bits / 8) * (d->stereo ? 2 : 1); if(*ioNumberDataPackets) { if(d->caSourceBuffer != NULL) { free(d->caSourceBuffer); d->caSourceBuffer = NULL; } d->caSourceBuffer = (void *) calloc (1, *ioNumberDataPackets * packetSize); ioData->mBuffers[0].mData = d->caSourceBuffer; // tell the Audio Converter where it's source data is ms_mutex_lock(&d->mutex); int readsize = ms_bufferizer_read(d->bufferizer,d->caSourceBuffer,*ioNumberDataPackets * packetSize); ms_mutex_unlock(&d->mutex); if(readsize != *ioNumberDataPackets * packetSize) { memset(d->caSourceBuffer, 0, *ioNumberDataPackets * packetSize); ioData->mBuffers[0].mDataByteSize = *ioNumberDataPackets * packetSize; // tell the Audio Converter how much source data there is } else { ioData->mBuffers[0].mDataByteSize = readsize; // tell the Audio Converter how much source data there is } } return err; }
static void enc_process(MSFilter *f){ static const int nsamples=160; EncState *s=(EncState*)f->data; mblk_t *im,*om; int16_t samples[nsamples]; while((im=ms_queue_get(f->inputs[0]))!=NULL){ ms_bufferizer_put (s->mb,im); } while((ms_bufferizer_read(s->mb,(uint8_t*)samples,nsamples*2))>=nsamples*2){ int ret; om=allocb(33,0); *om->b_wptr=0xf0; om->b_wptr++; ret=Encoder_Interface_Encode(s->enc,MR122,samples,om->b_wptr,0); if (ret<=0){ ms_warning("Encoder returned %i",ret); freemsg(om); continue; } om->b_wptr+=ret; mblk_set_timestamp_info(om,s->ts); s->ts+=nsamples; ms_queue_put(f->outputs[0],om); } }
static void filter_process ( MSFilter *f ) { isac_encoder_struct_t* obj = (isac_encoder_struct_t*)f->data; mblk_t *im; mblk_t *om=NULL; u_int8_t* input_buf = NULL; WebRtc_Word16 ret; static int out_count = 0; // get the input data and put it into our buffered input while( (im = ms_queue_get( f->inputs[0] ) ) != NULL ) { ms_bufferizer_put( obj->bufferizer, im ); } // feed the encoder with 160 16bit samples, until it has reached enough data // to produce a packet while( ms_bufferizer_get_avail(obj->bufferizer) > ISAC_SAMPLES_PER_ENCODE*2 ){ om = allocb( WebRtcIsacfix_GetNewFrameLen(obj->isac), 0 ); if(!input_buf) input_buf = ms_malloc( ISAC_SAMPLES_PER_ENCODE*2 ); ms_bufferizer_read(obj->bufferizer, input_buf, ISAC_SAMPLES_PER_ENCODE*2); ret = WebRtcIsacfix_Encode(obj->isac, (const WebRtc_Word16*)input_buf, (WebRtc_Word16*)om->b_wptr); if( ret < 0) { ms_error( "WebRtcIsacfix_Encode error: %d", WebRtcIsacfix_GetErrorCode(obj->isac) ); freeb(om); } else if( ret == 0 ) { // Encode() buffered the input, not yet able to produce a packet, continue feeding it // 160 samples per-call obj->ts += ISAC_SAMPLES_PER_ENCODE; freeb(om); } else { // a new packet has been encoded, send it obj->ts += ISAC_SAMPLES_PER_ENCODE; om->b_wptr += ret; out_count++; // ms_message("packet %d out, samples %d", out_count, obj->ts); mblk_set_timestamp_info( om, obj->ts ); ms_queue_put(f->outputs[0], om); om = NULL; } } if( input_buf ){ ms_free(input_buf); } }
static void enc_process(MSFilter *f){ SpeexEncState *s=(SpeexEncState*)f->data; mblk_t *im; int nbytes; uint8_t *buf; int frame_per_packet=1; if (s->frame_size<=0) return; ms_filter_lock(f); if (s->ptime>=20) { frame_per_packet = s->ptime/20; } if (frame_per_packet<=0) frame_per_packet=1; if (frame_per_packet>7) /* 7*20 == 140 ms max */ frame_per_packet=7; nbytes=s->frame_size*2; buf=(uint8_t*)alloca(nbytes*frame_per_packet); while((im=ms_queue_get(f->inputs[0]))!=NULL){ ms_bufferizer_put(s->bufferizer,im); } while(ms_bufferizer_read(s->bufferizer,buf,nbytes*frame_per_packet)==nbytes*frame_per_packet){ mblk_t *om=allocb(nbytes*frame_per_packet,0);//too large... int k; SpeexBits bits; speex_bits_init(&bits); for (k=0;k<frame_per_packet;k++) { speex_encode_int(s->state,(int16_t*)(buf + (k*s->frame_size*2)),&bits); s->ts+=s->frame_size; } speex_bits_insert_terminator(&bits); k=speex_bits_write(&bits, (char*)om->b_wptr, nbytes*frame_per_packet); om->b_wptr+=k; mblk_set_timestamp_info(om,s->ts-s->frame_size); ms_bufferizer_fill_current_metas(s->bufferizer, om); ms_queue_put(f->outputs[0],om); speex_bits_destroy(&bits); } ms_filter_unlock(f); }
static void volume_process(MSFilter *f){ mblk_t *m; Volume *v=(Volume*)f->data; float target_gain; /* Important notice: any processes called herein can modify v->target_gain, at * end of this function apply_gain() is called, thus: later process calls can * override this target gain, and order must be well thought out */ if (v->agc_enabled || v->peer!=NULL){ mblk_t *om; int nbytes=v->nsamples*2; ms_bufferizer_put_from_queue(v->buffer,f->inputs[0]); while(ms_bufferizer_get_avail(v->buffer)>=nbytes){ om=allocb(nbytes,0); ms_bufferizer_read(v->buffer,om->b_wptr,nbytes); om->b_wptr+=nbytes; update_energy((int16_t*)om->b_rptr, v->nsamples, v); target_gain = v->static_gain; if (v->peer) /* this ptr set = echo limiter enable flag */ target_gain = volume_echo_avoider_process(v, om); /* Multiply with gain from echo limiter, not "choose smallest". Why? * Remote talks, local echo suppress via mic path, but still audible in * remote speaker. AGC operates fully, too (local speaker close to local mic!); * having agc gain reduction also contribute to total reduction makes sense. */ if (v->agc_enabled) target_gain/= volume_agc_process(v, om); if (v->noise_gate_enabled) volume_noise_gate_process(v, v->level_pk, om); apply_gain(v, om, target_gain); ms_queue_put(f->outputs[0],om); } }else{ /*light processing: no agc. Work in place in the input buffer*/ while((m=ms_queue_get(f->inputs[0]))!=NULL){ update_energy((int16_t*)m->b_rptr, (m->b_wptr - m->b_rptr) / 2, v); target_gain = v->static_gain; if (v->noise_gate_enabled) volume_noise_gate_process(v, v->level_pk, m); apply_gain(v, m, target_gain); ms_queue_put(f->outputs[0],m); } } }
void alsa_read_process(MSFilter *obj){ AlsaReadData *ad=(AlsaReadData*)obj->data; mblk_t *om=NULL; int samples=(160*ad->rate)/8000; ms_mutex_lock(&ad->mutex); while (ms_bufferizer_get_avail(ad->bufferizer)>=samples*2){ om=allocb(samples*2,0); ms_bufferizer_read(ad->bufferizer,om->b_wptr,samples*2); om->b_wptr+=samples*2; /*ms_message("alsa_read_process: Outputing %i bytes",size);*/ ms_queue_put(obj->outputs[0],om); } ms_mutex_unlock(&ad->mutex); }
static OSStatus au_render_cb ( void *inRefCon, AudioUnitRenderActionFlags *ioActionFlags, const AudioTimeStamp *inTimeStamp, UInt32 inBusNumber, UInt32 inNumberFrames, AudioBufferList *ioData ) { ms_debug("render cb"); AUData *d=(AUData*)inRefCon; if (d->write_started == TRUE) { ioData->mBuffers[0].mDataByteSize=inNumberFrames*d->bits/8; ioData->mNumberBuffers=1; ms_mutex_lock(&d->mutex); if(ms_bufferizer_get_avail(d->bufferizer) >= inNumberFrames*d->bits/8) { ms_bufferizer_read(d->bufferizer, ioData->mBuffers[0].mData, inNumberFrames*d->bits/8); if (ms_bufferizer_get_avail(d->bufferizer) >10*inNumberFrames*d->bits/8) { ms_debug("we are late, bufferizer sise is %i bytes in framezize is %i bytes",ms_bufferizer_get_avail(d->bufferizer),inNumberFrames*d->bits/8); ms_bufferizer_flush(d->bufferizer); } ms_mutex_unlock(&d->mutex); } else { ms_mutex_unlock(&d->mutex); memset(ioData->mBuffers[0].mData, 0,ioData->mBuffers[0].mDataByteSize); ms_debug("nothing to write, pushing silences, bufferizer size is %i bytes in framezize is %i bytes mDataByteSize %i" ,ms_bufferizer_get_avail(d->bufferizer) ,inNumberFrames*d->bits/8 ,ioData->mBuffers[0].mDataByteSize); d->n_lost_frame+=inNumberFrames; } } if (!d->is_ringer) { // no need to read in ringer mode AudioBufferList readAudioBufferList; readAudioBufferList.mBuffers[0].mDataByteSize=inNumberFrames*d->bits/8; readAudioBufferList.mNumberBuffers=1; readAudioBufferList.mBuffers[0].mData=NULL; readAudioBufferList.mBuffers[0].mNumberChannels=d->nchannels; AudioUnitElement inputBus = 1; au_read_cb(d, ioActionFlags, inTimeStamp, inputBus, inNumberFrames, &readAudioBufferList); } return 0; }
static void enc_process(MSFilter *f){ EncState *s=(EncState*)f->data; mblk_t *im; int16_t buf[160]; while((im=ms_queue_get(f->inputs[0]))!=NULL){ ms_bufferizer_put(s->bufferizer,im); } while(ms_bufferizer_read(s->bufferizer,(uint8_t*)buf,sizeof(buf))==sizeof(buf)) { mblk_t *om=allocb(33,0); gsm_encode(s->state,(gsm_signal*)buf,(gsm_byte*)om->b_wptr); om->b_wptr+=33; mblk_set_timestamp_info(om,s->ts); ms_queue_put(f->outputs[0],om); s->ts+=sizeof(buf)/2; } }
static void detector_process(MSFilter *f){ DetectorState *s=(DetectorState *)f->data; mblk_t *m; while ((m=ms_queue_get(f->inputs[0]))!=NULL){ ms_queue_put(f->outputs[0],m); if (s->nscans>0){ ms_bufferizer_put(s->buf,dupmsg(m)); } } if (s->nscans>0){ uint8_t *buf=alloca(s->framesize); while(ms_bufferizer_read(s->buf,buf,s->framesize)!=0){ float en=compute_energy((int16_t*)buf,s->framesize/2); if (en>energy_min_threshold*(32767.0*32767.0*0.7)){ int i; for(i=0;i<s->nscans;++i){ GoertzelState *gs=&s->tone_gs[i]; MSToneDetectorDef *tone_def=&s->tone_def[i]; float freq_en=goertzel_state_run(gs,(int16_t*)buf,s->framesize/2,en); if (freq_en>=tone_def->min_amplitude){ if (gs->dur==0) gs->starttime=f->ticker->time; gs->dur+=s->frame_ms; if (gs->dur>=tone_def->min_duration && !gs->event_sent){ MSToneDetectorEvent event; strncpy(event.tone_name,tone_def->tone_name,sizeof(event.tone_name)); event.tone_start_time=gs->starttime; ms_filter_notify(f,MS_TONE_DETECTOR_EVENT,&event); gs->event_sent=TRUE; } }else{ gs->event_sent=FALSE; gs->dur=0; gs->starttime=0; } } }else end_all_tones(s); } } }
static OSStatus writeRenderProc(void *inRefCon, AudioUnitRenderActionFlags *inActionFlags, const AudioTimeStamp *inTimeStamp, UInt32 inBusNumber, UInt32 inNumFrames, AudioBufferList *ioData) { AUWrite *d=(AUWrite*)inRefCon; int read; if (ioData->mNumberBuffers!=1) ms_warning("writeRenderProc: %i buffers",ioData->mNumberBuffers); ms_mutex_lock(&d->common.mutex); read=ms_bufferizer_read(d->buffer,ioData->mBuffers[0].mData,ioData->mBuffers[0].mDataByteSize); ms_mutex_unlock(&d->common.mutex); if (read==0) { ms_warning("Silence inserted in audio output unit (%i bytes)",ioData->mBuffers[0].mDataByteSize); memset(ioData->mBuffers[0].mData,0,ioData->mBuffers[0].mDataByteSize); } return 0; }
static void enc_process(MSFilter *f){ EncState *s=(EncState*)f->data; mblk_t *im; int size = BV16_CODE_SIZE *2; int i,frames; uint8_t buf[BV16_FRAME_SIZE * 2 * 4]; while((im=ms_queue_get(f->inputs[0]))!=NULL){ ms_bufferizer_put(s->bufferizer,im); } while(ms_bufferizer_read(s->bufferizer,(uint8_t*)buf,sizeof(buf))==sizeof(buf)) { mblk_t *om=allocb(BV16_CODE_SIZE*8,0); uint8_t *in_buf = (uint8_t *)buf; frames = sizeof(buf)/(BV16_FRAME_SIZE * 2); for(i=0;i<frames;i++) { //BV16_Encode(&s->ebs, &s->cs,(short *)in_buf); //BV16_BitPack((unsigned short *)om->b_wptr, &s->ebs); bv16_encode(s->enc, (uint8_t *)om->b_wptr, (int16_t *) in_buf, BV16_FRAME_SIZE); //ms_error("BV16_Encode frames %d",i); in_buf += 80; om->b_wptr += 10; } mblk_set_timestamp_info(om,s->ts); ms_queue_put(f->outputs[0],om); s->ts+=sizeof(buf)/2; } }
static void enc_process(MSFilter *f) { EncState *s = (EncState*) f->data; unsigned int unitary_buff_size = sizeof (int16_t)*160; unsigned int buff_size = unitary_buff_size * s->ptime / 20; mblk_t *im; uint8_t tmp[OUT_MAX_SIZE]; int16_t samples[buff_size]; while ((im = ms_queue_get(f->inputs[0])) != NULL) { ms_bufferizer_put(s->mb, im); } while (ms_bufferizer_get_avail(s->mb) >= buff_size) { mblk_t *om = allocb(OUT_MAX_SIZE * buff_size / unitary_buff_size + 1, 0); ms_bufferizer_read(s->mb, (uint8_t*) samples, buff_size); *om->b_wptr = 0xf0; uint8_t *tocs = om->b_wptr++; om->b_wptr += buff_size / unitary_buff_size; int offset; for (offset = 0; offset < buff_size; offset += unitary_buff_size) { int ret = Encoder_Interface_Encode(s->enc, s->mode, &samples[offset / sizeof (int16_t)], tmp, s->dtx); if (ret <= 0 || ret > 32) { ms_warning("Encoder returned %i", ret); freemsg(om); continue; } memcpy(om->b_wptr, &tmp[1], ret - 1); om->b_wptr += ret - 1; *(++tocs) = tmp[0] | 0x80; // Not last payload } *tocs &= 0x7F; // last payload mblk_set_timestamp_info(om, s->ts); ms_queue_put(f->outputs[0], om); s->ts += buff_size / sizeof (int16_t)/*sizeof(buf)/2*/; } }
static void enc_process(MSFilter *f){ EncState *s=(EncState*)f->data; mblk_t *im,*om; int size=s->nsamples*2; int16_t samples[1610]; /* BLOCKL_MAX * 7 is the largest size for ptime == 140 */ float samples2[BLOCKL_MAX]; int i; int frame_per_packet=1; if (s->ptime>=20 && s->ms_per_frame>0 && s->ptime%s->ms_per_frame==0) { frame_per_packet = s->ptime/s->ms_per_frame; } if (frame_per_packet<=0) frame_per_packet=1; if (frame_per_packet>7) /* 7*20 == 140 ms max */ frame_per_packet=7; while((im=ms_queue_get(f->inputs[0]))!=NULL){ ms_bufferizer_put(s->bufferizer,im); } while(ms_bufferizer_read(s->bufferizer,(uint8_t*)samples,size*frame_per_packet)==(size*frame_per_packet)){ int k; om=allocb(s->nbytes*frame_per_packet,0); for (k=0;k<frame_per_packet;k++) { for (i=0;i<s->nsamples;i++){ samples2[i]=samples[i+(s->nsamples*k)]; } iLBC_encode((uint8_t*)om->b_wptr,samples2,&s->ilbc_enc); om->b_wptr+=s->nbytes; } s->ts+=s->nsamples*frame_per_packet; mblk_set_timestamp_info(om,s->ts); ms_bufferizer_fill_current_metas(s->bufferizer,om); ms_queue_put(f->outputs[0],om); } }
static void alaw_enc_process(MSFilter *obj){ AlawEncData *dt=(AlawEncData*)obj->data; MSBufferizer *bz=dt->bz; uint8_t buffer[2240]; int frame_per_packet=2; int size_of_pcm=320; mblk_t *m; if (dt->ptime>=10) { frame_per_packet = dt->ptime/10; } if (frame_per_packet<=0) frame_per_packet=1; if (frame_per_packet>14) /* 7*20 == 140 ms max */ frame_per_packet=14; size_of_pcm = 160*frame_per_packet; /* ex: for 20ms -> 160*2==320 */ while((m=ms_queue_get(obj->inputs[0]))!=NULL){ ms_bufferizer_put(bz,m); } while (ms_bufferizer_read(bz,buffer,size_of_pcm)==size_of_pcm){ mblk_t *o=allocb(size_of_pcm/2,0); int i; for (i=0;i<size_of_pcm/2;i++){ *o->b_wptr=Snack_Lin2Alaw(((int16_t*)buffer)[i]); o->b_wptr++; } ms_bufferizer_fill_current_metas(bz, o); mblk_set_timestamp_info(o,dt->ts); dt->ts+=size_of_pcm/2; ms_queue_put(obj->outputs[0],o); } }
static void sound_read_process(MSFilter *f){ msandroid_sound_read_data *d=(msandroid_sound_read_data*)f->data; int nbytes=d->framesize*d->nchannels*2; int avail; bool_t flush=FALSE; bool_t can_output=(d->start_time==-1 || ((f->ticker->time-d->start_time)%d->outgran_ms==0)); ms_mutex_lock(&d->mutex); if (!d->started) { ms_mutex_unlock(&d->mutex); return; } avail=ms_bufferizer_get_avail(&d->rb); if (f->ticker->time % 5000==0){ if (d->min_avail>=(sndread_flush_threshold*(float)d->rate*2.0*(float)d->nchannels)){ int excess_ms=(d->min_avail*1000)/(d->rate*2*d->nchannels); ms_warning("Excess of audio samples in capture side bytes=%i (%i ms)",d->min_avail,excess_ms); can_output=TRUE; flush=TRUE; } d->min_avail=-1; } do{ if (can_output && (avail>=nbytes*2)){//bytes*2 is to insure smooth output, we leave at least one packet in the buffer for next time*/ mblk_t *om=allocb(nbytes,0); ms_bufferizer_read(&d->rb,om->b_wptr,nbytes); om->b_wptr+=nbytes; ms_queue_put(f->outputs[0],om); //ms_message("Out time=%llu ",f->ticker->time); if (d->start_time==-1) d->start_time=f->ticker->time; avail-=nbytes; }else break; }while(flush); ms_mutex_unlock(&d->mutex); if (d->min_avail==-1 || avail<d->min_avail) d->min_avail=avail; }
static void * winsnd_thread(void *p){ MSSndCard *card=(MSSndCard*)p; WinSndData *d=(WinSndData*)card->data; int bsize=d->rate/8000 * 320; uint8_t *rtmpbuff=NULL; uint8_t *wtmpbuff=NULL; int err; MMRESULT mr = NOERROR; int pos_whdr=0; d->stat_input=0; d->stat_output=0; d->stat_notplayed=0; d->sound_err=winsnd_open(d, d->devid, d->bits,d->stereo,d->rate,&bsize); if (d->sound_err==0){ rtmpbuff=(uint8_t*)alloca(bsize); wtmpbuff=(uint8_t*)alloca(bsize); } while(d->read_started || d->write_started){ if (d->sound_err==0){ if (d->write_started){ #if 0 if (d->stat_output>0 && d->buffer_playing==0) { ms_error("No data currently playing in sound card" ); } if (d->stat_output>0 && (d->stat_input-d->stat_output>10 || d->stat_input-d->stat_output<-10)) ms_error("Not perfectly synchronized (input-output=%i)", d->stat_input-d->stat_output); #endif while (d->buffer_playing<6 && d->buffer_playing<MAX_WAVEHDR) { ms_mutex_lock(&d->mutex); err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize); ms_mutex_unlock(&d->mutex); if (err!=bsize) break; ms_mutex_lock(&d->mutex); /* write to sound devide! */ memcpy (d->waveouthdr[pos_whdr].lpData, wtmpbuff, bsize); mr = waveOutWrite (d->waveoutdev, &(d->waveouthdr[pos_whdr]), sizeof (d->waveouthdr[pos_whdr])); if (mr != MMSYSERR_NOERROR) { if (mr == WAVERR_STILLPLAYING) { /* retry later */ /* data should go back to queue */ /* TODO */ ms_warning("sound device write STILL_PLAYING (waveOutWrite:0x%i)", mr); } else { ms_warning("sound device write returned (waveOutWrite:0x%i)", mr); } } else { d->buffer_playing++; pos_whdr++; if (pos_whdr == MAX_WAVEHDR) pos_whdr = 0; /* loop over the prepared blocks */ } ms_mutex_unlock(&d->mutex); if (err<0){ #if !defined(_WIN32_WCE) ms_warning("Fail to write %i bytes from soundcard: %s", bsize,strerror(errno)); #else ms_warning("Fail to write %i bytes from soundcard: %i", bsize,WSAGetLastError()); #endif } } if (d->buffer_playing==6 || d->buffer_playing==MAX_WAVEHDR) { int discarded=0; ms_mutex_lock(&d->mutex); while (d->bufferizer->size>=bsize){ discarded++; d->stat_notplayed++; err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize); } ms_mutex_unlock(&d->mutex); if (discarded>0) ms_error("Extra data for sound card removed (%ims), (playing: %i) (input-output: %i)", (discarded*20*320)/320, d->buffer_playing, d->stat_input - d->stat_output); } #if !defined(_WIN32_WCE) Sleep(5); #endif #if defined(_WIN32_WCE) Sleep(10); #endif }else { int discarded=0; /* don't think this is usefull, anyway... */ ms_mutex_lock(&d->mutex); while (d->bufferizer->size>=bsize){ discarded++; err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize); } ms_mutex_unlock(&d->mutex); if (discarded>0) ms_error("Extra data for sound card removed (%ims), (playing: %i) (input-output: %i)", (discarded*20)/320, d->buffer_playing, d->stat_input - d->stat_output); Sleep(10); } }else Sleep(10); } if (d->sound_err==0) { int i; int count=0; /* close sound card */ ms_error("Shutting down sound device (playing: %i) (input-output: %i) (notplayed: %i)", d->buffer_playing, d->stat_input - d->stat_output, d->stat_notplayed); /* unprepare buffer */ for (i = 0; i < MAX_WAVEHDR; i++) { int counttry=0; for (counttry=0;counttry<10;counttry++) { mr = waveInUnprepareHeader (d->waveindev, &(d->waveinhdr[i]), sizeof (d->waveinhdr[i])); if (mr != MMSYSERR_NOERROR) { ms_error("Failed to unprepared %i buffer from sound card (waveInUnprepareHeader:0x%i", count, mr); Sleep (20); } else { count++; ms_message("successfully unprepared %i buffer from sound card.", count); break; } } } ms_warning("unprepared %i buffer from sound card.", count); mr = waveInStop (d->waveindev); if (mr != MMSYSERR_NOERROR) { ms_error("failed to stop recording sound card (waveInStop:0x%i)", mr); } else { ms_message("successfully stopped recording sound card"); } mr = waveInReset (d->waveindev); if (mr != MMSYSERR_NOERROR) { ms_warning("failed to reset recording sound card (waveInReset:0x%i)", mr); } else { ms_message("successful reset of recording sound card"); } mr = waveInClose (d->waveindev); if (mr != MMSYSERR_NOERROR) { ms_warning("failed to close recording sound card (waveInClose:0x%i)", mr); } else { ms_message("successfully closed recording sound card"); } d->sound_err=-1; } d->stat_input=0; d->stat_output=0; d->stat_notplayed=0; return NULL; }
static void conf_process(MSFilter *f){ int i; ConfState *s=(ConfState*)f->data; Channel *chan; Channel *chan0; ms_mutex_lock(&s->lock); /*read from all inputs and put into bufferizers*/ for (i=0;i<CONF_MAX_PINS;++i){ if (f->inputs[i]!=NULL){ chan=&s->channels[i]; ms_bufferizer_put_from_queue(&chan->buff,f->inputs[i]); if (ms_bufferizer_get_avail(&chan->buff)>0) { chan->missed=0; /* reset counter of missed packet */ if (/*i>0 && */chan->is_used==FALSE) { chan->is_used=TRUE; ms_message("msconf: new contributing stream (chan=%i) %i", ms_bufferizer_get_avail(&chan->buff), i); } } } } /*do the job */ while(should_process(f,s)==TRUE){ conf_sum(f, s); conf_dispatch(f,s); } #if 0 /* mixer is disabled! -> copy A->B and B->A*/ if (s->mix_mode == FALSE) { /* get the soundread data and copy it to pinX */ for (i=1;i<CONF_MAX_PINS;i=i+2){ if (f->inputs[i]!=NULL){ chan0=&s->channels[0]; chan=&s->channels[i]; if (chan->is_used==TRUE) { while (ms_bufferizer_read(&chan->buff,(uint8_t*)chan->input,s->conf_gran)==s->conf_gran) { if (f->outputs[0]!=NULL) { /* send in pin0 */ mblk_t *m=allocb(s->conf_gran,0); memcpy(m->b_wptr, chan->input, s->conf_gran); m->b_wptr+=s->conf_gran; ms_queue_put(f->outputs[0],m); } } } if (chan0->is_used==TRUE) { while (ms_bufferizer_read(&chan0->buff,(uint8_t*)chan0->input,s->conf_gran)==s->conf_gran) { if (f->outputs[i]!=NULL) { /* send in pinI */ mblk_t *m=allocb(s->conf_gran,0); memcpy(m->b_wptr, chan0->input, s->conf_gran); m->b_wptr+=s->conf_gran; ms_queue_put(f->outputs[i],m); } } } break; } } } #endif // 0 ms_mutex_unlock(&s->lock); }
/* inputs[0]= reference signal from far end (sent to soundcard) * inputs[1]= near speech & echo signal (read from soundcard) * outputs[0]= is a copy of inputs[0] to be sent to soundcard * outputs[1]= near end speech, echo removed - towards far end */ static void speex_ec_process(MSFilter *f){ SpeexECState *s=(SpeexECState*)f->data; int nbytes=s->framesize*2; mblk_t *refm; uint8_t *ref,*echo; if (s->bypass_mode) { while((refm=ms_queue_get(f->inputs[0]))!=NULL){ ms_queue_put(f->outputs[0],refm); } while((refm=ms_queue_get(f->inputs[1]))!=NULL){ ms_queue_put(f->outputs[1],refm); } return; } if (f->inputs[0]!=NULL){ if (s->echostarted){ while((refm=ms_queue_get(f->inputs[0]))!=NULL){ refm=audio_flow_controller_process(&s->afc,refm); if (refm){ mblk_t *cp=dupmsg(refm); ms_bufferizer_put(&s->delayed_ref,cp); ms_bufferizer_put(&s->ref,refm); } } }else{ ms_warning("Getting reference signal but no echo to synchronize on."); ms_queue_flush(f->inputs[0]); } } ms_bufferizer_put_from_queue(&s->echo,f->inputs[1]); ref=(uint8_t*)alloca(nbytes); echo=(uint8_t*)alloca(nbytes); while (ms_bufferizer_read(&s->echo,echo,nbytes)==nbytes){ mblk_t *oecho=allocb(nbytes,0); int avail; int avail_samples; if (!s->echostarted) s->echostarted=TRUE; if ((avail=ms_bufferizer_get_avail(&s->delayed_ref))<((s->nominal_ref_samples*2)+nbytes)){ /*we don't have enough to read in a reference signal buffer, inject silence instead*/ avail=nbytes; refm=allocb(nbytes,0); memset(refm->b_wptr,0,nbytes); refm->b_wptr+=nbytes; ms_bufferizer_put(&s->delayed_ref,refm); ms_queue_put(f->outputs[0],dupmsg(refm)); if (!s->using_zeroes){ ms_warning("Not enough ref samples, using zeroes"); s->using_zeroes=TRUE; } }else{ if (s->using_zeroes){ ms_message("Samples are back."); s->using_zeroes=FALSE; } /* read from our no-delay buffer and output */ refm=allocb(nbytes,0); if (ms_bufferizer_read(&s->ref,refm->b_wptr,nbytes)==0){ ms_fatal("Should never happen"); } refm->b_wptr+=nbytes; ms_queue_put(f->outputs[0],refm); } /*now read a valid buffer of delayed ref samples*/ if (ms_bufferizer_read(&s->delayed_ref,ref,nbytes)==0){ ms_fatal("Should never happen"); } avail-=nbytes; avail_samples=avail/2; /*ms_message("avail=%i",avail_samples);*/ if (avail_samples<s->min_ref_samples || s->min_ref_samples==-1){ s->min_ref_samples=avail_samples; } #ifdef EC_DUMP if (s->reffile) fwrite(ref,nbytes,1,s->reffile); if (s->echofile) fwrite(echo,nbytes,1,s->echofile); #endif speex_echo_cancellation(s->ecstate,(short*)echo,(short*)ref,(short*)oecho->b_wptr); speex_preprocess_run(s->den, (short*)oecho->b_wptr); #ifdef EC_DUMP if (s->cleanfile) fwrite(oecho->b_wptr,nbytes,1,s->cleanfile); #endif oecho->b_wptr+=nbytes; ms_queue_put(f->outputs[1],oecho); } /*verify our ref buffer does not become too big, meaning that we are receiving more samples than we are sending*/ if ((((uint32_t)(f->ticker->time - s->flow_control_time)) >= flow_control_interval_ms) && (s->min_ref_samples != -1)) { int diff=s->min_ref_samples-s->nominal_ref_samples; if (diff>(nbytes/2)){ int purge=diff-(nbytes/2); ms_warning("echo canceller: we are accumulating too much reference signal, need to throw out %i samples",purge); audio_flow_controller_set_target(&s->afc,purge,(flow_control_interval_ms*s->samplerate)/1000); } s->min_ref_samples=-1; s->flow_control_time = f->ticker->time; } }
static void * oss_thread(void *p){ MSSndCard *card=(MSSndCard*)p; OssData *d=(OssData*)card->data; int bsize=0; uint8_t *rtmpbuff=NULL; uint8_t *wtmpbuff=NULL; int err; mblk_t *rm=NULL; d->pcmfd=oss_open(d->pcmdev,d->bits,d->stereo,d->rate,&bsize); if (d->pcmfd>=0){ rtmpbuff=(uint8_t*)malloc(bsize); wtmpbuff=(uint8_t*)malloc(bsize); if(rtmpbuff == NULL || wtmpbuff == NULL) { free(rtmpbuff); free(wtmpbuff); return NULL; } } while(d->read_started || d->write_started){ if (d->pcmfd>=0){ if (d->read_started){ struct timeval timeout; fd_set read_fds; audio_buf_info info; if (rm==NULL) rm=allocb(bsize,0); timeout.tv_sec = 0; timeout.tv_usec = 0; FD_ZERO( &read_fds ); FD_SET( d->pcmfd, &read_fds ); if( select( d->pcmfd + 1, &read_fds, NULL, NULL, &timeout ) == -1 ) { } if (FD_ISSET( d->pcmfd, &read_fds ) && ioctl( d->pcmfd, SNDCTL_DSP_GETISPACE, &info ) != -1) { if (info.bytes>=bsize) { err=read(d->pcmfd,rm->b_wptr,bsize); if (err<0){ ms_warning("Fail to read %i bytes from soundcard: %s", bsize,strerror(errno)); }else{ rm->b_wptr+=err; ms_mutex_lock(&d->mutex); putq(&d->rq,rm); ms_mutex_unlock(&d->mutex); rm=NULL; } } else { timeout.tv_sec = 0; timeout.tv_usec = 5000; select(0, 0, NULL, NULL, &timeout ); } } else { timeout.tv_sec = 0; timeout.tv_usec = 5000; select(0, 0, NULL, NULL, &timeout ); } }else { int sz = read(d->pcmfd,rtmpbuff,bsize); if( sz!=bsize) ms_warning("sound device read returned %i !",sz); } if (d->write_started){ audio_buf_info info; if( ms_bufferizer_get_avail(d->bufferizer)>=bsize && ioctl( d->pcmfd, SNDCTL_DSP_GETOSPACE, &info ) == 0 ) { if( info.fragstotal - info.fragments > 15 ) { static int c=0; /* drop the fragment if the buffer starts to fill up */ /* we got too much data: I prefer to empty the incoming buffer */ while (ms_bufferizer_get_avail(d->bufferizer)>bsize*4){ ms_mutex_lock(&d->mutex); err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize); err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize); err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize); err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize); ms_mutex_unlock(&d->mutex); c=c+err*4; ms_warning("drop fragment when buffer gets too much data (%i - discarded:%i)", info.fragstotal - info.fragments, c); if (err==0) break; } }else { ms_mutex_lock(&d->mutex); err=ms_bufferizer_read(d->bufferizer,wtmpbuff,bsize); ms_mutex_unlock(&d->mutex); err=write(d->pcmfd,wtmpbuff,bsize); if (err<0){ ms_warning("Fail to write %i bytes from soundcard: %s", bsize,strerror(errno)); } } } }else { int sz; memset(wtmpbuff,0,bsize); sz = write(d->pcmfd,wtmpbuff,bsize); if( sz!=bsize) ms_warning("sound device write returned %i !",sz); } }else usleep(20000); } if (d->pcmfd>=0) { close(d->pcmfd); d->pcmfd=-1; } free(rtmpbuff); free(wtmpbuff); if (rm!=NULL) freemsg(rm); /*reset to default parameters */ //d->bits=16; //d->rate=8000; //d->stereo=FALSE; return NULL; }
static void ms_opus_enc_process(MSFilter *f) { OpusEncData *d = (OpusEncData *)f->data; mblk_t *im; mblk_t *om = NULL; int i; int frameNumber, packet_size; uint8_t *signalFrameBuffer = NULL; uint8_t *codedFrameBuffer[MAX_INPUT_FRAMES]; OpusRepacketizer *rp = opus_repacketizer_create(); opus_int32 ret = 0; opus_int32 totalLength = 0; int frame_size = d->samplerate * FRAME_LENGTH / 1000; /* in samples */ // lock the access while getting ptime ms_filter_lock(f); frameNumber = d->ptime/FRAME_LENGTH; /* encode 20ms frames, ptime is a multiple of 20ms */ packet_size = d->samplerate * d->ptime / 1000; /* in samples */ ms_filter_unlock(f); while ((im = ms_queue_get(f->inputs[0])) != NULL) { ms_bufferizer_put(d->bufferizer, im); } for (i=0; i<MAX_INPUT_FRAMES; i++) { codedFrameBuffer[i]=NULL; } while (ms_bufferizer_get_avail(d->bufferizer) >= (d->channels * packet_size * SIGNAL_SAMPLE_SIZE)) { totalLength = 0; opus_repacketizer_init(rp); for (i=0; i<frameNumber; i++) { /* encode 20ms by 20ms and repacketize all of them together */ if (!codedFrameBuffer[i]) codedFrameBuffer[i] = ms_malloc(MAX_BYTES_PER_FRAME); /* the repacketizer need the pointer to packet to remain valid, so we shall have a buffer for each coded frame */ if (!signalFrameBuffer) signalFrameBuffer = ms_malloc(frame_size * SIGNAL_SAMPLE_SIZE * d->channels); ms_bufferizer_read(d->bufferizer, signalFrameBuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels); ret = opus_encode(d->state, (opus_int16 *)signalFrameBuffer, frame_size, codedFrameBuffer[i], MAX_BYTES_PER_FRAME); if (ret < 0) { ms_error("Opus encoder error: %s", opus_strerror(ret)); break; } if (ret > 0) { int err = opus_repacketizer_cat(rp, codedFrameBuffer[i], ret); /* add the encoded frame into the current packet */ if (err != OPUS_OK) { ms_error("Opus repacketizer error: %s", opus_strerror(err)); break; } totalLength += ret; } } if (ret > 0) { om = allocb(totalLength+frameNumber + 1, 0); /* opus repacktizer API: allocate at leat number of frame + size of all data added before */ ret = opus_repacketizer_out(rp, om->b_wptr, totalLength+frameNumber); om->b_wptr += ret; mblk_set_timestamp_info(om, d->ts); ms_queue_put(f->outputs[0], om); d->ts += packet_size*48000/d->samplerate; /* RFC payload RTP opus 03 - section 4: RTP timestamp multiplier : WARNING works only with sr at 48000 */ ret = 0; } } opus_repacketizer_destroy(rp); if (signalFrameBuffer != NULL) { ms_free(signalFrameBuffer); } for (i=0; i<frameNumber; i++) { if (codedFrameBuffer[i] != NULL) { ms_free(codedFrameBuffer[i]); } } }
static void ms_opus_enc_process(MSFilter *f) { OpusEncData *d = (OpusEncData *)f->data; OpusRepacketizer *repacketizer = NULL; mblk_t *om = NULL; int packet_size, pcm_buffer_size; int max_frame_byte_size, ptime = 20; int frame_count = 0, frame_size = 0; opus_int32 total_length = 0; uint8_t *repacketizer_frame_buffer[MAX_INPUT_FRAMES]; int i; ms_filter_lock(f); ptime = d->ptime; packet_size = d->samplerate * ptime / 1000; /* in samples */ ms_filter_unlock(f); switch (ptime) { case 10: frame_size = d->samplerate * 10 / 1000; frame_count = 1; break; case 20: frame_size = d->samplerate * 20 / 1000; frame_count = 1; break; case 40: frame_size = d->samplerate * 40 / 1000; frame_count = 1; break; case 60: frame_size = d->samplerate * 60 / 1000; frame_count = 1; break; case 80: frame_size = d->samplerate * 40 / 1000; frame_count = 2; break; case 100: frame_size = d->samplerate * 20 / 1000; frame_count = 5; break; case 120: frame_size = d->samplerate * 60 / 1000; frame_count = 2; break; default: frame_size = d->samplerate * 20 / 1000; frame_count = 1; } max_frame_byte_size = MAX_BYTES_PER_MS * ptime/frame_count; pcm_buffer_size = d->channels * frame_size * SIGNAL_SAMPLE_SIZE; if (pcm_buffer_size > d->pcmbufsize){ if (d->pcmbuffer) ms_free(d->pcmbuffer); d->pcmbuffer = ms_malloc(pcm_buffer_size); d->pcmbufsize = pcm_buffer_size; } for (i=0; i<MAX_INPUT_FRAMES; i++) { repacketizer_frame_buffer[i]=NULL; } ms_bufferizer_put_from_queue(d->bufferizer, f->inputs[0]); while (ms_bufferizer_get_avail(d->bufferizer) >= (d->channels * packet_size * SIGNAL_SAMPLE_SIZE)) { opus_int32 ret = 0; if (frame_count == 1) { /* One Opus frame, not using the repacketizer */ om = allocb(max_frame_byte_size, 0); ms_bufferizer_read(d->bufferizer, d->pcmbuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels); ret = opus_encode(d->state, (opus_int16 *)d->pcmbuffer, frame_size, om->b_wptr, max_frame_byte_size); if (ret < 0) { freemsg(om); om=NULL; ms_error("Opus encoder error: %s", opus_strerror(ret)); break; } else { total_length = ret; om->b_wptr += total_length; } } else if(frame_count > 1) { /* We have multiple Opus frames we will use the opus repacketizer */ repacketizer = opus_repacketizer_create(); opus_repacketizer_init(repacketizer); /* Do not include FEC/LBRR in any frame after the first one since it will be sent with the previous one */ ret = opus_encoder_ctl(d->state, OPUS_SET_INBAND_FEC(0)); if (ret != OPUS_OK) { ms_error("could not set inband FEC to opus encoder: %s", opus_strerror(ret)); } for (i=0; i<frame_count; i++) { if(frame_count == i+1){ /* if configured, reactivate FEC on the last frame to tell the encoder he should restart saving LBRR frames */ ret = opus_encoder_ctl(d->state, OPUS_SET_INBAND_FEC(d->useinbandfec)); if (ret != OPUS_OK) { ms_error("could not set inband FEC to opus encoder: %s", opus_strerror(ret)); } } if (!repacketizer_frame_buffer[i]) repacketizer_frame_buffer[i] = ms_malloc(max_frame_byte_size); /* the repacketizer need the pointer to packet to remain valid, so we shall have a buffer for each coded frame */ ms_bufferizer_read(d->bufferizer, d->pcmbuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels); ret = opus_encode(d->state, (opus_int16 *)d->pcmbuffer, frame_size, repacketizer_frame_buffer[i], max_frame_byte_size); if (ret < 0) { ms_error("Opus encoder error: %s", opus_strerror(ret)); break; } else if (ret > 0) { int err = opus_repacketizer_cat(repacketizer, repacketizer_frame_buffer[i], ret); /* add the encoded frame into the current packet */ if (err != OPUS_OK) { ms_error("Opus repacketizer error: %s", opus_strerror(err)); break; } total_length += ret; } } om = allocb(total_length + frame_count + 1, 0); /* opus repacketizer API: allocate at least number of frame + size of all data added before */ ret = opus_repacketizer_out(repacketizer, om->b_wptr, total_length+frame_count); if(ret < 0){ freemsg(om); om=NULL; ms_error("Opus repacketizer out error: %s", opus_strerror(ret)); } else { om->b_wptr += ret; } opus_repacketizer_destroy(repacketizer); for (i=0; i<frame_count; i++) { if (repacketizer_frame_buffer[i] != NULL) { ms_free(repacketizer_frame_buffer[i]); } } } if(om) { /* we have an encoded output message */ mblk_set_timestamp_info(om, d->ts); ms_bufferizer_fill_current_metas(d->bufferizer, om); ms_queue_put(f->outputs[0], om); d->ts += packet_size*48000/d->samplerate; /* RFC payload RTP opus 03 - section 4: RTP timestamp multiplier : WARNING works only with sr at 48000 */ total_length = 0; } } }
static void* msandroid_write_cb(msandroid_sound_write_data* d) { jbyteArray write_buff; jmethodID write_id=0; jmethodID play_id=0; int min_size=-1; int count; int max_size=sndwrite_flush_threshold*(float)d->rate*(float)d->nchannels*2.0; int check_point_size=3*(float)d->rate*(float)d->nchannels*2.0; /*3 seconds*/ int nwrites=0; set_high_prio(); int buff_size = d->write_chunk_size; JNIEnv *jni_env = ms_get_jni_env(); // int write (byte[] audioData, int offsetInBytes, int sizeInBytes) write_id = jni_env->GetMethodID(d->audio_track_class,"write", "([BII)I"); if(write_id==0) { ms_error("cannot find AudioTrack.write() method"); goto end; } play_id = jni_env->GetMethodID(d->audio_track_class,"play", "()V"); if(play_id==0) { ms_error("cannot find AudioTrack.play() method"); goto end; } write_buff = jni_env->NewByteArray(buff_size); uint8_t tmpBuff[buff_size]; //start playing jni_env->CallVoidMethod(d->audio_track,play_id); ms_mutex_lock(&d->mutex); ms_bufferizer_flush(d->bufferizer); ms_mutex_unlock(&d->mutex); while(d->started) { int bufferizer_size; ms_mutex_lock(&d->mutex); min_size=-1; count=0; while((bufferizer_size = ms_bufferizer_get_avail(d->bufferizer)) >= d->write_chunk_size) { if (min_size==-1) min_size=bufferizer_size; else if (bufferizer_size<min_size) min_size=bufferizer_size; ms_bufferizer_read(d->bufferizer, tmpBuff, d->write_chunk_size); ms_mutex_unlock(&d->mutex); jni_env->SetByteArrayRegion(write_buff,0,d->write_chunk_size,(jbyte*)tmpBuff); int result = jni_env->CallIntMethod(d->audio_track,write_id,write_buff,0,d->write_chunk_size); d->writtenBytes+=result; if (result <= 0) { ms_error("write operation has failed [%i]",result); } nwrites++; ms_mutex_lock(&d->mutex); count+=d->write_chunk_size; if (count>check_point_size){ if (min_size > max_size) { ms_warning("we are late, flushing %i bytes",min_size); ms_bufferizer_skip_bytes(d->bufferizer,min_size); } count=0; } } if (d->started) { d->sleeping=true; ms_cond_wait(&d->cond,&d->mutex); d->sleeping=false; } ms_mutex_unlock(&d->mutex); } goto end; end: { ms_thread_exit(NULL); return NULL; } }
static void conf_sum(MSFilter *f, ConfState *s){ int i,j; Channel *chan; memset(s->sum,0,s->conf_nsamples*sizeof(int)); for (i=0;i<CONF_MAX_PINS;++i){ chan=&s->channels[i]; /* skip soundread and short buffer entry */ if (ms_bufferizer_get_avail(&chan->buff) > (s->conf_gran*CONF_BUFFER_SIZE) ) { #if 0 int loudness; #endif while (ms_bufferizer_get_avail(&chan->buff)> s->conf_gran) { ms_bufferizer_read(&chan->buff,(uint8_t*)chan->input,s->conf_gran); #if 0 speex_preprocess_ctl(chan->speex_pp, SPEEX_PREPROCESS_GET_AGC_LOUDNESS, &loudness); #endif /* we want to remove 4 packets (40ms) in a near future: */ #ifndef DISABLE_SPEEX if (chan->speex_pp!=NULL && s->enable_vad==TRUE) { int vad=0; vad = speex_preprocess(chan->speex_pp, (short*)chan->input, NULL); if (vad==1) break; /* voice detected: process as usual */ if (ms_bufferizer_get_avail(&chan->buff)<s->conf_gran) break; /* no more data to remove */ ms_message("No voice detected: discarding sample. (idx=%i - bufsize=%i sncardbufsize=%i)", i, ms_bufferizer_get_avail(&chan->buff), ms_bufferizer_get_avail(&chan->buff)); } if (ms_bufferizer_get_avail(&chan->buff) == (ms_bufferizer_get_avail(&chan->buff))) ms_message("same data in soundcard and incoming rtp. (idx=%i - bufsize=%i sncardbufsize=%i)", i, ms_bufferizer_get_avail(&chan->buff), ms_bufferizer_get_avail(&chan->buff)); #endif chan->stat_discarded++; } for(j=0;j<s->conf_nsamples;++j){ s->sum[j]+=chan->input[j]; } chan->has_contributed=TRUE; chan->stat_processed++; } else if (ms_bufferizer_get_avail(&chan->buff)>=s->conf_gran) { struct channel_volume { float energy; int channel; }; struct channel_volume vol; float en; ms_bufferizer_read(&chan->buff,(uint8_t*)chan->input,s->conf_gran); en=chan->energy; for(j=0;j<s->conf_nsamples;++j){ float s=chan->input[j]; en=(s*s*coef) + ((float)1.0-coef)*en; } chan->energy=en; vol.energy = chan->energy; //10*log10f(chan->energy/max_e); vol.channel = i; ms_filter_notify(f, MS_CONF_CHANNEL_VOLUME, (void*)&vol); { if (chan->energy>65) chan->count_speaking++; else chan->count_speaking=0; } #ifndef DISABLE_SPEEX if (chan->speex_pp!=NULL && s->enable_vad==TRUE && i==0) { int vad; vad = speex_preprocess(chan->speex_pp, (short*)chan->input, NULL); ms_filter_notify(f, MS_CONF_SPEEX_PREPROCESS_MIC, (void*)chan->speex_pp); if (s->enable_halfduplex>0) { powerspectrum_stat_beyond8K(chan); if (chan->average_psd>s->vad_prob_start) { if (chan->is_speaking<0) ms_message("MIC is turned on"); //ms_message("is_speaking (chan=%i) -> on/stat=%.3lf", i, chan->average_psd); chan->is_speaking=20; /* keep RTP unmuted for the next few ms */ } else if (chan->average_psd<s->vad_prob_continue) { if (chan->is_speaking==0) { ms_message("MIC is turned off"); chan->count_speaking=0; } chan->is_speaking--; } if (chan->is_speaking>0) { chan->count_speaking++; } } } else if (chan->speex_pp!=NULL && s->enable_vad==TRUE) { int vad; if (s->enable_halfduplex>0) { vad = speex_preprocess(chan->speex_pp, (short*)chan->input, NULL); } else { vad = speex_preprocess(chan->speex_pp, (short*)chan->input, NULL); //speex_preprocess_estimate_update(chan->speex_pp, (short*)chan->input); } } #endif for(j=0;j<s->conf_nsamples;++j){ s->sum[j]+=chan->input[j]; } chan->has_contributed=TRUE; chan->stat_processed++; } else { chan->stat_missed++; if (i>0 && chan->is_used == TRUE) { chan->missed++; /* delete stream if data is missing since a long time */ if (chan->missed>15) { chan->is_used=FALSE; ms_message("msconf: deleted contributing stream (pin=%i)", i); } /* couldn't we add confort noise for those outputs? */ } chan->has_contributed=FALSE; } } return; }