static void dec_process(MSFilter *f){
DecState *s=(DecState*)f->data;
mblk_t *inputMessage, *outputMessage;
struct BV16_Bit_Stream bs;
while((inputMessage=ms_queue_get(f->inputs[0]))!=NULL){
while(inputMessage->b_rptr<inputMessage->b_wptr) {
outputMessage = allocb(SIGNAL_FRAME_SIZE,0);
mblk_meta_copy(inputMessage, outputMessage);
BV16_BitUnPack((UWord8*)inputMessage->b_rptr, &bs);
BV16_Decode(&bs, &s->state, (short*)(outputMessage->b_wptr));
outputMessage->b_wptr+=SIGNAL_FRAME_SIZE;
inputMessage->b_rptr+=BITSTREAM_FRAME_SIZE;
ms_queue_put(f->outputs[0],outputMessage);
if (s->plc) ms_concealer_inc_sample_time(s->concealer,f->ticker->time, 5, 1);
}
freemsg(inputMessage);
}
// called every 10 ms
if (s->plc && ms_concealer_context_is_concealement_required(s->concealer, f->ticker->time)) {
int ms_concealed;
// need to conceal 10 ms
for (ms_concealed=0; ms_concealed<s->packet_ms_size; ms_concealed+=5){
outputMessage = allocb(SIGNAL_FRAME_SIZE,0);
BV16_PLC(&s->state,(short*)outputMessage->b_wptr);
outputMessage->b_wptr+=SIGNAL_FRAME_SIZE;
mblk_set_plc_flag(outputMessage, 1);
ms_queue_put(f->outputs[0],outputMessage);
}
ms_concealer_inc_sample_time(s->concealer,f->ticker->time,10, 0);
}
}
void SinkBase::cb_data(mblk_t *m)
{
uint32_t st = mblk_get_timestamp_info(m);
mblk_t *fm = dupmsg(m);
mblk_meta_copy(m, fm);
MSQueue *queue = post_handle(fm); // 此处 dupmsg(m) 将不会导致 m 被释放
bool first = true;
unsigned char *obuf = 0;
size_t olen = 0;
int index = 0;
size_t off = 0;
bool key = false;
while (mblk_t *om = ms_queue_get(queue)) {
int dlen = size_for(index, om);
obuf = (unsigned char*)realloc(obuf, off + dlen);
save_for(index, om, obuf + off);
if (index == 0)
key = is_key(index, om);
index++;
off += dlen;
// 处理时间戳回绕
if (first_frame_) {
first_frame_ = false;
}
else {
uint32_t delta = st - last_stamp_;
// 检查，是否乱序，乱序包直接扔掉！
if (delta > 0x80000000) {
fprintf(stderr, "??? maybe timestamp confusioned!, curr=%u, last=%u\n", st, last_stamp_);
return;
}
next_stamp_ += delta / payload_freq();
}
last_stamp_ = st;
freemsg(om);
}
if (cb_data_ && obuf) {
cb_data_(opaque_, next_stamp_, obuf, off, key);
}
if (obuf) free(obuf);
}
static void alaw_dec_process(MSFilter *obj){
mblk_t *m;
while((m=ms_queue_get(obj->inputs[0]))!=NULL){
mblk_t *o;
msgpullup(m,-1);
o=allocb((m->b_wptr-m->b_rptr)*2,0);
mblk_meta_copy(m, o);
for(;m->b_rptr<m->b_wptr;m->b_rptr++,o->b_wptr+=2){
*((int16_t*)(o->b_wptr))=Snack_Alaw2Lin(*m->b_rptr);
}
freemsg(m);
ms_queue_put(obj->outputs[0],o);
}
}
mblk_t *dupb(mblk_t *mp)
{
mblk_t *newm;
return_val_if_fail(mp->b_datap!=NULL,NULL);
return_val_if_fail(mp->b_datap->db_base!=NULL,NULL);
datab_ref(mp->b_datap);
newm=(mblk_t *) ortp_malloc(sizeof(mblk_t));
mblk_init(newm);
mblk_meta_copy(mp, newm);
newm->b_datap=mp->b_datap;
newm->b_rptr=mp->b_rptr;
newm->b_wptr=mp->b_wptr;
return newm;
}
static void process_cn(MSFilter *f, SenderData *d, uint32_t timestamp, mblk_t *im ){
if (d->cng_data.datasize>0){
rtp_header_t *rtp;
/* get CN payload type number */
int cn_pt=rtp_profile_find_payload_number(d->session->snd.profile, "CN", 8000, 1);
/* create the packet, payload type number is the one used for current codec */
mblk_t *m=rtp_session_create_packet(d->session, 12, d->cng_data.data, d->cng_data.datasize);
mblk_meta_copy(im, m);
/* replace payload type in RTP header */
rtp=(rtp_header_t*)m->b_rptr;
rtp->paytype = cn_pt;
rtp_session_sendm_with_ts(d->session,m,timestamp);
d->cng_data.datasize=0;
}
}
static int resample_channel_adapt(int in_nchannels, int out_nchannels, mblk_t *im, mblk_t **om) {
if ((in_nchannels == 2) && (out_nchannels == 1)) {
int msgsize = msgdsize(im) / 2;
*om = allocb(msgsize, 0);
for (; im->b_rptr < im->b_wptr; im->b_rptr += 4, (*om)->b_wptr += 2) {
*(int16_t *)(*om)->b_wptr = *(int16_t *)im->b_rptr;
}
return 1;
} else if ((in_nchannels == 1) && (out_nchannels == 2)) {
int msgsize = msgdsize(im) * 2;
*om = allocb(msgsize, 0);
for (; im->b_rptr < im->b_wptr; im->b_rptr += 2, (*om)->b_wptr += 4) {
((int16_t *)(*om)->b_wptr)[0] = *(int16_t *)im->b_rptr;
((int16_t *)(*om)->b_wptr)[1] = *(int16_t *)im->b_rptr;
}
mblk_meta_copy(im, *om);
return 1;
}
return 0;
}
static void decode(MSFilter *f, mblk_t *im) {
isac_decoder_t* obj = (isac_decoder_t*)f->data;
WebRtc_Word16 samples_nb, ret;
WebRtc_Word16 speech_type; // needed but not used..
// im is one packet from the encoder, so it's either 30 or 60 ms of audio
ret = WebRtcIsacfix_ReadFrameLen( (const WebRtc_Word16*)im->b_rptr, &samples_nb);
// ms_message("WebRtcIsacfix_ReadFrameLen -> %d", samples_nb);
if( ret == 0 ) {
mblk_t *om = allocb(samples_nb*2, 0);
mblk_meta_copy(im, om);
obj->ptime = (samples_nb == ISAC_30MS_SAMPLE_COUNT) ? 30 : 60; // update ptime
// ms_message("DECODED om datap @%p", om->b_datap);
ret = WebRtcIsacfix_Decode(obj->isac,
(const WebRtc_UWord16*)im->b_rptr,
(im->b_wptr - im->b_rptr),
(WebRtc_Word16*)om->b_wptr,
&speech_type );
if( ret < 0 ) {
ms_error( "WebRtcIsacfix_Decode error: %d", WebRtcIsacfix_GetErrorCode(obj->isac) );
freeb(om);
} else {
// ms_message("Decoded %d samples", ret);
om->b_wptr+= ret*2;
mblk_set_plc_flag(om, 0);
ms_queue_put(f->outputs[0], om);
}
} else {
ms_error( "WebRtcIsacfix_ReadFrameLen failed: %d", WebRtcIsacfix_GetErrorCode(obj->isac) );
}
obj->seq_nb = mblk_get_cseq(im);
ms_concealer_inc_sample_time(obj->plc_ctx, f->ticker->time, obj->ptime, TRUE);
return;
}
static void ms_opus_dec_process(MSFilter *f) {
OpusDecData *d = (OpusDecData *)f->data;
mblk_t *im;
mblk_t *om;
int frames;
if (!d->state) ms_queue_flush(f->inputs[0]);
/* decode available packets */
while ((im = ms_queue_get(f->inputs[0])) != NULL) {
om = allocb(5760 * d->channels * SIGNAL_SAMPLE_SIZE, 0); /* 5760 is the maximum number of sample in a packet (120ms at 48KHz) */
frames = opus_decode(d->state, (const unsigned char *)im->b_rptr, im->b_wptr - im->b_rptr, (opus_int16 *)om->b_wptr, 5760, 0);
if (frames < 0) {
ms_warning("Opus decoder error: %s", opus_strerror(frames));
freemsg(om);
} else {
d->lastPacketLength = frames; // store the packet length for eventual PLC if next two packets are missing
om->b_wptr += frames * d->channels * SIGNAL_SAMPLE_SIZE;
mblk_meta_copy(im,om);
ms_queue_put(f->outputs[0], om);
/*ms_message("Opus: outputing a normal frame of %i bytes (%i samples,%i ms)",(int)(om->b_wptr-om->b_rptr),frames,frames*1000/d->samplerate);*/
d->sequence_number = mblk_get_cseq(im); // used to get eventual FEC information if next packet is missing
ms_concealer_inc_sample_time(d->concealer,f->ticker->time, frames*1000/d->samplerate, 1);
}
freemsg(im);
}
/* Concealment if needed */
if (ms_concealer_context_is_concealement_required(d->concealer, f->ticker->time)) {
int imLength = 0;
uint8_t *payload = NULL;
im = NULL;
// try fec : info are stored in the next packet, do we have it?
if (d->rtp_picker_context.picker) {
/* FEC information is in the next packet, last valid packet was d->sequence_number, the missing one shall then be d->sequence_number+1, so check jitter buffer for d->sequence_number+2 */
/* but we may have the n+1 packet in the buffer and adaptative jitter control keeping it for later, in that case, just go for PLC */
if (d->rtp_picker_context.picker(&d->rtp_picker_context,d->sequence_number+1) == NULL) { /* missing packet is really missing */
im = d->rtp_picker_context.picker(&d->rtp_picker_context,d->sequence_number+2); /* try to get n+2 */
if (im) {
imLength=rtp_get_payload(im,&payload);
}
}
}
om = allocb(5760 * d->channels * SIGNAL_SAMPLE_SIZE, 0); /* 5760 is the maximum number of sample in a packet (120ms at 48KHz) */
/* call to the decoder, we'll have either FEC or PLC, do it on the same length that last received packet */
if (payload) { // found frame to try FEC
d->statsfec++;
frames = opus_decode(d->state, payload, imLength, (opus_int16 *)om->b_wptr, d->lastPacketLength, 1);
} else { // do PLC: PLC doesn't seem to be able to generate more than 960 samples (20 ms at 48000 Hz), get PLC until we have the correct number of sample
//frames = opus_decode(d->state, NULL, 0, (opus_int16 *)om->b_wptr, d->lastPacketLength, 0); // this should have work if opus_decode returns the requested number of samples
d->statsplc++;
frames = 0;
while (frames < d->lastPacketLength) {
frames += opus_decode(d->state, NULL, 0, (opus_int16 *)(om->b_wptr + (frames*d->channels*SIGNAL_SAMPLE_SIZE)), d->lastPacketLength-frames, 0);
}
}
if (frames < 0) {
ms_warning("Opus decoder error in concealment: %s", opus_strerror(frames));
freemsg(om);
} else {
om->b_wptr += frames * d->channels * SIGNAL_SAMPLE_SIZE;
/*ms_message("Opus: outputing a PLC frame of %i bytes (%i samples,%i ms)",(int)(om->b_wptr-om->b_rptr),frames,frames*1000/d->samplerate);*/
mblk_set_plc_flag(om,TRUE);
ms_queue_put(f->outputs[0], om);
d->sequence_number++;
ms_concealer_inc_sample_time(d->concealer,f->ticker->time, frames*1000/d->samplerate, 0);
}
}
}
static void dec_process(MSFilter *f){
DecState *s=(DecState*)f->data;
mblk_t *im;
mblk_t *om;
int err=-2;
SpeexBits bits;
int bytes=s->frsz*2;
bool_t bits_initd=FALSE;
while((im=ms_queue_get(f->inputs[0]))!=NULL){
int rem_bits=(im->b_wptr-im->b_rptr)*8;
if (!bits_initd) {
speex_bits_init(&bits);
bits_initd=TRUE;
}else speex_bits_reset(&bits);
speex_bits_read_from(&bits,(char*)im->b_rptr,im->b_wptr-im->b_rptr);
/* support for multiple frame in one RTP packet */
do{
om=allocb(bytes,0);
mblk_meta_copy(im, om);
err=speex_decode_int(s->state,&bits,(int16_t*)om->b_wptr);
om->b_wptr+=bytes;
if (err==0){
ms_queue_put(f->outputs[0],om);
if (s->sample_time==0) s->sample_time=f->ticker->time;
s->sample_time+=20;
if (s->plc_count>0){
// ms_warning("Did speex packet loss concealment during %i ms",s->plc_count*20);
s->plc_count=0;
}
}else {
if (err==-1)
ms_warning("speex end of stream");
else if (err==-2)
ms_warning("speex corrupted stream");
freemsg(om);
}
}while((rem_bits= speex_bits_remaining(&bits))>10);
freemsg(im);
}
if (s->plc && s->sample_time!=0 && f->ticker->time>=s->sample_time){
/* we should output a frame but no packet were decoded
thus do packet loss concealment*/
om=allocb(bytes,0);
err=speex_decode_int(s->state,NULL,(int16_t*)om->b_wptr);
om->b_wptr+=bytes;
mblk_set_plc_flag(om, 1);
ms_queue_put(f->outputs[0],om);
s->sample_time+=20;
s->plc_count++;
if (s->plc_count>=plc_max){
s->sample_time=0;
}
}
if (bits_initd) speex_bits_destroy(&bits);
}
static void _sender_process(MSFilter * f)
{
SenderData *d = (SenderData *) f->data;
RtpSession *s = d->session;
mblk_t *im;
uint32_t timestamp;
if (d->relay_session_id_size>0 &&
( (f->ticker->time-d->last_rsi_time)>5000 || d->last_rsi_time==0) ) {
ms_message("relay session id sent in RTCP APP");
rtp_session_send_rtcp_APP(s,0,"RSID",(const uint8_t *)d->relay_session_id,d->relay_session_id_size);
d->last_rsi_time=f->ticker->time;
}
ms_filter_lock(f);
im = ms_queue_get(f->inputs[0]);
do {
mblk_t *header;
timestamp = get_cur_timestamp(f, im);
if (d->dtmf != 0 && !d->skip) {
ms_debug("prepare to send RFC2833 dtmf.");
d->skip_until = timestamp + d->dtmf_duration;
d->dtmf_ts_cur=timestamp;
d->skip = TRUE;
}
if (d->skip) {
uint32_t origin_ts=d->skip_until-d->dtmf_duration;
if (RTP_TIMESTAMP_IS_NEWER_THAN(timestamp,d->dtmf_ts_cur)){
ms_debug("Sending RFC2833 packet, start_timestamp=%u, dtmf_ts_cur=%u",origin_ts,d->dtmf_ts_cur);
send_dtmf(f, origin_ts);
}
}
if (im){
if (d->skip == FALSE && d->mute==FALSE){
header = rtp_session_create_packet(s, 12, NULL, 0);
rtp_set_markbit(header, mblk_get_marker_info(im));
header->b_cont = im;
mblk_meta_copy(im, header);
rtp_session_sendm_with_ts(s, header, timestamp);
} else if (d->mute==TRUE && d->skip == FALSE) {
process_cn(f, d, timestamp, im);
freemsg(im);
//Send STUN packet as RTP keep alive
check_stun_sending(f);
}else{
freemsg(im);
}
} else if (d->skip == FALSE) {
// Send STUN packet as RTP keep alive even if there is no input
check_stun_sending(f);
}
}while ((im = ms_queue_get(f->inputs[0])) != NULL);
if (d->last_sent_time == -1) {
check_stun_sending(f);
}
/*every second, compute output bandwidth*/
if (f->ticker->time % 1000 == 0) rtp_session_compute_send_bandwidth(d->session);
ms_filter_unlock(f);
}
static void dec_process(MSFilter *f){
DecState *s=(DecState*)f->data;
mblk_t *im,*om;
int nbytes;
float samples[BLOCKL_MAX]={0};
int i;
while ((im=ms_queue_get(f->inputs[0]))!=NULL){
nbytes=msgdsize(im);
if (nbytes==0 || (nbytes%38!=0 && nbytes%50!=0)){
freemsg(im);
continue;
}
if (nbytes%38==0 && s->nbytes!=NO_OF_BYTES_20MS)
{
/* not yet configured, or misconfigured */
s->ms_per_frame=20;
s->nbytes=NO_OF_BYTES_20MS;
s->nsamples=BLOCKL_20MS;
s->ready=TRUE;
initDecode(&s->ilbc_dec,s->ms_per_frame,s->postfilter);
}
else if (nbytes%50==0 && s->nbytes!=NO_OF_BYTES_30MS)
{
/* not yet configured, or misconfigured */
s->ms_per_frame=30;
s->nbytes=NO_OF_BYTES_30MS;
s->nsamples=BLOCKL_30MS;
s->ready=TRUE;
initDecode(&s->ilbc_dec,s->ms_per_frame,s->postfilter);
}
if (s->nbytes>0 && nbytes>=s->nbytes){
int frame_per_packet = nbytes/s->nbytes;
int k;
int plctime;
for (k=0;k<frame_per_packet;k++)
{
om=allocb(s->nsamples*2,0);
iLBC_decode(samples,(uint8_t*)im->b_rptr+(k*s->nbytes),&s->ilbc_dec,1);
for (i=0;i<s->nsamples;i++,om->b_wptr+=2){
*((int16_t*)om->b_wptr)=samples[i];
}
mblk_meta_copy(im,om);
ms_queue_put(f->outputs[0],om);
}
if (s->plcctx){
plctime=ms_concealer_inc_sample_time(s->plcctx,f->ticker->time,frame_per_packet*s->ms_per_frame,1);
if (plctime>0){
ms_warning("ilbc: did plc during %i ms",plctime);
}
}
}else{
ms_warning("bad iLBC frame !");
}
freemsg(im);
}
if (s->plcctx && s->ready && ms_concealer_context_is_concealement_required(s->plcctx,f->ticker->time)){
om=allocb(s->nsamples*2,0);
iLBC_decode(samples,(uint8_t*)NULL,&s->ilbc_dec,0 /*PLC*/);
for (i=0;i<s->nsamples;i++,om->b_wptr+=2){
*((int16_t*)om->b_wptr)=samples[i];
}
mblk_set_plc_flag(om,TRUE);
ms_queue_put(f->outputs[0],om);
ms_concealer_inc_sample_time(s->plcctx,f->ticker->time,s->ms_per_frame,0);
}
}
static void resample_process_ms2(MSFilter *obj){
ResampleData *dt=(ResampleData*)obj->data;
mblk_t *im, *om = NULL, *om_chan = NULL;
if (dt->output_rate==dt->input_rate){
while((im=ms_queue_get(obj->inputs[0]))!=NULL){
if (resample_channel_adapt(dt->in_nchannels, dt->out_nchannels, im, &om) == 0) {
ms_queue_put(obj->outputs[0], im);
} else {
ms_queue_put(obj->outputs[0], om);
freemsg(im);
}
}
return;
}
ms_filter_lock(obj);
if (dt->handle!=NULL){
unsigned int inrate=0, outrate=0;
speex_resampler_get_rate(dt->handle,&inrate,&outrate);
if (inrate!=dt->input_rate || outrate!=dt->output_rate){
speex_resampler_destroy(dt->handle);
dt->handle=0;
}
}
if (dt->handle==NULL){
int err=0;
dt->handle=speex_resampler_init(dt->in_nchannels, dt->input_rate, dt->output_rate, SPEEX_RESAMPLER_QUALITY_VOIP, &err);
}
while((im=ms_queue_get(obj->inputs[0]))!=NULL){
unsigned int inlen=(im->b_wptr-im->b_rptr)/(2*dt->in_nchannels);
unsigned int outlen=((inlen*dt->output_rate)/dt->input_rate)+1;
unsigned int inlen_orig=inlen;
om=allocb(outlen*2*dt->in_nchannels,0);
mblk_meta_copy(im, om);
if (dt->in_nchannels==1){
speex_resampler_process_int(dt->handle,
0,
(int16_t*)im->b_rptr,
&inlen,
(int16_t*)om->b_wptr,
&outlen);
}else{
speex_resampler_process_interleaved_int(dt->handle,
(int16_t*)im->b_rptr,
&inlen,
(int16_t*)om->b_wptr,
&outlen);
}
if (inlen_orig!=inlen){
ms_error("Bug in resampler ! only %u samples consumed instead of %u, out=%u",
inlen,inlen_orig,outlen);
}
om->b_wptr+=outlen*2*dt->in_nchannels;
mblk_set_timestamp_info(om,dt->ts);
dt->ts+=outlen;
if (resample_channel_adapt(dt->in_nchannels, dt->out_nchannels, om, &om_chan) == 0) {
ms_queue_put(obj->outputs[0], om);
} else {
ms_queue_put(obj->outputs[0], om_chan);
freemsg(om);
}
freemsg(im);
}
ms_filter_unlock(obj);
}
