static void enc_process(MSFilter *f) { mblk_t *im; uint64_t timems = f->ticker->time; uint32_t timestamp = (uint32_t)(timems*90); EncState *s = (EncState *)f->data; unsigned int flags = 0; vpx_codec_err_t err; MSPicture yuv; bool_t is_ref_frame=FALSE; ms_filter_lock(f); #ifdef AVPF_DEBUG ms_message("VP8 enc_process:"); #endif if (!s->ready) { ms_queue_flush(f->inputs[0]); ms_filter_unlock(f); return; } if ((im = ms_queue_peek_last(f->inputs[0])) != NULL) { vpx_image_t img; flags = 0; ms_yuv_buf_init_from_mblk(&yuv, im); vpx_img_wrap(&img, VPX_IMG_FMT_I420, s->vconf.vsize.width, s->vconf.vsize.height, 1, yuv.planes[0]); if ((s->avpf_enabled != TRUE) && ms_video_starter_need_i_frame(&s->starter, f->ticker->time)) { s->force_keyframe = TRUE; } if (s->force_keyframe == TRUE) { ms_message("Forcing vp8 key frame for filter [%p]", f); flags = VPX_EFLAG_FORCE_KF; } else if (s->avpf_enabled == TRUE) { if (s->frame_count == 0) s->force_keyframe = TRUE; enc_fill_encoder_flags(s, &flags); } #ifdef AVPF_DEBUG ms_message("VP8 encoder frames state:"); ms_message("\tgolden: count=%" PRIi64 ", picture_id=0x%04x, ack=%s", s->frames_state.golden.count, s->frames_state.golden.picture_id, (s->frames_state.golden.acknowledged == TRUE) ? "Y" : "N"); ms_message("\taltref: count=%" PRIi64 ", picture_id=0x%04x, ack=%s", s->frames_state.altref.count, s->frames_state.altref.picture_id, (s->frames_state.altref.acknowledged == TRUE) ? "Y" : "N"); #endif err = vpx_codec_encode(&s->codec, &img, s->frame_count, 1, flags, 1000000LL/(2*(int)s->vconf.fps)); /*encoder has half a framerate interval to encode*/ if (err) { ms_error("vpx_codec_encode failed : %d %s (%s)\n", err, vpx_codec_err_to_string(err), vpx_codec_error_detail(&s->codec)); } else { vpx_codec_iter_t iter = NULL; const vpx_codec_cx_pkt_t *pkt; MSList *list = NULL; /* Update the frames state. */ is_ref_frame=FALSE; if (flags & VPX_EFLAG_FORCE_KF) { enc_mark_reference_frame_as_sent(s, VP8_GOLD_FRAME); enc_mark_reference_frame_as_sent(s, VP8_ALTR_FRAME); s->frames_state.golden.is_independant=TRUE; s->frames_state.altref.is_independant=TRUE; s->frames_state.last_independent_frame=s->frame_count; s->force_keyframe = FALSE; is_ref_frame=TRUE; }else if (flags & VP8_EFLAG_FORCE_GF) { enc_mark_reference_frame_as_sent(s, VP8_GOLD_FRAME); is_ref_frame=TRUE; }else if (flags & VP8_EFLAG_FORCE_ARF) { enc_mark_reference_frame_as_sent(s, VP8_ALTR_FRAME); is_ref_frame=TRUE; }else if (flags & VP8_EFLAG_NO_REF_LAST) { enc_mark_reference_frame_as_sent(s, VP8_LAST_FRAME); is_ref_frame=is_reconstruction_frame_sane(s,flags); } if (is_frame_independent(flags)){ s->frames_state.last_independent_frame=s->frame_count; } /* Pack the encoded frame. */ while( (pkt = vpx_codec_get_cx_data(&s->codec, &iter)) ) { if ((pkt->kind == VPX_CODEC_CX_FRAME_PKT) && (pkt->data.frame.sz > 0)) { Vp8RtpFmtPacket *packet = ms_new0(Vp8RtpFmtPacket, 1); packet->m = allocb(pkt->data.frame.sz, 0); memcpy(packet->m->b_wptr, pkt->data.frame.buf, pkt->data.frame.sz); packet->m->b_wptr += pkt->data.frame.sz; mblk_set_timestamp_info(packet->m, timestamp); packet->pd = ms_new0(Vp8RtpFmtPayloadDescriptor, 1); packet->pd->start_of_partition = TRUE; packet->pd->non_reference_frame = s->avpf_enabled && !is_ref_frame; if (s->avpf_enabled == TRUE) { packet->pd->extended_control_bits_present = TRUE; packet->pd->pictureid_present = TRUE; packet->pd->pictureid = s->picture_id; } else { packet->pd->extended_control_bits_present = FALSE; packet->pd->pictureid_present = FALSE; } if (s->flags & VPX_CODEC_USE_OUTPUT_PARTITION) { packet->pd->pid = (uint8_t)pkt->data.frame.partition_id; if (!(pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT)) { mblk_set_marker_info(packet->m, TRUE); } } else { packet->pd->pid = 0; mblk_set_marker_info(packet->m, TRUE); } list = ms_list_append(list, packet); } } #ifdef AVPF_DEBUG ms_message("VP8 encoder picture_id=%i ***| %s | %s | %s | %s", (int)s->picture_id, (flags & VPX_EFLAG_FORCE_KF) ? "KF " : (flags & VP8_EFLAG_FORCE_GF) ? "GF " : (flags & VP8_EFLAG_FORCE_ARF) ? "ARF" : " ", (flags & VP8_EFLAG_NO_REF_GF) ? "NOREFGF" : " ", (flags & VP8_EFLAG_NO_REF_ARF) ? "NOREFARF" : " ", (flags & VP8_EFLAG_NO_REF_LAST) ? "NOREFLAST" : " "); #endif vp8rtpfmt_packer_process(&s->packer, list, f->outputs[0], f->factory); /* Handle video starter if AVPF is not enabled. */ s->frame_count++; if ((s->avpf_enabled != TRUE) && (s->frame_count == 1)) { ms_video_starter_first_frame(&s->starter, f->ticker->time); } /* Increment the pictureID. */ s->picture_id++; #ifdef PICTURE_ID_ON_16_BITS if (s->picture_id == 0) s->picture_id = 0x8000; #else if (s->picture_id == 0x0080) s->picture_id = 0; #endif } } ms_filter_unlock(f); ms_queue_flush(f->inputs[0]); }
static void jpg_process(MSFilter *f){ JpegWriter *s=(JpegWriter*)f->data; ms_filter_lock(f); if (s->file!=NULL && s->codec!=NULL){ MSPicture yuvbuf, yuvjpeg; mblk_t *m=ms_queue_peek_last(f->inputs[0]); if (ms_yuv_buf_init_from_mblk(&yuvbuf,m)==0){ int error,got_pict; int comp_buf_sz=msgdsize(m); uint8_t *comp_buf=(uint8_t*)ms_malloc0(comp_buf_sz); mblk_t *jpegm; struct SwsContext *sws_ctx; struct AVPacket packet; AVCodecContext *avctx=avcodec_alloc_context3(s->codec); memset(&packet, 0, sizeof(packet)); avctx->width=yuvbuf.w; avctx->height=yuvbuf.h; avctx->time_base.num = 1; avctx->time_base.den =1; avctx->pix_fmt=AV_PIX_FMT_YUVJ420P; error=avcodec_open2(avctx,s->codec,NULL); if (error!=0) { ms_error("avcodec_open() failed: %i",error); cleanup(s,NULL, FALSE); av_free(avctx); goto end; } sws_ctx=sws_getContext(avctx->width,avctx->height,AV_PIX_FMT_YUV420P, avctx->width,avctx->height,avctx->pix_fmt,SWS_FAST_BILINEAR,NULL, NULL, NULL); if (sws_ctx==NULL) { ms_error(" sws_getContext() failed."); cleanup(s,avctx, FALSE); goto end; } jpegm=ms_yuv_buf_alloc (&yuvjpeg,avctx->width, avctx->height); #if LIBSWSCALE_VERSION_INT >= AV_VERSION_INT(0,9,0) if (sws_scale(sws_ctx,(const uint8_t *const*)yuvbuf.planes,yuvbuf.strides,0,avctx->height,yuvjpeg.planes,yuvjpeg.strides)<0){ #else if (sws_scale(sws_ctx,(uint8_t **)yuvbuf.planes,yuvbuf.strides,0,avctx->height,yuvjpeg.planes,yuvjpeg.strides)<0){ #endif ms_error("sws_scale() failed."); sws_freeContext(sws_ctx); cleanup(s,avctx, FALSE); freemsg(jpegm); goto end; } sws_freeContext(sws_ctx); av_frame_unref(s->pict); avpicture_fill((AVPicture*)s->pict,(uint8_t*)jpegm->b_rptr,avctx->pix_fmt,avctx->width,avctx->height); packet.data=comp_buf; packet.size=comp_buf_sz; error=avcodec_encode_video2(avctx, &packet, s->pict, &got_pict); if (error<0){ ms_error("Could not encode jpeg picture."); }else{ if (fwrite(comp_buf,packet.size,1,s->file)>0){ ms_message("Snapshot done"); }else{ ms_error("Error writing snapshot."); } } ms_free(comp_buf); cleanup(s,avctx, TRUE); freemsg(jpegm); } goto end; } end: ms_filter_unlock(f); ms_queue_flush(f->inputs[0]); } static MSFilterMethod jpg_methods[]={ { MS_JPEG_WRITER_TAKE_SNAPSHOT, take_snapshot }, { 0,NULL} }; #ifndef _MSC_VER MSFilterDesc ms_jpeg_writer_desc={ .id=MS_JPEG_WRITER_ID, .name="MSJpegWriter", .text="Take a video snapshot as jpg file", .category=MS_FILTER_OTHER, .ninputs=1, .noutputs=0, .init=jpg_init, .process=jpg_process, .uninit=jpg_uninit, .methods=jpg_methods }; #else MSFilterDesc ms_jpeg_writer_desc={ MS_JPEG_WRITER_ID, "MSJpegWriter", "Take a video snapshot as jpg file", MS_FILTER_OTHER, NULL, 1, 0, jpg_init, NULL, jpg_process, NULL, jpg_uninit, jpg_methods }; #endif MS_FILTER_DESC_EXPORT(ms_jpeg_writer_desc)
static void _sender_process(MSFilter * f) { SenderData *d = (SenderData *) f->data; RtpSession *s = d->session; mblk_t *im; uint32_t timestamp; if (d->relay_session_id_size>0 && ( (f->ticker->time-d->last_rsi_time)>5000 || d->last_rsi_time==0) ) { ms_message("relay session id sent in RTCP APP"); rtp_session_send_rtcp_APP(s,0,"RSID",(const uint8_t *)d->relay_session_id,d->relay_session_id_size); d->last_rsi_time=f->ticker->time; } ms_filter_lock(f); im = ms_queue_get(f->inputs[0]); do { mblk_t *header; timestamp = get_cur_timestamp(f, im); if (d->dtmf != 0 && !d->skip) { ms_debug("prepare to send RFC2833 dtmf."); d->skip_until = timestamp + d->dtmf_duration; d->dtmf_ts_cur=timestamp; d->skip = TRUE; } if (d->skip) { uint32_t origin_ts=d->skip_until-d->dtmf_duration; if (RTP_TIMESTAMP_IS_NEWER_THAN(timestamp,d->dtmf_ts_cur)){ ms_debug("Sending RFC2833 packet, start_timestamp=%u, dtmf_ts_cur=%u",origin_ts,d->dtmf_ts_cur); send_dtmf(f, origin_ts); } } if (im){ if (d->skip == FALSE && d->mute==FALSE){ header = rtp_session_create_packet(s, 12, NULL, 0); rtp_set_markbit(header, mblk_get_marker_info(im)); header->b_cont = im; mblk_meta_copy(im, header); rtp_session_sendm_with_ts(s, header, timestamp); } else if (d->mute==TRUE && d->skip == FALSE) { process_cn(f, d, timestamp, im); freemsg(im); //Send STUN packet as RTP keep alive check_stun_sending(f); }else{ freemsg(im); } } else if (d->skip == FALSE) { // Send STUN packet as RTP keep alive even if there is no input check_stun_sending(f); } }while ((im = ms_queue_get(f->inputs[0])) != NULL); if (d->last_sent_time == -1) { check_stun_sending(f); } /*every second, compute output bandwidth*/ if (f->ticker->time % 1000 == 0) rtp_session_compute_send_bandwidth(d->session); ms_filter_unlock(f); }
static void size_conv_process(MSFilter *f){ SizeConvState *s=(SizeConvState*)f->data; YuvBuf inbuf; mblk_t *im; int cur_frame; ms_filter_lock(f); if (s->frame_count==-1){ s->start_time=(float)f->ticker->time; s->frame_count=0; } while((im=ms_queue_get(f->inputs[0]))!=NULL ){ putq(&s->rq, im); } cur_frame=(int)((f->ticker->time-s->start_time)*s->fps/1000.0); if (cur_frame<=s->frame_count && s->fps>=0) { /* too much frame */ while(s->rq.q_mcount>1){ ms_message("MSSizeConv: extra frame removed."); im=getq(&s->rq); freemsg(im); } ms_filter_unlock(f); return; } if (cur_frame>s->frame_count && s->fps>=0) { /*keep the most recent frame if several frames have been captured */ while(s->rq.q_mcount>1){ ms_message("MSSizeConv: extra frame removed."); im=getq(&s->rq); freemsg(im); } } while((im=getq(&s->rq))!=NULL ){ if (ms_yuv_buf_init_from_mblk(&inbuf,im)==0){ if (inbuf.w==s->target_vsize.width && inbuf.h==s->target_vsize.height){ ms_queue_put(f->outputs[0],im); }else{ MSScalerContext *sws_ctx=get_resampler(s,inbuf.w,inbuf.h); mblk_t *om=size_conv_alloc_mblk(s); if (ms_scaler_process(sws_ctx,inbuf.planes,inbuf.strides,s->outbuf.planes, s->outbuf.strides)<0){ ms_error("MSSizeConv: error in ms_scaler_process()."); freemsg(om); }else{ ms_queue_put(f->outputs[0],om); } freemsg(im); } s->frame_count++; }else{ ms_warning("size_conv_process(): bad buffer."); freemsg(im); } } ms_filter_unlock(f); }
static void ms_opus_enc_preprocess(MSFilter *f) { int error; int opusComplexity = -1; const char *env = NULL; OpusEncData *d = (OpusEncData *)f->data; /* create the encoder */ d->state = opus_encoder_create(d->samplerate, d->channels, d->application, &error); if (error != OPUS_OK) { ms_error("Opus encoder creation failed: %s", opus_strerror(error)); return; } #ifndef MS2_WINDOWS_UNIVERSAL env = getenv("MS2_OPUS_COMPLEXITY"); #endif if (env != NULL) { opusComplexity = atoi(env); if (opusComplexity < -1) opusComplexity = -1; /*our default value*/ if (opusComplexity > 10) opusComplexity = 10; } if (opusComplexity == -1){ #if defined(__arm__) || defined(_M_ARM) int cpucount = ms_factory_get_cpu_count(f->factory); if (cpucount == 1){ opusComplexity = 0; /* set complexity to 0 for single processor arm devices */ }else if (cpucount == 2) { opusComplexity = 5; } #endif } if (opusComplexity != -1){ ms_message("Set Opus complexity to %d", opusComplexity); opus_encoder_ctl(d->state, OPUS_SET_COMPLEXITY(opusComplexity)); } /*otherwise we let opus with its default value, which is 9*/ error = opus_encoder_ctl(d->state, OPUS_SET_PACKET_LOSS_PERC(10)); if (error != OPUS_OK) { ms_error("Could not set default loss percentage to opus encoder: %s", opus_strerror(error)); } /* set the encoder parameters: VBR, IN_BAND_FEC, DTX and bitrate settings */ ms_opus_enc_set_vbr(f); ms_opus_enc_set_inbandfec(f); ms_opus_enc_set_dtx(f); /* if decoder prefers mono signal, force encoder to output mono signal */ if (d->stereo == 0) { error = opus_encoder_ctl(d->state, OPUS_SET_FORCE_CHANNELS(1)); if (error != OPUS_OK) { ms_error("could not force mono channel to opus encoder: %s", opus_strerror(error)); } if (d->channels == 2) ms_message("Opus encoder configured to encode mono despite it is feed with stereo."); }else if (d->channels == 2){ ms_message("Opus encoder configured to encode stereo."); } ms_filter_lock(f); // set bitrate wasn't call, compute it with the default network bitrate (36000) if (d->bitrate==-1) { compute_max_bitrate(d, 0); } apply_max_bitrate(d); ms_filter_unlock(f); }
static void ms_opus_enc_process(MSFilter *f) { OpusEncData *d = (OpusEncData *)f->data; OpusRepacketizer *repacketizer = NULL; mblk_t *om = NULL; int packet_size, pcm_buffer_size; int max_frame_byte_size, ptime = 20; int frame_count = 0, frame_size = 0; opus_int32 total_length = 0; uint8_t *repacketizer_frame_buffer[MAX_INPUT_FRAMES]; int i; ms_filter_lock(f); ptime = d->ptime; packet_size = d->samplerate * ptime / 1000; /* in samples */ ms_filter_unlock(f); switch (ptime) { case 10: frame_size = d->samplerate * 10 / 1000; frame_count = 1; break; case 20: frame_size = d->samplerate * 20 / 1000; frame_count = 1; break; case 40: frame_size = d->samplerate * 40 / 1000; frame_count = 1; break; case 60: frame_size = d->samplerate * 60 / 1000; frame_count = 1; break; case 80: frame_size = d->samplerate * 40 / 1000; frame_count = 2; break; case 100: frame_size = d->samplerate * 20 / 1000; frame_count = 5; break; case 120: frame_size = d->samplerate * 60 / 1000; frame_count = 2; break; default: frame_size = d->samplerate * 20 / 1000; frame_count = 1; } max_frame_byte_size = MAX_BYTES_PER_MS * ptime/frame_count; pcm_buffer_size = d->channels * frame_size * SIGNAL_SAMPLE_SIZE; if (pcm_buffer_size > d->pcmbufsize){ if (d->pcmbuffer) ms_free(d->pcmbuffer); d->pcmbuffer = ms_malloc(pcm_buffer_size); d->pcmbufsize = pcm_buffer_size; } for (i=0; i<MAX_INPUT_FRAMES; i++) { repacketizer_frame_buffer[i]=NULL; } ms_bufferizer_put_from_queue(d->bufferizer, f->inputs[0]); while (ms_bufferizer_get_avail(d->bufferizer) >= (d->channels * packet_size * SIGNAL_SAMPLE_SIZE)) { opus_int32 ret = 0; if (frame_count == 1) { /* One Opus frame, not using the repacketizer */ om = allocb(max_frame_byte_size, 0); ms_bufferizer_read(d->bufferizer, d->pcmbuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels); ret = opus_encode(d->state, (opus_int16 *)d->pcmbuffer, frame_size, om->b_wptr, max_frame_byte_size); if (ret < 0) { freemsg(om); om=NULL; ms_error("Opus encoder error: %s", opus_strerror(ret)); break; } else { total_length = ret; om->b_wptr += total_length; } } else if(frame_count > 1) { /* We have multiple Opus frames we will use the opus repacketizer */ repacketizer = opus_repacketizer_create(); opus_repacketizer_init(repacketizer); /* Do not include FEC/LBRR in any frame after the first one since it will be sent with the previous one */ ret = opus_encoder_ctl(d->state, OPUS_SET_INBAND_FEC(0)); if (ret != OPUS_OK) { ms_error("could not set inband FEC to opus encoder: %s", opus_strerror(ret)); } for (i=0; i<frame_count; i++) { if(frame_count == i+1){ /* if configured, reactivate FEC on the last frame to tell the encoder he should restart saving LBRR frames */ ret = opus_encoder_ctl(d->state, OPUS_SET_INBAND_FEC(d->useinbandfec)); if (ret != OPUS_OK) { ms_error("could not set inband FEC to opus encoder: %s", opus_strerror(ret)); } } if (!repacketizer_frame_buffer[i]) repacketizer_frame_buffer[i] = ms_malloc(max_frame_byte_size); /* the repacketizer need the pointer to packet to remain valid, so we shall have a buffer for each coded frame */ ms_bufferizer_read(d->bufferizer, d->pcmbuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels); ret = opus_encode(d->state, (opus_int16 *)d->pcmbuffer, frame_size, repacketizer_frame_buffer[i], max_frame_byte_size); if (ret < 0) { ms_error("Opus encoder error: %s", opus_strerror(ret)); break; } else if (ret > 0) { int err = opus_repacketizer_cat(repacketizer, repacketizer_frame_buffer[i], ret); /* add the encoded frame into the current packet */ if (err != OPUS_OK) { ms_error("Opus repacketizer error: %s", opus_strerror(err)); break; } total_length += ret; } } om = allocb(total_length + frame_count + 1, 0); /* opus repacketizer API: allocate at least number of frame + size of all data added before */ ret = opus_repacketizer_out(repacketizer, om->b_wptr, total_length+frame_count); if(ret < 0){ freemsg(om); om=NULL; ms_error("Opus repacketizer out error: %s", opus_strerror(ret)); } else { om->b_wptr += ret; } opus_repacketizer_destroy(repacketizer); for (i=0; i<frame_count; i++) { if (repacketizer_frame_buffer[i] != NULL) { ms_free(repacketizer_frame_buffer[i]); } } } if(om) { /* we have an encoded output message */ mblk_set_timestamp_info(om, d->ts); ms_bufferizer_fill_current_metas(d->bufferizer, om); ms_queue_put(f->outputs[0], om); d->ts += packet_size*48000/d->samplerate; /* RFC payload RTP opus 03 - section 4: RTP timestamp multiplier : WARNING works only with sr at 48000 */ total_length = 0; } } }
static int h264_dec_enable_avpf(MSFilter *f, const bool_t *enable) { ms_filter_lock(f); ((VTH264DecCtx *)f->data)->enable_avpf = *enable; ms_filter_unlock(f); return 0; }
static void resample_process_ms2(MSFilter *obj){ ResampleData *dt=(ResampleData*)obj->data; mblk_t *im, *om = NULL, *om_chan = NULL; if (dt->output_rate==dt->input_rate){ while((im=ms_queue_get(obj->inputs[0]))!=NULL){ if (resample_channel_adapt(dt->in_nchannels, dt->out_nchannels, im, &om) == 0) { ms_queue_put(obj->outputs[0], im); } else { ms_queue_put(obj->outputs[0], om); freemsg(im); } } return; } ms_filter_lock(obj); if (dt->handle!=NULL){ unsigned int inrate=0, outrate=0; speex_resampler_get_rate(dt->handle,&inrate,&outrate); if (inrate!=dt->input_rate || outrate!=dt->output_rate){ speex_resampler_destroy(dt->handle); dt->handle=0; } } if (dt->handle==NULL){ int err=0; dt->handle=speex_resampler_init(dt->in_nchannels, dt->input_rate, dt->output_rate, SPEEX_RESAMPLER_QUALITY_VOIP, &err); } while((im=ms_queue_get(obj->inputs[0]))!=NULL){ unsigned int inlen=(im->b_wptr-im->b_rptr)/(2*dt->in_nchannels); unsigned int outlen=((inlen*dt->output_rate)/dt->input_rate)+1; unsigned int inlen_orig=inlen; om=allocb(outlen*2*dt->in_nchannels,0); mblk_meta_copy(im, om); if (dt->in_nchannels==1){ speex_resampler_process_int(dt->handle, 0, (int16_t*)im->b_rptr, &inlen, (int16_t*)om->b_wptr, &outlen); }else{ speex_resampler_process_interleaved_int(dt->handle, (int16_t*)im->b_rptr, &inlen, (int16_t*)om->b_wptr, &outlen); } if (inlen_orig!=inlen){ ms_error("Bug in resampler ! only %u samples consumed instead of %u, out=%u", inlen,inlen_orig,outlen); } om->b_wptr+=outlen*2*dt->in_nchannels; mblk_set_timestamp_info(om,dt->ts); dt->ts+=outlen; if (resample_channel_adapt(dt->in_nchannels, dt->out_nchannels, om, &om_chan) == 0) { ms_queue_put(obj->outputs[0], om); } else { ms_queue_put(obj->outputs[0], om_chan); freemsg(om); } freemsg(im); } ms_filter_unlock(obj); }
static int h264_dec_get_video_size(MSFilter *f, MSVideoSize *vsize) { ms_filter_lock(f); *vsize = ((VTH264DecCtx *)f->data)->vsize; ms_filter_unlock(f); return 0; }
static int h264_dec_reset_first_image_notification(MSFilter *f) { ms_filter_lock(f); ((VTH264DecCtx *)f->data)->first_image = TRUE; ms_filter_unlock(f); return 0; }
static void h264_dec_process(MSFilter *f) { VTH264DecCtx *ctx = (VTH264DecCtx *)f->data; mblk_t *pkt; mblk_t *nalu; mblk_t *pixbuf; MSQueue q_nalus; MSQueue q_nalus2; CMBlockBufferRef stream = NULL; CMSampleBufferRef sample = NULL; CMSampleTimingInfo timing_info; MSPicture pixbuf_desc; OSStatus status; MSList *parameter_sets = NULL; bool_t unpacking_failed; ms_queue_init(&q_nalus); ms_queue_init(&q_nalus2); // unpack RTP packet unpacking_failed = FALSE; while((pkt = ms_queue_get(f->inputs[0]))) { unpacking_failed |= (rfc3984_unpack(&ctx->unpacker, pkt, &q_nalus) != 0); } if(unpacking_failed) { ms_error("VideoToolboxDecoder: error while unpacking RTP packets"); goto fail; } // Pull out SPSs and PPSs and put them into the filter context if necessary while((nalu = ms_queue_get(&q_nalus))) { MSH264NaluType nalu_type = ms_h264_nalu_get_type(nalu); if(nalu_type == MSH264NaluTypeSPS || nalu_type == MSH264NaluTypePPS) { parameter_sets = ms_list_append(parameter_sets, nalu); } else if(ctx->format_desc || parameter_sets) { ms_queue_put(&q_nalus2, nalu); } else { ms_free(nalu); } } if(parameter_sets) { CMFormatDescriptionRef last_format = ctx->format_desc ? CFRetain(ctx->format_desc) : NULL; h264_dec_update_format_description(ctx, parameter_sets); parameter_sets = ms_list_free_with_data(parameter_sets, (void (*)(void *))freemsg); if(ctx->format_desc == NULL) goto fail; if(last_format) { CMVideoDimensions last_vsize = CMVideoFormatDescriptionGetDimensions(last_format); CMVideoDimensions vsize = CMVideoFormatDescriptionGetDimensions(ctx->format_desc); if(last_vsize.width != vsize.width || last_vsize.height != vsize.height) { ms_message("VideoToolboxDecoder: new encoded video size %dx%d -> %dx%d", (int)last_vsize.width, (int)last_vsize.height, (int)vsize.width, (int)vsize.height); ms_message("VideoToolboxDecoder: destroying decoding session"); VTDecompressionSessionInvalidate(ctx->session); CFRelease(ctx->session); ctx->session = NULL; } CFRelease(last_format); } } /* Stops proccessing if no IDR has been received yet */ if(ctx->format_desc == NULL) { ms_warning("VideoToolboxDecoder: no IDR packet has been received yet"); goto fail; } /* Initializes the decoder if it has not be done yet or reconfigure it when the size of the encoded video change */ if(ctx->session == NULL) { if(!h264_dec_init_decoder(ctx)) { ms_error("VideoToolboxDecoder: failed to initialized decoder"); goto fail; } } // Pack all nalus in a VTBlockBuffer CMBlockBufferCreateEmpty(NULL, 0, kCMBlockBufferAssureMemoryNowFlag, &stream); while((nalu = ms_queue_get(&q_nalus2))) { CMBlockBufferRef nalu_block; size_t nalu_block_size = msgdsize(nalu) + H264_NALU_HEAD_SIZE; uint32_t nalu_size = htonl(msgdsize(nalu)); CMBlockBufferCreateWithMemoryBlock(NULL, NULL, nalu_block_size, NULL, NULL, 0, nalu_block_size, kCMBlockBufferAssureMemoryNowFlag, &nalu_block); CMBlockBufferReplaceDataBytes(&nalu_size, nalu_block, 0, H264_NALU_HEAD_SIZE); CMBlockBufferReplaceDataBytes(nalu->b_rptr, nalu_block, H264_NALU_HEAD_SIZE, msgdsize(nalu)); CMBlockBufferAppendBufferReference(stream, nalu_block, 0, nalu_block_size, 0); CFRelease(nalu_block); freemsg(nalu); } if(!CMBlockBufferIsEmpty(stream)) { timing_info.duration = kCMTimeInvalid; timing_info.presentationTimeStamp = CMTimeMake(f->ticker->time, 1000); timing_info.decodeTimeStamp = CMTimeMake(f->ticker->time, 1000); CMSampleBufferCreate( NULL, stream, TRUE, NULL, NULL, ctx->format_desc, 1, 1, &timing_info, 0, NULL, &sample); status = VTDecompressionSessionDecodeFrame(ctx->session, sample, 0, NULL, NULL); CFRelease(sample); if(status != noErr) { CFRelease(stream); ms_error("VideoToolboxDecoder: error while passing encoded frames to the decoder: %d", status); if(status == kVTInvalidSessionErr) { h264_dec_uninit_decoder(ctx); } goto fail; } } CFRelease(stream); goto put_frames_out; fail: ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_DECODING_ERRORS); ms_filter_lock(f); if(ctx->enable_avpf) { ms_message("VideoToolboxDecoder: sending PLI"); ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI); } ms_filter_unlock(f); put_frames_out: // Transfer decoded frames in the output queue ms_mutex_lock(&ctx->mutex); while((pixbuf = ms_queue_get(&ctx->queue))) { ms_mutex_unlock(&ctx->mutex); ms_yuv_buf_init_from_mblk(&pixbuf_desc, pixbuf); ms_filter_lock(f); if(pixbuf_desc.w != ctx->vsize.width || pixbuf_desc.h != ctx->vsize.height) { ctx->vsize = (MSVideoSize){ pixbuf_desc.w , pixbuf_desc.h }; } ms_average_fps_update(&ctx->fps, (uint32_t)f->ticker->time); if(ctx->first_image) { ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED); ctx->first_image = FALSE; } ms_filter_unlock(f); ms_queue_put(f->outputs[0], pixbuf); ms_mutex_lock(&ctx->mutex); } ms_mutex_unlock(&ctx->mutex); // Cleaning ms_queue_flush(&q_nalus); ms_queue_flush(&q_nalus2); ms_queue_flush(f->inputs[0]); return; }
static int h264_enc_req_vfu(MSFilter *f, void *ptr) { ms_filter_lock(f); ((VTH264EncCtx *)f->data)->vfu_requested = TRUE; ms_filter_unlock(f); return 0; }
static void h264_enc_process(MSFilter *f) { VTH264EncCtx *ctx = (VTH264EncCtx *)f->data; mblk_t *frame; OSStatus err; CMTime p_time = CMTimeMake(f->ticker->time, 1000); if(!ctx->is_configured) { ms_queue_flush(f->inputs[0]); return; } #if 0 && TARGET_OS_IPHONE CVPixelBufferPoolRef pixbuf_pool = VTCompressionSessionGetPixelBufferPool(ctx->session); if(pixbuf_pool == NULL) { ms_error("VideoToolbox: fails to get the pixel buffer pool"); return; } #endif while((frame = ms_queue_get(f->inputs[0]))) { YuvBuf src_yuv_frame, dst_yuv_frame = {0}; CVPixelBufferRef pixbuf; CFMutableDictionaryRef enc_param = NULL; int i, pixbuf_fmt = kCVPixelFormatType_420YpCbCr8Planar; CFNumberRef value; CFMutableDictionaryRef pixbuf_attr; ms_yuv_buf_init_from_mblk(&src_yuv_frame, frame); #if 0 && TARGET_OS_IPHONE CVPixelBufferPoolCreatePixelBuffer(NULL, pixbuf_pool, &pixbuf); #else pixbuf_attr = CFDictionaryCreateMutable(NULL, 0, NULL, NULL); value = CFNumberCreate(NULL, kCFNumberIntType, &pixbuf_fmt); CFDictionarySetValue(pixbuf_attr, kCVPixelBufferPixelFormatTypeKey, value); CVPixelBufferCreate(NULL, ctx->conf.vsize.width, ctx->conf.vsize.height, kCVPixelFormatType_420YpCbCr8Planar, pixbuf_attr, &pixbuf); CFRelease(pixbuf_attr); #endif CVPixelBufferLockBaseAddress(pixbuf, 0); dst_yuv_frame.w = (int)CVPixelBufferGetWidth(pixbuf); dst_yuv_frame.h = (int)CVPixelBufferGetHeight(pixbuf); for(i=0; i<3; i++) { dst_yuv_frame.planes[i] = CVPixelBufferGetBaseAddressOfPlane(pixbuf, i); dst_yuv_frame.strides[i] = (int)CVPixelBufferGetBytesPerRowOfPlane(pixbuf, i); } ms_yuv_buf_copy(src_yuv_frame.planes, src_yuv_frame.strides, dst_yuv_frame.planes, dst_yuv_frame.strides, (MSVideoSize){dst_yuv_frame.w, dst_yuv_frame.h}); CVPixelBufferUnlockBaseAddress(pixbuf, 0); freemsg(frame); ms_filter_lock(f); if(ctx->fps_changed || ctx->bitrate_changed || ctx->vfu_requested) { CFNumberRef value; enc_param = CFDictionaryCreateMutable(NULL, 0, NULL, NULL); if(ctx->fps_changed) { value = CFNumberCreate(NULL, kCFNumberFloatType, &ctx->conf.fps); CFDictionaryAddValue(enc_param, kVTCompressionPropertyKey_ExpectedFrameRate, value); ctx->fps_changed = FALSE; } if(ctx->bitrate_changed) { value = CFNumberCreate(NULL, kCFNumberIntType, &ctx->conf.required_bitrate); CFDictionaryAddValue(enc_param, kVTCompressionPropertyKey_AverageBitRate, value); ctx->bitrate_changed = FALSE; } if(ctx->vfu_requested) { int force_keyframe = 1; value = CFNumberCreate(NULL, kCFNumberIntType, &force_keyframe); CFDictionaryAddValue(enc_param, kVTEncodeFrameOptionKey_ForceKeyFrame, value); ctx->vfu_requested = FALSE; } } ms_filter_unlock(f); if(!ctx->enable_avpf) { if(ctx->first_frame) { ms_video_starter_first_frame(&ctx->starter, f->ticker->time); } if(ms_video_starter_need_i_frame(&ctx->starter, f->ticker->time)) { if(enc_param == NULL) enc_param = CFDictionaryCreateMutable(NULL, 0, NULL, NULL); if(CFDictionaryGetValue(enc_param, kVTEncodeFrameOptionKey_ForceKeyFrame) == NULL) { int force_keyframe = 1; CFNumberRef value = CFNumberCreate(NULL, kCFNumberIntType, &force_keyframe); CFDictionaryAddValue(enc_param, kVTEncodeFrameOptionKey_ForceKeyFrame, value); } } } if((err = VTCompressionSessionEncodeFrame(ctx->session, pixbuf, p_time, kCMTimeInvalid, enc_param, NULL, NULL)) != noErr) { ms_error("VideoToolbox: could not pass a pixbuf to the encoder: error code %d", err); } CFRelease(pixbuf); ctx->first_frame = FALSE; if(enc_param) CFRelease(enc_param); } ms_mutex_lock(&ctx->mutex); while ((frame = ms_queue_get(&ctx->queue))) { ms_mutex_unlock(&ctx->mutex); ms_queue_put(f->outputs[0], frame); ms_mutex_lock(&ctx->mutex); } ms_mutex_unlock(&ctx->mutex); }
static int dtmfgen_put(MSFilter *f, void *arg){ DtmfGenState *s=(DtmfGenState*)f->data; const char *dtmf=(char*)arg; switch(dtmf[0]){ case '0': s->lowfreq=941; s->highfreq=1336; break; case '1': s->lowfreq=697; s->highfreq=1209; break; case '2': s->lowfreq=697; s->highfreq=1336; break; case '3': s->lowfreq=697; s->highfreq=1477; break; case '4': s->lowfreq=770; s->highfreq=1209; break; case '5': s->lowfreq=770; s->highfreq=1336; break; case '6': s->lowfreq=770; s->highfreq=1477; break; case '7': s->lowfreq=852; s->highfreq=1209; break; case '8': s->lowfreq=852; s->highfreq=1336; break; case '9': s->lowfreq=852; s->highfreq=1477; break; case '*': s->lowfreq=941; s->highfreq=1209; break; case '#': s->lowfreq=941; s->highfreq=1477; break; case 'A': s->lowfreq=697; s->highfreq=1633; break; case 'B': s->lowfreq=770; s->highfreq=1633; break; case 'C': s->lowfreq=852; s->highfreq=1633; break; case 'D': s->lowfreq=941; s->highfreq=1633; break; case ' ': /*ignore*/ return 0; break; default: ms_warning("Not a dtmf key."); return -1; } ms_filter_lock(f); s->pos=0; s->lowfreq=s->lowfreq/s->rate; s->highfreq=s->highfreq/s->rate; s->dur=s->rate/10; /*100 ms duration */ s->silence=0; s->amplitude=s->default_amplitude*32767*0.7; s->dtmf=dtmf[0]; s->interval=0; ms_filter_unlock(f); return 0; }
static void x11video_process(MSFilter *f){ X11Video *obj=(X11Video*)f->data; mblk_t *inm; int update=0; MSPicture lsrc={0}; MSPicture src={0}; MSRect mainrect,localrect; bool_t precious=FALSE; bool_t local_precious=FALSE; XWindowAttributes wa; MSTickerLateEvent late_info; ms_filter_lock(f); if ((obj->window_id == 0) || (x11_error == TRUE)) goto end; XGetWindowAttributes(obj->display,obj->window_id,&wa); if (x11_error == TRUE) { ms_error("Could not get window attributes for window %lu", obj->window_id); goto end; } if (wa.width!=obj->wsize.width || wa.height!=obj->wsize.height){ ms_warning("Resized to %ix%i", wa.width,wa.height); obj->wsize.width=wa.width; obj->wsize.height=wa.height; XClearWindow(obj->display,obj->window_id); } ms_ticker_get_last_late_tick(f->ticker, &late_info); if(late_info.current_late_ms > 100) { ms_warning("Dropping frames because we're late"); goto end; } if (!obj->show) { goto end; } if (!obj->ready){ goto end; } if (f->inputs[0]!=NULL && (inm=ms_queue_peek_last(f->inputs[0]))!=0) { if (ms_yuv_buf_init_from_mblk(&src,inm)==0){ MSVideoSize newsize; newsize.width=src.w; newsize.height=src.h; precious=mblk_get_precious_flag(inm); if (!ms_video_size_equal(newsize,obj->vsize) ) { ms_message("received size is %ix%i",newsize.width,newsize.height); obj->vsize=newsize; if (obj->autofit){ MSVideoSize new_window_size; static const MSVideoSize min_size=MS_VIDEO_SIZE_QVGA; /*don't resize less than QVGA, it is too small*/ if (min_size.width*min_size.height>newsize.width*newsize.height){ new_window_size.width=newsize.width*2; new_window_size.height=newsize.height*2; }else new_window_size=newsize; obj->wsize=new_window_size; ms_message("autofit: new window size should be %ix%i",new_window_size.width,new_window_size.height); XResizeWindow(obj->display,obj->window_id,new_window_size.width,new_window_size.height); XSync(obj->display,FALSE); } x11video_unprepare(f); x11video_prepare(f); if (!obj->ready) goto end; } } update=1; } /*process last video message for local preview*/ if (obj->corner!=-1 && f->inputs[1]!=NULL && (inm=ms_queue_peek_last(f->inputs[1]))!=0) { if (ms_yuv_buf_init_from_mblk(&lsrc,inm)==0){ obj->lsize.width=lsrc.w; obj->lsize.height=lsrc.h; local_precious=mblk_get_precious_flag(inm); update=1; } } ms_layout_compute(obj->vsize, obj->vsize,obj->lsize,obj->corner,obj->scale_factor,&mainrect,&localrect); if (lsrc.w!=0 && obj->corner!=-1){ /* first reduce the local preview image into a temporary image*/ if (obj->local_msg==NULL){ obj->local_msg=ms_yuv_buf_alloc(&obj->local_pic,localrect.w,localrect.h); } if (obj->sws2==NULL){ obj->sws2=ms_scaler_create_context(lsrc.w,lsrc.h,MS_YUV420P,localrect.w,localrect.h,MS_YUV420P, MS_SCALER_METHOD_BILINEAR); } ms_scaler_process(obj->sws2,lsrc.planes,lsrc.strides,obj->local_pic.planes,obj->local_pic.strides); if (!local_precious) ms_yuv_buf_mirror(&obj->local_pic); } if (update && src.w!=0){ ms_yuv_buf_copy(src.planes,src.strides,obj->fbuf.planes,obj->fbuf.strides,obj->vsize); if (obj->mirror && !precious) ms_yuv_buf_mirror(&obj->fbuf); } /*copy resized local view into a corner:*/ if (update && obj->local_msg!=NULL && obj->corner!=-1){ MSPicture corner=obj->fbuf; MSVideoSize roi; roi.width=obj->local_pic.w; roi.height=obj->local_pic.h; corner.w=obj->local_pic.w; corner.h=obj->local_pic.h; corner.planes[0]+=localrect.x+(localrect.y*corner.strides[0]); corner.planes[1]+=(localrect.x/2)+((localrect.y/2)*corner.strides[1]); corner.planes[2]+=(localrect.x/2)+((localrect.y/2)*corner.strides[2]); corner.planes[3]=0; ms_yuv_buf_copy(obj->local_pic.planes,obj->local_pic.strides, corner.planes,corner.strides,roi); } if (update){ MSRect rect; ms_layout_center_rectangle(obj->wsize,obj->vsize,&rect); //ms_message("XvShmPutImage() %ix%i --> %ix%i",obj->fbuf.w,obj->fbuf.h,obj->wsize.width,obj->wsize.height); XvShmPutImage(obj->display,obj->port,obj->window_id,obj->gc, obj->xv_image, 0,0,obj->fbuf.w,obj->fbuf.h, rect.x,rect.y,rect.w,rect.h,TRUE); XSync(obj->display,FALSE); } end: ms_filter_unlock(f); if (f->inputs[0]!=NULL) ms_queue_flush(f->inputs[0]); if (f->inputs[1]!=NULL) ms_queue_flush(f->inputs[1]); }
static void ms_opus_enc_process(MSFilter *f) { OpusEncData *d = (OpusEncData *)f->data; mblk_t *im; mblk_t *om = NULL; int i; int frameNumber, packet_size; uint8_t *signalFrameBuffer = NULL; uint8_t *codedFrameBuffer[MAX_INPUT_FRAMES]; OpusRepacketizer *rp = opus_repacketizer_create(); opus_int32 ret = 0; opus_int32 totalLength = 0; int frame_size = d->samplerate * FRAME_LENGTH / 1000; /* in samples */ // lock the access while getting ptime ms_filter_lock(f); frameNumber = d->ptime/FRAME_LENGTH; /* encode 20ms frames, ptime is a multiple of 20ms */ packet_size = d->samplerate * d->ptime / 1000; /* in samples */ ms_filter_unlock(f); while ((im = ms_queue_get(f->inputs[0])) != NULL) { ms_bufferizer_put(d->bufferizer, im); } for (i=0; i<MAX_INPUT_FRAMES; i++) { codedFrameBuffer[i]=NULL; } while (ms_bufferizer_get_avail(d->bufferizer) >= (d->channels * packet_size * SIGNAL_SAMPLE_SIZE)) { totalLength = 0; opus_repacketizer_init(rp); for (i=0; i<frameNumber; i++) { /* encode 20ms by 20ms and repacketize all of them together */ if (!codedFrameBuffer[i]) codedFrameBuffer[i] = ms_malloc(MAX_BYTES_PER_FRAME); /* the repacketizer need the pointer to packet to remain valid, so we shall have a buffer for each coded frame */ if (!signalFrameBuffer) signalFrameBuffer = ms_malloc(frame_size * SIGNAL_SAMPLE_SIZE * d->channels); ms_bufferizer_read(d->bufferizer, signalFrameBuffer, frame_size * SIGNAL_SAMPLE_SIZE * d->channels); ret = opus_encode(d->state, (opus_int16 *)signalFrameBuffer, frame_size, codedFrameBuffer[i], MAX_BYTES_PER_FRAME); if (ret < 0) { ms_error("Opus encoder error: %s", opus_strerror(ret)); break; } if (ret > 0) { int err = opus_repacketizer_cat(rp, codedFrameBuffer[i], ret); /* add the encoded frame into the current packet */ if (err != OPUS_OK) { ms_error("Opus repacketizer error: %s", opus_strerror(err)); break; } totalLength += ret; } } if (ret > 0) { om = allocb(totalLength+frameNumber + 1, 0); /* opus repacktizer API: allocate at leat number of frame + size of all data added before */ ret = opus_repacketizer_out(rp, om->b_wptr, totalLength+frameNumber); om->b_wptr += ret; mblk_set_timestamp_info(om, d->ts); ms_queue_put(f->outputs[0], om); d->ts += packet_size*48000/d->samplerate; /* RFC payload RTP opus 03 - section 4: RTP timestamp multiplier : WARNING works only with sr at 48000 */ ret = 0; } } opus_repacketizer_destroy(rp); if (signalFrameBuffer != NULL) { ms_free(signalFrameBuffer); } for (i=0; i<frameNumber; i++) { if (codedFrameBuffer[i] != NULL) { ms_free(codedFrameBuffer[i]); } } }
static void glxvideo_process(MSFilter *f){ GLXVideo *obj=(GLXVideo*)f->data; mblk_t *inm; MSPicture src={0}; bool_t precious=FALSE; XWindowAttributes wa; XGetWindowAttributes(obj->display,obj->window_id,&wa); if (wa.width!=obj->wsize.width || wa.height!=obj->wsize.height){ ms_warning("Resized to %ix%i", wa.width,wa.height); obj->wsize.width=wa.width; obj->wsize.height=wa.height; ogl_display_init(obj->glhelper, wa.width, wa.height); } ms_filter_lock(f); if (!obj->show) { goto end; } if (!obj->ready) glxvideo_prepare(f); if (!obj->ready){ goto end; } glXMakeCurrent( obj->display, obj->window_id, obj->glContext ); if (f->inputs[0]!=NULL && (inm=ms_queue_peek_last(f->inputs[0]))!=0) { if (ms_yuv_buf_init_from_mblk(&src,inm)==0){ MSVideoSize newsize; newsize.width=src.w; newsize.height=src.h; precious=mblk_get_precious_flag(inm); if (!ms_video_size_equal(newsize,obj->vsize) ) { ms_message("received size is %ix%i",newsize.width,newsize.height); obj->vsize=newsize; if (obj->autofit){ MSVideoSize new_window_size; static const MSVideoSize min_size=MS_VIDEO_SIZE_QVGA; /*don't resize less than QVGA, it is too small*/ if (min_size.width*min_size.height>newsize.width*newsize.height){ new_window_size.width=newsize.width*2; new_window_size.height=newsize.height*2; }else new_window_size=newsize; obj->wsize=new_window_size; ms_message("autofit: new window size should be %ix%i",new_window_size.width,new_window_size.height); XResizeWindow(obj->display,obj->window_id,new_window_size.width,new_window_size.height); XSync(obj->display,FALSE); } glxvideo_unprepare(f); glxvideo_prepare(f); if (!obj->ready) goto end; } if (obj->mirror && !precious) ms_yuv_buf_mirror(&src); ogl_display_set_yuv_to_display(obj->glhelper, inm); } } if (f->inputs[1]!=NULL && (inm=ms_queue_peek_last(f->inputs[1]))!=0) { if (ms_yuv_buf_init_from_mblk(&src,inm)==0){ ogl_display_set_preview_yuv_to_display(obj->glhelper, inm); } } ogl_display_render(obj->glhelper, 0); glXSwapBuffers ( obj->display, obj->window_id ); end: ms_filter_unlock(f); if (f->inputs[0]!=NULL) ms_queue_flush(f->inputs[0]); if (f->inputs[1]!=NULL) ms_queue_flush(f->inputs[1]); }