/*****************************************************************************
* Filter
*****************************************************************************/
static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
{
picture_t * p_outpic;
filter_sys_t *p_sys = p_filter->p_sys;
if( !p_pic ) return NULL;
p_outpic = filter_NewPicture( p_filter );
if( !p_outpic )
{
picture_Release( p_pic );
return NULL;
}
if( p_sys->b_first )
{
picture_CopyPixels( p_sys->p_tmp, p_pic );
p_sys->b_first = false;
}
/* Get a new picture */
RenderBlur( p_sys, p_pic, p_outpic );
picture_CopyPixels( p_sys->p_tmp, p_outpic );
return CopyInfoAndRelease( p_outpic, p_pic );
}
static void Prepare(vout_display_t *vd, picture_t *pic, subpicture_t *subpic)
{
vout_display_sys_t *sys = vd->sys;
picture_resource_t rsc = { .p_sys = NULL };
void *planes[PICTURE_PLANE_MAX];
sys->pic_opaque = sys->lock(sys->opaque, planes);
for (unsigned i = 0; i < PICTURE_PLANE_MAX; i++) {
rsc.p[i].p_pixels = planes[i];
rsc.p[i].i_lines = sys->lines[i];
rsc.p[i].i_pitch = sys->pitches[i];
}
picture_t *locked = picture_NewFromResource(&vd->fmt, &rsc);
if (likely(locked != NULL)) {
picture_CopyPixels(locked, pic);
picture_Release(locked);
}
if (sys->unlock != NULL)
sys->unlock(sys->opaque, sys->pic_opaque, planes);
(void) subpic;
}
static void SwapUV( picture_t *p_dst, const picture_t *p_src )
{
picture_t tmp = *p_src;
tmp.p[1] = p_src->p[2];
tmp.p[2] = p_src->p[1];
picture_CopyPixels( p_dst, &tmp );
}
/*****************************************************************************
* PostprocPict
*****************************************************************************/
static picture_t *PostprocPict( filter_t *p_filter, picture_t *p_pic )
{
filter_sys_t *p_sys = p_filter->p_sys;
const uint8_t *src[3];
uint8_t *dst[3];
int i_plane;
int i_src_stride[3], i_dst_stride[3];
picture_t *p_outpic = filter_NewPicture( p_filter );
if( !p_outpic )
{
picture_Release( p_pic );
return NULL;
}
/* Lock to prevent issues if pp_mode is changed */
vlc_mutex_lock( &p_sys->lock );
if( !p_sys->pp_mode )
{
vlc_mutex_unlock( &p_sys->lock );
picture_CopyPixels( p_outpic, p_pic );
return CopyInfoAndRelease( p_outpic, p_pic );
}
for( i_plane = 0; i_plane < p_pic->i_planes; i_plane++ )
{
src[i_plane] = p_pic->p[i_plane].p_pixels;
dst[i_plane] = p_outpic->p[i_plane].p_pixels;
/* I'm not sure what happens if i_pitch != i_visible_pitch ...
* at least it shouldn't crash. */
i_src_stride[i_plane] = p_pic->p[i_plane].i_pitch;
i_dst_stride[i_plane] = p_outpic->p[i_plane].i_pitch;
}
if( !p_pic->p_q && p_sys->b_had_matrix )
{
msg_Warn( p_filter, "Quantification table was not set by video decoder. Postprocessing won't look good." );
p_sys->b_had_matrix = false;
}
else if( p_pic->p_q )
{
p_sys->b_had_matrix = true;
}
pp_postprocess( src, i_src_stride, dst, i_dst_stride,
p_filter->fmt_in.video.i_width,
p_filter->fmt_in.video.i_height,
p_pic->p_q, p_pic->i_qstride,
p_sys->pp_mode, p_sys->pp_context,
p_pic->i_qtype == QTYPE_MPEG2 ? PP_PICT_TYPE_QP2 : 0 );
vlc_mutex_unlock( &p_sys->lock );
return CopyInfoAndRelease( p_outpic, p_pic );
}
/****************************************************************************
* Filter: the whole thing
****************************************************************************
* This function is called just after the thread is launched.
****************************************************************************/
static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
{
filter_sys_t *p_sys = p_filter->p_sys;
const video_format_t *p_fmti = &p_filter->fmt_in.video;
const video_format_t *p_fmto = &p_filter->fmt_out.video;
picture_t *p_pic_dst;
/* Check if format properties changed */
if( Init( p_filter ) )
{
picture_Release( p_pic );
return NULL;
}
/* Request output picture */
p_pic_dst = filter_NewPicture( p_filter );
if( !p_pic_dst )
{
picture_Release( p_pic );
return NULL;
}
/* */
picture_t *p_src = p_pic;
picture_t *p_dst = p_pic_dst;
if( p_sys->i_extend_factor != 1 )
{
p_src = p_sys->p_src_e;
p_dst = p_sys->p_dst_e;
CopyPad( p_src, p_pic );
}
if( p_sys->b_copy && p_sys->b_swap_uvi == p_sys->b_swap_uvo )
picture_CopyPixels( p_dst, p_src );
else if( p_sys->b_copy )
SwapUV( p_dst, p_src );
else
Convert( p_filter, p_sys->ctx, p_dst, p_src, p_fmti->i_height, 0, 3,
p_sys->b_swap_uvi, p_sys->b_swap_uvo );
if( p_sys->ctxA )
{
/* We extract the A plane to rescale it, and then we reinject it. */
if( p_fmti->i_chroma == VLC_CODEC_RGBA )
ExtractA( p_sys->p_src_a, p_src, p_fmti->i_width * p_sys->i_extend_factor, p_fmti->i_height );
else
plane_CopyPixels( p_sys->p_src_a->p, p_src->p+A_PLANE );
Convert( p_filter, p_sys->ctxA, p_sys->p_dst_a, p_sys->p_src_a, p_fmti->i_height, 0, 1, false, false );
if( p_fmto->i_chroma == VLC_CODEC_RGBA )
InjectA( p_dst, p_sys->p_dst_a, p_fmto->i_width * p_sys->i_extend_factor, p_fmto->i_height );
else
plane_CopyPixels( p_dst->p+A_PLANE, p_sys->p_dst_a->p );
}
else if( p_sys->b_add_a )
{
/* We inject a complete opaque alpha plane */
if( p_fmto->i_chroma == VLC_CODEC_RGBA )
FillA( &p_dst->p[0], OFFSET_A );
else
FillA( &p_dst->p[A_PLANE], 0 );
}
if( p_sys->i_extend_factor != 1 )
{
picture_CopyPixels( p_pic_dst, p_dst );
}
picture_CopyProperties( p_pic_dst, p_pic );
picture_Release( p_pic );
return p_pic_dst;
}
void picture_Copy( picture_t *p_dst, const picture_t *p_src )
{
picture_CopyPixels( p_dst, p_src );
picture_CopyProperties( p_dst, p_src );
}
/*****************************************************************************
* Filter: displays previously rendered output
*****************************************************************************
* This function send the currently rendered image to the internal opencv
* filter for processing.
*****************************************************************************/
static picture_t* Filter( filter_t* p_filter, picture_t* p_pic )
{
picture_t* p_outpic = filter_NewPicture( p_filter );
if( p_outpic == NULL ) {
msg_Err( p_filter, "couldn't get a p_outpic!" );
picture_Release( p_pic );
return NULL;
}
video_format_t fmt_out;
// Make a copy if we want to show the original input
if (p_filter->p_sys->i_wrapper_output == VINPUT)
picture_Copy( p_outpic, p_pic );
VlcPictureToIplImage( p_filter, p_pic );
// Pass the image (as a pointer to the first IplImage*) to the
// internal OpenCV filter for processing.
p_filter->p_sys->p_opencv->pf_video_filter( p_filter->p_sys->p_opencv, (picture_t*)&(p_filter->p_sys->p_cv_image[0]) );
if(p_filter->p_sys->i_wrapper_output == PROCESSED) {
// Processed video
if( (p_filter->p_sys->p_proc_image) &&
(p_filter->p_sys->p_proc_image->i_planes > 0) &&
(p_filter->p_sys->i_internal_chroma != CINPUT) ) {
//p_filter->p_sys->p_proc_image->format.i_chroma = VLC_CODEC_RGB24;
memset( &fmt_out, 0, sizeof(video_format_t) );
fmt_out = p_pic->format;
//picture_Release( p_outpic );
/*
* We have to copy out the image from image_Convert(), otherwise
* you leak pictures for some reason:
* main video output error: pictures leaked, trying to workaround
*/
picture_t* p_outpic_tmp = image_Convert(
p_filter->p_sys->p_image,
p_filter->p_sys->p_proc_image,
&(p_filter->p_sys->p_proc_image->format),
&fmt_out );
picture_CopyPixels( p_outpic, p_outpic_tmp );
CopyInfoAndRelease( p_outpic, p_outpic_tmp );
} else if( p_filter->p_sys->i_internal_chroma == CINPUT ) {
picture_CopyPixels( p_outpic, p_filter->p_sys->p_proc_image );
picture_CopyProperties( p_outpic, p_filter->p_sys->p_proc_image );
}
}
ReleaseImages( p_filter );
picture_Release( p_pic );
#ifndef NDEBUG
msg_Dbg( p_filter, "Filter() done" );
#endif
if( p_filter->p_sys->i_wrapper_output != NONE ) {
return p_outpic;
} else { // NONE
picture_Release( p_outpic );
return NULL;
}
}
/**
* @brief VLC filter callback
* @return picture(s) containing the filtered frames
*/
static picture_t* y4m_filter(filter_t* intf, picture_t* srcPic)
{
filter_sys_t* sys = intf->p_sys;
//msg_Info(intf, ">>>> filter");
if (!srcPic)
{
//msg_Info(intf, ">>> filter: NULL INPUT");
return NULL;
}
// will be stored to sys->lastDate on return
mtime_t currDate = srcPic->date;
// if there was a problem with the subprocess then send back
// the picture unmodified, at least we won't freeze up vlc
if (sys->startFailed || sys->threadExit)
goto ECHO_RETURN;
// start subprocess and write the y4m header
// fixme: this can go in open() if fmt_in matches the srcPic
if (!sys->startFailed && !sys->childPid)
{
sys->startFailed = !startProcess(intf);
if (sys->startFailed)
goto ECHO_RETURN;
if (0 >= writeY4mHeader(intf, srcPic, sys->stdin))
{
msg_Err(intf, "writeY4mHeader failed: errno=%d %s",
errno, strerror(errno));
stopProcess(intf);
sys->startFailed = true;
goto ECHO_RETURN;
}
}
//
// control the buffering level by monitoring the input/output fifos
//
// the input/output fifos are emptied/filled by input/output threads
// in response to the subprocess reading/writing frames
//
// if the input fifo is empty, then probably the subprocess wants
// more input, so we should buffer more input
//
// if the output fifo is empty, and the input fifo is not empty, then
// probably the subprocess is about to write out a frame and we
// can wait for it to arrive. in practice, most of the time there
// is no waiting needed, unless the filter is too slow to keep up.
//
bool inputEmpty = true;
bool outputEmpty = true;
picture_t* tmp = picture_fifo_Peek(sys->inputFifo);
if (tmp)
{
picture_Release(tmp);
inputEmpty = false;
}
tmp = picture_fifo_Peek(sys->outputFifo);
if (tmp)
{
picture_Release(tmp);
outputEmpty = false;
}
// copy picture to input fifo, we can't use picture_Hold or else
// the decoder or vout would run out of pictures in its pool
picture_t* inPic = picture_NewFromFormat(&srcPic->format);
picture_Copy(inPic, srcPic);
picture_fifo_Push(sys->inputFifo, inPic);
// signal input thread to wake up and write some more data out
vlc_mutex_lock(&sys->inputMutex);
sys->bufferedIn++;
vlc_cond_signal(&sys->inputCond);
vlc_mutex_unlock(&sys->inputMutex);
// if echo is enabled, we're done
// todo: there should be a limiter on the input buffering in case
// the subprocess can't keep up
if (sys->echo)
goto ECHO_RETURN;
// keeps track of the number of buffered output pictures, assumes
// an integer ratio
// fixme: needs modification to support non-integer ratios
// and ratios < 1
sys->bufferedOut += sys->bufferRatio;
// handle buffering
if (outputEmpty && inputEmpty)
{
// we haven't supplied enough input, raise the minimum
// level of buffer to keep and return
sys->minBuffered += sys->bufferRatio;
msg_Info(intf, "buffer more input: buffers:%d:%d:%d",
sys->bufferedIn, sys->bufferedOut, sys->minBuffered);
goto NULL_RETURN;
}
if (outputEmpty)
waitForOutput(intf);
// if we don't know what the frame interval is, make it 0 which
// probably causes the next frames out to drop
// note: this happens at least every time y4m_flush() is called
// for example when seeking
if (currDate <= sys->lastDate || sys->lastDate == 0)
{
//msg_Err(intf, "currDate <= lastDate");
//goto ECHO_RETURN;
sys->lastDate = currDate;
}
// reference to first and last picture we are returning
picture_t* first = NULL;
picture_t* last = NULL;
picture_t* pic;
while( (pic = picture_fifo_Pop(sys->outputFifo)) )
{
// do output setup when we see the first frame out from the filter,
// it could have a different frame rate, chroma, size, etc than
// the frame going in
if (!sys->gotFirstOutput)
{
sys->gotFirstOutput = true;
// get the in/out frame ratio by comparing frame rates
float speed =
((float)srcPic->format.i_frame_rate_base * (float)pic->format.i_frame_rate) /
((float)srcPic->format.i_frame_rate * (float)pic->format.i_frame_rate_base);
if (speed < 1.0)
{
msg_Err(intf, "frame rate reduction isn't supported yet");
}
else
if (speed > 1.0)
{
if (ceil(speed) != speed)
msg_Err(intf, "frame rate change must be integer ratio");
sys->bufferRatio = speed;
// initial ratio was 1.0, need to correct the number of buffered frames
// now that we know what it is
sys->bufferedOut *= sys->bufferRatio;
sys->minBuffered *= sys->bufferRatio;
}
intf->fmt_out.video.i_frame_rate = pic->format.i_frame_rate;
intf->fmt_out.video.i_frame_rate_base = pic->format.i_frame_rate_base;
if (intf->fmt_out.video.i_chroma != pic->format.i_chroma)
msg_Err(intf, "filter changed the chroma, expect corruption");
// this can't be changed after open, crashes the GLX vout
//intf->fmt_out.i_codec = pic->format.i_chroma;
//intf->fmt_out.video.i_chroma = pic->format.i_chroma;
msg_Info(intf, "first output: buffers=%d:%d", sys->bufferedOut, sys->minBuffered);
}
sys->numFrames++;
sys->bufferedOut--;
// it seems filter_NewPicture is required now. however,
// sometimes it returns null in which case it seems like
// the best thing to do is dump frames
picture_t* copy = first == NULL ? srcPic : filter_NewPicture(intf);
if (!copy)
{
picture_Release(pic);
// throw away frames
// vlc already prints warning for this
//msg_Err(intf, "filter_NewPicture returns null");
if (sys->bufferedOut < sys->minBuffered)
break;
else
continue;
}
else
{
picture_CopyPixels(copy, pic);
picture_Release(pic);
pic = copy;
}
// the time per output frame interval is a fraction of the input frame time
int frameTime = (currDate - sys->lastDate) / sys->bufferRatio;
// the pts is whatever the current pts is minus any buffering
// introduced by the filter
pic->date = currDate - sys->bufferedOut*frameTime;
// msg_Info(intf, "frame=%d buffered=%d:%d frameTime=%d ratio:%d:1 fin=%d:%d fout=%d:%d pts=%u",
// sys->numFrames, sys->bufferedOut, sys->minBuffered,
// frameTime, sys->bufferRatio,
// srcPic->format.i_frame_rate, srcPic->format.i_frame_rate_base,
// pic->format.i_frame_rate, pic->format.i_frame_rate_base,
// (unsigned int)pic->date);
if (last)
last->p_next = pic;
else
first = pic;
last = pic;
// if we read too many frames on this iteration, on the next
// one we might not have any frames available which would be
// bad as vlc would think our intent was to drop frames
//
// if we stop reading before the output is completely empty,
// there will always be some frames for the next iteration,
// assuming the filter is fast enough to keep up
if (sys->bufferedOut < sys->minBuffered)
break;
// if there is still some input buffer left, but the fifo is
// empty, wait for next frame to arrive. otherwise we can
// build too much input buffering
if (sys->bufferedIn > 1)
waitForOutput(intf);
}
if (!first)
{
// the buffer checks should prevent from getting here, but
// just in case prevent leaking the input picture
picture_Release(srcPic);
sys->minBuffered++;
}
sys->lastDate = currDate;
return first;
ECHO_RETURN:
sys->lastDate = currDate;
//msg_Info(intf, "<<<< filter: ECHO");
return srcPic;
NULL_RETURN:
sys->lastDate = currDate;
picture_Release(srcPic);
//msg_Info(intf, "<<<< filter: NULL");
return NULL;
}