static GstFlowReturn gst_imx_compositor_aggregate_frames(GstImxBPVideoAggregator *videoaggregator, GstBuffer *outbuffer)
{
GstFlowReturn ret = GST_FLOW_OK;
GList *walk;
GstImxCompositor *compositor = GST_IMX_COMPOSITOR(videoaggregator);
GstImxCompositorClass *klass = GST_IMX_COMPOSITOR_CLASS(G_OBJECT_GET_CLASS(videoaggregator));
g_assert(klass->set_output_frame != NULL);
g_assert(klass->fill_region != NULL);
g_assert(klass->draw_frame != NULL);
/* This function is the heart of the compositor. Here, input frames
* are drawn on the output frame, with their specific parameters. */
/* Set the output buffer */
if (!(klass->set_output_frame(compositor, outbuffer)))
{
GST_ERROR_OBJECT(compositor, "could not set the output frame");
return GST_FLOW_ERROR;
}
/* TODO: are the update_overall_region calls here necessary?
* If the video aggregator calls update_caps when a pad is added/removed,
* there is no need for these calls */
/* Update the overall region first if necessary to ensure that it is valid
* and that the region_fill_necessary flag is set to the proper value */
gst_imx_compositor_update_overall_region(compositor);
GST_LOG_OBJECT(compositor, "aggregating frames, region_fill_necessary: %d", (gint)(compositor->region_fill_necessary));
/* Check if the overall region needs to be filled. This is the case if none
* of the input frames completely cover the overall region with 100% alpha
* (this is determined by gst_imx_compositor_update_overall_region() ) */
if (!(compositor->region_fill_necessary) || klass->fill_region(compositor, &(compositor->overall_region), compositor->background_color))
{
/* Lock object to ensure nothing is changed during composition */
GST_OBJECT_LOCK(compositor);
/* First walk: check if there is a new pad. If so, recompute the
* overall region, since it might need to be expanded to encompass
* the new additional input frames */
walk = GST_ELEMENT(videoaggregator)->sinkpads;
while (walk != NULL)
{
GstImxCompositorPad *compositor_pad = GST_IMX_COMPOSITOR_PAD_CAST(walk->data);
if (compositor_pad->pad_is_new)
{
GST_DEBUG_OBJECT(compositor, "there is a new pad; invalidate overall region");
compositor_pad->pad_is_new = FALSE;
compositor->overall_region_valid = FALSE;
/* While this call might seem redundant, there is one
* benefit in calling this function apparently twice
* (once above, and once here): the earlier call
* happens outside of the object lock. New pads are less
* common than overall region changes, so it is good
* if most update calls happen outside of the object
* lock (the overall_region_valid flag ensures redundant
* calls don't compute anything). */
gst_imx_compositor_update_overall_region(compositor);
break;
}
/* Move to next pad */
walk = g_list_next(walk);
}
/* Second walk: draw the input frames on the output frame */
walk = GST_ELEMENT(videoaggregator)->sinkpads;
while (walk != NULL)
{
GstImxBPVideoAggregatorPad *videoaggregator_pad = walk->data;
GstImxCompositorPad *compositor_pad = GST_IMX_COMPOSITOR_PAD_CAST(videoaggregator_pad);
/* If there actually is a buffer, draw it
* Sometimes, pads don't deliver data right from the start;
* in these cases, their buffers will be NULL
* Just skip to the next pad in that case */
if (videoaggregator_pad->buffer != NULL)
{
GstVideoCropMeta *video_crop_meta;
if (compositor_pad->input_crop && ((video_crop_meta = gst_buffer_get_video_crop_meta(videoaggregator_pad->buffer)) != NULL))
{
/* Crop metadata present. Reconfigure canvas. */
GstVideoInfo *info = &(videoaggregator_pad->info);
GstImxRegion source_region;
source_region.x1 = video_crop_meta->x;
source_region.y1 = video_crop_meta->y;
source_region.x2 = video_crop_meta->x + video_crop_meta->width;
source_region.y2 = video_crop_meta->y + video_crop_meta->height;
/* Make sure the source region does not exceed valid bounds */
source_region.x1 = MAX(0, source_region.x1);
source_region.y1 = MAX(0, source_region.y1);
source_region.x2 = MIN(GST_VIDEO_INFO_WIDTH(info), source_region.x2);
source_region.y2 = MIN(GST_VIDEO_INFO_HEIGHT(info), source_region.y2);
GST_LOG_OBJECT(compositor, "retrieved crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region));
/* Canvas needs to be updated if either one of these applies:
* - the current frame has crop metadata, the last one didn't
* - the new crop rectangle and the last are different */
if (!(compositor_pad->last_frame_with_cropdata) || !gst_imx_region_equal(&source_region, &(compositor_pad->last_source_region)))
{
GST_LOG_OBJECT(compositor, "using new crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region));
compositor_pad->last_source_region = source_region;
compositor_pad->canvas_needs_update = TRUE;
}
compositor_pad->last_frame_with_cropdata = TRUE;
/* Update canvas and input region if necessary */
if (compositor_pad->canvas_needs_update)
gst_imx_compositor_pad_update_canvas(compositor_pad, &(compositor_pad->last_source_region));
}
else
{
/* Force an update if this frame has no crop metadata but the last one did */
if (compositor_pad->last_frame_with_cropdata)
compositor_pad->canvas_needs_update = TRUE;
compositor_pad->last_frame_with_cropdata = FALSE;
/* Update the pad's canvas if necessary,
* to ensure there is a valid canvas to draw to */
gst_imx_compositor_pad_update_canvas(compositor_pad, NULL);
}
GST_LOG_OBJECT(
compositor,
"pad %p frame %p format: %s width/height: %d/%d regions: outer %" GST_IMX_REGION_FORMAT " inner %" GST_IMX_REGION_FORMAT " source subset %" GST_IMX_REGION_FORMAT,
(gpointer)(videoaggregator_pad),
(gpointer)(videoaggregator_pad->buffer),
gst_video_format_to_string(GST_VIDEO_INFO_FORMAT(&(videoaggregator_pad->info))),
GST_VIDEO_INFO_WIDTH(&(videoaggregator_pad->info)),
GST_VIDEO_INFO_HEIGHT(&(videoaggregator_pad->info)),
GST_IMX_REGION_ARGS(&(compositor_pad->canvas.outer_region)),
GST_IMX_REGION_ARGS(&(compositor_pad->canvas.inner_region)),
GST_IMX_REGION_ARGS(&(compositor_pad->source_subset))
);
if (!klass->draw_frame(
compositor,
&(videoaggregator_pad->info),
&(compositor_pad->source_subset),
&(compositor_pad->canvas),
videoaggregator_pad->buffer,
(guint8)(compositor_pad->alpha * 255.0)
))
{
GST_ERROR_OBJECT(compositor, "error while drawing composition frame");
ret = GST_FLOW_ERROR;
break;
}
}
else
{
GST_LOG_OBJECT(compositor, "pad %p buffer is NULL, no frame to aggregate - skipping to next pad", (gpointer)(videoaggregator_pad));
}
/* Move to next pad */
walk = g_list_next(walk);
}
GST_OBJECT_UNLOCK(compositor);
}
/* Release the output buffer, since we don't need it anymore, and
* there is no reason to retain it */
klass->set_output_frame(compositor, NULL);
return ret;
}
static GstFlowReturn gst_imx_blitter_video_sink_show_frame(GstVideoSink *video_sink, GstBuffer *buf)
{
GstImxBlitterVideoSink *blitter_video_sink = GST_IMX_BLITTER_VIDEO_SINK_CAST(video_sink);
GstVideoCropMeta *video_crop_meta;
GST_IMX_BLITTER_VIDEO_SINK_LOCK(blitter_video_sink);
if (blitter_video_sink->input_crop && ((video_crop_meta = gst_buffer_get_video_crop_meta(buf)) != NULL))
{
/* Crop metadata present. Reconfigure canvas. */
GstImxRegion source_region;
source_region.x1 = video_crop_meta->x;
source_region.y1 = video_crop_meta->y;
source_region.x2 = video_crop_meta->x + video_crop_meta->width;
source_region.y2 = video_crop_meta->y + video_crop_meta->height;
/* Make sure the source region does not exceed valid bounds */
source_region.x1 = MAX(0, source_region.x1);
source_region.y1 = MAX(0, source_region.y1);
source_region.x2 = MIN(GST_VIDEO_INFO_WIDTH(&(blitter_video_sink->input_video_info)), source_region.x2);
source_region.y2 = MIN(GST_VIDEO_INFO_HEIGHT(&(blitter_video_sink->input_video_info)), source_region.y2);
GST_LOG_OBJECT(blitter_video_sink, "retrieved crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region));
/* Canvas needs to be updated if either one of these applies:
* - the current frame has crop metadata, the last one didn't
* - the new crop rectangle and the last are different */
if (!(blitter_video_sink->last_frame_with_cropdata) || !gst_imx_region_equal(&source_region, &(blitter_video_sink->last_source_region)))
{
GST_LOG_OBJECT(blitter_video_sink, "using new crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region));
blitter_video_sink->last_source_region = source_region;
blitter_video_sink->canvas_needs_update = TRUE;
}
blitter_video_sink->last_frame_with_cropdata = TRUE;
/* Update canvas and input region if necessary */
if (blitter_video_sink->canvas_needs_update)
gst_imx_blitter_video_sink_update_canvas(blitter_video_sink, &(blitter_video_sink->last_source_region));
}
else
{
/* Force an update if this frame has no crop metadata but the last one did */
if (blitter_video_sink->last_frame_with_cropdata)
blitter_video_sink->canvas_needs_update = TRUE;
blitter_video_sink->last_frame_with_cropdata = FALSE;
/* Update canvas and input region if necessary */
if (blitter_video_sink->canvas_needs_update)
gst_imx_blitter_video_sink_update_canvas(blitter_video_sink, NULL);
}
if (blitter_video_sink->canvas.visibility_mask == 0)
{
/* Visibility mask 0 -> nothing to blit */
GST_IMX_BLITTER_VIDEO_SINK_UNLOCK(blitter_video_sink);
return GST_FLOW_OK;
}
gst_imx_blitter_set_input_frame(blitter_video_sink->blitter, buf);
/* If using vsync, blit to the backbuffer, and flip
* The flipping is done by scrolling in Y direction
* by the same number of rows as there are on screen
* The scrolling is implicitely vsync'ed */
if (blitter_video_sink->use_vsync)
{
/* Select which page to write/blit to */
++blitter_video_sink->old_fb_page;
blitter_video_sink->old_fb_page %= 3;
gst_imx_blitter_video_sink_select_fb_page(blitter_video_sink, blitter_video_sink->old_fb_page);
/* The actual blitting */
gst_imx_blitter_blit(blitter_video_sink->blitter, 255);
/* Flush the blitter to make sure it does not use any cached output
* information (for example, the physical address of the previously
* selected fb page) */
gst_imx_blitter_flush(blitter_video_sink->blitter);
/* Move the current_fb_page index to the next page. See the explanation
* at the set_property PROP_USE_VSYNC block for the reason why three
* pages are expected instead of 2. */
blitter_video_sink->current_fb_page++;
blitter_video_sink->current_fb_page %= 3;
/* Flip pages now */
gst_imx_blitter_video_sink_flip_to_selected_fb_page(blitter_video_sink);
}
else
{
gst_imx_blitter_blit(blitter_video_sink->blitter, 255);
}
GST_IMX_BLITTER_VIDEO_SINK_UNLOCK(blitter_video_sink);
return GST_FLOW_OK;
}
static GstFlowReturn gst_imx_blitter_video_transform_prepare_output_buffer(GstBaseTransform *transform, GstBuffer *input, GstBuffer **outbuf)
{
gboolean passthrough;
GstImxBlitterVideoTransform *blitter_video_transform = GST_IMX_BLITTER_VIDEO_TRANSFORM(transform);
GstImxBlitterVideoTransformClass *klass = GST_IMX_BLITTER_VIDEO_TRANSFORM_CLASS(G_OBJECT_GET_CLASS(transform));
GstVideoCropMeta *video_crop_meta;
gboolean update_canvas = FALSE;
/* If either there is no input buffer or in- and output info are not equal,
* it is clear there can be no passthrough mode */
passthrough = (input != NULL) && blitter_video_transform->inout_info_equal;
GST_IMX_BLITTER_VIDEO_TRANSFORM_LOCK(blitter_video_transform);
/* Check if cropping needs to be done */
if ((input != NULL) && blitter_video_transform->input_crop && ((video_crop_meta = gst_buffer_get_video_crop_meta(input)) != NULL))
{
GstImxRegion source_region;
gint in_width, in_height;
source_region.x1 = video_crop_meta->x;
source_region.y1 = video_crop_meta->y;
source_region.x2 = video_crop_meta->x + video_crop_meta->width;
source_region.y2 = video_crop_meta->y + video_crop_meta->height;
in_width = GST_VIDEO_INFO_WIDTH(&(blitter_video_transform->input_video_info));
in_height = GST_VIDEO_INFO_HEIGHT(&(blitter_video_transform->input_video_info));
/* Make sure the source region does not exceed valid bounds */
source_region.x1 = MAX(0, source_region.x1);
source_region.y1 = MAX(0, source_region.y1);
source_region.x2 = MIN(in_width, source_region.x2);
source_region.y2 = MIN(in_height, source_region.y2);
/* If the crop rectangle encompasses the entire frame, cropping is
* effectively a no-op, so make it passthrough in that case,
* unless passthrough is already FALSE */
passthrough = passthrough && (source_region.x1 == 0) && (source_region.y1 == 0) && (source_region.x2 == in_width) && (source_region.y2 == in_height);
GST_LOG_OBJECT(blitter_video_transform, "retrieved crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region));
/* Canvas needs to be updated if either one of these applies:
* - the current frame has crop metadata, the last one didn't
* - the new crop rectangle and the last are different */
if (!(blitter_video_transform->last_frame_with_cropdata) || !gst_imx_region_equal(&source_region, &(blitter_video_transform->last_source_region)))
{
GST_LOG_OBJECT(blitter_video_transform, "using new crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region));
blitter_video_transform->last_source_region = source_region;
update_canvas = TRUE;
}
blitter_video_transform->last_frame_with_cropdata = TRUE;
}
else
{
/* Force a canvas update if this frame has no crop metadata but the last one did */
if (blitter_video_transform->last_frame_with_cropdata)
update_canvas = TRUE;
blitter_video_transform->last_frame_with_cropdata = FALSE;
}
if (update_canvas)
{
GstImxRegion source_subset;
GstImxCanvas *canvas = &(blitter_video_transform->canvas);
gst_imx_canvas_clip(
canvas,
&(canvas->outer_region),
&(blitter_video_transform->input_video_info),
blitter_video_transform->last_frame_with_cropdata ? &(blitter_video_transform->last_source_region) : NULL,
&source_subset
);
gst_imx_blitter_set_input_region(blitter_video_transform->blitter, &source_subset);
gst_imx_blitter_set_output_canvas(blitter_video_transform->blitter, canvas);
}
if ((input != NULL) && passthrough)
{
/* test for additional special cases for passthrough must not be enabled
* such case are transforms like rotation, deinterlacing ... */
passthrough = passthrough && (blitter_video_transform->canvas.inner_rotation == GST_IMX_CANVAS_INNER_ROTATION_NONE) &&
(klass->are_transforms_necessary != NULL) &&
!(klass->are_transforms_necessary(blitter_video_transform, input));
}
else if (!blitter_video_transform->inout_info_equal)
GST_LOG_OBJECT(transform, "input and output caps are not equal");
else if (blitter_video_transform->last_frame_with_cropdata && !passthrough)
GST_LOG_OBJECT(transform, "cropping is performed");
else if (input == NULL)
GST_LOG_OBJECT(transform, "input buffer is NULL");
GST_IMX_BLITTER_VIDEO_TRANSFORM_UNLOCK(blitter_video_transform);
GST_LOG_OBJECT(transform, "passthrough: %s", passthrough ? "yes" : "no");
if (passthrough)
{
/* This instructs the base class to not allocate a new buffer for
* the output, and instead pass the input buffer as the output
* (this is used in the fransform_frame function below) */
*outbuf = input;
return GST_FLOW_OK;
}
else
return GST_BASE_TRANSFORM_CLASS(gst_imx_blitter_video_transform_parent_class)->prepare_output_buffer(transform, input, outbuf);
}