static void gst_imx_compositor_pad_compute_outer_region(GstImxCompositorPad *compositor_pad) { GstVideoInfo *info = &(GST_IMXBP_VIDEO_AGGREGATOR_PAD(compositor_pad)->info); /* Set the outer region's top left corner */ compositor_pad->canvas.outer_region.x1 = compositor_pad->xpos; compositor_pad->canvas.outer_region.y1 = compositor_pad->ypos; /* Check if width and/or height are 0. 0 means "use the video width/height". */ if (compositor_pad->width == 0) compositor_pad->canvas.outer_region.x2 = compositor_pad->xpos + GST_VIDEO_INFO_WIDTH(info); else compositor_pad->canvas.outer_region.x2 = compositor_pad->xpos + compositor_pad->width; if (compositor_pad->height == 0) compositor_pad->canvas.outer_region.y2 = compositor_pad->ypos + GST_VIDEO_INFO_HEIGHT(info); else compositor_pad->canvas.outer_region.y2 = compositor_pad->ypos + compositor_pad->height; GST_DEBUG_OBJECT(compositor_pad, "computed outer region: %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&(compositor_pad->canvas.outer_region))); }
static GstFlowReturn gst_imx_blitter_video_sink_show_frame(GstVideoSink *video_sink, GstBuffer *buf) { GstImxBlitterVideoSink *blitter_video_sink = GST_IMX_BLITTER_VIDEO_SINK_CAST(video_sink); GstVideoCropMeta *video_crop_meta; GST_IMX_BLITTER_VIDEO_SINK_LOCK(blitter_video_sink); if (blitter_video_sink->input_crop && ((video_crop_meta = gst_buffer_get_video_crop_meta(buf)) != NULL)) { /* Crop metadata present. Reconfigure canvas. */ GstImxRegion source_region; source_region.x1 = video_crop_meta->x; source_region.y1 = video_crop_meta->y; source_region.x2 = video_crop_meta->x + video_crop_meta->width; source_region.y2 = video_crop_meta->y + video_crop_meta->height; /* Make sure the source region does not exceed valid bounds */ source_region.x1 = MAX(0, source_region.x1); source_region.y1 = MAX(0, source_region.y1); source_region.x2 = MIN(GST_VIDEO_INFO_WIDTH(&(blitter_video_sink->input_video_info)), source_region.x2); source_region.y2 = MIN(GST_VIDEO_INFO_HEIGHT(&(blitter_video_sink->input_video_info)), source_region.y2); GST_LOG_OBJECT(blitter_video_sink, "retrieved crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region)); /* Canvas needs to be updated if either one of these applies: * - the current frame has crop metadata, the last one didn't * - the new crop rectangle and the last are different */ if (!(blitter_video_sink->last_frame_with_cropdata) || !gst_imx_region_equal(&source_region, &(blitter_video_sink->last_source_region))) { GST_LOG_OBJECT(blitter_video_sink, "using new crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region)); blitter_video_sink->last_source_region = source_region; blitter_video_sink->canvas_needs_update = TRUE; } blitter_video_sink->last_frame_with_cropdata = TRUE; /* Update canvas and input region if necessary */ if (blitter_video_sink->canvas_needs_update) gst_imx_blitter_video_sink_update_canvas(blitter_video_sink, &(blitter_video_sink->last_source_region)); } else { /* Force an update if this frame has no crop metadata but the last one did */ if (blitter_video_sink->last_frame_with_cropdata) blitter_video_sink->canvas_needs_update = TRUE; blitter_video_sink->last_frame_with_cropdata = FALSE; /* Update canvas and input region if necessary */ if (blitter_video_sink->canvas_needs_update) gst_imx_blitter_video_sink_update_canvas(blitter_video_sink, NULL); } if (blitter_video_sink->canvas.visibility_mask == 0) { /* Visibility mask 0 -> nothing to blit */ GST_IMX_BLITTER_VIDEO_SINK_UNLOCK(blitter_video_sink); return GST_FLOW_OK; } gst_imx_blitter_set_input_frame(blitter_video_sink->blitter, buf); /* If using vsync, blit to the backbuffer, and flip * The flipping is done by scrolling in Y direction * by the same number of rows as there are on screen * The scrolling is implicitely vsync'ed */ if (blitter_video_sink->use_vsync) { /* Select which page to write/blit to */ ++blitter_video_sink->old_fb_page; blitter_video_sink->old_fb_page %= 3; gst_imx_blitter_video_sink_select_fb_page(blitter_video_sink, blitter_video_sink->old_fb_page); /* The actual blitting */ gst_imx_blitter_blit(blitter_video_sink->blitter, 255); /* Flush the blitter to make sure it does not use any cached output * information (for example, the physical address of the previously * selected fb page) */ gst_imx_blitter_flush(blitter_video_sink->blitter); /* Move the current_fb_page index to the next page. See the explanation * at the set_property PROP_USE_VSYNC block for the reason why three * pages are expected instead of 2. */ blitter_video_sink->current_fb_page++; blitter_video_sink->current_fb_page %= 3; /* Flip pages now */ gst_imx_blitter_video_sink_flip_to_selected_fb_page(blitter_video_sink); } else { gst_imx_blitter_blit(blitter_video_sink->blitter, 255); } GST_IMX_BLITTER_VIDEO_SINK_UNLOCK(blitter_video_sink); return GST_FLOW_OK; }
static GstFlowReturn gst_imx_compositor_aggregate_frames(GstImxBPVideoAggregator *videoaggregator, GstBuffer *outbuffer) { GstFlowReturn ret = GST_FLOW_OK; GList *walk; GstImxCompositor *compositor = GST_IMX_COMPOSITOR(videoaggregator); GstImxCompositorClass *klass = GST_IMX_COMPOSITOR_CLASS(G_OBJECT_GET_CLASS(videoaggregator)); g_assert(klass->set_output_frame != NULL); g_assert(klass->fill_region != NULL); g_assert(klass->draw_frame != NULL); /* This function is the heart of the compositor. Here, input frames * are drawn on the output frame, with their specific parameters. */ /* Set the output buffer */ if (!(klass->set_output_frame(compositor, outbuffer))) { GST_ERROR_OBJECT(compositor, "could not set the output frame"); return GST_FLOW_ERROR; } /* TODO: are the update_overall_region calls here necessary? * If the video aggregator calls update_caps when a pad is added/removed, * there is no need for these calls */ /* Update the overall region first if necessary to ensure that it is valid * and that the region_fill_necessary flag is set to the proper value */ gst_imx_compositor_update_overall_region(compositor); GST_LOG_OBJECT(compositor, "aggregating frames, region_fill_necessary: %d", (gint)(compositor->region_fill_necessary)); /* Check if the overall region needs to be filled. This is the case if none * of the input frames completely cover the overall region with 100% alpha * (this is determined by gst_imx_compositor_update_overall_region() ) */ if (!(compositor->region_fill_necessary) || klass->fill_region(compositor, &(compositor->overall_region), compositor->background_color)) { /* Lock object to ensure nothing is changed during composition */ GST_OBJECT_LOCK(compositor); /* First walk: check if there is a new pad. If so, recompute the * overall region, since it might need to be expanded to encompass * the new additional input frames */ walk = GST_ELEMENT(videoaggregator)->sinkpads; while (walk != NULL) { GstImxCompositorPad *compositor_pad = GST_IMX_COMPOSITOR_PAD_CAST(walk->data); if (compositor_pad->pad_is_new) { GST_DEBUG_OBJECT(compositor, "there is a new pad; invalidate overall region"); compositor_pad->pad_is_new = FALSE; compositor->overall_region_valid = FALSE; /* While this call might seem redundant, there is one * benefit in calling this function apparently twice * (once above, and once here): the earlier call * happens outside of the object lock. New pads are less * common than overall region changes, so it is good * if most update calls happen outside of the object * lock (the overall_region_valid flag ensures redundant * calls don't compute anything). */ gst_imx_compositor_update_overall_region(compositor); break; } /* Move to next pad */ walk = g_list_next(walk); } /* Second walk: draw the input frames on the output frame */ walk = GST_ELEMENT(videoaggregator)->sinkpads; while (walk != NULL) { GstImxBPVideoAggregatorPad *videoaggregator_pad = walk->data; GstImxCompositorPad *compositor_pad = GST_IMX_COMPOSITOR_PAD_CAST(videoaggregator_pad); /* If there actually is a buffer, draw it * Sometimes, pads don't deliver data right from the start; * in these cases, their buffers will be NULL * Just skip to the next pad in that case */ if (videoaggregator_pad->buffer != NULL) { GstVideoCropMeta *video_crop_meta; if (compositor_pad->input_crop && ((video_crop_meta = gst_buffer_get_video_crop_meta(videoaggregator_pad->buffer)) != NULL)) { /* Crop metadata present. Reconfigure canvas. */ GstVideoInfo *info = &(videoaggregator_pad->info); GstImxRegion source_region; source_region.x1 = video_crop_meta->x; source_region.y1 = video_crop_meta->y; source_region.x2 = video_crop_meta->x + video_crop_meta->width; source_region.y2 = video_crop_meta->y + video_crop_meta->height; /* Make sure the source region does not exceed valid bounds */ source_region.x1 = MAX(0, source_region.x1); source_region.y1 = MAX(0, source_region.y1); source_region.x2 = MIN(GST_VIDEO_INFO_WIDTH(info), source_region.x2); source_region.y2 = MIN(GST_VIDEO_INFO_HEIGHT(info), source_region.y2); GST_LOG_OBJECT(compositor, "retrieved crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region)); /* Canvas needs to be updated if either one of these applies: * - the current frame has crop metadata, the last one didn't * - the new crop rectangle and the last are different */ if (!(compositor_pad->last_frame_with_cropdata) || !gst_imx_region_equal(&source_region, &(compositor_pad->last_source_region))) { GST_LOG_OBJECT(compositor, "using new crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region)); compositor_pad->last_source_region = source_region; compositor_pad->canvas_needs_update = TRUE; } compositor_pad->last_frame_with_cropdata = TRUE; /* Update canvas and input region if necessary */ if (compositor_pad->canvas_needs_update) gst_imx_compositor_pad_update_canvas(compositor_pad, &(compositor_pad->last_source_region)); } else { /* Force an update if this frame has no crop metadata but the last one did */ if (compositor_pad->last_frame_with_cropdata) compositor_pad->canvas_needs_update = TRUE; compositor_pad->last_frame_with_cropdata = FALSE; /* Update the pad's canvas if necessary, * to ensure there is a valid canvas to draw to */ gst_imx_compositor_pad_update_canvas(compositor_pad, NULL); } GST_LOG_OBJECT( compositor, "pad %p frame %p format: %s width/height: %d/%d regions: outer %" GST_IMX_REGION_FORMAT " inner %" GST_IMX_REGION_FORMAT " source subset %" GST_IMX_REGION_FORMAT, (gpointer)(videoaggregator_pad), (gpointer)(videoaggregator_pad->buffer), gst_video_format_to_string(GST_VIDEO_INFO_FORMAT(&(videoaggregator_pad->info))), GST_VIDEO_INFO_WIDTH(&(videoaggregator_pad->info)), GST_VIDEO_INFO_HEIGHT(&(videoaggregator_pad->info)), GST_IMX_REGION_ARGS(&(compositor_pad->canvas.outer_region)), GST_IMX_REGION_ARGS(&(compositor_pad->canvas.inner_region)), GST_IMX_REGION_ARGS(&(compositor_pad->source_subset)) ); if (!klass->draw_frame( compositor, &(videoaggregator_pad->info), &(compositor_pad->source_subset), &(compositor_pad->canvas), videoaggregator_pad->buffer, (guint8)(compositor_pad->alpha * 255.0) )) { GST_ERROR_OBJECT(compositor, "error while drawing composition frame"); ret = GST_FLOW_ERROR; break; } } else { GST_LOG_OBJECT(compositor, "pad %p buffer is NULL, no frame to aggregate - skipping to next pad", (gpointer)(videoaggregator_pad)); } /* Move to next pad */ walk = g_list_next(walk); } GST_OBJECT_UNLOCK(compositor); } /* Release the output buffer, since we don't need it anymore, and * there is no reason to retain it */ klass->set_output_frame(compositor, NULL); return ret; }
static void gst_imx_compositor_update_overall_region(GstImxCompositor *compositor) { GList *walk; gboolean first = TRUE; /* Catch redundant calls */ if (compositor->overall_region_valid) return; if ((compositor->overall_width != 0) && (compositor->overall_height != 0)) { /* If the width and height of the overall region are fixed to specific * values by the caller, use these, and don't look at the canvases * in the input pads. */ compositor->overall_region.x2 = compositor->overall_width; compositor->overall_region.y2 = compositor->overall_height; } else { /* Overall width and/or height are set to 0. This means the caller wants * the overall region to adapt to the sizes of the input canvases. The * overall region must encompass and show all of them (exception: * pads with negative xpos/ypos coordinates can have their canvas lie * either partially or fully outside of the overall region). * To compute this overall region, walk through all pads and merge their * outer canvas regions together. */ walk = GST_ELEMENT(compositor)->sinkpads; while (walk != NULL) { GstImxCompositorPad *compositor_pad = GST_IMX_COMPOSITOR_PAD_CAST(walk->data); GstImxRegion *outer_region = &(compositor_pad->canvas.outer_region); /* Update the outer region, since the xpos/ypos/width/height pad properties * might have changed */ gst_imx_compositor_pad_compute_outer_region(compositor_pad); /* The pad canvasses are *not* updated here. This is because in order for * these updates to be done, a valid overall region needs to exist first. * And the whole point of this loop is to compute said region. * Furthermore, canvas updates anyway are unnecessary here. They'll be * done later, during frame aggregation, when necessary. The only * value that is needed here from the canvas is the outer region, and * this one is already computed above. */ if (first) { /* This is the first visited pad, so just copy its outer region */ compositor->overall_region = *outer_region; first = FALSE; } else gst_imx_region_merge(&(compositor->overall_region), &(compositor->overall_region), outer_region); GST_DEBUG_OBJECT(compositor, "current outer region: %" GST_IMX_REGION_FORMAT " merged overall region: %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(outer_region), GST_IMX_REGION_ARGS(&(compositor->overall_region))); /* Move to next pad */ walk = g_list_next(walk); } } /* Make sure the overall region starts at (0,0), since any other topleft * coordinates make little sense */ compositor->overall_region.x1 = 0; compositor->overall_region.y1 = 0; /* Now that the overall region is computed, walk through the individual * outer regions, and check if any of them completely cover the overall region * If so, the compositor does not have to clear the frame first (= filling * the overall region with fill_region), thus saving bandwidth */ compositor->region_fill_necessary = TRUE; walk = GST_ELEMENT(compositor)->sinkpads; while (walk != NULL) { GstImxCompositorPad *compositor_pad = GST_IMX_COMPOSITOR_PAD_CAST(walk->data); GstImxRegion *outer_region = &(compositor_pad->canvas.outer_region); /* Check if the outer region completely contains the overall region */ if (gst_imx_region_contains(&(compositor->overall_region), outer_region) == GST_IMX_REGION_CONTAINS_FULL) { /* disable filling if this outer region is opaque * (because it will completely cover the overall region) */ compositor->region_fill_necessary = (compositor_pad->alpha < 1.0); break; } walk = g_list_next(walk); } compositor->overall_region_valid = TRUE; }
static GstFlowReturn gst_imx_blitter_video_transform_prepare_output_buffer(GstBaseTransform *transform, GstBuffer *input, GstBuffer **outbuf) { gboolean passthrough; GstImxBlitterVideoTransform *blitter_video_transform = GST_IMX_BLITTER_VIDEO_TRANSFORM(transform); GstImxBlitterVideoTransformClass *klass = GST_IMX_BLITTER_VIDEO_TRANSFORM_CLASS(G_OBJECT_GET_CLASS(transform)); GstVideoCropMeta *video_crop_meta; gboolean update_canvas = FALSE; /* If either there is no input buffer or in- and output info are not equal, * it is clear there can be no passthrough mode */ passthrough = (input != NULL) && blitter_video_transform->inout_info_equal; GST_IMX_BLITTER_VIDEO_TRANSFORM_LOCK(blitter_video_transform); /* Check if cropping needs to be done */ if ((input != NULL) && blitter_video_transform->input_crop && ((video_crop_meta = gst_buffer_get_video_crop_meta(input)) != NULL)) { GstImxRegion source_region; gint in_width, in_height; source_region.x1 = video_crop_meta->x; source_region.y1 = video_crop_meta->y; source_region.x2 = video_crop_meta->x + video_crop_meta->width; source_region.y2 = video_crop_meta->y + video_crop_meta->height; in_width = GST_VIDEO_INFO_WIDTH(&(blitter_video_transform->input_video_info)); in_height = GST_VIDEO_INFO_HEIGHT(&(blitter_video_transform->input_video_info)); /* Make sure the source region does not exceed valid bounds */ source_region.x1 = MAX(0, source_region.x1); source_region.y1 = MAX(0, source_region.y1); source_region.x2 = MIN(in_width, source_region.x2); source_region.y2 = MIN(in_height, source_region.y2); /* If the crop rectangle encompasses the entire frame, cropping is * effectively a no-op, so make it passthrough in that case, * unless passthrough is already FALSE */ passthrough = passthrough && (source_region.x1 == 0) && (source_region.y1 == 0) && (source_region.x2 == in_width) && (source_region.y2 == in_height); GST_LOG_OBJECT(blitter_video_transform, "retrieved crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region)); /* Canvas needs to be updated if either one of these applies: * - the current frame has crop metadata, the last one didn't * - the new crop rectangle and the last are different */ if (!(blitter_video_transform->last_frame_with_cropdata) || !gst_imx_region_equal(&source_region, &(blitter_video_transform->last_source_region))) { GST_LOG_OBJECT(blitter_video_transform, "using new crop rectangle %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(&source_region)); blitter_video_transform->last_source_region = source_region; update_canvas = TRUE; } blitter_video_transform->last_frame_with_cropdata = TRUE; } else { /* Force a canvas update if this frame has no crop metadata but the last one did */ if (blitter_video_transform->last_frame_with_cropdata) update_canvas = TRUE; blitter_video_transform->last_frame_with_cropdata = FALSE; } if (update_canvas) { GstImxRegion source_subset; GstImxCanvas *canvas = &(blitter_video_transform->canvas); gst_imx_canvas_clip( canvas, &(canvas->outer_region), &(blitter_video_transform->input_video_info), blitter_video_transform->last_frame_with_cropdata ? &(blitter_video_transform->last_source_region) : NULL, &source_subset ); gst_imx_blitter_set_input_region(blitter_video_transform->blitter, &source_subset); gst_imx_blitter_set_output_canvas(blitter_video_transform->blitter, canvas); } if ((input != NULL) && passthrough) { /* test for additional special cases for passthrough must not be enabled * such case are transforms like rotation, deinterlacing ... */ passthrough = passthrough && (blitter_video_transform->canvas.inner_rotation == GST_IMX_CANVAS_INNER_ROTATION_NONE) && (klass->are_transforms_necessary != NULL) && !(klass->are_transforms_necessary(blitter_video_transform, input)); } else if (!blitter_video_transform->inout_info_equal) GST_LOG_OBJECT(transform, "input and output caps are not equal"); else if (blitter_video_transform->last_frame_with_cropdata && !passthrough) GST_LOG_OBJECT(transform, "cropping is performed"); else if (input == NULL) GST_LOG_OBJECT(transform, "input buffer is NULL"); GST_IMX_BLITTER_VIDEO_TRANSFORM_UNLOCK(blitter_video_transform); GST_LOG_OBJECT(transform, "passthrough: %s", passthrough ? "yes" : "no"); if (passthrough) { /* This instructs the base class to not allocate a new buffer for * the output, and instead pass the input buffer as the output * (this is used in the fransform_frame function below) */ *outbuf = input; return GST_FLOW_OK; } else return GST_BASE_TRANSFORM_CLASS(gst_imx_blitter_video_transform_parent_class)->prepare_output_buffer(transform, input, outbuf); }
static void gst_imx_video_compositor_update_overall_region(GstImxVideoCompositor *compositor) { GList *walk; gboolean first = TRUE; /* Catch redundant calls */ if (compositor->overall_region_valid) return; if ((compositor->overall_width != 0) && (compositor->overall_height != 0)) { /* If the width and height of the overall region are fixed to specific * values by the caller, use these, and don't look at the canvases * in the input pads. */ compositor->overall_region.x2 = compositor->overall_width; compositor->overall_region.y2 = compositor->overall_height; } else { /* Overall width and/or height are set to 0. This means the caller wants * the overall region to adapt to the sizes of the input canvases. The * overall region must encompass and show all of them (exception: * pads with negative xpos/ypos coordinates can have their canvas lie * either partially or fully outside of the overall region). * To compute this overall region, walk through all pads and merge their * outer canvas regions together. */ walk = GST_ELEMENT(compositor)->sinkpads; while (walk != NULL) { GstImxVideoCompositorPad *compositor_pad = GST_IMX_VIDEO_COMPOSITOR_PAD_CAST(walk->data); GstImxRegion *outer_region = &(compositor_pad->canvas.outer_region); /* Update the outer region, since the xpos/ypos/width/height pad properties * might have changed */ gst_imx_video_compositor_pad_compute_outer_region(compositor_pad); /* The pad canvasses are *not* updated here. This is because in order for * these updates to be done, a valid overall region needs to exist first. * And the whole point of this loop is to compute said region. * Furthermore, canvas updates anyway are unnecessary here. They'll be * done later, during frame aggregation, when necessary. The only * value that is needed here from the canvas is the outer region, and * this one is already computed above. */ if (first) { /* This is the first visited pad, so just copy its outer region */ compositor->overall_region = *outer_region; first = FALSE; } else gst_imx_region_merge(&(compositor->overall_region), &(compositor->overall_region), outer_region); GST_DEBUG_OBJECT(compositor, "current outer region: %" GST_IMX_REGION_FORMAT " merged overall region: %" GST_IMX_REGION_FORMAT, GST_IMX_REGION_ARGS(outer_region), GST_IMX_REGION_ARGS(&(compositor->overall_region))); /* Move to next pad */ walk = g_list_next(walk); } } /* Make sure the overall region starts at (0,0), since any other topleft * coordinates make little sense */ compositor->overall_region.x1 = 0; compositor->overall_region.y1 = 0; /* Now that the overall region is computed, walk through the individual * outer regions, and check if any of them completely cover the overall region * If so, the compositor does not have to clear the frame first (= filling * the overall region with fill_region), thus saving bandwidth */ compositor->region_fill_necessary = TRUE; walk = GST_ELEMENT(compositor)->sinkpads; while (walk != NULL) { GstImxVideoCompositorPad *compositor_pad = GST_IMX_VIDEO_COMPOSITOR_PAD_CAST(walk->data); GstImxRegion *outer_region = &(compositor_pad->canvas.outer_region); GstVideoInfo *info = &(GST_IMXBP_VIDEO_AGGREGATOR_PAD(compositor_pad)->info); /* Check if the outer region completely contains the overall region */ if (gst_imx_region_contains(&(compositor->overall_region), outer_region) == GST_IMX_REGION_CONTAINS_FULL) { /* The outer region completely contains the inner region. * If the video frames are opaque, then this means that * their pixels will fully overwrite the entire overall * region, so it does not have to be initially filled * with the background color. This is used as a way for * improving performance. Even if there are multiple * input video streams which each have outer regions * that fully contain the overall region, all it takes * is for just one of these to be 100% opaque, and the * region fill is not necessary. So, if such a fully * opaque region that completely covers the overall * region is found, exit the loop immediately, otherwise * keep looking at the other pads. * * Also, blending may be necessary even if alpha is 1.0, * since video frames might themselves have alpha values * per-pixel (GST_VIDEO_INFO_HAS_ALPHA() returns TRUE in * this case then). */ compositor->region_fill_necessary = (compositor_pad->alpha < 1.0) || GST_VIDEO_INFO_HAS_ALPHA(info); if (!compositor->region_fill_necessary) break; } walk = g_list_next(walk); } compositor->overall_region_valid = TRUE; }