static gboolean
gst_gl_view_convert_element_set_caps (GstGLFilter * filter, GstCaps * incaps,
GstCaps * outcaps)
{
GstGLViewConvertElement *viewconvert_filter =
GST_GL_VIEW_CONVERT_ELEMENT (filter);
GstCapsFeatures *gl_features;
gboolean ret;
GST_DEBUG_OBJECT (filter, "incaps %" GST_PTR_FORMAT
" outcaps %" GST_PTR_FORMAT, incaps, outcaps);
/* The view_convert component needs RGBA caps */
incaps = gst_caps_copy (incaps);
outcaps = gst_caps_copy (outcaps);
gst_caps_set_simple (incaps, "format", G_TYPE_STRING, "RGBA", NULL);
gl_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
gst_caps_set_features (incaps, 0, gl_features);
gst_caps_set_simple (outcaps, "format", G_TYPE_STRING, "RGBA", NULL);
gl_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
gst_caps_set_features (outcaps, 0, gl_features);
ret = gst_gl_view_convert_set_caps (viewconvert_filter->viewconvert,
incaps, outcaps);
gst_caps_unref (incaps);
gst_caps_unref (outcaps);
return ret;
}
/* Called after videoaggregator fixates our caps */
static gboolean
_negotiated_caps (GstAggregator * agg, GstCaps * caps)
{
GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg);
GstCaps *in_caps;
GST_LOG_OBJECT (mix, "Configured output caps %" GST_PTR_FORMAT, caps);
if (GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps)
if (!GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps))
return FALSE;
/* Update the glview_convert output */
/* We can configure the view_converter now */
gst_gl_view_convert_set_context (mix->viewconvert,
GST_GL_BASE_MIXER (mix)->context);
in_caps = gst_video_info_to_caps (&mix->mix_info);
gst_caps_set_features (in_caps, 0,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
gst_caps_set_simple (in_caps, "texture-target", G_TYPE_STRING,
GST_GL_TEXTURE_TARGET_2D_STR, NULL);
gst_gl_view_convert_set_caps (mix->viewconvert, in_caps, caps);
return TRUE;
}
GstCaps *
gst_gl_caps_replace_all_caps_features (const GstCaps * caps,
const gchar * feature_name)
{
GstCaps *tmp = gst_caps_copy (caps);
guint n = gst_caps_get_size (tmp);
guint i = 0;
for (i = 0; i < n; i++) {
gst_caps_set_features (tmp, i,
gst_caps_features_from_string (feature_name));
}
return tmp;
}
GstCaps *
gst_vulkan_swapper_get_supported_caps (GstVulkanSwapper * swapper,
GError ** error)
{
GstStructure *s;
GstCaps *caps;
g_return_val_if_fail (GST_IS_VULKAN_SWAPPER (swapper), NULL);
if (!_vulkan_swapper_retrieve_surface_properties (swapper, error))
return NULL;
caps = gst_caps_new_empty_simple ("video/x-raw");
gst_caps_set_features (caps, 0,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_VULKAN_BUFFER));
s = gst_caps_get_structure (caps, 0);
{
int i;
GValue list = G_VALUE_INIT;
g_value_init (&list, GST_TYPE_LIST);
if (swapper->n_surf_formats
&& swapper->surf_formats[0].format == VK_FORMAT_UNDEFINED) {
_add_vk_format_to_list (&list, VK_FORMAT_B8G8R8A8_UNORM);
} else {
for (i = 0; i < swapper->n_surf_formats; i++) {
_add_vk_format_to_list (&list, swapper->surf_formats[i].format);
}
}
gst_structure_set_value (s, "format", &list);
g_value_unset (&list);
}
{
guint32 max_dim = swapper->device->gpu_props.limits.maxImageDimension2D;
gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, (gint) max_dim,
"height", GST_TYPE_INT_RANGE, 1, (gint) max_dim, "pixel-aspect-ratio",
GST_TYPE_FRACTION, 1, 1, "framerate", GST_TYPE_FRACTION_RANGE, 0, 1,
G_MAXINT, 1, NULL);
}
GST_INFO_OBJECT (swapper, "Probed the following caps %" GST_PTR_FORMAT, caps);
return caps;
}
static void
_init_upload (GstGLStereoSplit * split)
{
GstGLContext *context = split->context;
if (!split->upload) {
GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (split->sink_pad));
GstCaps *split_caps = gst_pad_get_current_caps (split->left_pad);
GstCaps *upload_caps = gst_caps_copy (in_caps);
GstCapsFeatures *gl_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
GstCaps *gl_caps;
split->upload = gst_gl_upload_new (context);
gst_caps_set_features (upload_caps, 0,
gst_caps_features_copy (gl_features));
gst_gl_upload_set_caps (split->upload, in_caps, upload_caps);
gst_caps_unref (in_caps);
gl_caps = gst_caps_copy (upload_caps);
gst_caps_set_simple (gl_caps, "format", G_TYPE_STRING, "RGBA", NULL);
gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features));
if (!split->convert) {
split->convert = gst_gl_color_convert_new (context);
gst_gl_color_convert_set_caps (split->convert, upload_caps, gl_caps);
}
gst_caps_unref (upload_caps);
gst_caps_features_free (gl_features);
gst_gl_view_convert_set_context (split->viewconvert, split->context);
split_caps = gst_caps_make_writable (split_caps);
gst_caps_set_simple (split_caps, "multiview-mode", G_TYPE_STRING,
"separated", "views", G_TYPE_INT, 2, NULL);
gst_gl_view_convert_set_caps (split->viewconvert, gl_caps, split_caps);
gst_caps_unref (split_caps);
gst_caps_unref (gl_caps);
}
}
static gboolean
pad_can_dmabuf (GstMsdkVPP * thiz, GstPadDirection direction, GstCaps * filter)
{
gboolean ret = FALSE;
GstCaps *caps, *out_caps;
GstPad *pad;
GstBaseTransform *trans = GST_BASE_TRANSFORM (thiz);
if (direction == GST_PAD_SRC)
pad = GST_BASE_TRANSFORM_SRC_PAD (trans);
else
pad = GST_BASE_TRANSFORM_SINK_PAD (trans);
/* make a copy of filter caps since we need to alter the structure
* by adding dmabuf-capsfeatures */
caps = gst_caps_copy (filter);
gst_caps_set_features (caps, 0,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_DMABUF));
out_caps = gst_pad_peer_query_caps (pad, caps);
if (!out_caps)
goto done;
if (gst_caps_is_any (out_caps) || gst_caps_is_empty (out_caps)
|| out_caps == caps)
goto done;
if (_gst_caps_has_feature (out_caps, GST_CAPS_FEATURE_MEMORY_DMABUF))
ret = TRUE;
done:
if (caps)
gst_caps_unref (caps);
if (out_caps)
gst_caps_unref (out_caps);
return ret;
}
/* Return the possible output caps based on inputs and downstream prefs */
static GstCaps *
_update_caps (GstVideoAggregator * vagg, GstCaps * caps)
{
GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg);
GList *l;
gint best_width = -1, best_height = -1;
gdouble best_fps = -1, cur_fps;
gint best_fps_n = 0, best_fps_d = 1;
GstVideoInfo *mix_info;
GstCaps *blend_caps, *tmp_caps;
GstCaps *out_caps;
GST_OBJECT_LOCK (vagg);
for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
GstVideoAggregatorPad *pad = l->data;
GstVideoInfo tmp = pad->info;
gint this_width, this_height;
gint fps_n, fps_d;
if (!pad->info.finfo)
continue;
/* This can happen if we release a pad and another pad hasn't been negotiated_caps yet */
if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN)
continue;
/* Convert to per-view width/height for unpacked forms */
gst_video_multiview_video_info_change_mode (&tmp,
GST_VIDEO_MULTIVIEW_MODE_SEPARATED, GST_VIDEO_MULTIVIEW_FLAGS_NONE);
this_width = GST_VIDEO_INFO_WIDTH (&tmp);
this_height = GST_VIDEO_INFO_HEIGHT (&tmp);
fps_n = GST_VIDEO_INFO_FPS_N (&tmp);
fps_d = GST_VIDEO_INFO_FPS_D (&tmp);
GST_INFO_OBJECT (vagg, "Input pad %" GST_PTR_FORMAT
" w %u h %u", pad, this_width, this_height);
if (this_width == 0 || this_height == 0)
continue;
if (best_width < this_width)
best_width = this_width;
if (best_height < this_height)
best_height = this_height;
if (fps_d == 0)
cur_fps = 0.0;
else
gst_util_fraction_to_double (fps_n, fps_d, &cur_fps);
if (best_fps < cur_fps) {
best_fps = cur_fps;
best_fps_n = fps_n;
best_fps_d = fps_d;
}
/* FIXME: Preserve PAR for at least one input when different sized inputs */
}
GST_OBJECT_UNLOCK (vagg);
mix_info = &mix->mix_info;
gst_video_info_set_format (mix_info, GST_VIDEO_FORMAT_RGBA, best_width,
best_height);
GST_VIDEO_INFO_FPS_N (mix_info) = best_fps_n;
GST_VIDEO_INFO_FPS_D (mix_info) = best_fps_d;
GST_VIDEO_INFO_MULTIVIEW_MODE (mix_info) = GST_VIDEO_MULTIVIEW_MODE_SEPARATED;
GST_VIDEO_INFO_VIEWS (mix_info) = 2;
/* FIXME: If input is marked as flipped or flopped, preserve those flags */
GST_VIDEO_INFO_MULTIVIEW_FLAGS (mix_info) = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
/* Choose our output format based on downstream preferences */
blend_caps = gst_video_info_to_caps (mix_info);
gst_caps_set_features (blend_caps, 0,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
tmp_caps = get_converted_caps (GST_GL_STEREO_MIX (vagg), blend_caps);
gst_caps_unref (blend_caps);
out_caps = gst_caps_intersect (caps, tmp_caps);
gst_caps_unref (tmp_caps);
GST_DEBUG_OBJECT (vagg, "Possible output caps %" GST_PTR_FORMAT, out_caps);
return out_caps;
}
static void
check_conversion (TestFrame * frames, guint size)
{
gint i, j, k, l;
gint ref_count = 0;
for (i = 0; i < size; i++) {
GstBuffer *inbuf;
GstVideoInfo in_info;
gint in_width = frames[i].width;
gint in_height = frames[i].height;
GstVideoFormat in_v_format = frames[i].v_format;
gchar *in_data[GST_VIDEO_MAX_PLANES] = { 0 };
GstGLMemory *in_mem[GST_VIDEO_MAX_PLANES] = { 0 };
GstVideoFrame in_frame;
GstCaps *in_caps;
gst_video_info_set_format (&in_info, in_v_format, in_width, in_height);
in_caps = gst_video_info_to_caps (&in_info);
gst_caps_set_features (in_caps, 0,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
for (j = 0; j < GST_VIDEO_INFO_N_PLANES (&in_info); j++) {
in_data[j] = frames[i].data[j];
}
/* create GL buffer */
ref_count += GST_VIDEO_INFO_N_PLANES (&in_info);
inbuf = gst_buffer_new ();
fail_unless (gst_gl_memory_setup_wrapped (context, GST_GL_TEXTURE_TARGET_2D,
&in_info, NULL, (gpointer *) in_data, in_mem, &ref_count,
_frame_unref));
for (j = 0; j < GST_VIDEO_INFO_N_PLANES (&in_info); j++) {
gst_buffer_append_memory (inbuf, (GstMemory *) in_mem[j]);
}
fail_unless (gst_video_frame_map (&in_frame, &in_info, inbuf,
GST_MAP_READ));
/* sanity check that the correct values were wrapped */
for (j = 0; j < GST_VIDEO_INFO_N_PLANES (&in_info); j++) {
for (k = 0; k < _video_info_plane_size (&in_info, j); k++) {
if (in_data[j][k] != IGNORE_MAGIC)
fail_unless (((gchar *) in_frame.data[j])[k] == in_data[j][k]);
}
}
for (j = 0; j < size; j++) {
GstBuffer *outbuf;
GstVideoInfo out_info;
gint out_width = frames[j].width;
gint out_height = frames[j].height;
GstVideoFormat out_v_format = frames[j].v_format;
gchar *out_data[GST_VIDEO_MAX_PLANES] = { 0 };
GstVideoFrame out_frame;
GstCaps *out_caps;
gst_video_info_set_format (&out_info, out_v_format, out_width,
out_height);
out_caps = gst_video_info_to_caps (&out_info);
gst_caps_set_features (out_caps, 0,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
for (k = 0; k < GST_VIDEO_INFO_N_PLANES (&out_info); k++) {
out_data[k] = frames[j].data[k];
}
gst_gl_color_convert_set_caps (convert, in_caps, out_caps);
/* convert the data */
outbuf = gst_gl_color_convert_perform (convert, inbuf);
if (outbuf == NULL) {
const gchar *in_str = gst_video_format_to_string (in_v_format);
const gchar *out_str = gst_video_format_to_string (out_v_format);
GST_WARNING ("failed to convert from %s to %s", in_str, out_str);
}
fail_unless (gst_video_frame_map (&out_frame, &out_info, outbuf,
GST_MAP_READ));
/* check that the converted values are correct */
for (k = 0; k < GST_VIDEO_INFO_N_PLANES (&out_info); k++) {
for (l = 0; l < _video_info_plane_size (&out_info, k); l++) {
gchar out_pixel = ((gchar *) out_frame.data[k])[l];
if (out_data[k][l] != IGNORE_MAGIC && out_pixel != IGNORE_MAGIC)
fail_unless (out_pixel == out_data[k][l]);
/* FIXME: check alpha clobbering */
}
}
gst_caps_unref (out_caps);
gst_video_frame_unmap (&out_frame);
gst_buffer_unref (outbuf);
}
gst_caps_unref (in_caps);
gst_video_frame_unmap (&in_frame);
gst_buffer_unref (inbuf);
fail_unless_equals_int (ref_count, 0);
}
}
static void
check_conversion (TestFrame * frames, guint size)
{
GstGLBaseMemoryAllocator *base_mem_alloc;
gint i, j, k, l;
gint ref_count = 0;
base_mem_alloc =
GST_GL_BASE_MEMORY_ALLOCATOR (gst_allocator_find
(GST_GL_MEMORY_ALLOCATOR_NAME));
for (i = 0; i < size; i++) {
GstBuffer *inbuf;
GstVideoInfo in_info;
gint in_width = frames[i].width;
gint in_height = frames[i].height;
GstVideoFormat in_v_format = frames[i].v_format;
GstVideoFrame in_frame;
GstCaps *in_caps;
gst_video_info_set_format (&in_info, in_v_format, in_width, in_height);
in_caps = gst_video_info_to_caps (&in_info);
gst_caps_set_features (in_caps, 0,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
/* create GL buffer */
inbuf = gst_buffer_new ();
for (j = 0; j < GST_VIDEO_INFO_N_PLANES (&in_info); j++) {
GstVideoGLTextureType tex_type = gst_gl_texture_type_from_format (context,
GST_VIDEO_INFO_FORMAT (&in_info), j);
GstGLVideoAllocationParams *params;
GstGLBaseMemory *mem;
ref_count++;
params = gst_gl_video_allocation_params_new_wrapped_data (context, NULL,
&in_info, j, NULL, GST_GL_TEXTURE_TARGET_2D, tex_type,
frames[i].data[j], &ref_count, _frame_unref);
mem = gst_gl_base_memory_alloc (base_mem_alloc,
(GstGLAllocationParams *) params);
gst_buffer_append_memory (inbuf, GST_MEMORY_CAST (mem));
gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
}
fail_unless (gst_video_frame_map (&in_frame, &in_info, inbuf,
GST_MAP_READ));
/* sanity check that the correct values were wrapped */
for (j = 0; j < GST_VIDEO_INFO_N_PLANES (&in_info); j++) {
for (k = 0; k < _video_info_plane_size (&in_info, j); k++) {
if (frames[i].data[j][k] != IGNORE_MAGIC)
fail_unless (((gchar *) in_frame.data[j])[k] == frames[i].data[j][k]);
}
}
for (j = 0; j < size; j++) {
GstBuffer *outbuf;
GstVideoInfo out_info;
gint out_width = frames[j].width;
gint out_height = frames[j].height;
GstVideoFormat out_v_format = frames[j].v_format;
gchar *out_data[GST_VIDEO_MAX_PLANES] = { 0 };
GstVideoFrame out_frame;
GstCaps *out_caps;
gst_video_info_set_format (&out_info, out_v_format, out_width,
out_height);
out_caps = gst_video_info_to_caps (&out_info);
gst_caps_set_features (out_caps, 0,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
for (k = 0; k < GST_VIDEO_INFO_N_PLANES (&out_info); k++) {
out_data[k] = frames[j].data[k];
}
gst_gl_color_convert_set_caps (convert, in_caps, out_caps);
/* convert the data */
outbuf = gst_gl_color_convert_perform (convert, inbuf);
if (outbuf == NULL) {
const gchar *in_str = gst_video_format_to_string (in_v_format);
const gchar *out_str = gst_video_format_to_string (out_v_format);
GST_WARNING ("failed to convert from %s to %s", in_str, out_str);
}
fail_unless (gst_video_frame_map (&out_frame, &out_info, outbuf,
GST_MAP_READ));
/* check that the converted values are correct */
for (k = 0; k < GST_VIDEO_INFO_N_PLANES (&out_info); k++) {
for (l = 0; l < _video_info_plane_size (&out_info, k); l++) {
gchar out_pixel = ((gchar *) out_frame.data[k])[l];
if (out_data[k][l] != IGNORE_MAGIC && out_pixel != IGNORE_MAGIC)
fail_unless (out_pixel == out_data[k][l]);
/* FIXME: check alpha clobbering */
}
}
gst_caps_unref (out_caps);
gst_video_frame_unmap (&out_frame);
gst_buffer_unref (outbuf);
}
gst_caps_unref (in_caps);
gst_video_frame_unmap (&in_frame);
gst_buffer_unref (inbuf);
fail_unless_equals_int (ref_count, 0);
}
gst_object_unref (base_mem_alloc);
}
static GstBuffer *
_default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame,
GstBuffer * buffer)
{
GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad);
GstGLMixerPad *pad = frame->pad;
GstBuffer *uploaded_buf, *gl_buffer;
GstCaps *gl_caps;
GstCapsFeatures *gl_features;
GstVideoInfo gl_info;
GstVideoFrame gl_frame;
GstGLSyncMeta *sync_meta;
gst_video_info_set_format (&gl_info,
GST_VIDEO_FORMAT_RGBA,
GST_VIDEO_INFO_WIDTH (&vaggpad->info),
GST_VIDEO_INFO_HEIGHT (&vaggpad->info));
gl_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
gl_caps = gst_video_info_to_caps (&gl_info);
gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features));
if (!pad->upload) {
GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad));
GstCaps *upload_caps = gst_caps_copy (in_caps);
pad->upload = gst_gl_upload_new (mix->context);
gst_caps_set_features (upload_caps, 0,
gst_caps_features_copy (gl_features));
gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps);
if (!pad->convert) {
pad->convert = gst_gl_color_convert_new (mix->context);
gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps);
}
gst_caps_unref (upload_caps);
gst_caps_unref (in_caps);
}
gst_caps_features_free (gl_features);
gst_caps_unref (gl_caps);
sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer);
if (sync_meta)
gst_gl_sync_meta_wait (sync_meta);
if (gst_gl_upload_perform_with_buffer (pad->upload,
vaggpad->buffer, &uploaded_buf) != GST_GL_UPLOAD_DONE) {
return NULL;
}
if (!(gl_buffer = gst_gl_color_convert_perform (pad->convert, uploaded_buf))) {
gst_buffer_unref (uploaded_buf);
return NULL;
}
if (!gst_video_frame_map (&gl_frame, &gl_info, gl_buffer,
GST_MAP_READ | GST_MAP_GL)) {
gst_buffer_unref (uploaded_buf);
gst_buffer_unref (gl_buffer);
return NULL;
}
frame->texture = *(guint *) gl_frame.data[0];
gst_buffer_unref (uploaded_buf);
gst_video_frame_unmap (&gl_frame);
return gl_buffer;
}