void GStreamerBaseFrameSourceImpl::FinalizeGstPipeLine()
{
if (pipeline)
{
handleGStreamerMessages();
gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_NULL);
handleGStreamerMessages();
gst_object_unref(GST_OBJECT(bus));
bus = NULL;
gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_NULL);
gst_object_unref(GST_OBJECT(pipeline));
pipeline = NULL;
}
}
void GStreamerBaseFrameSourceImpl::close()
{
handleGStreamerMessages();
FinalizeGstPipeLine();
if (devMem != NULL)
{
cudaFree(devMem);
devMem = NULL;
}
if (scaledImage)
{
vxReleaseImage(&scaledImage);
scaledImage = NULL;
}
}
bool GStreamerCameraFrameSourceImpl::InitializeGstPipeLine()
{
GstStateChangeReturn status;
end = true;
pipeline = GST_PIPELINE(gst_pipeline_new(NULL));
if (pipeline == NULL)
{
printf("Cannot create Gstreamer pipeline\n");
return false;
}
bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
// create v4l2src
GstElement * v4l2src = gst_element_factory_make("v4l2src", NULL);
if (v4l2src == NULL)
{
printf("Cannot create v4l2src\n");
FinalizeGstPipeLine();
return false;
}
std::ostringstream cameraDev;
cameraDev << "/dev/video" << cameraIdx;
g_object_set(G_OBJECT(v4l2src), "device", cameraDev.str().c_str(), NULL);
gst_bin_add(GST_BIN(pipeline), v4l2src);
// create color convert element
GstElement * color = gst_element_factory_make(COLOR_ELEM, NULL);
if (color == NULL)
{
printf("Cannot create %s element\n", COLOR_ELEM);
FinalizeGstPipeLine();
return false;
}
gst_bin_add(GST_BIN(pipeline), color);
// create appsink element
sink = gst_element_factory_make("appsink", NULL);
if (sink == NULL)
{
printf("Cannot create appsink element\n");
FinalizeGstPipeLine();
return false;
}
gst_bin_add(GST_BIN(pipeline), sink);
// if initial values for FrameSource::Parameters are not
// specified, let's set them manually to prevent very huge images
if (configuration.frameWidth == (vx_uint32)-1)
configuration.frameWidth = 1920;
if (configuration.frameHeight == (vx_uint32)-1)
configuration.frameHeight = 1080;
if (configuration.fps == (vx_uint32)-1)
configuration.fps = 30;
#if GST_VERSION_MAJOR == 0
GstCaps* caps_v42lsrc = gst_caps_new_simple ("video/x-raw-rgb",
"width", GST_TYPE_INT_RANGE, 1, (int)configuration.frameWidth,
"height", GST_TYPE_INT_RANGE, 1, (int)configuration.frameHeight,
"framerate", GST_TYPE_FRACTION, (int)configuration.fps,
NULL);
#else
std::ostringstream stream;
stream << "video/x-raw, format=(string){RGB, GRAY8}, width=[1," << configuration.frameWidth <<
"], height=[1," << configuration.frameHeight << "], framerate=" << configuration.fps << "/1;";
GstCaps* caps_v42lsrc = gst_caps_from_string(stream.str().c_str());
#endif
if (caps_v42lsrc == NULL)
{
printf("Failed to create caps\n");
FinalizeGstPipeLine();
return false;
}
// link elements
if (!gst_element_link_filtered(v4l2src, color, caps_v42lsrc))
{
printf("GStreamer: cannot link v4l2src -> color using caps\n");
FinalizeGstPipeLine();
gst_caps_unref(caps_v42lsrc);
return false;
}
gst_caps_unref(caps_v42lsrc);
// link elements
if (!gst_element_link(color, sink))
{
printf("GStreamer: cannot link color -> appsink\n");
FinalizeGstPipeLine();
return false;
}
gst_app_sink_set_max_buffers (GST_APP_SINK(sink), 1);
gst_app_sink_set_drop (GST_APP_SINK(sink), true);
// do not emit signals: all calls will be synchronous and blocking
gst_app_sink_set_emit_signals (GST_APP_SINK(sink), 0);
#if GST_VERSION_MAJOR == 0
GstCaps* caps_appsink = gst_caps_new_simple("video/x-raw-rgb",
"bpp", G_TYPE_INT, 24,
"red_mask", G_TYPE_INT, 0xFF0000,
"green_mask", G_TYPE_INT, 0x00FF00,
"blue_mask", G_TYPE_INT, 0x0000FF,
NULL);
#else
// support 1 and 3 channel 8 bit data
GstCaps* caps_appsink = gst_caps_from_string("video/x-raw, format=(string){RGB, GRAY8};");
#endif
gst_app_sink_set_caps(GST_APP_SINK(sink), caps_appsink);
gst_caps_unref(caps_appsink);
// Force pipeline to play video as fast as possible, ignoring system clock
gst_pipeline_use_clock(pipeline, NULL);
status = gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PLAYING);
handleGStreamerMessages();
if (status == GST_STATE_CHANGE_ASYNC)
{
// wait for status update
status = gst_element_get_state(GST_ELEMENT(pipeline), NULL, NULL, GST_CLOCK_TIME_NONE);
}
if (status == GST_STATE_CHANGE_FAILURE)
{
printf("GStreamer: unable to start playback\n");
FinalizeGstPipeLine();
return false;
}
std::unique_ptr<GstPad, GStreamerObjectDeleter> pad(gst_element_get_static_pad(color, "src"));
#if GST_VERSION_MAJOR == 0
std::unique_ptr<GstCaps, GStreamerObjectDeleter> bufferCaps(gst_pad_get_caps(pad.get()));
#else
std::unique_ptr<GstCaps, GStreamerObjectDeleter> bufferCaps(gst_pad_get_current_caps(pad.get()));
#endif
const GstStructure *structure = gst_caps_get_structure(bufferCaps.get(), 0);
int width, height;
if (!gst_structure_get_int(structure, "width", &width))
{
handleGStreamerMessages();
printf("Cannot query video width\n");
}
if (!gst_structure_get_int(structure, "height", &height))
{
handleGStreamerMessages();
printf("Cannot query video height\n");
}
configuration.frameWidth = static_cast<vx_uint32>(width);
configuration.frameHeight = static_cast<vx_uint32>(height);
gint num = 0, denom = 1;
if (!gst_structure_get_fraction(structure, "framerate", &num, &denom))
{
handleGStreamerMessages();
printf("Cannot query video fps\n");
}
configuration.fps = static_cast<float>(num) / denom;
end = false;
return true;
}
FrameSource::FrameStatus GStreamerBaseFrameSourceImpl::fetch(vx_image image, vx_uint32 /*timeout*/)
{
if (end)
{
close();
return FrameSource::CLOSED;
}
handleGStreamerMessages();
if (gst_app_sink_is_eos(GST_APP_SINK(sink)))
{
close();
return FrameSource::CLOSED;
}
if ((lastFrameTimestamp.toc()/1000.0) > Application::get().getSourceDefaultTimeout())
{
close();
return FrameSource::CLOSED;
}
lastFrameTimestamp.tic();
#if GST_VERSION_MAJOR == 0
std::unique_ptr<GstBuffer, GStreamerObjectDeleter> bufferHolder(
gst_app_sink_pull_buffer(GST_APP_SINK(sink)));
GstBuffer* buffer = bufferHolder.get();
#else
std::unique_ptr<GstSample, GStreamerObjectDeleter> sample(gst_app_sink_pull_sample(GST_APP_SINK(sink)));
if (!sample)
{
close();
return FrameSource::CLOSED;
}
GstBuffer* buffer = gst_sample_get_buffer(sample.get());
#endif
gint width;
gint height;
#if GST_VERSION_MAJOR == 0
std::unique_ptr<GstCaps, GStreamerObjectDeleter> bufferCapsHolder(gst_buffer_get_caps(buffer));
GstCaps* bufferCaps = bufferCapsHolder.get();
#else
GstCaps* bufferCaps = gst_sample_get_caps(sample.get());
#endif
// bail out in no caps
assert(gst_caps_get_size(bufferCaps) == 1);
GstStructure* structure = gst_caps_get_structure(bufferCaps, 0);
// bail out if width or height are 0
if (!gst_structure_get_int(structure, "width", &width) ||
!gst_structure_get_int(structure, "height", &height))
{
close();
return FrameSource::CLOSED;
}
int depth = 3;
#if GST_VERSION_MAJOR > 0
depth = 0;
const gchar* name = gst_structure_get_name(structure);
const gchar* format = gst_structure_get_string(structure, "format");
if (!name || !format)
{
close();
return FrameSource::CLOSED;
}
// we support 2 types of data:
// video/x-raw, format=BGR -> 8bit, 3 channels
// video/x-raw, format=GRAY8 -> 8bit, 1 channel
if (strcasecmp(name, "video/x-raw") == 0)
{
if (strcasecmp(format, "RGB") == 0)
{
depth = 3;
}
else if(strcasecmp(format, "GRAY8") == 0)
{
depth = 1;
}
}
#endif
if (depth == 0)
{
close();
return FrameSource::CLOSED;
}
vx_imagepatch_addressing_t decodedImageAddr;
decodedImageAddr.dim_x = width;
decodedImageAddr.dim_y = height;
decodedImageAddr.stride_x = depth;
// GStreamer uses as stride width rounded up to the nearest multiple of 4
decodedImageAddr.stride_y = ((width*depth+3)/4)*4;
decodedImageAddr.scale_x = 1;
decodedImageAddr.scale_y = 1;
vx_image decodedImage = NULL;
vx_df_image_e vx_type_map[5] = { VX_DF_IMAGE_VIRT, VX_DF_IMAGE_U8,
VX_DF_IMAGE_VIRT, VX_DF_IMAGE_RGB, VX_DF_IMAGE_RGBX };
// fetch image width and height
vx_uint32 actual_width, actual_height;
vx_df_image_e actual_format;
NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_WIDTH, (void *)&actual_width, sizeof(actual_width)) );
NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_HEIGHT, (void *)&actual_height, sizeof(actual_height)) );
NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_FORMAT, (void *)&actual_format, sizeof(actual_format)) );
bool needScale = width != (int)configuration.frameWidth || height != (int)configuration.frameHeight;
// config and actual image sized must be the same!
if ((actual_height != configuration.frameHeight) ||
(actual_width != configuration.frameWidth) ||
(actual_format != configuration.format))
{
close();
NVXIO_THROW_EXCEPTION("Actual image [ " << actual_width << " x " << actual_height <<
" ] does not equal configuration one [ " << configuration.frameWidth
<< " x " << configuration.frameHeight << " ]");
}
// we assume that decoced image will have no more than 3 channels per pixel
if (!devMem)
{
NVXIO_ASSERT( cudaSuccess == cudaMallocPitch(&devMem, &devMemPitch, width * 3, height) );
}
// check if decoded image format has changed
if (scaledImage)
{
vx_df_image_e scaled_format;
NVXIO_SAFE_CALL( vxQueryImage(scaledImage, VX_IMAGE_ATTRIBUTE_FORMAT, (void *)&scaled_format, sizeof(scaled_format)) );
if (scaled_format != vx_type_map[depth])
{
vxReleaseImage(&scaledImage);
scaledImage = NULL;
}
}
if (needScale && !scaledImage)
{
scaledImage = vxCreateImage(vxContext, configuration.frameWidth,
configuration.frameHeight, vx_type_map[depth]);
NVXIO_CHECK_REFERENCE( scaledImage );
}
#if GST_VERSION_MAJOR == 0
bool needConvert = configuration.format != VX_DF_IMAGE_RGB;
void * decodedPtr = GST_BUFFER_DATA(buffer);
#else
GstMapInfo info;
gboolean success = gst_buffer_map(buffer, &info, (GstMapFlags)GST_MAP_READ);
if (!success)
{
printf("GStreamer: unable to map buffer\n");
close();
return FrameSource::CLOSED;
}
bool needConvert = configuration.format != vx_type_map[depth];
void * decodedPtr = info.data;
#endif
if (!needConvert && !needScale)
{
decodedImage = vxCreateImageFromHandle(vxContext, vx_type_map[depth], &decodedImageAddr,
&decodedPtr, VX_IMPORT_TYPE_HOST);
NVXIO_CHECK_REFERENCE( decodedImage );
NVXIO_SAFE_CALL( nvxuCopyImage(vxContext, decodedImage, image) );
}
else
{
// 1. upload decoced image to CUDA buffer
NVXIO_ASSERT( cudaSuccess == cudaMemcpy2D(devMem, devMemPitch,
decodedPtr, decodedImageAddr.stride_y,
decodedImageAddr.dim_x * depth, decodedImageAddr.dim_y,
cudaMemcpyHostToDevice) );
// 2. create vx_image wrapper for decoded buffer
decodedImageAddr.stride_y = static_cast<vx_int32>(devMemPitch);
decodedImage = vxCreateImageFromHandle(vxContext, vx_type_map[depth], &decodedImageAddr,
&devMem, NVX_IMPORT_TYPE_CUDA);
NVXIO_CHECK_REFERENCE( decodedImage );
if (needScale)
{
// 3. scale image
NVXIO_SAFE_CALL( vxuScaleImage(vxContext, decodedImage, scaledImage, VX_INTERPOLATION_TYPE_BILINEAR) );
// 4. convert to dst image
NVXIO_SAFE_CALL( vxuColorConvert(vxContext, scaledImage, image) );
}
else
{
// 3. convert to dst image
NVXIO_SAFE_CALL( vxuColorConvert(vxContext, decodedImage, image) );
}
}
#if GST_VERSION_MAJOR != 0
gst_buffer_unmap(buffer, &info);
#endif
NVXIO_SAFE_CALL( vxReleaseImage(&decodedImage) );
return FrameSource::OK;
}
bool GStreamerNvCameraFrameSourceImpl::InitializeGstPipeLine()
{
// select config with max FPS value to be default
NvCameraConfigs nvcameraconfig = configs[2];
// use user specified camera config
if ( (configuration.frameWidth != (vx_uint32)-1) &&
(configuration.frameHeight != (vx_uint32)-1) )
{
nvcameraconfig.frameWidth = configuration.frameWidth;
nvcameraconfig.frameHeight = configuration.frameHeight;
nvcameraconfig.fps = 30;
// select FPS default for the specified config
for (vx_size i = 0; i < dimOf(configs); ++i)
{
if ((nvcameraconfig.frameWidth == configs[i].frameWidth) &&
(nvcameraconfig.frameHeight == configs[i].frameHeight))
{
nvcameraconfig.fps = configs[i].fps;
break;
}
}
}
if (configuration.fps == (vx_uint32)-1)
configuration.fps = nvcameraconfig.fps;
end = true;
pipeline = GST_PIPELINE(gst_pipeline_new(NULL));
if (pipeline == NULL)
{
NVXIO_PRINT("Cannot create Gstreamer pipeline");
return false;
}
bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
// create nvcamerasrc
GstElement * nvcamerasrc = gst_element_factory_make("nvcamerasrc", NULL);
if (nvcamerasrc == NULL)
{
NVXIO_PRINT("Cannot create nvcamerasrc");
NVXIO_PRINT("\"nvcamerasrc\" element is not available on this platform");
FinalizeGstPipeLine();
return false;
}
std::ostringstream stream;
stream << configuration.fps << " " << configuration.fps;
std::string fpsRange = stream.str();
g_object_set(G_OBJECT(nvcamerasrc), "sensor-id", cameraIdx, NULL);
g_object_set(G_OBJECT(nvcamerasrc), "fpsRange", fpsRange.c_str(), NULL);
gst_bin_add(GST_BIN(pipeline), nvcamerasrc);
// create nvvideosink element
GstElement * nvvideosink = gst_element_factory_make("nvvideosink", NULL);
if (nvvideosink == NULL)
{
NVXIO_PRINT("Cannot create nvvideosink element");
FinalizeGstPipeLine();
return false;
}
g_object_set(G_OBJECT(nvvideosink), "display", context.display, NULL);
g_object_set(G_OBJECT(nvvideosink), "stream", context.stream, NULL);
g_object_set(G_OBJECT(nvvideosink), "fifo", fifoMode, NULL);
g_object_set(G_OBJECT(nvvideosink), "max-lateness", -1, NULL);
g_object_set(G_OBJECT(nvvideosink), "throttle-time", 0, NULL);
g_object_set(G_OBJECT(nvvideosink), "render-delay", 0, NULL);
g_object_set(G_OBJECT(nvvideosink), "qos", FALSE, NULL);
g_object_set(G_OBJECT(nvvideosink), "sync", FALSE, NULL);
g_object_set(G_OBJECT(nvvideosink), "async", TRUE, NULL);
gst_bin_add(GST_BIN(pipeline), nvvideosink);
// link elements
stream.str(std::string());
stream << "video/x-raw(memory:NVMM), width=(int)" << nvcameraconfig.frameWidth << ", "
"height=(int)" << nvcameraconfig.frameHeight << ", format=(string){I420}, "
"framerate=(fraction)" << nvcameraconfig.fps << "/1;";
std::unique_ptr<GstCaps, GStreamerObjectDeleter> caps_nvvidconv(
gst_caps_from_string(stream.str().c_str()));
if (!caps_nvvidconv)
{
NVXIO_PRINT("Failed to create caps");
FinalizeGstPipeLine();
return false;
}
if (!gst_element_link_filtered(nvcamerasrc, nvvideosink, caps_nvvidconv.get()))
{
NVXIO_PRINT("GStreamer: cannot link nvvidconv -> nvvideosink using caps");
FinalizeGstPipeLine();
return false;
}
// Force pipeline to play video as fast as possible, ignoring system clock
gst_pipeline_use_clock(pipeline, NULL);
GstStateChangeReturn status = gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PLAYING);
handleGStreamerMessages();
if (status == GST_STATE_CHANGE_ASYNC)
{
// wait for status update
status = gst_element_get_state(GST_ELEMENT(pipeline), NULL, NULL, GST_CLOCK_TIME_NONE);
}
if (status == GST_STATE_CHANGE_FAILURE)
{
NVXIO_PRINT("GStreamer: unable to start playback");
FinalizeGstPipeLine();
return false;
}
vx_uint32 initialFPS = configuration.fps;
if (!updateConfiguration(nvcamerasrc, configuration))
{
FinalizeGstPipeLine();
return false;
}
// if initialFPS is specified, we should use this, because
// retrieved via the updateConfiguration function FPS corresponds
// to camera config FPS
if (initialFPS != (vx_uint32)-1)
configuration.fps = initialFPS;
end = false;
return true;
}
bool GStreamerOpenMAXFrameSourceImpl::InitializeGstPipeLine()
{
GstStateChangeReturn status;
end = true;
std::string uri;
if (!gst_uri_is_valid(fileName.c_str()))
{
char* real = realpath(fileName.c_str(), NULL);
if (!real)
{
NVXIO_PRINT("Can't resolve path \"%s\": %s", fileName.c_str(), strerror(errno));
return false;
}
std::unique_ptr<char[], GlibDeleter> pUri(g_filename_to_uri(real, NULL, NULL));
free(real);
uri = pUri.get();
}
else
{
uri = fileName;
}
pipeline = GST_PIPELINE(gst_pipeline_new(NULL));
if (pipeline == NULL)
{
NVXIO_PRINT("Cannot create Gstreamer pipeline");
return false;
}
bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
// create uridecodebin
GstBin * uriDecodeBin = GST_BIN(gst_element_factory_make("uridecodebin", NULL));
if (uriDecodeBin == NULL)
{
NVXIO_PRINT("Cannot create uridecodebin");
FinalizeGstPipeLine();
return false;
}
g_object_set(G_OBJECT(uriDecodeBin), "uri", uri.c_str(), NULL);
g_object_set(G_OBJECT(uriDecodeBin), "message-forward", TRUE, NULL);
gst_bin_add(GST_BIN(pipeline), GST_ELEMENT(uriDecodeBin));
// create nvvidconv
GstElement * nvvidconv = gst_element_factory_make("nvvidconv", NULL);
if (nvvidconv == NULL)
{
NVXIO_PRINT("Cannot create nvvidconv");
FinalizeGstPipeLine();
return false;
}
gst_bin_add(GST_BIN(pipeline), nvvidconv);
// create nvvideosink element
GstElement * nvvideosink = gst_element_factory_make("nvvideosink", NULL);
if (nvvideosink == NULL)
{
NVXIO_PRINT("Cannot create nvvideosink element");
FinalizeGstPipeLine();
return false;
}
g_object_set(G_OBJECT(nvvideosink), "display", context.display, NULL);
g_object_set(G_OBJECT(nvvideosink), "stream", context.stream, NULL);
g_object_set(G_OBJECT(nvvideosink), "fifo", fifoMode, NULL);
gst_bin_add(GST_BIN(pipeline), nvvideosink);
g_signal_connect(uriDecodeBin, "autoplug-select", G_CALLBACK(GStreamerOpenMAXFrameSourceImpl::autoPlugSelect), NULL);
g_signal_connect(uriDecodeBin, "pad-added", G_CALLBACK(GStreamerBaseFrameSourceImpl::newGstreamerPad), nvvidconv);
std::unique_ptr<GstCaps, GStreamerObjectDeleter> caps_nvvidconv(
gst_caps_from_string("video/x-raw(memory:NVMM), format=(string){I420}"));
// link nvvidconv using caps
if (!gst_element_link_filtered(nvvidconv, nvvideosink, caps_nvvidconv.get()))
{
NVXIO_PRINT("GStreamer: cannot link nvvidconv -> nvvideosink");
FinalizeGstPipeLine();
return false;
}
// Force pipeline to play video as fast as possible, ignoring system clock
gst_pipeline_use_clock(pipeline, NULL);
status = gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PLAYING);
handleGStreamerMessages();
if (status == GST_STATE_CHANGE_ASYNC)
{
// wait for status update
status = gst_element_get_state(GST_ELEMENT(pipeline), NULL, NULL, GST_CLOCK_TIME_NONE);
}
if (status == GST_STATE_CHANGE_FAILURE)
{
NVXIO_PRINT("GStreamer: unable to start playback");
FinalizeGstPipeLine();
return false;
}
// GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "gst_pipeline");
if (!updateConfiguration(nvvidconv, configuration))
{
FinalizeGstPipeLine();
return false;
}
end = false;
return true;
}
