FrameSource::FrameStatus GStreamerBaseFrameSourceImpl::fetch(vx_image image, vx_uint32 /*timeout*/)
{
if (end)
{
close();
return FrameSource::CLOSED;
}
handleGStreamerMessages();
if (gst_app_sink_is_eos(GST_APP_SINK(sink)))
{
close();
return FrameSource::CLOSED;
}
if ((lastFrameTimestamp.toc()/1000.0) > Application::get().getSourceDefaultTimeout())
{
close();
return FrameSource::CLOSED;
}
lastFrameTimestamp.tic();
#if GST_VERSION_MAJOR == 0
std::unique_ptr<GstBuffer, GStreamerObjectDeleter> bufferHolder(
gst_app_sink_pull_buffer(GST_APP_SINK(sink)));
GstBuffer* buffer = bufferHolder.get();
#else
std::unique_ptr<GstSample, GStreamerObjectDeleter> sample(gst_app_sink_pull_sample(GST_APP_SINK(sink)));
if (!sample)
{
close();
return FrameSource::CLOSED;
}
GstBuffer* buffer = gst_sample_get_buffer(sample.get());
#endif
gint width;
gint height;
#if GST_VERSION_MAJOR == 0
std::unique_ptr<GstCaps, GStreamerObjectDeleter> bufferCapsHolder(gst_buffer_get_caps(buffer));
GstCaps* bufferCaps = bufferCapsHolder.get();
#else
GstCaps* bufferCaps = gst_sample_get_caps(sample.get());
#endif
// bail out in no caps
assert(gst_caps_get_size(bufferCaps) == 1);
GstStructure* structure = gst_caps_get_structure(bufferCaps, 0);
// bail out if width or height are 0
if (!gst_structure_get_int(structure, "width", &width) ||
!gst_structure_get_int(structure, "height", &height))
{
close();
return FrameSource::CLOSED;
}
int depth = 3;
#if GST_VERSION_MAJOR > 0
depth = 0;
const gchar* name = gst_structure_get_name(structure);
const gchar* format = gst_structure_get_string(structure, "format");
if (!name || !format)
{
close();
return FrameSource::CLOSED;
}
// we support 2 types of data:
// video/x-raw, format=BGR -> 8bit, 3 channels
// video/x-raw, format=GRAY8 -> 8bit, 1 channel
if (strcasecmp(name, "video/x-raw") == 0)
{
if (strcasecmp(format, "RGB") == 0)
{
depth = 3;
}
else if(strcasecmp(format, "GRAY8") == 0)
{
depth = 1;
}
}
#endif
if (depth == 0)
{
close();
return FrameSource::CLOSED;
}
vx_imagepatch_addressing_t decodedImageAddr;
decodedImageAddr.dim_x = width;
decodedImageAddr.dim_y = height;
decodedImageAddr.stride_x = depth;
// GStreamer uses as stride width rounded up to the nearest multiple of 4
decodedImageAddr.stride_y = ((width*depth+3)/4)*4;
decodedImageAddr.scale_x = 1;
decodedImageAddr.scale_y = 1;
vx_image decodedImage = NULL;
vx_df_image_e vx_type_map[5] = { VX_DF_IMAGE_VIRT, VX_DF_IMAGE_U8,
VX_DF_IMAGE_VIRT, VX_DF_IMAGE_RGB, VX_DF_IMAGE_RGBX };
// fetch image width and height
vx_uint32 actual_width, actual_height;
vx_df_image_e actual_format;
NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_WIDTH, (void *)&actual_width, sizeof(actual_width)) );
NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_HEIGHT, (void *)&actual_height, sizeof(actual_height)) );
NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_FORMAT, (void *)&actual_format, sizeof(actual_format)) );
bool needScale = width != (int)configuration.frameWidth || height != (int)configuration.frameHeight;
// config and actual image sized must be the same!
if ((actual_height != configuration.frameHeight) ||
(actual_width != configuration.frameWidth) ||
(actual_format != configuration.format))
{
close();
NVXIO_THROW_EXCEPTION("Actual image [ " << actual_width << " x " << actual_height <<
" ] does not equal configuration one [ " << configuration.frameWidth
<< " x " << configuration.frameHeight << " ]");
}
// we assume that decoced image will have no more than 3 channels per pixel
if (!devMem)
{
NVXIO_ASSERT( cudaSuccess == cudaMallocPitch(&devMem, &devMemPitch, width * 3, height) );
}
// check if decoded image format has changed
if (scaledImage)
{
vx_df_image_e scaled_format;
NVXIO_SAFE_CALL( vxQueryImage(scaledImage, VX_IMAGE_ATTRIBUTE_FORMAT, (void *)&scaled_format, sizeof(scaled_format)) );
if (scaled_format != vx_type_map[depth])
{
vxReleaseImage(&scaledImage);
scaledImage = NULL;
}
}
if (needScale && !scaledImage)
{
scaledImage = vxCreateImage(vxContext, configuration.frameWidth,
configuration.frameHeight, vx_type_map[depth]);
NVXIO_CHECK_REFERENCE( scaledImage );
}
#if GST_VERSION_MAJOR == 0
bool needConvert = configuration.format != VX_DF_IMAGE_RGB;
void * decodedPtr = GST_BUFFER_DATA(buffer);
#else
GstMapInfo info;
gboolean success = gst_buffer_map(buffer, &info, (GstMapFlags)GST_MAP_READ);
if (!success)
{
printf("GStreamer: unable to map buffer\n");
close();
return FrameSource::CLOSED;
}
bool needConvert = configuration.format != vx_type_map[depth];
void * decodedPtr = info.data;
#endif
if (!needConvert && !needScale)
{
decodedImage = vxCreateImageFromHandle(vxContext, vx_type_map[depth], &decodedImageAddr,
&decodedPtr, VX_IMPORT_TYPE_HOST);
NVXIO_CHECK_REFERENCE( decodedImage );
NVXIO_SAFE_CALL( nvxuCopyImage(vxContext, decodedImage, image) );
}
else
{
// 1. upload decoced image to CUDA buffer
NVXIO_ASSERT( cudaSuccess == cudaMemcpy2D(devMem, devMemPitch,
decodedPtr, decodedImageAddr.stride_y,
decodedImageAddr.dim_x * depth, decodedImageAddr.dim_y,
cudaMemcpyHostToDevice) );
// 2. create vx_image wrapper for decoded buffer
decodedImageAddr.stride_y = static_cast<vx_int32>(devMemPitch);
decodedImage = vxCreateImageFromHandle(vxContext, vx_type_map[depth], &decodedImageAddr,
&devMem, NVX_IMPORT_TYPE_CUDA);
NVXIO_CHECK_REFERENCE( decodedImage );
if (needScale)
{
// 3. scale image
NVXIO_SAFE_CALL( vxuScaleImage(vxContext, decodedImage, scaledImage, VX_INTERPOLATION_TYPE_BILINEAR) );
// 4. convert to dst image
NVXIO_SAFE_CALL( vxuColorConvert(vxContext, scaledImage, image) );
}
else
{
// 3. convert to dst image
NVXIO_SAFE_CALL( vxuColorConvert(vxContext, decodedImage, image) );
}
}
#if GST_VERSION_MAJOR != 0
gst_buffer_unmap(buffer, &info);
#endif
NVXIO_SAFE_CALL( vxReleaseImage(&decodedImage) );
return FrameSource::OK;
}
void convertFrame(vx_context vxContext,
vx_image frame,
const FrameSource::Parameters & configuration,
vx_imagepatch_addressing_t & decodedImageAddr,
void * decodedPtr,
bool is_cuda,
void *& devMem,
size_t & devMemPitch,
vx_image & scaledImage
)
{
vx_df_image_e vx_type_map[5] = { VX_DF_IMAGE_VIRT, VX_DF_IMAGE_U8,
VX_DF_IMAGE_VIRT, VX_DF_IMAGE_RGB, VX_DF_IMAGE_RGBX };
vx_df_image_e decodedFormat = vx_type_map[decodedImageAddr.stride_x];
// fetch image width and height
vx_uint32 frameWidth, frameHeight;
vx_df_image_e frameFormat;
NVXIO_SAFE_CALL( vxQueryImage(frame, VX_IMAGE_ATTRIBUTE_WIDTH, (void *)&frameWidth, sizeof(frameWidth)) );
NVXIO_SAFE_CALL( vxQueryImage(frame, VX_IMAGE_ATTRIBUTE_HEIGHT, (void *)&frameHeight, sizeof(frameHeight)) );
NVXIO_SAFE_CALL( vxQueryImage(frame, VX_IMAGE_ATTRIBUTE_FORMAT, (void *)&frameFormat, sizeof(frameFormat)) );
bool needScale = frameWidth != decodedImageAddr.dim_x ||
frameHeight != decodedImageAddr.dim_y;
bool needConvert = frameFormat != decodedFormat;
// config and actual image sized must be the same!
if ((frameWidth != configuration.frameWidth) ||
(frameHeight != configuration.frameHeight))
{
NVXIO_THROW_EXCEPTION("Actual image [ " << frameWidth << " x " << frameHeight <<
" ] is not equal to configuration one [ " << configuration.frameWidth
<< " x " << configuration.frameHeight << " ]");
}
// allocate CUDA memory to copy decoded image to
if (!is_cuda)
{
if (!devMem)
{
// we assume that decoded image will have no more than 4 channels per pixel
NVXIO_ASSERT( cudaSuccess == cudaMallocPitch(&devMem, &devMemPitch, decodedImageAddr.dim_x * 4,
decodedImageAddr.dim_y) );
}
}
// check if decoded image format has changed
if (scaledImage)
{
vx_df_image_e scaledFormat;
NVXIO_SAFE_CALL( vxQueryImage(scaledImage, VX_IMAGE_ATTRIBUTE_FORMAT, (void *)&scaledFormat, sizeof(scaledFormat)) );
if (scaledFormat != decodedFormat)
{
NVXIO_SAFE_CALL( vxReleaseImage(&scaledImage) );
scaledImage = NULL;
}
}
if (needScale && !scaledImage)
{
scaledImage = vxCreateImage(vxContext, frameWidth, frameHeight, decodedFormat);
NVXIO_CHECK_REFERENCE( scaledImage );
}
vx_image decodedImage = NULL;
// 1. create vx_image wrapper
if (is_cuda)
{
// a. create vx_image wrapper from CUDA pointer
decodedImage = vxCreateImageFromHandle(vxContext, decodedFormat, &decodedImageAddr,
&decodedPtr, NVX_IMPORT_TYPE_CUDA);
}
else
{
// a. upload decoded image to CUDA buffer
NVXIO_ASSERT( cudaSuccess == cudaMemcpy2D(devMem, devMemPitch,
decodedPtr, decodedImageAddr.stride_y,
decodedImageAddr.dim_x * decodedImageAddr.stride_x,
decodedImageAddr.dim_y, cudaMemcpyHostToDevice) );
// b. create vx_image wrapper for decoded buffer
decodedImageAddr.stride_y = static_cast<vx_int32>(devMemPitch);
decodedImage = vxCreateImageFromHandle(vxContext, decodedFormat, &decodedImageAddr,
&devMem, NVX_IMPORT_TYPE_CUDA);
}
NVXIO_CHECK_REFERENCE( decodedImage );
// 2. scale if necessary
if (needScale)
{
// a. scale image
NVXIO_SAFE_CALL( vxuScaleImage(vxContext, decodedImage, scaledImage, VX_INTERPOLATION_TYPE_BILINEAR) );
}
else
{
scaledImage = decodedImage;
}
// 3. convert / copy to dst image
if (needConvert)
{
NVXIO_SAFE_CALL( vxuColorConvert(vxContext, scaledImage, frame) );
}
else
{
NVXIO_SAFE_CALL( nvxuCopyImage(vxContext, scaledImage, frame) );
}
if (!needScale)
scaledImage = NULL;
NVXIO_SAFE_CALL( vxReleaseImage(&decodedImage) );
}