// // decode buffer // IplImage * CvCapture_GStreamer::retrieveFrame(int) { if(!buffer) return false; if(!frame) { gint height, width; GstCaps *buff_caps = gst_buffer_get_caps(buffer); assert(gst_caps_get_size(buff_caps) == 1); GstStructure* structure = gst_caps_get_structure(buff_caps, 0); if(!gst_structure_get_int(structure, "width", &width) || !gst_structure_get_int(structure, "height", &height)) return false; frame = cvCreateImageHeader(cvSize(width, height), IPL_DEPTH_8U, 3); gst_caps_unref(buff_caps); } // no need to memcpy, just use gstreamer's buffer :-) frame->imageData = (char *)GST_BUFFER_DATA(buffer); //memcpy (frame->imageData, GST_BUFFER_DATA(buffer), GST_BUFFER_SIZE (buffer)); //gst_buffer_unref(buffer); //buffer = 0; return frame; }
static void swfdec_video_decoder_gst_decode (SwfdecVideoDecoder *dec, SwfdecBuffer *buffer) { SwfdecVideoDecoderGst *player = SWFDEC_VIDEO_DECODER_GST (dec); #define SWFDEC_ALIGN(x, n) (((x) + (n) - 1) & (~((n) - 1))) GstBuffer *buf; GstCaps *caps; GstStructure *structure; buf = swfdec_gst_buffer_new (swfdec_buffer_ref (buffer)); if (!swfdec_gst_decoder_push (&player->dec, buf)) { swfdec_video_decoder_error (dec, "failed to push buffer"); return; } buf = swfdec_gst_decoder_pull (&player->dec); if (buf == NULL) { SWFDEC_ERROR ("failed to pull decoded buffer. Broken stream?"); return; } else { if (player->last) gst_buffer_unref (player->last); player->last = buf; } while ((buf = swfdec_gst_decoder_pull (&player->dec))) { SWFDEC_ERROR ("too many output buffers!"); gst_buffer_unref (buf); } caps = gst_buffer_get_caps (player->last); if (caps == NULL) { swfdec_video_decoder_error (dec, "no caps on decoded buffer"); return; } structure = gst_caps_get_structure (caps, 0); if (!gst_structure_get_int (structure, "width", (int *) &dec->width) || !gst_structure_get_int (structure, "height", (int *) &dec->height)) { swfdec_video_decoder_error (dec, "invalid caps on decoded buffer"); return; } buf = player->last; switch (swfdec_video_codec_get_format (dec->codec)) { case SWFDEC_VIDEO_FORMAT_RGBA: dec->plane[0] = buf->data; dec->rowstride[0] = dec->width * 4; break; case SWFDEC_VIDEO_FORMAT_I420: dec->plane[0] = buf->data; dec->rowstride[0] = SWFDEC_ALIGN (dec->width, 4); dec->plane[1] = dec->plane[0] + dec->rowstride[0] * SWFDEC_ALIGN (dec->height, 2); dec->rowstride[1] = SWFDEC_ALIGN (dec->width, 8) / 2; dec->plane[2] = dec->plane[1] + dec->rowstride[1] * SWFDEC_ALIGN (dec->height, 2) / 2; dec->rowstride[2] = dec->rowstride[1]; g_assert (dec->plane[2] + dec->rowstride[2] * SWFDEC_ALIGN (dec->height, 2) / 2 == dec->plane[0] + buf->size); break; default: g_return_if_reached (); } #undef SWFDEC_ALIGN }
static GstFlowReturn gst_test_reverse_negotiation_sink_render (GstBaseSink * bsink, GstBuffer * buffer) { GstTestReverseNegotiationSink *sink = GST_TEST_REVERSE_NEGOTIATION_SINK (bsink); GstCaps *caps = gst_buffer_get_caps (buffer); GstVideoFormat fmt; gint width, height; fail_unless (caps != NULL); fail_unless (gst_video_format_parse_caps (caps, &fmt, &width, &height)); sink->nbuffers++; /* The third buffer is still in the old size * because the ffmpegcolorspaces can't convert * the frame sizes */ if (sink->nbuffers > 3) { fail_unless_equals_int (width, 512); fail_unless_equals_int (height, 128); } gst_caps_unref (caps); return GST_FLOW_OK; }
static GstFlowReturn gst_nle_source_push_still_picture (GstNleSource * nlesrc, GstNleSrcItem * item, GstBuffer * buf) { GstCaps *bcaps, *ncaps; guint64 buf_dur; gint i, n_bufs; GstFlowReturn ret = GST_FLOW_OK; buf_dur = GST_SECOND * nlesrc->fps_d / nlesrc->fps_n; n_bufs = item->duration / buf_dur; bcaps = gst_buffer_get_caps (buf); ncaps = gst_caps_make_writable (bcaps); gst_caps_set_simple (ncaps, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL); gst_buffer_set_caps (buf, ncaps); gst_caps_unref (ncaps); nlesrc->video_seek_done = TRUE; for (i = 0; i < n_bufs; i++) { GstBuffer *new_buf; new_buf = gst_buffer_copy (buf); GST_BUFFER_TIMESTAMP (new_buf) = item->start + buf_dur * i; GST_BUFFER_DURATION (new_buf) = buf_dur; ret = gst_nle_source_push_buffer (nlesrc, new_buf, FALSE); if (ret <= GST_FLOW_UNEXPECTED) { break; } } gst_buffer_unref (buf); return ret; }
/*! Returns audio format for a buffer. If the buffer doesn't have a valid audio format, an empty QAudioFormat is returned. */ QAudioFormat QGstUtils::audioFormatForBuffer(GstBuffer *buffer) { GstCaps* caps = gst_buffer_get_caps(buffer); if (!caps) return QAudioFormat(); QAudioFormat format = QGstUtils::audioFormatForCaps(caps); gst_caps_unref(caps); return format; }
/* chain function * this function does the actual processing */ static GstFlowReturn gst_throttle_chain(GstPad * pad, GstBuffer * buf) { GstThrottle * filter = GST_THROTTLE(GST_OBJECT_PARENT(pad)); if (filter->printOnly) { GstCaps * caps = gst_buffer_get_caps(buf); gchar * capsStr = gst_caps_to_string(caps); gst_caps_unref(caps); GST_LOG_OBJECT(filter, "ts: %" GST_TIME_FORMAT " %sof type %s", GST_TIME_ARGS(buf->timestamp), GST_BUFFER_IS_DISCONT(buf) ? "and discontinuity " : "", capsStr ); g_free(capsStr); GstFlowReturn ret = gst_pad_push(filter->srcpad, buf); GST_TRACE_OBJECT(filter, "ts: %" GST_TIME_FORMAT " processed with status %d", GST_TIME_ARGS(buf->timestamp), ret); return ret; } if (filter->clock == NULL) { return gst_pad_push(filter->srcpad, buf); } GstClockTime realTs = gst_clock_get_time(filter->clock); if (filter->haveStartTime) { const char * discont = GST_BUFFER_IS_DISCONT(buf) ? " with discotinuity" : ""; GstClockTime expectedRealTs = filter->streamStartRealTime + buf->timestamp; gboolean early = realTs < expectedRealTs; if (early) { GstClockID * cid = gst_clock_new_single_shot_id(filter->clock, expectedRealTs); GST_TRACE_OBJECT(filter, "ts: %" GST_TIME_FORMAT " %s, waiting for %ld ms", GST_TIME_ARGS(buf->timestamp), discont, (expectedRealTs - realTs)/1000000); gst_clock_id_wait(cid, NULL); gst_clock_id_unref(cid); } else { GST_TRACE_OBJECT(filter, "ts: %" GST_TIME_FORMAT " %s, pad on time", GST_TIME_ARGS(buf->timestamp), discont); } } else { filter->streamStartRealTime = realTs - buf->timestamp; filter->haveStartTime = TRUE; } return gst_pad_push(filter->srcpad, buf); }
GstFlowReturn AudioFileReader::handleBuffer(GstAppSink* sink) { GstBuffer* buffer = gst_app_sink_pull_buffer(sink); if (!buffer) return GST_FLOW_ERROR; GstCaps* caps = gst_buffer_get_caps(buffer); GstStructure* structure = gst_caps_get_structure(caps, 0); gint channels = 0; if (!gst_structure_get_int(structure, "channels", &channels) || !channels) { gst_caps_unref(caps); gst_buffer_unref(buffer); return GST_FLOW_ERROR; } gint sampleRate = 0; if (!gst_structure_get_int(structure, "rate", &sampleRate) || !sampleRate) { gst_caps_unref(caps); gst_buffer_unref(buffer); return GST_FLOW_ERROR; } gint width = 0; if (!gst_structure_get_int(structure, "width", &width) || !width) { gst_caps_unref(caps); gst_buffer_unref(buffer); return GST_FLOW_ERROR; } GstClockTime duration = (static_cast<guint64>(GST_BUFFER_SIZE(buffer)) * 8 * GST_SECOND) / (sampleRate * channels * width); int frames = GST_CLOCK_TIME_TO_FRAMES(duration, sampleRate); // Check the first audio channel. The buffer is supposed to store // data of a single channel anyway. GstAudioChannelPosition* positions = gst_audio_get_channel_positions(structure); switch (positions[0]) { case GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT: gst_buffer_list_iterator_add(m_frontLeftBuffersIterator, buffer); m_channelSize += frames; break; case GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT: gst_buffer_list_iterator_add(m_frontRightBuffersIterator, buffer); break; default: gst_buffer_unref(buffer); break; } g_free(positions); gst_caps_unref(caps); return GST_FLOW_OK; }
void QGstreamerAudioProbeControl::bufferProbed(GstBuffer* buffer) { GstCaps* caps = gst_buffer_get_caps(buffer); if (!caps) return; QAudioFormat format = QGstUtils::audioFormatForCaps(caps); gst_caps_unref(caps); if (!format.isValid()) return; QAudioBuffer audioBuffer = QAudioBuffer(QByteArray((const char*)buffer->data, buffer->size), format); { QMutexLocker locker(&m_bufferMutex); m_pendingBuffer = audioBuffer; QMetaObject::invokeMethod(this, "bufferProbed", Qt::QueuedConnection); } }
PassRefPtr<ImageGStreamer> ImageGStreamer::createImage(GstBuffer* buffer) { int width = 0, height = 0; GstCaps* caps = gst_buffer_get_caps(buffer); GstVideoFormat format; if (!gst_video_format_parse_caps(caps, &format, &width, &height)) { gst_caps_unref(caps); return 0; } gst_caps_unref(caps); cairo_format_t cairoFormat; if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) cairoFormat = CAIRO_FORMAT_ARGB32; else cairoFormat = CAIRO_FORMAT_RGB24; return adoptRef(new ImageGStreamer(buffer, IntSize(width, height), cairoFormat)); }
/* Saves photos */ void acam_webcam_take_photo (GstElement *element, GstBuffer *buffer, GstPad *pad, acam_webcam_device_s *acam_webcam_device) { GstCaps *caps; const GstStructure *structure; int width, height, stride; GdkPixbuf *pixbuf; const int bits_per_pixel = 8; time_t now; struct tm *d; char li[30]; gchar *filename; gchar *home_directory; time (&now); d = localtime (&now); strftime (li, 30, "%d-%m-%Y-%H-%M-%S", d); /* Create name photo */ filename = g_strdup_printf("%s/dev%d-%s.jpg", acam_webcam_device->photo_directory, acam_webcam_device->dev_num + 1, li); g_print (" Taking photo | device %d | %s\n", acam_webcam_device->dev_num + 1, filename); caps = gst_buffer_get_caps (buffer); structure = gst_caps_get_structure (caps, 0); gst_structure_get_int (structure, "width", &width); gst_structure_get_int (structure, "height", &height); stride = buffer->size / height; pixbuf = gdk_pixbuf_new_from_data (GST_BUFFER_DATA (buffer), GDK_COLORSPACE_RGB, FALSE, bits_per_pixel, width, height, stride, NULL, NULL); gdk_pixbuf_save (pixbuf, filename, "jpeg", NULL, NULL); g_object_unref (G_OBJECT (pixbuf)); g_signal_handler_disconnect (G_OBJECT (acam_webcam_device->photo_sink), acam_webcam_device->photo_handler_signal_id); }
static gboolean query_data(struct input_handle* ih) { GstBuffer *preroll; GstCaps *src_caps; GstStructure *s; int i; ih->n_channels = 0; ih->sample_rate = 0; ih->channel_positions = NULL; preroll = gst_app_sink_pull_preroll(GST_APP_SINK(ih->appsink)); src_caps = gst_buffer_get_caps(preroll); s = gst_caps_get_structure(src_caps, 0); gst_structure_get_int(s, "rate", &(ih->sample_rate)); gst_structure_get_int(s, "channels", &(ih->n_channels)); if (!ih->sample_rate || !ih->n_channels) { gst_caps_unref(src_caps); gst_buffer_unref(preroll); return FALSE; } ih->channel_positions = gst_audio_get_channel_positions(s); if (verbose) { if (ih->channel_positions) { for (i = 0; i < ih->n_channels; ++i) { printf("Channel %d: %d\n", i, ih->channel_positions[i]); } } g_print ("%d channels @ %d Hz\n", ih->n_channels, ih->sample_rate); } gst_caps_unref(src_caps); gst_buffer_unref(preroll); return TRUE; }
std::auto_ptr<GnashImage> VideoDecoderGst::pop() { GstBuffer * buffer = swfdec_gst_decoder_pull (&_decoder); if (!buffer) { return std::auto_ptr<GnashImage>(); } GstCaps* caps = gst_buffer_get_caps(buffer); assert(gst_caps_get_size(caps) == 1); GstStructure* structure = gst_caps_get_structure (caps, 0); gst_structure_get_int (structure, "width", &_width); gst_structure_get_int (structure, "height", &_height); gst_caps_unref(caps); std::auto_ptr<GnashImage> ret(new gnashGstBuffer(buffer, _width, _height)); return ret; }
/* * Test if the parser pushes clean data properly. */ void gst_parser_test_run (GstParserTest * test, GstCaps ** out_caps) { buffer_verify_data_s vdata = { 0, 0, 0, NULL, 0, NULL, FALSE }; GstElement *element; GstBuffer *buffer = NULL; GstCaps *src_caps; guint i, j, k; guint frames = 0, size = 0; element = setup_element (test->factory, test->sink_template, NULL, test->src_template, test->src_caps); /* push some setup headers */ for (j = 0; j < G_N_ELEMENTS (test->headers) && test->headers[j].data; j++) { buffer = buffer_new (test->headers[j].data, test->headers[j].size); fail_unless_equals_int (gst_pad_push (srcpad, buffer), GST_FLOW_OK); } for (j = 0; j < 3; j++) { for (i = 0; i < test->series[j].num; i++) { /* sanity enforcing */ for (k = 0; k < MAX (1, test->series[j].fpb); k++) { if (!k) buffer = buffer_new (test->series[j].data, test->series[j].size); else { GstCaps *caps = gst_buffer_get_caps (buffer); buffer = gst_buffer_join (buffer, buffer_new (test->series[j].data, test->series[j].size)); if (caps) { gst_buffer_set_caps (buffer, caps); gst_caps_unref (caps); } } } fail_unless_equals_int (gst_pad_push (srcpad, buffer), GST_FLOW_OK); if (j == 0) vdata.buffers_before_offset_skip++; else if (j == 1) vdata.offset_skip_amount += test->series[j].size * test->series[j].fpb; if (j != 1) { frames += test->series[j].fpb; size += test->series[j].size * test->series[j].fpb; } } } gst_pad_push_event (srcpad, gst_event_new_eos ()); if (G_LIKELY (test->framed)) fail_unless_equals_int (g_list_length (buffers) - test->discard, frames); /* if all frames are identical, do extended test, * otherwise only verify total data size */ if (test->series[0].data && (!test->series[2].size || (test->series[0].size == test->series[2].size && test->series[2].data && !memcmp (test->series[0].data, test->series[2].data, test->series[0].size)))) { vdata.data_to_verify = test->series[0].data; vdata.data_to_verify_size = test->series[0].size; vdata.caps = test->sink_caps; vdata.discard = test->discard; vdata.no_metadata = test->no_metadata; g_list_foreach (buffers, buffer_verify_data, &vdata); } else { guint datasum = 0; g_list_foreach (buffers, buffer_count_size, &datasum); size -= test->dropped; fail_unless_equals_int (datasum, size); } src_caps = gst_pad_get_negotiated_caps (sinkpad); GST_LOG ("output caps: %" GST_PTR_FORMAT, src_caps); if (test->sink_caps) { GST_LOG ("%" GST_PTR_FORMAT " = %" GST_PTR_FORMAT " ?", src_caps, test->sink_caps); fail_unless (gst_caps_is_equal (src_caps, test->sink_caps)); } if (out_caps) *out_caps = src_caps; else gst_caps_unref (src_caps); cleanup_element (element); }
static void bp_vis_pcm_handoff (GstElement *sink, GstBuffer *buffer, GstPad *pad, gpointer userdata) { BansheePlayer *player = (BansheePlayer*)userdata; GstStructure *structure; gint channels, wanted_size; gfloat *data; BansheePlayerVisDataCallback vis_data_cb; g_return_if_fail (IS_BANSHEE_PLAYER (player)); vis_data_cb = player->vis_data_cb; if (vis_data_cb == NULL) { return; } if (player->vis_thawing) { // Flush our buffers out. gst_adapter_clear (player->vis_buffer); memset (player->vis_fft_sample_buffer, 0, sizeof(gfloat) * SLICE_SIZE); player->vis_thawing = FALSE; } structure = gst_caps_get_structure (gst_buffer_get_caps (buffer), 0); gst_structure_get_int (structure, "channels", &channels); wanted_size = channels * SLICE_SIZE * sizeof (gfloat); gst_adapter_push (player->vis_buffer, gst_buffer_copy (buffer)); while ((data = (gfloat *)gst_adapter_peek (player->vis_buffer, wanted_size)) != NULL) { gfloat *deinterlaced = g_malloc (wanted_size); gfloat *specbuf = g_new (gfloat, SLICE_SIZE * 2); gint i, j; memcpy (specbuf, player->vis_fft_sample_buffer, SLICE_SIZE * sizeof(gfloat)); for (i = 0; i < SLICE_SIZE; i++) { gfloat avg = 0.0f; for (j = 0; j < channels; j++) { gfloat sample = data[i * channels + j]; deinterlaced[j * SLICE_SIZE + i] = sample; avg += sample; } avg /= channels; specbuf[i + SLICE_SIZE] = avg; } memcpy (player->vis_fft_sample_buffer, &specbuf[SLICE_SIZE], SLICE_SIZE * sizeof(gfloat)); gst_fft_f32_window (player->vis_fft, specbuf, GST_FFT_WINDOW_HAMMING); gst_fft_f32_fft (player->vis_fft, specbuf, player->vis_fft_buffer); for (i = 0; i < SLICE_SIZE; i++) { gfloat val; GstFFTF32Complex cplx = player->vis_fft_buffer[i]; val = cplx.r * cplx.r + cplx.i * cplx.i; val /= SLICE_SIZE * SLICE_SIZE; val = 10.0f * log10f(val); val = (val + 60.0f) / 60.0f; if (val < 0.0f) val = 0.0f; specbuf[i] = val; } vis_data_cb (player, channels, SLICE_SIZE, deinterlaced, SLICE_SIZE, specbuf); g_free (deinterlaced); g_free (specbuf); gst_adapter_flush (player->vis_buffer, wanted_size); } }
void FarsightChannel::OnFakeSinkHandoff(GstElement *fakesink, GstBuffer *buffer, GstPad *pad, gpointer user_data) { FarsightChannel* self = (FarsightChannel*)user_data; static GStaticMutex mutex = G_STATIC_MUTEX_INIT; g_static_mutex_lock (&mutex); gst_buffer_ref(buffer); int rate = 0; int channels = 0; int width = 0; GstCaps *caps; GstStructure *structure; caps = gst_buffer_get_caps(buffer); structure = gst_caps_get_structure(caps, 0); gst_structure_get_int(structure, "rate", &rate); gst_structure_get_int(structure, "channels", &channels); gst_structure_get_int(structure, "width", &width); gst_caps_unref(caps); if (GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_PREROLL)) { LogInfo("Drop fakesink buffer: Preroll audio data packet."); gst_buffer_unref(buffer); g_static_mutex_unlock (&mutex); return; } if (GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_GAP)) { LogInfo("Drop fakesink buffer: Caps audio data packet."); gst_buffer_unref(buffer); g_static_mutex_unlock (&mutex); return; } if (GST_BUFFER_DURATION(buffer) == 0) { LogInfo("Drop fakesink buffer: Got audio data packet with 0 duration"); gst_buffer_unref(buffer); g_static_mutex_unlock (&mutex); return; } if (GST_BUFFER_IS_DISCONT(buffer)) { LogInfo("Drop fakesink buffer: Got disconnect audio data packet."); gst_buffer_unref(buffer); g_static_mutex_unlock (&mutex); return; } if (GST_BUFFER_OFFSET_IS_VALID(buffer)) { guint64 offset = GST_BUFFER_OFFSET (buffer); } if (GST_BUFFER_OFFSET_END_IS_VALID(buffer)) { guint64 offset = GST_BUFFER_OFFSET_END(buffer); } u8* data = GST_BUFFER_DATA(buffer); u32 size = GST_BUFFER_SIZE(buffer); self->HandleAudioData(data, size, rate, width, channels); gst_buffer_unref(buffer); g_static_mutex_unlock (&mutex); }
void MediaPlayerPrivate::paint(GraphicsContext* context, const IntRect& rect) { if (context->paintingDisabled()) return; if (!m_player->visible()) return; if (!m_buffer) return; int width = 0, height = 0; int pixelAspectRatioNumerator = 0; int pixelAspectRatioDenominator = 0; double doublePixelAspectRatioNumerator = 0; double doublePixelAspectRatioDenominator = 0; double displayWidth; double displayHeight; double scale, gapHeight, gapWidth; GstCaps *caps = gst_buffer_get_caps(m_buffer); if (!gst_video_format_parse_caps(caps, NULL, &width, &height) || !gst_video_parse_caps_pixel_aspect_ratio(caps, &pixelAspectRatioNumerator, &pixelAspectRatioDenominator)) { gst_caps_unref(caps); return; } displayWidth = width; displayHeight = height; doublePixelAspectRatioNumerator = pixelAspectRatioNumerator; doublePixelAspectRatioDenominator = pixelAspectRatioDenominator; cairo_t* cr = context->platformContext(); cairo_surface_t* src = cairo_image_surface_create_for_data(GST_BUFFER_DATA(m_buffer), CAIRO_FORMAT_RGB24, width, height, 4 * width); cairo_save(cr); cairo_set_operator(cr, CAIRO_OPERATOR_SOURCE); displayWidth *= doublePixelAspectRatioNumerator / doublePixelAspectRatioDenominator; displayHeight *= doublePixelAspectRatioDenominator / doublePixelAspectRatioNumerator; scale = MIN (rect.width () / displayWidth, rect.height () / displayHeight); displayWidth *= scale; displayHeight *= scale; // Calculate gap between border an picture gapWidth = (rect.width() - displayWidth) / 2.0; gapHeight = (rect.height() - displayHeight) / 2.0; // paint the rectangle on the context and draw the surface inside. cairo_translate(cr, rect.x() + gapWidth, rect.y() + gapHeight); cairo_rectangle(cr, 0, 0, rect.width(), rect.height()); cairo_scale(cr, doublePixelAspectRatioNumerator / doublePixelAspectRatioDenominator, doublePixelAspectRatioDenominator / doublePixelAspectRatioNumerator); cairo_scale(cr, scale, scale); cairo_set_source_surface(cr, src, 0, 0); cairo_fill(cr); cairo_restore(cr); cairo_surface_destroy(src); gst_caps_unref(caps); }
FrameSource::FrameStatus GStreamerBaseFrameSourceImpl::fetch(vx_image image, vx_uint32 /*timeout*/) { if (end) { close(); return FrameSource::CLOSED; } handleGStreamerMessages(); if (gst_app_sink_is_eos(GST_APP_SINK(sink))) { close(); return FrameSource::CLOSED; } if ((lastFrameTimestamp.toc()/1000.0) > Application::get().getSourceDefaultTimeout()) { close(); return FrameSource::CLOSED; } lastFrameTimestamp.tic(); #if GST_VERSION_MAJOR == 0 std::unique_ptr<GstBuffer, GStreamerObjectDeleter> bufferHolder( gst_app_sink_pull_buffer(GST_APP_SINK(sink))); GstBuffer* buffer = bufferHolder.get(); #else std::unique_ptr<GstSample, GStreamerObjectDeleter> sample(gst_app_sink_pull_sample(GST_APP_SINK(sink))); if (!sample) { close(); return FrameSource::CLOSED; } GstBuffer* buffer = gst_sample_get_buffer(sample.get()); #endif gint width; gint height; #if GST_VERSION_MAJOR == 0 std::unique_ptr<GstCaps, GStreamerObjectDeleter> bufferCapsHolder(gst_buffer_get_caps(buffer)); GstCaps* bufferCaps = bufferCapsHolder.get(); #else GstCaps* bufferCaps = gst_sample_get_caps(sample.get()); #endif // bail out in no caps assert(gst_caps_get_size(bufferCaps) == 1); GstStructure* structure = gst_caps_get_structure(bufferCaps, 0); // bail out if width or height are 0 if (!gst_structure_get_int(structure, "width", &width) || !gst_structure_get_int(structure, "height", &height)) { close(); return FrameSource::CLOSED; } int depth = 3; #if GST_VERSION_MAJOR > 0 depth = 0; const gchar* name = gst_structure_get_name(structure); const gchar* format = gst_structure_get_string(structure, "format"); if (!name || !format) { close(); return FrameSource::CLOSED; } // we support 2 types of data: // video/x-raw, format=BGR -> 8bit, 3 channels // video/x-raw, format=GRAY8 -> 8bit, 1 channel if (strcasecmp(name, "video/x-raw") == 0) { if (strcasecmp(format, "RGB") == 0) { depth = 3; } else if(strcasecmp(format, "GRAY8") == 0) { depth = 1; } } #endif if (depth == 0) { close(); return FrameSource::CLOSED; } vx_imagepatch_addressing_t decodedImageAddr; decodedImageAddr.dim_x = width; decodedImageAddr.dim_y = height; decodedImageAddr.stride_x = depth; // GStreamer uses as stride width rounded up to the nearest multiple of 4 decodedImageAddr.stride_y = ((width*depth+3)/4)*4; decodedImageAddr.scale_x = 1; decodedImageAddr.scale_y = 1; vx_image decodedImage = NULL; vx_df_image_e vx_type_map[5] = { VX_DF_IMAGE_VIRT, VX_DF_IMAGE_U8, VX_DF_IMAGE_VIRT, VX_DF_IMAGE_RGB, VX_DF_IMAGE_RGBX }; // fetch image width and height vx_uint32 actual_width, actual_height; vx_df_image_e actual_format; NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_WIDTH, (void *)&actual_width, sizeof(actual_width)) ); NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_HEIGHT, (void *)&actual_height, sizeof(actual_height)) ); NVXIO_SAFE_CALL( vxQueryImage(image, VX_IMAGE_ATTRIBUTE_FORMAT, (void *)&actual_format, sizeof(actual_format)) ); bool needScale = width != (int)configuration.frameWidth || height != (int)configuration.frameHeight; // config and actual image sized must be the same! if ((actual_height != configuration.frameHeight) || (actual_width != configuration.frameWidth) || (actual_format != configuration.format)) { close(); NVXIO_THROW_EXCEPTION("Actual image [ " << actual_width << " x " << actual_height << " ] does not equal configuration one [ " << configuration.frameWidth << " x " << configuration.frameHeight << " ]"); } // we assume that decoced image will have no more than 3 channels per pixel if (!devMem) { NVXIO_ASSERT( cudaSuccess == cudaMallocPitch(&devMem, &devMemPitch, width * 3, height) ); } // check if decoded image format has changed if (scaledImage) { vx_df_image_e scaled_format; NVXIO_SAFE_CALL( vxQueryImage(scaledImage, VX_IMAGE_ATTRIBUTE_FORMAT, (void *)&scaled_format, sizeof(scaled_format)) ); if (scaled_format != vx_type_map[depth]) { vxReleaseImage(&scaledImage); scaledImage = NULL; } } if (needScale && !scaledImage) { scaledImage = vxCreateImage(vxContext, configuration.frameWidth, configuration.frameHeight, vx_type_map[depth]); NVXIO_CHECK_REFERENCE( scaledImage ); } #if GST_VERSION_MAJOR == 0 bool needConvert = configuration.format != VX_DF_IMAGE_RGB; void * decodedPtr = GST_BUFFER_DATA(buffer); #else GstMapInfo info; gboolean success = gst_buffer_map(buffer, &info, (GstMapFlags)GST_MAP_READ); if (!success) { printf("GStreamer: unable to map buffer\n"); close(); return FrameSource::CLOSED; } bool needConvert = configuration.format != vx_type_map[depth]; void * decodedPtr = info.data; #endif if (!needConvert && !needScale) { decodedImage = vxCreateImageFromHandle(vxContext, vx_type_map[depth], &decodedImageAddr, &decodedPtr, VX_IMPORT_TYPE_HOST); NVXIO_CHECK_REFERENCE( decodedImage ); NVXIO_SAFE_CALL( nvxuCopyImage(vxContext, decodedImage, image) ); } else { // 1. upload decoced image to CUDA buffer NVXIO_ASSERT( cudaSuccess == cudaMemcpy2D(devMem, devMemPitch, decodedPtr, decodedImageAddr.stride_y, decodedImageAddr.dim_x * depth, decodedImageAddr.dim_y, cudaMemcpyHostToDevice) ); // 2. create vx_image wrapper for decoded buffer decodedImageAddr.stride_y = static_cast<vx_int32>(devMemPitch); decodedImage = vxCreateImageFromHandle(vxContext, vx_type_map[depth], &decodedImageAddr, &devMem, NVX_IMPORT_TYPE_CUDA); NVXIO_CHECK_REFERENCE( decodedImage ); if (needScale) { // 3. scale image NVXIO_SAFE_CALL( vxuScaleImage(vxContext, decodedImage, scaledImage, VX_INTERPOLATION_TYPE_BILINEAR) ); // 4. convert to dst image NVXIO_SAFE_CALL( vxuColorConvert(vxContext, scaledImage, image) ); } else { // 3. convert to dst image NVXIO_SAFE_CALL( vxuColorConvert(vxContext, decodedImage, image) ); } } #if GST_VERSION_MAJOR != 0 gst_buffer_unmap(buffer, &info); #endif NVXIO_SAFE_CALL( vxReleaseImage(&decodedImage) ); return FrameSource::OK; }
// // decode buffer // static IplImage *icvRetrieveFrame_GStreamer(CvCapture_GStreamer *cap) { if(!cap->buffer) return 0; // printf("getting buffercaps\n"); GstCaps* caps = gst_buffer_get_caps(cap->buffer); assert(gst_caps_get_size(caps) == 1); GstStructure* structure = gst_caps_get_structure(caps, 0); gint bpp, endianness, redmask, greenmask, bluemask; if(!gst_structure_get_int(structure, "bpp", &bpp) || !gst_structure_get_int(structure, "endianness", &endianness) || !gst_structure_get_int(structure, "red_mask", &redmask) || !gst_structure_get_int(structure, "green_mask", &greenmask) || !gst_structure_get_int(structure, "blue_mask", &bluemask)) { printf("missing essential information in buffer caps, %s\n", gst_caps_to_string(caps)); return 0; } printf("buffer has %d bpp, endianness %d, rgb %x %x %x, %s\n", bpp, endianness, redmask, greenmask, bluemask, gst_caps_to_string(caps)); if(!redmask || !greenmask || !bluemask) return 0; if(!cap->frame) { gint height, width; if(!gst_structure_get_int(structure, "width", &width) || !gst_structure_get_int(structure, "height", &height)) return 0; // printf("creating frame %dx%d\n", width, height); cap->frame = cvCreateImage(cvSize(width, height), IPL_DEPTH_8U, 3); } gst_caps_unref(caps); unsigned char *data = GST_BUFFER_DATA(cap->buffer); printf("generating shifts\n"); IplImage *frame = cap->frame; unsigned nbyte = bpp >> 3; unsigned redshift, blueshift, greenshift; unsigned mask = redmask; for(redshift = 0, mask = redmask; (mask & 1) == 0; mask >>= 1, redshift++) ; for(greenshift = 0, mask = greenmask; (mask & 1) == 0; mask >>= 1, greenshift++) ; for(blueshift = 0, mask = bluemask; (mask & 1) == 0; mask >>= 1, blueshift++) ; printf("shifts: %u %u %u\n", redshift, greenshift, blueshift); for(int r = 0; r < frame->height; r++) { for(int c = 0; c < frame->width; c++, data += nbyte) { int at = r * frame->widthStep + c * 3; frame->imageData[at] = ((*((gint *)data)) & redmask) >> redshift; frame->imageData[at+1] = ((*((gint *)data)) & greenmask) >> greenshift; frame->imageData[at+2] = ((*((gint *)data)) & bluemask) >> blueshift; } } // printf("converted buffer\n"); gst_buffer_unref(cap->buffer); cap->buffer = 0; return cap->frame; }
void GStreamerWrapper::newAudioSinkPrerollCallback( GstBuffer* audioSinkBuffer ) { std::lock_guard< std::mutex > lock( m_AudioMutex ); if ( m_cAudioBuffer == NULL ) { m_iAudioBufferSize = GST_BUFFER_SIZE( audioSinkBuffer ); m_cAudioBuffer = new unsigned char[m_iAudioBufferSize]; ////////////////////////////////////////////////////////////////////////// AUDIO DATA /* Note: For some reason, with this version of GStreamer the only way to retrieve the audio metadata is to read the caps from the audio appsink buffer and via a GstStructure we can retrieve the needed values from the caps. After lots of research I also found another possibility by using GstAudioInfo but this struct is not available in this version. If a later version of GStreamer is ever compiled in a valid way so it can be used with Visual Studio it would definitely be a good idea to retrieve the audio information somewhere else in the code. But this piece of code does it well for now. */ // Get Audio metadata // http://gstreamer.freedesktop.org/data/doc/gstreamer/head/pwg/html/section-types-definitions.html GstCaps* audioCaps = gst_buffer_get_caps( audioSinkBuffer ); GstStructure* gstStructure = gst_caps_get_structure( audioCaps, 0 ); // Is audio data signed or not? gboolean isAudioSigned; gst_structure_get_boolean( gstStructure, "signed", &isAudioSigned ); m_bIsAudioSigned = isAudioSigned; // Number of channels gst_structure_get_int( gstStructure, "channels", &m_iNumAudioChannels ); // Audio sample rate gst_structure_get_int( gstStructure, "rate", &m_iAudioSampleRate ); // Audio width gst_structure_get_int( gstStructure, "width", &m_iAudioWidth ); // Calculate the audio buffer size without the number of channels and audio width m_iAudioDecodeBufferSize = m_iAudioBufferSize / m_iNumAudioChannels / ( m_iAudioWidth / 8 ); // Audio endianness gint audioEndianness; gst_structure_get_int( gstStructure, "endianness", &audioEndianness ); m_AudioEndianness = (Endianness)audioEndianness; gst_caps_unref( audioCaps ); } else { // The Audio Buffer size may change during runtime so we keep track if the buffer changes // If so, delete the old buffer and re-allocate it with the respective new buffer size int bufferSize = GST_BUFFER_SIZE( audioSinkBuffer ); if ( m_iAudioBufferSize != bufferSize ) { // Allocate the audio data array according to the audio appsink buffer size m_iAudioBufferSize = bufferSize; delete [] m_cAudioBuffer; m_cAudioBuffer = NULL; m_cAudioBuffer = new unsigned char[m_iAudioBufferSize]; } } // Copy the audio appsink buffer data to our unsigned char array memcpy( (unsigned char *)m_cAudioBuffer, (unsigned char *)GST_BUFFER_DATA( audioSinkBuffer ), GST_BUFFER_SIZE( audioSinkBuffer ) ); }
/*! * \brief CvCapture_GStreamer::retrieveFrame * \return IplImage pointer. [Transfer Full] * Retreive the previously grabbed buffer, and wrap it in an IPLImage structure */ IplImage * CvCapture_GStreamer::retrieveFrame(int) { if(!buffer) return 0; //construct a frame header if we did not have any yet if(!frame) { gint height, width; //reuse the caps ptr if (buffer_caps) gst_caps_unref(buffer_caps); #if GST_VERSION_MAJOR == 0 buffer_caps = gst_buffer_get_caps(buffer); #else buffer_caps = gst_sample_get_caps(sample); #endif // bail out in no caps assert(gst_caps_get_size(buffer_caps) == 1); GstStructure* structure = gst_caps_get_structure(buffer_caps, 0); // bail out if width or height are 0 if(!gst_structure_get_int(structure, "width", &width) || !gst_structure_get_int(structure, "height", &height)) { return 0; } int depth = 3; #if GST_VERSION_MAJOR > 0 depth = 0; const gchar* name = gst_structure_get_name(structure); const gchar* format = gst_structure_get_string(structure, "format"); if (!name || !format) return 0; // we support 3 types of data: // video/x-raw, format=BGR -> 8bit, 3 channels // video/x-raw, format=GRAY8 -> 8bit, 1 channel // video/x-bayer -> 8bit, 1 channel // bayer data is never decoded, the user is responsible for that // everything is 8 bit, so we just test the caps for bit depth if (strcasecmp(name, "video/x-raw") == 0) { if (strcasecmp(format, "BGR") == 0) { depth = 3; } else if(strcasecmp(format, "GRAY8") == 0){ depth = 1; } } else if (strcasecmp(name, "video/x-bayer") == 0) { depth = 1; } #endif if (depth > 0) { frame = cvCreateImageHeader(cvSize(width, height), IPL_DEPTH_8U, depth); }else{ return 0; } } // gstreamer expects us to handle the memory at this point // so we can just wrap the raw buffer and be done with it #if GST_VERSION_MAJOR == 0 frame->imageData = (char *)GST_BUFFER_DATA(buffer); #else // the data ptr in GstMapInfo is only valid throughout the mapifo objects life. // TODO: check if reusing the mapinfo object is ok. gboolean success = gst_buffer_map(buffer,info, (GstMapFlags)GST_MAP_READ); if (!success){ //something weird went wrong here. abort. abort. //fprintf(stderr,"GStreamer: unable to map buffer"); return 0; } frame->imageData = (char*)info->data; gst_buffer_unmap(buffer,info); #endif return frame; }