VideoFrame VideoDecoderVDA::frame() { DPTR_D(VideoDecoderVDA); CVPixelBufferRef cv_buffer = (CVPixelBufferRef)d.frame->data[3]; if (!cv_buffer) { qDebug("Frame buffer is empty."); return VideoFrame(); } if (CVPixelBufferGetDataSize(cv_buffer) <= 0) { qDebug("Empty frame buffer"); return VideoFrame(); } VideoFormat::PixelFormat pixfmt = format_from_cv(CVPixelBufferGetPixelFormatType(cv_buffer)); if (pixfmt == VideoFormat::Format_Invalid) { qWarning("unsupported vda pixel format: %#x", CVPixelBufferGetPixelFormatType(cv_buffer)); return VideoFrame(); } // we can map the cv buffer addresses to video frame in SurfaceInteropCVBuffer. (may need VideoSurfaceInterop::mapToTexture() class SurfaceInteropCVBuffer Q_DECL_FINAL: public VideoSurfaceInterop { bool glinterop; CVPixelBufferRef cvbuf; // keep ref until video frame is destroyed public: SurfaceInteropCVBuffer(CVPixelBufferRef cv, bool gl) : glinterop(gl), cvbuf(cv) { //CVPixelBufferRetain(cvbuf); } ~SurfaceInteropCVBuffer() { CVPixelBufferRelease(cvbuf); } void* mapToHost(const VideoFormat &format, void *handle, int plane) { Q_UNUSED(plane); CVPixelBufferLockBaseAddress(cvbuf, 0); const VideoFormat fmt(format_from_cv(CVPixelBufferGetPixelFormatType(cvbuf))); if (!fmt.isValid()) { CVPixelBufferUnlockBaseAddress(cvbuf, 0); return NULL; } const int w = CVPixelBufferGetWidth(cvbuf); const int h = CVPixelBufferGetHeight(cvbuf); uint8_t *src[3]; int pitch[3]; for (int i = 0; i <fmt.planeCount(); ++i) { // get address results in internal copy src[i] = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cvbuf, i); pitch[i] = CVPixelBufferGetBytesPerRowOfPlane(cvbuf, i); } CVPixelBufferUnlockBaseAddress(cvbuf, 0); //CVPixelBufferRelease(cv_buffer); // release when video frame is destroyed VideoFrame frame(VideoFrame::fromGPU(fmt, w, h, h, src, pitch)); if (fmt != format) frame = frame.to(format); VideoFrame *f = reinterpret_cast<VideoFrame*>(handle); frame.setTimestamp(f->timestamp()); frame.setDisplayAspectRatio(f->displayAspectRatio()); *f = frame; return f; }
bool getImage(const VideoFrame &videoFrame, void *dest, ImgScaler *nv12ToRGB32) override { { QMutexLocker locker(&m_buffersMutex); if (m_buffers.indexOf(videoFrame.surfaceId) < 0) return false; } CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)videoFrame.surfaceId; if (CVPixelBufferGetPixelFormatType(pixelBuffer) == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); const quint8 *srcData[2] = { (const quint8 *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0), (const quint8 *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1) }; const qint32 srcLinesize[2] = { (qint32)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0), (qint32)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1) }; nv12ToRGB32->scale((const void **)srcData, srcLinesize, dest); CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); return true; } return false; }
void dump_cvpixel_buffer (CVPixelBufferRef pixbuf) { gsize left, right, top, bottom; GST_LOG ("buffer %p", pixbuf); if (CVPixelBufferLockBaseAddress (pixbuf, 0)) { GST_WARNING ("Couldn't lock base adress on pixel buffer !"); return; } GST_LOG ("Width:%" G_GSIZE_FORMAT " , Height:%" G_GSIZE_FORMAT, CVPixelBufferGetWidth (pixbuf), CVPixelBufferGetHeight (pixbuf)); GST_LOG ("Format:%" GST_FOURCC_FORMAT, GST_FOURCC_ARGS (CVPixelBufferGetPixelFormatType (pixbuf))); GST_LOG ("base address:%p", CVPixelBufferGetBaseAddress (pixbuf)); GST_LOG ("Bytes per row:%" G_GSIZE_FORMAT, CVPixelBufferGetBytesPerRow (pixbuf)); GST_LOG ("Data Size:%" G_GSIZE_FORMAT, CVPixelBufferGetDataSize (pixbuf)); GST_LOG ("Plane count:%" G_GSIZE_FORMAT, CVPixelBufferGetPlaneCount (pixbuf)); CVPixelBufferGetExtendedPixels (pixbuf, &left, &right, &top, &bottom); GST_LOG ("Extended pixels. left/right/top/bottom : %" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT, left, right, top, bottom); CVPixelBufferUnlockBaseAddress (pixbuf, 0); }
void VdaMixer::adjust(VideoFormatData *data, const mp_image *mpi) { Q_ASSERT(data->imgfmt == IMGFMT_VDA); auto buffer = (CVPixelBufferRef)mpi->planes[3]; switch (CVPixelBufferGetPixelFormatType(buffer)) { case kCVPixelFormatType_422YpCbCr8: data->type = IMGFMT_UYVY; break; case kCVPixelFormatType_422YpCbCr8_yuvs: data->type = IMGFMT_YUYV; break; case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: data->type = IMGFMT_NV12; break; case kCVPixelFormatType_420YpCbCr8Planar: data->type = IMGFMT_420P; break; default: _Error("Not supported format."); data->type = IMGFMT_NONE; } auto desc = mp_imgfmt_get_desc(data->type); if (CVPixelBufferIsPlanar(buffer)) { data->planes = CVPixelBufferGetPlaneCount(buffer); Q_ASSERT(data->planes == desc.num_planes); for (int i=0; i<data->planes; ++i) { data->alignedByteSize[i].rwidth() = CVPixelBufferGetBytesPerRowOfPlane(buffer, i); data->alignedByteSize[i].rheight() = CVPixelBufferGetHeightOfPlane(buffer, i); data->bpp += desc.bpp[i] >> (desc.xs[i] + desc.ys[i]); } } else {
static int vda_retrieve_data(AVCodecContext *s, AVFrame *frame) { InputStream *ist = s->opaque; VDAContext *vda = ist->hwaccel_ctx; CVPixelBufferRef pixbuf = (CVPixelBufferRef)frame->data[3]; OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf); CVReturn err; uint8_t *data[4] = { 0 }; int linesize[4] = { 0 }; int planes, ret, i; av_frame_unref(vda->tmp_frame); switch (pixel_format) { case kCVPixelFormatType_420YpCbCr8Planar: vda->tmp_frame->format = AV_PIX_FMT_YUV420P; break; case kCVPixelFormatType_422YpCbCr8: vda->tmp_frame->format = AV_PIX_FMT_UYVY422; break; default: av_log(NULL, AV_LOG_ERROR, "Unsupported pixel format: %u\n", pixel_format); return AVERROR(ENOSYS); } vda->tmp_frame->width = frame->width; vda->tmp_frame->height = frame->height; ret = av_frame_get_buffer(vda->tmp_frame, 32); if (ret < 0) return ret; err = CVPixelBufferLockBaseAddress(pixbuf, kCVPixelBufferLock_ReadOnly); if (err != kCVReturnSuccess) { av_log(NULL, AV_LOG_ERROR, "Error locking the pixel buffer.\n"); return AVERROR_UNKNOWN; } if (CVPixelBufferIsPlanar(pixbuf)) { planes = CVPixelBufferGetPlaneCount(pixbuf); for (i = 0; i < planes; i++) { data[i] = CVPixelBufferGetBaseAddressOfPlane(pixbuf, i); linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixbuf, i); } } else { data[0] = CVPixelBufferGetBaseAddress(pixbuf); linesize[0] = CVPixelBufferGetBytesPerRow(pixbuf); } av_image_copy(vda->tmp_frame->data, vda->tmp_frame->linesize, data, linesize, vda->tmp_frame->format, frame->width, frame->height); ret = av_frame_copy_props(vda->tmp_frame, frame); if (ret < 0) return ret; av_frame_unref(frame); av_frame_move_ref(frame, vda->tmp_frame); return 0; }
/* Decoder callback that adds the vda frame to the queue in display order. */ static void vda_decoder_callback (void *vda_hw_ctx, CFDictionaryRef user_info, OSStatus status, uint32_t infoFlags, CVImageBufferRef image_buffer) { struct vda_context *vda_ctx = vda_hw_ctx; if (!image_buffer) return; if (vda_ctx->cv_pix_fmt_type != CVPixelBufferGetPixelFormatType(image_buffer)) return; if (vda_ctx->use_sync_decoding) { vda_ctx->cv_buffer = CVPixelBufferRetain(image_buffer); } else { vda_frame *new_frame; vda_frame *queue_walker; if (!(new_frame = av_mallocz(sizeof(*new_frame)))) return; new_frame->next_frame = NULL; new_frame->cv_buffer = CVPixelBufferRetain(image_buffer); new_frame->pts = vda_pts_from_dictionary(user_info); pthread_mutex_lock(&vda_ctx->queue_mutex); queue_walker = vda_ctx->queue; if (!queue_walker || (new_frame->pts < queue_walker->pts)) { /* we have an empty queue, or this frame earlier than the current queue head */ new_frame->next_frame = queue_walker; vda_ctx->queue = new_frame; } else { /* walk the queue and insert this frame where it belongs in display order */ vda_frame *next_frame; while (1) { next_frame = queue_walker->next_frame; if (!next_frame || (new_frame->pts < next_frame->pts)) { new_frame->next_frame = next_frame; queue_walker->next_frame = new_frame; break; } queue_walker = next_frame; } } pthread_mutex_unlock(&vda_ctx->queue_mutex); } }
/* Decoder callback that adds the vda frame to the queue in display order. */ static void vda_decoder_callback (void *vda_hw_ctx, CFDictionaryRef /*user_info*/, OSStatus /*status*/, uint32_t /*infoFlags*/, CVImageBufferRef image_buffer) { vda_context *vda_ctx = (vda_context*)vda_hw_ctx; if (!image_buffer) return; const auto fmt = CVPixelBufferGetPixelFormatType(image_buffer); if (!_Contains(cvpixfmts, fmt)) { qDebug() << "vda: not supported format!!"; return; } vda_ctx->cv_pix_fmt_type = fmt; vda_ctx->cv_buffer = CVPixelBufferRetain(image_buffer); }
/* Decoder callback that adds the VDA frame to the queue in display order. */ static void vda_decoder_callback(void *vda_hw_ctx, CFDictionaryRef user_info, OSStatus status, uint32_t infoFlags, CVImageBufferRef image_buffer) { struct vda_context *vda_ctx = vda_hw_ctx; if (!image_buffer) return; if (vda_ctx->cv_pix_fmt_type != CVPixelBufferGetPixelFormatType(image_buffer)) return; vda_ctx->cv_buffer = CVPixelBufferRetain(image_buffer); }
static void VDADecoderCallback (void *decompressionOutputRefCon, CFDictionaryRef frameInfo, OSStatus status, uint32_t infoFlags, CVImageBufferRef imageBuffer) { MoonVDADecoder *decoder = (MoonVDADecoder *) decompressionOutputRefCon; VideoStream *vs = (VideoStream *) decoder->GetStream (); // FIXME: Is this always 1 thread? Can we optimize this decoder->GetDeployment ()->RegisterThread (); Deployment::SetCurrent (decoder->GetDeployment ()); if (imageBuffer == NULL) { return; } OSType format_type = CVPixelBufferGetPixelFormatType (imageBuffer); if (format_type != kCVPixelFormatType_422YpCbCr8) { g_warning ("Mismatched format in VDA"); return; } MediaFrame *mf = (MediaFrame *) CFDictionaryGetValue (frameInfo, CFSTR ("MoonMediaFrame")); mf->AddState (MediaFrameVUY2); mf->FreeBuffer (); mf->SetBufLen (0); mf->srcSlideY = 0; mf->srcSlideH = vs->GetHeight (); mf->width = vs->GetWidth (); mf->height = vs->GetHeight (); CVPixelBufferLockBaseAddress (imageBuffer, 0); mf->data_stride [0] = (uint8_t *) CVPixelBufferGetBaseAddress (imageBuffer); mf->srcStride [0] = CVPixelBufferGetBytesPerRow (imageBuffer); mf->AddState (MediaFrameDecoded); mf->decoder_specific_data = imageBuffer; CVPixelBufferRetain (imageBuffer); decoder->ReportDecodeFrameCompleted (mf); mf->unref (); }
static void _frame_decompressed(void *decompressionTrackingRefCon, OSStatus err, ICMDecompressionTrackingFlags dtf, CVPixelBufferRef pixelBuffer, TimeValue64 displayTime, TimeValue64 displayDuration, ICMValidTimeFlags validTimeFlags, void *reserved, void *sourceFrameRefCon) { dbg_printf("[ vOE] >> [%08lx] :: _frame_decompressed()\n", (UInt32) -1); if (!err) { StreamInfoPtr si = (StreamInfoPtr) decompressionTrackingRefCon; if (dtf & kICMDecompressionTracking_ReleaseSourceData) { // if we were responsible for managing source data buffers, // we should release the source buffer here, // using sourceFrameRefCon to identify it. } if ((dtf & kICMDecompressionTracking_EmittingFrame) && pixelBuffer) { ICMCompressionFrameOptionsRef frameOptions = NULL; OSType pf = CVPixelBufferGetPixelFormatType(pixelBuffer); dbg_printf("[ vOE] > [%08lx] :: _frame_decompressed() = %ld; %ld," " %lld, %lld, %ld [%ld '%4.4s' (%ld x %ld)]\n", (UInt32) -1, err, dtf, displayTime, displayDuration, validTimeFlags, CVPixelBufferGetDataSize(pixelBuffer), (char *) &pf, CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); displayDuration = 25; //? // Feed the frame to the compression session. err = ICMCompressionSessionEncodeFrame(si->si_v.cs, pixelBuffer, displayTime, displayDuration, validTimeFlags, frameOptions, NULL, NULL ); } } dbg_printf("[ vOE] < [%08lx] :: _frame_decompressed() = %ld\n", (UInt32) -1, err); }
Surface8uRef convertCVPixelBufferToSurface( CVPixelBufferRef pixelBufferRef ) { CVPixelBufferLockBaseAddress( pixelBufferRef, 0 ); uint8_t *ptr = reinterpret_cast<uint8_t*>( CVPixelBufferGetBaseAddress( pixelBufferRef ) ); int32_t rowBytes = CVPixelBufferGetBytesPerRow( pixelBufferRef ); OSType type = CVPixelBufferGetPixelFormatType( pixelBufferRef ); size_t width = CVPixelBufferGetWidth( pixelBufferRef ); size_t height = CVPixelBufferGetHeight( pixelBufferRef ); SurfaceChannelOrder sco; if( type == k24RGBPixelFormat ) sco = SurfaceChannelOrder::RGB; else if( type == k32ARGBPixelFormat ) sco = SurfaceChannelOrder::ARGB; else if( type == k24BGRPixelFormat ) sco = SurfaceChannelOrder::BGR; else if( type == k32BGRAPixelFormat ) sco = SurfaceChannelOrder::BGRA; Surface8u *newSurface = new Surface8u( ptr, width, height, rowBytes, sco ); return Surface8uRef( newSurface, [=] ( Surface8u *s ) { ::CVBufferRelease( pixelBufferRef ); delete s; } ); }
Surface8u convertCVPixelBufferToSurface( CVPixelBufferRef pixelBufferRef ) { CVPixelBufferLockBaseAddress( pixelBufferRef, 0 ); uint8_t *ptr = reinterpret_cast<uint8_t*>( CVPixelBufferGetBaseAddress( pixelBufferRef ) ); int32_t rowBytes = CVPixelBufferGetBytesPerRow( pixelBufferRef ); OSType type = CVPixelBufferGetPixelFormatType( pixelBufferRef ); size_t width = CVPixelBufferGetWidth( pixelBufferRef ); size_t height = CVPixelBufferGetHeight( pixelBufferRef ); SurfaceChannelOrder sco; if( type == k24RGBPixelFormat ) sco = SurfaceChannelOrder::RGB; else if( type == k32ARGBPixelFormat ) sco = SurfaceChannelOrder::ARGB; else if( type == k24BGRPixelFormat ) sco = SurfaceChannelOrder::BGR; else if( type == k32BGRAPixelFormat ) sco = SurfaceChannelOrder::BGRA; Surface result( ptr, width, height, rowBytes, sco ); result.setDeallocator( CVPixelBufferDealloc, pixelBufferRef ); return result; }
static gboolean gst_video_info_init_from_pixel_buffer (GstVideoInfo * info, CVPixelBufferRef pixel_buf) { size_t width, height; OSType format_type; GstVideoFormat video_format; width = CVPixelBufferGetWidth (pixel_buf); height = CVPixelBufferGetHeight (pixel_buf); format_type = CVPixelBufferGetPixelFormatType (pixel_buf); video_format = gst_core_media_buffer_get_video_format (format_type); if (video_format == GST_VIDEO_FORMAT_UNKNOWN) { return FALSE; } gst_video_info_init (info); gst_video_info_set_format (info, video_format, width, height); return TRUE; }
bool QTPixelBuffer::pixelFormatIs32BGRA() const { return CVPixelBufferGetPixelFormatType(m_pixelBuffer) == k32BGRAPixelFormat; }
void jit_gl_hap_draw_frame(void *jitob, CVImageBufferRef frame) { t_jit_gl_hap * x = (t_jit_gl_hap*)jitob; CFTypeID imageType = CFGetTypeID(frame); OSType newPixelFormat; if(x->validframe) return; if (imageType == CVPixelBufferGetTypeID()) { // Update the texture CVBufferRetain(frame); if(x->buffer) { CVPixelBufferUnlockBaseAddress(x->buffer, kCVPixelBufferLock_ReadOnly); CVBufferRelease(x->buffer); } x->buffer = frame; CVPixelBufferLockBaseAddress(x->buffer, kCVPixelBufferLock_ReadOnly); x->dim[0] = CVPixelBufferGetWidth(x->buffer); x->dim[1] = CVPixelBufferGetHeight(x->buffer); newPixelFormat = CVPixelBufferGetPixelFormatType(x->buffer); if(x->buffer && x->hap_format==JIT_GL_HAP_PF_HAP) { size_t extraRight, extraBottom; unsigned int bitsPerPixel; size_t bytesPerRow; size_t actualBufferSize; CVPixelBufferGetExtendedPixels(x->buffer, NULL, &extraRight, NULL, &extraBottom); x->roundedWidth = x->dim[0] + extraRight; x->roundedHeight = x->dim[1] + extraBottom; if (x->roundedWidth % 4 != 0 || x->roundedHeight % 4 != 0) { x->validframe = 0; return; } switch (newPixelFormat) { case kHapPixelFormatTypeRGB_DXT1: x->newInternalFormat = GL_COMPRESSED_RGB_S3TC_DXT1_EXT; bitsPerPixel = 4; break; case kHapPixelFormatTypeRGBA_DXT5: case kHapPixelFormatTypeYCoCg_DXT5: x->newInternalFormat = GL_COMPRESSED_RGBA_S3TC_DXT5_EXT; bitsPerPixel = 8; break; default: // we don't support non-DXT pixel buffers x->validframe = 0; return; break; } x->useshader = (newPixelFormat == kHapPixelFormatTypeYCoCg_DXT5); bytesPerRow = (x->roundedWidth * bitsPerPixel) / 8; x->newDataLength = bytesPerRow * x->roundedHeight; // usually not the full length of the buffer actualBufferSize = CVPixelBufferGetDataSize(x->buffer); // Check the buffer is as large as we expect it to be if (x->newDataLength > actualBufferSize) { x->validframe = 0; return; } // If we got this far we're good to go x->validframe = 1; x->target = GL_TEXTURE_2D; if(!x->flipped) { jit_attr_setlong(x->texoutput, gensym("flip"), 1); x->flipped = 1; } //x->drawhap = 1; } else if(x->buffer) {// && x->hap_format==JIT_GL_HAP_PF_HAP) { if( newPixelFormat == k24RGBPixelFormat ) x->newInternalFormat = GL_RGB8; else if( newPixelFormat == k32BGRAPixelFormat ) x->newInternalFormat = GL_RGBA8; else { x->validframe = 0; return; } x->roundedWidth = x->dim[0]; x->roundedHeight = x->dim[1]; x->newDataLength = CVPixelBufferGetDataSize(x->buffer); x->rowLength = CVPixelBufferGetBytesPerRow( x->buffer ) / (x->hap_format==JIT_GL_HAP_PF_RGB ? 3 : 4); x->target = GL_TEXTURE_RECTANGLE_EXT; if(!x->flipped) { jit_attr_setlong(x->texoutput, gensym("flip"), 1); x->flipped = 1; } x->validframe = 1; } } else { #ifdef MAC_VERSION CGSize imageSize = CVImageBufferGetEncodedSize(frame); bool flipped = CVOpenGLTextureIsFlipped(frame); x->texture = CVOpenGLTextureGetName(frame); x->useshader = 0; x->dim[0] = (t_atom_long)imageSize.width; x->dim[1] = (t_atom_long)imageSize.height; x->validframe = 1; x->target = GL_TEXTURE_RECTANGLE_ARB; if(x->flipped!=flipped) { jit_attr_setlong(x->texoutput, gensym("flip"), flipped); x->flipped = flipped; } #endif } }
static GF_Err VTBDec_ProcessData(GF_MediaDecoder *ifcg, char *inBuffer, u32 inBufferLength, u16 ES_ID, u32 *CTS, char *outBuffer, u32 *outBufferLength, u8 PaddingBits, u32 mmlevel) { OSStatus status; CMSampleBufferRef sample = NULL; CMBlockBufferRef block_buffer = NULL; OSType type; char *in_data; u32 in_data_size; GF_Err e; VTBDec *ctx = (VTBDec *)ifcg->privateStack; if (ctx->skip_mpeg4_vosh) { GF_M4VDecSpecInfo dsi; dsi.width = dsi.height = 0; e = gf_m4v_get_config(inBuffer, inBufferLength, &dsi); //found a vosh - remove it from payload, init decoder if needed if ((e==GF_OK) && dsi.width && dsi.height) { if (!ctx->vtb_session) { ctx->vosh = inBuffer; ctx->vosh_size = dsi.next_object_start; e = VTBDec_InitDecoder(ctx, GF_FALSE); if (e) return e; //enfoce removal for all frames ctx->skip_mpeg4_vosh = GF_TRUE; if (ctx->out_size != *outBufferLength) { *outBufferLength = ctx->out_size; return GF_BUFFER_TOO_SMALL; } } ctx->vosh_size = dsi.next_object_start; } else if (!ctx->vtb_session) { *outBufferLength=0; return GF_OK; } } if (ctx->init_mpeg12) { GF_M4VDecSpecInfo dsi; dsi.width = dsi.height = 0; e = gf_mpegv12_get_config(inBuffer, inBufferLength, &dsi); if ((e==GF_OK) && dsi.width && dsi.height) { ctx->width = dsi.width; ctx->height = dsi.height; ctx->pixel_ar = dsi.par_num; ctx->pixel_ar <<= 16; ctx->pixel_ar |= dsi.par_den; e = VTBDec_InitDecoder(ctx, GF_FALSE); if (e) return e; if (ctx->out_size != *outBufferLength) { *outBufferLength = ctx->out_size; return GF_BUFFER_TOO_SMALL; } } if (!ctx->vtb_session) { *outBufferLength=0; return GF_OK; } } if (ctx->is_annex_b || (!ctx->vtb_session && ctx->nalu_size_length) ) { if (ctx->cached_annex_b) { in_data = ctx->cached_annex_b; in_data_size = ctx->cached_annex_b_size; ctx->cached_annex_b = NULL; } else { e = VTB_RewriteNALs(ctx, inBuffer, inBufferLength, &in_data, &in_data_size); if (e) return e; } if (ctx->out_size != *outBufferLength) { *outBufferLength = ctx->out_size; ctx->cached_annex_b = in_data; ctx->cached_annex_b_size = in_data_size; return GF_BUFFER_TOO_SMALL; } } else if (ctx->vosh_size) { in_data = inBuffer + ctx->vosh_size; in_data_size = inBufferLength - ctx->vosh_size; ctx->vosh_size = 0; } else { in_data = inBuffer; in_data_size = inBufferLength; } if (!ctx->vtb_session) { *outBufferLength=0; return GF_OK; } status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, in_data, in_data_size, kCFAllocatorNull, NULL, 0, in_data_size, 0, &block_buffer); if (status) { return GF_IO_ERR; } *outBufferLength=0; if (block_buffer == NULL) return GF_OK; status = CMSampleBufferCreate(kCFAllocatorDefault, block_buffer, TRUE, NULL, NULL, ctx->fmt_desc, 1, 0, NULL, 0, NULL, &sample); if (status || (sample==NULL)) { if (block_buffer) CFRelease(block_buffer); return GF_IO_ERR; } ctx->last_error = GF_OK; status = VTDecompressionSessionDecodeFrame(ctx->vtb_session, sample, 0, NULL, 0); if (!status) status = VTDecompressionSessionWaitForAsynchronousFrames(ctx->vtb_session); CFRelease(block_buffer); CFRelease(sample); if (ctx->cached_annex_b) gf_free(in_data); if (ctx->last_error) return ctx->last_error; if (status) return GF_NON_COMPLIANT_BITSTREAM; if (!ctx->frame) { *outBufferLength=0; return ctx->last_error; } *outBufferLength = ctx->out_size; status = CVPixelBufferLockBaseAddress(ctx->frame, kCVPixelBufferLock_ReadOnly); if (status != kCVReturnSuccess) { GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error locking frame data\n")); return GF_IO_ERR; } type = CVPixelBufferGetPixelFormatType(ctx->frame); if (CVPixelBufferIsPlanar(ctx->frame)) { u32 i, j, nb_planes = (u32) CVPixelBufferGetPlaneCount(ctx->frame); char *dst = outBuffer; Bool needs_stride=GF_FALSE; if ((type==kCVPixelFormatType_420YpCbCr8Planar) || (type==kCVPixelFormatType_420YpCbCr8PlanarFullRange) || (type==kCVPixelFormatType_422YpCbCr8_yuvs) || (type==kCVPixelFormatType_444YpCbCr8) || (type=='444v') ) { u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 0); //TOCHECK - for now the 3 planes are consecutive in VideoToolbox if (stride==ctx->width) { char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 0); memcpy(dst, data, sizeof(char)*ctx->out_size); } else { for (i=0; i<nb_planes; i++) { char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, i); u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, i); u32 w, h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, i); w = ctx->width; if (i) { switch (ctx->pix_fmt) { case GF_PIXEL_YUV444: break; case GF_PIXEL_YUV422: case GF_PIXEL_YV12: w /= 2; break; } } if (stride != w) { needs_stride=GF_TRUE; for (j=0; j<h; j++) { memcpy(dst, data, sizeof(char)*w); dst += w; data += stride; } } else { memcpy(dst, data, sizeof(char)*h*stride); dst += sizeof(char)*h*stride; } } } } else if ((type==kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) || (type==kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) { char *dst_v; char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 0); u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 0); u32 i, h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, 0); if (stride==ctx->width) { memcpy(dst, data, sizeof(char)*h*stride); dst += sizeof(char)*h*stride; } else { for (i=0; i<h; i++) { memcpy(dst, data, sizeof(char)*ctx->width); dst += ctx->width; data += stride; } needs_stride=GF_TRUE; } data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 1); stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 1); h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, 1); dst_v = dst+sizeof(char) * h*stride/2; for (i=0; i<ctx->width * h / 2; i++) { *dst = data[0]; *dst_v = data[1]; data += 2; dst_v++; dst++; if (!(i%ctx->width)) data += (stride - ctx->width); } } } CVPixelBufferUnlockBaseAddress(ctx->frame, kCVPixelBufferLock_ReadOnly); return GF_OK; }
unsigned long QTPixelBuffer::pixelFormatType() const { return CVPixelBufferGetPixelFormatType(m_pixelBuffer); }
void CDVDVideoCodecVDA::VDADecoderCallback( void *decompressionOutputRefCon, CFDictionaryRef frameInfo, OSStatus status, uint32_t infoFlags, CVImageBufferRef imageBuffer) { CCocoaAutoPool pool; // Warning, this is an async callback. There can be multiple frames in flight. CDVDVideoCodecVDA *ctx = (CDVDVideoCodecVDA*)decompressionOutputRefCon; if (imageBuffer == NULL) { //CLog::Log(LOGDEBUG, "%s - imageBuffer is NULL", __FUNCTION__); return; } OSType format_type = CVPixelBufferGetPixelFormatType(imageBuffer); if ((format_type != kCVPixelFormatType_422YpCbCr8) && (format_type != kCVPixelFormatType_32BGRA) ) { CLog::Log(LOGERROR, "%s - imageBuffer format is not '2vuy' or 'BGRA',is reporting 0x%x", __FUNCTION__, format_type); return; } if (kVDADecodeInfo_FrameDropped & infoFlags) { CLog::Log(LOGDEBUG, "%s - frame dropped", __FUNCTION__); return; } // allocate a new frame and populate it with some information. // this pointer to a frame_queue type keeps track of the newest decompressed frame // and is then inserted into a linked list of frame pointers depending on the display time // parsed out of the bitstream and stored in the frameInfo dictionary by the client frame_queue *newFrame = (frame_queue*)calloc(sizeof(frame_queue), 1); newFrame->nextframe = NULL; newFrame->pixel_buffer_format = format_type; newFrame->pixel_buffer_ref = CVPixelBufferRetain(imageBuffer); GetFrameDisplayTimeFromDictionary(frameInfo, newFrame); // if both dts or pts are good we use those, else use decoder insert time for frame sort if ((newFrame->pts != DVD_NOPTS_VALUE) || (newFrame->dts != DVD_NOPTS_VALUE)) { // if pts is borked (stupid avi's), use dts for frame sort if (newFrame->pts == DVD_NOPTS_VALUE) newFrame->sort_time = newFrame->dts; else newFrame->sort_time = newFrame->pts; } // since the frames we get may be in decode order rather than presentation order // our hypothetical callback places them in a queue of frames which will // hold them in display order for display on another thread pthread_mutex_lock(&ctx->m_queue_mutex); // frame_queue *queueWalker = ctx->m_display_queue; if (!queueWalker || (newFrame->sort_time < queueWalker->sort_time)) { // we have an empty queue, or this frame earlier than the current queue head. newFrame->nextframe = queueWalker; ctx->m_display_queue = newFrame; } else { // walk the queue and insert this frame where it belongs in display order. bool frameInserted = false; frame_queue *nextFrame = NULL; // while (!frameInserted) { nextFrame = queueWalker->nextframe; if (!nextFrame || (newFrame->sort_time < nextFrame->sort_time)) { // if the next frame is the tail of the queue, or our new frame is earlier. newFrame->nextframe = nextFrame; queueWalker->nextframe = newFrame; frameInserted = true; } queueWalker = nextFrame; } } ctx->m_queue_depth++; // pthread_mutex_unlock(&ctx->m_queue_mutex); }
static int videotoolbox_retrieve_data(AVCodecContext *s, AVFrame *frame) { InputStream *ist = s->opaque; VTContext *vt = ist->hwaccel_ctx; CVPixelBufferRef pixbuf = (CVPixelBufferRef)frame->data[3]; OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf); CVReturn err; uint8_t *data[4] = { 0 }; int linesize[4] = { 0 }; int planes, ret, i; char codec_str[32]; av_frame_unref(vt->tmp_frame); switch (pixel_format) { case kCVPixelFormatType_420YpCbCr8Planar: vt->tmp_frame->format = AV_PIX_FMT_YUV420P; break; case kCVPixelFormatType_422YpCbCr8: vt->tmp_frame->format = AV_PIX_FMT_UYVY422; break; case kCVPixelFormatType_32BGRA: vt->tmp_frame->format = AV_PIX_FMT_BGRA; break; #ifdef kCFCoreFoundationVersionNumber10_7 case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: vt->tmp_frame->format = AV_PIX_FMT_NV12; break; #endif default: av_get_codec_tag_string(codec_str, sizeof(codec_str), s->codec_tag); av_log(NULL, AV_LOG_ERROR, "%s: Unsupported pixel format: %s\n", codec_str, videotoolbox_pixfmt); return AVERROR(ENOSYS); } vt->tmp_frame->width = frame->width; vt->tmp_frame->height = frame->height; ret = av_frame_get_buffer(vt->tmp_frame, 32); if (ret < 0) return ret; err = CVPixelBufferLockBaseAddress(pixbuf, kCVPixelBufferLock_ReadOnly); if (err != kCVReturnSuccess) { av_log(NULL, AV_LOG_ERROR, "Error locking the pixel buffer.\n"); return AVERROR_UNKNOWN; } if (CVPixelBufferIsPlanar(pixbuf)) { planes = CVPixelBufferGetPlaneCount(pixbuf); for (i = 0; i < planes; i++) { data[i] = CVPixelBufferGetBaseAddressOfPlane(pixbuf, i); linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixbuf, i); } } else { data[0] = CVPixelBufferGetBaseAddress(pixbuf); linesize[0] = CVPixelBufferGetBytesPerRow(pixbuf); } av_image_copy(vt->tmp_frame->data, vt->tmp_frame->linesize, (const uint8_t **)data, linesize, vt->tmp_frame->format, frame->width, frame->height); ret = av_frame_copy_props(vt->tmp_frame, frame); CVPixelBufferUnlockBaseAddress(pixbuf, kCVPixelBufferLock_ReadOnly); if (ret < 0) return ret; av_frame_unref(frame); av_frame_move_ref(frame, vt->tmp_frame); return 0; }
void CDVDVideoCodecVDA::VDADecoderCallback( void *decompressionOutputRefCon, CFDictionaryRef frameInfo, OSStatus status, uint32_t infoFlags, CVImageBufferRef imageBuffer) { CCocoaAutoPool pool; // Warning, this is an async callback. There can be multiple frames in flight. CDVDVideoCodecVDA *ctx = (CDVDVideoCodecVDA*)decompressionOutputRefCon; if (imageBuffer == NULL) { //CLog::Log(LOGDEBUG, "%s - imageBuffer is NULL", __FUNCTION__); return; } OSType format_type = CVPixelBufferGetPixelFormatType(imageBuffer); if ((format_type != kCVPixelFormatType_422YpCbCr8) && (format_type != kCVPixelFormatType_32BGRA) ) { CLog::Log(LOGERROR, "%s - imageBuffer format is not '2vuy' or 'BGRA',is reporting 0x%x", __FUNCTION__, (unsigned int)format_type); return; } if (kVDADecodeInfo_FrameDropped & infoFlags) { if (g_advancedSettings.CanLogComponent(LOGVIDEO)) CLog::Log(LOGDEBUG, "%s - frame dropped", __FUNCTION__); return; } // allocate a new frame and populate it with some information. // this pointer to a frame_queue type keeps track of the newest decompressed frame // and is then inserted into a linked list of frame pointers depending on the display time // parsed out of the bitstream and stored in the frameInfo dictionary by the client frame_queue *newFrame = (frame_queue*)calloc(sizeof(frame_queue), 1); newFrame->nextframe = NULL; newFrame->pixel_buffer_format = format_type; newFrame->pixel_buffer_ref = CVBufferRetain(imageBuffer); GetFrameDisplayTimeFromDictionary(frameInfo, newFrame); // since the frames we get may be in decode order rather than presentation order // our hypothetical callback places them in a queue of frames which will // hold them in display order for display on another thread pthread_mutex_lock(&ctx->m_queue_mutex); frame_queue base; base.nextframe = ctx->m_display_queue; frame_queue *ptr = &base; for(; ptr->nextframe; ptr = ptr->nextframe) { if(ptr->nextframe->pts == DVD_NOPTS_VALUE || newFrame->pts == DVD_NOPTS_VALUE) continue; if(ptr->nextframe->pts > newFrame->pts) break; } /* insert after ptr */ newFrame->nextframe = ptr->nextframe; ptr->nextframe = newFrame; /* update anchor if needed */ if(newFrame->nextframe == ctx->m_display_queue) ctx->m_display_queue = newFrame; ctx->m_queue_depth++; // pthread_mutex_unlock(&ctx->m_queue_mutex); }
void PrivateDecoderVDA::VDADecoderCallback(void *decompressionOutputRefCon, CFDictionaryRef frameInfo, OSStatus status, uint32_t infoFlags, CVImageBufferRef imageBuffer) { CocoaAutoReleasePool pool; PrivateDecoderVDA *decoder = (PrivateDecoderVDA*)decompressionOutputRefCon; if (kVDADecodeInfo_FrameDropped & infoFlags) { LOG(VB_GENERAL, LOG_ERR, LOC + "Callback: Decoder dropped frame"); return; } if (!imageBuffer) { LOG(VB_GENERAL, LOG_ERR, LOC + "Callback: decoder returned empty buffer."); return; } INIT_ST; vda_st = status; CHECK_ST; OSType format_type = CVPixelBufferGetPixelFormatType(imageBuffer); if ((format_type != '2vuy') && (format_type != 'BGRA')) { LOG(VB_GENERAL, LOG_ERR, LOC + QString("Callback: image buffer format unknown (%1)") .arg(format_type)); return; } int64_t pts = AV_NOPTS_VALUE; int8_t interlaced = 0; int8_t topfirst = 0; int8_t repeatpic = 0; CFNumberRef ptsref = (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_PTS")); CFNumberRef intref = (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_INTERLACED")); CFNumberRef topref = (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_TFF")); CFNumberRef repref = (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_REPEAT")); if (ptsref) { CFNumberGetValue(ptsref, kCFNumberSInt64Type, &pts); CFRelease(ptsref); } if (intref) { CFNumberGetValue(intref, kCFNumberSInt8Type, &interlaced); CFRelease(intref); } if (topref) { CFNumberGetValue(topref, kCFNumberSInt8Type, &topfirst); CFRelease(topref); } if (repref) { CFNumberGetValue(repref, kCFNumberSInt8Type, &repeatpic); CFRelease(repref); } int64_t time = (pts != (int64_t)AV_NOPTS_VALUE) ? pts : 0; { QMutexLocker lock(&decoder->m_frame_lock); bool found = false; int i = 0; for (; i < decoder->m_decoded_frames.size(); i++) { int64_t pts = decoder->m_decoded_frames[i].pts; if (pts != (int64_t)AV_NOPTS_VALUE && time > pts) { found = true; break; } } VDAFrame frame(CVPixelBufferRetain(imageBuffer), format_type, pts, interlaced, topfirst, repeatpic); if (!found) i = decoder->m_decoded_frames.size(); decoder->m_decoded_frames.insert(i, frame); decoder->m_frames_decoded++; } }