VideoFrame VideoDecoderVAAPI::frame() { DPTR_D(VideoDecoderVAAPI); if (!d.frame->opaque || !d.frame->data[0]) return VideoFrame(); VASurfaceID surface_id = (VASurfaceID)(uintptr_t)d.frame->data[3]; VAStatus status = VA_STATUS_SUCCESS; if (display() == GLX) { d.surface_interop->setSurface((va_surface_t*)d.frame->opaque, d.surface_width, d.surface_height); VideoFrame f(d.surface_width, d.surface_height, VideoFormat::Format_RGB32); f.setBytesPerLine(d.surface_width*4); //used by gl to compute texture size f.setSurfaceInterop(d.surface_interop); return f; } #if VA_CHECK_VERSION(0,31,0) if ((status = vaSyncSurface(d.display, surface_id)) != VA_STATUS_SUCCESS) { qWarning("vaSyncSurface(VADisplay:%p, VASurfaceID:%#x) == %#x", d.display, surface_id, status); #else if (vaSyncSurface(d.display, d.context_id, surface_id)) { qWarning("vaSyncSurface(VADisplay:%#x, VAContextID:%#x, VASurfaceID:%#x) == %#x", d.display, d.context_id, surface_id, status); #endif return VideoFrame(); } if (!d.disable_derive && d.supports_derive) { /* * http://web.archiveorange.com/archive/v/OAywENyq88L319OcRnHI * vaDeriveImage is faster than vaGetImage. But VAImage is uncached memory and copying from it would be terribly slow * TODO: copy from USWC, see vlc and https://github.com/OpenELEC/OpenELEC.tv/pull/2937.diff * https://software.intel.com/en-us/articles/increasing-memory-throughput-with-intel-streaming-simd-extensions-4-intel-sse4-streaming-load */ status = vaDeriveImage(d.display, surface_id, &d.image); if (status != VA_STATUS_SUCCESS) { qWarning("vaDeriveImage(VADisplay:%p, VASurfaceID:%#x, VAImage*:%p) == %#x", d.display, surface_id, &d.image, status); return VideoFrame(); } } else { status = vaGetImage(d.display, surface_id, 0, 0, d.surface_width, d.surface_height, d.image.image_id); if (status != VA_STATUS_SUCCESS) { qWarning("vaGetImage(VADisplay:%p, VASurfaceID:%#x, 0,0, %d, %d, VAImageID:%#x) == %#x", d.display, surface_id, d.surface_width, d.surface_height, d.image.image_id, status); return VideoFrame(); } } void *p_base; if ((status = vaMapBuffer(d.display, d.image.buf, &p_base)) != VA_STATUS_SUCCESS) { qWarning("vaMapBuffer(VADisplay:%p, VABufferID:%#x, pBuf:%p) == %#x", d.display, d.image.buf, &p_base, status); return VideoFrame(); } VideoFormat::PixelFormat pixfmt = VideoFormat::Format_Invalid; bool swap_uv = false; switch (d.image.format.fourcc) { case VA_FOURCC_YV12: swap_uv |= d.disable_derive || !d.supports_derive; pixfmt = VideoFormat::Format_YUV420P; break; case VA_FOURCC_IYUV: swap_uv = true; pixfmt = VideoFormat::Format_YUV420P; break; case VA_FOURCC_NV12: pixfmt = VideoFormat::Format_NV12; break; default: break; } if (pixfmt == VideoFormat::Format_Invalid) { qWarning("unsupported vaapi pixel format: %#x", d.image.format.fourcc); return VideoFrame(); } const VideoFormat fmt(pixfmt); uint8_t *src[3]; int pitch[3]; for (int i = 0; i < fmt.planeCount(); ++i) { src[i] = (uint8_t*)p_base + d.image.offsets[i]; pitch[i] = d.image.pitches[i]; } if (swap_uv) { std::swap(src[1], src[2]); std::swap(pitch[1], pitch[2]); } VideoFrame frame; if (d.copy_uswc && d.gpu_mem.isReady()) { int yuv_size = 0; if (pixfmt == VideoFormat::Format_NV12) yuv_size = pitch[0]*d.surface_height*3/2; else yuv_size = pitch[0]*d.surface_height + pitch[1]*d.surface_height/2 + pitch[2]*d.surface_height/2; // additional 15 bytes to ensure 16 bytes aligned QByteArray buf(15 + yuv_size, 0); const int offset_16 = (16 - ((uintptr_t)buf.data() & 0x0f)) & 0x0f; // plane 1, 2... is aligned? uchar* plane_ptr = (uchar*)buf.data() + offset_16; QVector<uchar*> dst(fmt.planeCount(), 0); for (int i = 0; i < dst.size(); ++i) { dst[i] = plane_ptr; // TODO: add VideoFormat::planeWidth/Height() ? const int plane_w = pitch[i];//(i == 0 || pixfmt == VideoFormat::Format_NV12) ? d.surface_width : fmt.chromaWidth(d.surface_width); const int plane_h = i == 0 ? d.surface_height : fmt.chromaHeight(d.surface_height); plane_ptr += pitch[i] * plane_h; d.gpu_mem.copyFrame(src[i], dst[i], plane_w, plane_h, pitch[i]); } frame = VideoFrame(buf, d.width, d.height, fmt); frame.setBits(dst); frame.setBytesPerLine(pitch); } else { frame = VideoFrame(d.width, d.height, fmt); frame.setBits(src); frame.setBytesPerLine(pitch); // TODO: why clone is faster()? frame = frame.clone(); } if ((status = vaUnmapBuffer(d.display, d.image.buf)) != VA_STATUS_SUCCESS) { qWarning("vaUnmapBuffer(VADisplay:%p, VABufferID:%#x) == %#x", d.display, d.image.buf, status); return VideoFrame(); } if (!d.disable_derive && d.supports_derive) { vaDestroyImage(d.display, d.image.image_id); d.image.image_id = VA_INVALID_ID; } return frame; } struct display_names_t { VideoDecoderVAAPI::DisplayType display; QString name; }; static const display_names_t display_names[] = { { VideoDecoderVAAPI::GLX, "GLX" }, { VideoDecoderVAAPI::X11, "X11" }, { VideoDecoderVAAPI::DRM, "DRM" } }; static VideoDecoderVAAPI::DisplayType displayFromName(QString name) { for (unsigned int i = 0; i < sizeof(display_names)/sizeof(display_names[0]); ++i) { if (name.toUpper().contains(display_names[i].name.toUpper())) { return display_names[i].display; } } return VideoDecoderVAAPI::X11; } static QString displayToName(VideoDecoderVAAPI::DisplayType t) { for (unsigned int i = 0; i < sizeof(display_names)/sizeof(display_names[0]); ++i) { if (t == display_names[i].display) { return display_names[i].name; } } return QString(); } void VideoDecoderVAAPI::setDisplayPriority(const QStringList &priority) { DPTR_D(VideoDecoderVAAPI); d.display_priority.clear(); foreach (QString disp, priority) { d.display_priority.push_back(displayFromName(disp)); }
void push(int field) { queue->push_back(VideoFrame(false, in->mpi(), p->nextPTS(), field | (in->field() & ~VideoFrame::Interlaced))); ++pushed; }
void QuickVideoPreview::displayNoFrame() { receive(VideoFrame()); }
void VideoRendererItem::present(const QImage &image) { present(VideoFrame(image)); }
// ###################################################################### VideoFrame XCgrabberFlex::grabRaw() { #ifndef HAVE_XCLIB LFATAL("you must have XC support and the xclib library in order to use XCgrabberFlex"); return VideoFrame(); /* can't happen */ #else ASSERT(itsCameraOk); int i = 0; struct xclib::xcdevservice xcdev = itsXclib.xcdev; struct xclib::pxlibservice pxlib = itsXclib.pxlib; // get the captured buffer ID xclib::pxbuffer_t bufferID = (xclib::pxbuffer_t)xcdev.getLiveStatus (&xcdev, UNITMAP, 0, PXVIST_DONE | PXVIST_BUFFER); while( bufferID == itsLastBuf) { bufferID = (xclib::pxbuffer_t)xcdev.getLiveStatus (&xcdev, UNITMAP, 0, PXVIST_DONE | PXVIST_BUFFER); usleep(100); } if(itsLastBuf != 0 && bufferID != (itsLastBuf)%USEDBUFFER + 1) { LINFO("last buf id= %4d, curr buf id= %4d",(int)itsLastBuf,(int)bufferID); LERROR("buffer error: buffer mis order"); } pthread_mutex_lock(&qmutex_buf); itsLastBuf = bufferID; pthread_mutex_unlock(&qmutex_buf); // is the captured image base on byte or uint16 type int dataMode = (itsBitDepth == 8 ? PXDATUINT8:PXDATUINT16); const unsigned int bufSz = itsDims.getVal().sz() * (int)ceil(itsBitDepth/8); const unsigned int imgSz = itsDims.getVal().sz(); //! define the image from frame buffer struct xclib::pximage pximg; i = pxlib.initPximage(&pxlib, UNITMAP, &pximg, 1, PXHINTBAYER, 0, itsStateid, bufferID, 0); pximg.wind.nw.x = 1920/2 - itsDims.getVal().w()/2; pximg.wind.nw.y = 1080/2 - itsDims.getVal().h()/2; pximg.wind.se.x = 1920/2 + itsDims.getVal().w()/2; pximg.wind.se.y = 1080/2 + itsDims.getVal().h()/2; LINFO("pximgsize %d,%d", pximg.wind.nw.x,pximg.wind.se.x); if (i<1) LFATAL("error, can not define a pximage, code: %d",i); if(pximg.ioset(&pximg, PXRXSCAN | PXIWRAP, dataMode, 0x01) < 0) { LFATAL("error in ioset, can not set frame buffer read"); return VideoFrame(); } if(imgSz != pximg.ioread(&pximg, PXRXSCAN | PXIWRAP, itsImgBuf,bufSz,0,0)) { LFATAL("error in reading frame buffer(size error)," "expected size = %d", imgSz); return VideoFrame(); } return VideoFrame(itsImgBuf, bufSz, itsDims.getVal(), itsGrabMode.getVal(), itsByteSwap.getVal(), false); #endif // HAVE_XCLIB }
VideoFrame VideoDecoderVAAPI::frame() { DPTR_D(VideoDecoderVAAPI); if (!d.frame->opaque || !d.frame->data[0]) return VideoFrame(); VASurfaceID surface_id = (VASurfaceID)(uintptr_t)d.frame->data[3]; VAStatus status = VA_STATUS_SUCCESS; if (display() == GLX || (copyMode() == ZeroCopy && display() == X11)) { surface_ptr p; std::list<surface_ptr>::iterator it = d.surfaces_used.begin(); for (; it != d.surfaces_used.end() && !p; ++it) { if((*it)->get() == surface_id) { p = *it; break; } } if (!p) { for (it = d.surfaces_free.begin(); it != d.surfaces_free.end() && !p; ++it) { if((*it)->get() == surface_id) { p = *it; break; } } } if (!p) { qWarning("VAAPI - Unable to find surface"); return VideoFrame(); } ((SurfaceInteropVAAPI*)d.surface_interop.data())->setSurface(p); VideoFrame f(d.width, d.height, VideoFormat::Format_RGB32); //p->width() f.setBytesPerLine(d.width*4); //used by gl to compute texture size f.setMetaData("surface_interop", QVariant::fromValue(d.surface_interop)); f.setTimestamp(double(d.frame->pkt_pts)/1000.0); return f; } #if VA_CHECK_VERSION(0,31,0) if ((status = vaSyncSurface(d.display->get(), surface_id)) != VA_STATUS_SUCCESS) { qWarning("vaSyncSurface(VADisplay:%p, VASurfaceID:%#x) == %#x", d.display->get(), surface_id, status); #else if (vaSyncSurface(d.display->get(), d.context_id, surface_id)) { qWarning("vaSyncSurface(VADisplay:%#x, VAContextID:%#x, VASurfaceID:%#x) == %#x", d.display, d.context_id, surface_id, status); #endif return VideoFrame(); } if (!d.disable_derive && d.supports_derive) { /* * http://web.archiveorange.com/archive/v/OAywENyq88L319OcRnHI * vaDeriveImage is faster than vaGetImage. But VAImage is uncached memory and copying from it would be terribly slow * TODO: copy from USWC, see vlc and https://github.com/OpenELEC/OpenELEC.tv/pull/2937.diff * https://software.intel.com/en-us/articles/increasing-memory-throughput-with-intel-streaming-simd-extensions-4-intel-sse4-streaming-load */ VA_ENSURE_TRUE(vaDeriveImage(d.display->get(), surface_id, &d.image), VideoFrame()); } else { VA_ENSURE_TRUE(vaGetImage(d.display->get(), surface_id, 0, 0, d.width, d.height, d.image.image_id), VideoFrame()); } void *p_base; VA_ENSURE_TRUE(vaMapBuffer(d.display->get(), d.image.buf, &p_base), VideoFrame()); VideoFormat::PixelFormat pixfmt = VideoFormat::Format_Invalid; bool swap_uv = false; switch (d.image.format.fourcc) { case VA_FOURCC_YV12: swap_uv |= d.disable_derive || !d.supports_derive; pixfmt = VideoFormat::Format_YUV420P; break; case VA_FOURCC_IYUV: swap_uv = true; pixfmt = VideoFormat::Format_YUV420P; break; case VA_FOURCC_NV12: pixfmt = VideoFormat::Format_NV12; break; default: break; } if (pixfmt == VideoFormat::Format_Invalid) { qWarning("unsupported vaapi pixel format: %#x", d.image.format.fourcc); return VideoFrame(); } const VideoFormat fmt(pixfmt); uint8_t *src[3]; int pitch[3]; for (int i = 0; i < fmt.planeCount(); ++i) { src[i] = (uint8_t*)p_base + d.image.offsets[i]; pitch[i] = d.image.pitches[i]; } VideoFrame frame(copyToFrame(fmt, d.surface_height, src, pitch, swap_uv)); VAWARN(vaUnmapBuffer(d.display->get(), d.image.buf)); if (!d.disable_derive && d.supports_derive) { vaDestroyImage(d.display->get(), d.image.image_id); d.image.image_id = VA_INVALID_ID; } return frame; } void VideoDecoderVAAPI::setDisplayPriority(const QStringList &priority) { DPTR_D(VideoDecoderVAAPI); d.display_priority.clear(); int idx = staticMetaObject.indexOfEnumerator("DisplayType"); const QMetaEnum me = staticMetaObject.enumerator(idx); foreach (const QString& disp, priority) { d.display_priority.push_back((DisplayType)me.keyToValue(disp.toUtf8().constData())); }
// return the key frame position bool extractInPrecision(qint64 value, int range) { frame = VideoFrame(); if (value < demuxer.startTime()) value += demuxer.startTime(); demuxer.seek(value); const int vstream = demuxer.videoStream(); Packet pkt; qint64 pts0 = -1; bool warn_bad_seek = true; bool warn_out_of_range = true; while (!demuxer.atEnd()) { if (!demuxer.readFrame()) continue; if (demuxer.stream() != vstream) continue; pkt = demuxer.packet(); if (pts0 < 0LL) pts0 = (qint64)(pkt.pts*1000.0); if ((qint64)(pkt.pts*1000.0) - value > (qint64)range) { if (warn_out_of_range) qDebug("read packet out of range"); warn_out_of_range = false; // No return because decoder needs more packets before the desired frame is decoded //return false; } //qDebug("video packet: %f", pkt.pts); // TODO: always key frame? if (pkt.hasKeyFrame) break; if (warn_bad_seek) qWarning("Not seek to key frame!!!"); warn_bad_seek = false; } // enlarge range if seek to key-frame failed const qint64 key_pts = (qint64)(pkt.pts*1000.0); const bool enlarge_range = pts0 >= 0LL && key_pts - pts0 > 0LL; if (enlarge_range) { range = qMax<qint64>(key_pts - value, range); qDebug() << "enlarge range ==>>>> " << range; } if (!pkt.isValid()) { qWarning("VideoFrameExtractor failed to get a packet at %lld", value); return false; } decoder->flush(); //must flush otherwise old frames will be decoded at the beginning decoder->setOptions(dec_opt_normal); // must decode key frame int k = 0; while (k < 2 && !frame.isValid()) { //qWarning("invalid key frame!!!!! undecoded: %d", decoder->undecodedSize()); if (decoder->decode(pkt)) { frame = decoder->frame(); } ++k; } // if seek backward correctly to key frame, diff0 = t - value <= 0 // but sometimes seek to no-key frame(and range is enlarged), diff0 >= 0 // decode key frame const int diff0 = qint64(frame.timestamp()*1000.0) - value; if (qAbs(diff0) <= range) { //TODO: flag forward: result pts must >= value if (frame.isValid()) { qDebug() << "VideoFrameExtractor: key frame found @" << frame.timestamp() <<" diff=" << diff0 << ". format: " << frame.format(); return true; } } QVariantHash* dec_opt = &dec_opt_normal; // 0: default, 1: framedrop // decode at the given position while (!demuxer.atEnd()) { if (!demuxer.readFrame()) continue; if (demuxer.stream() != vstream) continue; pkt = demuxer.packet(); const qreal t = pkt.pts; //qDebug("video packet: %f, delta=%lld", t, value - qint64(t*1000.0)); if (!pkt.isValid()) { qWarning("invalid packet. no decode"); continue; } if (pkt.hasKeyFrame) { // FIXME: //qCritical("Internal error. Can not be a key frame!!!!"); //return false; //?? } qint64 diff = qint64(t*1000.0) - value; QVariantHash *dec_opt_old = dec_opt; if (seek_count == 0 || diff >= 0) dec_opt = &dec_opt_normal; else dec_opt = &dec_opt_framedrop; if (dec_opt != dec_opt_old) decoder->setOptions(*dec_opt); // invalid packet? if (!decoder->decode(pkt)) { qWarning("!!!!!!!!!decode failed!!!!"); frame = VideoFrame(); return false; } // store the last decoded frame because next frame may be out of range const VideoFrame f = decoder->frame(); if (!f.isValid()) { //qDebug("VideoFrameExtractor: invalid frame!!!"); continue; } frame = f; const qreal pts = frame.timestamp(); const qint64 pts_ms = pts*1000.0; if (pts_ms < value) continue; // diff = pts_ms - value; if (qAbs(diff) <= (qint64)range) { qDebug("got frame at %fs, diff=%lld", pts, diff); break; } // if decoder was not flushed, we may get old frame which is acceptable if (diff > range && t > pts) { qWarning("out pts out of range. diff=%lld, range=%d", diff, range); frame = VideoFrame(); return false; } } ++seek_count; // now we get the final frame return true; }
// ###################################################################### GenericFrame MgzJDecoder::readFrame() { // Grab the journal entry for this frame and allocate an appropriate GenericFrame MgzJEncoder::journalEntry entry = itsJournal.at(itsFrameNum); const Dims dims(entry.width, entry.height); const GenericFrame::NativeType pix_type = GenericFrame::NativeType(entry.pix_type); const int num_pix = dims.sz(); GenericFrame frame; //Read in the compressed image to a buffer uint64 comp_image_buf_size = entry.end_byte - entry.start_byte; byte * comp_image_buf = new byte[comp_image_buf_size]; itsFile.seekg(entry.start_byte, std::ios::beg); itsFile.read((char*)comp_image_buf, comp_image_buf_size); //Prepare zlib to do the decompression z_stream strm; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = 0; strm.next_in = Z_NULL; int ret = inflateInit(&strm); if(ret != Z_OK) LFATAL("Could not initialize zlib!"); strm.avail_in = comp_image_buf_size; strm.next_in = comp_image_buf; switch(pix_type) { case GenericFrame::GRAY_U8: { Image<byte> img(dims, NO_INIT); strm.avail_out = num_pix * sizeof(byte); strm.next_out = (unsigned char*)img.getArrayPtr(); ret = inflate(&strm, Z_FINISH); if(ret != Z_STREAM_END) { LFATAL("Could Not Inflate Frame! %d, %s", ret, strm.msg); } frame = GenericFrame(img); break; } case GenericFrame::GRAY_U16: { Image<uint16> img(dims, NO_INIT); strm.avail_out = num_pix * sizeof(uint16); strm.next_out = (unsigned char*)img.getArrayPtr(); ret = inflate(&strm, Z_FINISH); if(ret != Z_STREAM_END) { LFATAL("Could Not Inflate Frame! %d, %s", ret, strm.msg); } frame = GenericFrame(img); break; } case GenericFrame::GRAY_F32: { Image<float> img(dims, NO_INIT); strm.avail_out = num_pix * sizeof(float); strm.next_out = (unsigned char*)img.getArrayPtr(); ret = inflate(&strm, Z_FINISH); if(ret != Z_STREAM_END) { LFATAL("Could Not Inflate Frame! %d, %s", ret, strm.msg); } frame = GenericFrame(img, entry.flags); break; } case GenericFrame::RGB_U8: { Image<PixRGB<byte> > img(dims, NO_INIT); strm.avail_out = num_pix * sizeof(PixRGB<byte>); strm.next_out = (unsigned char*)img.getArrayPtr(); ret = inflate(&strm, Z_FINISH); if(ret != Z_STREAM_END) { LFATAL("Could Not Inflate Frame! %d, %s", ret, strm.msg); } frame = GenericFrame(img); break; } case GenericFrame::RGB_U16: { Image<PixRGB<uint16> > img(dims, NO_INIT); strm.avail_out = num_pix * sizeof(PixRGB<uint16>); strm.next_out = (unsigned char*)img.getArrayPtr(); ret = inflate(&strm, Z_FINISH); if(ret != Z_STREAM_END) { LFATAL("Could Not Inflate Frame! %d, %s", ret, strm.msg); } frame = GenericFrame(img); break; } case GenericFrame::RGB_F32: { Image<PixRGB<float> > img(dims, NO_INIT); strm.avail_out = num_pix * sizeof(PixRGB<float>); strm.next_out = (unsigned char*)img.getArrayPtr(); ret = inflate(&strm, Z_FINISH); if(ret != Z_STREAM_END) { LFATAL("Could Not Inflate Frame! %d, %s", ret, strm.msg); } frame = GenericFrame(img, entry.flags); break; } case GenericFrame::VIDEO: { const size_t vidSize = getFrameSize(VideoFormat(entry.flags), dims); ArrayHandle<byte> vidBuffer(new ArrayData<byte>(Dims(vidSize,1), NO_INIT)); strm.avail_out = vidSize; strm.next_out = (unsigned char*)vidBuffer.uniq().dataw(); ret = inflate(&strm, Z_FINISH); if(ret != Z_STREAM_END) { LFATAL("Could Not Inflate Frame! %d, %s", ret, strm.msg); } frame = GenericFrame(VideoFrame(vidBuffer, dims, VideoFormat(entry.flags), bool(entry.byte_swap))); break; } default: LFATAL("Could Not Open Frame Of Type: %d!", pix_type); } inflateEnd(&strm); delete [] comp_image_buf; return frame; }