bool QPainterRenderer::preparePixmap(const VideoFrame &frame)
{
DPTR_D(QPainterRenderer);
// already locked in a larger scope of receive()
QImage::Format imgfmt = frame.imageFormat();
if (frame.constBits(0)) {
d.video_frame = frame;
} else {
if (imgfmt == QImage::Format_Invalid) {
d.video_frame = frame.to(VideoFormat::Format_RGB32);
imgfmt = d.video_frame.imageFormat();
} else {
d.video_frame = frame.to(frame.pixelFormat());
}
}
const bool swapRGB = (int)imgfmt < 0;
if (swapRGB) {
imgfmt = (QImage::Format)(-imgfmt);
}
// DO NOT use frameData().data() because it's temp ptr while d.image does not deep copy the data
QImage image = QImage((uchar*)d.video_frame.constBits(), d.video_frame.width(), d.video_frame.height(), d.video_frame.bytesPerLine(), imgfmt);
if (swapRGB)
image = image.rgbSwapped();
d.pixmap = QPixmap::fromImage(image);
//Format_RGB32 is fast. see document
return true;
}
VideoFrame VideoFrame::to(const VideoFormat &fmt, const QSize& dstSize, const QRectF& roi) const
{
if (!isValid() || !constBits(0)) {// hw surface. map to host. only supports rgb packed formats now
Q_D(const VideoFrame);
const QVariant v = d->metadata.value(QStringLiteral("surface_interop"));
if (!v.isValid())
return VideoFrame();
VideoSurfaceInteropPtr si = v.value<VideoSurfaceInteropPtr>();
if (!si)
return VideoFrame();
VideoFrame f;
f.setDisplayAspectRatio(displayAspectRatio());
f.setTimestamp(timestamp());
if (si->map(HostMemorySurface, fmt, &f)) {
if ((!dstSize.isValid() ||dstSize == QSize(width(), height())) && (!roi.isValid() || roi == QRectF(0, 0, width(), height()))) //roi is not supported now
return f;
return f.to(fmt, dstSize, roi);
}
return VideoFrame();
}
const int w = dstSize.width() > 0 ? dstSize.width() : width();
const int h = dstSize.height() > 0 ? dstSize.height() : height();
if (fmt.pixelFormatFFmpeg() == pixelFormatFFmpeg()
&& w == width() && h == height()
// TODO: roi check.
)
return *this;
Q_D(const VideoFrame);
ImageConverterSWS conv;
conv.setInFormat(pixelFormatFFmpeg());
conv.setOutFormat(fmt.pixelFormatFFmpeg());
conv.setInSize(width(), height());
conv.setOutSize(w, h);
conv.setInRange(colorRange());
if (!conv.convert(d->planes.constData(), d->line_sizes.constData())) {
qWarning() << "VideoFrame::to error: " << format() << "=>" << fmt;
return VideoFrame();
}
VideoFrame f(w, h, fmt, conv.outData());
f.setBits(conv.outPlanes());
f.setBytesPerLine(conv.outLineSizes());
if (fmt.isRGB()) {
f.setColorSpace(fmt.isPlanar() ? ColorSpace_GBR : ColorSpace_RGB);
} else {
f.setColorSpace(ColorSpace_Unknown);
}
// TODO: color range
f.setTimestamp(timestamp());
f.setDisplayAspectRatio(displayAspectRatio());
f.d_ptr->metadata = d->metadata; // need metadata?
return f;
}
VideoFrame VideoFrameConverter::convert(const VideoFrame &frame, int fffmt) const
{
if (!frame.isValid() || fffmt == QTAV_PIX_FMT_C(NONE))
return VideoFrame();
if (!frame.constBits(0)) // hw surface
return frame.to(VideoFormat::pixelFormatFromFFmpeg(fffmt));
const VideoFormat format(frame.format());
//if (fffmt == format.pixelFormatFFmpeg())
// return *this;
if (!m_cvt) {
m_cvt = new ImageConverterSWS();
}
m_cvt->setBrightness(m_eq[0]);
m_cvt->setContrast(m_eq[1]);
m_cvt->setSaturation(m_eq[2]);
m_cvt->setInFormat(format.pixelFormatFFmpeg());
m_cvt->setOutFormat(fffmt);
m_cvt->setInSize(frame.width(), frame.height());
m_cvt->setOutSize(frame.width(), frame.height());
m_cvt->setInRange(frame.colorRange());
const int pal = format.hasPalette();
QVector<const uchar*> pitch(format.planeCount() + pal);
QVector<int> stride(format.planeCount() + pal);
for (int i = 0; i < format.planeCount(); ++i) {
pitch[i] = frame.constBits(i);
stride[i] = frame.bytesPerLine(i);
}
const QByteArray paldata(frame.metaData(QStringLiteral("pallete")).toByteArray());
if (pal > 0) {
pitch[1] = (const uchar*)paldata.constData();
stride[1] = paldata.size();
}
if (!m_cvt->convert(pitch.constData(), stride.constData())) {
return VideoFrame();
}
const VideoFormat fmt(fffmt);
VideoFrame f(frame.width(), frame.height(), fmt, m_cvt->outData());
f.setBits(m_cvt->outPlanes());
f.setBytesPerLine(m_cvt->outLineSizes());
f.setTimestamp(frame.timestamp());
f.setDisplayAspectRatio(frame.displayAspectRatio());
// metadata?
if (fmt.isRGB()) {
f.setColorSpace(fmt.isPlanar() ? ColorSpace_GBR : ColorSpace_RGB);
} else {
f.setColorSpace(ColorSpace_Unknown);
}
// TODO: color range
return f;
}
VideoFrame VideoFrameConverter::convert(const VideoFrame &frame, int fffmt) const
{
if (!frame.isValid() || fffmt == QTAV_PIX_FMT_C(NONE))
return VideoFrame();
if (!frame.bits(0)) // hw surface
return frame.to(VideoFormat::pixelFormatFromFFmpeg(fffmt));
const VideoFormat format(frame.format());
//if (fffmt == format.pixelFormatFFmpeg())
// return *this;
if (!m_cvt) {
m_cvt = new ImageConverterSWS();
}
m_cvt->setBrightness(m_eq[0]);
m_cvt->setContrast(m_eq[1]);
m_cvt->setSaturation(m_eq[2]);
m_cvt->setInFormat(format.pixelFormatFFmpeg());
m_cvt->setOutFormat(fffmt);
m_cvt->setInSize(frame.width(), frame.height());
m_cvt->setOutSize(frame.width(), frame.height());
QVector<const uchar*> pitch(format.planeCount());
QVector<int> stride(format.planeCount());
for (int i = 0; i < format.planeCount(); ++i) {
pitch[i] = frame.bits(i);
stride[i] = frame.bytesPerLine(i);
}
if (!m_cvt->convert(pitch.constData(), stride.constData())) {
return VideoFrame();
}
const VideoFormat fmt(fffmt);
VideoFrame f(m_cvt->outData(), frame.width(), frame.height(), fmt);
f.setBits(m_cvt->outPlanes());
f.setBytesPerLine(m_cvt->outLineSizes());
f.setTimestamp(frame.timestamp());
// metadata?
if (fmt.isRGB()) {
f.setColorSpace(fmt.isPlanar() ? ColorSpace_GBR : ColorSpace_RGB);
} else {
f.setColorSpace(ColorSpace_Unknow);
}
return f;
}
void* SurfaceInteropVAAPI::mapToHost(const VideoFormat &format, void *handle, int plane)
{
Q_UNUSED(plane);
VAImage image;
static const unsigned int fcc[] = { VA_FOURCC_NV12, VA_FOURCC_YV12, VA_FOURCC_IYUV, 0};
va_new_image(m_surface->vadisplay(), fcc, &image, m_surface->width(), m_surface->height());
if (image.image_id == VA_INVALID_ID)
return NULL;
void *p_base;
VA_ENSURE(vaGetImage(m_surface->vadisplay(), m_surface->get(), 0, 0, m_surface->width(), m_surface->height(), image.image_id), NULL);
VA_ENSURE(vaMapBuffer(m_surface->vadisplay(), image.buf, &p_base), NULL); //TODO: destroy image before return
VideoFormat::PixelFormat pixfmt = pixelFormatFromVA(image.format.fourcc);
bool swap_uv = image.format.fourcc != VA_FOURCC_NV12;
if (pixfmt == VideoFormat::Format_Invalid) {
qWarning("unsupported vaapi pixel format: %#x", image.format.fourcc);
VA_ENSURE(vaDestroyImage(m_surface->vadisplay(), image.image_id), NULL);
return NULL;
}
const VideoFormat fmt(pixfmt);
uint8_t *src[3];
int pitch[3];
for (int i = 0; i < fmt.planeCount(); ++i) {
src[i] = (uint8_t*)p_base + image.offsets[i];
pitch[i] = image.pitches[i];
}
VideoFrame frame = VideoFrame::fromGPU(fmt, frame_width, frame_height, m_surface->height(), src, pitch, true, swap_uv);
if (format != fmt)
frame = frame.to(format);
VAWARN(vaUnmapBuffer(m_surface->vadisplay(), image.buf));
VAWARN(vaDestroyImage(m_surface->vadisplay(), image.image_id));
image.image_id = VA_INVALID_ID;
VideoFrame *f = reinterpret_cast<VideoFrame*>(handle);
frame.setTimestamp(f->timestamp());
*f = frame;
return f;
}
bool VideoEncoderFFmpeg::encode(const VideoFrame &frame)
{
DPTR_D(VideoEncoderFFmpeg);
QScopedPointer<AVFrame, ScopedAVFrameDeleter> f;
// hwupload
AVPixelFormat pixfmt = AVPixelFormat(frame.pixelFormatFFmpeg());
if (frame.isValid()) {
f.reset(av_frame_alloc());
f->format = pixfmt;
f->width = frame.width();
f->height = frame.height();
// f->quality = d.avctx->global_quality;
switch (timestampMode()) {
case TimestampCopy:
f->pts = int64_t(frame.timestamp()*frameRate()); // TODO: check monotically increase and fix if not. or another mode?
break;
case TimestampMonotonic:
f->pts = d.nb_encoded+1;
break;
default:
break;
}
// pts is set in muxer
const int nb_planes = frame.planeCount();
for (int i = 0; i < nb_planes; ++i) {
f->linesize[i] = frame.bytesPerLine(i);
f->data[i] = (uint8_t*)frame.constBits(i);
}
if (d.avctx->width <= 0) {
d.avctx->width = frame.width();
}
if (d.avctx->height <= 0) {
d.avctx->height = frame.width();
}
#ifdef HAVE_AVHWCTX
if (d.avctx->hw_frames_ctx) {
// TODO: try to map to SourceSurface
// checl valid sw_formats
if (!d.hwframes_ref) {
qWarning("no hw frame context for uploading");
return false;
}
if (pixfmt != d.hwframes->sw_format) {
// reinit or got an unsupported format. assume parameters will not change, so it's the 1st init
// check constraints
bool init_frames_ctx = d.hwframes->sw_format == AVPixelFormat(-1);
if (d.sw_fmts.contains(pixfmt)) { // format changed
init_frames_ctx = true;
} else { // convert to supported sw format
pixfmt = d.sw_fmts[0];
f->format = pixfmt;
VideoFrame converted = frame.to(VideoFormat::pixelFormatFromFFmpeg(pixfmt));
for (int i = 0; i < converted.planeCount(); ++i) {
f->linesize[i] = converted.bytesPerLine(i);
f->data[i] = (uint8_t*)frame.constBits(i);
}
}
if (init_frames_ctx) {
d.hwframes->sw_format = pixfmt;
d.hwframes->width = frame.width();
d.hwframes->height = frame.height();
AV_ENSURE(av_hwframe_ctx_init(d.hwframes_ref), false);
}
}
// upload
QScopedPointer<AVFrame, ScopedAVFrameDeleter> hwf( av_frame_alloc());
AV_ENSURE(av_hwframe_get_buffer(d.hwframes_ref, hwf.data(), 0), false);
//hwf->format = d.hwframes->format; // not necessary
//hwf->width = f->width;
//hwf->height = f->height;
AV_ENSURE(av_hwframe_transfer_data(hwf.data(), f.data(), 0), false);
AV_ENSURE(av_frame_copy_props(hwf.data(), f.data()), false);
av_frame_unref(f.data());
av_frame_move_ref(f.data(), hwf.data());
}
#endif //HAVE_AVHWCTX
}
AVPacket pkt;
av_init_packet(&pkt);
pkt.data = (uint8_t*)d.buffer.constData();
pkt.size = d.buffer.size();
int got_packet = 0;
int ret = avcodec_encode_video2(d.avctx, &pkt, f.data(), &got_packet);
if (ret < 0) {
qWarning("error avcodec_encode_video2: %s" ,av_err2str(ret));
return false; //false
}
d.nb_encoded++;
if (!got_packet) {
qWarning("no packet got");
d.packet = Packet();
// invalid frame means eof
return frame.isValid();
}
// qDebug("enc avpkt.pts: %lld, dts: %lld.", pkt.pts, pkt.dts);
d.packet = Packet::fromAVPacket(&pkt, av_q2d(d.avctx->time_base));
// qDebug("enc packet.pts: %.3f, dts: %.3f.", d.packet.pts, d.packet.dts);
return true;
}