void print_active_format (const VideoFormat& format)
{
std::cout << "Active format:\n"
<< "Format: \t" << fourcc_to_description(format.get_fourcc())
<< "\nResolution: \t" << format.get_size().width << "x" << format.get_size().height
<< "\nFramerate: \t" << format.get_framerate() << "\n" << std::endl;
}
void VideoShader::update(VideoMaterial *material)
{
if (!material->bind())
return;
const VideoFormat fmt(material->currentFormat());
//format is out of date because we may use the same shader for different formats
setVideoFormat(fmt);
// uniforms begin
program()->bind(); //glUseProgram(id). for glUniform
// all texture ids should be binded when renderering even for packed plane!
const int nb_planes = fmt.planeCount(); //number of texture id
for (int i = 0; i < nb_planes; ++i) {
// use glUniform1i to swap planes. swap uv: i => (3-i)%3
// TODO: in shader, use uniform sample2D u_Texture[], and use glUniform1iv(u_Texture, 3, {...})
program()->setUniformValue(textureLocation(i), (GLint)i);
}
if (nb_planes < textureLocationCount()) {
for (int i = nb_planes; i < textureLocationCount(); ++i) {
program()->setUniformValue(textureLocation(i), (GLint)(nb_planes - 1));
}
}
//qDebug() << "color mat " << material->colorMatrix();
program()->setUniformValue(colorMatrixLocation(), material->colorMatrix());
program()->setUniformValue(bppLocation(), (GLfloat)material->bpp());
//program()->setUniformValue(matrixLocation(), material->matrix()); //what about sgnode? state.combindMatrix()?
// uniform end. attribute begins
}
void VideoMaterial::setCurrentFrame(const VideoFrame &frame)
{
DPTR_D(VideoMaterial);
d.update_texure = true;
d.bpp = frame.format().bitsPerPixel(0);
d.width = frame.width();
d.height = frame.height();
const VideoFormat fmt(frame.format());
// http://forum.doom9.org/archive/index.php/t-160211.html
ColorTransform::ColorSpace cs = ColorTransform::RGB;
if (fmt.isRGB()) {
if (fmt.isPlanar())
cs = ColorTransform::GBR;
} else {
if (frame.width() >= 1280 || frame.height() > 576) //values from mpv
cs = ColorTransform::BT709;
else
cs = ColorTransform::BT601;
}
d.colorTransform.setInputColorSpace(cs);
d.frame = frame;
if (fmt != d.video_format) {
qDebug("pixel format changed: %s => %s", qPrintable(d.video_format.name()), qPrintable(fmt.name()));
d.video_format = fmt;
}
}
QDebug operator<< ( QDebug os, const VideoFormat & videoFormat ){
os << "w: " << videoFormat.frameWidth()
<< ", h: " << videoFormat.frameHeight()
<< ", fps: " << videoFormat.framesPerSecond()
<< ", count: " << videoFormat.frameCount()
<< ", type: " << videoFormat.type();
return os;
}
VideoFrame VideoDecoderDXVA::frame()
{
DPTR_D(VideoDecoderDXVA);
//qDebug("frame size: %dx%d", d.frame->width, d.frame->height);
if (!d.frame->opaque || !d.frame->data[0])
return VideoFrame();
if (d.frame->width <= 0 || d.frame->height <= 0 || !d.codec_ctx)
return VideoFrame();
IDirect3DSurface9 *d3d = (IDirect3DSurface9*)(uintptr_t)d.frame->data[3];
if (copyMode() == ZeroCopy && d.interop_res) {
dxva::SurfaceInteropDXVA *interop = new dxva::SurfaceInteropDXVA(d.interop_res);
interop->setSurface(d3d, width(), height());
VideoFrame f(width(), height(), VideoFormat::Format_RGB32); //p->width()
f.setBytesPerLine(d.width * 4); //used by gl to compute texture size
f.setMetaData(QStringLiteral("surface_interop"), QVariant::fromValue(VideoSurfaceInteropPtr(interop)));
f.setTimestamp(d.frame->pkt_pts/1000.0);
f.setDisplayAspectRatio(d.getDAR(d.frame));
return f;
}
class ScopedD3DLock {
IDirect3DSurface9 *mpD3D;
public:
ScopedD3DLock(IDirect3DSurface9* d3d, D3DLOCKED_RECT *rect) : mpD3D(d3d) {
if (FAILED(mpD3D->LockRect(rect, NULL, D3DLOCK_READONLY))) {
qWarning("Failed to lock surface");
mpD3D = 0;
}
}
~ScopedD3DLock() {
if (mpD3D)
mpD3D->UnlockRect();
}
};
D3DLOCKED_RECT lock;
ScopedD3DLock(d3d, &lock);
if (lock.Pitch == 0) {
return VideoFrame();
}
//picth >= desc.Width
D3DSURFACE_DESC desc;
d3d->GetDesc(&desc);
const VideoFormat fmt = VideoFormat(pixelFormatFromD3D(desc.Format));
if (!fmt.isValid()) {
qWarning("unsupported dxva pixel format: %#x", desc.Format);
return VideoFrame();
}
//YV12 need swap, not imc3?
// imc3 U V pitch == Y pitch, but half of the U/V plane is space. we convert to yuv420p here
// nv12 bpp(1)==1
// 3rd plane is not used for nv12
int pitch[3] = { lock.Pitch, 0, 0}; //compute chroma later
uint8_t *src[] = { (uint8_t*)lock.pBits, 0, 0}; //compute chroma later
const bool swap_uv = desc.Format == MAKEFOURCC('I','M','C','3');
return copyToFrame(fmt, d.surface_height, src, pitch, swap_uv);
}
VideoFrame VideoDecoderFFmpegHW::copyToFrame(const VideoFormat& fmt, int surface_h, quint8 *src[], int pitch[], bool swapUV)
{
DPTR_D(VideoDecoderFFmpegHW);
Q_ASSERT_X(src[0] && pitch[0] > 0, "VideoDecoderFFmpegHW::copyToFrame", "src[0] and pitch[0] must be set");
const int nb_planes = fmt.planeCount();
const int chroma_pitch = nb_planes > 1 ? fmt.bytesPerLine(pitch[0], 1) : 0;
const int chroma_h = fmt.chromaHeight(surface_h);
int h[] = { surface_h, 0, 0};
for (int i = 1; i < nb_planes; ++i) {
h[i] = chroma_h;
// set chroma address and pitch if not set
if (pitch[i] <= 0)
pitch[i] = chroma_pitch;
if (!src[i])
src[i] = src[i-1] + pitch[i-1]*h[i-1];
}
if (swapUV) {
std::swap(src[1], src[2]);
std::swap(pitch[1], pitch[2]);
}
VideoFrame frame;
if (copyMode() == VideoDecoderFFmpegHW::OptimizedCopy && d.gpu_mem.isReady()) {
int yuv_size = 0;
for (int i = 0; i < nb_planes; ++i) {
yuv_size += pitch[i]*h[i];
}
// additional 15 bytes to ensure 16 bytes aligned
QByteArray buf(15 + yuv_size, 0);
const int offset_16 = (16 - ((uintptr_t)buf.data() & 0x0f)) & 0x0f;
// plane 1, 2... is aligned?
uchar* plane_ptr = (uchar*)buf.data() + offset_16;
QVector<uchar*> dst(nb_planes, 0);
for (int i = 0; i < nb_planes; ++i) {
dst[i] = plane_ptr;
// TODO: add VideoFormat::planeWidth/Height() ?
// pitch instead of surface_width
plane_ptr += pitch[i] * h[i];
d.gpu_mem.copyFrame(src[i], dst[i], pitch[i], h[i], pitch[i]);
}
frame = VideoFrame(buf, width(), height(), fmt);
frame.setBits(dst);
frame.setBytesPerLine(pitch);
} else {
frame = VideoFrame(width(), height(), fmt);
frame.setBits(src);
frame.setBytesPerLine(pitch);
// TODO: why clone is faster()?
// TODO: buffer pool and create VideoFrame when needed to avoid copy? also for other va
frame = frame.clone();
}
frame.setTimestamp(double(d.frame->pkt_pts)/1000.0);
frame.setDisplayAspectRatio(d.getDAR(d.frame));
d.updateColorDetails(&frame);
return frame;
}
VideoFrame VideoFrame::to(const VideoFormat &fmt, const QSize& dstSize, const QRectF& roi) const
{
if (!isValid() || !constBits(0)) {// hw surface. map to host. only supports rgb packed formats now
Q_D(const VideoFrame);
const QVariant v = d->metadata.value(QStringLiteral("surface_interop"));
if (!v.isValid())
return VideoFrame();
VideoSurfaceInteropPtr si = v.value<VideoSurfaceInteropPtr>();
if (!si)
return VideoFrame();
VideoFrame f;
f.setDisplayAspectRatio(displayAspectRatio());
f.setTimestamp(timestamp());
if (si->map(HostMemorySurface, fmt, &f)) {
if ((!dstSize.isValid() ||dstSize == QSize(width(), height())) && (!roi.isValid() || roi == QRectF(0, 0, width(), height()))) //roi is not supported now
return f;
return f.to(fmt, dstSize, roi);
}
return VideoFrame();
}
const int w = dstSize.width() > 0 ? dstSize.width() : width();
const int h = dstSize.height() > 0 ? dstSize.height() : height();
if (fmt.pixelFormatFFmpeg() == pixelFormatFFmpeg()
&& w == width() && h == height()
// TODO: roi check.
)
return *this;
Q_D(const VideoFrame);
ImageConverterSWS conv;
conv.setInFormat(pixelFormatFFmpeg());
conv.setOutFormat(fmt.pixelFormatFFmpeg());
conv.setInSize(width(), height());
conv.setOutSize(w, h);
conv.setInRange(colorRange());
if (!conv.convert(d->planes.constData(), d->line_sizes.constData())) {
qWarning() << "VideoFrame::to error: " << format() << "=>" << fmt;
return VideoFrame();
}
VideoFrame f(w, h, fmt, conv.outData());
f.setBits(conv.outPlanes());
f.setBytesPerLine(conv.outLineSizes());
if (fmt.isRGB()) {
f.setColorSpace(fmt.isPlanar() ? ColorSpace_GBR : ColorSpace_RGB);
} else {
f.setColorSpace(ColorSpace_Unknown);
}
// TODO: color range
f.setTimestamp(timestamp());
f.setDisplayAspectRatio(displayAspectRatio());
f.d_ptr->metadata = d->metadata; // need metadata?
return f;
}
bool tcam::save_image (CaptureDevice& g, const std::string& filename)
{
struct data
{
CaptureDevice* g;
std::string filename;
int count;
bool wait;
};
auto f = [] (MemoryBuffer* buf, void* userdata)
{
data* d = static_cast<data*>(userdata);
if (d->count >= 2)
{
//save_jpeg(*buf, d->filename);
d->wait = false;
}
else
{
d->count++;
}
};
auto sink = std::make_shared<ImageSink>();
data d;
d.g = &g;
d.filename = filename;
d.count = 0;
d.wait = true;
sink->registerCallback(f, &d);
VideoFormat v;
v.from_string("format=RGB24,width=640,height=480,binning=0,framerate=30.000000");
g.set_video_format(v);
g.start_stream(sink);
// wait until image was captured
while(d.wait)
{
usleep(500);
}
d.g->stop_stream();
std::cout << "Successfully saved image" << std::endl;
return true;
}
static QMatrix4x4 channelMap(const VideoFormat& fmt)
{
if (fmt.isPlanar()) //currently only for planar
return QMatrix4x4();
switch (fmt.pixelFormat()) {
case VideoFormat::Format_UYVY:
return QMatrix4x4(0.0f, 0.5f, 0.0f, 0.5f,
1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f);
case VideoFormat::Format_YUYV:
return QMatrix4x4(0.5f, 0.0f, 0.5f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f,
0.0f, 0.0f, 0.0f, 1.0f);
case VideoFormat::Format_VYUY:
return QMatrix4x4(0.0f, 0.5f, 0.0f, 0.5f,
0.0f, 0.0f, 1.0f, 0.0f,
1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f);
case VideoFormat::Format_YVYU:
return QMatrix4x4(0.5f, 0.0f, 0.5f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f,
0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f);
case VideoFormat::Format_VYU:
return QMatrix4x4(0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f,
1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f);
default:
break;
}
const quint8 *channels = NULL;//{ 0, 1, 2, 3};
for (int i = 0; gl_channel_maps[i].pixfmt != VideoFormat::Format_Invalid; ++i) {
if (gl_channel_maps[i].pixfmt == fmt.pixelFormat()) {
channels = gl_channel_maps[i].channels;
break;
}
}
QMatrix4x4 m;
if (!channels)
return m;
m.fill(0);
for (int i = 0; i < 4; ++i) {
m(i, channels[i]) = 1;
}
qDebug() << m;
return m;
}
VideoFrame VideoDecoderDXVA::frame()
{
DPTR_D(VideoDecoderDXVA);
if (!d.frame->opaque || !d.frame->data[0])
return VideoFrame();
if (d.width <= 0 || d.height <= 0 || !d.codec_ctx)
return VideoFrame();
class ScopedD3DLock {
public:
ScopedD3DLock(IDirect3DSurface9* d3d, D3DLOCKED_RECT *rect)
: mpD3D(d3d)
{
if (FAILED(mpD3D->LockRect(rect, NULL, D3DLOCK_READONLY))) {
qWarning("Failed to lock surface");
mpD3D = 0;
}
}
~ScopedD3DLock() {
if (mpD3D)
mpD3D->UnlockRect();
}
private:
IDirect3DSurface9 *mpD3D;
};
IDirect3DSurface9 *d3d = (IDirect3DSurface9*)(uintptr_t)d.frame->data[3];
//picth >= desc.Width
//D3DSURFACE_DESC desc;
//d3d->GetDesc(&desc);
D3DLOCKED_RECT lock;
ScopedD3DLock(d3d, &lock);
if (lock.Pitch == 0) {
return VideoFrame();
}
const VideoFormat fmt = VideoFormat((int)D3dFindFormat(d.render)->avpixfmt);
if (!fmt.isValid()) {
qWarning("unsupported dxva pixel format: %#x", d.render);
return VideoFrame();
}
//YV12 need swap, not imc3?
// imc3 U V pitch == Y pitch, but half of the U/V plane is space. we convert to yuv420p here
// nv12 bpp(1)==1
// 3rd plane is not used for nv12
int pitch[3] = { lock.Pitch, 0, 0}; //compute chroma later
uint8_t *src[] = { (uint8_t*)lock.pBits, 0, 0}; //compute chroma later
const bool swap_uv = d.render == MAKEFOURCC('I','M','C','3');
return copyToFrame(fmt, d.surface_height, src, pitch, swap_uv);
}
VideoFrame VideoFrameConverter::convert(const VideoFrame &frame, int fffmt) const
{
if (!frame.isValid() || fffmt == QTAV_PIX_FMT_C(NONE))
return VideoFrame();
if (!frame.constBits(0)) // hw surface
return frame.to(VideoFormat::pixelFormatFromFFmpeg(fffmt));
const VideoFormat format(frame.format());
//if (fffmt == format.pixelFormatFFmpeg())
// return *this;
if (!m_cvt) {
m_cvt = new ImageConverterSWS();
}
m_cvt->setBrightness(m_eq[0]);
m_cvt->setContrast(m_eq[1]);
m_cvt->setSaturation(m_eq[2]);
m_cvt->setInFormat(format.pixelFormatFFmpeg());
m_cvt->setOutFormat(fffmt);
m_cvt->setInSize(frame.width(), frame.height());
m_cvt->setOutSize(frame.width(), frame.height());
m_cvt->setInRange(frame.colorRange());
const int pal = format.hasPalette();
QVector<const uchar*> pitch(format.planeCount() + pal);
QVector<int> stride(format.planeCount() + pal);
for (int i = 0; i < format.planeCount(); ++i) {
pitch[i] = frame.constBits(i);
stride[i] = frame.bytesPerLine(i);
}
const QByteArray paldata(frame.metaData(QStringLiteral("pallete")).toByteArray());
if (pal > 0) {
pitch[1] = (const uchar*)paldata.constData();
stride[1] = paldata.size();
}
if (!m_cvt->convert(pitch.constData(), stride.constData())) {
return VideoFrame();
}
const VideoFormat fmt(fffmt);
VideoFrame f(frame.width(), frame.height(), fmt, m_cvt->outData());
f.setBits(m_cvt->outPlanes());
f.setBytesPerLine(m_cvt->outLineSizes());
f.setTimestamp(frame.timestamp());
f.setDisplayAspectRatio(frame.displayAspectRatio());
// metadata?
if (fmt.isRGB()) {
f.setColorSpace(fmt.isPlanar() ? ColorSpace_GBR : ColorSpace_RGB);
} else {
f.setColorSpace(ColorSpace_Unknown);
}
// TODO: color range
return f;
}
void* SurfaceInteropDXVA::mapToHost(const VideoFormat &format, void *handle, int plane)
{
Q_UNUSED(plane);
class ScopedD3DLock {
IDirect3DSurface9 *mpD3D;
public:
ScopedD3DLock(IDirect3DSurface9* d3d, D3DLOCKED_RECT *rect) : mpD3D(d3d) {
if (FAILED(mpD3D->LockRect(rect, NULL, D3DLOCK_READONLY))) {
qWarning("Failed to lock surface");
mpD3D = 0;
}
}
~ScopedD3DLock() {
if (mpD3D)
mpD3D->UnlockRect();
}
};
D3DLOCKED_RECT lock;
ScopedD3DLock(m_surface, &lock);
if (lock.Pitch == 0)
return NULL;
//picth >= desc.Width
D3DSURFACE_DESC desc;
m_surface->GetDesc(&desc);
const VideoFormat fmt = VideoFormat(pixelFormatFromFourcc(desc.Format));
if (!fmt.isValid()) {
qWarning("unsupported dxva pixel format: %#x", desc.Format);
return NULL;
}
//YV12 need swap, not imc3?
// imc3 U V pitch == Y pitch, but half of the U/V plane is space. we convert to yuv420p here
// nv12 bpp(1)==1
// 3rd plane is not used for nv12
int pitch[3] = { lock.Pitch, 0, 0}; //compute chroma later
quint8 *src[] = { (quint8*)lock.pBits, 0, 0}; //compute chroma later
Q_ASSERT(src[0] && pitch[0] > 0);
const bool swap_uv = desc.Format == MAKEFOURCC('I','M','C','3');
// try to use SSE. fallback to normal copy if SSE is not supported
VideoFrame frame(VideoFrame::fromGPU(fmt, frame_width, frame_height, desc.Height, src, pitch, true, swap_uv));
// TODO: check rgb32 because d3d can use hw to convert
if (format != fmt)
frame = frame.to(format);
VideoFrame *f = reinterpret_cast<VideoFrame*>(handle);
frame.setTimestamp(f->timestamp());
*f = frame;
return f;
}
void GLWidgetRendererPrivate::updateShaderIfNeeded()
{
const VideoFormat& fmt(video_frame.format());
if (fmt != video_format) {
qDebug("pixel format changed: %s => %s", qPrintable(video_format.name()), qPrintable(fmt.name()));
}
VideoMaterialType *newType = materialType(fmt);
if (material_type == newType)
return;
material_type = newType;
// http://forum.doom9.org/archive/index.php/t-160211.html
ColorTransform::ColorSpace cs = ColorTransform::RGB;
if (fmt.isRGB()) {
if (fmt.isPlanar())
cs = ColorTransform::GBR;
} else {
if (video_frame.width() >= 1280 || video_frame.height() > 576) //values from mpv
cs = ColorTransform::BT709;
else
cs = ColorTransform::BT601;
}
if (!prepareShaderProgram(fmt, cs)) {
qWarning("shader program create error...");
return;
} else {
qDebug("shader program created!!!");
}
}
bool set_active_format (std::shared_ptr<CaptureDevice> dev, const std::string& new_format)
{
VideoFormat v;
bool ret = v.from_string(new_format);
if (ret)
{
return dev->set_video_format(v);
}
else
{
std::cout << "Invalid string description!" << std::endl;
}
return false;
}
void VideoFrame::allocate(const VideoFormat &format) {
if (format.isEmpty() && d->buffer.isEmpty())
return;
if (!d->buffer.isEmpty() && d->format == format)
return;
d.detach();
d->format = format;
int len = 0;
int offsets[4] = {0};
for (int i=0; i<format.planes(); ++i) {
offsets[i] = len;
len += format.bytesPerPlain(i);
}
d->buffer.resize(len);
for (int i=0; i< format.planes(); ++i)
d->data[i] = (uchar*)d->buffer.data() + offsets[i];
}
void QPainterFilterContext::initializeOnFrame(VideoFrame *vframe)
{
if (!vframe) {
if (!painter) {
painter = new QPainter(); //warning: more than 1 painter on 1 device
}
if (!paint_device) {
paint_device = painter->device();
}
if (!paint_device && !painter->isActive()) {
qWarning("No paint device and painter is not active. No painting!");
return;
}
if (!painter->isActive())
painter->begin(paint_device);
return;
}
VideoFormat format = vframe->format();
if (!format.isValid()) {
qWarning("Not a valid format");
return;
}
if (format.imageFormat() == QImage::Format_Invalid) {
format.setPixelFormat(VideoFormat::Format_RGB32);
if (!cvt) {
cvt = new VideoFrameConverter();
}
*vframe = cvt->convert(*vframe, format);
}
if (paint_device) {
if (painter && painter->isActive()) {
painter->end(); //destroy a paint device that is being painted is not allowed!
}
delete paint_device;
paint_device = 0;
}
Q_ASSERT(video_width > 0 && video_height > 0);
// direct draw on frame data, so use VideoFrame::constBits()
paint_device = new QImage((uchar*)vframe->constBits(0), video_width, video_height, vframe->bytesPerLine(0), format.imageFormat());
if (!painter)
painter = new QPainter();
own_painter = true;
own_paint_device = true; //TODO: what about renderer is not a widget?
painter->begin((QImage*)paint_device);
}
void MPTranscoder::start()
{
m_composer = IMediaComposer::create(ProgressListener(new ProgressListenerWrapper(*this)));
m_composer->addSourceFile(m_input.toStdString());
m_composer->setTargetFile(m_output.toStdString());
VideoFormat videoFormat = IVideoFormat::create(MIMETypeAVC, 640, 480);
videoFormat->setVideoBitRateInKBytes(1500);
videoFormat->setVideoFrameRate(25);
videoFormat->setVideoIFrameInterval(1);
m_composer->setTargetVideoFormat(videoFormat);
AudioFormat audioFormat = IAudioFormat::create(MIMETypeAAC, 48000, 2);
audioFormat->setAudioBitrateInBytes(96 * 1024);
m_composer->setTargetAudioFormat(audioFormat);
m_composer->start();
}
bool convertTo(const VideoFormat& fmt, const QSizeF &dstSize, const QRectF &roi) {
if (fmt == format.pixelFormatFFmpeg()
&& roi == QRectF(0, 0, width, height)
&& dstSize == roi.size())
return true;
if (!conv) {
format.setPixelFormat(VideoFormat::Format_Invalid);
return false;
}
format = fmt;
data = conv->outData();
planes = conv->outPlanes();
line_sizes = conv->outLineSizes();
planes.resize(fmt.planeCount());
line_sizes.resize(fmt.planeCount());
textures.resize(fmt.planeCount());
return false;
}
VideoFrame VideoFrameConverter::convert(const VideoFrame &frame, int fffmt) const
{
if (!frame.isValid() || fffmt == QTAV_PIX_FMT_C(NONE))
return VideoFrame();
if (!frame.bits(0)) // hw surface
return frame.to(VideoFormat::pixelFormatFromFFmpeg(fffmt));
const VideoFormat format(frame.format());
//if (fffmt == format.pixelFormatFFmpeg())
// return *this;
if (!m_cvt) {
m_cvt = new ImageConverterSWS();
}
m_cvt->setBrightness(m_eq[0]);
m_cvt->setContrast(m_eq[1]);
m_cvt->setSaturation(m_eq[2]);
m_cvt->setInFormat(format.pixelFormatFFmpeg());
m_cvt->setOutFormat(fffmt);
m_cvt->setInSize(frame.width(), frame.height());
m_cvt->setOutSize(frame.width(), frame.height());
QVector<const uchar*> pitch(format.planeCount());
QVector<int> stride(format.planeCount());
for (int i = 0; i < format.planeCount(); ++i) {
pitch[i] = frame.bits(i);
stride[i] = frame.bytesPerLine(i);
}
if (!m_cvt->convert(pitch.constData(), stride.constData())) {
return VideoFrame();
}
const VideoFormat fmt(fffmt);
VideoFrame f(m_cvt->outData(), frame.width(), frame.height(), fmt);
f.setBits(m_cvt->outPlanes());
f.setBytesPerLine(m_cvt->outLineSizes());
f.setTimestamp(frame.timestamp());
// metadata?
if (fmt.isRGB()) {
f.setColorSpace(fmt.isPlanar() ? ColorSpace_GBR : ColorSpace_RGB);
} else {
f.setColorSpace(ColorSpace_Unknow);
}
return f;
}
void GLWidgetRendererPrivate::updateTexturesIfNeeded()
{
const VideoFormat &fmt = video_frame.format();
bool update_textures = false;
if (fmt != video_format) {
update_textures = true; //FIXME
qDebug("updateTexturesIfNeeded pixel format changed: %s => %s", qPrintable(video_format.name()), qPrintable(fmt.name()));
}
// effective size may change even if plane size not changed
if (update_textures
|| video_frame.bytesPerLine(0) != plane0Size.width() || video_frame.height() != plane0Size.height()
|| (plane1_linesize > 0 && video_frame.bytesPerLine(1) != plane1_linesize)) { // no need to check height if plane 0 sizes are equal?
update_textures = true;
qDebug("---------------------update texture: %dx%d, %s", video_frame.width(), video_frame.height(), video_frame.format().name().toUtf8().constData());
const int nb_planes = fmt.planeCount();
texture_size.resize(nb_planes);
texture_upload_size.resize(nb_planes);
effective_tex_width.resize(nb_planes);
for (int i = 0; i < nb_planes; ++i) {
qDebug("plane linesize %d: padded = %d, effective = %d", i, video_frame.bytesPerLine(i), video_frame.effectiveBytesPerLine(i));
qDebug("plane width %d: effective = %d", video_frame.planeWidth(i), video_frame.effectivePlaneWidth(i));
qDebug("planeHeight %d = %d", i, video_frame.planeHeight(i));
// we have to consider size of opengl format. set bytesPerLine here and change to width later
texture_size[i] = QSize(video_frame.bytesPerLine(i), video_frame.planeHeight(i));
texture_upload_size[i] = texture_size[i];
effective_tex_width[i] = video_frame.effectiveBytesPerLine(i); //store bytes here, modify as width later
// TODO: ratio count the GL_UNPACK_ALIGN?
//effective_tex_width_ratio = qMin((qreal)1.0, (qreal)video_frame.effectiveBytesPerLine(i)/(qreal)video_frame.bytesPerLine(i));
}
plane1_linesize = 0;
if (nb_planes > 1) {
texture_size[0].setWidth(texture_size[1].width() * effective_tex_width[0]/effective_tex_width[1]);
// height? how about odd?
plane1_linesize = video_frame.bytesPerLine(1);
}
effective_tex_width_ratio = (qreal)video_frame.effectiveBytesPerLine(nb_planes-1)/(qreal)video_frame.bytesPerLine(nb_planes-1);
qDebug("effective_tex_width_ratio=%f", effective_tex_width_ratio);
plane0Size.setWidth(video_frame.bytesPerLine(0));
plane0Size.setHeight(video_frame.height());
}
if (update_textures) {
initTextures(fmt);
}
}
bool videoFormatToGL(const VideoFormat& fmt, GLint* internal_format, GLenum* data_format, GLenum* data_type)
{
struct fmt_entry {
VideoFormat::PixelFormat pixfmt;
GLint internal_format;
GLenum format;
GLenum type;
};
// Very special formats, for which OpenGL happens to have direct support
static const struct fmt_entry pixfmt_to_gl_formats[] = {
#ifdef QT_OPENGL_ES_2
{VideoFormat::Format_ARGB32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_RGB32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE },
#else
{VideoFormat::Format_RGB32, GL_RGBA, GL_BGRA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_ARGB32, GL_RGBA, GL_BGRA, GL_UNSIGNED_BYTE },
#endif
{VideoFormat::Format_RGB24, GL_RGB, GL_RGB, GL_UNSIGNED_BYTE },
#ifdef GL_UNSIGNED_SHORT_1_5_5_5_REV
{VideoFormat::Format_RGB555, GL_RGBA, GL_BGRA, GL_UNSIGNED_SHORT_1_5_5_5_REV},
#endif
{VideoFormat::Format_RGB565, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT_5_6_5}, //GL_UNSIGNED_SHORT_5_6_5_REV?
//{VideoFormat::Format_BGRA32, GL_RGBA, GL_BGRA, GL_UNSIGNED_BYTE },
//{VideoFormat::Format_BGR32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_BGR24, GL_RGB, GL_BGR, GL_UNSIGNED_BYTE },
#ifdef GL_UNSIGNED_SHORT_1_5_5_5_REV
{VideoFormat::Format_BGR555, GL_RGBA, GL_BGRA, GL_UNSIGNED_SHORT_1_5_5_5_REV},
#endif
{VideoFormat::Format_BGR565, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT_5_6_5}, // need swap r b?
};
for (unsigned int i = 0; i < sizeof(pixfmt_to_gl_formats)/sizeof(pixfmt_to_gl_formats[0]); ++i) {
if (pixfmt_to_gl_formats[i].pixfmt == fmt.pixelFormat()) {
*internal_format = pixfmt_to_gl_formats[i].internal_format;
*data_format = pixfmt_to_gl_formats[i].format;
*data_type = pixfmt_to_gl_formats[i].type;
return true;
}
}
return false;
}
bool GLWidgetRendererPrivate::releaseShaderProgram()
{
video_format.setPixelFormat(VideoFormat::Format_Invalid);
#if NO_QGL_SHADER
if (vert) {
if (program)
glDetachShader(program, vert);
glDeleteShader(vert);
}
if (frag) {
if (program)
glDetachShader(program, frag);
glDeleteShader(frag);
}
if (program) {
glDeleteProgram(program);
program = 0;
}
#else
shader_program->removeAllShaders();
#endif
return true;
}
void GLWidgetRendererPrivate::uploadPlane(int p, GLint internalFormat, GLenum format, const QRect& roi)
{
// FIXME: why happens on win?
if (video_frame.bytesPerLine(p) <= 0)
return;
glActiveTexture(GL_TEXTURE0 + p); //xbmc: only for es, not for desktop?
glBindTexture(GL_TEXTURE_2D, textures[p]);
////nv12: 2
//glPixelStorei(GL_UNPACK_ALIGNMENT, 1);//GetAlign(video_frame.bytesPerLine(p)));
#if defined(GL_UNPACK_ROW_LENGTH)
// glPixelStorei(GL_UNPACK_ROW_LENGTH, video_frame.bytesPerLine(p));
#endif
setupQuality();
//qDebug("bpl[%d]=%d width=%d", p, video_frame.bytesPerLine(p), video_frame.planeWidth(p));
// This is necessary for non-power-of-two textures
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//uploading part of image eats less gpu memory, but may be more cpu(gles)
//FIXME: more cpu usage then qpainter. FBO, VBO?
//roi for planes?
if (ROI_TEXCOORDS || roi.size() == video_frame.size()) {
glTexSubImage2D(GL_TEXTURE_2D
, 0 //level
, 0 // xoffset
, 0 // yoffset
, texture_size[p].width()
, texture_size[p].height()
, format //format, must the same as internal format?
, data_type[p]
, video_frame.bits(p));
} else {
int roi_x = roi.x();
int roi_y = roi.y();
int roi_w = roi.width();
int roi_h = roi.height();
int plane_w = video_frame.planeWidth(p);
VideoFormat fmt = video_frame.format();
if (p == 0) {
plane0Size = QSize(roi_w, roi_h);
} else {
roi_x = fmt.chromaWidth(roi_x);
roi_y = fmt.chromaHeight(roi_y);
roi_w = fmt.chromaWidth(roi_w);
roi_h = fmt.chromaHeight(roi_h);
}
qDebug("roi: %d, %d %dx%d", roi_x, roi_y, roi_w, roi_h);
#if 0// defined(GL_UNPACK_ROW_LENGTH) && defined(GL_UNPACK_SKIP_PIXELS)
// http://stackoverflow.com/questions/205522/opengl-subtexturing
glPixelStorei(GL_UNPACK_ROW_LENGTH, plane_w);
//glPixelStorei or compute pointer
glPixelStorei(GL_UNPACK_SKIP_PIXELS, roi_x);
glPixelStorei(GL_UNPACK_SKIP_ROWS, roi_y);
glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, roi_w, roi_h, 0, format, GL_UNSIGNED_BYTE, video_frame.bits(p));
//glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, roi_w, roi_h, format, GL_UNSIGNED_BYTE, video_frame.bits(p));
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
glPixelStorei(GL_UNPACK_SKIP_PIXELS, 0);
glPixelStorei(GL_UNPACK_SKIP_ROWS, 0);
#else // GL ES
//define it? or any efficient way?
//glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, roi_w, roi_h, 0, format, GL_UNSIGNED_BYTE, NULL);
const char *src = (char*)video_frame.bits(p) + roi_y*plane_w + roi_x*fmt.bytesPerPixel(p);
#define UPLOAD_LINE 1
#if UPLOAD_LINE
for (int y = 0; y < roi_h; y++) {
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, y, roi_w, 1, format, GL_UNSIGNED_BYTE, src);
}
#else
int line_size = roi_w*fmt.bytesPerPixel(p);
char *sub = (char*)malloc(roi_h*line_size);
char *dst = sub;
for (int y = 0; y < roi_h; y++) {
memcpy(dst, src, line_size);
src += video_frame.bytesPerLine(p);
dst += line_size;
}
// FIXME: crash
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, roi_w, roi_h, format, GL_UNSIGNED_BYTE, sub);
free(sub);
#endif //UPLOAD_LINE
#endif //GL_UNPACK_ROW_LENGTH
}
#if defined(GL_UNPACK_ROW_LENGTH)
// glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
#endif
//glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
glBindTexture(GL_TEXTURE_2D, 0);
}
bool GLWidgetRendererPrivate::initTextures(const VideoFormat &fmt)
{
// isSupported(pixfmt)
if (!fmt.isValid())
return false;
video_format.setPixelFormatFFmpeg(fmt.pixelFormatFFmpeg());
//http://www.berkelium.com/OpenGL/GDC99/internalformat.html
//NV12: UV is 1 plane. 16 bits as a unit. GL_LUMINANCE4, 8, 16, ... 32?
//GL_LUMINANCE, GL_LUMINANCE_ALPHA are deprecated in GL3, removed in GL3.1
//replaced by GL_RED, GL_RG, GL_RGB, GL_RGBA? for 1, 2, 3, 4 channel image
//http://www.gamedev.net/topic/634850-do-luminance-textures-still-exist-to-opengl/
//https://github.com/kivy/kivy/issues/1738: GL_LUMINANCE does work on a Galaxy Tab 2. LUMINANCE_ALPHA very slow on Linux
//ALPHA: vec4(1,1,1,A), LUMINANCE: (L,L,L,1), LUMINANCE_ALPHA: (L,L,L,A)
/*
* To support both planar and packed use GL_ALPHA and in shader use r,g,a like xbmc does.
* or use Swizzle_mask to layout the channels: http://www.opengl.org/wiki/Texture#Swizzle_mask
* GL ES2 support: GL_RGB, GL_RGBA, GL_LUMINANCE, GL_LUMINANCE_ALPHA, GL_ALPHA
* http://stackoverflow.com/questions/18688057/which-opengl-es-2-0-texture-formats-are-color-depth-or-stencil-renderable
*/
internal_format = QVector<GLint>(fmt.planeCount(), FMT_INTERNAL);
data_format = QVector<GLenum>(fmt.planeCount(), FMT);
data_type = QVector<GLenum>(fmt.planeCount(), GL_UNSIGNED_BYTE);
if (fmt.isPlanar()) {
/*!
* GLES internal_format == data_format, GL_LUMINANCE_ALPHA is 2 bytes
* so if NV12 use GL_LUMINANCE_ALPHA, YV12 use GL_ALPHA
*/
qDebug("///////////bpp %d", fmt.bytesPerPixel());
internal_format[0] = data_format[0] = GL_LUMINANCE; //or GL_RED for GL
if (fmt.planeCount() == 2) {
internal_format[1] = data_format[1] = GL_LUMINANCE_ALPHA;
} else {
if (fmt.bytesPerPixel(1) == 2) {
// read 16 bits and compute the real luminance in shader
internal_format[0] = data_format[0] = GL_LUMINANCE_ALPHA;
internal_format[1] = data_format[1] = GL_LUMINANCE_ALPHA; //vec4(L,L,L,A)
internal_format[2] = data_format[2] = GL_LUMINANCE_ALPHA;
} else {
internal_format[1] = data_format[1] = GL_LUMINANCE; //vec4(L,L,L,1)
internal_format[2] = data_format[2] = GL_ALPHA;//GL_ALPHA;
}
}
for (int i = 0; i < internal_format.size(); ++i) {
// xbmc use bpp not bpp(plane)
//internal_format[i] = GetGLInternalFormat(data_format[i], fmt.bytesPerPixel(i));
//data_format[i] = internal_format[i];
}
} else {
//glPixelStorei(GL_UNPACK_ALIGNMENT, fmt.bytesPerPixel());
// TODO: if no alpha, data_fmt is not GL_BGRA. align at every upload?
}
for (int i = 0; i < fmt.planeCount(); ++i) {
//qDebug("format: %#x GL_LUMINANCE_ALPHA=%#x", data_format[i], GL_LUMINANCE_ALPHA);
if (fmt.bytesPerPixel(i) == 2 && fmt.planeCount() == 3) {
//data_type[i] = GL_UNSIGNED_SHORT;
}
int bpp_gl = bytesOfGLFormat(data_format[i], data_type[i]);
int pad = qCeil((qreal)(texture_size[i].width() - effective_tex_width[i])/(qreal)bpp_gl);
texture_size[i].setWidth(qCeil((qreal)texture_size[i].width()/(qreal)bpp_gl));
effective_tex_width[i] /= bpp_gl; //fmt.bytesPerPixel(i);
//effective_tex_width_ratio =
qDebug("texture width: %d - %d = pad: %d. bpp(gl): %d", texture_size[i].width(), effective_tex_width[i], pad, bpp_gl);
}
/*
* there are 2 fragment shaders: rgb and yuv.
* only 1 texture for packed rgb. planar rgb likes yuv
* To support both planar and packed yuv, and mixed yuv(NV12), we give a texture sample
* for each channel. For packed, each (channel) texture sample is the same. For planar,
* packed channels has the same texture sample.
* But the number of actural textures we upload is plane count.
* Which means the number of texture id equals to plane count
*/
if (textures.size() != fmt.planeCount()) {
glDeleteTextures(textures.size(), textures.data());
qDebug("delete %d textures", textures.size());
textures.clear();
textures.resize(fmt.planeCount());
glGenTextures(textures.size(), textures.data());
}
if (!hasGLSL) {
initTexture(textures[0], internal_format[0], data_format[0], data_type[0], texture_size[0].width(), texture_size[0].height());
// more than 1?
qWarning("Does not support GLSL!");
return false;
}
qDebug("init textures...");
initTexture(textures[0], internal_format[0], data_format[0], data_type[0], texture_size[0].width(), texture_size[0].height());
for (int i = 1; i < textures.size(); ++i) {
initTexture(textures[i], internal_format[i], data_format[i], data_type[i], texture_size[i].width(), texture_size[i].height());
}
return true;
}
bool GLWidgetRendererPrivate::prepareShaderProgram(const VideoFormat &fmt)
{
// isSupported(pixfmt)
if (!fmt.isValid())
return false;
releaseShaderProgram();
video_format.setPixelFormatFFmpeg(fmt.pixelFormatFFmpeg());
// TODO: only to kinds, packed.glsl, planar.glsl
QString frag;
if (fmt.isPlanar()) {
frag = getShaderFromFile("shaders/yuv_rgb.f.glsl");
} else {
frag = getShaderFromFile("shaders/rgb.f.glsl");
}
if (frag.isEmpty())
return false;
if (!fmt.isRGB() && fmt.isPlanar() && fmt.bytesPerPixel(0) == 2) {
if (fmt.isBigEndian())
frag.prepend("#define YUV16BITS_BE_LUMINANCE_ALPHA\n");
else
frag.prepend("#define YUV16BITS_LE_LUMINANCE_ALPHA\n");
frag.prepend(QString("#define YUV%1P\n").arg(fmt.bitsPerPixel(0)));
}
#if NO_QGL_SHADER
program = createProgram(kVertexShader, frag.toUtf8().constData());
if (!program) {
qWarning("Could not create shader program.");
return false;
}
// vertex shader
a_Position = glGetAttribLocation(program, "a_Position");
a_TexCoords = glGetAttribLocation(program, "a_TexCoords");
u_matrix = glGetUniformLocation(program, "u_MVP_matrix");
// fragment shader
u_colorMatrix = glGetUniformLocation(program, "u_colorMatrix");
#else
if (!shader_program->addShaderFromSourceCode(QGLShader::Vertex, kVertexShader)) {
qWarning("Failed to add vertex shader: %s", shader_program->log().toUtf8().constData());
return false;
}
if (!shader_program->addShaderFromSourceCode(QGLShader::Fragment, frag)) {
qWarning("Failed to add fragment shader: %s", shader_program->log().toUtf8().constData());
return false;
}
if (!shader_program->link()) {
qWarning("Failed to link shader program...%s", shader_program->log().toUtf8().constData());
return false;
}
// vertex shader
a_Position = shader_program->attributeLocation("a_Position");
a_TexCoords = shader_program->attributeLocation("a_TexCoords");
u_matrix = shader_program->uniformLocation("u_MVP_matrix");
// fragment shader
u_colorMatrix = shader_program->uniformLocation("u_colorMatrix");
#endif //NO_QGL_SHADER
qDebug("glGetAttribLocation(\"a_Position\") = %d\n", a_Position);
qDebug("glGetAttribLocation(\"a_TexCoords\") = %d\n", a_TexCoords);
qDebug("glGetUniformLocation(\"u_MVP_matrix\") = %d\n", u_matrix);
qDebug("glGetUniformLocation(\"u_colorMatrix\") = %d\n", u_colorMatrix);
if (fmt.isRGB())
u_Texture.resize(1);
else
u_Texture.resize(fmt.channels());
for (int i = 0; i < u_Texture.size(); ++i) {
QString tex_var = QString("u_Texture%1").arg(i);
#if NO_QGL_SHADER
u_Texture[i] = glGetUniformLocation(program, tex_var.toUtf8().constData());
#else
u_Texture[i] = shader_program->uniformLocation(tex_var);
#endif
qDebug("glGetUniformLocation(\"%s\") = %d\n", tex_var.toUtf8().constData(), u_Texture[i]);
}
return true;
}
bool videoFormatToGL(const VideoFormat& fmt, GLint* internal_format, GLenum* data_format, GLenum* data_type, QMatrix4x4* mat)
{
typedef struct fmt_entry {
VideoFormat::PixelFormat pixfmt;
GLint internal_format;
GLenum format;
GLenum type;
} fmt_entry;
static const fmt_entry pixfmt_to_gles[] = {
{VideoFormat::Format_BGRA32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE }, //tested for angle
{VideoFormat::Format_RGB32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_Invalid, 0, 0, 0}
};
Q_UNUSED(pixfmt_to_gles);
static const fmt_entry pixfmt_to_desktop[] = {
{VideoFormat::Format_BGRA32, GL_RGBA, GL_BGRA, GL_UNSIGNED_BYTE }, //bgra bgra works on win but not osx
{VideoFormat::Format_RGB32, GL_RGBA, GL_BGRA, GL_UNSIGNED_BYTE }, //FIXMEL endian check
//{VideoFormat::Format_BGRA32, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE }, //{2,1,0,3}
//{VideoFormat::Format_BGR24, GL_RGB, GL_BGR, GL_UNSIGNED_BYTE }, //{0,1,2,3}
#ifdef GL_UNSIGNED_SHORT_5_6_5_REV
{VideoFormat::Format_BGR565, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT_5_6_5_REV}, // es error, use channel map
#endif
#ifdef GL_UNSIGNED_SHORT_1_5_5_5_REV
{VideoFormat::Format_RGB555, GL_RGBA, GL_BGRA, GL_UNSIGNED_SHORT_1_5_5_5_REV},
#endif
#ifdef GL_UNSIGNED_SHORT_1_5_5_5_REV
{VideoFormat::Format_BGR555, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT_1_5_5_5_REV},
#endif
// TODO: BE formats not implemeted
{VideoFormat::Format_RGB48, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT }, //TODO: they are not work for ANGLE, and rgb16 works on desktop gl, so remove these lines to use rgb16?
{VideoFormat::Format_RGB48LE, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT },
{VideoFormat::Format_RGB48BE, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT },
{VideoFormat::Format_BGR48, GL_RGB, GL_BGR, GL_UNSIGNED_SHORT }, //RGB16?
{VideoFormat::Format_BGR48LE, GL_RGB, GL_BGR, GL_UNSIGNED_SHORT },
{VideoFormat::Format_BGR48BE, GL_RGB, GL_BGR, GL_UNSIGNED_SHORT },
{VideoFormat::Format_RGBA64LE, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT },
{VideoFormat::Format_RGBA64BE, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT },
{VideoFormat::Format_BGRA64LE, GL_RGBA, GL_BGRA, GL_UNSIGNED_SHORT },
{VideoFormat::Format_BGRA64BE, GL_RGBA, GL_BGRA, GL_UNSIGNED_SHORT },
{VideoFormat::Format_Invalid, 0, 0, 0}
};
Q_UNUSED(pixfmt_to_desktop);
const fmt_entry *pixfmt_gl_entry = pixfmt_to_desktop;
if (OpenGLHelper::isOpenGLES())
pixfmt_gl_entry = pixfmt_to_gles;
// Very special formats, for which OpenGL happens to have direct support
static const fmt_entry pixfmt_gl_base[] = {
{VideoFormat::Format_RGBA32, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE }, // only tested for osx, win, angle
{VideoFormat::Format_RGB24, GL_RGB, GL_RGB, GL_UNSIGNED_BYTE },
{VideoFormat::Format_RGB565, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT_5_6_5},
{VideoFormat::Format_BGR32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE }, //rgba(tested) or abgr, depending on endian
};
const VideoFormat::PixelFormat pixfmt = fmt.pixelFormat();
// can not use array size because pixfmt_gl_entry is set on runtime
for (const fmt_entry* e = pixfmt_gl_entry; e->pixfmt != VideoFormat::Format_Invalid; ++e) {
if (e->pixfmt == pixfmt) {
*internal_format = e->internal_format;
*data_format = e->format;
*data_type = e->type;
if (mat)
*mat = QMatrix4x4();
return true;
}
}
for (size_t i = 0; i < ARRAY_SIZE(pixfmt_gl_base); ++i) {
const fmt_entry& e = pixfmt_gl_base[i];
if (e.pixfmt == pixfmt) {
*internal_format = e.internal_format;
*data_format = e.format;
*data_type = e.type;
if (mat)
*mat = QMatrix4x4();
return true;
}
}
static const fmt_entry pixfmt_to_gl_swizzele[] = {
{VideoFormat::Format_UYVY, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_YUYV, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_VYUY, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_YVYU, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_BGR565, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT_5_6_5}, //swizzle
{VideoFormat::Format_RGB555, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT_5_5_5_1}, //not working
{VideoFormat::Format_BGR555, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT_5_5_5_1}, //not working
};
for (size_t i = 0; i < ARRAY_SIZE(pixfmt_to_gl_swizzele); ++i) {
const fmt_entry& e = pixfmt_to_gl_swizzele[i];
if (e.pixfmt == pixfmt) {
*internal_format = e.internal_format;
*data_format = e.format;
*data_type = e.type;
if (mat)
*mat = channelMap(fmt);
return true;
}
}
GLint *i_f = internal_format;
GLenum *d_f = data_format;
GLenum *d_t = data_type;
gl_param_t* gp = (gl_param_t*)get_gl_param();
if (gp == gl_param_desktop && (
fmt.planeCount() == 2 // nv12 UV plane is 16bit, but we use rg
|| (OpenGLHelper::depth16BitTexture() == 16 && OpenGLHelper::has16BitTexture() && fmt.isBigEndian() && fmt.bitsPerComponent() > 8) // 16bit texture does not support be channel now
)) {
gp = (gl_param_t*)gl_param_desktop_fallback;
qDebug("desktop_fallback for %s", fmt.planeCount() == 2 ? "bi-plane format" : "16bit big endian channel");
}
for (int p = 0; p < fmt.planeCount(); ++p) {
// for packed rgb(swizzle required) and planar formats
const int c = (fmt.channels(p)-1) + 4*((fmt.bitsPerComponent() + 7)/8 - 1);
if (gp[c].format == 0)
return false;
const gl_param_t& f = gp[c];
*(i_f++) = f.internal_format;
*(d_f++) = f.format;
*(d_t++) = f.type;
}
if (mat)
*mat = channelMap(fmt);
return true;
}
bool PipelineManager::validate_pipeline ()
{
// check if pipeline is valid
if (source.get() == nullptr || sink.get() == nullptr)
{
return false;
}
// check source format
auto in_format = source->getVideoFormat();
if (in_format != this->input_format)
{
tcam_log(TCAM_LOG_DEBUG,
"Video format in source does not match pipeline: '%s' != '%s'",
in_format.to_string().c_str(),
input_format.to_string().c_str());
return false;
}
else
{
tcam_log(TCAM_LOG_DEBUG,
"Starting pipeline with format: '%s'",
in_format.to_string().c_str());
}
VideoFormat in;
VideoFormat out;
for (auto f : filter_pipeline)
{
f->getVideoFormat(in, out);
if (in != in_format)
{
tcam_log(TCAM_LOG_ERROR,
"Ingoing video format for filter %s is not compatible with previous element. '%s' != '%s'",
f->getDescription().name.c_str(),
in_format.to_string().c_str(),
in.to_string().c_str());
return false;
}
else
{
tcam_log(TCAM_LOG_DEBUG, "Filter %s connected to pipeline -- %s",
f->getDescription().name.c_str(),
out.to_string().c_str());
// save output for next comparison
in_format = out;
}
}
if (in_format != this->output_format)
{
tcam_log(TCAM_LOG_ERROR, "Video format in sink does not match pipeline '%s' != '%s'",
in_format.to_string().c_str(),
output_format.to_string().c_str());
return false;
}
return true;
}
void ImageConverter::setOutFormat(const VideoFormat& format)
{
setOutFormat(format.pixelFormatFFmpeg());
}
bool convertTo(const VideoFormat& fmt) {
return convertTo(fmt.pixelFormatFFmpeg());
}
VideoFrame VideoFrameConverter::convert(const VideoFrame& frame, const VideoFormat &fmt) const
{
return convert(frame, fmt.pixelFormatFFmpeg());
}
