VideoFrame VideoDecoderDXVA::frame()
{
DPTR_D(VideoDecoderDXVA);
//qDebug("frame size: %dx%d", d.frame->width, d.frame->height);
if (!d.frame->opaque || !d.frame->data[0])
return VideoFrame();
if (d.frame->width <= 0 || d.frame->height <= 0 || !d.codec_ctx)
return VideoFrame();
IDirect3DSurface9 *d3d = (IDirect3DSurface9*)(uintptr_t)d.frame->data[3];
if (copyMode() == ZeroCopy && d.interop_res) {
dxva::SurfaceInteropDXVA *interop = new dxva::SurfaceInteropDXVA(d.interop_res);
interop->setSurface(d3d, width(), height());
VideoFrame f(width(), height(), VideoFormat::Format_RGB32); //p->width()
f.setBytesPerLine(d.width * 4); //used by gl to compute texture size
f.setMetaData(QStringLiteral("surface_interop"), QVariant::fromValue(VideoSurfaceInteropPtr(interop)));
f.setTimestamp(d.frame->pkt_pts/1000.0);
f.setDisplayAspectRatio(d.getDAR(d.frame));
return f;
}
class ScopedD3DLock {
IDirect3DSurface9 *mpD3D;
public:
ScopedD3DLock(IDirect3DSurface9* d3d, D3DLOCKED_RECT *rect) : mpD3D(d3d) {
if (FAILED(mpD3D->LockRect(rect, NULL, D3DLOCK_READONLY))) {
qWarning("Failed to lock surface");
mpD3D = 0;
}
}
~ScopedD3DLock() {
if (mpD3D)
mpD3D->UnlockRect();
}
};
D3DLOCKED_RECT lock;
ScopedD3DLock(d3d, &lock);
if (lock.Pitch == 0) {
return VideoFrame();
}
//picth >= desc.Width
D3DSURFACE_DESC desc;
d3d->GetDesc(&desc);
const VideoFormat fmt = VideoFormat(pixelFormatFromD3D(desc.Format));
if (!fmt.isValid()) {
qWarning("unsupported dxva pixel format: %#x", desc.Format);
return VideoFrame();
}
//YV12 need swap, not imc3?
// imc3 U V pitch == Y pitch, but half of the U/V plane is space. we convert to yuv420p here
// nv12 bpp(1)==1
// 3rd plane is not used for nv12
int pitch[3] = { lock.Pitch, 0, 0}; //compute chroma later
uint8_t *src[] = { (uint8_t*)lock.pBits, 0, 0}; //compute chroma later
const bool swap_uv = desc.Format == MAKEFOURCC('I','M','C','3');
return copyToFrame(fmt, d.surface_height, src, pitch, swap_uv);
}
VideoFrame VideoDecoderDXVA::frame()
{
DPTR_D(VideoDecoderDXVA);
if (!d.frame->opaque || !d.frame->data[0])
return VideoFrame();
if (d.width <= 0 || d.height <= 0 || !d.codec_ctx)
return VideoFrame();
class ScopedD3DLock {
public:
ScopedD3DLock(IDirect3DSurface9* d3d, D3DLOCKED_RECT *rect)
: mpD3D(d3d)
{
if (FAILED(mpD3D->LockRect(rect, NULL, D3DLOCK_READONLY))) {
qWarning("Failed to lock surface");
mpD3D = 0;
}
}
~ScopedD3DLock() {
if (mpD3D)
mpD3D->UnlockRect();
}
private:
IDirect3DSurface9 *mpD3D;
};
IDirect3DSurface9 *d3d = (IDirect3DSurface9*)(uintptr_t)d.frame->data[3];
//picth >= desc.Width
//D3DSURFACE_DESC desc;
//d3d->GetDesc(&desc);
D3DLOCKED_RECT lock;
ScopedD3DLock(d3d, &lock);
if (lock.Pitch == 0) {
return VideoFrame();
}
const VideoFormat fmt = VideoFormat((int)D3dFindFormat(d.render)->avpixfmt);
if (!fmt.isValid()) {
qWarning("unsupported dxva pixel format: %#x", d.render);
return VideoFrame();
}
//YV12 need swap, not imc3?
// imc3 U V pitch == Y pitch, but half of the U/V plane is space. we convert to yuv420p here
// nv12 bpp(1)==1
// 3rd plane is not used for nv12
int pitch[3] = { lock.Pitch, 0, 0}; //compute chroma later
uint8_t *src[] = { (uint8_t*)lock.pBits, 0, 0}; //compute chroma later
const bool swap_uv = d.render == MAKEFOURCC('I','M','C','3');
return copyToFrame(fmt, d.surface_height, src, pitch, swap_uv);
}
void* SurfaceInteropDXVA::mapToHost(const VideoFormat &format, void *handle, int plane)
{
Q_UNUSED(plane);
class ScopedD3DLock {
IDirect3DSurface9 *mpD3D;
public:
ScopedD3DLock(IDirect3DSurface9* d3d, D3DLOCKED_RECT *rect) : mpD3D(d3d) {
if (FAILED(mpD3D->LockRect(rect, NULL, D3DLOCK_READONLY))) {
qWarning("Failed to lock surface");
mpD3D = 0;
}
}
~ScopedD3DLock() {
if (mpD3D)
mpD3D->UnlockRect();
}
};
D3DLOCKED_RECT lock;
ScopedD3DLock(m_surface, &lock);
if (lock.Pitch == 0)
return NULL;
//picth >= desc.Width
D3DSURFACE_DESC desc;
m_surface->GetDesc(&desc);
const VideoFormat fmt = VideoFormat(pixelFormatFromFourcc(desc.Format));
if (!fmt.isValid()) {
qWarning("unsupported dxva pixel format: %#x", desc.Format);
return NULL;
}
//YV12 need swap, not imc3?
// imc3 U V pitch == Y pitch, but half of the U/V plane is space. we convert to yuv420p here
// nv12 bpp(1)==1
// 3rd plane is not used for nv12
int pitch[3] = { lock.Pitch, 0, 0}; //compute chroma later
quint8 *src[] = { (quint8*)lock.pBits, 0, 0}; //compute chroma later
Q_ASSERT(src[0] && pitch[0] > 0);
const bool swap_uv = desc.Format == MAKEFOURCC('I','M','C','3');
// try to use SSE. fallback to normal copy if SSE is not supported
VideoFrame frame(VideoFrame::fromGPU(fmt, frame_width, frame_height, desc.Height, src, pitch, true, swap_uv));
// TODO: check rgb32 because d3d can use hw to convert
if (format != fmt)
frame = frame.to(format);
VideoFrame *f = reinterpret_cast<VideoFrame*>(handle);
frame.setTimestamp(f->timestamp());
*f = frame;
return f;
}
void QPainterFilterContext::initializeOnFrame(VideoFrame *vframe)
{
if (!vframe) {
if (!painter) {
painter = new QPainter(); //warning: more than 1 painter on 1 device
}
if (!paint_device) {
paint_device = painter->device();
}
if (!paint_device && !painter->isActive()) {
qWarning("No paint device and painter is not active. No painting!");
return;
}
if (!painter->isActive())
painter->begin(paint_device);
return;
}
VideoFormat format = vframe->format();
if (!format.isValid()) {
qWarning("Not a valid format");
return;
}
if (format.imageFormat() == QImage::Format_Invalid) {
format.setPixelFormat(VideoFormat::Format_RGB32);
if (!cvt) {
cvt = new VideoFrameConverter();
}
*vframe = cvt->convert(*vframe, format);
}
if (paint_device) {
if (painter && painter->isActive()) {
painter->end(); //destroy a paint device that is being painted is not allowed!
}
delete paint_device;
paint_device = 0;
}
Q_ASSERT(video_width > 0 && video_height > 0);
// direct draw on frame data, so use VideoFrame::constBits()
paint_device = new QImage((uchar*)vframe->constBits(0), video_width, video_height, vframe->bytesPerLine(0), format.imageFormat());
if (!painter)
painter = new QPainter();
own_painter = true;
own_paint_device = true; //TODO: what about renderer is not a widget?
painter->begin((QImage*)paint_device);
}
bool GLWidgetRendererPrivate::initTextures(const VideoFormat &fmt)
{
// isSupported(pixfmt)
if (!fmt.isValid())
return false;
video_format.setPixelFormatFFmpeg(fmt.pixelFormatFFmpeg());
//http://www.berkelium.com/OpenGL/GDC99/internalformat.html
//NV12: UV is 1 plane. 16 bits as a unit. GL_LUMINANCE4, 8, 16, ... 32?
//GL_LUMINANCE, GL_LUMINANCE_ALPHA are deprecated in GL3, removed in GL3.1
//replaced by GL_RED, GL_RG, GL_RGB, GL_RGBA? for 1, 2, 3, 4 channel image
//http://www.gamedev.net/topic/634850-do-luminance-textures-still-exist-to-opengl/
//https://github.com/kivy/kivy/issues/1738: GL_LUMINANCE does work on a Galaxy Tab 2. LUMINANCE_ALPHA very slow on Linux
//ALPHA: vec4(1,1,1,A), LUMINANCE: (L,L,L,1), LUMINANCE_ALPHA: (L,L,L,A)
/*
* To support both planar and packed use GL_ALPHA and in shader use r,g,a like xbmc does.
* or use Swizzle_mask to layout the channels: http://www.opengl.org/wiki/Texture#Swizzle_mask
* GL ES2 support: GL_RGB, GL_RGBA, GL_LUMINANCE, GL_LUMINANCE_ALPHA, GL_ALPHA
* http://stackoverflow.com/questions/18688057/which-opengl-es-2-0-texture-formats-are-color-depth-or-stencil-renderable
*/
internal_format = QVector<GLint>(fmt.planeCount(), FMT_INTERNAL);
data_format = QVector<GLenum>(fmt.planeCount(), FMT);
data_type = QVector<GLenum>(fmt.planeCount(), GL_UNSIGNED_BYTE);
if (fmt.isPlanar()) {
/*!
* GLES internal_format == data_format, GL_LUMINANCE_ALPHA is 2 bytes
* so if NV12 use GL_LUMINANCE_ALPHA, YV12 use GL_ALPHA
*/
qDebug("///////////bpp %d", fmt.bytesPerPixel());
internal_format[0] = data_format[0] = GL_LUMINANCE; //or GL_RED for GL
if (fmt.planeCount() == 2) {
internal_format[1] = data_format[1] = GL_LUMINANCE_ALPHA;
} else {
if (fmt.bytesPerPixel(1) == 2) {
// read 16 bits and compute the real luminance in shader
internal_format[0] = data_format[0] = GL_LUMINANCE_ALPHA;
internal_format[1] = data_format[1] = GL_LUMINANCE_ALPHA; //vec4(L,L,L,A)
internal_format[2] = data_format[2] = GL_LUMINANCE_ALPHA;
} else {
internal_format[1] = data_format[1] = GL_LUMINANCE; //vec4(L,L,L,1)
internal_format[2] = data_format[2] = GL_ALPHA;//GL_ALPHA;
}
}
for (int i = 0; i < internal_format.size(); ++i) {
// xbmc use bpp not bpp(plane)
//internal_format[i] = GetGLInternalFormat(data_format[i], fmt.bytesPerPixel(i));
//data_format[i] = internal_format[i];
}
} else {
//glPixelStorei(GL_UNPACK_ALIGNMENT, fmt.bytesPerPixel());
// TODO: if no alpha, data_fmt is not GL_BGRA. align at every upload?
}
for (int i = 0; i < fmt.planeCount(); ++i) {
//qDebug("format: %#x GL_LUMINANCE_ALPHA=%#x", data_format[i], GL_LUMINANCE_ALPHA);
if (fmt.bytesPerPixel(i) == 2 && fmt.planeCount() == 3) {
//data_type[i] = GL_UNSIGNED_SHORT;
}
int bpp_gl = bytesOfGLFormat(data_format[i], data_type[i]);
int pad = qCeil((qreal)(texture_size[i].width() - effective_tex_width[i])/(qreal)bpp_gl);
texture_size[i].setWidth(qCeil((qreal)texture_size[i].width()/(qreal)bpp_gl));
effective_tex_width[i] /= bpp_gl; //fmt.bytesPerPixel(i);
//effective_tex_width_ratio =
qDebug("texture width: %d - %d = pad: %d. bpp(gl): %d", texture_size[i].width(), effective_tex_width[i], pad, bpp_gl);
}
/*
* there are 2 fragment shaders: rgb and yuv.
* only 1 texture for packed rgb. planar rgb likes yuv
* To support both planar and packed yuv, and mixed yuv(NV12), we give a texture sample
* for each channel. For packed, each (channel) texture sample is the same. For planar,
* packed channels has the same texture sample.
* But the number of actural textures we upload is plane count.
* Which means the number of texture id equals to plane count
*/
if (textures.size() != fmt.planeCount()) {
glDeleteTextures(textures.size(), textures.data());
qDebug("delete %d textures", textures.size());
textures.clear();
textures.resize(fmt.planeCount());
glGenTextures(textures.size(), textures.data());
}
if (!hasGLSL) {
initTexture(textures[0], internal_format[0], data_format[0], data_type[0], texture_size[0].width(), texture_size[0].height());
// more than 1?
qWarning("Does not support GLSL!");
return false;
}
qDebug("init textures...");
initTexture(textures[0], internal_format[0], data_format[0], data_type[0], texture_size[0].width(), texture_size[0].height());
for (int i = 1; i < textures.size(); ++i) {
initTexture(textures[i], internal_format[i], data_format[i], data_type[i], texture_size[i].width(), texture_size[i].height());
}
return true;
}
bool GLWidgetRendererPrivate::prepareShaderProgram(const VideoFormat &fmt)
{
// isSupported(pixfmt)
if (!fmt.isValid())
return false;
releaseShaderProgram();
video_format.setPixelFormatFFmpeg(fmt.pixelFormatFFmpeg());
// TODO: only to kinds, packed.glsl, planar.glsl
QString frag;
if (fmt.isPlanar()) {
frag = getShaderFromFile("shaders/yuv_rgb.f.glsl");
} else {
frag = getShaderFromFile("shaders/rgb.f.glsl");
}
if (frag.isEmpty())
return false;
if (!fmt.isRGB() && fmt.isPlanar() && fmt.bytesPerPixel(0) == 2) {
if (fmt.isBigEndian())
frag.prepend("#define YUV16BITS_BE_LUMINANCE_ALPHA\n");
else
frag.prepend("#define YUV16BITS_LE_LUMINANCE_ALPHA\n");
frag.prepend(QString("#define YUV%1P\n").arg(fmt.bitsPerPixel(0)));
}
#if NO_QGL_SHADER
program = createProgram(kVertexShader, frag.toUtf8().constData());
if (!program) {
qWarning("Could not create shader program.");
return false;
}
// vertex shader
a_Position = glGetAttribLocation(program, "a_Position");
a_TexCoords = glGetAttribLocation(program, "a_TexCoords");
u_matrix = glGetUniformLocation(program, "u_MVP_matrix");
// fragment shader
u_colorMatrix = glGetUniformLocation(program, "u_colorMatrix");
#else
if (!shader_program->addShaderFromSourceCode(QGLShader::Vertex, kVertexShader)) {
qWarning("Failed to add vertex shader: %s", shader_program->log().toUtf8().constData());
return false;
}
if (!shader_program->addShaderFromSourceCode(QGLShader::Fragment, frag)) {
qWarning("Failed to add fragment shader: %s", shader_program->log().toUtf8().constData());
return false;
}
if (!shader_program->link()) {
qWarning("Failed to link shader program...%s", shader_program->log().toUtf8().constData());
return false;
}
// vertex shader
a_Position = shader_program->attributeLocation("a_Position");
a_TexCoords = shader_program->attributeLocation("a_TexCoords");
u_matrix = shader_program->uniformLocation("u_MVP_matrix");
// fragment shader
u_colorMatrix = shader_program->uniformLocation("u_colorMatrix");
#endif //NO_QGL_SHADER
qDebug("glGetAttribLocation(\"a_Position\") = %d\n", a_Position);
qDebug("glGetAttribLocation(\"a_TexCoords\") = %d\n", a_TexCoords);
qDebug("glGetUniformLocation(\"u_MVP_matrix\") = %d\n", u_matrix);
qDebug("glGetUniformLocation(\"u_colorMatrix\") = %d\n", u_colorMatrix);
if (fmt.isRGB())
u_Texture.resize(1);
else
u_Texture.resize(fmt.channels());
for (int i = 0; i < u_Texture.size(); ++i) {
QString tex_var = QString("u_Texture%1").arg(i);
#if NO_QGL_SHADER
u_Texture[i] = glGetUniformLocation(program, tex_var.toUtf8().constData());
#else
u_Texture[i] = shader_program->uniformLocation(tex_var);
#endif
qDebug("glGetUniformLocation(\"%s\") = %d\n", tex_var.toUtf8().constData(), u_Texture[i]);
}
return true;
}