void GLWidgetRendererPrivate::uploadPlane(int p, GLint internalFormat, GLenum format, const QRect& roi)
{
// FIXME: why happens on win?
if (video_frame.bytesPerLine(p) <= 0)
return;
glActiveTexture(GL_TEXTURE0 + p); //xbmc: only for es, not for desktop?
glBindTexture(GL_TEXTURE_2D, textures[p]);
////nv12: 2
//glPixelStorei(GL_UNPACK_ALIGNMENT, 1);//GetAlign(video_frame.bytesPerLine(p)));
#if defined(GL_UNPACK_ROW_LENGTH)
// glPixelStorei(GL_UNPACK_ROW_LENGTH, video_frame.bytesPerLine(p));
#endif
setupQuality();
//qDebug("bpl[%d]=%d width=%d", p, video_frame.bytesPerLine(p), video_frame.planeWidth(p));
// This is necessary for non-power-of-two textures
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//uploading part of image eats less gpu memory, but may be more cpu(gles)
//FIXME: more cpu usage then qpainter. FBO, VBO?
//roi for planes?
if (ROI_TEXCOORDS || roi.size() == video_frame.size()) {
glTexSubImage2D(GL_TEXTURE_2D
, 0 //level
, 0 // xoffset
, 0 // yoffset
, texture_size[p].width()
, texture_size[p].height()
, format //format, must the same as internal format?
, data_type[p]
, video_frame.bits(p));
} else {
int roi_x = roi.x();
int roi_y = roi.y();
int roi_w = roi.width();
int roi_h = roi.height();
int plane_w = video_frame.planeWidth(p);
VideoFormat fmt = video_frame.format();
if (p == 0) {
plane0Size = QSize(roi_w, roi_h);
} else {
roi_x = fmt.chromaWidth(roi_x);
roi_y = fmt.chromaHeight(roi_y);
roi_w = fmt.chromaWidth(roi_w);
roi_h = fmt.chromaHeight(roi_h);
}
qDebug("roi: %d, %d %dx%d", roi_x, roi_y, roi_w, roi_h);
#if 0// defined(GL_UNPACK_ROW_LENGTH) && defined(GL_UNPACK_SKIP_PIXELS)
// http://stackoverflow.com/questions/205522/opengl-subtexturing
glPixelStorei(GL_UNPACK_ROW_LENGTH, plane_w);
//glPixelStorei or compute pointer
glPixelStorei(GL_UNPACK_SKIP_PIXELS, roi_x);
glPixelStorei(GL_UNPACK_SKIP_ROWS, roi_y);
glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, roi_w, roi_h, 0, format, GL_UNSIGNED_BYTE, video_frame.bits(p));
//glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, roi_w, roi_h, format, GL_UNSIGNED_BYTE, video_frame.bits(p));
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
glPixelStorei(GL_UNPACK_SKIP_PIXELS, 0);
glPixelStorei(GL_UNPACK_SKIP_ROWS, 0);
#else // GL ES
//define it? or any efficient way?
//glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, roi_w, roi_h, 0, format, GL_UNSIGNED_BYTE, NULL);
const char *src = (char*)video_frame.bits(p) + roi_y*plane_w + roi_x*fmt.bytesPerPixel(p);
#define UPLOAD_LINE 1
#if UPLOAD_LINE
for (int y = 0; y < roi_h; y++) {
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, y, roi_w, 1, format, GL_UNSIGNED_BYTE, src);
}
#else
int line_size = roi_w*fmt.bytesPerPixel(p);
char *sub = (char*)malloc(roi_h*line_size);
char *dst = sub;
for (int y = 0; y < roi_h; y++) {
memcpy(dst, src, line_size);
src += video_frame.bytesPerLine(p);
dst += line_size;
}
// FIXME: crash
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, roi_w, roi_h, format, GL_UNSIGNED_BYTE, sub);
free(sub);
#endif //UPLOAD_LINE
#endif //GL_UNPACK_ROW_LENGTH
}
#if defined(GL_UNPACK_ROW_LENGTH)
// glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
#endif
//glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
glBindTexture(GL_TEXTURE_2D, 0);
}
bool GLWidgetRendererPrivate::prepareShaderProgram(const VideoFormat &fmt)
{
// isSupported(pixfmt)
if (!fmt.isValid())
return false;
releaseShaderProgram();
video_format.setPixelFormatFFmpeg(fmt.pixelFormatFFmpeg());
// TODO: only to kinds, packed.glsl, planar.glsl
QString frag;
if (fmt.isPlanar()) {
frag = getShaderFromFile("shaders/yuv_rgb.f.glsl");
} else {
frag = getShaderFromFile("shaders/rgb.f.glsl");
}
if (frag.isEmpty())
return false;
if (!fmt.isRGB() && fmt.isPlanar() && fmt.bytesPerPixel(0) == 2) {
if (fmt.isBigEndian())
frag.prepend("#define YUV16BITS_BE_LUMINANCE_ALPHA\n");
else
frag.prepend("#define YUV16BITS_LE_LUMINANCE_ALPHA\n");
frag.prepend(QString("#define YUV%1P\n").arg(fmt.bitsPerPixel(0)));
}
#if NO_QGL_SHADER
program = createProgram(kVertexShader, frag.toUtf8().constData());
if (!program) {
qWarning("Could not create shader program.");
return false;
}
// vertex shader
a_Position = glGetAttribLocation(program, "a_Position");
a_TexCoords = glGetAttribLocation(program, "a_TexCoords");
u_matrix = glGetUniformLocation(program, "u_MVP_matrix");
// fragment shader
u_colorMatrix = glGetUniformLocation(program, "u_colorMatrix");
#else
if (!shader_program->addShaderFromSourceCode(QGLShader::Vertex, kVertexShader)) {
qWarning("Failed to add vertex shader: %s", shader_program->log().toUtf8().constData());
return false;
}
if (!shader_program->addShaderFromSourceCode(QGLShader::Fragment, frag)) {
qWarning("Failed to add fragment shader: %s", shader_program->log().toUtf8().constData());
return false;
}
if (!shader_program->link()) {
qWarning("Failed to link shader program...%s", shader_program->log().toUtf8().constData());
return false;
}
// vertex shader
a_Position = shader_program->attributeLocation("a_Position");
a_TexCoords = shader_program->attributeLocation("a_TexCoords");
u_matrix = shader_program->uniformLocation("u_MVP_matrix");
// fragment shader
u_colorMatrix = shader_program->uniformLocation("u_colorMatrix");
#endif //NO_QGL_SHADER
qDebug("glGetAttribLocation(\"a_Position\") = %d\n", a_Position);
qDebug("glGetAttribLocation(\"a_TexCoords\") = %d\n", a_TexCoords);
qDebug("glGetUniformLocation(\"u_MVP_matrix\") = %d\n", u_matrix);
qDebug("glGetUniformLocation(\"u_colorMatrix\") = %d\n", u_colorMatrix);
if (fmt.isRGB())
u_Texture.resize(1);
else
u_Texture.resize(fmt.channels());
for (int i = 0; i < u_Texture.size(); ++i) {
QString tex_var = QString("u_Texture%1").arg(i);
#if NO_QGL_SHADER
u_Texture[i] = glGetUniformLocation(program, tex_var.toUtf8().constData());
#else
u_Texture[i] = shader_program->uniformLocation(tex_var);
#endif
qDebug("glGetUniformLocation(\"%s\") = %d\n", tex_var.toUtf8().constData(), u_Texture[i]);
}
return true;
}
bool GLWidgetRendererPrivate::initTextures(const VideoFormat &fmt)
{
// isSupported(pixfmt)
if (!fmt.isValid())
return false;
video_format.setPixelFormatFFmpeg(fmt.pixelFormatFFmpeg());
//http://www.berkelium.com/OpenGL/GDC99/internalformat.html
//NV12: UV is 1 plane. 16 bits as a unit. GL_LUMINANCE4, 8, 16, ... 32?
//GL_LUMINANCE, GL_LUMINANCE_ALPHA are deprecated in GL3, removed in GL3.1
//replaced by GL_RED, GL_RG, GL_RGB, GL_RGBA? for 1, 2, 3, 4 channel image
//http://www.gamedev.net/topic/634850-do-luminance-textures-still-exist-to-opengl/
//https://github.com/kivy/kivy/issues/1738: GL_LUMINANCE does work on a Galaxy Tab 2. LUMINANCE_ALPHA very slow on Linux
//ALPHA: vec4(1,1,1,A), LUMINANCE: (L,L,L,1), LUMINANCE_ALPHA: (L,L,L,A)
/*
* To support both planar and packed use GL_ALPHA and in shader use r,g,a like xbmc does.
* or use Swizzle_mask to layout the channels: http://www.opengl.org/wiki/Texture#Swizzle_mask
* GL ES2 support: GL_RGB, GL_RGBA, GL_LUMINANCE, GL_LUMINANCE_ALPHA, GL_ALPHA
* http://stackoverflow.com/questions/18688057/which-opengl-es-2-0-texture-formats-are-color-depth-or-stencil-renderable
*/
internal_format = QVector<GLint>(fmt.planeCount(), FMT_INTERNAL);
data_format = QVector<GLenum>(fmt.planeCount(), FMT);
data_type = QVector<GLenum>(fmt.planeCount(), GL_UNSIGNED_BYTE);
if (fmt.isPlanar()) {
/*!
* GLES internal_format == data_format, GL_LUMINANCE_ALPHA is 2 bytes
* so if NV12 use GL_LUMINANCE_ALPHA, YV12 use GL_ALPHA
*/
qDebug("///////////bpp %d", fmt.bytesPerPixel());
internal_format[0] = data_format[0] = GL_LUMINANCE; //or GL_RED for GL
if (fmt.planeCount() == 2) {
internal_format[1] = data_format[1] = GL_LUMINANCE_ALPHA;
} else {
if (fmt.bytesPerPixel(1) == 2) {
// read 16 bits and compute the real luminance in shader
internal_format[0] = data_format[0] = GL_LUMINANCE_ALPHA;
internal_format[1] = data_format[1] = GL_LUMINANCE_ALPHA; //vec4(L,L,L,A)
internal_format[2] = data_format[2] = GL_LUMINANCE_ALPHA;
} else {
internal_format[1] = data_format[1] = GL_LUMINANCE; //vec4(L,L,L,1)
internal_format[2] = data_format[2] = GL_ALPHA;//GL_ALPHA;
}
}
for (int i = 0; i < internal_format.size(); ++i) {
// xbmc use bpp not bpp(plane)
//internal_format[i] = GetGLInternalFormat(data_format[i], fmt.bytesPerPixel(i));
//data_format[i] = internal_format[i];
}
} else {
//glPixelStorei(GL_UNPACK_ALIGNMENT, fmt.bytesPerPixel());
// TODO: if no alpha, data_fmt is not GL_BGRA. align at every upload?
}
for (int i = 0; i < fmt.planeCount(); ++i) {
//qDebug("format: %#x GL_LUMINANCE_ALPHA=%#x", data_format[i], GL_LUMINANCE_ALPHA);
if (fmt.bytesPerPixel(i) == 2 && fmt.planeCount() == 3) {
//data_type[i] = GL_UNSIGNED_SHORT;
}
int bpp_gl = bytesOfGLFormat(data_format[i], data_type[i]);
int pad = qCeil((qreal)(texture_size[i].width() - effective_tex_width[i])/(qreal)bpp_gl);
texture_size[i].setWidth(qCeil((qreal)texture_size[i].width()/(qreal)bpp_gl));
effective_tex_width[i] /= bpp_gl; //fmt.bytesPerPixel(i);
//effective_tex_width_ratio =
qDebug("texture width: %d - %d = pad: %d. bpp(gl): %d", texture_size[i].width(), effective_tex_width[i], pad, bpp_gl);
}
/*
* there are 2 fragment shaders: rgb and yuv.
* only 1 texture for packed rgb. planar rgb likes yuv
* To support both planar and packed yuv, and mixed yuv(NV12), we give a texture sample
* for each channel. For packed, each (channel) texture sample is the same. For planar,
* packed channels has the same texture sample.
* But the number of actural textures we upload is plane count.
* Which means the number of texture id equals to plane count
*/
if (textures.size() != fmt.planeCount()) {
glDeleteTextures(textures.size(), textures.data());
qDebug("delete %d textures", textures.size());
textures.clear();
textures.resize(fmt.planeCount());
glGenTextures(textures.size(), textures.data());
}
if (!hasGLSL) {
initTexture(textures[0], internal_format[0], data_format[0], data_type[0], texture_size[0].width(), texture_size[0].height());
// more than 1?
qWarning("Does not support GLSL!");
return false;
}
qDebug("init textures...");
initTexture(textures[0], internal_format[0], data_format[0], data_type[0], texture_size[0].width(), texture_size[0].height());
for (int i = 1; i < textures.size(); ++i) {
initTexture(textures[i], internal_format[i], data_format[i], data_type[i], texture_size[i].width(), texture_size[i].height());
}
return true;
}
bool videoFormatToGL(const VideoFormat& fmt, GLint* internal_format, GLenum* data_format, GLenum* data_type, QMatrix4x4* mat)
{
typedef struct fmt_entry {
VideoFormat::PixelFormat pixfmt;
GLint internal_format;
GLenum format;
GLenum type;
} fmt_entry;
static const fmt_entry pixfmt_to_gles[] = {
{VideoFormat::Format_BGRA32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE }, //tested for angle
{VideoFormat::Format_RGB32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_Invalid, 0, 0, 0}
};
Q_UNUSED(pixfmt_to_gles);
static const fmt_entry pixfmt_to_desktop[] = {
{VideoFormat::Format_BGRA32, GL_RGBA, GL_BGRA, GL_UNSIGNED_BYTE }, //bgra bgra works on win but not osx
{VideoFormat::Format_RGB32, GL_RGBA, GL_BGRA, GL_UNSIGNED_BYTE }, //FIXMEL endian check
//{VideoFormat::Format_BGRA32, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE }, //{2,1,0,3}
//{VideoFormat::Format_BGR24, GL_RGB, GL_BGR, GL_UNSIGNED_BYTE }, //{0,1,2,3}
#ifdef GL_UNSIGNED_SHORT_5_6_5_REV
{VideoFormat::Format_BGR565, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT_5_6_5_REV}, // es error, use channel map
#endif
#ifdef GL_UNSIGNED_SHORT_1_5_5_5_REV
{VideoFormat::Format_RGB555, GL_RGBA, GL_BGRA, GL_UNSIGNED_SHORT_1_5_5_5_REV},
#endif
#ifdef GL_UNSIGNED_SHORT_1_5_5_5_REV
{VideoFormat::Format_BGR555, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT_1_5_5_5_REV},
#endif
// TODO: BE formats not implemeted
{VideoFormat::Format_RGB48, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT }, //TODO: they are not work for ANGLE, and rgb16 works on desktop gl, so remove these lines to use rgb16?
{VideoFormat::Format_RGB48LE, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT },
{VideoFormat::Format_RGB48BE, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT },
{VideoFormat::Format_BGR48, GL_RGB, GL_BGR, GL_UNSIGNED_SHORT }, //RGB16?
{VideoFormat::Format_BGR48LE, GL_RGB, GL_BGR, GL_UNSIGNED_SHORT },
{VideoFormat::Format_BGR48BE, GL_RGB, GL_BGR, GL_UNSIGNED_SHORT },
{VideoFormat::Format_RGBA64LE, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT },
{VideoFormat::Format_RGBA64BE, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT },
{VideoFormat::Format_BGRA64LE, GL_RGBA, GL_BGRA, GL_UNSIGNED_SHORT },
{VideoFormat::Format_BGRA64BE, GL_RGBA, GL_BGRA, GL_UNSIGNED_SHORT },
{VideoFormat::Format_Invalid, 0, 0, 0}
};
Q_UNUSED(pixfmt_to_desktop);
const fmt_entry *pixfmt_gl_entry = pixfmt_to_desktop;
if (OpenGLHelper::isOpenGLES())
pixfmt_gl_entry = pixfmt_to_gles;
// Very special formats, for which OpenGL happens to have direct support
static const fmt_entry pixfmt_gl_base[] = {
{VideoFormat::Format_RGBA32, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE }, // only tested for osx, win, angle
{VideoFormat::Format_RGB24, GL_RGB, GL_RGB, GL_UNSIGNED_BYTE },
{VideoFormat::Format_RGB565, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT_5_6_5},
{VideoFormat::Format_BGR32, GL_BGRA, GL_BGRA, GL_UNSIGNED_BYTE }, //rgba(tested) or abgr, depending on endian
};
const VideoFormat::PixelFormat pixfmt = fmt.pixelFormat();
// can not use array size because pixfmt_gl_entry is set on runtime
for (const fmt_entry* e = pixfmt_gl_entry; e->pixfmt != VideoFormat::Format_Invalid; ++e) {
if (e->pixfmt == pixfmt) {
*internal_format = e->internal_format;
*data_format = e->format;
*data_type = e->type;
if (mat)
*mat = QMatrix4x4();
return true;
}
}
for (size_t i = 0; i < ARRAY_SIZE(pixfmt_gl_base); ++i) {
const fmt_entry& e = pixfmt_gl_base[i];
if (e.pixfmt == pixfmt) {
*internal_format = e.internal_format;
*data_format = e.format;
*data_type = e.type;
if (mat)
*mat = QMatrix4x4();
return true;
}
}
static const fmt_entry pixfmt_to_gl_swizzele[] = {
{VideoFormat::Format_VYU, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_UYVY, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_YUYV, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_VYUY, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_YVYU, GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE },
{VideoFormat::Format_BGR565, GL_RGB, GL_RGB, GL_UNSIGNED_SHORT_5_6_5}, //swizzle
{VideoFormat::Format_RGB555, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT_5_5_5_1}, //not working
{VideoFormat::Format_BGR555, GL_RGBA, GL_RGBA, GL_UNSIGNED_SHORT_5_5_5_1}, //not working
};
for (size_t i = 0; i < ARRAY_SIZE(pixfmt_to_gl_swizzele); ++i) {
const fmt_entry& e = pixfmt_to_gl_swizzele[i];
if (e.pixfmt == pixfmt) {
*internal_format = e.internal_format;
*data_format = e.format;
*data_type = e.type;
if (mat)
*mat = channelMap(fmt);
return true;
}
}
GLint *i_f = internal_format;
GLenum *d_f = data_format;
GLenum *d_t = data_type;
gl_param_t* gp = (gl_param_t*)get_gl_param();
const int nb_planes = fmt.planeCount();
if (gp == gl_param_desktop && (
nb_planes == 2 // nv12 UV plane is 16bit, but we use rg
|| (OpenGLHelper::depth16BitTexture() == 16 && OpenGLHelper::has16BitTexture() && fmt.isBigEndian() && fmt.bitsPerComponent() > 8) // 16bit texture does not support be channel now
)) {
gp = (gl_param_t*)gl_param_desktop_fallback;
qDebug("desktop_fallback for %s", nb_planes == 2 ? "bi-plane format" : "16bit big endian channel");
}
for (int p = 0; p < nb_planes; ++p) {
// for packed rgb(swizzle required) and planar formats
const int c = (fmt.channels(p)-1) + 4*((fmt.bitsPerComponent() + 7)/8 - 1);
if (gp[c].format == 0)
return false;
const gl_param_t& f = gp[c];
*(i_f++) = f.internal_format;
*(d_f++) = f.format;
*(d_t++) = f.type;
}
if (nb_planes > 2 && data_format[2] == GL_LUMINANCE && fmt.bytesPerPixel(1) == 1) { // QtAV uses the same shader for planar and semi-planar yuv format
internal_format[2] = data_format[2] = GL_ALPHA;
if (nb_planes == 4)
internal_format[3] = data_format[3] = data_format[2]; // vec4(,,,A)
}
if (mat)
*mat = channelMap(fmt);
return true;
}