Related
I’m struggling with rendering a simple overlay image on top of an already rendered frame. The frame is rendered by a mapbox library which I modified by calling my rendering code just after mapbox renders the frame. I don't know whether the fact that I’m rendering on top of an already rendered frame by mapbox library is important, but maybe it is, so I’m mentioning it here.
The problem seems somewhat related to texturing. The image I try to render is a checkerboard with red and transparent squares. The image bitmap does not have any edges. The image is visible but the edges are also coloured red which is not what I’ve intended and I've already spent a couple of days trying to figure out the reason why these edges are being rendered. I’ll be more than grateful if someone could enlighten me on what I’m doing wrong.
checkerboard with edges. Notice that one edge fades out at the top
namespace mbgl {
void MapOverlayRenderer::arm() { // called when rendering surface has changed
releaseLog("MOR a");
program = createProgram();
vertexBuffer = createVertexBuffer();
indexBuffer = createIndexBuffer();
glGenTextures(1, &texture);
fillTexture();
}
void MapOverlayRenderer::render() { // called just after mapbox frame has rendered
if (!_image.valid()) {
return;
}
glUseProgram(program);
auto positionHandle = glGetAttribLocation(program, "vertexData");
glBindBuffer(GL_ARRAY_BUFFER, vertexBuffer);
glEnableVertexAttribArray(positionHandle);
GLint numberOfValuesPerVertex = 4;
GLsizei stride = numberOfValuesPerVertex * sizeof(GLfloat);
glVertexAttribPointer(positionHandle, numberOfValuesPerVertex, GL_FLOAT, false, stride, nullptr);
auto imagePositionHandle = glGetUniformLocation(program, "imagePosition");
GLfloat imagePosition[] = {_leftPadding, _topPadding, _rightPadding, _bottomPadding};
glUniform4fv(imagePositionHandle, 1, imagePosition);
auto imageTextureHandle = glGetUniformLocation(program, "imageTexture");
glUniform1i(imageTextureHandle, 0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, indexBuffer);
auto numberOfVertices = 6;
glDrawElements(GL_TRIANGLES, numberOfVertices, GL_UNSIGNED_INT, nullptr);
glDisableVertexAttribArray(positionHandle);
}
void MapOverlayRenderer::setImage(float leftPadding, float topPadding, float rightPadding,
float bottomPadding, PremultipliedImage &&image) { // is called when I pass new image to be rendered
this->_image = std::move(image);
this->_leftPadding = leftPadding;
this->_topPadding = topPadding;
this->_rightPadding = rightPadding;
this->_bottomPadding = bottomPadding;
fillTexture();
}
void MapOverlayRenderer::removeImage() {
this->_leftPadding = 0.0f;
this->_topPadding = 0.0f;
this->_rightPadding = 0.0f;
this->_bottomPadding = 0.0f;
this->_image = PremultipliedImage();
}
GLuint MapOverlayRenderer::createVertexBuffer() {
GLuint result;
glGenBuffers(1, &result);
std::array<GLfloat, 16u> vertices{
-1.0f, -1.0f, 0.0f, 0.0f,
1.0f, -1.0f, 1.0f, 0.0f,
-1.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 1.0f, 1.0f
};
glBindBuffer(GL_ARRAY_BUFFER, result);
glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), &vertices, GL_STATIC_DRAW);
return result;
}
GLuint MapOverlayRenderer::createIndexBuffer() {
GLuint result;
glGenBuffers(1, &result);
std::array<GLuint, 6u> indices{
0u, 1u, 2u,
2u, 1u, 3u
};
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, result);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), &indices, GL_STATIC_DRAW);
return result;
}
GLuint MapOverlayRenderer::createVertexShader() {
auto vertexShaderCode = R"(
precision highp float;
precision highp int;
precision highp sampler2D;
attribute vec4 vertexData;
varying vec2 fullscreenTextureCoordinates;
void main() {
fullscreenTextureCoordinates = vertexData.zw;
gl_Position = vec4(vertexData.xy, 0.0, 1.0);
})";
auto shader = loadShader(GL_VERTEX_SHADER, &vertexShaderCode);
return shader;
}
GLuint MapOverlayRenderer::createFragmentShader() {
auto fragmentShaderCode = R"(
precision highp float;
precision highp int;
precision highp sampler2D;
varying vec2 fullscreenTextureCoordinates;
uniform vec4 imagePosition;
uniform sampler2D imageTexture;
void main() {
float left = imagePosition[0];
float top = imagePosition[1];
float right = imagePosition[2];
float bottom = imagePosition[3];
if (fullscreenTextureCoordinates[0] < left ||
fullscreenTextureCoordinates[0] > right ||
fullscreenTextureCoordinates[1] > 1.0 - top ||
fullscreenTextureCoordinates[1] < 1.0 - bottom) {
gl_FragColor = vec4(0.0, 0.0, 0.0, 0.0);
} else {
float imageHorizontalCoordinate = (fullscreenTextureCoordinates[0] - left) / (right - left);
float imageVerticalCoordinate = 1.0 - (fullscreenTextureCoordinates[1] - top) / (bottom - top);
vec2 imageTextureCoordinates = vec2(imageHorizontalCoordinate, imageVerticalCoordinate);
gl_FragColor = texture2D(imageTexture, imageTextureCoordinates);
}
})";
auto shader = loadShader(GL_FRAGMENT_SHADER, &fragmentShaderCode);
return shader;
}
GLuint MapOverlayRenderer::createProgram() {
auto vertexShader = createVertexShader();
auto fragmentShader = createFragmentShader();
auto result = glCreateProgram();
glAttachShader(result, vertexShader);
glAttachShader(result, fragmentShader);
glLinkProgram(result);
return result;
}
GLuint MapOverlayRenderer::loadShader(GLenum type, GLchar const *const *shaderCode) {
auto shader = glCreateShader(type);
glShaderSource(shader, 1, shaderCode, nullptr);
glCompileShader(shader);
return shader;
}
void MapOverlayRenderer::fillTexture() {
if (!_image.valid()) {
return;
}
glBindTexture(GL_TEXTURE_2D, texture);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, static_cast<GLsizei>(_image.size.width), static_cast<GLsizei>(_image.size.height), 0, GL_RGBA,
GL_UNSIGNED_BYTE, _image.data.get());
glGenerateMipmap(GL_TEXTURE_2D);
}
}
The default texture wrapping mode is set to wrap around to the other side of the texture. If any filtering is being done, this will end up sampling pixels from the opposite side of the texture, which explains the thin red border.
To change it to clamping, use this in fillTexture after binding the texture:
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
I am currently working on a rtsp player on android using ffmpeg to connect and decode the video stream. I would like to use OpenGL es 2.0 to convert the YUV frame to RGB frame and display it but i am blocked (it's the first time i use opengl).
I will try to explain clearly what is my problem.
From the NDK android i initialize an opengl context (from the thread i want to use to display images) using this method :
//
EGLint attribs[] = {
EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL_BLUE_SIZE, 8,
EGL_GREEN_SIZE, 8,
EGL_RED_SIZE, 8,
EGL_ALPHA_SIZE, 8,
EGL_NONE
};
EGLint contextAttrs[] = {
EGL_CONTEXT_CLIENT_VERSION, 2,
EGL_NONE
};
LOGI("Initializing context");
if((display = eglGetDisplay(EGL_DEFAULT_DISPLAY)) == EGL_NO_DISPLAY)
{
closeContext();
return;
}
if(!eglInitialize(display, 0, 0))
{
closeContext();
return;
}
if(!eglChooseConfig(display, attribs, &config, 1, &numConfigs))
{
closeContext();
return;
}
if(!eglGetConfigAttrib(display, config, EGL_NATIVE_VISUAL_ID, &format))
{
closeContext();
return;
}
ANativeWindow_setBuffersGeometry(window, 0, 0, format);
if(!(surface = eglCreateWindowSurface(display, config, window, 0)))
{
closeContext();
return;
}
if(!(context = eglCreateContext(display, config, 0, contextAttrs)))
{
closeContext();
return;
}
if(!eglMakeCurrent(display, surface, surface, context))
{
closeContext();
return;
}
if(!eglQuerySurface(display, surface, EGL_WIDTH, &width) || !eglQuerySurface(display, surface, EGL_HEIGHT, &height))
{
closeContext();
return;
}
LOGI("EGLWIDTH : %d EGLHEIGHT : %d ", (int)width, (int)height);
isInitEGLContext = 1;
Then i setup the graphics using this method :
//
//Load Vertex and Fragment Shader, attach shader and link program
programId = createProgram(kVertexShader, kFragmentShader);
LOGI("Program id : %d error : %d",(int) programId, glGetError());
if(!programId)
{
LOGI("Could not create program");
return;
}
// get index of the generic vertex attribute bound to vPosition
positionObject = (int) glGetAttribLocation(programId, "vPosition");
// get index of the generic vertex attribute bound to vTexCoord
texturePosition = (int) glGetAttribLocation(programId, "vTexCoord");
// get the location of yTexture within the program (corresponding to program id)
yuv_texture_object[0] = glGetUniformLocation(programId, "yTexture");
// get the location of uTexture within the program
yuv_texture_object[1] = glGetUniformLocation(programId, "uTexture");
// get the location of vTexture within the program
yuv_texture_object[2] = glGetUniformLocation(programId, "vTexture");
// Setup width of each planes (display size)
stream_yuv_width[0] = 800;
stream_yuv_width[1] = 400;
stream_yuv_width[2] = 400;
// Setup height of each planes (display size)
stream_yuv_height[0] = 600;
stream_yuv_height[1] = 300;
stream_yuv_height[2] = 300;
//set the view port
glViewport(0,0,stream_yuv_width[0],stream_yuv_height[0]);
LOGI("glViewPort() %d ", glGetError());
I have hardcoded the display size (for now) until i get something that work.
The createProgram method, load the shaders, create the program, compile and link the shaders successfully.
Here are my shaders :
const char kVertexShader[] =
"attribute vec4 vPosition;\n"
"attribute vec2 vTexCoord;\n"
"varying vec2 v_vTexCoord;\n"
"void main() {\n"
"gl_Position = vPosition;\n"
"v_vTexCoord = vTexCoord;\n"
"}\n";
const char kFragmentShader[] =
"precision mediump float; \n"
"varying vec2 v_vTexCoord;\n"
"uniform sampler2D yTexture;\n"
"uniform sampler2D uTexture;\n"
"uniform sampler2D vTexture;\n"
"void main() {\n"
"float nx, ny; \n"
"nx = v_vTexCoord.x; \n"
"ny = v_vTexCoord.y; \n"
"float y=texture2D(yTexture, v_vTexCoord).r;\n"
"float u=texture2D(uTexture, vec2(nx / 2.0, ny / 2.0)).r;\n"
"float v=texture2D(vTexture, vec2(nx / 2.0, ny / 2.0)).r;\n"
"y = 1.1643 * (y - 0.0625);\n"
"u = u - 0.5; \n"
"v = v - 0.5; \n"
"float r=y + 1.5958 * v;\n"
"float g=y - 0.39173 * u - 0.81290 * v;\n"
"float b=y + 2.017 * u;\n"
"gl_FragColor = vec4(r, g, b, 1.0);\n"
"}\n";
const GLfloat kVertexInformation[] = {
-1.0f, 1.0f, // TexCoord 0 top left
-1.0f,-1.0f, // TexCoord 1 bottom left
1.0f,-1.0f, // TexCoord 2 bottom right
1.0f, 1.0f // TexCoord 3 top right
};
const GLshort kTextureCoordinateInformation[] = {
0, 0, // TexCoord 0 top left
0, 1, // TexCoord 1 bottom left
1, 1, // TexCoord 2 bottom right
1, 0 // TexCoord 3 top right
};
const GLuint kStride = 0;//COORDS_PER_VERTEX * 4;
const GLshort kIndicesInformation[] = {
0, 1, 2,
0, 2, 3
};
Then i setup the yuv textures and the render to textures, at this moment yuv_width[i] and yuv_height[i] are set to correct value:
void setupYUVTexture()
{
//Setup the pixel alignement
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
LOGI("glPixelStorei() : %d ", glGetError());
int i = 0;
for(i = 0 ; i < 3 ; ++i)
{
//Check if the texture already setup
if(yuv_texture_id[i] != 0)
{
glDeleteTextures(1, &yuv_texture_id[i]);
yuv_texture_id[i] = 0;
}
// Active the i texture
glActiveTexture(GL_TEXTURE0 + i);
//Generate the texture name
glGenTextures(1, &yuv_texture_id[i]);
// Bind the texture
glBindTexture(GL_TEXTURE_2D, yuv_texture_id[i]);
// Setup the texture parameters
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//Define the texture image
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, yuv_width[i], yuv_height[i], 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
LOGI("glTexImage2D() %d ", glGetError());
}
}
void renderToTexture()
{
// Generate framebuffer object name
glGenFramebuffers(1, &frameBufferObject);
//Bind the framebuffer
glBindFramebuffer(GL_FRAMEBUFFER, frameBufferObject);
//Generate render buffer object name
glGenRenderbuffers(1, &renderBufferObject);
//Bind render buffer
glBindRenderbuffer(GL_RENDERBUFFER, renderBufferObject);
//Create and initialize render buffer for display RGBA with the same size of the viewport
glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA4, 800, 600);
//Attach render buffer to frame buffer object
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, renderBufferObject);
//Attach y plane to frame buffer object
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id[0], 0);
//Attach u plane to frame buffer object
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id[1], 0);
//Attach v plane to frame buffer object
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id[2], 0);
// Bind the framebuffer
glBindFramebuffer(GL_FRAMEBUFFER, 0);
//Check if the framebuffer is correctly setup
GLint status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if(status != GL_FRAMEBUFFER_COMPLETE)
{
LOGI(" FBO setting fault : %d ", status);
return;
}
}
To finish, my draw frame method :
void drawFrame()
{
LOGI("DrawFrame");
glBindFramebuffer(GL_FRAMEBUFFER, frameBufferObject);
printGLError("glBindFramebuffer");
glUseProgram(programId);
printGLError("glUseProgram");
int i = 0;
for(i = 0 ; i < 3 ; ++i)
{
glActiveTexture(GL_TEXTURE0 + i);
printGLError("glActiveTexture");
glBindTexture(GL_TEXTURE_2D, yuv_texture_object[i]);
printGLError("glBindTexture");
glUniform1i(yuv_texture_object[i], i);
printGLError("glUniform1i");
LOGI("Plan : %d Largeur : %d Hauteur : %d ", i, yuv_width[i], yuv_height[i]);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0,yuv_width[i], yuv_height[i], GL_LUMINANCE, GL_UNSIGNED_BYTE, yuv_planes[i]);
printGLError("glTexSubImage2D");
}
glVertexAttribPointer(positionObject, 2, GL_FLOAT, GL_FALSE, kStride, kVertexInformation);
printGLError("glVertexAttribPointer");
glVertexAttribPointer(texturePosition, 2, GL_SHORT, GL_FALSE, kStride, kTextureCoordinateInformation);
printGLError("glVertexAttribPointer");
glEnableVertexAttribArray(positionObject);
printGLError("glVertexAttribArray");
glEnableVertexAttribArray(texturePosition);
printGLError("glVertexAttribArray");
glBindFramebuffer(GL_FRAMEBUFFER, 0);
printGLError("glBindFramebuffer");
glDrawElements(GL_TRIANGLE_STRIP, 6, GL_UNSIGNED_SHORT, kIndicesInformation);
printGLError("glDrawElements");
eglSwapBuffers(display, surface);
printGLError("eglSwapBuffers");
}
I initialize once the opengl textures and other attributes which are necessary, then when a frame is decode i recopy y buffer into yuv_planes[0], u buffer in yuv_planes[ 1] and v buffer in yuv_planes[2].
Once a frame is correctly decoded using ffmpeg i call in this order :
- initContext()
- setupGraphics()
- setupYUVTexture()
- renderToTexture()
then i call drawFrame. Of course, when everything is initialize i call directly drawFrame after each decoded frame.
There is the output i have now.
The size of the image is correct, but now i am block here. I don't understand why the image display is green !
Any ideas
That's a lot of code to go through and a lot of things that can go wrong ;). To debug these kind of issues, I would go step by step.
just output red (gl_FragColor = vec4(1.0, 0.5, 0.5, 1.0)) to make sure your configuration is working properly.
try to output every texture in grayscale. (gl_FragColor = vec4(y, y, y, 1.0))
If all that is working then it most likely means your yuv => rgb conversion is wrong somewhere.
If that's not working, then I would suspect something in the texture mapping. Double check your glTexSubImage2D call. You might need to pass a different stride or use a different coordinate system.
I made a render-to-texture test using Qt, running it both on macOS and Android.
The test creates 2 textures and an FBO, and then in a loop attaches alternating textures as rendering target.
On Android the code is surprisingly slow though. On a Samsung Galaxy Tab S, it about 30-40 times slower than on my mac, so I suspect something is wrong.
The actual code inside the fragment shader does not seem to matter.
Adding a glClear after the glBindFramebuffer makes it a bit faster but still very slow. Any clues on where to look for the cause?
// create textures
glEnable(GL_TEXTURE_2D);
GLuint tex1, tex2;
// define texture properties
glGenTextures(1, &tex1);
glBindTexture(GL_TEXTURE_2D, tex1);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER_EXT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER_EXT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, 1280, 800, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0);
// define texture properties
glGenTextures(1, &tex2);
glBindTexture(GL_TEXTURE_2D, tex2);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER_EXT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER_EXT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, 1280, 800, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0);
// create framebuffer
GLuint fbo;
glGenFramebuffers(1, &fbo);
glBindFramebuffer(GL_FRAMEBUFFER, fbo);
glClear(GL_COLOR_BUFFER_BIT);
// create program
QGLShaderProgram program;
program.addShaderFromSourceFile(QGLShader::Fragment, ":/shaders/fshader.glsl");
program.addShaderFromSourceFile(QGLShader::Vertex, ":/shaders/vshader.glsl");
program.link();
program.bind();
//
float vertices[16];
int i = 0;
vertices[i++] = 0.0f; vertices[i++] = 0.0f; vertices[i++] = 0.0; vertices[i++] = 1.0;
vertices[i++] = 0.0f; vertices[i++] = 1280.0f; vertices[i++] = 0.0; vertices[i++] = 0.0;
vertices[i++] = 1280.0f; vertices[i++] = 0.0f; vertices[i++] = 1.0; vertices[i++] = 1.0;
vertices[i++] = 1280.0f; vertices[i++] = 1280.0f; vertices[i++] = 1.0; vertices[i++] = 0.0;
int vertexLocation = program.attributeLocation("a_position");
program.enableAttributeArray(vertexLocation);
glVertexAttribPointer(vertexLocation, 2, GL_FLOAT, GL_FALSE, 4*sizeof(float), (const void *)vertices);
int texcoordLocation = program.attributeLocation("a_texcoord");
program.enableAttributeArray(texcoordLocation);
glVertexAttribPointer(texcoordLocation, 2, GL_FLOAT, GL_FALSE, 4*sizeof(float), (const void *)(vertices + 2));
QMatrix4x4 textureMatrix, modelViewMatrix;
// do loop test
int count = 10000;
bool swapped = false;
cout << "Start fbo test" << endl;
QTime myTimer;
myTimer.start();
textureMatrix.setToIdentity();
program.setUniformValue("textureMatrix", textureMatrix);
modelViewMatrix.setToIdentity();
program.setUniformValue("modelViewProjectionMatrix", modelViewMatrix);
program.setUniformValue("srcTex", 0);
for(int i = 0; i < count; i++)
{
glBindFramebuffer(GL_FRAMEBUFFER, fbo);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_TEXTURE_2D, swapped? tex2 : tex1, 0);
// check completeness
GLenum status;
status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
switch(status)
{
case GL_FRAMEBUFFER_COMPLETE:
break;
default:
log("Framebuffer error");
}
glEnable(GL_TEXTURE_2D);
glBindTexture(GL_TEXTURE_2D, swapped? tex1 : tex2);
// draw slab
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
swapped = !swapped;
}
This is the vertex shader:
#ifdef GL_ES
// Set default precision to medium
precision mediump int;
precision mediump float;
#endif
uniform mat4 modelViewProjectionMatrix;
uniform mat4 textureMatrix;
attribute vec4 a_position;
attribute vec2 a_texcoord;
varying vec2 v_texcoord;
void main()
{
// Calculate vertex position in screen space
gl_Position = modelViewProjectionMatrix* a_position;
v_texcoord = vec4(textureMatrix * vec4(a_texcoord, 0.0, 1.0)).xy;
}
And the fragment shader:
#ifdef GL_ES
// Set default precision to medium
precision mediump int;
precision mediump float;
#endif
uniform sampler2D srcTex;
varying vec2 v_texcoord;
void main(void)
{
vec4 f = texture2D(srcTex, v_texcoord);
gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);
}
Use two FBOs, each binds to a separate texture instead of swapping within the loop. Call glClear() after glBindFramebuffer() within the loop if you don't want to preserve the content in texture.
Riddle me this,
I recently published a game for Android (play.google.com/store/apps/details?id=com.quackers if you want to witness the problems first hand), and initial feedback suggests that the thing doesn't run properly on some devices. I've since got ahold of one of the offending tablets (a Samsung Galaxy Tab 2 7.0) and it turns out it does run, it just wasn't rendering things properly.
A few digs later and I've discovered that it's a texturing issue. Textures are being loaded okay, but they're not being rendered - not the usual black squares you often get with OpenGL when something goes wrong - nothing at all.
This is OpenGL ES 2.0, doing an SDL/C++/ndk thing. While there are similar problems on the net, much of it involves ES 1.0 and regards a different issue - texture dimensions not being powers of two (e.g. 64x64, 128x128, 256x256 etc.) or some wacky compression stuff which doesn't apply here.
I've stripped out all of my rendering code and have gone back to basics - rendering a textured square (in a not particularly optimised manner).
Pre-loop code:
SDL_Init(SDL_INIT_VIDEO);
SDL_LogSetAllPriority(SDL_LOG_PRIORITY_VERBOSE);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0);
SDL_DisplayMode mode;
SDL_GetDisplayMode(0,0, &mode);
_currentWidth = mode.w;
_currentHeight = mode.h;
SDL_GL_SetAttribute(SDL_GL_ACCELERATED_VISUAL, 1);
SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1);
SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE, 24);
_screen = SDL_CreateWindow("window", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, _currentWidth, _currentHeight, SDL_WINDOW_OPENGL | SDL_WINDOW_FULLSCREEN | SDL_WINDOW_RESIZABLE);
SDL_GLContext context = SDL_GL_CreateContext(_screen);
SDL_GL_MakeCurrent(_screen, context);
glViewport(0, 0, _currentWidth, _currentHeight);
//---
GLuint vs = glCreateShader(GL_VERTEX_SHADER);
const char *vs_source = "attribute highp vec2 coord2d; "
"attribute highp vec2 texcoord;"
"varying highp vec2 f_texcoord;"
"void main(void) { "
"gl_Position = vec4(coord2d, 0.0, 1.0); "
"f_texcoord = texcoord;"
"}";
glShaderSource(vs, 1, &vs_source, NULL);
glCompileShader(vs);
GLuint fs = glCreateShader(GL_FRAGMENT_SHADER);
const char *fs_source = "varying highp vec2 f_texcoord;"
"uniform sampler2D texture;"
"void main(void) { "
"vec2 flipped_texcoord = vec2(f_texcoord.x, 1.0 - f_texcoord.y);"
"gl_FragColor = texture2D(texture, flipped_texcoord);"
"}";
glShaderSource(fs, 1, &fs_source, NULL);
glCompileShader(fs);
_program = glCreateProgram();
glAttachShader(_program, vs);
glAttachShader(_program, fs);
glLinkProgram(_program);
//---
GLuint vs2 = glCreateShader(GL_VERTEX_SHADER);
const char *vs_source2 = "attribute vec2 coord2d; "
"void main(void) { "
"gl_Position = vec4(coord2d, 0.0, 1.0); "
"}";
glShaderSource(vs2, 1, &vs_source2, NULL);
glCompileShader(vs2);
GLuint fs2 = glCreateShader(GL_FRAGMENT_SHADER);
const char *fs_source2 = "uniform lowp vec4 u_colour;"
"void main(void) { "
"gl_FragColor = u_colour;"
"}";
glShaderSource(fs2, 1, &fs_source2, NULL);
glCompileShader(fs2);
_flatProgram = glCreateProgram();
glAttachShader(_flatProgram, vs2);
glAttachShader(_flatProgram, fs2);
glLinkProgram(_flatProgram);
glEnable(GL_BLEND);
glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
//---------------------------------------
_screenRect.x = -1.0;
_screenRect.y = -1.0;
_screenRect.w = 2.0;
_screenRect.h = 2.0;
_superDuperFrameBuffer = 0;
_depthRenderBuffer = 0;
glGenTextures(1, &_screenTexture);
glBindTexture(GL_TEXTURE_2D, _screenTexture);
if(_currentWidth < SCREENWIDTH*2 || _currentHeight < SCREENHEIGHT*2) {
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
}
else {
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
}
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, SCREENWIDTH, SCREENHEIGHT, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0);
glBindTexture(GL_TEXTURE_2D, 0);
glGenRenderbuffers(1, &_depthRenderBuffer);
glBindRenderbuffer(GL_RENDERBUFFER, _depthRenderBuffer);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8_OES, SCREENWIDTH, SCREENHEIGHT);
glBindRenderbuffer(GL_RENDERBUFFER, 0);
// create a framebuffer object
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glGenFramebuffers(1, &_superDuperFrameBuffer);
glBindFramebuffer(GL_FRAMEBUFFER, _superDuperFrameBuffer);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _screenTexture, 0);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, _depthRenderBuffer);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_RENDERBUFFER, _depthRenderBuffer);
glBindFramebuffer(GL_FRAMEBUFFER, 0);
_defaultFrameBuffer = 0;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &_defaultFrameBuffer);
glClearColor(0.0f, 0.0f, 1.0f, 1.0f);
glEnable(GL_BLEND);
glEnable(GL_TEXTURE_2D);
SDL_Surface* testSurface = IMG_Load("graphics/bg_01_0.png");
uint32_t rmask;
uint32_t gmask;
uint32_t bmask;
uint32_t amask;
#if SDL_BYTEORDER == SDL_BIG_ENDIAN
rmask = 0xff000000;
gmask = 0x00ff0000;
bmask = 0x0000ff00;
amask = 0x000000ff;
#else
rmask = 0x000000ff;
gmask = 0x0000ff00;
bmask = 0x00ff0000;
amask = 0xff000000;
#endif
SDL_Surface *tempSurface = SDL_CreateRGBSurface(0, testSurface->w, testSurface->h, 32, rmask, gmask, bmask, amask);
SDL_SetSurfaceBlendMode(tempSurface, SDL_BLENDMODE_BLEND);
SDL_BlitSurface(testSurface, NULL, tempSurface, NULL);
testSurface = tempSurface;
SDL_FreeSurface(tempSurface);
GLint uniformTexture = glGetUniformLocation(_program, "texture");
_testTexture = 0;
glGenTextures(1, &_testTexture);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, _testTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
//glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
//glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glUniform1i(uniformTexture, /*GL_TEXTURE*/0);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, 256, 256, 0, GL_RGBA, GL_UNSIGNED_BYTE, testSurface->pixels);
_vboTest = 0;
_vbo_cube_texcoords = 0;
glGenBuffers(1, &_vboTest);
glGenBuffers(1, &_vbo_cube_texcoords);
loop:
...
_quadColour[0] = 0.0f;
_quadColour[1] = 255.0f;
_quadColour[2] = 0.0f;
_quadColour[3] = 1.0f;
drawSquare(0, 0, 20, 20);
glViewport(0, 0, SCREENWIDTH, SCREENHEIGHT);
GLfloat x1 = 0, x2 = 8, y1 = 0, y2 = 8;
glUseProgram(_program);
GLint attributeCoord2d = glGetAttribLocation(_program, "coord2d");
GLint attributeTexcoord = glGetAttribLocation(_program, "texcoord");
glEnableVertexAttribArray(attributeTexcoord);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, _testTexture);
GLfloat cube_texcoords[] = {
0.0, 1.0,
0.0, 0.0,
1.0, 0.0,
1.0, 1.0,
};
glBindBuffer(GL_ARRAY_BUFFER, _vbo_cube_texcoords);
glVertexAttribPointer(attributeTexcoord, 2, GL_FLOAT, GL_FALSE, 0, 0);
glBufferData(GL_ARRAY_BUFFER, sizeof(cube_texcoords), cube_texcoords, GL_STATIC_DRAW);
glEnableVertexAttribArray(attributeCoord2d);
GLfloat triangle_vertices[] = {
x1, y2,
x1, y1,
x2, y1,
x2, y2,
};
glBindBuffer(GL_ARRAY_BUFFER, _vboTest);
glVertexAttribPointer(attributeCoord2d, 2, GL_FLOAT, GL_FALSE, 0, 0);
glBufferData(GL_ARRAY_BUFFER, sizeof(triangle_vertices), triangle_vertices, GL_STATIC_DRAW);
glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
SDL_GL_SwapWindow(_screen);
...
obviously there's some stuff I've only added for testing purposes, like converting textures to RGBA and whatever. It draws a little green square, then a textured square.
Code might be messy but the point is this - two different results:
Galaxy Tab 2 7.0 (bork)
http://i.imgur.com/ht6LvFV.png
Nexus 7 (correct)
http://i.imgur.com/p4acmIq.png
How do I fix this?
Power-of-two textures are still required on many GLES 2.0 devices. From section 3.8.2 of the GLES 2.0 spec (https://www.khronos.org/registry/gles/specs/2.0/es_full_spec_2.0.25.pdf):
"Calling a sampler from a fragment shader will return (R; G;B;A) =
(0; 0; 0; 1) if any of the following conditions are true: ... A two-dimensional sampler is called, the corresponding texture image is a non-power-of-two image (as described in the Mipmapping discussion of section 3.7.7), and either the texture wrap mode is not CLAMP_TO_EDGE, or the minification filter is neither NEAREST nor LINEAR."
Assuming SCREENHEIGHT/SCREENWIDTH are the dimensions of your device, you're violating this restriction. You can ignore this restriction if your device supports some NPOT extension, for instance GL_OES_texture_npot (https://www.khronos.org/registry/gles/extensions/OES/OES_texture_npot.txt), although in my experience, some devices that report this extension still sample textures as black when the npot texture is the color target of a framebuffer. The best course is to just always use POT render targets in ES 2.0.
There are a few errors and possible issues in this code:
Your fragment shader should not compile if the GLSL compiler is strict about error checking. Since you do not specify a default precision, and there is no default precision for float/vector/matrix types, you need an explicit precision for all declarations. It is missing for this variable:
vec2 flipped_texcoord = vec2(f_texcoord.x, 1.0 - f_texcoord.y);
If you want to stick with highp, this should be:
highp vec2 flipped_texcoord = vec2(f_texcoord.x, 1.0 - f_texcoord.y);
This call has a bad argument:
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _screenTexture, 0);
Since you're attaching a texture, the 3rd argument must be GL_TEXTURE_2D (you would need to use glFramebufferRenderbuffer for attaching a renderbuffer):
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, screenTexture, 0);
Make sure that the OES_packed_depth_stencil extension is supported on the device, since you're using it here:
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8_OES, SCREENWIDTH, SCREENHEIGHT);
This code sequence does not make much sense:
glBindFramebuffer(GL_FRAMEBUFFER, 0);
_defaultFrameBuffer = 0;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &_defaultFrameBuffer);
You just bound framebuffer 0, so the current framebuffer binding will always be 0 here. If you're concerned that the default framebuffer might not be 0, you have to query the value before the first time you change the binding.
This is not a valid call in ES 2.0:
glEnable(GL_TEXTURE_2D);
Enabling textures was only needed in fixed function OpenGL. Once you use shader, it will just use textures anytime the shader... uses textures. No need to explicitly enable anything.
I have a 2D game project that I'm porting to Android that utilizes OpenGL ES 2.0. I am having trouble getting anything drawn on the screen (except for a solid color from clearing the screen). Everything renders just fine when running in my Windows environment, but of course the environment is set up differently for the different version of OpenGL.
I followed the native-activity sample and took advice from several other OpenGL ES 2.0 resources to compose what I currently have.
I have checked everything I know how to with no anomalous results. As mentioned, glClear works, and displays the color set by glClearColor. I also know that every frame is being rendered, as changing glClearColor frame-by-frame displays the different colors. Of course, the application properly compiles. My textures are loaded from the proper location in the app's cache. glGetError is returning GL_NO_ERROR at every step in the process, so what I am doing appears to be accepted by OpenGL. My shaders are loaded without error. I have also tested this on both a few emulators and my physical android device, so it isn't localized to a specific device configuration.
I speculate that it must be some mistake in how I initialize and set up OpenGL. I am hoping someone more versed in OpenGL ES than I am will be able to help root out my problem. I am pasting the different relevant sections of my code below. engine is a global struct I am presently using out of laziness.
Initializing the display
static int AND_InitDisplay() {
// Desired display attributes
const EGLint attribs[] = {
EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL_BLUE_SIZE, 8,
EGL_GREEN_SIZE, 8,
EGL_RED_SIZE, 8,
EGL_ALPHA_SIZE, 8,
EGL_DEPTH_SIZE, 16,
EGL_NONE
};
EGLint w, h, dummy, format;
EGLint numConfigs;
EGLConfig config;
EGLSurface surface;
EGLContext context;
EGLDisplay display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
eglInitialize(display, 0, 0);
eglChooseConfig(display, attribs, &config, 1, &numConfigs);
eglGetConfigAttrib(display, config, EGL_NATIVE_VISUAL_ID, &format);
ANativeWindow_setBuffersGeometry(engine->app->window, 0, 0, format);
surface = eglCreateWindowSurface(display, config, engine->app->window, NULL);
EGLint const attrib_list[3] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE};
context = eglCreateContext(display, config, NULL, attrib_list);
if (eglMakeCurrent(display, surface, surface, context) == EGL_FALSE) {
LOGW("Unable to eglMakeCurrent");
return -1;
}
eglQuerySurface(display, surface, EGL_WIDTH, &w);
eglQuerySurface(display, surface, EGL_HEIGHT, &h);
engine->display = display;
engine->context = context;
engine->surface = surface;
engine->width = w;
engine->height = h;
// Initialize GL state.
glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST);
return 0;
}
Drawing a frame
static void AND_drawFrame() {
if (engine->display == NULL) {
LOGW("DB E: DISPLAY IS NULL");
// No display.
return;
}
// Clearing with red color. This displays properly.
glClearColor(1.f, 0.f, 0.f, 1.f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
// eglSwapBuffers results in no visible change
eglSwapBuffers(engine->display, engine->surface);
}
Example of preparing VBO data
I understand many wouldn't like the idea of using multiple VBOs for the same geometry. I would love to hear if this code isn't orthodox or is incorrect, but I am not focused on this unless it the root of my problem.
GLfloat charPosVerts[] = {
p0.x, p0.y, 0.f,
p1.x, p0.y, 0.f,
p1.x, p1.y, 0.f,
p0.x, p0.y, 0.f,
p1.x, p1.y, 0.f,
p0.x, p1.y, 0.f
};
GLfloat charTexVerts[] = {
0.0, 0.0,
textures[texid].w, 0.0,
textures[texid].w, textures[texid].h,
0.0, 0.0,
textures[texid].w, textures[texid].h,
0.0, textures[texid].h
};
GLfloat charColorVerts[] = {
e->color.r, e->color.g, e->color.b, e->color.a,
e->color.r, e->color.g, e->color.b, e->color.a,
e->color.r, e->color.g, e->color.b, e->color.a,
e->color.r, e->color.g, e->color.b, e->color.a,
e->color.r, e->color.g, e->color.b, e->color.a,
e->color.r, e->color.g, e->color.b, e->color.a
};
glGenBuffers(1, &(e->vboPos));
glGenBuffers(1, &(e->vboTex));
glGenBuffers(1, &(e->vboColor));
glBindBuffer(GL_ARRAY_BUFFER, e->vboPos);
glBufferData(GL_ARRAY_BUFFER, sizeof(charPosVerts), charPosVerts, GL_DYNAMIC_DRAW);
glVertexAttribPointer(shaderIDs.attribPosition, 3, GL_FLOAT, GL_FALSE, 0, 0);
glEnableVertexAttribArray(shaderIDs.attribPosition);
glBindBuffer(GL_ARRAY_BUFFER, e->vboTex);
glBufferData(GL_ARRAY_BUFFER, sizeof(charTexVerts), charTexVerts, GL_DYNAMIC_DRAW);
glVertexAttribPointer(shaderIDs.attribTexCoord, 2, GL_FLOAT, GL_FALSE, 0, 0);
glEnableVertexAttribArray(shaderIDs.attribTexCoord);
glBindBuffer(GL_ARRAY_BUFFER, e->vboColor);
glBufferData(GL_ARRAY_BUFFER, sizeof(charColorVerts), charColorVerts, GL_DYNAMIC_DRAW);
glVertexAttribPointer(shaderIDs.attribColors, 4, GL_FLOAT, GL_FALSE, 0, 0);
glEnableVertexAttribArray(shaderIDs.attribColors);
Example of drawing VBO
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, CORE_GetBmpOpenGLTex(texix));
glUniform1i(shaderIDs.uniTexture, 0);
// Draw the sprite
glBindBuffer(GL_ARRAY_BUFFER, e->vboPos);
glVertexAttribPointer(shaderIDs.attribPosition, 3, GL_FLOAT, GL_FALSE, 0, 0);
glEnableVertexAttribArray(shaderIDs.attribPosition);
glBindBuffer(GL_ARRAY_BUFFER, e->vboTex);
glVertexAttribPointer(shaderIDs.attribTexCoord, 2, GL_FLOAT, GL_FALSE, 0, 0);
glEnableVertexAttribArray(shaderIDs.attribTexCoord);
glBindBuffer(GL_ARRAY_BUFFER, e->vboColor);
glVertexAttribPointer(shaderIDs.attribColors, 4, GL_FLOAT, GL_FALSE, 0, 0);
glEnableVertexAttribArray(shaderIDs.attribColors);
glDrawArrays(GL_TRIANGLES, 0, 18);
Vertex Shader
The shaders are very simple.
attribute vec3 position;
attribute vec2 texCoord;
attribute vec4 colors;
varying vec2 texCoordVar;
varying vec4 colorsVar;
void main() {
gl_Position = vec4(position, 1.0);
texCoordVar = texCoord;
colorsVar = colors;
}
Fragment Shader
uniform sampler2D texture;
varying vec2 texCoordVar;
varying vec4 colorsVar;
void main()
{
gl_FragColor = texture2D(texture, texCoordVar) * colorsVar;
}
Thanks for looking at this long post. Help is very much appreciated.
The posted code is not drawing anything. From the AND_drawFrame() function:
// Clearing with red color. This displays properly.
glClearColor(1.f, 0.f, 0.f, 1.f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
// eglSwapBuffers results in no visible change
eglSwapBuffers(engine->display, engine->surface);
Based on this, the draw code is either never invoked, or the window is cleared after drawing, which would wipe out everything that was drawn before.