Develop Reference

How to provide a shader function to perform the alpha masking for video

I am working on Vr app using virocore library in android. I have to show video over sphere. The video which i have to implement is not actually video but the two frames provided are the colour frame (left) and the alpha mask (right) frame. I have not worked with openGl but seems like I will need to provide a shader function to perform the alpha masking.
I have used this for shader Adding transparency to a video from black and white (and gray) alpha information video images
but how can i use it with OpenGL in on draw method? or If there is any way in virocore using which i can do alpha masking. I have tried chroma filtering method in virocore but that makes whole video transparent.
public class VideoSurfaceView extends GLSurfaceView {
VideoRender mRenderer;
private MediaPlayer mMediaPlayer = null;
public VideoSurfaceView(Context context, MediaPlayer mp) {
super(context);
setEGLContextClientVersion(2);
mMediaPlayer = mp;
mRenderer = new VideoRender(context);
this.getHolder().setFormat(PixelFormat.RGB_565);
this.getHolder().setFormat(PixelFormat.TRANSPARENT);
setEGLConfigChooser(8,8,8,8,16,0);
setEGLContextClientVersion(2);
setRenderer(mRenderer);
}
#Override
public void onResume() {
Log.e("onResume ", "onResume");
queueEvent(new Runnable(){
public void run() {
Log.e("runnable ", "runnable");
mRenderer.setMediaPlayer(mMediaPlayer);
}});
super.onResume();
}
private static class VideoRender
implements Renderer, SurfaceTexture.OnFrameAvailableListener, MediaPlayer.OnPreparedListener {
private static String TAG = "VideoRender";
private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 0.f, 0.f,
1.0f, -1.0f, 0, 1.f, 0.f,
-1.0f, 1.0f, 0, 0.f, 1.f,
1.0f, 1.0f, 0, 1.f, 1.f,
};
private FloatBuffer mTriangleVertices;
private static final String mVertexShader =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 position;\n" +
"attribute vec4 inputTextureCoordinate;\n" +
" \n" +
"varying vec2 textureCoordinate;\n" +
"varying vec2 textureCoordinate2;\n" +
" \n" +
"void main()\n" +
"{\n" +
" gl_Position = uMVPMatrix * position;\n" +
" vec4 texCoord = uSTMatrix * inputTextureCoordinate;\n"+
"textureCoordinate = vec2(inputTextureCoordinate.x * 0.5, inputTextureCoordinate.y);\n" +
" textureCoordinate2 = vec2(inputTextureCoordinate.x * 0.5 + 0.5, inputTextureCoordinate.y);\n" +
"}";
public static final String mFragmentShader = "#extension GL_OES_EGL_image_external : require\n"+
"varying highp vec2 textureCoordinate;\n"+
"varying highp vec2 textureCoordinate2;\n"+
"uniform samplerExternalOES inputImageTexture;\n" +
"void main() {\n"+
" lowp vec4 rgbcolor = texture2D(inputImageTexture, textureCoordinate);\n"+
" lowp vec4 alphaValue = texture2D(inputImageTexture, textureCoordinate2);\n"+
" if (alphaValue.g < 0.5)\n"+
" discard;\n"+
" gl_FragColor = vec4(rgbcolor.rgb, 1.0);\n"+
"}";
private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private int mProgram;
private int mTextureID;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
private SurfaceTexture mSurface;
private boolean updateSurface = false;
private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
private MediaPlayer mMediaPlayer;
public VideoRender(Context context) {
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(mTriangleVerticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
}
public void setMediaPlayer(MediaPlayer player) {
mMediaPlayer = player;
}
#Override
public void onDrawFrame(GL10 glUnused) {
synchronized(this) {
if (updateSurface) {
mSurface.updateTexImage();
mSurface.getTransformMatrix(mSTMatrix);
updateSurface = false;
}
}
GLES20.glClearColor(0.0f, 0.0f, 0.0f, .0f);
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT|GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glEnable(GLES20.GL_BLEND);
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
}
#Override
public void onSurfaceChanged(GL10 glUnused, int width, int height) {
}
#Override
public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
mProgram = createProgram(mVertexShader, mFragmentShader);
if (mProgram == 0) {
return;
}
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "position");
checkGlError("glGetAttribLocation aPosition");
if (maPositionHandle == -1) {
throw new RuntimeException("Could not get attrib location for aPosition");
}
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "inputTextureCoordinate");
checkGlError("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1) {
throw new RuntimeException("Could not get attrib location for aTextureCoord");
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
checkGlError("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uMVPMatrix");
}
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
checkGlError("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uSTMatrix");
}
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
mTextureID = textures[0];
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError("glBindTexture mTextureID");
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST);
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
/*
* Create the SurfaceTexture that will feed this textureID,
* and pass it to the MediaPlayer
*/
mSurface = new SurfaceTexture(mTextureID);
mSurface.setOnFrameAvailableListener(this);
Log.e("surface ", "surface");
Surface surface = new Surface(mSurface);
mMediaPlayer.setSurface(surface);
mMediaPlayer.setScreenOnWhilePlaying(true);
surface.release();
mMediaPlayer.setOnPreparedListener(this);
mMediaPlayer.prepareAsync();
synchronized(this) {
updateSurface = false;
}
}
synchronized public void onFrameAvailable(SurfaceTexture surface) {
updateSurface = true;
}
private int loadShader(int shaderType, String source) {
int shader = GLES20.glCreateShader(shaderType);
if (shader != 0) {
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e(TAG, "Could not compile shader " + shaderType + ":");
Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
}
return shader;
}
private int createProgram(String vertexSource, String fragmentSource) {
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if (vertexShader == 0) {
return 0;
}
int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if (pixelShader == 0) {
return 0;
}
int program = GLES20.glCreateProgram();
if (program != 0) {
GLES20.glAttachShader(program, vertexShader);
checkGlError("glAttachShader");
GLES20.glAttachShader(program, pixelShader);
checkGlError("glAttachShader");
GLES20.glLinkProgram(program);
int[] linkStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] != GLES20.GL_TRUE) {
Log.e(TAG, "Could not link program: ");
Log.e(TAG, GLES20.glGetProgramInfoLog(program));
GLES20.glDeleteProgram(program);
program = 0;
}
}
return program;
}
private void checkGlError(String op) {
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e(TAG, op + ": glError " + error);
throw new RuntimeException(op + ": glError " + error);
}
}
#Override
public void onPrepared(MediaPlayer mediaPlayer) {
mediaPlayer.start();
}
} // End of class VideoRender.
} // End of class VideoSurfaceView.
With this code, video looks inverted

If you want to discard fragments, then you can use the discard keyword in the fragment shader.
e.g. discard all fragments with an alpha value less than 0.5:
void main()
{
lowp vec4 rgbcolor = texture2D(inputImageTexture, textureCoordinate);
lowp vec4 alphaValue = texture2D(inputImageTexture, textureCoordinate2);
if (alphaValue.g < 0.5)
discard;
gl_FragColor = vec4(rgbcolor.rgb, 1.0);
}
See also OpenGL ES Shading Language 1.00 Specification; 6.4 Jumps; page 58:
The discard keyword is only allowed within fragment shaders. It can be used within a fragment shader to abandon the operation on the current fragment. This keyword causes the fragment to be discarded and no updates to any buffers will occur. It would typically be used within a conditional statement, for example:
if (intensity < 0.0)
discard;
In reference to the comment
As there is no matrix so what will i use in GLES20.glGetUniformLocation?
Of course you can add the matrices to the vertex shader:
attribute vec4 inputTextureCoordinate;
varying vec2 textureCoordinate;
varying vec2 textureCoordinate2;
uniform mat4 matMVP;
uniform mat4 matST;
void main()
{
gl_Position = matMVP * position;
vec4 texCoord = matST * inputTextureCoordinate;
textureCoordinate = vec2(texCoord.x * 0.5, 1.0 - texCoord.y);
textureCoordinate2 = vec2(texCoord.x * 0.5 + 0.5, 1.0 - texCoord.y);
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "matMVP");
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "matST");
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

OpenGL ES Fatal signal 11 (SIGSEGV), code 1, fault addr 0x8 in tid 8367 (GLThread 3562)

I have a crash in my call to glDrawArrays when trying to render a .obj loaded using a library. I dont know what is happening since it is my first time using openGL ES. My guess was that the number of triangles was wrong and after trying glDrawArrays with 10 as parameter, I realised this might not be the problem.
The code of my renderer:
private class RocketArrowRenderer implements GLSurfaceView.Renderer {
private final int mBytesPerFloat = 4;
private Context mContext;
private FloatBuffer mVertices;
private int mMVPMatrixHandle;
private int mPositionHandle;
private float[] mModelMatrix = new float[16];
private int mColorHandle;
private float[] mMVPMatrix = new float[16];
private final int mStrideBytes = 3 * mBytesPerFloat;
private final int mPositionOffset = 0;
private final int mPositionDataSize = 3;
private final int mColorOffset = 3;
private final int mColorDataSize = 4;
private float[] mViewMatrix = new float[16];
final String vertexShader =
"uniform mat4 u_MVPMatrix; \n" // A constant representing the combined model/view/projection matrix.
+ "attribute vec4 a_Position; \n" // Per-vertex position information we will pass in.
+ "uniform vec4 a_Color; \n" // Per-vertex color information we will pass in.
+ "varying vec4 v_Color; \n" // This will be passed into the fragment shader.
+ "void main() \n" // The entry point for our vertex shader.
+ "{ \n"
+ " v_Color = a_Color; \n" // Pass the color through to the fragment shader.
// It will be interpolated across the triangle.
+ " gl_Position = u_MVPMatrix \n" // gl_Position is a special variable used to store the final position.
+ " * a_Position; \n" // Multiply the vertex by the matrix to get the final point in
+ "} \n"; // normalized screen coordinates.
final String fragmentShader =
"precision mediump float; \n" // Set the default precision to medium. We don't need as high of a
// precision in the fragment shader.
+ "varying vec4 v_Color; \n" // This is the color from the vertex shader interpolated across the
// triangle per fragment.
+ "void main() \n" // The entry point for our fragment shader.
+ "{ \n"
+ " gl_FragColor = v_Color; \n" // Pass the color directly through the pipeline.
+ "} \n";
final float eyeX = 0.0f;
final float eyeY = 0.0f;
final float eyeZ = 25.0f;
final float lookX = 0.0f;
final float lookY = 0.0f;
final float lookZ = 0.0f;
final float upX = 0.0f;
final float upY = 1.0f;
final float upZ = 0.0f;
private boolean mObjLoaded = false;
public RocketArrowRenderer(Context context)
{
mContext = context;
new Thread(new Runnable() {
public void run() {
Resources res = mContext.getResources();
InputStream inputStream = res.openRawResource(R.raw.falcon_heavy_obj);
Obj obj = null;
try {
obj = ObjUtils.convertToRenderable(ObjReader.read(inputStream));
} catch (IOException e) {
e.printStackTrace();
}
mVertices = ObjData.getVertices(obj);
mObjLoaded = true;
}
}).start();
}
#Override
public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
GLES20.glClearColor(0.5f, 0.5f, 0.5f, 0.5f);
Matrix.setLookAtM(mViewMatrix, 0, eyeX, eyeY, eyeZ, lookX, lookY, lookZ, upX, upY, upZ);
int vertexShaderHandle = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
if (vertexShaderHandle != 0)
{
GLES20.glShaderSource(vertexShaderHandle, vertexShader);
GLES20.glCompileShader(vertexShaderHandle);
final int[] compileStatus = new int[1];
GLES20.glGetShaderiv(vertexShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
if (compileStatus[0] == 0)
{
GLES20.glDeleteShader(vertexShaderHandle);
vertexShaderHandle = 0;
}
}
if (vertexShaderHandle == 0)
{
throw new RuntimeException("Error creating vertex shader.");
}
// Load in the vertex shader.
int fragmentShaderHandle = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
if(fragmentShader != null){
GLES20.glShaderSource(fragmentShaderHandle, fragmentShader);
GLES20.glCompileShader(fragmentShaderHandle);
final int[] compileStatus2 = new int[1];
GLES20.glGetShaderiv(vertexShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus2, 0);
if(compileStatus2[0] == 0){
GLES20.glDeleteShader(fragmentShaderHandle);
fragmentShaderHandle = 0;
}
}
if(fragmentShader == null){
throw new RuntimeException("Error creating fragment shader");
}
int programHandle = GLES20.glCreateProgram();
if (programHandle != 0)
{
GLES20.glAttachShader(programHandle, vertexShaderHandle);
GLES20.glAttachShader(programHandle, fragmentShaderHandle);
GLES20.glBindAttribLocation(programHandle, 0, "a_Position");
GLES20.glBindAttribLocation(programHandle, 1, "a_Color");
GLES20.glLinkProgram(programHandle);
final int[] linkStatus = new int[1];
GLES20.glGetProgramiv(programHandle, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] == 0)
{
GLES20.glDeleteProgram(programHandle);
programHandle = 0;
}
}
if (programHandle == 0) {
throw new RuntimeException("Error creating program.");
}
mMVPMatrixHandle = GLES20.glGetUniformLocation(programHandle, "u_MVPMatrix");
mPositionHandle = GLES20.glGetAttribLocation(programHandle, "a_Position");
mColorHandle = GLES20.glGetAttribLocation(programHandle, "a_Color");
GLES20.glUseProgram(programHandle);
}
private float[] mProjectionMatrix = new float[16];
#Override
public void onSurfaceChanged(GL10 glUnused, int width, int height)
{
GLES20.glViewport(0, 0, width, height);
final float ratio = (float) width / height;
final float left = -ratio;
final float right = ratio;
final float bottom = -1.0f;
final float top = 8.0f;
final float near = 1.0f;
final float far = 10.0f;
Matrix.frustumM(mProjectionMatrix, 0, left, right, bottom, top, near, far);
}
#Override
public void onDrawFrame(GL10 glUnused)
{
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
Matrix.setIdentityM(mModelMatrix, 0);
if(mObjLoaded){
draw();
}
}
private void draw() {
int numberOfTriangles = mVertices.position(0).remaining() / 3;
//mVertices.position(mPositionOffset);
GLES20.glVertexAttribPointer(mPositionHandle, mPositionDataSize, GLES20.GL_FLOAT, false,
0, 0);
GLES20.glEnableVertexAttribArray(mPositionHandle);
//and later, in draw
GLES20.glUniform4f(mColorHandle, 1.0f, 0.0f, 0.0f, 1.0f); //red!
Matrix.multiplyMM(mMVPMatrix, 0, mViewMatrix, 0, mModelMatrix, 0);
Matrix.multiplyMM(mMVPMatrix, 0, mProjectionMatrix, 0, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, numberOfTriangles);
}
}

You need to call GLES20.glEnableVertexAttribArray(mPositionHandle);
before using
GLES20.glVertexAttribPointer(mPositionHandle, mPositionDataSize, GLES20.GL_FLOAT, false,
0, 0);

Applying Effects on Video being Played

I am new to the world of Open Gl and I have googled a lot but i am unable to find a way to implement Effects on a video being played. After some research i have finally found a class that can be used to play video on a GLSurfaceView. And i know from Google documentation and that we can apply effects on a video.
By following this post i was able to successfully apply effects on bitmaps. Now i want to do that for my video so any help or pointers is appreciated.
Here is the VideoSurfaceView that i am using to Render Video that is being played
package me.crossle.demo.surfacetexture;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
import android.annotation.SuppressLint;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.MediaPlayer;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.util.Log;
import android.view.Surface;
#SuppressLint("ViewConstructor")
class VideoSurfaceView extends GLSurfaceView {
VideoRender mRenderer;
private MediaPlayer mMediaPlayer = null;
public VideoSurfaceView(Context context, MediaPlayer mp) {
super(context);
setEGLContextClientVersion(2);
mMediaPlayer = mp;
mRenderer = new VideoRender(context);
setRenderer(mRenderer);
}
#Override
public void onResume() {
queueEvent(new Runnable(){
public void run() {
mRenderer.setMediaPlayer(mMediaPlayer);
}});
super.onResume();
}
private static class VideoRender
implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
private static String TAG = "VideoRender";
private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 0.f, 0.f,
1.0f, -1.0f, 0, 1.f, 0.f,
-1.0f, 1.0f, 0, 0.f, 1.f,
1.0f, 1.0f, 0, 1.f, 1.f,
};
private FloatBuffer mTriangleVertices;
private final String mVertexShader =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"}\n";
private final String mFragmentShader =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private int mProgram;
private int mTextureID;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
private SurfaceTexture mSurface;
private boolean updateSurface = false;
private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
private MediaPlayer mMediaPlayer;
public VideoRender(Context context) {
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(mTriangleVerticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
}
public void setMediaPlayer(MediaPlayer player) {
mMediaPlayer = player;
}
#Override
public void onDrawFrame(GL10 glUnused) {
synchronized(this) {
if (updateSurface) {
mSurface.updateTexImage();
mSurface.getTransformMatrix(mSTMatrix);
updateSurface = false;
}
}
GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
}
#Override
public void onSurfaceChanged(GL10 glUnused, int width, int height) {
}
#Override
public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
mProgram = createProgram(mVertexShader, mFragmentShader);
if (mProgram == 0) {
return;
}
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
checkGlError("glGetAttribLocation aPosition");
if (maPositionHandle == -1) {
throw new RuntimeException("Could not get attrib location for aPosition");
}
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
checkGlError("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1) {
throw new RuntimeException("Could not get attrib location for aTextureCoord");
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
checkGlError("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uMVPMatrix");
}
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
checkGlError("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uSTMatrix");
}
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
mTextureID = textures[0];
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError("glBindTexture mTextureID");
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST);
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
/*
* Create the SurfaceTexture that will feed this textureID,
* and pass it to the MediaPlayer
*/
mSurface = new SurfaceTexture(mTextureID);
mSurface.setOnFrameAvailableListener(this);
Surface surface = new Surface(mSurface);
mMediaPlayer.setSurface(surface);
mMediaPlayer.setScreenOnWhilePlaying(true);
surface.release();
try {
mMediaPlayer.prepare();
} catch (IOException t) {
Log.e(TAG, "media player prepare failed");
}
synchronized(this) {
updateSurface = false;
}
mMediaPlayer.start();
}
synchronized public void onFrameAvailable(SurfaceTexture surface) {
updateSurface = true;
}
private int loadShader(int shaderType, String source) {
int shader = GLES20.glCreateShader(shaderType);
if (shader != 0) {
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e(TAG, "Could not compile shader " + shaderType + ":");
Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
}
return shader;
}
private int createProgram(String vertexSource, String fragmentSource) {
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if (vertexShader == 0) {
return 0;
}
int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if (pixelShader == 0) {
return 0;
}
int program = GLES20.glCreateProgram();
if (program != 0) {
GLES20.glAttachShader(program, vertexShader);
checkGlError("glAttachShader");
GLES20.glAttachShader(program, pixelShader);
checkGlError("glAttachShader");
GLES20.glLinkProgram(program);
int[] linkStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] != GLES20.GL_TRUE) {
Log.e(TAG, "Could not link program: ");
Log.e(TAG, GLES20.glGetProgramInfoLog(program));
GLES20.glDeleteProgram(program);
program = 0;
}
}
return program;
}
private void checkGlError(String op) {
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e(TAG, op + ": glError " + error);
throw new RuntimeException(op + ": glError " + error);
}
}
} // End of class VideoRender.
} // End of class VideoSurfaceView.
And Here is my MainActivity
package me.crossle.demo.surfacetexture;
import java.io.File;
import android.app.Activity;
import android.content.res.AssetFileDescriptor;
import android.content.res.Resources;
import android.media.MediaPlayer;
import android.os.Bundle;
import android.os.Environment;
import android.util.Log;
public class MainActivity extends Activity {
private static final String TAG = "MainActivity";
protected Resources mResources;
private VideoSurfaceView mVideoView = null;
private MediaPlayer mMediaPlayer = null;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
mResources = getResources();
mMediaPlayer = new MediaPlayer();
try {
File dir = Environment
.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS);
File file = new File(dir,
"video.mp4");
mMediaPlayer.setDataSource(file.getAbsolutePath());
} catch (Exception e) {
Log.e(TAG, e.getMessage(), e);
}
mVideoView = new VideoSurfaceView(this, mMediaPlayer);
setContentView(mVideoView);
}
#Override
protected void onResume() {
super.onResume();
mVideoView.onResume();
}
}

I have solved the issue and i am posting the answer in case anyone else is also looking for a way to apply different Filters on their video.
After being pointed out in the right direction by Lunero and Fadden i am now able to apply almost all EffectFactory effects to the video being played. Though these effects are only meant for preview purpose and do not change the original video but still they do the job for me.
What i did was that I changed the FragmentShaders code that was applied to the video being rendered and i was able to achieve different effects.
Here is the code for some fragmentShaders.
Black and White Effect
String fragmentShader = "#extension GL_OES_EGL_image_external : require\n"
+ "precision mediump float;\n"
+ "varying vec2 vTextureCoord;\n"
+ "uniform samplerExternalOES sTexture;\n"
+ "void main() {\n"
+ " vec4 color = texture2D(sTexture, vTextureCoord);\n"
+ " float colorR = (color.r + color.g + color.b) / 3.0;\n"
+ " float colorG = (color.r + color.g + color.b) / 3.0;\n"
+ " float colorB = (color.r + color.g + color.b) / 3.0;\n"
+ " gl_FragColor = vec4(colorR, colorG, colorB, color.a);\n"
+ "}\n";
Negative Effect
String fragmentShader = "#extension GL_OES_EGL_image_external : require\n"
+ "precision mediump float;\n"
+ "varying vec2 vTextureCoord;\n"
+ "uniform samplerExternalOES sTexture;\n"
+ "void main() {\n"
+ " vec4 color = texture2D(sTexture, vTextureCoord);\n"
+ " float colorR = (1.0 - color.r) / 1.0;\n"
+ " float colorG = (1.0 - color.g) / 1.0;\n"
+ " float colorB = (1.0 - color.b) / 1.0;\n"
+ " gl_FragColor = vec4(colorR, colorG, colorB, color.a);\n"
+ "}\n";
Original Video without any Effect
Video with Black and White Effect
Video with Negative Effect
If you like to apply more effects then i suggest you look at VidEffects on github. It will help you apply many different effects on your video.

CardboardView opentok stereo renderer

I'm testing out OpenTok for use in a VR app we are building. Using the opentok android SDK samples as a staring point I have modified the CustomVideoRenderer class to support CardboardView and corresponding StereoRenderer interface instead of GLSurfaceView and its Render interface. The switch between CardboardView and GLSurfaceView is almost trivial and works fine in other cases (eg in other apps that do not use opentok). In this case it didn't work at all and only after some tinkering with shaders and texture setup properties I was actually able to get anything to appear on the screen! However, much to my disappointment, my attempt at the renderer code(pasted below) results in a weird half-way-there VR view. The stereoscopic view is rendered with the desired distortion on each eye however the underlying frame images are not in duplicated for each eye -instead it the full screen single image is rendered with the the eye portals sitting over the top! Also the frames are rendered in grayscale! If anyone has got this to work properly I would really appreciate some help :)
Here is the code:
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.ShortBuffer;
import java.util.concurrent.locks.ReentrantLock;
import javax.microedition.khronos.egl.EGLConfig;
import android.content.Context;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.util.Log;
import android.view.View;
import com.google.vrtoolkit.cardboard.CardboardView;
import com.google.vrtoolkit.cardboard.EyeTransform;
import com.google.vrtoolkit.cardboard.HeadTransform;
import com.google.vrtoolkit.cardboard.Viewport;
import com.opentok.android.BaseVideoRenderer;
public class CustomVideoStereoRendererWorking extends BaseVideoRenderer {
Context mContext;
CardboardView mView;
MyRenderer mRenderer;
static class MyRenderer implements CardboardView.StereoRenderer {
int mTextureIds[] = new int[3];
float[] mScaleMatrix = new float[16];
private FloatBuffer mVertexBuffer;
private FloatBuffer mTextureBuffer;
private ShortBuffer mDrawListBuffer;
boolean mVideoFitEnabled = true;
boolean mVideoDisabled = false;
// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static final int TEXTURECOORDS_PER_VERTEX = 2;
static float mXYZCoords[] = { -1.0f, 1.0f, 0.0f, // top left
-1.0f, -1.0f, 0.0f, // bottom left
1.0f, -1.0f, 0.0f, // bottom right
1.0f, 1.0f, 0.0f // top right
};
static float mUVCoords[] = { 0, 0, // top left
0, 1, // bottom left
1, 1, // bottom right
1, 0 }; // top right
private short mVertexIndex[] = { 0, 1, 2, 0, 2, 3 }; // order to draw
// vertices
private final String vertexShaderCode =
"uniform mat4 uMVPMatrix;" +
//"attribute vec4 aPosition;\n" +
"attribute vec2 aPosition;\n" +
"attribute vec2 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() {\n" +
//" gl_Position = uMVPMatrix * aPosition;\n" +
" gl_Position = vec4 ( aPosition.x, aPosition.y, 0.0, 1.0 );\n" + //from working renderer
" vTextureCoord = aTextureCoord;\n" +
"}\n";
/*"attribute vec2 vPosition;\n" +
* "attribute vec2 vTexCoord;\n" +
* "varying vec2 texCoord;\n" +
* "void main() {\n" +
* " texCoord = vTexCoord;\n" +
* " gl_Position = vec4 ( vPosition.x, vPosition.y, 0.0, 1.0 );\n" +
* "}";*/
private final String fragmentShaderCode =
"precision mediump float;\n" +
"uniform sampler2D Ytex;\n" +
"uniform sampler2D Utex,Vtex;\n" +
"varying vec2 vTextureCoord;\n" +
"void main(void) {\n" +
" float nx,ny,r,g,b,y,u,v;\n" +
" mediump vec4 txl,ux,vx;" +
" nx=vTextureCoord[0];\n" +
" ny=vTextureCoord[1];\n" +
" y=texture2D(Ytex,vec2(nx,ny)).r;\n" +
" u=texture2D(Utex,vec2(nx,ny)).r;\n" +
" v=texture2D(Vtex,vec2(nx,ny)).r;\n" +
//" y=1.0-1.1643*(y-0.0625);\n" + // Invert effect
//" y=1.1643*(y-0.0625);\n" + // Normal renderer
" u=u-0.5;\n" +
" v=v-0.5;\n" +
" r=y+1.5958*v;\n" +
" g=y-0.39173*u-0.81290*v;\n" +
" b=y+2.017*u;\n" +
" gl_FragColor=vec4(r,g,b,1.0);\n" +
"}\n";
/*
private final String fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"uniform samplerExternalOES Ytex;\n" +
"uniform samplerExternalOES Utex,Vtex;\n" +
"varying vec2 vTextureCoord;\n" +
"void main(void) {\n" +
" float nx,ny,r,g,b,y,u,v;\n" +
" mediump vec4 txl,ux,vx;" +
" nx=vTextureCoord[0];\n" +
" ny=vTextureCoord[1];\n" +
" y=texture2D(Ytex,vec2(nx,ny)).r;\n" +
" u=texture2D(Utex,vec2(nx,ny)).r;\n" +
" v=texture2D(Vtex,vec2(nx,ny)).r;\n" +
//" y=1.0-1.1643*(y-0.0625);\n" + // Invert effect
//" y=1.1643*(y-0.0625);\n" + // Normal renderer
" u=u-0.5;\n" +
" v=v-0.5;\n" +
" r=y+1.5958*v;\n" +
" g=y-0.39173*u-0.81290*v;\n" +
" b=y+2.017*u;\n" +
" gl_FragColor=vec4(r,g,b,1.0);\n" +
"}\n";
*/
/*"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"uniform samplerExternalOES sTexture;\n" +
"varying vec2 texCoord;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture,texCoord);\n" +
"}"*/
ReentrantLock mFrameLock = new ReentrantLock();
Frame mCurrentFrame;
private int mProgram;
private int mTextureWidth;
private int mTextureHeight;
private int mViewportWidth;
private int mViewportHeight;
public MyRenderer() {
ByteBuffer bb = ByteBuffer.allocateDirect(mXYZCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
mVertexBuffer = bb.asFloatBuffer();
mVertexBuffer.put(mXYZCoords);
mVertexBuffer.position(0);
ByteBuffer tb = ByteBuffer.allocateDirect(mUVCoords.length * 4);
tb.order(ByteOrder.nativeOrder());
mTextureBuffer = tb.asFloatBuffer();
mTextureBuffer.put(mUVCoords);
mTextureBuffer.position(0);
ByteBuffer dlb = ByteBuffer.allocateDirect(mVertexIndex.length * 2);
dlb.order(ByteOrder.nativeOrder());
mDrawListBuffer = dlb.asShortBuffer();
mDrawListBuffer.put(mVertexIndex);
mDrawListBuffer.position(0);
}
public void onSurfaceCreated(EGLConfig config) {
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
mProgram = GLES20.glCreateProgram(); // create empty OpenGL ES
// Program
GLES20.glAttachShader(mProgram, vertexShader); // add the vertex
// shader to program
GLES20.glAttachShader(mProgram, fragmentShader); // add the fragment
// shader to
// program
GLES20.glLinkProgram(mProgram);
int positionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
int textureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
GLES20.glVertexAttribPointer(positionHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT, false, COORDS_PER_VERTEX * 4, mVertexBuffer);
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(textureHandle, TEXTURECOORDS_PER_VERTEX, GLES20.GL_FLOAT, false, TEXTURECOORDS_PER_VERTEX * 4,
mTextureBuffer);
GLES20.glEnableVertexAttribArray(textureHandle);
GLES20.glUseProgram(mProgram);
int i = GLES20.glGetUniformLocation(mProgram, "Ytex");
GLES20.glUniform1i(i, 0); /* Bind Ytex to texture unit 0 */
i = GLES20.glGetUniformLocation(mProgram, "Utex");
GLES20.glUniform1i(i, 1); /* Bind Utex to texture unit 1 */
i = GLES20.glGetUniformLocation(mProgram, "Vtex");
GLES20.glUniform1i(i, 2); /* Bind Vtex to texture unit 2 */
mTextureWidth = 0;
mTextureHeight = 0;
}
static void initializeTexture(int name, int id, int width, int height) {
GLES20.glActiveTexture(name);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, id);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, width, height, 0, GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE, null);
}
void setupTextures(Frame frame) {
if (mTextureIds[0] != 0) {
GLES20.glDeleteTextures(3, mTextureIds, 0);
}
GLES20.glGenTextures(3, mTextureIds, 0);
int w = frame.getWidth();
int h = frame.getHeight();
int hw = (w + 1) >> 1;
int hh = (h + 1) >> 1;
initializeTexture(GLES20.GL_TEXTURE0, mTextureIds[0], w, h);
initializeTexture(GLES20.GL_TEXTURE1, mTextureIds[1], hw, hh);
initializeTexture(GLES20.GL_TEXTURE2, mTextureIds[2], hw, hh);
mTextureWidth = frame.getWidth();
mTextureHeight = frame.getHeight();
}
void updateTextures(Frame frame) {
int width = frame.getWidth();
int height = frame.getHeight();
int half_width = (width + 1) >> 1;
int half_height = (height + 1) >> 1;
int y_size = width * height;
int uv_size = half_width * half_height;
ByteBuffer bb = frame.getBuffer();
// If we are reusing this frame, make sure we reset position and
// limit
bb.clear();
if (bb.remaining() == y_size + uv_size * 2) {
bb.position(0);
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[0]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[1]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size + uv_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE2);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[2]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE, bb);
} else {
mTextureWidth = 0;
mTextureHeight = 0;
}
}
public void onSurfaceChanged(int width, int height) {
GLES20.glViewport(0, 0, width, height);
mViewportWidth = width;
mViewportHeight = height;
}
public void onDrawEye(EyeTransform arg0) {
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
mFrameLock.lock();
if (mCurrentFrame != null && !mVideoDisabled) {
GLES20.glUseProgram(mProgram);
if (mTextureWidth != mCurrentFrame.getWidth() || mTextureHeight != mCurrentFrame.getHeight()) {
setupTextures(mCurrentFrame);
}
updateTextures(mCurrentFrame);
Matrix.setIdentityM(mScaleMatrix, 0);
float scaleX = 1.0f, scaleY = 1.0f;
float ratio = (float) mCurrentFrame.getWidth() / mCurrentFrame.getHeight();
float vratio = (float) mViewportWidth / mViewportHeight;
if (mVideoFitEnabled) {
if (ratio > vratio) {
scaleY = vratio / ratio;
} else {
scaleX = ratio / vratio;
}
} else {
if (ratio < vratio) {
scaleY = vratio / ratio;
} else {
scaleX = ratio / vratio;
}
}
Matrix.scaleM(mScaleMatrix, 0, scaleX * (mCurrentFrame.isMirroredX() ? -1.0f : 1.0f), scaleY, 1);
int mMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false, mScaleMatrix, 0);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);
}
mFrameLock.unlock();
}
public void displayFrame(Frame frame) {
mFrameLock.lock();
if (this.mCurrentFrame != null) {
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = frame;
mFrameLock.unlock();
}
public static int loadShader(int type, String shaderCode) {
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e("Shader", "Could not compile vshader");
Log.v("Shader", "Could not compile vshader:" + GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
return shader;
}
public void disableVideo(boolean b) {
mFrameLock.lock();
mVideoDisabled = b;
if (mVideoDisabled) {
if (this.mCurrentFrame != null) {
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = null;
}
mFrameLock.unlock();
}
public void enableVideoFit(boolean enableVideoFit) {
mVideoFitEnabled = enableVideoFit;
}
public void onFinishFrame(Viewport arg0) {
// TODO Auto-generated method stub
}
public void onNewFrame(HeadTransform arg0) {
// TODO Auto-generated method stub
}
public void onRendererShutdown() {
// TODO Auto-generated method stub
}
};
public CustomVideoStereoRendererWorking(Context context) {
this.mContext = context;
mView = new CardboardView(context);
mView.setEGLContextClientVersion(2);
mView.setVRModeEnabled(true);
mRenderer = new MyRenderer();
mView.setRenderer(mRenderer);
mView.setRenderMode(CardboardView.RENDERMODE_WHEN_DIRTY);
}
#Override
public void onFrame(Frame frame) {
mRenderer.displayFrame(frame);
mView.requestRender();
}
#Override
public void setStyle(String key, String value) {
if (BaseVideoRenderer.STYLE_VIDEO_SCALE.equals(key)) {
if (BaseVideoRenderer.STYLE_VIDEO_FIT.equals(value)) {
mRenderer.enableVideoFit(true);
} else if (BaseVideoRenderer.STYLE_VIDEO_FILL.equals(value)) {
mRenderer.enableVideoFit(false);
}
}
}
#Override
public void onVideoPropertiesChanged(boolean videoEnabled) {
mRenderer.disableVideo(!videoEnabled);
}
#Override
public View getView() {
return mView;
}
#Override
public void onPause() {
mView.onPause();
}
#Override
public void onResume() {
mView.onResume();
}
}

Point is not showing in opengles 2.0

I am trying to learn OpenGLES 2.0 in android in depth. I am trying to draw a Simple Point at the center of screen but somehow , the point is not showing .
public class MyRenderer implements GLSurfaceView.Renderer {
Context context;
private int mProgram;
private final float[] mViewMatrix=new float[16];
private float[] mProjectionMatrix=new float[16];
private final float[] mPointModelMatrix=new float[16];
private final float[] mMVPMatrix=new float[16];
private final float[] mPointPosInModelSpace = new float[] {0.0f, 0.0f, 0.0f, 1.0f};
private final float[] mPointPosInWorldSpace = new float[4];
private final float[] mPointPosInEyeSpace = new float[4];
private int pointMVPMatrixHandle;
private int pointPositionHandle;
public MyRenderer(Context context){
this.context=context;
}
public void onDrawFrame(GL10 arg0) {
// TODO Auto-generated method stub
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
Matrix.setIdentityM(mPointModelMatrix, 0);
Matrix.translateM(mPointModelMatrix, 0, 0.0f, 0.0f, -3.0f);
Matrix.multiplyMV(mPointPosInWorldSpace, 0, mPointModelMatrix, 0, mPointPosInModelSpace, 0);
Matrix.multiplyMV(mPointPosInEyeSpace, 0, mViewMatrix, 0, mPointPosInWorldSpace, 0);
GLES20.glUseProgram(mProgram);
drawPoint();
}
private void drawPoint(){
pointMVPMatrixHandle=GLES20.glGetUniformLocation(mProgram, "u_MVPMatrix");
pointPositionHandle=GLES20.glGetAttribLocation(mProgram, "a_position");
GLES20.glVertexAttrib3f(pointPositionHandle, mPointPosInEyeSpace[0], mPointPosInEyeSpace[1], mPointPosInEyeSpace[2]);
GLES20.glDisableVertexAttribArray(pointPositionHandle);
Matrix.multiplyMM(mMVPMatrix, 0, mViewMatrix, 0, mPointModelMatrix, 0);
Matrix.multiplyMM(mMVPMatrix, 0, mProjectionMatrix, 0, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(pointMVPMatrixHandle, 1, false, mMVPMatrix, 0);
// Draw the point.
GLES20.glDrawArrays(GLES20.GL_POINTS, 0, 1);
}
public void onSurfaceChanged(GL10 gl, int width, int height) {
// TODO Auto-generated method stub
GLES20.glViewport(0, 0, width, height);
final float ratio=(float)width/height;
Log.d("Ratio is", " "+ratio);
Log.d("Width is"," "+width+" and "+height);
final float left = -ratio;
final float right = ratio;
final float bottom= -1.0f;
final float top = 1.0f;
final float near = 1.0f;
final float far = 10.0f;
Matrix.frustumM(mProjectionMatrix, 0, left, right, bottom, top, near, far);
}
public void onSurfaceCreated(GL10 gl, EGLConfig config) {
// TODO Auto-generated method stub
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
GLES20.glEnable(GLES20.GL_CULL_FACE);
GLES20.glEnable(GLES20.GL_DEPTH_TEST);
float eyeX=0.0f;
float eyeY=0.0f;
float eyeZ=-0.5f;
float centerX=0.0f;
float centerY=0.0f;
float centerZ=-5.0f;
float upX=0.0f;
float upY=1.0f;
float upZ=0.0f;
Matrix.setLookAtM(mViewMatrix, 0, eyeX, eyeY, eyeZ, centerX, centerY, centerZ, upX, upY, upZ);
final String vertexShader=this.getVertexShader();
final String fragmentShader=this.getFragmentShader();
final int vertexShaderHandle=ShaderHelper.compileShader(GLES20.GL_VERTEX_SHADER, vertexShader);
final int fragmentShaderHandle=ShaderHelper.compileShader(GLES20.GL_FRAGMENT_SHADER, fragmentShader);
mProgram=ShaderHelper.createAndLinkProgram(vertexShaderHandle, fragmentShaderHandle, new String[]{"a_position"});
}
private String getVertexShader(){
final String vertexShader="uniform mat4 u_MVPMatrix; \n"
+ "attribute vec4 a_Position; \n"
+ "void main() \n"
+ "{ \n"
+ " gl_Position = u_MVPMatrix \n"
+ " * a_Position; \n"
+ " gl_PointSize = 10.0; \n"
+ "} \n";
return vertexShader;
}
private String getFragmentShader(){
final String fragmentShader="precision mediump float; \n"
+ "void main() \n"
+ "{ \n"
+ " gl_FragColor = vec4(1.0, \n"
+ " 1.0, 1.0, 1.0); \n"
+ "} \n";
return fragmentShader;
}
}
I am pretty much sure that I am pointing the both eye and point in negative Z direction(further from viewer) . The point should show up as in vertex Shader, point size is 10.0 but somehow, no luck.
Note: ShaderHelper is a class with static method compileShader and createAndLinkProgram where code for compiling shaders, checking for errors are written. (No Errors in Program)

Here is a class I built based around your code for displaying the point. Since you will probably use more than one point eventually it is better to push the points into a vertex array as shown.
package point.example.point;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
public class PointRenderer implements GLSurfaceView.Renderer
{
private float[] mModelMatrix = new float[16];
private float[] mViewMatrix = new float[16];
private float[] mProjectionMatrix = new float[16];
private float[] mMVPMatrix = new float[16];
private int mMVPMatrixHandle;
private int mPositionHandle;
float[] vertices = {
0.0f,0.0f,0.0f
};
FloatBuffer vertexBuf;
#Override
public void onSurfaceCreated(GL10 glUnused, EGLConfig config)
{
vertexBuf = ByteBuffer.allocateDirect(vertices.length * 4).order(ByteOrder.nativeOrder()).asFloatBuffer();
vertexBuf.put(vertices).position(0);
// Set the background clear color to gray.
GLES20.glClearColor(0.5f, 0.5f, 0.5f, 0.5f);
float eyeX=0.0f;
float eyeY=0.0f;
float eyeZ=0.0f;
float centerX=0.0f;
float centerY=0.0f;
float centerZ=-5.0f;
float upX=0.0f;
float upY=1.0f;
float upZ=0.0f;
// Set the view matrix. This matrix can be said to represent the camera position.
// NOTE: In OpenGL 1, a ModelView matrix is used, which is a combination of a model and
// view matrix. In OpenGL 2, we can keep track of these matrices separately if we choose.
Matrix.setLookAtM(mViewMatrix, 0, eyeX, eyeY, eyeZ, centerX, centerY, centerZ, upX, upY, upZ);
final String vertexShader =
"uniform mat4 u_MVPMatrix; \n"
+ "attribute vec4 a_Position; \n"
+ "void main() \n"
+ "{ \n"
+ " gl_Position = u_MVPMatrix \n"
+ " * a_Position; \n"
+ " gl_PointSize = 10.0; \n"
+ "} \n";
final String fragmentShader =
"precision mediump float; \n"
+ "void main() \n"
+ "{ \n"
+ " gl_FragColor = vec4(1.0, \n"
+ " 1.0, 1.0, 1.0); \n"
+ "} \n";
// Load in the vertex shader.
int vertexShaderHandle = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
if (vertexShaderHandle != 0)
{
// Pass in the shader source.
GLES20.glShaderSource(vertexShaderHandle, vertexShader);
// Compile the shader.
GLES20.glCompileShader(vertexShaderHandle);
// Get the compilation status.
final int[] compileStatus = new int[1];
GLES20.glGetShaderiv(vertexShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
// If the compilation failed, delete the shader.
if (compileStatus[0] == 0)
{
GLES20.glDeleteShader(vertexShaderHandle);
vertexShaderHandle = 0;
}
}
if (vertexShaderHandle == 0)
{
throw new RuntimeException("Error creating vertex shader.");
}
// Load in the fragment shader shader.
int fragmentShaderHandle = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
if (fragmentShaderHandle != 0)
{
// Pass in the shader source.
GLES20.glShaderSource(fragmentShaderHandle, fragmentShader);
// Compile the shader.
GLES20.glCompileShader(fragmentShaderHandle);
// Get the compilation status.
final int[] compileStatus = new int[1];
GLES20.glGetShaderiv(fragmentShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
// If the compilation failed, delete the shader.
if (compileStatus[0] == 0)
{
GLES20.glDeleteShader(fragmentShaderHandle);
fragmentShaderHandle = 0;
}
}
if (fragmentShaderHandle == 0)
{
throw new RuntimeException("Error creating fragment shader.");
}
// Create a program object and store the handle to it.
int programHandle = GLES20.glCreateProgram();
if (programHandle != 0)
{
// Bind the vertex shader to the program.
GLES20.glAttachShader(programHandle, vertexShaderHandle);
// Bind the fragment shader to the program.
GLES20.glAttachShader(programHandle, fragmentShaderHandle);
// Bind attributes
GLES20.glBindAttribLocation(programHandle, 0, "a_Position");
// Link the two shaders together into a program.
GLES20.glLinkProgram(programHandle);
// Get the link status.
final int[] linkStatus = new int[1];
GLES20.glGetProgramiv(programHandle, GLES20.GL_LINK_STATUS, linkStatus, 0);
// If the link failed, delete the program.
if (linkStatus[0] == 0)
{
GLES20.glDeleteProgram(programHandle);
programHandle = 0;
}
}
if (programHandle == 0)
{
throw new RuntimeException("Error creating program.");
}
// Set program handles. These will later be used to pass in values to the program.
mMVPMatrixHandle = GLES20.glGetUniformLocation(programHandle, "u_MVPMatrix");
mPositionHandle = GLES20.glGetAttribLocation(programHandle, "a_Position");
// Tell OpenGL to use this program when rendering.
GLES20.glUseProgram(programHandle);
}
#Override
public void onSurfaceChanged(GL10 glUnused, int width, int height)
{
// Set the OpenGL viewport to the same size as the surface.
GLES20.glViewport(0, 0, width, height);
// Create a new perspective projection matrix. The height will stay the same
// while the width will vary as per aspect ratio.
final float ratio = (float) width / height;
final float left = -ratio;
final float right = ratio;
final float bottom = -1.0f;
final float top = 1.0f;
final float near = 1.0f;
final float far = 100.0f;
Matrix.frustumM(mProjectionMatrix, 0, left, right, bottom, top, near, far);
}
#Override
public void onDrawFrame(GL10 glUnused)
{
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
Matrix.setIdentityM(mModelMatrix, 0);
//Push to the distance - note this will have no effect on a point size
Matrix.translateM(mModelMatrix, 0, 0.0f, 0.0f, -5.0f);
Matrix.multiplyMV(mMVPMatrix, 0, mViewMatrix, 0, mModelMatrix, 0);
Matrix.multiplyMV(mMVPMatrix, 0, mProjectionMatrix, 0, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false, mMVPMatrix, 0);
//Send the vertex
GLES20.glVertexAttribPointer(mPositionHandle, 3, GLES20.GL_FLOAT, false, 0, vertexBuf);
GLES20.glEnableVertexAttribArray(mPositionHandle);
//Draw the point
GLES20.glDrawArrays(GLES20.GL_POINTS, 0, 1);
}
}