Develop Reference

How to provide a shader function to perform the alpha masking for video

I am working on Vr app using virocore library in android. I have to show video over sphere. The video which i have to implement is not actually video but the two frames provided are the colour frame (left) and the alpha mask (right) frame. I have not worked with openGl but seems like I will need to provide a shader function to perform the alpha masking.
I have used this for shader Adding transparency to a video from black and white (and gray) alpha information video images
but how can i use it with OpenGL in on draw method? or If there is any way in virocore using which i can do alpha masking. I have tried chroma filtering method in virocore but that makes whole video transparent.
public class VideoSurfaceView extends GLSurfaceView {
VideoRender mRenderer;
private MediaPlayer mMediaPlayer = null;
public VideoSurfaceView(Context context, MediaPlayer mp) {
super(context);
setEGLContextClientVersion(2);
mMediaPlayer = mp;
mRenderer = new VideoRender(context);
this.getHolder().setFormat(PixelFormat.RGB_565);
this.getHolder().setFormat(PixelFormat.TRANSPARENT);
setEGLConfigChooser(8,8,8,8,16,0);
setEGLContextClientVersion(2);
setRenderer(mRenderer);
}
#Override
public void onResume() {
Log.e("onResume ", "onResume");
queueEvent(new Runnable(){
public void run() {
Log.e("runnable ", "runnable");
mRenderer.setMediaPlayer(mMediaPlayer);
}});
super.onResume();
}
private static class VideoRender
implements Renderer, SurfaceTexture.OnFrameAvailableListener, MediaPlayer.OnPreparedListener {
private static String TAG = "VideoRender";
private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 0.f, 0.f,
1.0f, -1.0f, 0, 1.f, 0.f,
-1.0f, 1.0f, 0, 0.f, 1.f,
1.0f, 1.0f, 0, 1.f, 1.f,
};
private FloatBuffer mTriangleVertices;
private static final String mVertexShader =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 position;\n" +
"attribute vec4 inputTextureCoordinate;\n" +
" \n" +
"varying vec2 textureCoordinate;\n" +
"varying vec2 textureCoordinate2;\n" +
" \n" +
"void main()\n" +
"{\n" +
" gl_Position = uMVPMatrix * position;\n" +
" vec4 texCoord = uSTMatrix * inputTextureCoordinate;\n"+
"textureCoordinate = vec2(inputTextureCoordinate.x * 0.5, inputTextureCoordinate.y);\n" +
" textureCoordinate2 = vec2(inputTextureCoordinate.x * 0.5 + 0.5, inputTextureCoordinate.y);\n" +
"}";
public static final String mFragmentShader = "#extension GL_OES_EGL_image_external : require\n"+
"varying highp vec2 textureCoordinate;\n"+
"varying highp vec2 textureCoordinate2;\n"+
"uniform samplerExternalOES inputImageTexture;\n" +
"void main() {\n"+
" lowp vec4 rgbcolor = texture2D(inputImageTexture, textureCoordinate);\n"+
" lowp vec4 alphaValue = texture2D(inputImageTexture, textureCoordinate2);\n"+
" if (alphaValue.g < 0.5)\n"+
" discard;\n"+
" gl_FragColor = vec4(rgbcolor.rgb, 1.0);\n"+
"}";
private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private int mProgram;
private int mTextureID;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
private SurfaceTexture mSurface;
private boolean updateSurface = false;
private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
private MediaPlayer mMediaPlayer;
public VideoRender(Context context) {
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(mTriangleVerticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
}
public void setMediaPlayer(MediaPlayer player) {
mMediaPlayer = player;
}
#Override
public void onDrawFrame(GL10 glUnused) {
synchronized(this) {
if (updateSurface) {
mSurface.updateTexImage();
mSurface.getTransformMatrix(mSTMatrix);
updateSurface = false;
}
}
GLES20.glClearColor(0.0f, 0.0f, 0.0f, .0f);
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT|GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glEnable(GLES20.GL_BLEND);
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
}
#Override
public void onSurfaceChanged(GL10 glUnused, int width, int height) {
}
#Override
public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
mProgram = createProgram(mVertexShader, mFragmentShader);
if (mProgram == 0) {
return;
}
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "position");
checkGlError("glGetAttribLocation aPosition");
if (maPositionHandle == -1) {
throw new RuntimeException("Could not get attrib location for aPosition");
}
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "inputTextureCoordinate");
checkGlError("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1) {
throw new RuntimeException("Could not get attrib location for aTextureCoord");
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
checkGlError("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uMVPMatrix");
}
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
checkGlError("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uSTMatrix");
}
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
mTextureID = textures[0];
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError("glBindTexture mTextureID");
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST);
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
/*
* Create the SurfaceTexture that will feed this textureID,
* and pass it to the MediaPlayer
*/
mSurface = new SurfaceTexture(mTextureID);
mSurface.setOnFrameAvailableListener(this);
Log.e("surface ", "surface");
Surface surface = new Surface(mSurface);
mMediaPlayer.setSurface(surface);
mMediaPlayer.setScreenOnWhilePlaying(true);
surface.release();
mMediaPlayer.setOnPreparedListener(this);
mMediaPlayer.prepareAsync();
synchronized(this) {
updateSurface = false;
}
}
synchronized public void onFrameAvailable(SurfaceTexture surface) {
updateSurface = true;
}
private int loadShader(int shaderType, String source) {
int shader = GLES20.glCreateShader(shaderType);
if (shader != 0) {
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e(TAG, "Could not compile shader " + shaderType + ":");
Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
}
return shader;
}
private int createProgram(String vertexSource, String fragmentSource) {
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if (vertexShader == 0) {
return 0;
}
int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if (pixelShader == 0) {
return 0;
}
int program = GLES20.glCreateProgram();
if (program != 0) {
GLES20.glAttachShader(program, vertexShader);
checkGlError("glAttachShader");
GLES20.glAttachShader(program, pixelShader);
checkGlError("glAttachShader");
GLES20.glLinkProgram(program);
int[] linkStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] != GLES20.GL_TRUE) {
Log.e(TAG, "Could not link program: ");
Log.e(TAG, GLES20.glGetProgramInfoLog(program));
GLES20.glDeleteProgram(program);
program = 0;
}
}
return program;
}
private void checkGlError(String op) {
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e(TAG, op + ": glError " + error);
throw new RuntimeException(op + ": glError " + error);
}
}
#Override
public void onPrepared(MediaPlayer mediaPlayer) {
mediaPlayer.start();
}
} // End of class VideoRender.
} // End of class VideoSurfaceView.
With this code, video looks inverted

If you want to discard fragments, then you can use the discard keyword in the fragment shader.
e.g. discard all fragments with an alpha value less than 0.5:
void main()
{
lowp vec4 rgbcolor = texture2D(inputImageTexture, textureCoordinate);
lowp vec4 alphaValue = texture2D(inputImageTexture, textureCoordinate2);
if (alphaValue.g < 0.5)
discard;
gl_FragColor = vec4(rgbcolor.rgb, 1.0);
}
See also OpenGL ES Shading Language 1.00 Specification; 6.4 Jumps; page 58:
The discard keyword is only allowed within fragment shaders. It can be used within a fragment shader to abandon the operation on the current fragment. This keyword causes the fragment to be discarded and no updates to any buffers will occur. It would typically be used within a conditional statement, for example:
if (intensity < 0.0)
discard;
In reference to the comment
As there is no matrix so what will i use in GLES20.glGetUniformLocation?
Of course you can add the matrices to the vertex shader:
attribute vec4 inputTextureCoordinate;
varying vec2 textureCoordinate;
varying vec2 textureCoordinate2;
uniform mat4 matMVP;
uniform mat4 matST;
void main()
{
gl_Position = matMVP * position;
vec4 texCoord = matST * inputTextureCoordinate;
textureCoordinate = vec2(texCoord.x * 0.5, 1.0 - texCoord.y);
textureCoordinate2 = vec2(texCoord.x * 0.5 + 0.5, 1.0 - texCoord.y);
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "matMVP");
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "matST");
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

Render 3d Objects into Cameraview

I tried to develop a mobile cardboard application, which renders 3d objects into a camera view (some kind of ar).
I used this project and tried to render a simple cube in the camera:
https://github.com/Sveder/CardboardPassthrough/
I didn't get it working, the background is always black or the app wrecked.
I would be very grateful for any help or suggestions.
Thanks
Thats what i have
Origin CardboardPassthrough
here is the working code, with the cubes
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.hardware.Camera;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.os.Bundle;
import android.os.Vibrator;
import android.util.Log;
import android.view.KeyEvent;
import com.google.vrtoolkit.cardboard.*;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
public class Card extends CardboardActivity implements CardboardView.StereoRenderer, SurfaceTexture.OnFrameAvailableListener {
private static final float CAMERA_Z = 0.01f;
private static final float TIME_DELTA = 0.3f;
private static final float YAW_LIMIT = 0.12f;
private static final float PITCH_LIMIT = 0.12f;
//---------------------------------------------------
private int intCurrentI = -1;
private int intCurrentI1 = -1;
//---------------------------------------------------
// We keep the light always position just above the user.
private final float[] mLightPosInWorldSpace = new float[]{0.0f, 2.0f, 0.0f, 1.0f};
private final float[] mLightPosInEyeSpace = new float[4];
private static final int COORDS_PER_VERTEX = 3;
private final WorldLayoutData DATA = new WorldLayoutData();
private FloatBuffer mCubeVertices;
private FloatBuffer mCubeColors;
private FloatBuffer mCubeFoundColors;
private FloatBuffer mCubeNormals;
private int mGlProgram;
private int mPositionParam;
private int mNormalParam;
private int mColorParam;
private int mModelViewProjectionParam;
private int mLightPosParam;
private int mModelViewParam;
private int mModelParam;
private int mIsFloorParam;
private float[] mModelCube;
private float[] mCamera;
private float[] mView;
private float[] mHeadView;
private float[] mModelViewProjection;
private float[] mModelView;
private float[] mModelCube2;
private float[] mModelFloor;
private float mObjectDistance = 12f;
private float mFloorDepth = 20f;
private Vibrator mVibrator;
private CardboardOverlayView mOverlayView;
private SurfaceTexture surface;
private Camera camera;
private static final int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
private final String vertexShaderCode =
"attribute vec4 position;" +
"attribute vec2 inputTextureCoordinate;" +
"varying vec2 textureCoordinate;" +
"void main()" +
"{" +
"gl_Position = position;" +
"textureCoordinate = inputTextureCoordinate;" +
"}";
private final String fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;" +
"varying vec2 textureCoordinate; \n" +
"uniform samplerExternalOES s_texture; \n" +
"void main(void) {" +
" gl_FragColor = texture2D( s_texture, textureCoordinate );\n" +
//" gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);\n" +
"}";
private int texture;
private CardboardView cardboardView;
/**
* Converts a raw text file, saved as a resource, into an OpenGL ES shader
*
* #param type The type of shader we will be creating.
* #param resId The resource ID of the raw text file about to be turned into a shader.
* #return
*/
private int loadGLShader(int type, int resId) {
String code = readRawTextFile(resId);
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, code);
GLES20.glCompileShader(shader);
// Get the compilation status.
final int[] compileStatus = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
// If the compilation failed, delete the shader.
if (compileStatus[0] == 0) {
GLES20.glDeleteShader(shader);
shader = 0;
}
if (shader == 0) {
throw new RuntimeException("Error creating shader.");
}
return shader;
}
/**
* Checks if we've had an error inside of OpenGL ES, and if so what that error is.
*
* #param func
*/
private static void checkGLError(String func) {
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
throw new RuntimeException(func + ": glError " + error);
}
}
/**
* Sets the view to our CardboardView and initializes the transformation matrices we will use
* to render our scene.
*
* #param savedInstanceState
*/
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.common_ui);
cardboardView = (CardboardView) findViewById(R.id.cardboard_view);
/*********************/
cardboardView.setEGLConfigChooser(8, 8, 8, 8, 16, 0);
/*********************/
cardboardView.setRenderer(this);
setCardboardView(cardboardView);
/* 2014-10-16 */
mModelCube2 = new float[16];
/* 2014-10-16 */
mModelCube = new float[16];
mCamera = new float[16];
mView = new float[16];
mModelViewProjection = new float[16];
mModelView = new float[16];
mModelFloor = new float[16];
mHeadView = new float[16];
mVibrator = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE);
mOverlayView = (CardboardOverlayView) findViewById(R.id.overlay);
mOverlayView.show3DToast("VR-Test");
}
#Override
public void onRendererShutdown() {
}
#Override
public void onSurfaceChanged(int width, int height) {
}
/**
* Creates the buffers we use to store information about the 3D world. OpenGL doesn't use Java
* arrays, but rather needs data in a format it can understand. Hence we use ByteBuffers.
*
* #param config The EGL configuration used when creating the surface.
*/
#Override
public void onSurfaceCreated(EGLConfig config) {
GLES20.glClearColor(0.1f, 0.1f, 0.1f, 0.5f); // Dark background so text shows up well
ByteBuffer bbVertices = ByteBuffer.allocateDirect(DATA.CUBE_COORDS.length * 4);
bbVertices.order(ByteOrder.nativeOrder());
mCubeVertices = bbVertices.asFloatBuffer();
mCubeVertices.put(DATA.CUBE_COORDS);
mCubeVertices.position(0);
ByteBuffer bbColors = ByteBuffer.allocateDirect(DATA.CUBE_COLORS.length * 4);
bbColors.order(ByteOrder.nativeOrder());
mCubeColors = bbColors.asFloatBuffer();
mCubeColors.put(DATA.CUBE_COLORS);
mCubeColors.position(0);
ByteBuffer bbFoundColors = ByteBuffer.allocateDirect(DATA.CUBE_FOUND_COLORS.length * 4);
bbFoundColors.order(ByteOrder.nativeOrder());
mCubeFoundColors = bbFoundColors.asFloatBuffer();
mCubeFoundColors.put(DATA.CUBE_FOUND_COLORS);
mCubeFoundColors.position(0);
ByteBuffer bbNormals = ByteBuffer.allocateDirect(DATA.CUBE_NORMALS.length * 4);
bbNormals.order(ByteOrder.nativeOrder());
mCubeNormals = bbNormals.asFloatBuffer();
mCubeNormals.put(DATA.CUBE_NORMALS);
mCubeNormals.position(0);
int vertexShader = loadGLShader(GLES20.GL_VERTEX_SHADER, R.raw.light_vertex);
int gridShader = loadGLShader(GLES20.GL_FRAGMENT_SHADER, R.raw.grid_fragment);
mGlProgram = GLES20.glCreateProgram();
GLES20.glAttachShader(mGlProgram, vertexShader);
GLES20.glAttachShader(mGlProgram, gridShader);
GLES20.glLinkProgram(mGlProgram);
texture = createTexture();
startCamera(texture);
GLES20.glEnable(GLES20.GL_DEPTH_TEST);
// Object first appears directly in front of user
Matrix.setIdentityM(mModelCube, 0);
Matrix.translateM(mModelCube, 0, 0, 0, -mObjectDistance);
Matrix.setIdentityM(mModelCube2, 0);
Matrix.translateM(mModelCube2, 0, -10.0f, -10.0f, -mObjectDistance - 12.0f);
Matrix.setIdentityM(mModelFloor, 0);
Matrix.translateM(mModelFloor, 0, 0, -mFloorDepth, 0); // Floor appears below user
checkGLError("onSurfaceCreated");
}
/**
* Converts a raw text file into a string.
*
* #param resId The resource ID of the raw text file about to be turned into a shader.
* #return
*/
private String readRawTextFile(int resId) {
InputStream inputStream = getResources().openRawResource(resId);
try {
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
StringBuilder sb = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("\n");
}
reader.close();
return sb.toString();
} catch (IOException e) {
e.printStackTrace();
}
return "";
}
/**
* Prepares OpenGL ES before we draw a frame.
*
* #param headTransform The head transformation in the new frame.
*/
#Override
public void onNewFrame(HeadTransform headTransform) {
GLES20.glUseProgram(mGlProgram);
mModelViewProjectionParam = GLES20.glGetUniformLocation(mGlProgram, "u_MVP");
mLightPosParam = GLES20.glGetUniformLocation(mGlProgram, "u_LightPos");
mModelViewParam = GLES20.glGetUniformLocation(mGlProgram, "u_MVMatrix");
mModelParam = GLES20.glGetUniformLocation(mGlProgram, "u_Model");
mIsFloorParam = GLES20.glGetUniformLocation(mGlProgram, "u_IsFloor");
// Build the Model part of the ModelView matrix.
Matrix.rotateM(mModelCube, 0, TIME_DELTA, 0.5f, 0.5f, 1.0f);
Matrix.rotateM(mModelCube2, 0, TIME_DELTA, 0.5f, 0.5f, 1.0f);
//--------------------------------------
// Build the camera matrix and apply it to the ModelView.
Matrix.setLookAtM(mCamera, 0, 0.0f, 0.0f, CAMERA_Z, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f);
headTransform.getHeadView(mHeadView, 0);
checkGLError("onReadyToDraw");
}
/**
* Draws a frame for an eye. The transformation for that eye (from the camera) is passed in as
* a parameter.
*
* #param transform The transformations to apply to render this eye.
*/
#Override
public void onDrawEye(EyeTransform transform) {
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
mPositionParam = GLES20.glGetAttribLocation(mGlProgram, "a_Position");
mNormalParam = GLES20.glGetAttribLocation(mGlProgram, "a_Normal");
mColorParam = GLES20.glGetAttribLocation(mGlProgram, "a_Color");
GLES20.glEnableVertexAttribArray(mPositionParam);
GLES20.glEnableVertexAttribArray(mNormalParam);
GLES20.glEnableVertexAttribArray(mColorParam);
checkGLError("mColorParam");
// Apply the eye transformation to the camera.
Matrix.multiplyMM(mView, 0, transform.getEyeView(), 0, mCamera, 0);
// Set the position of the light
Matrix.multiplyMV(mLightPosInEyeSpace, 0, mView, 0, mLightPosInWorldSpace, 0);
GLES20.glUniform3f(mLightPosParam, mLightPosInEyeSpace[0], mLightPosInEyeSpace[1],
mLightPosInEyeSpace[2]);
// Build the ModelView and ModelViewProjection matrices
// for calculating cube position and light.
Matrix.multiplyMM(mModelView, 0, mView, 0, mModelCube, 0);
Matrix.multiplyMM(mModelViewProjection, 0, transform.getPerspective(), 0,
mModelView, 0);
drawCube(1);
//--------------------------------------
Matrix.multiplyMM(mModelView, 0, mView, 0, mModelCube2, 0);
Matrix.multiplyMM(mModelViewProjection, 0, transform.getPerspective(), 0,
mModelView, 0);
drawCube(0);
//--------------------------------------
}
#Override
public void onFinishFrame(Viewport viewport) {
}
public void drawCube(int i1) {
// This is not the floor!
GLES20.glUniform1f(mIsFloorParam, 0f);
// Set the Model in the shader, used to calculate lighting
if (i1 == 1) {
GLES20.glUniformMatrix4fv(mModelParam, 1, false, mModelCube, 0);
} else if (i1 == 0) {
//--2014-10-16 ??--------------------------------
GLES20.glUniformMatrix4fv(mModelParam, 1, false, mModelCube2, 0);
//-------------------------------------------------
}
// Set the ModelView in the shader, used to calculate lighting
GLES20.glUniformMatrix4fv(mModelViewParam, 1, false, mModelView, 0);
// Set the position of the cube
GLES20.glVertexAttribPointer(mPositionParam, COORDS_PER_VERTEX, GLES20.GL_FLOAT,
false, 0, mCubeVertices);
// Set the ModelViewProjection matrix in the shader.
GLES20.glUniformMatrix4fv(mModelViewProjectionParam, 1, false, mModelViewProjection, 0);
// Set the normal positions of the cube, again for shading
GLES20.glVertexAttribPointer(mNormalParam, 3, GLES20.GL_FLOAT,
false, 0, mCubeNormals);
if (isLookingAtObject(i1)) {
GLES20.glVertexAttribPointer(mColorParam, 4, GLES20.GL_FLOAT, false,
0, mCubeFoundColors);
if (i1 == 1)
intCurrentI1 = i1;
else
intCurrentI1 = -1;
intCurrentI = i1;
System.out.println("drawCube->intCurrentI2:" + intCurrentI);
} else {
GLES20.glVertexAttribPointer(mColorParam, 4, GLES20.GL_FLOAT, false,
0, mCubeColors);
intCurrentI = -1;
}
GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, 36);
checkGLError("Drawing cube");
if (intCurrentI1 != -1)
intCurrentI = intCurrentI1;
System.out.println("drawCube_out_if->intCurrentI4:" + intCurrentI);
}
private boolean isLookingAtObject(int i1) {
float[] initVec = {0, 0, 0, 1.0f};
float[] objPositionVec = new float[4];
System.out.println("isLookingAtObject1->i1:" + i1);
// Convert object space to camera space. Use the headView from onNewFrame.
if (i1 == 1) {
Matrix.multiplyMM(mModelView, 0, mHeadView, 0, mModelCube, 0);
Matrix.multiplyMV(objPositionVec, 0, mModelView, 0, initVec, 0);
intCurrentI = i1;
} else if (i1 == 0) {
Matrix.multiplyMM(mModelView, 0, mHeadView, 0, mModelCube2, 0);
Matrix.multiplyMV(objPositionVec, 0, mModelView, 0, initVec, 0);
intCurrentI = i1;
}
float pitch = (float) Math.atan2(objPositionVec[1], -objPositionVec[2]);
float yaw = (float) Math.atan2(objPositionVec[0], -objPositionVec[2]);
boolean bool1 = (Math.abs(pitch) < PITCH_LIMIT) && (Math.abs(yaw) < YAW_LIMIT);
return bool1;
}
public void startCamera(int texture) {
surface = new SurfaceTexture(texture);
surface.setOnFrameAvailableListener(this);
camera = Camera.open();
try {
camera.setPreviewTexture(surface);
camera.startPreview();
} catch (IOException ioe) {
Log.w("MainActivity", "CAM LAUNCH FAILED");
}
}
static private int createTexture() {
int[] texture = new int[1];
GLES20.glGenTextures(1, texture, 0);
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture[0]);
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_LINEAR);
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_MAG_FILTER, GL10.GL_LINEAR);
GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_WRAP_S, GL10.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_WRAP_T, GL10.GL_CLAMP_TO_EDGE);
return texture[0];
}
#Override
public void onFrameAvailable(SurfaceTexture surfaceTexture) {
this.cardboardView.requestRender();
}
}
Edit 11.07.16
I cut the vertex in half, to see posible vertex behind it.
But i think the problem is vertex and fragment-shader
The Cube is only showing up, when im using this vertex and fragment-shader
simple_fragment.shader
precision mediump float;
varying vec4 v_Color;
void main() {
gl_FragColor = v_Color;
}
light_vertex.shader
uniform mat4 u_MVP;
uniform mat4 u_MVMatrix;
uniform mat4 u_Model;
uniform vec3 u_LightPos;
uniform float u_IsFloor;
attribute vec4 a_Position;
attribute vec4 a_Color;
attribute vec3 a_Normal;
varying vec4 v_Color;
varying vec3 v_Grid;
varying float v_isFloor;
void main()
{
vec3 modelVertex = vec3(u_Model * a_Position);
v_Grid = modelVertex;
vec3 modelViewVertex = vec3(u_MVMatrix * a_Position);
vec3 modelViewNormal = vec3(u_MVMatrix * vec4(a_Normal, 0.0));
float distance = length(u_LightPos - modelViewVertex);
vec3 lightVector = normalize(u_LightPos - modelViewVertex);
float diffuse = max(dot(modelViewNormal, lightVector), 0.5 );
diffuse = diffuse * (1.0 / (1.0 + (0.00001 * distance * distance)));
v_Color = a_Color * diffuse;
gl_Position = u_MVP * a_Position;
v_isFloor = u_IsFloor;
}
and the camera is only showing up when im using this shaders
vertex.shader
attribute vec4 position;
attribute vec2 inputTextureCoordinate;
varying vec2 textureCoordinate;
void main()
{
gl_Position = position;
textureCoordinate = inputTextureCoordinate;
}
fragment.shader
#extension GL_OES_EGL_image_external : require
precision mediump float;
varying vec2 textureCoordinate;
varying vec4 v_Color;
uniform samplerExternalOES s_texture;
void main(void) {
gl_FragColor = texture2D( s_texture, textureCoordinate );
}
i don't know how to fix the shaders

I would suggest that you disable glEnable(GL_DEPTH_TEST) to render background objects in the foreground then switch between shaders using:
GLES20.glUseProgram();
For your example this might be:
#Override
public void onSurfaceCreated(EGLConfig config) {
Log.i(TAG, "onSurfaceCreated");
GLES20.glClearColor(0.1f, 0.1f, 0.1f, 0.5f); // Dark background so text shows up well
ByteBuffer bb = ByteBuffer.allocateDirect(squareVertices.length * 4);
bb.order(ByteOrder.nativeOrder());
vertexBuffer = bb.asFloatBuffer();
vertexBuffer.put(squareVertices);
vertexBuffer.position(0);
ByteBuffer dlb = ByteBuffer.allocateDirect(drawOrder.length * 2);
dlb.order(ByteOrder.nativeOrder());
drawListBuffer = dlb.asShortBuffer();
drawListBuffer.put(drawOrder);
drawListBuffer.position(0);
ByteBuffer bb2 = ByteBuffer.allocateDirect(textureVertices.length * 4);
bb2.order(ByteOrder.nativeOrder());
textureVerticesBuffer = bb2.asFloatBuffer();
textureVerticesBuffer.put(textureVertices);
textureVerticesBuffer.position(0);
//Cube
ByteBuffer bbVertices = ByteBuffer.allocateDirect(DATA.CUBE_COORDS.length * 4);
bbVertices.order(ByteOrder.nativeOrder());
mCubeVertices = bbVertices.asFloatBuffer();
mCubeVertices.put(DATA.CUBE_COORDS);
mCubeVertices.position(0);
ByteBuffer bbColors = ByteBuffer.allocateDirect(DATA.CUBE_COLORS.length * 4);
bbColors.order(ByteOrder.nativeOrder());
mCubeColors = bbColors.asFloatBuffer();
mCubeColors.put(DATA.CUBE_COLORS);
mCubeColors.position(0);
ByteBuffer bbFoundColors = ByteBuffer.allocateDirect(DATA.CUBE_FOUND_COLORS.length * 4);
bbFoundColors.order(ByteOrder.nativeOrder());
mCubeFoundColors = bbFoundColors.asFloatBuffer();
mCubeFoundColors.put(DATA.CUBE_FOUND_COLORS);
mCubeFoundColors.position(0);
ByteBuffer bbNormals = ByteBuffer.allocateDirect(DATA.CUBE_NORMALS.length * 4);
bbNormals.order(ByteOrder.nativeOrder());
mCubeNormals = bbNormals.asFloatBuffer();
mCubeNormals.put(DATA.CUBE_NORMALS);
mCubeNormals.position(0);
int vertexShader = loadGLShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = loadGLShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
mCameraProgram = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mCameraProgram, vertexShader); // add the vertex shader to program
GLES20.glAttachShader(mCameraProgram, fragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mCameraProgram);
vertexShader = loadGLShader(GLES20.GL_VERTEX_SHADER, R.raw.light_vertex);
fragmentShader = loadGLShader(GLES20.GL_FRAGMENT_SHADER, R.raw.grid_fragment);
mCubeProgram = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mCubeProgram, vertexShader); // add the vertex shader to program
GLES20.glAttachShader(mCubeProgram, fragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mCubeProgram);
texture = createTexture();
startCamera(texture);
Matrix.setIdentityM(mModelCube, 0);
Matrix.translateM(mModelCube, 0, 0, 0, -mObjectDistance);
checkGLError("onSurfaceCreated");
}
#Override
public void onNewFrame(HeadTransform headTransform) {
GLES20.glUseProgram(mCubeProgram);
mModelViewProjectionParam = GLES20.glGetUniformLocation(mCubeProgram, "u_MVP");
mLightPosParam = GLES20.glGetUniformLocation(mCubeProgram, "u_LightPos");
mModelViewParam = GLES20.glGetUniformLocation(mCubeProgram, "u_MVMatrix");
mModelParam = GLES20.glGetUniformLocation(mCubeProgram, "u_Model");
mIsFloorParam = GLES20.glGetUniformLocation(mCubeProgram, "u_IsFloor");
// Build the Model part of the ModelView matrix.
Matrix.rotateM(mModelCube, 0, TIME_DELTA, 0.5f, 0.5f, 1.0f);
// Build the camera matrix and apply it to the ModelView.
Matrix.setLookAtM(mCamera, 0, 0.0f, 0.0f, CAMERA_Z, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f);
headTransform.getHeadView(mHeadView, 0);
GLES20.glUseProgram(mCameraProgram);
float[] mtx = new float[16];
//GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
surface.updateTexImage();
surface.getTransformMatrix(mtx);
}
#Override
public void onDrawEye(EyeTransform transform) {
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
//Camera
GLES20.glUseProgram(mCameraProgram);
GLES20.glActiveTexture(GL_TEXTURE_EXTERNAL_OES);
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture);
mPositionHandle = GLES20.glGetAttribLocation(mCameraProgram, "position");
GLES20.glEnableVertexAttribArray(mPositionHandle);
GLES20.glVertexAttribPointer(mPositionHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT,
false, vertexStride, vertexBuffer);
mTextureCoordHandle = GLES20.glGetAttribLocation(mCameraProgram, "inputTextureCoordinate");
GLES20.glEnableVertexAttribArray(mTextureCoordHandle);
GLES20.glVertexAttribPointer(mTextureCoordHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT,
false, vertexStride, textureVerticesBuffer);
mColorHandle = GLES20.glGetAttribLocation(mCameraProgram, "s_texture");
GLES20.glDrawElements(GLES20.GL_TRIANGLES, drawOrder.length,
GLES20.GL_UNSIGNED_SHORT, drawListBuffer);
// Disable vertex array
GLES20.glDisableVertexAttribArray(mPositionHandle);
GLES20.glDisableVertexAttribArray(mTextureCoordHandle);
//cube
GLES20.glUseProgram(mCubeProgram);
mPositionParam = GLES20.glGetAttribLocation(mCubeProgram, "a_Position");
mNormalParam = GLES20.glGetAttribLocation(mCubeProgram, "a_Normal");
mColorParam = GLES20.glGetAttribLocation(mCubeProgram, "a_Color");
GLES20.glEnableVertexAttribArray(mPositionParam);
GLES20.glEnableVertexAttribArray(mNormalParam);
GLES20.glEnableVertexAttribArray(mColorParam);
// Set the position of the light
Matrix.multiplyMV(mLightPosInEyeSpace, 0, mView, 0, mLightPosInWorldSpace, 0);
GLES20.glUniform3f(mLightPosParam, mLightPosInEyeSpace[0], mLightPosInEyeSpace[1],
mLightPosInEyeSpace[2]);
Matrix.multiplyMM(mModelView, 0, mView, 0, mModelCube, 0);
Matrix.multiplyMM(mModelViewProjection, 0, transform.getPerspective(), 0,
mModelView, 0);
drawCube(1);
Matrix.multiplyMM(mView, 0, transform.getEyeView(), 0, mCamera, 0);

Just an open suggestion. I developed an AR project for a University assignment I had, a couple of months ago. In my case I used a tool called Vuforia and integrated it with Unity for it to work on mobile devices. You can get your app to work on both Android and iOS devices. The latest releases of both Unity and Vuforia both help in the development of AR projects, since it is currently at its hype.
Depending on the work you need you AR project to perform, and your experience with Unity, the learning curve increases. In my case, I augmented the construction of a roof for a neolithical site. I also used a third party software called makehuman and Blender to create a walking human being. In all of my project, I didn't need to touch a line of code at all :)
Hope this helps.

Android - How to renderer video texture on 3D cube with opengl-es?

I draw a 3D cube by using opengl-es
https://db.tt/ktcbwtnD //this is the picture, because I'm new in stackoverflow so i cant paste picture
and I also renderer video by reference this code
public class VideoTextureRender implements Renderer, SurfaceTexture.OnFrameAvailableListener {
private static String TAG = "VideoRender";
private static final int FLOAT_SIZE_BYTES = 4;
public static final int BYTES_PER_FLOAT = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private static int count = 1;
private final float[] mTriangleVerticesData = {
// X, Y, Z, U, V
-0.5f, -0.5f, 0, 0.f, 0.f,
0.5f, -0.5f, 0, 1.f, 0.f,
-0.5f, 0.5f, 0, 0.f, 1.f,
0.5f, 0.5f, 0, 1.f, 1.f,
};
private FloatBuffer mTriangleVertices;
private final String mVertexShader =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"}\n";
private final String mFragmentShader =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private int mProgram;
private int mTextureID;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
private GLSurfaceViewActivity mGLSurfaceViewActivity;
private SurfaceTexture mSurface;
private boolean updateSurface = false;
public static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
private MediaPlayer mMediaPlayer;
private Context context;
public VideoTextureRender(Context Context) {
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(mTriangleVerticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
}
public void setMediaPlayer(MediaPlayer player) {
mMediaPlayer = player;
}
#Override
public void onDrawFrame(GL10 glUnused) {
synchronized(this) {
if (updateSurface) {
mSurface.updateTexImage();
mSurface.getTransformMatrix(mSTMatrix);
updateSurface = false;
}
}
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
//rotateM(mSTMatrix, 0, count, 1f, 1f, 0f);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
}
#Override
public void onSurfaceChanged(GL10 glUnused, int width, int height) {
}
#Override
public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
mProgram = createProgram(mVertexShader, mFragmentShader);
if (mProgram == 0) {
return;
}
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
checkGlError("glGetAttribLocation aPosition");
if (maPositionHandle == -1) {
throw new RuntimeException("Could not get attrib location for aPosition");
}
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
checkGlError("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1) {
throw new RuntimeException("Could not get attrib location for aTextureCoord");
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
checkGlError("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uMVPMatrix");
}
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
checkGlError("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uSTMatrix");
}
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
mTextureID = textures[0];
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError("glBindTexture mTextureID");
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST);
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
mSurface = new SurfaceTexture(mTextureID);
mSurface.setOnFrameAvailableListener(this);
Surface surface = new Surface(mSurface);
mMediaPlayer.setSurface(surface);
mMediaPlayer.setScreenOnWhilePlaying(true);
surface.release();
try {
mMediaPlayer.prepare();
} catch (IOException t) {
Log.e(TAG, "media player prepare failed");
synchronized(this) {
updateSurface = false;
}
}
mMediaPlayer.start();
}
synchronized public void onFrameAvailable(SurfaceTexture surface) {
updateSurface = true;
}
private int loadShader(int shaderType, String source) {
int shader = GLES20.glCreateShader(shaderType);
if (shader != 0) {
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e(TAG, "Could not compile shader " + shaderType + ":");
Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
}
return shader;
}
private int createProgram(String vertexSource, String fragmentSource) {
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if (vertexShader == 0) {
return 0;
}
int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if (pixelShader == 0) {
return 0;
}
int program = GLES20.glCreateProgram();
if (program != 0) {
GLES20.glAttachShader(program, vertexShader);
checkGlError("glAttachShader");
GLES20.glAttachShader(program, pixelShader);
checkGlError("glAttachShader");
GLES20.glLinkProgram(program);
int[] linkStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] != GLES20.GL_TRUE) {
Log.e(TAG, "Could not link program: ");
Log.e(TAG, GLES20.glGetProgramInfoLog(program));
GLES20.glDeleteProgram(program);
program = 0;
}
}
return program;
}
private void checkGlError(String op) {
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e(TAG, op + ": glError " + error);
throw new RuntimeException(op + ": glError " + error);
}
}
}
https://db.tt/rDjbtYjE
My problem is how to set video as a texture and bind video texture on cube ?
Thank You!

Using video with OpenGL ES is more complex than you probably expect. The key issues are:
Video frames must be converted from YUV to RGB color space. This is best done in 2.0 or 3.0 GLSL shader code or using external textures.
The glTexImage2D() function is too slow to handle HD video frame rates because it copies the data. Use the EGL Image Extension instead named EGL_NATIVE_BUFFER_ANDROID.
The decoding of video frames must be synchronized with OpenGL ES's texture loading. This can be done with the fence sync extensions.
This answer has some links with further information to get you started.

Android OpenGL Camera Preview Issue

I'm working on an Android camera app that modifies the camera feed and displays it live on the screen. I have it working and doing what I want perfectly on my DROID RAZR MAXX running 4.3, and it works perfect on other phones, but unfortunately I have ran into a problem on several phones and am unable to track down the issue.
I've attached a screenshot showing what the issue is.
It's very hard to tell what the green "artifacts" are, but it almost looks like it's blocks from the camera feed from when it first turned on. The colors flicker, but the shapes inside the blocks don't really change.
I've stripped out everything that isn't needed and cleaned up the code as best I can, but I honestly have zero clue as to why this is happening, especially since it seems to work on some phones fine, while other phones it doesn't.
If I need to give more information just comment and I will add it!
CameraActivity.java
public class CameraActivity extends Activity
{
private MyGLSurfaceView glSurfaceView;
private MyCamera mCamera;
#Override
protected void onCreate(Bundle savedInstanceState)
{
super.onCreate(savedInstanceState);
mCamera = new MyCamera();
glSurfaceView = new MyGLSurfaceView(this, mCamera);
setContentView(glSurfaceView);
}
#Override
protected void onPause()
{
super.onPause();
mCamera.stop();
}
}
MyCamera.java
public class MyCamera
{
private final static String LOG_TAG = "MyCamera";
private Camera mCamera;
private Parameters mCameraParams;
private Boolean running = false;
void start(SurfaceTexture surface)
{
Log.v(LOG_TAG, "Starting Camera");
mCamera = Camera.open(0);
mCameraParams = mCamera.getParameters();
Log.v(LOG_TAG, mCameraParams.getPreviewSize().width + " x " + mCameraParams.getPreviewSize().height);
try {
mCamera.setPreviewTexture(surface);
mCamera.startPreview();
running = true;
} catch (IOException e) {
e.printStackTrace();
}
}
void stop()
{
if (running) {
Log.v(LOG_TAG, "Stopping Camera");
mCamera.stopPreview();
mCamera.release();
running = false;
}
}
}
MyGLSurfaceView.java
class MyGLSurfaceView extends GLSurfaceView implements Renderer
{
private final static String LOG_TAG = "MyGLSurfaceView";
private MyCamera mCamera;
private SurfaceTexture mSurface;
private DirectVideo mDirectVideo;
public MyGLSurfaceView(Context context, MyCamera camera)
{
super(context);
mCamera = camera;
setEGLContextClientVersion(2);
setRenderer(this);
}
#Override
public void onDrawFrame(GL10 gl)
{
float[] mtx = new float[16];
mSurface.updateTexImage();
mSurface.getTransformMatrix(mtx);
mDirectVideo.draw();
}
#Override
public void onSurfaceChanged(GL10 gl, int width, int height)
{
Log.v(LOG_TAG, "Surface Changed");
GLES20.glViewport(0, 0, width, height);
}
#Override
public void onSurfaceCreated(GL10 gl, EGLConfig config)
{
Log.v(LOG_TAG, "Surface Created");
int texture = createTexture();
mDirectVideo = new DirectVideo(texture);
mSurface = new SurfaceTexture(texture);
mCamera.start(mSurface);
}
private int createTexture()
{
int[] textures = new int[1];
// generate one texture pointer and bind it as an external texture.
GLES20.glGenTextures(1, textures, 0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textures[0]);
// No mip-mapping with camera source.
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_LINEAR);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL10.GL_TEXTURE_MAG_FILTER, GL10.GL_LINEAR);
// Clamp to edge is only option.
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL10.GL_TEXTURE_WRAP_S, GL10.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL10.GL_TEXTURE_WRAP_T, GL10.GL_CLAMP_TO_EDGE);
return textures[0];
}
public static int loadShader(int type, String shaderCode){
// create a vertex shader type (GLES20.GL_VERTEX_SHADER)
// or a fragment shader type (GLES20.GL_FRAGMENT_SHADER)
int shader = GLES20.glCreateShader(type);
// add the source code to the shader and compile it
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
}
DirectVideo.java
public class DirectVideo
{
private final String vertexShaderCode =
"attribute vec4 vPosition;" +
"attribute vec2 inputTextureCoordinate;" +
"varying vec2 textureCoordinate;" +
"void main()" +
"{"+
"gl_Position = vPosition;"+
"textureCoordinate = inputTextureCoordinate;" +
"}";
private final String fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require\n"+
"precision mediump float;" +
"varying vec2 textureCoordinate;\n" +
"uniform samplerExternalOES s_texture;\n" +
"void main() {" +
" gl_FragColor = texture2D( s_texture, textureCoordinate );\n" +
"}";
private FloatBuffer vertexBuffer, textureVerticesBuffer;
private ShortBuffer drawListBuffer;
private final int mProgram;
private int mPositionHandle;
private int mTextureCoordHandle;
private short drawOrder[] = { 0, 1, 2, 0, 2, 3 }; // order to draw vertices
// number of coordinates per vertex in this array
private static final int COORDS_PER_VERTEX = 2;
private final int vertexStride = COORDS_PER_VERTEX * 4; // 4 bytes per vertex
static float squareCoords[] = {
-1.0f, 1.0f,
-1.0f, -1.0f,
1.0f, -1.0f,
1.0f, 1.0f,
};
static float textureVertices[] = {
0.0f, 1.0f,
1.0f, 1.0f,
1.0f, 0.0f,
0.0f, 0.0f,
};
private int texture;
public DirectVideo(int texture)
{
this.texture = texture;
// initialize vertex byte buffer for shape coordinates
ByteBuffer bb = ByteBuffer.allocateDirect(squareCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
vertexBuffer = bb.asFloatBuffer();
vertexBuffer.put(squareCoords);
vertexBuffer.position(0);
// initialize byte buffer for the draw list
ByteBuffer dlb = ByteBuffer.allocateDirect(drawOrder.length * 2);
dlb.order(ByteOrder.nativeOrder());
drawListBuffer = dlb.asShortBuffer();
drawListBuffer.put(drawOrder);
drawListBuffer.position(0);
ByteBuffer bb2 = ByteBuffer.allocateDirect(textureVertices.length * 4);
bb2.order(ByteOrder.nativeOrder());
textureVerticesBuffer = bb2.asFloatBuffer();
textureVerticesBuffer.put(textureVertices);
textureVerticesBuffer.position(0);
int vertexShader = MyGLSurfaceView.loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = MyGLSurfaceView.loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
mProgram = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mProgram, vertexShader); // add the vertex shader to program
GLES20.glAttachShader(mProgram, fragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mProgram); // creates OpenGL ES program executables
}
public void draw()
{
GLES20.glUseProgram(mProgram);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texture);
// get handle to vertex shader's vPosition member
mPositionHandle = GLES20.glGetAttribLocation(mProgram, "vPosition");
// Enable a handle to the triangle vertices
GLES20.glEnableVertexAttribArray(mPositionHandle);
// Prepare the <insert shape here> coordinate data
GLES20.glVertexAttribPointer(mPositionHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT, false, vertexStride, vertexBuffer);
mTextureCoordHandle = GLES20.glGetAttribLocation(mProgram, "inputTextureCoordinate");
GLES20.glEnableVertexAttribArray(mTextureCoordHandle);
GLES20.glVertexAttribPointer(mTextureCoordHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT, false, vertexStride, textureVerticesBuffer);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, drawOrder.length, GLES20.GL_UNSIGNED_SHORT, drawListBuffer);
// Disable vertex array
GLES20.glDisableVertexAttribArray(mPositionHandle);
GLES20.glDisableVertexAttribArray(mTextureCoordHandle);
}
}

In the onDrawFrame method you get a transformation matrix, but you don't use it.
This matrix should be used to transform texture coordinates.
For details see the documentation of the SurfaceTexture class.
Here is the fix:
pass the matrix to the the draw method:
#Override
public void onDrawFrame(GL10 gl)
{
float[] mtx = new float[16];
mSurface.updateTexImage();
mSurface.getTransformMatrix(mtx);
mDirectVideo.draw(mtx);
}
Add the following method to the DirectVideo class:
private float[] transformTextureCoordinates( float[] coords, float[] matrix)
{
float[] result = new float[ coords.length ];
float[] vt = new float[4];
for ( int i = 0 ; i < coords.length ; i += 2 ) {
float[] v = { coords[i], coords[i+1], 0 , 1 };
Matrix.multiplyMV(vt, 0, matrix, 0, v, 0);
result[i] = vt[0];
result[i+1] = vt[1];
}
return result;
}
In the draw method Convert the textureVertices list before adding to the buffer ( you should do this conversion at every paint since the matrix can change):
textureVerticesBuffer.clear();
textureVerticesBuffer.put( transformTextureCoordinates( textureVertices, mtx ));
textureVerticesBuffer.position(0);
An alternative solution is if you pass the matrix to the shader.

implements SurfaceTexture.OnFrameAvailableListener
#Override
public void onFrameAvailable(SurfaceTexture surfaceTexture) {
// TODO Auto-generated method stub
}
setOnFrameAvailableListener(this);

Using SurfaceTexture in Android

I want to play a video into an OpenGL texture on XOOM using Android 3.0.
I have come across SurfaceTexture in the goole developer docs which has been added in API 11
http://developer.android.com/reference/android/graphics/SurfaceTexture.html
The documentation talks about using GL_TEXTURE_EXTERNAL_OES instead of GL_TEXTURE_2D, I cannot find any reference to this define in the latest android-ndk-r5c, it's not defined in in the ndk and it only goes up to platform-9, so I guess I would need platform-11 and there appears to be no android.opengl.GLES20Ext which I guess is where it would reside on the java side.
I have the latest of all released android tools from google and I cannot find any extra things from NVidia or Motorola on their developer sites.
Do anyone have a working example of using SurfaceTexture to either put the camera image or video onto an OpenGL texture? and/or know what I am missing to be able to use this new functionality?

A demo locate https://github.com/crossle/MediaPlayerSurface
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.MediaPlayer;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.util.Log;
import android.view.Surface;
class VideoSurfaceView extends GLSurfaceView {
VideoRender mRenderer;
private MediaPlayer mMediaPlayer = null;
public VideoSurfaceView(Context context, MediaPlayer mp) {
super(context);
setEGLContextClientVersion(2);
mMediaPlayer = mp;
mRenderer = new VideoRender(context);
setRenderer(mRenderer);
}
#Override
public void onResume() {
queueEvent(new Runnable(){
public void run() {
mRenderer.setMediaPlayer(mMediaPlayer);
}});
super.onResume();
}
private static class VideoRender
implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
private static String TAG = "VideoRender";
private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 0.f, 0.f,
1.0f, -1.0f, 0, 1.f, 0.f,
-1.0f, 1.0f, 0, 0.f, 1.f,
1.0f, 1.0f, 0, 1.f, 1.f,
};
private FloatBuffer mTriangleVertices;
private final String mVertexShader =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"}\n";
private final String mFragmentShader =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private int mProgram;
private int mTextureID;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
private SurfaceTexture mSurface;
private boolean updateSurface = false;
private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
private MediaPlayer mMediaPlayer;
public VideoRender(Context context) {
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(mTriangleVerticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
}
public void setMediaPlayer(MediaPlayer player) {
mMediaPlayer = player;
}
public void onDrawFrame(GL10 glUnused) {
synchronized(this) {
if (updateSurface) {
mSurface.updateTexImage();
mSurface.getTransformMatrix(mSTMatrix);
updateSurface = false;
}
}
GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
}
public void onSurfaceChanged(GL10 glUnused, int width, int height) {
}
public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
mProgram = createProgram(mVertexShader, mFragmentShader);
if (mProgram == 0) {
return;
}
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
checkGlError("glGetAttribLocation aPosition");
if (maPositionHandle == -1) {
throw new RuntimeException("Could not get attrib location for aPosition");
}
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
checkGlError("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1) {
throw new RuntimeException("Could not get attrib location for aTextureCoord");
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
checkGlError("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uMVPMatrix");
}
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
checkGlError("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1) {
throw new RuntimeException("Could not get attrib location for uSTMatrix");
}
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
mTextureID = textures[0];
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError("glBindTexture mTextureID");
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST);
GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
/*
* Create the SurfaceTexture that will feed this textureID,
* and pass it to the MediaPlayer
*/
mSurface = new SurfaceTexture(mTextureID);
mSurface.setOnFrameAvailableListener(this);
Surface surface = new Surface(mSurface);
mMediaPlayer.setSurface(surface);
surface.release();
try {
mMediaPlayer.prepare();
} catch (IOException t) {
Log.e(TAG, "media player prepare failed");
}
synchronized(this) {
updateSurface = false;
}
mMediaPlayer.start();
}
synchronized public void onFrameAvailable(SurfaceTexture surface) {
updateSurface = true;
}
private int loadShader(int shaderType, String source) {
int shader = GLES20.glCreateShader(shaderType);
if (shader != 0) {
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e(TAG, "Could not compile shader " + shaderType + ":");
Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
}
return shader;
}
private int createProgram(String vertexSource, String fragmentSource) {
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if (vertexShader == 0) {
return 0;
}
int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if (pixelShader == 0) {
return 0;
}
int program = GLES20.glCreateProgram();
if (program != 0) {
GLES20.glAttachShader(program, vertexShader);
checkGlError("glAttachShader");
GLES20.glAttachShader(program, pixelShader);
checkGlError("glAttachShader");
GLES20.glLinkProgram(program);
int[] linkStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] != GLES20.GL_TRUE) {
Log.e(TAG, "Could not link program: ");
Log.e(TAG, GLES20.glGetProgramInfoLog(program));
GLES20.glDeleteProgram(program);
program = 0;
}
}
return program;
}
private void checkGlError(String op) {
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e(TAG, op + ": glError " + error);
throw new RuntimeException(op + ": glError " + error);
}
}
} // End of class VideoRender.
} // End of class VideoSurfaceView.

Example code. This creates a new external texture suitable for use in a SurfaceTexture, then wraps it in said SurfaceTexture and passes it to the camera as a surface to write the preview into.
int[] textures = new int[1];
// generate one texture pointer and bind it as an external texture.
GLES20.glGenTextures(1, textures, 0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textures[0]);
// No mip-mapping with camera source.
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_MIN_FILTER,
GL10.GL_LINEAR);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_MAG_FILTER, GL10.GL_LINEAR);
// Clamp to edge is only option.
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_WRAP_S, GL10.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_WRAP_T, GL10.GL_CLAMP_TO_EDGE);
int texture_id = textures[0];
SurfaceTexture mTexture = new SurfaceTexture(texture_id);
mTexture.setOnFrameAvailableListener(this);
Camera cam = Camera.open();
cam.setPreviewTexture(mTexture);
Note that if you render this object, you'll need to be careful: It's NOT a 2D texture, so it needs special treatment in the shader.

If using API level 11 to 14 you can just define GL_TEXTURE_EXTERNAL_OES yourself by placing
private static final int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
in your code. This seems to work just fine for me.

NVIDIA has a full and working sample in their TEGRA Android Developer pack. The sample is written pure Java and runs in Standard Eclipse+Android SDK - so you just need to install the samples.
The name of the project is surfacetexture (or something like it). It works nice!
Hopes it helps you.