Develop Reference

How to correctly pass YUV_420_888 Image Buffer from Java through JNI to OpenCV, accounting for stride/padding

I'm trying to convert the image data from an Android device from YUV_420_888 to an RGB matrix on the C++ side. On some devices, this is working flawlessly. On a Note 10, the image comes out looking like this:
My guess here is that the stride is causing this issue. How do I remove this extra data and then pass the correct buffer through JNI?
Here is the Java code:
IntBuffer rgb = image.getPlanes()[0].getBuffer().asIntBuffer();
NativeLib.passImageBuffer(rgb);
And here is the C++ code:
cv::Mat outputRGB;
cv::cvtColor(cv::Mat(height+height/2, width, CV_8UC1, inputRGB), outputRGB, CV_YUV2BGR_NV21);
I've tried some different image formats on the C++ side, but they all come back with the same band on the side of the screen.
I've implemented this answer, in order to remove the extra padding, but the image that is passed ends up being completely green. Do some corresponding edits need to be made to the C++ code? I've tried using a 3 channel format, but that crashes at runtime. I'm thinking that since passing the buffer works with the 1 channel matrix on phones that have 8 bits per pixel, that it should be possible to do that with the note 10?
Image.Plane Y = image.getPlanes()[0];
Image.Plane U = image.getPlanes()[1];
Image.Plane V = image.getPlanes()[2];
int[] rgbBytes = new int[image.getHeight()*image.getWidth()*4];
int idx = 0;
ByteBuffer yBuffer = Y.getBuffer();
int yPixelStride = Y.getPixelStride();
int yRowStride = Y.getRowStride();
ByteBuffer uBuffer = U.getBuffer();
int uPixelStride = U.getPixelStride();
int uRowStride = U.getRowStride();
ByteBuffer vBuffer = V.getBuffer();
int vPixelStride = V.getPixelStride();
int vRowStride = V.getRowStride();
ByteBuffer rgbBuffer = ByteBuffer.allocateDirect(rgb.limit());
for (int row = 0; row < image.getHeight(); row++) {
for (int col = 0; col < image.getWidth(); col++) {
int y = yBuffer.get(col*yPixelStride + row*yRowStride) & 0xff;
int u = uBuffer.get(col/2*uPixelStride + row/2*uRowStride) & 0xff;
int v = vBuffer.get(col/2*vPixelStride + row/2*vRowStride) & 0xff;
int y1 = ((19077 << 8) * y) >> 16;
int r = (y1 + (((26149 << 8) * v) >> 16) - 14234) >> 6;
int g = (y1 - (((6419 << 8) * u) >> 16) - (((13320 << 8) * v) >> 16) + 8708) >> 6;
int b = (y1 + (((33050 << 8) * u) >> 16) - 17685) >> 6;
if (r < 0) r = 0;
if (g < 0) g = 0;
if (b < 0) b = 0;
if (r > 255) r = 255;
if (g > 255) g = 255;
if (b > 255) b = 255;
byte pixel = (byte)(0xff000000 + b + 256 * (g + 256 * r));
rgbBuffer.put(pixel);
}
}

Look at this repo
https://github.com/quickbirdstudios/yuvToMat/
It supports different formats (YUV420, NV12) and variety of pixel and row strides.

Android decodeYUV420SP results in green images?

Ok so my question is pretty much identical to this:
Converting preview frame to bitmap
However his answer is no good, and trying to use it doesn't solve my problem.
So what I'm trying to do at the moment is to send each frame as a bitmap to a method to detect if there are any faces, but first I need to create a bitmap which means I have to use the decodeYUV420sp method, which doesn't seem to be working properly and all my images just come out as a green and yellow tie dye looking image. Here is my code:
This is from onPreviewFrame:
Parameters parameters = cam.getParameters();
Integer width = parameters.getPreviewSize().width;
Integer height = parameters.getPreviewSize().height;
Log.i("preview size: ", String.valueOf(width) + "x" + String.valueOf(height));
int[] mIntArray = new int[width*height];
// Decode Yuv data to integer array
decodeYUV420SP(mIntArray, data, width, height);
//Initialize the bitmap, with the replaced color
Bitmap bmp = Bitmap.createBitmap(mIntArray, width, height, Bitmap.Config.ARGB_8888);
saveImage(bmp);
This is decodeYUV method:
static public void decodeYUV420SP(int[] rgba, byte[] yuv420sp, int width,
int height) {
final int frameSize = width * height;
for (int j = 0, yp = 0; j < height; j++) {
int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
for (int i = 0; i < width; i++, yp++) {
int y = (0xff & ((int) yuv420sp[yp])) - 16;
if (y < 0)
y = 0;
if ((i & 1) == 0) {
v = (0xff & yuv420sp[uvp++]) - 128;
u = (0xff & yuv420sp[uvp++]) - 128;
}
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0)
r = 0;
else if (r > 262143)
r = 262143;
if (g < 0)
g = 0;
else if (g > 262143)
g = 262143;
if (b < 0)
b = 0;
else if (b > 262143)
b = 262143;
// rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) &
// 0xff00) | ((b >> 10) & 0xff);
// rgba, divide 2^10 ( >> 10)
rgba[yp] = ((r << 14) & 0xff000000) | ((g << 6) & 0xff0000)
| ((b >> 2) | 0xff00);
}
}
}
and this is the method I'm calling to save the bitmaps to see what they look like:
private void saveImage(Bitmap bmp) {
File myDir=new File("/sdcard/saved_images");
myDir.mkdirs();
Random generator = new Random();
int n = 10000;
n = generator.nextInt(n);
String fname = "Image-"+ n +".jpg";
File file = new File (myDir, fname);
if (file.exists ()) file.delete ();
try {
FileOutputStream out = new FileOutputStream(file);
bmp.compress(Bitmap.CompressFormat.JPEG, 90, out);
out.flush();
out.close();
} catch (Exception e) {
e.printStackTrace();
}
}
Here is a resulting image:
https://docs.google.com/drawings/d/1kyIvb4oHHInW_c71mjfFSVCxVopBgBWX3k1OR_nMgRA/edit

The key point here is that there are a (large) number of different YUV encodings, and an even larger list of names used for them. A lot of information about all the different variants (and their names) is given by fourcc, although 420SP isn't mentionned explicitly. Looking here, it looks like:
420P is the same as YV12. 'P' appears to stand for planar: there are three 'planes' of data one after the other: Y, U and then V. (Or, in YV21, which is also a 420P encoding, Y, V and then U.)
420SP is the same as NV12 (which is the same as NV21 but with U and V swapped around). 'SP' appears to stand for 'semi-planar', and so '420SP' could, technically, refer to either NV21 or NV12.
In this case, therefore, you are decoding NV12 (as opposed to NV21) and so the order of U and V is swapped around compared to the answer you quote in your answer. In case it helps, I have provided some code here.

Ok so the problem was the decodeYUV method which I got from a different stackoverflow post here:
Converting YUV->RGB(Image processing)->YUV during onPreviewFrame in android? didn't quite work.
But I replaced that with what I think must be the original decodeYUV method from here:
http://code.google.com/p/android/issues/detail?id=823
an

Android getPixels

Im currently working on a program which applies edge detection to an area of the preview frame. I have used previewcallback and got my cropped bitmap, have converted to grayscale using the following method.
int height1=120;
int width2=120;
final Bitmap resizedBitmap = Bitmap.createBitmap(bmp, 260, 15,
width2, height1);
try {
int bWidth = resizedBitmap.getWidth();
int bHeight = resizedBitmap.getHeight();
int[] pixels = new int[bWidth * bHeight];
resizedBitmap.getPixels(pixels, 0, bWidth, 0, 0, bWidth, bHeight);
for (int y = 0; y < bHeight; y++){
for (int x = 0; x < bWidth; x++){
int index = y * bWidth + x;
int R = (pixels[index] >> 16) & 0xff; //bitwise shifting
int G = (pixels[index] >> 8) & 0xff;
int B = pixels[index] & 0xff;
int gray = (int) (.299 * R + .587 * G + .114 * B);
}
}
I am very new to this, and would like to know whether gray is a 2D array of 120x120 pixels, or whether the value of gray is just being overwritten for each loop.
Apologies if this is very basic

Well, maybe I'm missing something, but as far as I can see gray is overwritten. You'd need something like
int[][] gray = new int[width][height];
// start loop
In the Loop:
gray[x][y] = ...;

How to retrieve RGB value for each color apart from one dimensional integer RGB array

I am not too familiar with bit shifting so I have the following question.
I use the function below (found elsewhere) to decode from YUV to an RGB int array.
Now I want to adjust red or green or blue values to create some custom filter effect.
I need to retrieve the R value, G value, and B value. Each value ranging from 0-255.
After that I need to restore it in the rgb array at the specified index.
So I need to retrieve each color from rgb[i] and than be able to store it again in rgb[i]
void decodeYUV420SP(int[] rgb, byte[] yuv420sp, int width, int height) {
final int frameSize = width * height;
for (int j = 0, yp = 0; j < height; j++) {
int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
for (int i = 0; i < width; i++, yp++) {
int y = (0xff & ((int) yuv420sp[yp])) - 16;
if (y < 0)
y = 0;
if ((i & 1) == 0) {
v = (0xff & yuv420sp[uvp++]) - 128;
u = (0xff & yuv420sp[uvp++]) - 128;
}
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0)
r = 0;
else if (r > 262143)
r = 262143;
if (g < 0)
g = 0;
else if (g > 262143)
g = 262143;
if (b < 0)
b = 0;
else if (b > 262143)
b = 262143;
rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);
}
}
}
}

Basically color transformation is multiplication of color vector by matrix, and android offers support for it. Look:
http://developer.android.com/reference/android/graphics/ColorMatrix.html
It offers several convenience methods to create desired transformations. ( see sample on end)
How to change colors of a Drawable in Android?

For your convenience the Color-class contains many methods for getting and setting specific parts of a 32bit color integer, such as
Color.red(color)
returns the red part of the int color.
Or
Color.argb(alpha, red, green, blue)
to create an integer from the specified values (range 0-255).
The documentation for the different methods also specifies how to implement the bit-shifting yourself, if that is what you want to do.

How do you get the RGB values from a Bitmap on Android?

I want to get RGB values of a Bitmap on Android but I can't do this so far. My aim is to obtain RGB values for each pixel of a Bitmap. Is there any specific function for Android or anything else?
Also I wonder that do I need colorMatrix() function?
It is very important for my project.

Bitmap#getPixel(x, y) returns an int with the colour values and alpha value embedded into it.
int colour = bitmap.getPixel(x, y);
int red = Color.red(colour);
int green = Color.green(colour);
int blue = Color.blue(colour);
int alpha = Color.alpha(colour);

This may be slightly late, but to clear up the confusion with the use of &0xff:
In Java ints are 32 bits, so the (A)RGB values for each pixel are packed in 4 bytes.
In other words, a pixel with the values R(123), G(93), B(49) = FF7B 5D31 in the ARGB_8888 model. Where Alpha = FF, R = 7B, G = 5D, B = 31. But this is stored as an int as -8692431.
So, to extract the Green value from -8692431, we need to shift the 5D by 8 bits to the right, as you know. This gives 00FF 7B5D. So, if we were just to take that value we would be left with 16743261 as our Green value. Therefore, we bitwise-and that value with the mask of 0xFF (which is equivalent to 0000 00FF) and will result in 00FF 7B5D being 'masked' to 0000 005D. So we have extracted our Green value of 5D (or 93 decimal).
We can use the same mask of 0xFF for each extraction because the values have all been shifted to expose the desired two bytes as the least significant. Hence the previously suggested code of:
int p = pixel[index];
int R = (p >> 16) & 0xff;
int G = (p >> 8) & 0xff;
int B = p & 0xff;
If it makes it clearer, you can perform the equivalent operation of:
int R = (p & 0xff0000) >> 16;
int G = (p & 0x00ff00) >> 8;
int B = (p & 0x0000ff) >> 0;
For brevity, the extra 0s can be dropped, and it can be written as
int R = (p & 0xff0000) >> 16;
int G = (p & 0xff00) >> 8;
int B = p & 0xff;
Note however, that alternative colour models may be used, such as RGB_555 which stores each pixel as just 2 bytes, with varying precision for the RGB channels. So you should check the model that your bitmap is using before you perform the extraction, because the colours may be stored differently.

This is how I am trying to get that value. Use bitmap.getPixel() to get the corresponding bitmap in
integer array. By using bitwise rotation operation, we will get RGB values.
int[] pix = new int[picw * pich];
bitmap.getPixels(pix, 0, picw, 0, 0, picw, pich);
int R, G, B,Y;
for (int y = 0; y < pich; y++){
for (int x = 0; x < picw; x++)
{
int index = y * picw + x;
int R = (pix[index] >> 16) & 0xff; //bitwise shifting
int G = (pix[index] >> 8) & 0xff;
int B = pix[index] & 0xff;
//R,G.B - Red, Green, Blue
//to restore the values after RGB modification, use
//next statement
pix[index] = 0xff000000 | (R << 16) | (G << 8) | B;
}}

Arbitrary Bitmap Color Handling
You can read about the various Color methods here that will extract the components of color from a pixel int.
You might want to apply a filter to the bitmap, and return a byte array. Otherwise, you can cut this example down to the for-loop and roll through the pixels generating your array of bytes.
private byte[] rgbValuesFromBitmap(Bitmap bitmap)
{
ColorMatrix colorMatrix = new ColorMatrix();
ColorFilter colorFilter = new ColorMatrixColorFilter(
colorMatrix);
Bitmap argbBitmap = Bitmap.createBitmap(bitmap.getWidth(), bitmap.getHeight(),
Bitmap.Config.ARGB_8888);
Canvas canvas = new Canvas(argbBitmap);
Paint paint = new Paint();
paint.setColorFilter(colorFilter);
canvas.drawBitmap(bitmap, 0, 0, paint);
int width = bitmap.getWidth();
int height = bitmap.getHeight();
int componentsPerPixel = 3;
int totalPixels = width * height;
int totalBytes = totalPixels * componentsPerPixel;
byte[] rgbValues = new byte[totalBytes];
#ColorInt int[] argbPixels = new int[totalPixels];
argbBitmap.getPixels(argbPixels, 0, width, 0, 0, width, height);
for (int i = 0; i < totalPixels; i++) {
#ColorInt int argbPixel = argbPixels[i];
int red = Color.red(argbPixel);
int green = Color.green(argbPixel);
int blue = Color.blue(argbPixel);
rgbValues[i * componentsPerPixel + 0] = (byte) red;
rgbValues[i * componentsPerPixel + 1] = (byte) green;
rgbValues[i * componentsPerPixel + 2] = (byte) blue;
}
return rgbValues;
}

One for statement less :D
imagen.getPixels(pix, 0, picw, 0, 0, picw, pich);
for (i = 0; i < pix.length; i++) {
r = (pix[i]) >> 16 & 0xff;
g = (pix[i]) >> 8 & 0xff;
b = (pix[i]) & 0xff;
}

In addition to #Cobbles' answer, you can also use Bitmap#getColor(x, y) and a Color object.
for (int y = 0; y < bitmap.getHeight(); y++) {
for (int x = 0; x < bitmap.getWidth(); x++) {
Color color = bitmap.getColor(x, y);
float red = color.red();
float green = color.green();
float blue = color.blue();
float alpha = color.alpha();
Log.d(TAG, String.format(
"(R, G, B, A) = (%f, %f, %f, %f)", red, green, blue, alpha
));
}
}
With above code, you can get float (0..1) RGBA values. When you just want to get integer (0..255) values, #Cobble's way is rather straight forward (recommended). Though there is still a way to get integer values with this Color object by using Color#toArgb.
for (int y = 0; y < bitmap.getHeight(); y++) {
for (int x = 0; x < bitmap.getWidth(); x++) {
int color = bitmap.getColor(x, y).toArgb();
int red = Color.red(color);
int green = Color.green(color);
int blue = Color.blue(color);
int alpha = Color.alpha(color);
Log.d(TAG, String.format(
"(R, G, B, A) = (%3d, %3d, %3d, %3d)", red, green, blue, alpha
));
}
}