Related
I am trying to floodfill a bitmap using Renderscript. and my renderscript file progress.rs is
#pragma version(1)
#pragma rs java_package_name(com.intel.sample.androidbasicrs)
rs_allocation input;
int width;
int height;
int xTouchApply;
int yTouchApply;
static int same(uchar4 pixel, uchar4 in);
uchar4 __attribute__((kernel)) root(const uchar4 in, uint32_t x, uint32_t y) {
uchar4 out = in;
rsDebug("Process.rs : image width: ", width);
rsDebug("Process.rs : image height: ", height);
rsDebug("Process.rs : image pointX: ", xTouchApply);
rsDebug("Process.rs : image pointY: ", yTouchApply);
if(xTouchApply >= 0 && xTouchApply < width && yTouchApply >=0 && yTouchApply < height){
// getting touched pixel
uchar4 pixel = rsGetElementAt_uchar4(input, xTouchApply, yTouchApply);
rsDebug("Process.rs : getting touched pixel", 0);
// resets the pixel stack
int topOfStackIndex = 0;
// creating pixel stack
int pixelStack[width*height];
// Pushes the touched pixel onto the stack
pixelStack[topOfStackIndex] = xTouchApply;
pixelStack[topOfStackIndex+1] = yTouchApply;
topOfStackIndex += 2;
//four way stack floodfill algorithm
while(topOfStackIndex>0){
rsDebug("Process.rs : looping while", 0);
// Pops a pixel from the stack
int x = pixelStack[topOfStackIndex - 2];
int y1 = pixelStack[topOfStackIndex - 1];
topOfStackIndex -= 2;
while (y1 >= 0 && same(rsGetElementAt_uchar4(input, x, y1), pixel)) {
y1--;
}
y1++;
int spanLeft = 0;
int spanRight = 0;
while (y1 < height && same(rsGetElementAt_uchar4(input, x, y1), pixel)) {
rsDebug("Process.rs : pointX: ", x);
rsDebug("Process.rs : pointY: ", y1);
float3 outPixel = dot(f4.rgb, channelWeights);
out = rsPackColorTo8888(outPixel);
// conditions to traverse skipPixels to check threshold color(Similar color)
if (!spanLeft && x > 0 && same(rsGetElementAt_uchar4(input, x - 1, y1), pixel)) {
// Pixel to the left must also be changed, pushes it to the stack
pixelStack[topOfStackIndex] = x - 1;
pixelStack[topOfStackIndex + 1] = y1;
topOfStackIndex += 2;
spanLeft = 1;
} else if (spanLeft && !same(rsGetElementAt_uchar4(input, x - 1, y1), pixel)) {
// Pixel to the left has already been changed
spanLeft = 0;
}
// conditions to traverse skipPixels to check threshold color(Similar color)
if (!spanRight && x < width - 1 && same(rsGetElementAt_uchar4(input, x + 1, y1), pixel)) {
// Pixel to the right must also be changed, pushes it to the stack
pixelStack[topOfStackIndex] = x + 1;
pixelStack[topOfStackIndex + 1] = y1;
topOfStackIndex += 2;
spanRight = 1;
} else if (spanRight && x < width - 1 && !same(rsGetElementAt_uchar4(input, x + 1, y1), pixel)) {
// Pixel to the right has already been changed
spanRight = 0;
}
y1++;
}
}
}
return out;
}
static int same(uchar4 px, uchar4 inPx){
int isSame = 0;
if((px.r == inPx.r) && (px.g == inPx.g) && (px.b == inPx.b) && (px.a == inPx.a)) {
isSame = 1;
// rsDebug("Process.rs : matching pixel: ", isSame);
} else {
isSame = 0;
}
// rsDebug("Process.rs : matching pixel: ", isSame);
return isSame;
}
and my Activity's code is:
inputBitmap = Bitmap.createScaledBitmap(inputBitmap, displayWidth, displayHeight, false);
// Create an allocation (which is memory abstraction in the RenderScript)
// that corresponds to the inputBitmap.
allocationIn = Allocation.createFromBitmap(
rs,
inputBitmap,
Allocation.MipmapControl.MIPMAP_NONE,
Allocation.USAGE_SCRIPT
);
allocationOut = Allocation.createTyped(rs, allocationIn.getType());
int imageWidth = inputBitmap.getWidth();
int imageHeight = inputBitmap.getHeight();
script.set_width(imageWidth);
script.set_height(imageHeight);
script.set_input(allocationIn);
//....
//....
// and my onTouchEvent Code is
script.set_xTouchApply(xTouchApply);
script.set_yTouchApply(yTouchApply);
// Run the script.
script.forEach_root(allocationIn, allocationOut);
allocationOut.copyTo(outputBitmap);
when I touched bitmap it is showing Application not responding. It is because of root method is calling for every pixels. How can I optimize this code. And how can I compare two uchar4 variables in Renderscript? How can I improve my same method? Or How can I find similar neighbor pixels using threshold value? I got stuck. Please guys help me.
I don't have much knowledge of c99 programming language and Renderscript. Can you guys debug my renderscript code. and please tell me what's wrong in this code. Or can I improve this renderscript code to floodfill the bitmap. Any help will be appreciated And sorry for my poor English ;-) . Thanks
Renderscript is Android's front-end to GPU-instructions. And it is extremely good if you want to perform operations on each pixel because it uses the massive GPU-parallelism-capabilities. So, you can run an operation on each pixel. For this purpose, you start a program in Renderscript with sth like "for all pixels, do the following".
The flood fill algorithm though cannot run in such a parallel environment because you only know which pixel to paint after painting another pixel before it. This is not only true for renderscript but all GPU-related libraries, like CUDA or others.
I am developing an Android application using Java. In my application, I am doing some image processing. So I am using c++ and Open CV for it and calling the c++ function through JNI. I am trying to convert equirectangular/spherical image to cubemap image.
I found this link for conversion, https://code.i-harness.com/en/q/1c4dbae/. I am passing Mat from Java and trying to return the converted image back to Java.
This is my C++ code
#include <jni.h>
#include <string>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/features2d/features2d.hpp>
using namespace std;
using namespace cv;
extern "C"
JNIEXPORT jstring
JNICALL
Java_media_memento_memento_VRPhotoSphereActivity_convertEquiRectToCubeMap(
JNIEnv *env,
jobject /* this */, jlong addrMat, jlong addrNewMat) {
Mat& mat = *(Mat*)addrMat;
Mat& newMat = *(Mat*)addrNewMat;
newMat.create(mat.rows, mat.cols, mat.type());
memcpy(newMat.data, mat.data , sizeof(mat.data) -1);
//EquiRec to Cubemap conversion starts from here
float faceTransform[6][2] =
{
{0, 0},
{M_PI / 2, 0},
{M_PI, 0},
{-M_PI / 2, 0},
{0, -M_PI / 2},
{0, M_PI / 2}
};
//conversion ends here
const Mat &in= mat;
Mat face = newMat;
int faceId = 0;
const int width = -1;
const int height = -1;
float inWidth = in.cols;
float inHeight = in.rows;
// Allocate map
Mat mapx(height, width, CV_32F);
Mat mapy(height, width, CV_32F);
// Calculate adjacent (ak) and opposite (an) of the
// triangle that is spanned from the sphere center
//to our cube face.
const float an = sin(M_PI / 4);
const float ak = cos(M_PI / 4);
const float ftu = faceTransform[faceId][0];
const float ftv = faceTransform[faceId][1];
// For each point in the target image,
// calculate the corresponding source coordinates.
for(int y = 0; y < height; y++) {
for(int x = 0; x < width; x++) {
// Map face pixel coordinates to [-1, 1] on plane
float nx = (float)y / (float)height - 0.5f;
float ny = (float)x / (float)width - 0.5f;
nx *= 2;
ny *= 2;
// Map [-1, 1] plane coords to [-an, an]
// thats the coordinates in respect to a unit sphere
// that contains our box.
nx *= an;
ny *= an;
float u, v;
// Project from plane to sphere surface.
if(ftv == 0) {
// Center faces
u = atan2(nx, ak);
v = atan2(ny * cos(u), ak);
u += ftu;
} else if(ftv > 0) {
// Bottom face
float d = sqrt(nx * nx + ny * ny);
v = M_PI / 2 - atan2(d, ak);
u = atan2(ny, nx);
} else {
// Top face
float d = sqrt(nx * nx + ny * ny);
v = -M_PI / 2 + atan2(d, ak);
u = atan2(-ny, nx);
}
// Map from angular coordinates to [-1, 1], respectively.
u = u / (M_PI);
v = v / (M_PI / 2);
// Warp around, if our coordinates are out of bounds.
while (v < -1) {
v += 2;
u += 1;
}
while (v > 1) {
v -= 2;
u += 1;
}
while(u < -1) {
u += 2;
}
while(u > 1) {
u -= 2;
}
// Map from [-1, 1] to in texture space
u = u / 2.0f + 0.5f;
v = v / 2.0f + 0.5f;
u = u * (inWidth - 1);
v = v * (inHeight - 1);
// Save the result for this pixel in map
mapx.at<float>(x, y) = u;
mapy.at<float>(x, y) = v;
}
}
// Recreate output image if it has wrong size or type.
if(face.cols != width || face.rows != height ||
face.type() != in.type()) {
face = Mat(width, height, in.type());
}
// Do actual resampling using OpenCV's remap
Mat i = in;
Mat f = face;
remap(i, f, mapx, mapy,
CV_INTER_LINEAR, BORDER_CONSTANT, Scalar(0, 0, 0));
//send the image back here. For now the feature is not implemented yet.
std::string hello = "Spherical equirectangular photo converted to cubemap face photo";
return env->NewStringUTF(hello.c_str());
}
When I tried to run my application, it is giving me this compilation error.
Error:(146) undefined reference to `cv::remap(cv::_InputArray const&, cv::_OutputArray const&, cv::_InputArray const&, cv::_InputArray const&, int, int, cv::Scalar_<double> const&)'
This is the screenshot.
How can I fix that error?
Edit
Actually, it is throwing error starting from this line
Mat mapx(height, width, CV_32F);
Mat mapy(height, width, CV_32F);
This is the screenshot
I have to provide a YUV(NV21) byte array to a recognition solution and I'd like, to reduce processing time, to down scale the preview frame.
From solutions gathered here and there on SO, I manage to convert on a 1:1 ratio and I get recognition hits. But if I'd like to scale the intermediate bitmap down, I get no result. Even if I scale it down to 95% only.
Any help would be appreciated.
Thus, every 400-ish ms I take the preview frame to convert it asynchronously. I convert it to ARGB using RenderScript, scale it down and then convert it back.
// Camera callback
#Override
public void onPreviewFrame(byte[] frame, Camera camera) {
if (camera != null) {
// Debounce
if ((System.currentTimeMillis() - mStart) > 400) {
mStart = System.currentTimeMillis();
Camera.Size size = camera.getParameters().getPreviewSize();
new FrameScaleAsyncTask(frame, size.width, size.height).execute();
}
}
if (mCamera != null) {
mCamera.addCallbackBuffer(mBuffer);
}
}
// In FrameScaleAsyncTask
#Override
protected Void doInBackground(Void... params) {
// Create YUV type for in-allocation
Type yuvType = new Type.Builder(mRenderScript, Element.U8(mRenderScript))
.setX(mFrame.length)
.create();
mAllocationIn = Allocation.createTyped(mRenderScript, yuvType, Allocation.USAGE_SCRIPT);
// Create ARGB-8888 type for out-allocation
Type rgbType = new Type.Builder(mRenderScript, Element.RGBA_8888(mRenderScript))
.setX(mWidth)
.setY(mHeight)
.create();
mAllocationOut = Allocation.createTyped(mRenderScript, rgbType, Allocation.USAGE_SCRIPT);
// Copy frame data into in-allocation
mAllocationIn.copyFrom(mFrame);
// Set script input and fire !
mScript.setInput(mAllocationIn);
mScript.forEach(mAllocationOut);
// Create a bitmap of camera preview size (see camera setup) and copy out-allocation to it
Bitmap bitmap = Bitmap.createBitmap(mWidth, mHeight, Bitmap.Config.ARGB_8888);
mAllocationOut.copyTo(bitmap);
// Scale bitmap down
double scaleRatio = 1;
Bitmap scaledBitmap = Bitmap.createScaledBitmap(
bitmap,
(int) (bitmap.getWidth() * scaleRatio),
(int) (bitmap.getHeight() * scaleRatio),
false
);
bitmap.recycle();
int size = scaledBitmap.getRowBytes() * scaledBitmap.getHeight();
int scaledWidth = scaledBitmap.getWidth();
int scaledHeight = scaledBitmap.getHeight();
int[] pixels = new int[scaledWidth * scaledHeight];
// Put bitmap pixels into an int array
scaledBitmap.getPixels(pixels, 0, scaledWidth, 0, 0, scaledWidth, scaledHeight);
mFrame = new byte[pixels.length * 3 / 2];
ImageHelper.encodeYUV420SPAlt(mFrame, pixels, scaledWidth, scaledHeight);
return null;
}
The RGB to YUV algorithm (see : this answer ):
public static void encodeYUV420SPAlt(byte[] yuv420sp, int[] argb, int width, int height) {
final int frameSize = width * height;
int yIndex = 0;
int uvIndex = frameSize;
int a, R, G, B, Y, U, V;
int index = 0;
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
a = (argb[index] & 0xff000000) >> 24; // a is not used obviously
R = (argb[index] & 0xff0000) >> 16;
G = (argb[index] & 0xff00) >> 8;
B = (argb[index] & 0xff) >> 0;
// well known RGB to YUV algorithm
Y = ((66 * R + 129 * G + 25 * B + 128) >> 8) + 16;
U = ((-38 * R - 74 * G + 112 * B + 128) >> 8) + 128;
V = ((112 * R - 94 * G - 18 * B + 128) >> 8) + 128;
// NV21 has a plane of Y and interleaved planes of VU each sampled by a factor of 2
// meaning for every 4 Y pixels there are 1 V and 1 U. Note the sampling is every other
// pixel AND every other scanline.
yuv420sp[yIndex++] = (byte) ((Y < 0) ? 0 : ((Y > 255) ? 255 : Y));
if (j % 2 == 0 && index % 2 == 0) {
yuv420sp[uvIndex++] = (byte) ((V < 0) ? 0 : ((V > 255) ? 255 : V));
yuv420sp[uvIndex++] = (byte) ((U < 0) ? 0 : ((U > 255) ? 255 : U));
}
index++;
}
}
}
I finally end up resizing my image (as a OpenCV.Mat) directly in C++. This was way easier and faster.
Size size(correctedWidth, correctedHeight);
Mat dst;
resize(image, dst, size);
My Problem is: I've set up a camera in Android and receive the preview data by using an onPreviewFrame-listener which passes me an byte[] array containing the image data in the default android YUV-format (device does not support R5G6B5-format). Each pixel consists of 12bits which makes the thing a little tricky. Now what I want to do is converting the YUV-data into ARGB-data in order to do image processing with it. This has to be done with renderscript, in order to maintain a high performance.
My idea was to pass two pixels in one element (which would be 24bits = 3 bytes) and then return two ARGB pixels. The problem is, that in Renderscript a u8_3 (a 3dimensional 8bit vector) is stored in 32bit, which means that the last 8 bits are unused. But when copying the image data into the allocation all of the 32bits are used, so the last 8bit get lost. Even if I used a 32bit input data, the last 8bit are useless, because they're only 2/3 of a pixel. When defining an element consisting a 3-byte-array it actually has a real size of 3 bytes. But then the Allocation.copyFrom()-method doesn't fill the in-Allocation with data, argueing it doesn't has the right data type to be filled with a byte[].
The renderscript documentation states, that there is a ScriptIntrinsicYuvToRGB which should do exactly that in API Level 17. But in fact the class doesn't exist. I've downloaded API Level 17 even though it seems not to be downloadable any more. Does anyone have any information about it? Does anyone have ever tried out a ScriptIntrinsic?
So in conclusion my question is: How to convert the camera data into ARGB data fast, hardwareaccelerated?
That's how to do it in Dalvik VM (found the code somewhere online, it works):
#SuppressWarnings("unused")
private void decodeYUV420SP(int[] rgb, byte[] yuv420sp, int width, int height) {
final int frameSize = width * height;
for (int j = 0, yp = 0; j < height; j++) {
int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
for (int i = 0; i < width; i++, yp++) {
int y = (0xff & ((int) yuv420sp[yp])) - 16;
if (y < 0)
y = 0;
if ((i & 1) == 0) {
v = (0xff & yuv420sp[uvp++]) - 128;
u = (0xff & yuv420sp[uvp++]) - 128;
}
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0)
r = 0;
else if (r > 262143)
r = 262143;
if (g < 0)
g = 0;
else if (g > 262143)
g = 262143;
if (b < 0)
b = 0;
else if (b > 262143)
b = 262143;
rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);
}
}
}
I'm sure you will find the LivePreview test application interesting ... it's part of the Android source code in the latest Jelly Bean (MR1). It implements a camera preview and uses ScriptIntrinsicYuvToRgb to convert the preview data with Renderscript. You can browse the source online here:
LivePreview
I was not able to get running ScriptInstrinsicYuvToRgb, so I decided to write my own RS solution.
Here's ready script (named yuv.rs):
#pragma version(1)
#pragma rs java_package_name(com.package.name)
rs_allocation gIn;
int width;
int height;
int frameSize;
void yuvToRgb(const uchar *v_in, uchar4 *v_out, const void *usrData, uint32_t x, uint32_t y) {
uchar yp = rsGetElementAtYuv_uchar_Y(gIn, x, y) & 0xFF;
int index = frameSize + (x & (~1)) + (( y>>1) * width );
int v = (int)( rsGetElementAt_uchar(gIn, index) & 0xFF ) -128;
int u = (int)( rsGetElementAt_uchar(gIn, index+1) & 0xFF ) -128;
int r = (int) (1.164f * yp + 1.596f * v );
int g = (int) (1.164f * yp - 0.813f * v - 0.391f * u);
int b = (int) (1.164f * yp + 2.018f * u );
r = r>255? 255 : r<0 ? 0 : r;
g = g>255? 255 : g<0 ? 0 : g;
b = b>255? 255 : b<0 ? 0 : b;
uchar4 res4;
res4.r = (uchar)r;
res4.g = (uchar)g;
res4.b = (uchar)b;
res4.a = 0xFF;
*v_out = res4;
}
Don't forget to set camera preview format to NV21:
Parameters cameraParameters = camera.getParameters();
cameraParameters.setPreviewFormat(ImageFormat.NV21);
// Other camera init stuff: preview size, framerate, etc.
camera.setParameters(cameraParameters);
Allocations initialization and script usage:
// Somewhere in initialization section
// w and h are variables for selected camera preview size
rs = RenderScript.create(this);
Type.Builder tbIn = new Type.Builder(rs, Element.U8(rs));
tbIn.setX(w);
tbIn.setY(h);
tbIn.setYuvFormat(ImageFormat.NV21);
Type.Builder tbOut = new Type.Builder(rs, Element.RGBA_8888(rs));
tbOut.setX(w);
tbOut.setY(h);
inData = Allocation.createTyped(rs, tbIn.create(), Allocation.MipmapControl.MIPMAP_NONE, Allocation.USAGE_SCRIPT & Allocation.USAGE_SHARED);
outData = Allocation.createTyped(rs, tbOut.create(), Allocation.MipmapControl.MIPMAP_NONE, Allocation.USAGE_SCRIPT & Allocation.USAGE_SHARED);
outputBitmap = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888);
yuvScript = new ScriptC_yuv(rs);
yuvScript.set_gIn(inData);
yuvScript.set_width(w);
yuvScript.set_height(h);
yuvScript.set_frameSize(previewSize);
//.....
Camera callback method:
public void onPreviewFrame(byte[] data, Camera camera) {
// In your camera callback, data
inData.copyFrom(data);
yuvScript.forEach_yuvToRgb(inData, outData);
outData.copyTo(outputBitmap);
// draw your bitmap where you want to
// .....
}
For anyone who didn't know, RenderScript is now in the Android Support Library, including intrinsics.
http://android-developers.blogspot.com.au/2013/09/renderscript-in-android-support-library.html
http://android-developers.blogspot.com.au/2013/08/renderscript-intrinsics.html
We now have the new renderscript-intrinsics-replacement-toolkit to do it. First, build and import the renderscript module to your project and add it as a dependency to your app module. Then, go to Toolkit.kt and add the following:
fun toNv21(image: Image): ByteArray? {
val nv21 = ByteArray((image.width * image.height * 1.5f).toInt())
return if (!nativeYuv420toNv21(
nativeHandle,
image.width,
image.height,
image.planes[0].buffer, // Y buffer
image.planes[1].buffer, // U buffer
image.planes[2].buffer, // V buffer
image.planes[0].pixelStride, // Y pixel stride
image.planes[1].pixelStride, // U/V pixel stride
image.planes[0].rowStride, // Y row stride
image.planes[1].rowStride, // U/V row stride
nv21
)
) {
null
} else nv21
}
private external fun nativeYuv420toNv21(
nativeHandle: Long,
imageWidth: Int,
imageHeight: Int,
yByteBuffer: ByteBuffer,
uByteBuffer: ByteBuffer,
vByteBuffer: ByteBuffer,
yPixelStride: Int,
uvPixelStride: Int,
yRowStride: Int,
uvRowStride: Int,
nv21Output: ByteArray
): Boolean
Now, go to JniEntryPoints.cpp and add the following:
extern "C" JNIEXPORT jboolean JNICALL Java_com_google_android_renderscript_Toolkit_nativeYuv420toNv21(
JNIEnv *env, jobject/*thiz*/, jlong native_handle,
jint image_width, jint image_height, jobject y_byte_buffer,
jobject u_byte_buffer, jobject v_byte_buffer, jint y_pixel_stride,
jint uv_pixel_stride, jint y_row_stride, jint uv_row_stride,
jbyteArray nv21_array) {
auto y_buffer = static_cast<jbyte*>(env->GetDirectBufferAddress(y_byte_buffer));
auto u_buffer = static_cast<jbyte*>(env->GetDirectBufferAddress(u_byte_buffer));
auto v_buffer = static_cast<jbyte*>(env->GetDirectBufferAddress(v_byte_buffer));
jbyte* nv21 = env->GetByteArrayElements(nv21_array, nullptr);
if (nv21 == nullptr || y_buffer == nullptr || u_buffer == nullptr
|| v_buffer == nullptr) {
// Log this.
return false;
}
RenderScriptToolkit* toolkit = reinterpret_cast<RenderScriptToolkit*>(native_handle);
toolkit->yuv420toNv21(image_width, image_height, y_buffer, u_buffer, v_buffer,
y_pixel_stride, uv_pixel_stride, y_row_stride, uv_row_stride,
nv21);
env->ReleaseByteArrayElements(nv21_array, nv21, 0);
return true;
}
Go to YuvToRgb.cpp and add the following:
void RenderScriptToolkit::yuv420toNv21(int image_width, int image_height, const int8_t* y_buffer,
const int8_t* u_buffer, const int8_t* v_buffer, int y_pixel_stride,
int uv_pixel_stride, int y_row_stride, int uv_row_stride,
int8_t *nv21) {
// Copy Y channel.
for(int y = 0; y < image_height; ++y) {
int destOffset = image_width * y;
int yOffset = y * y_row_stride;
memcpy(nv21 + destOffset, y_buffer + yOffset, image_width);
}
if (v_buffer - u_buffer == sizeof(int8_t)) {
// format = nv21
// TODO: If the format is VUVUVU & pixel stride == 1 we can simply the copy
// with memcpy. In Android Camera2 I have mostly come across UVUVUV packaging
// though.
}
// Copy UV Channel.
int idUV = image_width * image_height;
int uv_width = image_width / 2;
int uv_height = image_height / 2;
for(int y = 0; y < uv_height; ++y) {
int uvOffset = y * uv_row_stride;
for (int x = 0; x < uv_width; ++x) {
int bufferIndex = uvOffset + (x * uv_pixel_stride);
// V channel.
nv21[idUV++] = v_buffer[bufferIndex];
// U channel.
nv21[idUV++] = u_buffer[bufferIndex];
}
}
}
Finally, go to RenderscriptToolkit.h and add the following:
/**
* https://blog.minhazav.dev/how-to-use-renderscript-to-convert-YUV_420_888-yuv-image-to-bitmap/#tobitmapimage-image-method
* #param image_width width of the image you want to convert to byte array
* #param image_height height of the image you want to convert to byte array
* #param y_buffer Y buffer
* #param u_buffer U buffer
* #param v_buffer V buffer
* #param y_pixel_stride Y pixel stride
* #param uv_pixel_stride UV pixel stride
* #param y_row_stride Y row stride
* #param uv_row_stride UV row stride
* #param nv21 the output byte array
*/
void yuv420toNv21(int image_width, int image_height, const int8_t* y_buffer,
const int8_t* u_buffer, const int8_t* v_buffer, int y_pixel_stride,
int uv_pixel_stride, int y_row_stride, int uv_row_stride,
int8_t *nv21);
You are now ready to harness the full power of renderscript. Below, I am providing an example with the ARCore Camera Image object (replace the first line with whatever code gives you your camera image):
val cameraImage = arFrame.frame.acquireCameraImage()
val width = cameraImage.width
val height = cameraImage.height
val byteArray = Toolkit.toNv21(cameraImage)
byteArray?.let {
Toolkit.yuvToRgbBitmap(
byteArray,
width,
height,
YuvFormat.NV21
).let { bitmap ->
saveBitmapToDevice(
name,
session,
bitmap,
context
)}}
I am capturing image using SurfaceView and getting Yuv Raw preview data in public void onPreviewFrame4(byte[] data, Camera camera)
I have to perform some image preprocessing in onPreviewFrame so i need to convert Yuv preview data to RGB data than image preprocessing and back to Yuv data.
I have used both function for encoding and decoding Yuv data to RGB as following :
public void onPreviewFrame(byte[] data, Camera camera) {
Point cameraResolution = configManager.getCameraResolution();
if (data != null) {
Log.i("DEBUG", "data Not Null");
// Preprocessing
Log.i("DEBUG", "Try For Image Processing");
Camera.Parameters mParameters = camera.getParameters();
Size mSize = mParameters.getPreviewSize();
int mWidth = mSize.width;
int mHeight = mSize.height;
int[] mIntArray = new int[mWidth * mHeight];
// Decode Yuv data to integer array
decodeYUV420SP(mIntArray, data, mWidth, mHeight);
// Converting int mIntArray to Bitmap and
// than image preprocessing
// and back to mIntArray.
// Encode intArray to Yuv data
encodeYUV420SP(data, mIntArray, mWidth, mHeight);
}
}
static public void decodeYUV420SP(int[] rgba, byte[] yuv420sp, int width,
int height) {
final int frameSize = width * height;
for (int j = 0, yp = 0; j < height; j++) {
int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
for (int i = 0; i < width; i++, yp++) {
int y = (0xff & ((int) yuv420sp[yp])) - 16;
if (y < 0)
y = 0;
if ((i & 1) == 0) {
v = (0xff & yuv420sp[uvp++]) - 128;
u = (0xff & yuv420sp[uvp++]) - 128;
}
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0)
r = 0;
else if (r > 262143)
r = 262143;
if (g < 0)
g = 0;
else if (g > 262143)
g = 262143;
if (b < 0)
b = 0;
else if (b > 262143)
b = 262143;
// rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) &
// 0xff00) | ((b >> 10) & 0xff);
// rgba, divide 2^10 ( >> 10)
rgba[yp] = ((r << 14) & 0xff000000) | ((g << 6) & 0xff0000)
| ((b >> 2) | 0xff00);
}
}
}
static public void encodeYUV420SP_original(byte[] yuv420sp, int[] rgba,
int width, int height) {
final int frameSize = width * height;
int[] U, V;
U = new int[frameSize];
V = new int[frameSize];
final int uvwidth = width / 2;
int r, g, b, y, u, v;
for (int j = 0; j < height; j++) {
int index = width * j;
for (int i = 0; i < width; i++) {
r = (rgba[index] & 0xff000000) >> 24;
g = (rgba[index] & 0xff0000) >> 16;
b = (rgba[index] & 0xff00) >> 8;
// rgb to yuv
y = (66 * r + 129 * g + 25 * b + 128) >> 8 + 16;
u = (-38 * r - 74 * g + 112 * b + 128) >> 8 + 128;
v = (112 * r - 94 * g - 18 * b + 128) >> 8 + 128;
// clip y
yuv420sp[index++] = (byte) ((y < 0) ? 0 : ((y > 255) ? 255 : y));
U[index] = u;
V[index++] = v;
}
}
The problem is that encoding and decoding Yuv data might have some mistake because if i skip the preprocessing step than also encoded Yuv data are differ from original data of PreviewCallback.
Please help me to resolve this issue. I have to used this code in OCR scanning so i need to implement this type of logic.
If any other way of doing same thing than please provide me.
Thanks in advance. :)
Although the documentation suggests that you can set which format the image data should arrive from the camera in, in practice you often have a choice of one: NV21, a YUV format. For lots of information on this format see http://www.fourcc.org/yuv.php#NV21 and for information on the theory behind converting it to RGB see http://www.fourcc.org/fccyvrgb.php. There is a picture based explanation at Extract black and white image from android camera's NV21 format. There is an android specific section on a wikipedia page about the subject (thanks #AlexCohn): YUV#Y'UV420sp (NV21) to RGB conversion (Android).
However, once you've set up your onPreviewFrame routine, the mechanics of going from the byte array it sends you to useful data is somewhat, ummmm, unclear. From API 8 onwards, the following solution is available, to get to a ByteStream holiding a JPEG of the image (compressToJpeg is the only conversion option offered by YuvImage):
// pWidth and pHeight define the size of the preview Frame
ByteArrayOutputStream out = new ByteArrayOutputStream();
// Alter the second parameter of this to the actual format you are receiving
YuvImage yuv = new YuvImage(data, ImageFormat.NV21, pWidth, pHeight, null);
// bWidth and bHeight define the size of the bitmap you wish the fill with the preview image
yuv.compressToJpeg(new Rect(0, 0, bWidth, bHeight), 50, out);
This JPEG may then need to be converted into the format you want. If you want a Bitmap:
byte[] bytes = out.toByteArray();
Bitmap bitmap= BitmapFactory.decodeByteArray(bytes, 0, bytes.length);
If, for whatever reason, you are unable to do this, you can do the conversion manually. Some problems to be overcome in doing this:
The data arrives in a byte array. By definition, bytes are signed numbers, meaning that they go from -128 to 127. However, the data is actually unsigned bytes (0 to 255). If this isn't dealt with, the outcome is doomed to have some odd clipping effects.
The data is in a very specific order (as per the previously mentioned web pages) and each pixel needs to be extracted carefully.
Each pixel needs to be put into the right place on a bitmap, say. This also requires a rather messy (in my view) approach of building a buffer of the data and then filling a bitmap from it.
In principle, the values should be stored [16..240], but it appears that they are stored [0..255] in the data sent to onPreviewFrame
Just about every web page on the matter proposes different coefficients, even allowing for [16..240] vs [0..255] options.
If you've actually got NV12 (another variant on YUV420), then you will need to swap the reads for U and V.
I present a solution (which seems to work), with requests for corrections, improvements and ways of making the whole thing less costly to run. I have set it out to hopefully make clear what is happening, rather than to optimise it for speed. It creates a bitmap the size of the preview image:
The data variable is coming from the call to onPreviewFrame
// Define whether expecting [16..240] or [0..255]
boolean dataIs16To240 = false;
// the bitmap we want to fill with the image
Bitmap bitmap = Bitmap.createBitmap(imageWidth, imageHeight, Bitmap.Config.ARGB_8888);
int numPixels = imageWidth*imageHeight;
// the buffer we fill up which we then fill the bitmap with
IntBuffer intBuffer = IntBuffer.allocate(imageWidth*imageHeight);
// If you're reusing a buffer, next line imperative to refill from the start,
// if not good practice
intBuffer.position(0);
// Set the alpha for the image: 0 is transparent, 255 fully opaque
final byte alpha = (byte) 255;
// Holding variables for the loop calculation
int R = 0;
int G = 0;
int B = 0;
// Get each pixel, one at a time
for (int y = 0; y < imageHeight; y++) {
for (int x = 0; x < imageWidth; x++) {
// Get the Y value, stored in the first block of data
// The logical "AND 0xff" is needed to deal with the signed issue
float Y = (float) (data[y*imageWidth + x] & 0xff);
// Get U and V values, stored after Y values, one per 2x2 block
// of pixels, interleaved. Prepare them as floats with correct range
// ready for calculation later.
int xby2 = x/2;
int yby2 = y/2;
// make this V for NV12/420SP
float U = (float)(data[numPixels + 2*xby2 + yby2*imageWidth] & 0xff) - 128.0f;
// make this U for NV12/420SP
float V = (float)(data[numPixels + 2*xby2 + 1 + yby2*imageWidth] & 0xff) - 128.0f;
if (dataIs16To240) {
// Correct Y to allow for the fact that it is [16..235] and not [0..255]
Y = 1.164*(Y - 16.0);
// Do the YUV -> RGB conversion
// These seem to work, but other variations are quoted
// out there.
R = (int)(Yf + 1.596f*V);
G = (int)(Yf - 0.813f*V - 0.391f*U);
B = (int)(Yf + 2.018f*U);
}
else {
// No need to correct Y
// These are the coefficients proposed by #AlexCohn
// for [0..255], as per the wikipedia page referenced
// above
R = (int)(Yf + 1.370705f*V);
G = (int)(Yf - 0.698001f*V - 0.337633f*U);
B = (int)(Yf + 1.732446f*U);
}
// Clip rgb values to 0-255
R = R < 0 ? 0 : R > 255 ? 255 : R;
G = G < 0 ? 0 : G > 255 ? 255 : G;
B = B < 0 ? 0 : B > 255 ? 255 : B;
// Put that pixel in the buffer
intBuffer.put(alpha*16777216 + R*65536 + G*256 + B);
}
}
// Get buffer ready to be read
intBuffer.flip();
// Push the pixel information from the buffer onto the bitmap.
bitmap.copyPixelsFromBuffer(intBuffer);
As #Timmmm points out below, you could do the conversion in int by multiplying the scaling factors by 1000 (ie. 1.164 becomes 1164) and then dividng the end results by 1000.
Why not specify that camera preview should provide RGB images?
i.e. Camera.Parameters.setPreviewFormat(ImageFormat.RGB_565);
You can use RenderScript -> ScriptIntrinsicYuvToRGB
Kotlin Sample
val rs = RenderScript.create(CONTEXT_HERE)
val yuvToRgbIntrinsic = ScriptIntrinsicYuvToRGB.create(rs, Element.U8_4(rs))
val yuvType = Type.Builder(rs, Element.U8(rs)).setX(byteArray.size)
val inData = Allocation.createTyped(rs, yuvType.create(), Allocation.USAGE_SCRIPT)
val rgbaType = Type.Builder(rs, Element.RGBA_8888(rs)).setX(width).setY(height)
val outData = Allocation.createTyped(rs, rgbaType.create(), Allocation.USAGE_SCRIPT)
inData.copyFrom(byteArray)
yuvToRgbIntrinsic.setInput(inData)
yuvToRgbIntrinsic.forEach(outData)
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
outData.copyTo(bitmap)
After some tests on Samsung S4 mini fastest code is (120% faster then Neil's [floats!] and 30% faster then original Hitesh's):
static public void decodeYUV420SP(int[] rgba, byte[] yuv420sp, int width,
int height) {
final int frameSize = width * height;
// define variables before loops (+ 20-30% faster algorithm o0`)
int r, g, b, y1192, y, i, uvp, u, v;
for (int j = 0, yp = 0; j < height; j++) {
uvp = frameSize + (j >> 1) * width;
u = 0;
v = 0;
for (i = 0; i < width; i++, yp++) {
y = (0xff & ((int) yuv420sp[yp])) - 16;
if (y < 0)
y = 0;
if ((i & 1) == 0) {
v = (0xff & yuv420sp[uvp++]) - 128;
u = (0xff & yuv420sp[uvp++]) - 128;
}
y1192 = 1192 * y;
r = (y1192 + 1634 * v);
g = (y1192 - 833 * v - 400 * u);
b = (y1192 + 2066 * u);
// Java's functions are faster then 'IFs'
r = Math.max(0, Math.min(r, 262143));
g = Math.max(0, Math.min(g, 262143));
b = Math.max(0, Math.min(b, 262143));
// rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) &
// 0xff00) | ((b >> 10) & 0xff);
// rgba, divide 2^10 ( >> 10)
rgba[yp] = ((r << 14) & 0xff000000) | ((g << 6) & 0xff0000)
| ((b >> 2) | 0xff00);
}
}
}
Speed is comparable to YuvImage.compressToJpeg() with ByteArrayOutputStream as output (30-50 ms for 640x480 image).
Result: Samsung S4 mini (2x1.7GHz) can't compress to JPEG/convert YUV to RGB in real time (640x480#30fps)
Java implementation is 10 times slow than the c version, I suggest you use GPUImage library or just move this part of code.
There is a android version of GPUImage:
https://github.com/CyberAgent/android-gpuimage
You can include this library if you use gradle, and call the method:
GPUImageNativeLibrary.YUVtoRBGA( inputArray, WIDTH, HEIGHT, outputArray);
I compare the time, for a NV21 image which is 960x540, use above java code, it cost 200ms+, with GPUImage version, just 10ms~20ms.
You can use ColorHelper library for this:
using ColorHelper;
YUV yuv = new YUV(0.1, 0.1, 0.2);
RGB rgb = ColorConverter.YuvToRgb(yuv);
Links:
Github
Nuget
Fixup the above code snippet
static public void decodeYUV420SP(int[] rgba, byte[] yuv420sp, int width,
int height) {
final int frameSize = width * height;
int r, g, b, y1192, y, i, uvp, u, v;
for (int j = 0, yp = 0; j < height; j++) {
uvp = frameSize + (j >> 1) * width;
u = 0;
v = 0;
for (i = 0; i < width; i++, yp++) {
y = (0xff & ((int) yuv420sp[yp])) - 16;
if (y < 0)
y = 0;
if ((i & 1) == 0) {
// above answer is wrong at the following lines. just swap ***u*** and ***v***
u = (0xff & yuv420sp[uvp++]) - 128;
v = (0xff & yuv420sp[uvp++]) - 128;
}
y1192 = 1192 * y;
r = (y1192 + 1634 * v);
g = (y1192 - 833 * v - 400 * u);
b = (y1192 + 2066 * u);
r = Math.max(0, Math.min(r, 262143));
g = Math.max(0, Math.min(g, 262143));
b = Math.max(0, Math.min(b, 262143));
// combine ARGB
rgba[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00)
| ((b >> 10) | 0xff);
}
}
}
Try RenderScript ScriptIntrinsicYuvToRGB, which comes with JellyBean 4.2 (Api 17+).
https://developer.android.com/reference/android/renderscript/ScriptIntrinsicYuvToRGB.html
On Nexus 7 (2013, JellyBean 4.3) a 1920x1080 image conversion (full HD camera preview) takes about 7 ms.
You can get the bitmap directly from the TextureView. Which is really fast.
Bitmap bitmap = textureview.getBitmap()
After reading many suggested links, articles, etc. I found the following great Android example app which captures the YUV Image from the camera and converts it into RGB Bitmap:
https://github.com/android/camera-samples/tree/main/CameraXTfLite
Nice things about this:
It uses the aforementioned RenderScript framework and the code can be easily reused - check out the YuvToRgbConverter.kt class
according to their documentation, this code achieves " ~30 FPS # 640x480 on a Pixel 3 phone"
After switching to this code (especially the YUV to RGB conversion part) my framerate doubled! I am not quite reaching 30 FPS overall since I am doing a bit more things after capturing the image, but the speed-up is remarkable!