Related
What I am trying?
I am trying to take picture with drawable/paint on face but, i am not able to get both on same picture.
What I have tried?
I have tried using CameraSource.takePicture but i am just getting face without any drawable/paint on it.
mCameraSource.takePicture(shutterCallback, new CameraSource.PictureCallback() {
#Override
public void onPictureTaken(byte[] bytes) {
try {
String mainpath = getExternalStorageDirectory() + separator + "TestXyz" + separator + "images" + separator;
File basePath = new File(mainpath);
if (!basePath.exists())
Log.d("CAPTURE_BASE_PATH", basePath.mkdirs() ? "Success": "Failed");
String path = mainpath + "photo_" + getPhotoTime() + ".jpg";
File captureFile = new File(path);
captureFile.createNewFile();
if (!captureFile.exists())
Log.d("CAPTURE_FILE_PATH", captureFile.createNewFile() ? "Success": "Failed");
FileOutputStream stream = new FileOutputStream(captureFile);
stream.write(bytes);
stream.flush();
stream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
});
I also tried using :
mPreview.setDrawingCacheEnabled(true);
Bitmap drawingCache = mPreview.getDrawingCache();
try {
String mainpath = getExternalStorageDirectory() + separator + "TestXyz" + separator + "images" + separator;
File basePath = new File(mainpath);
if (!basePath.exists())
Log.d("CAPTURE_BASE_PATH", basePath.mkdirs() ? "Success": "Failed");
String path = mainpath + "photo_" + getPhotoTime() + ".jpg";
File captureFile = new File(path);
captureFile.createNewFile();
if (!captureFile.exists())
Log.d("CAPTURE_FILE_PATH", captureFile.createNewFile() ? "Success": "Failed");
FileOutputStream stream = new FileOutputStream(captureFile);
drawingCache.compress(Bitmap.CompressFormat.PNG, 100, stream);
stream.flush();
stream.close();
} catch (IOException e) {
e.printStackTrace();
}
in this case i am only getting what i draw on face. Here, mPreview is the CameraSourcePreview.
Just added capture button and added above code in this google example.
You are very close to achieve what you need :)
You have:
An image from the Camera of the face (First code snippet)
An image from the Canvas of the eyes overlay (Second code snippet)
What you need:
An image that has the face with the eyes overlay on top - A merged image.
How to merge?
To merge 2 images simply use a canvas, like so:
public Bitmap mergeBitmaps(Bitmap face, Bitmap overlay) {
// Create a new image with target size
int width = face.getWidth();
int height = face.getHeight();
Bitmap newBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
Rect faceRect = new Rect(0,0,width,height);
Rect overlayRect = new Rect(0,0,overlay.getWidth(),overlay.getHeight());
// Draw face and then overlay (Make sure rects are as needed)
Canvas canvas = new Canvas(newBitmap);
canvas.drawBitmap(face, faceRect, faceRect, null);
canvas.drawBitmap(overlay, overlayRect, faceRect, null);
return newBitmap
}
Then you can save the new image, as you are doing now.
Full code would look like:
mCameraSource.takePicture(shutterCallback, new
CameraSource.PictureCallback() {
#Override
public void onPictureTaken(byte[] bytes) {
// Generate the Face Bitmap
BitmapFactory.Options options = new BitmapFactory.Options();
Bitmap face = BitmapFactory.decodeByteArray(bytes, 0, bytes.length, options);
// Generate the Eyes Overlay Bitmap
mPreview.setDrawingCacheEnabled(true);
Bitmap overlay = mPreview.getDrawingCache();
// Generate the final merged image
Bitmap result = mergeBitmaps(face, overlay);
// Save result image to file
try {
String mainpath = getExternalStorageDirectory() + separator + "TestXyz" + separator + "images" + separator;
File basePath = new File(mainpath);
if (!basePath.exists())
Log.d("CAPTURE_BASE_PATH", basePath.mkdirs() ? "Success": "Failed");
String path = mainpath + "photo_" + getPhotoTime() + ".jpg";
File captureFile = new File(path);
captureFile.createNewFile();
if (!captureFile.exists())
Log.d("CAPTURE_FILE_PATH", captureFile.createNewFile() ? "Success": "Failed");
FileOutputStream stream = new FileOutputStream(captureFile);
result.compress(Bitmap.CompressFormat.PNG, 100, stream);
stream.flush();
stream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
});
Note that the above is just an example code.
You should probably move the merging and saving to a file to a background thread.
I am able to capture image with drawable/paint on it by below solution :
private void captureImage() {
mPreview.setDrawingCacheEnabled(true);
Bitmap drawingCache = mPreview.getDrawingCache();
mCameraSource.takePicture(shutterCallback, new CameraSource.PictureCallback() {
#Override
public void onPictureTaken(byte[] bytes) {
int orientation = Exif.getOrientation(bytes);
Bitmap temp = BitmapFactory.decodeByteArray(bytes, 0, bytes.length);
Bitmap picture = rotateImage(temp,orientation);
Bitmap overlay = Bitmap.createBitmap(mGraphicOverlay.getWidth(),mGraphicOverlay.getHeight(),picture.getConfig());
Canvas canvas = new Canvas(overlay);
Matrix matrix = new Matrix();
matrix.setScale((float)overlay.getWidth()/(float)picture.getWidth(),(float)overlay.getHeight()/(float)picture.getHeight());
// mirror by inverting scale and translating
matrix.preScale(-1, 1);
matrix.postTranslate(canvas.getWidth(), 0);
Paint paint = new Paint();
canvas.drawBitmap(picture,matrix,paint);
canvas.drawBitmap(drawingCache,0,0,paint);
try {
String mainpath = getExternalStorageDirectory() + separator + "MaskIt" + separator + "images" + separator;
File basePath = new File(mainpath);
if (!basePath.exists())
Log.d("CAPTURE_BASE_PATH", basePath.mkdirs() ? "Success": "Failed");
String path = mainpath + "photo_" + getPhotoTime() + ".jpg";
File captureFile = new File(path);
captureFile.createNewFile();
if (!captureFile.exists())
Log.d("CAPTURE_FILE_PATH", captureFile.createNewFile() ? "Success": "Failed");
FileOutputStream stream = new FileOutputStream(captureFile);
overlay.compress(Bitmap.CompressFormat.PNG, 100, stream);
stream.flush();
stream.close();
picture.recycle();
drawingCache.recycle();
mPreview.setDrawingCacheEnabled(false);
} catch (IOException e) {
e.printStackTrace();
}
}
});
}
Sometimes orientation issue also occurs on some devices. For that i used Exif class and rotateImage() function.
Exif Class (reference from here) :
public class Exif {
private static final String TAG = "CameraExif";
// Returns the degrees in clockwise. Values are 0, 90, 180, or 270.
public static int getOrientation(byte[] jpeg) {
if (jpeg == null) {
return 0;
}
int offset = 0;
int length = 0;
// ISO/IEC 10918-1:1993(E)
while (offset + 3 < jpeg.length && (jpeg[offset++] & 0xFF) == 0xFF) {
int marker = jpeg[offset] & 0xFF;
// Check if the marker is a padding.
if (marker == 0xFF) {
continue;
}
offset++;
// Check if the marker is SOI or TEM.
if (marker == 0xD8 || marker == 0x01) {
continue;
}
// Check if the marker is EOI or SOS.
if (marker == 0xD9 || marker == 0xDA) {
break;
}
// Get the length and check if it is reasonable.
length = pack(jpeg, offset, 2, false);
if (length < 2 || offset + length > jpeg.length) {
Log.e(TAG, "Invalid length");
return 0;
}
// Break if the marker is EXIF in APP1.
if (marker == 0xE1 && length >= 8 &&
pack(jpeg, offset + 2, 4, false) == 0x45786966 &&
pack(jpeg, offset + 6, 2, false) == 0) {
offset += 8;
length -= 8;
break;
}
// Skip other markers.
offset += length;
length = 0;
}
// JEITA CP-3451 Exif Version 2.2
if (length > 8) {
// Identify the byte order.
int tag = pack(jpeg, offset, 4, false);
if (tag != 0x49492A00 && tag != 0x4D4D002A) {
Log.e(TAG, "Invalid byte order");
return 0;
}
boolean littleEndian = (tag == 0x49492A00);
// Get the offset and check if it is reasonable.
int count = pack(jpeg, offset + 4, 4, littleEndian) + 2;
if (count < 10 || count > length) {
Log.e(TAG, "Invalid offset");
return 0;
}
offset += count;
length -= count;
// Get the count and go through all the elements.
count = pack(jpeg, offset - 2, 2, littleEndian);
while (count-- > 0 && length >= 12) {
// Get the tag and check if it is orientation.
tag = pack(jpeg, offset, 2, littleEndian);
if (tag == 0x0112) {
// We do not really care about type and count, do we?
int orientation = pack(jpeg, offset + 8, 2, littleEndian);
switch (orientation) {
case 1:
return 0;
case 3:
return 3;
case 6:
return 6;
case 8:
return 8;
}
Log.i(TAG, "Unsupported orientation");
return 0;
}
offset += 12;
length -= 12;
}
}
Log.i(TAG, "Orientation not found");
return 0;
}
private static int pack(byte[] bytes, int offset, int length,
boolean littleEndian) {
int step = 1;
if (littleEndian) {
offset += length - 1;
step = -1;
}
int value = 0;
while (length-- > 0) {
value = (value << 8) | (bytes[offset] & 0xFF);
offset += step;
}
return value;
}
}
rotateImage function :
private Bitmap rotateImage(Bitmap bm, int i) {
Matrix matrix = new Matrix();
switch (i) {
case ExifInterface.ORIENTATION_NORMAL:
return bm;
case ExifInterface.ORIENTATION_FLIP_HORIZONTAL:
matrix.setScale(-1, 1);
break;
case ExifInterface.ORIENTATION_ROTATE_180:
matrix.setRotate(180);
break;
case ExifInterface.ORIENTATION_FLIP_VERTICAL:
matrix.setRotate(180);
matrix.postScale(-1, 1);
break;
case ExifInterface.ORIENTATION_TRANSPOSE:
matrix.setRotate(90);
matrix.postScale(-1, 1);
break;
case ExifInterface.ORIENTATION_ROTATE_90:
matrix.setRotate(90);
break;
case ExifInterface.ORIENTATION_TRANSVERSE:
matrix.setRotate(-90);
matrix.postScale(-1, 1);
break;
case ExifInterface.ORIENTATION_ROTATE_270:
matrix.setRotate(-90);
break;
default:
return bm;
}
try {
Bitmap bmRotated = Bitmap.createBitmap(bm, 0, 0, bm.getWidth(), bm.getHeight(), matrix, true);
bm.recycle();
return bmRotated;
} catch (OutOfMemoryError e) {
e.printStackTrace();
return null;
}
}
You can achieve the effect that you want by breaking it into smaller steps.
Take the picture
Send the bitmap to Google Mobile Vision to detect the "landmarks" in the face and the probability that each eye is open
Paint the appropriate "eyes" onto your image
When using Google Mobile Vision's FaceDetector, you'll get back a SparseArray of Face objects (which may contain more than one face, or which may be empty). So you'll need to handle these cases. But you can loop through the SparseArray and find the Face object that you want to play with.
static Bitmap processFaces(Context context, Bitmap picture) {
// Create a "face detector" object, using the builder pattern
FaceDetector detector = new FaceDetector.Builder(context)
.setTrackingEnabled(false) // disable tracking to improve performance
.setClassificationType(FaceDetector.ALL_CLASSIFICATIONS)
.build();
// create a "Frame" object, again using a builder pattern (and passing in our picture)
Frame frame = new Frame.Builder().setBitmap(picture).build(); // build frame
// get a sparse array of face objects
SparseArray<Face> faces = detector.detect(frame); // detect the faces
// This example just deals with a single face for the sake of simplicity,
// but you can change this to deal with multiple faces.
if (faces.size() != 1) return picture;
// make a mutable copy of the background image that we can modify
Bitmap bmOverlay = Bitmap.createBitmap(picture.getWidth(), picture.getHeight(), picture.getConfig());
Canvas canvas = new Canvas(bmOverlay);
canvas.drawBitmap(picture, 0, 0, null);
// get the Face object that we want to manipulate, and process it
Face face = faces.valueAt(0);
processFace(face, canvas);
detector.release();
return bmOverlay;
}
Once you've got a Face object, you can find the features that interest you like this
private static void processFace(Face face, Canvas canvas) {
// The Face object can tell you the probability that each eye is open.
// I'm comparing this probability to an arbitrary threshold of 0.6 here,
// but you can vary it between 0 and 1 as you please.
boolean leftEyeClosed = face.getIsLeftEyeOpenProbability() < .6;
boolean rightEyeClosed = face.getIsRightEyeOpenProbability() < .6;
// Loop through the face's "landmarks" (eyes, nose, etc) to find the eyes.
// landmark.getPosition() gives you the (x,y) coordinates of each feature.
for (Landmark landmark : face.getLandmarks()) {
if (landmark.getType() == Landmark.LEFT_EYE)
overlayEyeBitmap(canvas, leftEyeClosed, landmark.getPosition().x, landmark.getPosition().y);
if (landmark.getType() == Landmark.RIGHT_EYE)
overlayEyeBitmap(canvas, rightEyeClosed, landmark.getPosition().x, landmark.getPosition().y);
}
}
Then you can add your paint!
private static void overlayEyeBitmap(Canvas canvas, boolean eyeClosed, float cx, float cy) {
float radius = 40;
// draw the eye's background circle with appropriate color
Paint paintFill = new Paint();
paintFill.setStyle(Paint.Style.FILL);
if (eyeClosed)
paintFill.setColor(Color.YELLOW);
else
paintFill.setColor(Color.WHITE);
canvas.drawCircle(cx, cy, radius, paintFill);
// draw a black border around the eye
Paint paintStroke = new Paint();
paintStroke.setColor(Color.BLACK);
paintStroke.setStyle(Paint.Style.STROKE);
paintStroke.setStrokeWidth(5);
canvas.drawCircle(cx, cy, radius, paintStroke);
if (eyeClosed)
// draw horizontal line across closed eye
canvas.drawLine(cx - radius, cy, cx + radius, cy, paintStroke);
else {
// draw big off-center pupil on open eye
paintFill.setColor(Color.BLACK);
float cxPupil = cx - 10;
float cyPupil = cy + 10;
canvas.drawCircle(cxPupil, cyPupil, 25, paintFill);
}
}
In the snippet above, I just hardcoded the eye radii, to show proof of concept. You'll probably want to do some more flexible scaling, using some percentage of face.getWidth() to determine the appropriate values. But here's what this image processing can do:
Some more details about the Mobile Vision API are here, and Udacity's current Advanced Android course has a nice walkthrough of this stuff (taking a picture, sending it to Mobile Vision, and adding a bitmap onto it). The course is free, or you can just look at what they did on Github.
Note: All info in my post only goes for Samsung Galaxy S7 device. I do not know how emulators and other devices behave.
In onImageAvailable I convert continuously each image to a NV21 byte array and forward it to an API expecting raw NV21 format.
This is how I initialize the image reader and receive the images:
private void openCamera() {
...
mImageReader = ImageReader.newInstance(WIDTH, HEIGHT,
ImageFormat.YUV_420_888, 1); // only 1 for best performance
mImageReader.setOnImageAvailableListener(
mOnImageAvailableListener, mBackgroundHandler);
...
}
private final ImageReader.OnImageAvailableListener mOnImageAvailableListener
= new ImageReader.OnImageAvailableListener() {
#Override
public void onImageAvailable(ImageReader reader) {
Image image = reader.acquireLatestImage();
if (image != null) {
byte[] data = convertYUV420ToNV21_ALL_PLANES(image); // this image is turned 90 deg using front cam in portrait mode
byte[] data_rotated = rotateNV21_working(data, WIDTH, HEIGHT, 270);
ForwardToAPI(data_rotated); // image data is being forwarded to api and received later on
image.close();
}
}
};
The function converting the image to raw NV21 (from here), working fine, the image is (due to android?) turned by 90 degrees when using front cam in portrait mode:
(I modified it, slightly according to comments of Alex Cohn)
private byte[] convertYUV420ToNV21_ALL_PLANES(Image imgYUV420) {
byte[] rez;
ByteBuffer buffer0 = imgYUV420.getPlanes()[0].getBuffer();
ByteBuffer buffer1 = imgYUV420.getPlanes()[1].getBuffer();
ByteBuffer buffer2 = imgYUV420.getPlanes()[2].getBuffer();
// actually here should be something like each second byte
// however I simply get the last byte of buffer 2 and the entire buffer 1
int buffer0_size = buffer0.remaining();
int buffer1_size = buffer1.remaining(); // / 2 + 1;
int buffer2_size = 1;//buffer2.remaining(); // / 2 + 1;
byte[] buffer0_byte = new byte[buffer0_size];
byte[] buffer1_byte = new byte[buffer1_size];
byte[] buffer2_byte = new byte[buffer2_size];
buffer0.get(buffer0_byte, 0, buffer0_size);
buffer1.get(buffer1_byte, 0, buffer1_size);
buffer2.get(buffer2_byte, buffer2_size-1, buffer2_size);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try {
// swap 1 and 2 as blue and red colors are swapped
outputStream.write(buffer0_byte);
outputStream.write(buffer2_byte);
outputStream.write(buffer1_byte);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
rez = outputStream.toByteArray();
return rez;
}
Hence "data" needs to be rotated. Using this function (from here), I get a weird 3-times interlaced picture error:
public static byte[] rotateNV21(byte[] input, int width, int height, int rotation) {
byte[] output = new byte[input.length];
boolean swap = (rotation == 90 || rotation == 270);
// **EDIT:** in portrait mode & front cam this needs to be set to true:
boolean yflip = true;// (rotation == 90 || rotation == 180);
boolean xflip = (rotation == 270 || rotation == 180);
for (int x = 0; x < width; x++) {
for (int y = 0; y < height; y++) {
int xo = x, yo = y;
int w = width, h = height;
int xi = xo, yi = yo;
if (swap) {
xi = w * yo / h;
yi = h * xo / w;
}
if (yflip) {
yi = h - yi - 1;
}
if (xflip) {
xi = w - xi - 1;
}
output[w * yo + xo] = input[w * yi + xi];
int fs = w * h;
int qs = (fs >> 2);
xi = (xi >> 1);
yi = (yi >> 1);
xo = (xo >> 1);
yo = (yo >> 1);
w = (w >> 1);
h = (h >> 1);
// adjust for interleave here
int ui = fs + (w * yi + xi) * 2;
int uo = fs + (w * yo + xo) * 2;
// and here
int vi = ui + 1;
int vo = uo + 1;
output[uo] = input[ui];
output[vo] = input[vi];
}
}
return output;
}
Resulting into this picture:
Note: it is still the same cup, however you see it 3-4 times.
Using another suggested rotate function from here gives the proper result:
public static byte[] rotateNV21_working(final byte[] yuv,
final int width,
final int height,
final int rotation)
{
if (rotation == 0) return yuv;
if (rotation % 90 != 0 || rotation < 0 || rotation > 270) {
throw new IllegalArgumentException("0 <= rotation < 360, rotation % 90 == 0");
}
final byte[] output = new byte[yuv.length];
final int frameSize = width * height;
final boolean swap = rotation % 180 != 0;
final boolean xflip = rotation % 270 != 0;
final boolean yflip = rotation >= 180;
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
final int yIn = j * width + i;
final int uIn = frameSize + (j >> 1) * width + (i & ~1);
final int vIn = uIn + 1;
final int wOut = swap ? height : width;
final int hOut = swap ? width : height;
final int iSwapped = swap ? j : i;
final int jSwapped = swap ? i : j;
final int iOut = xflip ? wOut - iSwapped - 1 : iSwapped;
final int jOut = yflip ? hOut - jSwapped - 1 : jSwapped;
final int yOut = jOut * wOut + iOut;
final int uOut = frameSize + (jOut >> 1) * wOut + (iOut & ~1);
final int vOut = uOut + 1;
output[yOut] = (byte)(0xff & yuv[yIn]);
output[uOut] = (byte)(0xff & yuv[uIn]);
output[vOut] = (byte)(0xff & yuv[vIn]);
}
}
return output;
}
The result is fine now:
The top image shows the direct stream using a texture view's surface and adding it to the captureRequestBuilder. The bottom image shows the raw image data after rotating.
The questions are:
Does this hack in "convertYUV420ToNV21_ALL_PLANES" work on any
device/emulator?
Why does rotateNV21 not work, while rotateNV21_working works fine.
Edit: The mirror issue is fixed, see code comment. The squeeze issue is fixed, it was caused by the API it gets forwarded.
The actual open issue is a proper not too expensive function, converting and rotating an image into raw NV21 working on any device.
Here is the code to convert the Image to NV21 byte[]. This will work when the imgYUV420 U and V planes have pixelStride=1 (as on emulator) or pixelStride=2 (as on Nexus):
private byte[] convertYUV420ToNV21_ALL_PLANES(Image imgYUV420) {
assert(imgYUV420.getFormat() == ImageFormat.YUV_420_888);
Log.d(TAG, "image: " + imgYUV420.getWidth() + "x" + imgYUV420.getHeight() + " " + imgYUV420.getFormat());
Log.d(TAG, "planes: " + imgYUV420.getPlanes().length);
for (int nplane = 0; nplane < imgYUV420.getPlanes().length; nplane++) {
Log.d(TAG, "plane[" + nplane + "]: length " + imgYUV420.getPlanes()[nplane].getBuffer().remaining() + ", strides: " + imgYUV420.getPlanes()[nplane].getPixelStride() + " " + imgYUV420.getPlanes()[nplane].getRowStride());
}
byte[] rez = new byte[imgYUV420.getWidth() * imgYUV420.getHeight() * 3 / 2];
ByteBuffer buffer0 = imgYUV420.getPlanes()[0].getBuffer();
ByteBuffer buffer1 = imgYUV420.getPlanes()[1].getBuffer();
ByteBuffer buffer2 = imgYUV420.getPlanes()[2].getBuffer();
int n = 0;
assert(imgYUV420.getPlanes()[0].getPixelStride() == 1);
for (int row = 0; row < imgYUV420.getHeight(); row++) {
for (int col = 0; col < imgYUV420.getWidth(); col++) {
rez[n++] = buffer0.get();
}
}
assert(imgYUV420.getPlanes()[2].getPixelStride() == imgYUV420.getPlanes()[1].getPixelStride());
int stride = imgYUV420.getPlanes()[1].getPixelStride();
for (int row = 0; row < imgYUV420.getHeight(); row += 2) {
for (int col = 0; col < imgYUV420.getWidth(); col += 2) {
rez[n++] = buffer1.get();
rez[n++] = buffer2.get();
for (int skip = 1; skip < stride; skip++) {
if (buffer1.remaining() > 0) {
buffer1.get();
}
if (buffer2.remaining() > 0) {
buffer2.get();
}
}
}
}
Log.w(TAG, "total: " + rez.length);
return rez;
}
optimized Java code is available here.
As you can see, it is very easy to change this code to produce a rotated image in a single step:
private byte[] rotateYUV420ToNV21(Image imgYUV420) {
Log.d(TAG, "image: " + imgYUV420.getWidth() + "x" + imgYUV420.getHeight() + " " + imgYUV420.getFormat());
Log.d(TAG, "planes: " + imgYUV420.getPlanes().length);
for (int nplane = 0; nplane < imgYUV420.getPlanes().length; nplane++) {
Log.d(TAG, "plane[" + nplane + "]: length " + imgYUV420.getPlanes()[nplane].getBuffer().remaining() + ", strides: " + imgYUV420.getPlanes()[nplane].getPixelStride() + " " + imgYUV420.getPlanes()[nplane].getRowStride());
}
byte[] rez = new byte[imgYUV420.getWidth() * imgYUV420.getHeight() * 3 / 2];
ByteBuffer buffer0 = imgYUV420.getPlanes()[0].getBuffer();
ByteBuffer buffer1 = imgYUV420.getPlanes()[1].getBuffer();
ByteBuffer buffer2 = imgYUV420.getPlanes()[2].getBuffer();
int width = imgYUV420.getHeight();
assert(imgYUV420.getPlanes()[0].getPixelStride() == 1);
for (int row = imgYUV420.getHeight()-1; row >=0; row--) {
for (int col = 0; col < imgYUV420.getWidth(); col++) {
rez[col*width+row] = buffer0.get();
}
}
int uv_offset = imgYUV420.getWidth()*imgYUV420.getHeight();
assert(imgYUV420.getPlanes()[2].getPixelStride() == imgYUV420.getPlanes()[1].getPixelStride());
int stride = imgYUV420.getPlanes()[1].getPixelStride();
for (int row = imgYUV420.getHeight() - 2; row >= 0; row -= 2) {
for (int col = 0; col < imgYUV420.getWidth(); col += 2) {
rez[uv_offset+col/2*width+row] = buffer1.get();
rez[uv_offset+col/2*width+row+1] = buffer2.get();
for (int skip = 1; skip < stride; skip++) {
if (buffer1.remaining() > 0) {
buffer1.get();
}
if (buffer2.remaining() > 0) {
buffer2.get();
}
}
}
}
Log.w(TAG, "total rotated: " + rez.length);
return rez;
}
I sincerely recommend the site http://rawpixels.net/ to see the actual structure of your raw images.
With OpenCV and Android Camera API 2 this task is very fast and you don't need YUV420toNV21 Java conversion, and with OpenCV this convertion is 4x more fast:
Java side:
//Starts a builtin camera with api camera 2
public void startCamera() {
CameraManager manager = (CameraManager) AppData.getAppContext().getSystemService(Context.CAMERA_SERVICE);
try {
String pickedCamera = getCamera(manager);
manager.openCamera(pickedCamera, cameraStateCallback, null);
// set image format on YUV
mImageReader = ImageReader.newInstance(mWidth,mHeight, ImageFormat.YUV_420_888, 4);
mImageReader.setOnImageAvailableListener(onImageAvailableListener, null);
Log.d(TAG, "imageReader created");
} catch (CameraAccessException e) {
Log.e(TAG, e.getMessage());
}
}
//Listens for frames and send them to be processed
protected ImageReader.OnImageAvailableListener onImageAvailableListener = new ImageReader.OnImageAvailableListener() {
#Override
public void onImageAvailable(ImageReader reader) {
Image image = null;
try {
image = reader.acquireLatestImage();
ByteBuffer buffer = image.getPlanes()[0].getBuffer();
byte[] frameData = new byte[buffer.capacity()];
buffer.get(frameData);
// Native process (see below)
processAndRotateFrame(frameData);
image.close();
} catch (Exception e) {
Logger.e(TAG, "imageReader exception: "+e.getMessage());
} finally {
if (image != null) {
image.close();
}
}
}
};
Native side (NDK or Cmake):
JNIEXPORT jint JNICALL com_android_mvf_Utils_ProccessAndRotateFrame
(JNIEnv *env, jobject object, jint width, jint height, jbyteArray frame, jint rotation) {
// load data from JAVA side
jbyte *pFrameData = env->GetByteArrayElements(frame, 0);
// convert array to Mat, for example GRAY or COLOR
Mat mGray(height, width, cv::IMREAD_GRAYSCALE, (unsigned char *)pFrameData);
// rotate image
rotateMat(mGray, rotation);
int objects = your_function(env, mGray);
env->ReleaseByteArrayElements(frame, pFrameData, 0);
return objects;
}
void rotateMat(cv::Mat &matImage, int rotFlag) {
if (rotFlag != 0 && rotFlag != 360) {
if (rotFlag == 90) {
cv::transpose(matImage, matImage);
cv::flip(matImage, matImage, 1);
} else if (rotFlag == 270 || rotFlag == -90) {
cv::transpose(matImage, matImage);
cv::flip(matImage, matImage, 0);
} else if (rotFlag == 180) {
cv::flip(matImage, matImage, -1);
}
}
}
I would really appreciate if you can help me with this little problem I`ve had for a while now.
The thing is that when I print an image or qr code I only gets a lot of 'y' as result.
Here the function that I used to print qr code:
printQR() {
QRCodeEncoder qrCodeEncoder = new QRCodeEncoder("stringQR",
null,
"TEXT_TYPE",
BarcodeFormat.QR_CODE.toString(),
250);
try {
Bitmap bitmapQR = qrCodeEncoder.encodeAsBitmap(); // Here
int size = bitmapQR.getRowBytes() * bitmapQR.getHeight();
ByteBuffer byteBuffer = ByteBuffer.allocate(size);
bitmapQR.copyPixelsToBuffer(byteBuffer);
byte[] byteArray = byteBuffer.array();
// I think here is the problem
byte[] formats = {(byte)0x1B,(byte)0x58,(byte)0x31,(byte)0x20,(byte)0xFA};
byte[] bytes = new byte[formats.length + byteArray.length];
System.arraycopy(formats, 0, bytes, 0, formats.length);
System.arraycopy(byteArray, 0, bytes, formats.length, byteArray.length);
try {
BluetoothService.write(bytes);
BluetoothService.write("\n\n".getBytes());
} catch (IOException e) {
e.printStackTrace();
}
} catch (WriterException e) {
e.printStackTrace();
}
}
public QREncoder(String data, Bundle bundle, String type, String format, int dimension) {
this.dimension = dimension;
encoded = encodeContents(data, bundle, type, format);
}
private boolean encodeContents(String data, Bundle bundle, String type, String formatString) {
format = null;
if (formatString != null) {
try {
format = BarcodeFormat.valueOf(formatString);
} catch (IllegalArgumentException iae) { }
}
if (format == null || format == BarcodeFormat.QR_CODE) {
this.format = BarcodeFormat.QR_CODE;
encodeQRCodeContents(data, bundle, type);
} else if (data != null && data.length() > 0) {
contents = data;
displayContents = data;
title = "Text";
}
return contents != null && contents.length() > 0;
}
public Bitmap encodeAsBitmap() throws WriterException {
if (!encoded) return null;
Map<EncodeHintType, Object> hints = null;
String encoding = guessAppropriateEncoding(contents);
if (encoding != null) {
hints = new EnumMap<EncodeHintType, Object>(EncodeHintType.class);
hints.put(EncodeHintType.CHARACTER_SET, encoding);
}
MultiFormatWriter writer = new MultiFormatWriter();
BitMatrix result = writer.encode(contents, format, dimension, dimension, hints);
int width = result.getWidth();
int height = result.getHeight();
int[] pixels = new int[width * height];
// All are 0, or black, by default
for (int y = 0; y < height; y++) {
int offset = y * width;
for (int x = 0; x < width; x++) {
pixels[offset + x] = result.get(x, y) ? BLACK : WHITE;
}
}
Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
bitmap.setPixels(pixels, 0, width, 0, 0, width, height);
return bitmap;
}
The size of the QR image is 250 x 250, thats why I put 0x20 0xFA in byte[] formats
I am working on an android app. I can detect lines using Canny and Hough transform but I don't know how to get the angles and lengths of the detected lines, can you help please? Here's the code:
public void countStraightLines(View view) {
try{
int iCannyLowerThreshold = 45;
int iCannyUpperThreshold = 75;
Mat rgba = Utils.loadResource(getApplicationContext(), res);
Bitmap bmp = Bitmap.createBitmap(rgba.width(), rgba.height(), Bitmap.Config.ARGB_8888);
Mat gray = new Mat ();
Imgproc.cvtColor(rgba, gray, Imgproc.COLOR_BGRA2GRAY, 4);
Imgproc.Canny(gray, gray, iCannyLowerThreshold, iCannyUpperThreshold);
Utils.matToBitmap(gray, bmp);
imgSource.setImageBitmap(bmp);
} catch (IOException e) {
Log.e(TAG, "ERROR Loading Mat");
e.printStackTrace();
}
}
#Override
public void onItemSelected(AdapterView<?> parent, View view, int position,
long id) {
Globals.pictSelected=parent.getItemAtPosition(position).toString();
res = getResources().getIdentifier(parent.getItemAtPosition(position).toString(), "drawable", this.getPackageName());
try {
Mat rgba = Utils.loadResource(getApplicationContext(), res);
Bitmap bmp = Bitmap.createBitmap(rgba.width(), rgba.height(), Bitmap.Config.ARGB_8888);
Mat gray = new Mat ();
Imgproc.cvtColor(rgba, gray, Imgproc.COLOR_BGRA2GRAY, 4);
int iCannyLowerThreshold = 45;
int iCannyUpperThreshold = 75;
int iHoughLinesThreshold = 50;
int iHoughLinesMinLineSize = 40;
int iHoughLinesGap = 20;
Imgproc.Canny(gray, gray, iCannyLowerThreshold, iCannyUpperThreshold);
Mat lines = new Mat();
Imgproc.HoughLinesP(gray, lines, 1, Math.PI/180, iHoughLinesThreshold, iHoughLinesMinLineSize, iHoughLinesGap);
int x = 0;
char s = 'N';
for (; x < Math.min(lines.cols(), 100); x++)
{
double[] vec = lines.get(0, x);
if (vec == null)
break;
double x1 = vec[0], y1 = vec[1], x2 = vec[2], y2 = vec[3];
Point start = new Point(x1, y1);
Point end = new Point(x2, y2);
Core.line(rgba, start, end, new Scalar(255, 0, 0, 255), 1);
if (x >= 40){s = 'C';}
else {s = 'S';}
}
text.setText("Line Count: " + x + " and The Picture is " + s);
Utils.matToBitmap(rgba, bmp);
imgSource.setImageBitmap(bmp);
} catch (IOException e) {
Log.e(TAG, "ERROR Loading Mat");
e.printStackTrace();
}
}
Thank you!
Well, it looks like you have a start point and an end point- the length is just ((starty-endy)^2+(startx-endx)^2)^(1/2). You can get the angle between them by using the fact that the dot product of two vectors A and B A.B=|A|*|B|*cos(alpha) where alpha is the angle between them. So alpha= arccos((A.B)/(|A|*|B|)).
I am using android "opencv samples - face detection" to detect the face. It is working good. Now I want to include "eye detection" with "face detection". Is it possible? if yes can anyone give me some samples to detect the eye. I have searched over the internet. But I didn't get any samples.
You can detect the eyes using this code...(on number of faces)
//code...
public void setFace() {
FaceDetector fd;
mFaceBitmap = mBitmap.copy(Bitmap.Config.RGB_565, true);
mFaceWidth = mFaceBitmap.getWidth();
mFaceHeight = mFaceBitmap.getHeight();
FaceDetector.Face [] faces = new FaceDetector.Face[MAX_FACES];
PointF eyescenter = new PointF();
float eyesdist = 0.0f;
int [] fpx = null;
int [] fpy = null;
try {
Log.e("setFace(): ", "try");
fd = new FaceDetector(mFaceWidth, mFaceHeight, MAX_FACES);
count = fd.findFaces(mFaceBitmap, faces);
} catch (Exception e) {
return;
}
// check if we detect any faces
if (count > 0) {
fpx = new int[count * 1];
fpy = new int[count * 1];
for (int i = 0; i < count; i++) {
try {
Log.e("setFace(): ", "loop");
faces[i].getMidPoint(eyescenter);
eyesdist = faces[i].eyesDistance();
// set up left eye location
fpx[1 * i] = (int)(eyescenter.x - eyesdist / 2);
fpy[1 * i] = (int)eyescenter.y;
// set up right eye location
fpx[1 * i + 1] = (int)(eyescenter.x + eyesdist / 2);
fpy[1 * i + 1] = (int)eyescenter.y;
} catch (Exception e) {
}
}
MainActivity.clicked = true;
}
}
I think that is possible. You need to load another cascade (nested cascade) and use the detectMultiScale in the ROI detected by the face cascade. Take a look at the facedetect.cpp example in the c samples.
Yes, you can do this. There is both a generic eye detector, and specific left/right eye detectors available.
You can use the same Haar cascade code you used for faces, just substituting the eye detectors.
In practice, I've found that it's better to try both - start with the generic eye detector, then if nothing is found in a reasonable location, move on to the left and right eye detectors.
You can go through this code
public FdView(Context context) {
super(context);
this.context = context;
try {
InputStream is = context.getResources().openRawResource(
R.raw.lbpcascade_frontalface);
File cascadeDir = context.getDir("cascade", Context.MODE_PRIVATE);
mCascadeFile = new File(cascadeDir, "lbpcascade_frontalface.xml");
FileOutputStream os = new FileOutputStream(mCascadeFile);
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = is.read(buffer)) != -1) {
os.write(buffer, 0, bytesRead);
}
is.close();
os.close();
// ------------------------- load left eye classificator//
// -----------------------------------
InputStream iser = context.getResources().openRawResource(
R.raw.haarcascade_lefteye_2splits);
File cascadeDirER = context.getDir("cascadeER",
Context.MODE_PRIVATE);
File cascadeFileER = new File(cascadeDirER,
"haarcascade_eye_right.xml");
FileOutputStream oser = new FileOutputStream(cascadeFileER);
byte[] bufferER = new byte[4096];
int bytesReadER;
while ((bytesReadER = iser.read(bufferER)) != -1) {
oser.write(bufferER, 0, bytesReadER);
}
iser.close();
oser.close();
// ----------------------------------------------------------------------------------------------------
// --------------------------------- load right eye classificator//
// ------------------------------------
InputStream isel = context.getResources().openRawResource(
R.raw.haarcascade_lefteye_2splits);
File cascadeDirEL = context.getDir("cascadeEL",
Context.MODE_PRIVATE);
File cascadeFileEL = new File(cascadeDirEL,
"haarcascade_eye_left.xml");
FileOutputStream osel = new FileOutputStream(cascadeFileEL);
byte[] bufferEL = new byte[4096];
int bytesReadEL;
while ((bytesReadEL = isel.read(bufferEL)) != -1) {
osel.write(bufferEL, 0, bytesReadEL);
}
isel.close();
osel.close();
// ------------------------------------------------------------------------------------------------------
mJavaDetector = new CascadeClassifier(
mCascadeFile.getAbsolutePath());
mCascadeER = new CascadeClassifier(cascadeFileER.getAbsolutePath());
mCascadeEL = new CascadeClassifier(cascadeFileER.getAbsolutePath());
if (mJavaDetector.empty() || mCascadeER.empty()
|| mCascadeEL.empty()) {
Log.e(TAG, "Failed to load cascade classifier");
mJavaDetector = null;
mCascadeER = null;
mCascadeEL = null;
} else
Log.i(TAG,
"Loaded cascade classifier from "
+ mCascadeFile.getAbsolutePath());
mNativeDetector = new DetectionBasedTracker(
mCascadeFile.getAbsolutePath(), 0);
cascadeDir.delete();
cascadeFileER.delete();
cascadeDirER.delete();
cascadeFileEL.delete();
cascadeDirEL.delete();
} catch (IOException e) {
e.printStackTrace();
Log.e(TAG, "Failed to load cascade. Exception thrown: " + e);
}
}
#Override
public void surfaceCreated(SurfaceHolder holder) {
synchronized (this) {
// initialize Mats before usage
mGray = new Mat();
mRgba = new Mat();
}
super.surfaceCreated(holder);
}
#Override
protected Bitmap processFrame(VideoCapture capture) {
capture.retrieve(mRgba, Highgui.CV_CAP_ANDROID_COLOR_FRAME_RGBA);
capture.retrieve(mGray, Highgui.CV_CAP_ANDROID_GREY_FRAME);
if (mAbsoluteFaceSize == 0) {
int height = mGray.rows();
if (Math.round(height * mRelativeFaceSize) > 0)
{
mAbsoluteFaceSize = Math.round(height * mRelativeFaceSize);
}
mNativeDetector.setMinFaceSize(mAbsoluteFaceSize);
}
MatOfRect faces = new MatOfRect();
if (mDetectorType == JAVA_DETECTOR) {
if (mJavaDetector != null)
mJavaDetector.detectMultiScale(mGray, faces, 1.1, 2, 2,
new Size(mAbsoluteFaceSize, mAbsoluteFaceSize),
new Size());
/*
* if (mZoomCorner == null || mZoomWindow == null)
* CreateAuxiliaryMats();
*/
Rect[] facesArray = faces.toArray();
for (int i = 0; i < facesArray.length; i++) {
Rect r = facesArray[i];
Core.rectangle(mGray, r.tl(), r.br(),new Scalar(255, 0, 255, 0), 3);
Core.rectangle(mRgba, r.tl(), r.br(),new Scalar(46, 139, 87, 5), 3);
//eyearea = new Rect(r.x + r.width / 8,(int) (r.y + (r.height / 4.5)), r.width - 10 * r.width/ 8, (int) (r.height / 3.0));
// Core.rectangle(mRgba, eyearea.tl(), eyearea.br(), new Scalar(255, 0, 0, 255), 3);
Rect eyearea_right = new Rect(r.x + r.width / 16,
(int) (r.y + (r.height / 4.5)),
(r.width - 2 * r.width / 16) / 2,(int) (r.height / 3.0));
Rect eyearea_left = new Rect(r.x + r.width / 16+ (r.width - 2 * r.width / 16) / 2,(int) (r.y + (r.height / 4.5)),(r.width - 2 * r.width / 16) / 2,(int) (r.height / 3.0));
// outer recTangles
//Core.rectangle(mRgba, eyearea_left.tl(), eyearea_left.br(),new Scalar(0, 255, 255, 255), 3);
//Core.rectangle(mRgba, eyearea_right.tl(), eyearea_right.br(),new Scalar(0, 255, 255, 255), 3);
teplateR = get_template(mCascadeER, eyearea_right, 3);
teplateL = get_template(mCascadeEL, eyearea_left, 3);
learn_frames++;
/*
* match_value = match_eye(eyearea_right, teplateR,
* FdActivity.method);
*
* match_value = match_eye(eyearea_left, teplateL,
* FdActivity.method); ; }
*/
/************************************************************************************************/
/*
* Imgproc.resize(mRgba.submat(eyearea_left), mZoomWindow2,
* mZoomWindow2.size());
* Imgproc.resize(mRgba.submat(eyearea_right), mZoomWindow,
* mZoomWindow.size());
*/
}
} else if (mDetectorType == NATIVE_DETECTOR) {
if (mNativeDetector != null)
mNativeDetector.detect(mGray, faces);
} else {
Log.e(TAG, "Detection method is not selected!");
}
Rect[] facesArray = faces.toArray();
for (int i = 0; i < facesArray.length; i++)
Core.rectangle(mRgba, facesArray[i].tl(), facesArray[i].br(),
FACE_RECT_COLOR, 3);
Bitmap bmp = Bitmap.createBitmap(mRgba.cols(), mRgba.rows(),
Bitmap.Config.ARGB_8888);
try {
Utils.matToBitmap(mRgba, bmp);
} catch (Exception e) {
Log.e(TAG,
"Utils.matToBitmap() throws an exception: "
+ e.getMessage());
bmp.recycle();
bmp = null;
}
return bmp;
}
Hope this will help you.