Develop Reference

Android Opencv save mat as a picture without drawn rectangle

I want to save a Mat mRgba as a picture with Imgcodecs.imwrite(Environment.getExternalStorageDirectory() + "/final-image.jpg", mRgba); and in general it saves more that I want to. I want to save image without rectangle Imgproc.rectangle(mRgba, new Point(touchedYD, touchedXL), new Point(touchedYU, touchedXR), Util.WHITE, 2); that is drawn on screen before saving. How to achieve that?
Here is my code.
Fragment:
public class StageTwo extends Fragment implements CameraBridgeViewBase.CvCameraViewListener2, OnSwitchFragmentFromStageTwo {
#Override
public void onViewCreated(#NonNull View view, #Nullable Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
((Stages) getActivity()).onSwitchFragmentFromStageTwo = this;
view.setOnTouchListener(new View.OnTouchListener() {
#Override
public boolean onTouch(View v, MotionEvent event) {
// poziomo
if (-event.getX() + camLayHeight + (xCorrection * 10) < (camLayHeight / 2)) {
touchedXR = -event.getX() + camLayHeight + (xCorrection * 10);
if (touchedXR < 0) touchedXR = 0;
} else {
touchedXL = -event.getX() + camLayHeight + (xCorrection * 10);
if (touchedXL > camLayHeight) touchedXL = camLayHeight;
}
// pionowo
if (event.getY() - (yCorrection * 10) < (camLayWidth / 2)) {
touchedYU = event.getY() - (yCorrection * 10);
if (touchedYU < 0) touchedYU = 0;
} else {
touchedYD = event.getY() - (yCorrection * 10);
if (touchedYD > camLayWidth) touchedYD = camLayWidth;
}
return true;
}
});
kamera = view.findViewById(R.id.java_surface_view);
kamera.setCvCameraViewListener(this);
Display display = getActivity().getWindowManager().getDefaultDisplay();
android.graphics.Point size = new android.graphics.Point();
display.getSize(size);
int height = size.y;
kamera.getLayoutParams().height = height / 2;
}
#Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
Core.transpose(mGray, mGray);
Core.flip(mGray, mGray, -1);
Imgproc.cvtColor(inputFrame.rgba(), mRgba, Imgproc.COLOR_RGBA2RGB, 1);
if (gridPref.equals(getString(R.string.yes))) {
Imgproc.line(mRgba, p1, p2, Util.BLUE);
Imgproc.line(mRgba, p3, p4, Util.BLUE);
}
Imgproc.rectangle(mRgba, new Point(touchedYD, touchedXL), new Point(touchedYU, touchedXR), Util.WHITE, 2);
rozmiar_y = (int) ((touchedYU - touchedYD));
rozmiar_x = (int) ((touchedXL - touchedXR));
if (rozmiar_x > rozmiar_y)
px_cm = (double) Math.round((rozmiar_x / Integer.parseInt(rozmiar)) * 100000) / 100000d;
if (rozmiar_x < rozmiar_y)
px_cm = (double) Math.round((rozmiar_y / Integer.parseInt(rozmiar)) * 100000) / 100000d;
return mRgba;
}
#Override
public void onCameraViewStarted(int width, int height) {
mRgba = new Mat(height, width, CvType.CV_8UC3);
new Mat(height, width, CvType.CV_8UC4);
mGray = new Mat(height, width, CvType.CV_8UC1);
camLayHeight = height; // mniejsza wartosc 480
camLayWidth = width;
touchedXL = camLayHeight / 2;
touchedXR = camLayHeight / 2;
touchedYD = camLayWidth / 2;
touchedYU = camLayWidth / 2;
}
#Override
public void onCameraViewStopped() {
}
#Override
public double onSwitchFragmentFromFragmentTwo() {
if (px_cm > 0.5) {
(...)
Imgcodecs.imwrite(Environment.getExternalStorageDirectory() + "/final-image.jpg", mRgba);
}
return px_cm;
}
}
Activity
OnSwitchFragmentFromStageTwo onSwitchFragmentFromStageTwo;
#Override
protected void onCreate(#Nullable Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.stages);
bottomNavigationView = findViewById(R.id.bottom_navigation);
bottomNavigationView.setItemIconTintList(null);
SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(Stages.this);
FragmentTransaction ft = getSupportFragmentManager().beginTransaction();
if (!sp.getBoolean("correctionDone", false))
ft.replace(R.id.content_frame, new StageZero(this));
else {
ft.replace(R.id.content_frame, new StageOne());
bottomNavigationView.setSelectedItemId(R.id.navigation_stage_one);
}
ft.commit();
SharedPreferences.Editor editor = sp.edit();
bottomNavigationView.setEnabled(false);
bottomNavigationView.setOnNavigationItemSelectedListener(new BottomNavigationView.OnNavigationItemSelectedListener() {
#Override
public boolean onNavigationItemSelected(#NonNull MenuItem item) {
(...)
if (bottomNavigationView.getSelectedItemId() == R.id.navigation_stage_two) {
if (onSwitchFragmentFromStageTwo.onSwitchFragmentFromFragmentTwo() <= 0.5) {
(...)
return false;
} else {
(...)
return true;
}
} else {
(...)
}
return true;
}
});
}
I was trying to solve this problem by setting touchedXL = 0; touchedXR = 0; touchedYD = 0; touchedYU = 0; right before saving but it did not help, picture is still saved with this rectangle. If you need something more just ask. Thank you in advance! :)

You may create a copy of mRgba before drawing the rectangle.
Add a new private class member mRgbNoRect:
private Mat mRgbNoRect; //mRgba before drawing rectangle
Initialize mRgbNoRect in onCameraViewStarted:
public void onCameraViewStarted(int width, int height) {
mRgba = new Mat(height, width, CvType.CV_8UC3);
mRgbNoRect = new Mat(height, width, CvType.CV_8UC3);
}
Create a copy of mRgba before drawing the rectangle in onCameraFrame:
Imgproc.cvtColor(inputFrameRgba, mRgba, Imgproc.COLOR_RGBA2RGB);
mRgba.copyTo(mRgbNoRect); //Copy mRgba content to mRgbNoRect before drawing a rectangle
Imgproc.rectangle(mRgba, new Point(20, 20), new Point(100, 100), new Scalar(255, 255, 255), 2);
Note: It's just an example (not your original code).
Add a "get" function getRgbNoRect():
public Mat getRgbNoRect() {
return mRgbNoRect;
}
Get mRgbNoRect and save it (example):
Mat rgbNoRect = sample.getRgbNoRect();
Imgcodecs.imwrite("rgbNoRect.png", rgbNoRect);
Here is a complete code sample (simple sample without a camera):
package myproject;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.CvType;
import org.opencv.core.Scalar;
import org.opencv.core.Point;
import org.opencv.imgproc.Imgproc;
import org.opencv.imgcodecs.Imgcodecs;
class Sample {
private Mat mRgba;
private Mat mRgbNoRect; //mRgba before drawing rectangle
static { System.loadLibrary(Core.NATIVE_LIBRARY_NAME); }
public Mat onCameraFrame() {
//Create RGBA matrix filled with grin color - simulating inputFrame.rgba().
Mat inputFrameRgba = Mat.zeros(200, 250, CvType.CV_8UC4);
inputFrameRgba.setTo(new Scalar(0, 255, 0, 255));
Imgproc.cvtColor(inputFrameRgba, mRgba, Imgproc.COLOR_RGBA2RGB);
mRgba.copyTo(mRgbNoRect); //Copy mRgba content to mRgbNoRect before drawing a rectangle
Imgproc.rectangle(mRgba, new Point(20, 20), new Point(100, 100), new Scalar(255, 255, 255), 2);
return mRgba;
}
public Mat getRgbNoRect() {
return mRgbNoRect;
}
public void onCameraViewStarted(int width, int height) {
mRgba = new Mat(height, width, CvType.CV_8UC3);
mRgbNoRect = new Mat(height, width, CvType.CV_8UC3);
}
public static void main(String[] args) {
Sample sample = new Sample();
sample.onCameraViewStarted(250, 200);
Mat rgbWithRect = sample.onCameraFrame();
Mat rgbNoRect = sample.getRgbNoRect();
Imgcodecs.imwrite("rgbWithRect.png", rgbWithRect);
Imgcodecs.imwrite("rgbNoRect.png", rgbNoRect);
}
}
Notes:
The code sample is tested in Windows, and I am not sure if it can be executed in Android as is.
The last time I programmed in JAVA was many years ago, so I hope I didn't do some rookie's mistakes.

Open CV detecting paper returns strange/wrong coordinates

I'm using the following class to detect/receive a found document/paper:
public class DetekPs {
/**
* Object that encapsulates the contour and 4 points that makes the larger
* rectangle on the image
*/
public static class Quadrilateral {
public MatOfPoint contour;
public Point[] points;
public Quadrilateral(MatOfPoint contour, Point[] points) {
this.contour = contour;
this.points = points;
}
}
public static Quadrilateral findDocument( Mat inputRgba ) {
ArrayList<MatOfPoint> contours = findContours(inputRgba);
Quadrilateral quad = getQuadrilateral(contours);
return quad;
}
private static ArrayList<MatOfPoint> findContours(Mat src) {
double ratio = src.size().height / 500;
int height = Double.valueOf(src.size().height / ratio).intValue();
int width = Double.valueOf(src.size().width / ratio).intValue();
Size size = new Size(width,height);
Mat resizedImage = new Mat(size, CvType.CV_8UC4);
Mat grayImage = new Mat(size, CvType.CV_8UC4);
Mat cannedImage = new Mat(size, CvType.CV_8UC1);
Imgproc.resize(src,resizedImage,size);
Imgproc.cvtColor(resizedImage, grayImage, Imgproc.COLOR_RGBA2GRAY, 4);
Imgproc.GaussianBlur(grayImage, grayImage, new Size(5, 5), 0);
Imgproc.Canny(grayImage, cannedImage, 75, 200);
ArrayList<MatOfPoint> contours = new ArrayList<MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(cannedImage, contours, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
hierarchy.release();
Collections.sort(contours, new Comparator<MatOfPoint>() {
#Override
public int compare(MatOfPoint lhs, MatOfPoint rhs) {
return Double.valueOf(Imgproc.contourArea(rhs)).compareTo(Imgproc.contourArea(lhs));
}
});
resizedImage.release();
grayImage.release();
cannedImage.release();
return contours;
}
private static Quadrilateral getQuadrilateral(ArrayList<MatOfPoint> contours) {
for ( MatOfPoint c: contours ) {
MatOfPoint2f c2f = new MatOfPoint2f(c.toArray());
double peri = Imgproc.arcLength(c2f, true);
MatOfPoint2f approx = new MatOfPoint2f();
Imgproc.approxPolyDP(c2f, approx, 0.02 * peri, true);
Point[] points = approx.toArray();
// select biggest 4 angles polygon
if (points.length == 4) {
Point[] foundPoints = sortPoints(points);
return new Quadrilateral(c, foundPoints);
}
}
return null;
}
private static Point[] sortPoints(Point[] src) {
ArrayList<Point> srcPoints = new ArrayList<>(Arrays.asList(src));
Point[] result = { null , null , null , null };
Comparator<Point> sumComparator = new Comparator<Point>() {
#Override
public int compare(Point lhs, Point rhs) {
return Double.valueOf(lhs.y + lhs.x).compareTo(rhs.y + rhs.x);
}
};
Comparator<Point> diffComparator = new Comparator<Point>() {
#Override
public int compare(Point lhs, Point rhs) {
return Double.valueOf(lhs.y - lhs.x).compareTo(rhs.y - rhs.x);
}
};
// top-left corner = minimal sum
result[0] = Collections.min(srcPoints, sumComparator);
// bottom-right corner = maximal sum
result[2] = Collections.max(srcPoints, sumComparator);
// top-right corner = minimal diference
result[1] = Collections.min(srcPoints, diffComparator);
// bottom-left corner = maximal diference
result[3] = Collections.max(srcPoints, diffComparator);
return result;
}
}
Calling the detectormethod and drawing:
takenPicture.setImageBitmap(bmp); //default imageview which displays the entire image
if (OpenCVLoader.initDebug()) {
Mat m = new Mat();
Utils.bitmapToMat(bmp,m);
DetekPs.Quadrilateral h = DetekPs.findDocument(m);
new Handler().postDelayed(new Runnable() {
#Override
public void run() {
takenPicture2.setPointsAndDraw(h.points); //draws only the points of the detected object
}
}, 2500);
}
Drawing the points:
#Override
protected void onDraw(Canvas canvas) {
Log.e("i","he ra hu" );
Paint paint2 = new Paint();
paint2.setColor(Color.RED);
paint2.setStrokeWidth(3);
for(int idx=0;idx<po.length;idx++){
canvas.drawCircle((float)po[idx].x,(float)po[idx].y, 6, paint2);
Log.e("x",""+xx);
Log.e("y",""+yy);
}
}
The points I receive look like this:
x: 23.0
y: 122.0
x: 249.0
y: 110.0
x: 249.0
y: 110.0
x: 0.0
y: 182.0
Which causes the points get drawn like this:
As you can see the points a receive and thus draw are far away from the image i have.
The question is rather what can the cause be. am i drawing wrong am i receiving the false points? or is is an issue on how large/width my image/bitmap is so i need to set the right aspect/resolution?

How to find contours in tilted document

Right now I am using
Imgproc.findContours(cannedImage, contours, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
to find the contours and it's working fine when the document is put straight in front of the camera. But it's not detecting tilted document. Any idea how can I find the contours in a tilted document?
This is my complete function:
private ArrayList<MatOfPoint> findContours(Mat src) {
double ratio = src.size().height / 500;
int height = Double.valueOf(src.size().height / ratio).intValue();
int width = Double.valueOf(src.size().width / ratio).intValue();
Size size = new Size(width, height);
Mat resizedImage = new Mat(size, CvType.CV_8UC4);
Mat grayImage = new Mat(size, CvType.CV_8UC4);
Mat cannedImage = new Mat(size, CvType.CV_8UC1);
Imgproc.resize(src, resizedImage, size);
Imgproc.cvtColor(resizedImage, grayImage, Imgproc.COLOR_RGBA2GRAY, 4);
Imgproc.GaussianBlur(grayImage, grayImage, new Size(5, 5), 0);
Imgproc.Canny(grayImage, cannedImage, 75, 200);
ArrayList<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
Imgproc.findContours(cannedImage, contours, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
hierarchy.release();
Collections.sort(contours, new Comparator<MatOfPoint>() {
#Override
public int compare(MatOfPoint lhs, MatOfPoint rhs) {
return Double.valueOf(Imgproc.contourArea(rhs)).compareTo(Imgproc.contourArea(lhs));
}
});
resizedImage.release();
grayImage.release();
cannedImage.release();
return contours;
}

OpenCV image Comparison And Similarity in Android

I'm OpenCV learner. I was trying Image Comparison. I have used OpenCV 2.4.13.3
I have these two images 1.jpg and cam1.jpg.
When I use the following command in openCV
File sdCard = Environment.getExternalStorageDirectory();
String path1, path2;
path1 = sdCard.getAbsolutePath() + "/1.jpg";
path2 = sdCard.getAbsolutePath() + "/cam1.jpg";
FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.BRIEF);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
Mat img1 = Highgui.imread(path1);
Mat img2 = Highgui.imread(path2);
Mat descriptors1 = new Mat();
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
detector.detect(img1, keypoints1);
extractor.compute(img1, keypoints1, descriptors1);
//second image
// Mat img2 = Imgcodecs.imread(path2);
Mat descriptors2 = new Mat();
MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
detector.detect(img2, keypoints2);
extractor.compute(img2, keypoints2, descriptors2);
//matcher image descriptors
MatOfDMatch matches = new MatOfDMatch();
matcher.match(descriptors1,descriptors2,matches);
// Filter matches by distance
MatOfDMatch filtered = filterMatchesByDistance(matches);
int total = (int) matches.size().height;
int Match= (int) filtered.size().height;
Log.d("LOG", "total:" + total + " Match:"+Match);
Method filterMatchesByDistance
static MatOfDMatch filterMatchesByDistance(MatOfDMatch matches){
List<DMatch> matches_original = matches.toList();
List<DMatch> matches_filtered = new ArrayList<DMatch>();
int DIST_LIMIT = 30;
// Check all the matches distance and if it passes add to list of filtered matches
Log.d("DISTFILTER", "ORG SIZE:" + matches_original.size() + "");
for (int i = 0; i < matches_original.size(); i++) {
DMatch d = matches_original.get(i);
if (Math.abs(d.distance) <= DIST_LIMIT) {
matches_filtered.add(d);
}
}
Log.d("DISTFILTER", "FIL SIZE:" + matches_filtered.size() + "");
MatOfDMatch mat = new MatOfDMatch();
mat.fromList(matches_filtered);
return mat;
}
Log
total:122 Match:30
As we can see from the log match is 30.
But as we can see both images have same visual element (in).
How can I get match=90 using openCV?
It would be great if somebody can help with code snippet.
If using opencv it is not possible then what are the other
alternatives we can look for?

But as we can see both images have same visual element (in).
So, we should compare not whole images, but "same visual element" on it. You can improve Match value more if you do not compare the "template" and "camera" images themselves, but processed same way (converted to binary black/white, for example) "template" and "camera" images. For example, try to find blue (background of template logo) square on both ("template" and "camera") images and compare that squares (Region Of Interest). The code may be something like that:
Bitmap bmImageTemplate = <get your template image Bitmap>;
Bitmap bmTemplate = findLogo(bmImageTemplate); // process template image
Bitmap bmImage = <get your camera image Bitmap>;
Bitmap bmLogo = findLogo(bmImage); // process camera image same way
compareBitmaps(bmTemplate, bmLogo);
where
private Bitmap findLogo(Bitmap sourceBitmap) {
Bitmap roiBitmap = null;
Mat sourceMat = new Mat(sourceBitmap.getWidth(), sourceBitmap.getHeight(), CvType.CV_8UC3);
Utils.bitmapToMat(sourceBitmap, sourceMat);
Mat roiTmp = sourceMat.clone();
final Mat hsvMat = new Mat();
sourceMat.copyTo(hsvMat);
// convert mat to HSV format for Core.inRange()
Imgproc.cvtColor(hsvMat, hsvMat, Imgproc.COLOR_RGB2HSV);
Scalar lowerb = new Scalar(85, 50, 40); // lower color border for BLUE
Scalar upperb = new Scalar(135, 255, 255); // upper color border for BLUE
Core.inRange(hsvMat, lowerb, upperb, roiTmp); // select only blue pixels
// find contours
List<MatOfPoint> contours = new ArrayList<>();
List<Rect> squares = new ArrayList<>();
Imgproc.findContours(roiTmp, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
// find appropriate bounding rectangles
for (MatOfPoint contour : contours) {
MatOfPoint2f areaPoints = new MatOfPoint2f(contour.toArray());
RotatedRect boundingRect = Imgproc.minAreaRect(areaPoints);
double rectangleArea = boundingRect.size.area();
// test min ROI area in pixels
if (rectangleArea > 400) {
Point rotated_rect_points[] = new Point[4];
boundingRect.points(rotated_rect_points);
Rect rect = Imgproc.boundingRect(new MatOfPoint(rotated_rect_points));
double aspectRatio = rect.width > rect.height ?
(double) rect.height / (double) rect.width : (double) rect.width / (double) rect.height;
if (aspectRatio >= 0.9) {
squares.add(rect);
}
}
}
Mat logoMat = extractSquareMat(roiTmp, getBiggestSquare(squares));
roiBitmap = Bitmap.createBitmap(logoMat.cols(), logoMat.rows(), Bitmap.Config.ARGB_8888);
Utils.matToBitmap(logoMat, roiBitmap);
return roiBitmap;
}
method extractSquareMat() just extract Region of Interest (logo) from whole image
public static Mat extractSquareMat(Mat sourceMat, Rect rect) {
Mat squareMat = null;
int padding = 50;
if (rect != null) {
Rect truncatedRect = new Rect((int) rect.tl().x + padding, (int) rect.tl().y + padding,
rect.width - 2 * padding, rect.height - 2 * padding);
squareMat = new Mat(sourceMat, truncatedRect);
}
return squareMat ;
}
and compareBitmaps() just wrapper for your code:
private void compareBitmaps(Bitmap bitmap1, Bitmap bitmap2) {
Mat mat1 = new Mat(bitmap1.getWidth(), bitmap1.getHeight(), CvType.CV_8UC3);
Utils.bitmapToMat(bitmap1, mat1);
Mat mat2 = new Mat(bitmap2.getWidth(), bitmap2.getHeight(), CvType.CV_8UC3);
Utils.bitmapToMat(bitmap2, mat2);
compareMats(mat1, mat2);
}
your code as method:
private void compareMats(Mat img1, Mat img2) {
FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.BRIEF);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
Mat descriptors1 = new Mat();
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
detector.detect(img1, keypoints1);
extractor.compute(img1, keypoints1, descriptors1);
//second image
// Mat img2 = Imgcodecs.imread(path2);
Mat descriptors2 = new Mat();
MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
detector.detect(img2, keypoints2);
extractor.compute(img2, keypoints2, descriptors2);
//matcher image descriptors
MatOfDMatch matches = new MatOfDMatch();
matcher.match(descriptors1,descriptors2,matches);
// Filter matches by distance
MatOfDMatch filtered = filterMatchesByDistance(matches);
int total = (int) matches.size().height;
int Match= (int) filtered.size().height;
Log.d("LOG", "total:" + total + " Match:" + Match);
}
static MatOfDMatch filterMatchesByDistance(MatOfDMatch matches){
List<DMatch> matches_original = matches.toList();
List<DMatch> matches_filtered = new ArrayList<DMatch>();
int DIST_LIMIT = 30;
// Check all the matches distance and if it passes add to list of filtered matches
Log.d("DISTFILTER", "ORG SIZE:" + matches_original.size() + "");
for (int i = 0; i < matches_original.size(); i++) {
DMatch d = matches_original.get(i);
if (Math.abs(d.distance) <= DIST_LIMIT) {
matches_filtered.add(d);
}
}
Log.d("DISTFILTER", "FIL SIZE:" + matches_filtered.size() + "");
MatOfDMatch mat = new MatOfDMatch();
mat.fromList(matches_filtered);
return mat;
}
As result for resized (scaled for 50%) images saved from your question result is:
D/DISTFILTER: ORG SIZE:237
D/DISTFILTER: FIL SIZE:230
D/LOG: total:237 Match:230
NB! This is a quick and dirty example just to demonstrate approach only for given template.
P.S. getBiggestSquare() can be like that (based on compare by area):
public static Rect getBiggestSquare(List<Rect> squares) {
Rect biggestSquare = null;
if (squares != null && squares.size() >= 1) {
Rect square;
double maxArea;
int ixMaxArea = 0;
square = squares.get(ixMaxArea);
maxArea = square.area();
for (int ix = 1; ix < squares.size(); ix++) {
square = squares.get(ix);
if (square.area() > maxArea) {
maxArea = square.area();
ixMaxArea = ix;
}
}
biggestSquare = squares.get(ixMaxArea);
}
return biggestSquare;
}

Object Tracking using Opencv and JavaCamera

As I use the back camera to cap a frame, by default the Android application is landscape so to get the input frame I use
Core.flip(currentFrame, currentFrame, 1);//flip around Y-axi
After some image enhancement and findcontour using opencv,
I have the following problems:
a. Object moves left hand side, drawcirle moves downward.
b. Object moves right hand side, drawcircle moves upward.
c. Object moves upward, drawcircile moves left hand side.
d. Object moves downward, drawcircle moves right hand side.
In other word, the drawcircle (output) should be clockwise 90 to get the image of the source 1.
Code shown as follows:
package com.mtyiuaa.writingintheair;
import android.support.v7.app.AppCompatActivity;
import android.os.Bundle;
import android.util.Log;
import android.view.SurfaceView;
import android.view.View;
import android.content.Intent;
import android.view.ViewDebug;
import android.widget.Button;
import java.util.ArrayList;
import java.util.List;
import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.JavaCameraView;
import org.opencv.android.LoaderCallbackInterface;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Core;
import org.opencv.core.Point;
import org.opencv.core.MatOfPoint;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.core.Rect;
import org.opencv.imgproc.Imgproc;
import org.opencv.imgproc.Moments;
import org.opencv.highgui.VideoCapture;
public class MainActivity extends AppCompatActivity implements CameraBridgeViewBase.CvCameraViewListener2{
private static final int THRESH_BINARY = 1;
private static final int THRESH_TOZERO = 4;
private static String TAG = "MainActivity";
JavaCameraView javaCameraView;
JavaCameraView javaCameraView2;
VideoCapture videoCapture;
Mat mRgba;
Mat temp;
Mat previousFrame;
Mat GpreviousFrame; // gray-level frame of previous Frame
Mat currentFrame;
Mat GcurrentFrame; // gray-level frame of current Frame
Mat diffFrame;
Mat imgGray;
Mat imgHSV;
Mat imgCanny;
Mat inputFrame;
Mat FlipFrame;
Mat outputFrame;
Mat imgthresholding;
Mat imgNormalization;
Mat imgGaussianSmothing;
int max_Binary_value = 255;
int thresh = 20;
Boolean CameraActive;
Boolean firstIteration= true;
int[] theObject = {0,0};
int x=0, y=0;
int FRAME_WIDTH = 1280;
int FRAME_HEIGHT = 720;
//max number of objects to be detected in frame
int MAX_NUM_OBJECTS=50;
//Minimum and Maximum object area
int MIN_OBJECT_AREA = 20*20;
int MAX_OBJECT_AREA = (int) ((FRAME_HEIGHT*FRAME_WIDTH)/1.5);
//MatOfPoint allcontours = new MatOfPoint();
//bounding rectangle of the object, we will use the center of this as its position.
BaseLoaderCallback mLoaderCallBack = new BaseLoaderCallback(this) {
#Override
public void onManagerConnected(int status) {
switch(status){
case BaseLoaderCallback.SUCCESS:{
javaCameraView.enableView();
//javaCameraView2.enableView();
break;
}
default:{
super.onManagerConnected(status);
break;
}
}
}
};
static{
}
//JavaCameraView javaCameraView;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
javaCameraView = (JavaCameraView)findViewById(R.id.java_camera_view);
javaCameraView.setVisibility(SurfaceView.VISIBLE);
javaCameraView.setCvCameraViewListener(this);
#Override
protected void onPause(){
super.onPause();
if(javaCameraView!=null) {
CameraActive = false;
javaCameraView.disableView();
}
}
#Override
protected void onDestroy(){
super.onDestroy(); // call the basic function
if(javaCameraView!=null){
javaCameraView.disableView();
}
}
#Override
protected void onResume(){
super.onResume(); //call based class
if(OpenCVLoader.initDebug()){
Log.i(TAG, "OpenCV loaded successfully");
mLoaderCallBack.onManagerConnected(LoaderCallbackInterface.SUCCESS);
//grab a new instance by using Basecallbackloader
}
else {
Log.i(TAG, "OpenCV not loaded");
//recall opencvLoader if not loaded
OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_10, this, mLoaderCallBack);
}
}
#Override
public void onCameraViewStarted(int width, int height) {
//Mat::Mat(int rows, int cols, int type)
// initialize all Mat object when onCamera starts
CameraActive = true;
// 4 channels are used
mRgba = new Mat(height, width, CvType.CV_8SC4);
FlipFrame = new Mat(height, width, CvType.CV_8SC4);
previousFrame =new Mat(height, width, CvType.CV_8SC4);
currentFrame = new Mat(height, width, CvType.CV_8SC4);
diffFrame =new Mat(height, width, CvType.CV_8SC4);
// 1 channel is used.
GcurrentFrame = new Mat(height, width, CvType.CV_8SC1);
GpreviousFrame = new Mat(height, width, CvType.CV_8SC1);
imgGray= new Mat(height, width, CvType.CV_8SC1);
imgHSV = new Mat (height, width, CvType.CV_8SC1);
imgCanny = new Mat(height, width, CvType.CV_8SC1);
imgGaussianSmothing = new Mat(height, width, CvType.CV_8SC1);
imgthresholding = new Mat(height, width, CvType.CV_8SC1);
imgNormalization = new Mat(height,width, CvType.CV_8SC1);
inputFrame = new Mat(height, width, CvType.CV_8SC1);
outputFrame = new Mat(height, width, CvType.CV_8SC1);
temp = new Mat(height, width, CvType.CV_8SC1);
}
#Override
public void onCameraViewStopped() {
mRgba.release();
FlipFrame.release();
previousFrame.release();
currentFrame.release();
diffFrame.release();
GcurrentFrame.release();
GpreviousFrame.release();
imgGray.release();
imgHSV.release();
imgCanny.release();
imgGaussianSmothing.release();
imgthresholding.release();
imgNormalization.release();
inputFrame.release();
outputFrame.release();
temp.release();
CameraActive = false;
}
#Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
while(CameraActive) {
Mat temp2 = new Mat();
Mat temp3 = new Mat();
currentFrame = inputFrame.rgba();
Core.flip(currentFrame, currentFrame, 1);//flip aroud Y-axis
RGB2HSV(currentFrame).copyTo(temp2);
FilterHSVImage(temp2).copyTo(temp2);
//CannyDetector(temp2).copyTo(temp4);
MorphOperation(temp2).copyTo(temp2);
List<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
Imgproc.findContours(temp2,contours,hierarchy,Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
temp2.copyTo(temp3);
FindLargestContours(temp3, contours);
//return outputFrame;
}
return null;
}
// Edge Detector using Canny
// Goal: Edge image is less sensitive to lighting conditon
public Mat CannyDetector(Mat inputFrame) {
Imgproc.Canny(inputFrame, imgCanny, 50, 150);
return imgCanny;
}
private Mat RGB2Gray (Mat inputFrame){
Imgproc.cvtColor(inputFrame, imgGray, Imgproc.COLOR_RGB2GRAY);
return imgGray;
}
private Mat RGB2HSV (Mat inputFrame){
Imgproc.cvtColor(inputFrame, imgHSV, Imgproc.COLOR_RGB2HSV);
return imgHSV;
}
private Mat FilterHSVImage(Mat inputFrame){
Core.inRange(inputFrame, new Scalar(0, 100, 100), new Scalar(10, 255, 255), imgthresholding);
//Core.inRange(temp2, new Scalar(160, 100, 100), new Scalar(179, 255, 255), temp2);
return imgthresholding;
}
private Mat MorphOperation (Mat inputFrame){
//Mat element1 = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(2*dilation_size + 1, 2*dilation_size+1));
//Imgproc.dilate(source, destination, element1);
//Highgui.imwrite("dilation.jpg", destination);
Mat erodeElement =Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3,3));
Mat dilateElement = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size (8,8));
Imgproc.dilate(inputFrame, inputFrame, erodeElement);
Imgproc.dilate(inputFrame, inputFrame, erodeElement);
Imgproc.erode(inputFrame, inputFrame, dilateElement);
Imgproc.erode(inputFrame, inputFrame, dilateElement);
return inputFrame;
}
private Mat Threshold(Mat inputFrame){
Imgproc.threshold(inputFrame, imgthresholding, thresh, max_Binary_value, Imgproc.THRESH_TOZERO);
return imgthresholding;
}
private Mat ThresholdToBinary(Mat inputFrame){
Imgproc.threshold(inputFrame, imgthresholding, thresh, max_Binary_value, Imgproc.THRESH_BINARY);
//Imgproc.threshold(inputFrame, imgthresholding, thresh, max_Binary_value, THRESH_BINARY);
return imgthresholding;
}
private Mat Normalization(Mat inputFrame, double min, double max){
//double E_Max =
Core.normalize(inputFrame, imgNormalization, min, max, Core.NORM_MINMAX);
return imgNormalization;
}
private Mat drawObject(int x, int y, Mat inputFrame) {
Point point = new Point(x, y);
Point pointA = new Point(x, y - 25);
Point pointB = new Point(x, y + 25);
Point pointC = new Point(x - 25, y);
Point pointD = new Point(x + 25, y);
Scalar scalar = new Scalar(255, 0, 0);
Core.circle(inputFrame,point,20,scalar,2);
if(y-25>0) Core.line(inputFrame,point,pointA,scalar,2);
else Core.line(inputFrame,point,new Point(x,0),scalar,2);
if(y+25<FRAME_HEIGHT) Core.line(inputFrame,point,pointB,scalar,2);
else Core.line(inputFrame,point,new Point(x,FRAME_HEIGHT),scalar,2);
if(x-25>0)Core.line(inputFrame,point,pointC,scalar,2);
else Core.line(inputFrame,point,new Point(0,y),scalar,2);
if(x+25<FRAME_WIDTH) Core.line(inputFrame,point,pointD,scalar,2);
else Core.line(inputFrame,point,new Point(FRAME_WIDTH,y),scalar,2);
Core.putText(inputFrame, "Tracking object at (" + Integer.toString(x)+" , "+ Integer.toString(y)+ ")",point, 1, 1,scalar, 2);
// putText(inputFrame,intToString(x)+","+intToString(y),Point(x,y+30),1,1,Scalar(0,255,0),2);
Log.i(TAG, "Draw x at "+Integer.toString(x)+ " Draw y at "+ Integer.toString(y));
inputFrame.copyTo(outputFrame);
return outputFrame;
}
private void TrackFilteredObject (int x, int y, Mat filteredImage, Mat sourceImage){
boolean objectFound = false;
Mat temp3 = new Mat();
filteredImage.copyTo(temp3);
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(temp3,contours,hierarchy,Imgproc.RETR_CCOMP, Imgproc.CHAIN_APPROX_SIMPLE);
//Point[] contourPoints = (Point[]) contours.toArray();
double refArea = 0;
if (hierarchy.size().height>0 && hierarchy.size().width>0){
// int numObjects = hierarchy.size();
//if number of objects greater than MAX_NUM_OBJECTS we have a noisy filter
//if(numObjects<MAX_NUM_OBJECTS) {
for (int index = 0; index >= 0; index =(int)hierarchy.get(index,0)[0]){
//hierarchy[index][0]) {
Moments moment = Imgproc.moments(contours.get(index), true);
double area = moment.get_m00();
//if the area is less than 20 px by 20px then it is probably just noise
//if the area is the same as the 3/2 of the image size, probably just a bad filter
//we only want the object with the largest area so we safe a reference area each
//iteration and compare it to the area in the next iteration.
if (area > MIN_OBJECT_AREA && area < MAX_OBJECT_AREA && area > refArea) {
// x = moment.m10 / area;
x= (int) (moment.get_m10()/area);
y = (int) (moment.get_m01()/area);
objectFound = true;
refArea = area;
} else objectFound = false;
}
//}
}
}
}

Replace x with y, it's pretty simple man come on