Develop Reference

How can I change this project to a OpenCV realtime face detection app?

Project is a realtime image processer same as my Project but it uses two values which are input and output(I remember that these projects are using frame for processing like that).
I changed its native-lib.cpp file this
#include <jni.h>
#include "opencv2/objdetect.hpp"
#include "opencv2/highgui.hpp"
#include <android/log.h>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#define TAG "NativeLib"
using namespace std;
using namespace cv;
extern "C" {
//void detect(Mat& input);
void JNICALL Java_com_example_nativeopencvandroidtemplate_MainActivity_adaptiveThresholdFromJNI(JNIEnv *env, jobject instance, jlong inputAddr, jlong outputAddr) {
Mat &input = *(Mat *) inputAddr;
Mat &output = *(Mat *) outputAddr;
clock_t begin = clock();
cv::adaptiveThreshold(input, output, 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV, 21, 5);
double total_time = double (clock() - begin ) / CLOCKS_PER_SEC;
__android_log_print(ANDROID_LOG_INFO, TAG, "adaptiveThreshold computation time = %f seconds\n", total_time);
}
}
to this
#include <jni.h>
#include "opencv2/objdetect.hpp"
#include "opencv2/highgui.hpp"
#include <android/log.h>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
using namespace std;
using namespace cv;
extern "C" {
void detect(Mat &input);
void JNICALL
Java_com_example_nativeopencvandroidtemplate_MainActivity_adaptiveThresholdFromJNI(JNIEnv *env, jobject instance,
jlong inputAddr) {
Mat &input = *(Mat *) inputAddr;
detect(input);
}
void detect(Mat &input) {
String face_cascade_name = "/storage/emulated/0/ony.xml";
String eyes_cascade_name = "/storage/emulated/0/moe.xml";
CascadeClassifier face_cascade;
CascadeClassifier eyes_cascade;
if (!face_cascade.load(face_cascade_name)) {
printf("--(!)Error loading\n");
return;
};
if (!eyes_cascade.load(eyes_cascade_name)) {
printf("--(!)Error loading\n");
return;
};
std::vector<Rect> faces;
Mat frame_gray;
cvtColor( input, frame_gray, COLOR_RGB2GRAY );
equalizeHist(frame_gray, frame_gray);
//-- Detect faces
face_cascade.detectMultiScale(frame_gray, faces, 1.1, 2, 0 | CASCADE_SCALE_IMAGE, Size(30, 30));
for (size_t i = 0; i < faces.size(); i++) {
Point center(faces[i].x + faces[i].width * 0.5, faces[i].y + faces[i].height * 0.5);
ellipse(input, center, Size(faces[i].width * 0.5, faces[i].height * 0.5), 0, 0, 360, Scalar(255, 0, 255), 4, 8,
0);
Mat faceROI = frame_gray(faces[i]);
std::vector<Rect> eyes;
//-- In each face, detect eyes
eyes_cascade.detectMultiScale(faceROI, eyes, 1.1, 2, 0 | CASCADE_SCALE_IMAGE, Size(30, 30));
for (size_t j = 0; j < eyes.size(); j++) {
Point center(faces[i].x + eyes[j].x + eyes[j].width * 0.5, faces[i].y + eyes[j].y + eyes[j].height * 0.5);
int radius = cvRound((eyes[j].width + eyes[j].height) * 0.25);
circle(input, center, radius, Scalar(255, 0, 0), 4, 8, 0);
}
}
}
}
But in my phone there was a black screen for probably five seconds and app stopping repeatedly.
Note: Sync and build were successfully and before I changed cpp file app was working successfully
Please help me about my Project.
Thanks

Here you made some changes to the definition of the C++ method Java_com_example_nativeopencvandroidtemplate_MainActivity_adaptiveThresholdFromJNI, you will therefore have to reflect these changes on the Kotlin side, as this method is called from your MainActivity.kt using JNI. Here is the code you'll have to adapt in MainActivity.kt:
class MainActivity : Activity(), CameraBridgeViewBase.CvCameraViewListener2 {
...
override fun onCameraFrame(inputFrame: CameraBridgeViewBase.CvCameraViewFrame): Mat {
val mat = inputFrame.gray()
adaptiveThresholdFromJNI(mat.nativeObjAddr)
return mat
}
private external fun adaptiveThresholdFromJNI(matAddr: Long)
...
}
Here adaptiveThresholdFromJNI was adapted to handle only one argument (as you did with the C++ equivalent) and that one argument is then returned from onCameraFrame to be displayed on the screen.
I see in your C++ code that the first thing you try to do is to convert your input Mat to gray, but this is not necessary because the Mat passed to your C++ code is already gray (see val mat = inputFrame.gray()).
If you want to keep your C++ code intact, you can also pass the colorized version of the camera view frame to your C++ code by using val mat = inputFrame.rgba().

How to count time in a process JNI?

I want to make logging time which a JNI's cpp system needed for every process.This is my code :
/*
* ImageProcessing.cpp
*/
#include <jni.h>
#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <android/log.h>
#include <strstream>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc_c.h>
#include <opencv2/objdetect/objdetect.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <vector>
#define LOG_TAG "TourGuide"
#define LOGD(...) ((void)__android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__))
using namespace std;
using namespace cv;
std::vector<float> parse_delimeted_list_of_numbers(char* line, char delimeter)
{
std::vector<float> vector_of_numbers;
std::istrstream input_stream(line);
std::string text;
float number;
while (std::getline(input_stream, text, delimeter)) {
sscanf(text.c_str(), "%f", &number, text.size());
vector_of_numbers.push_back(number);
}
return vector_of_numbers;
}
extern "C"
jboolean
Java_com_example_franksyesipangkar_tourguide_CameraPreview_ImageProcessing
(JNIEnv* env, jobject thiz, jint width, jint height, jbyteArray NV21FrameData, jintArray outPixels, jbyteArray b)
{
LOGD("JNIEnv");
//convert jbyteArray to char
jbyte *cmd = env->GetByteArrayElements(b, 0);
LOGD("JNIEnvFeature");
char feature[90600];//[819000] sejumlah 800kb untuk ukuran file
memset(feature,0, sizeof(feature));
memcpy(feature, cmd, strlen((char*)cmd));
LOGD("OutFeature: %s", feature);
//LOGD("OutCMD: %s", cmd);
vector<float> vectorHOGSVM;
vectorHOGSVM = parse_delimeted_list_of_numbers(feature, ' ');
LOGD("Parsing Vector Success ");
/* Mengambil data pointer */
jbyte * pNV21FrameData = env->GetByteArrayElements(NV21FrameData, 0);
jint * poutPixels = env->GetIntArrayElements(outPixels, 0);
/* Membuat matrix dari input frame gambar */
Mat mGray(height, width, CV_8UC1, (unsigned char *)pNV21FrameData);
Mat mResult(height, width, CV_8UC4, (unsigned char *)poutPixels);
/* Mengubah matrix kembali menjadi frame gambar */
IplImage GrayImg = mGray;
IplImage ResultImg = mResult;
HOGDescriptor hog;
//hog.winSize = Size(56,40);
// Set our custom detecting vector
hog.setSVMDetector(vectorHOGSVM);
//hog.setSVMDetector(HOGDescriptor::getDefaultPeopleDetector());
/* Deklarasi variable vector untuk menggambar kotak deteksi */
vector<Rect> found, found_filtered;
size_t i, j;
hog.detectMultiScale(mGray, found, 0, Size(8,8), Size(32,32), 1.05, 2);
double t = (double)getTickCount();
t = (double)getTickCount() - t;
LOGD("Detection Time: %gms", t*1000./cv::getTickFrequency());
LOGD("Animal: %d", found.size());
for( i = 0; i < found.size(); i++ )
{
Rect r = found[i];
for( j = 0; j < found.size(); j++ )
if( j != i && (r & found[j]) == r)
break;
if( j == found.size() )
found_filtered.push_back(r);
}
if(found.size()) {
Rect r = found[0];
r.x += cvRound(r.width*0.1);
r.width = cvRound(r.width*0.8);
r.y += cvRound(r.height*0.07);
r.height = cvRound(r.height*0.8);
LOGD("c : %d, r : %d",r.height,r.width);
cvCvtColor(&GrayImg, &ResultImg, CV_GRAY2BGR);
env->ReleaseByteArrayElements(NV21FrameData, pNV21FrameData, 0);
env->ReleaseIntArrayElements(outPixels, poutPixels, 0);
env->ReleaseByteArrayElements(b, cmd, 0);
return true;
}
For detection process, I make logging time with cv::getTickFrequency() (OpenCV function) like this :
double t = (double)getTickCount();
t = (double)getTickCount() - t;
LOGD("Detection Time: %gms", t*1000./cv::getTickFrequency());
But, in another process I don't understand for making log time.So, I want to make logging time in every process, in process LOGD("OutFeature: %s", feature);, and process LOGD("Parsing Vector Success ");.Do you have an idea for it ?

OpenCV - Android - low latency Camera Preview in setting pixel value

I'm working with OpenCV with Android version, it's a RGB Mat and when I'm trying to set new value pixel with 2 'for' to browse the camera preview. The camera display is lagging.
My goal is to change pixels value near to Red colors with a standard Blue color for example. When i tried to focus on the problem, I noticed that it's the browsing of display which is lagging the result.
Above, my browsing code with these for :
for (int i=0; i<480; i++)
{
for (int j=0; j<720; j++)
{
int red = (int) rgba.get(j,i)[0];
int green = (int) rgba.get(j,i)[1];
int blue = (int) rgba.get(j,i)[2];
Log.v(TAG, "RED=" + red);
if(red <= 50){
rgba.put(j, i, 0, 0, 255, 0);
}
}
}
// THEN THE RESULT LAGS !
Is there an other method to browse without latency or an issue ?
Thanks guys!

This is a C++ implementation. You could write one yourself in Java or use native code.
#include "opencv2/highgui/highgui.hpp"
#include <opencv2/imgproc/imgproc.hpp>
#include <stdio.h>
using namespace cv;
using namespace std;
int main(int argc, char* argv[]) {
Mat img = imread(argv[1]);
Mat lut(1, 256, CV_8UC3);
for(int i=0; i<256; i++) {
if (i <= 50) {
lut.at<Vec3b>(i) = Vec3b(255, 0, 0); // bgr
} else {
lut.at<Vec3b>(i) = Vec3b(i, i, i);
}
}
Mat retVal;
LUT(img, lut, retVal);
imshow("img", img);
imshow("retVal", retVal);
waitKey(0);
}

Accessing to OpenCv cell values of Mat from native c++ function of android app

:)
I'm programming an algorithm for detect symmetric radial center of an image, to detect eyes position into face framePicture. I know that already exist a project of public domain that do this work, but i would base my work about another kind of studies.
This is the scenario:
Doing by hand this manipulation frame by frame, i have seen that writting code on java layer, like this:
private Mat mGray = new Mat(height,width,CvType.CV_8U);
private Mat mOut = new Mat(height,width,CvType.CV_8U);
private Mat mIntermediateMat = Mat.zeros(height,width,CvType.CV_32F);
[...common methods of opencv app...]
public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
switch (ImageManipulationsActivity.viewMode) {
case ImageManipulationsActivity.VIEW_MODE_RGBA:
mOut = inputFrame.rgba();
break;
case ImageManipulationsActivity.VIEW_MODE_MODIFY:
mGray = inputFrame.gray();
int h = mGray.rows();
int w = mGray.cols();
int sobxVal,sobyVal;
/**
* Appling manually sobel filtering to calculate dx and dy image,
* moreover calculate magnitudo matrix and cosValue and sinValue
* matrices for computing, using lut techniques.
*/
for(int i = 1; i < h-1; i++)
for(int j = 1; j < w-1; j++) {
sobxVal = (int) (
((int)mGray.get(i-1,j)[0] << 1) +
mGray.get(i-1,j-1)[0] +
mGray.get(i-1,j+1)[0] - (
((int)mGray.get(i+1,j)[0] << 1) +
mGray.get(i+1,j-1)[0] +
mGray.get(i+1,j+1)[0] ) );
sobyVal = (int) (
((int)mGray.get(i,j-1)[0] << 1) +
mGray.get(i-1,j-1)[0] +
mGray.get(i+1,j-1)[0] - (
((int)mGray.get(i,j+1)[0] << 1) +
mGray.get(i-1,j+1)[0] +
mGray.get(i+1,j+1)[0] ) );
// compute magnitudo and atan2
}
// ...other calculations...
Core.convertScaleAbs(mIntermediateMat, mOut);
}
return mOut;
}
is not all efficient! So I decided to write c++ for a native function to manipulate matrix in this way:
Code by c++ side
#include <jni.h>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/features2d/features2d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <math.h>
#include <vector>
#include <android/log.h>
#define LOG_TAG "Example Filter"
#define LOGD(...) ((void)__android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__))
#define RADIUS 10
#define _K 9.9
#define _A 2 //radial strictness parameter, found experimentally
using namespace std;
using namespace cv;
extern "C" {
JNIEXPORT void JNICALL Java_org_opencv_samples_tutorial2_Tutorial2Activity_FindFeatures(
JNIEnv* env,
jobject,
jlong addrGray,
jlong addrRgba,
jlong addrlutCosSin,
jlong addrlutM );
JNIEXPORT void JNICALL Java_org_opencv_samples_tutorial2_Tutorial2Activity_FindFeatures(
JNIEnv* env,
jobject,
jlong addrGray,
jlong addrOut,
jlong addrlutCosSin,
jlong addrlutM )
{
Mat& mGr = *(Mat*)addrGray;
Mat& mOut = *(Mat*)addrOut;
Mat& lutCosSin = *(Mat*)addrlutCosSin;
Mat& lutM = *(Mat*)addrlutM;
int w = mGr.cols;
int h = mGr.rows;
double sobelxVal,sobelyVal,angle;
Mat magnitudo(h,w,CV_32F,Scalar(0));
Mat xMat(h,w,CV_8S,Scalar(0));
Mat yMat(h,w,CV_8S,Scalar(0));
Mat oMat(h,w,CV_32F,Scalar(0));
Mat mMat(h,w,CV_32F,Scalar(0));
/*
* Convolves Matrix with Sobel ky kernel and Sobel kx kernel
*ky = [ 1 2 1 ;
* 0 0 0 ;
* -1 -2 -1 ]
*
*kx = [ 1 0 -1 ;
* 2 0 -2 ;
* 1 0 -1 ]
*
* doing dedicated computation
*/
for( int i = 1; i < h-1; i++ )
{
for (int j = 1; j < w-1; j++ )
{
sobelxVal = (mGr.at<int>(i-1,j) << 1) +
mGr.at<int>(i-1,j-1) +
mGr.at<int>(i-1,j+1) - (
(mGr.at<int>(i+1,j) << 1) +
mGr.at<int>(i+1,j-1) +
mGr.at<int>(i+1,j+1) );
sobelyVal = (mGr.at<int>(i,j-1) << 1) +
mGr.at<int>(i-1,j-1) +
mGr.at<int>(i+1,j-1) - (
(mGr.at<int>(i,j+1) << 1) +
mGr.at<int>(i-1,j+1) +
mGr.at<int>(i+1,j+1) );
magnitudo.at<double>(i,j) = lutM.at<double>((int)sobelxVal+255/4,(int)sobelxVal+255/4);
angle = floor(atan2(sobelyVal,sobelxVal)*180/M_PI);
xMat.at<double>(i,j) = lutCosSin.at<double>(0,angle);
yMat.at<double>(i,j) = lutCosSin.at<double>(1,angle);
}
}
// other code calculation to determine mOut matrix values
}
}
By java code side
private Mat mRgba;
private Mat mGray;
/*
* Matrix of 360 cols and 2 rows
* row[0] cos values
* row[1] sin values
*/
private static Mat lutCosSin;
/*
* Matrix 510 x 510
* where lutM(i,j) = atan2(i,j)
*/
private static Mat lutMagnitudo
// common methods and declarations...
public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
final int viewMode = mViewMode;
switch (viewMode) {
case VIEW_MODE_RGBA:
// input frame has RBGA format
mRgba = inputFrame.rgba();
break;
case VIEW_MODE_FEATURES:
// input frame has RGBA format
mGray = inputFrame.gray();
mRgba = Mat.zeros(mGray.rows(), mGray.cols(), mGray.type());
FindFeatures(
mGray.getNativeObjAddr(),
mRgba.getNativeObjAddr(),
lutCosSin.getNativeObjAddr(),
lutMagnitudo.getNativeObjAddr()
);
//Core.convertScaleAbs(mRgba, mRgba);
// Log.d(TAG, "Called native function :"+mRgba.submat(new Range(0,5), new Range(0,5)).toString()+
// "\nAngles matrix:"+mGray);
break;
}
return mRgba;
}
public native void FindFeatures(long matAddrGr, long matAddrRgba, long matAddrlutCS, long matAddrlutM);
the first important issue of this snipped code is that on accessing to mGr cells with "at" method in this way:
mGr.at<int>(i,j)
previus check that mGr type is int, the returned values isn't the effective gray level pixel of gray frame (i saw it by log).
I suppose that there are linkin bug of matrix from java code to c++ code, but i'm not sure for this.
I hope may anyone help me to solve this issue XD !!

C code not compiling correctly

I wrote a simple loop to aid in billboarding that will check if a pixel is white. if so, it will set it to 100% transparency. i wrote it in native code because the java equivalent of this loop took 19 seconds to run for a 256x256 bitmap, too slow.
when compiling:
#include "org_me_renderscene_Billboard.h"
#include <stdio.h>
#include <stdlib.h>
JNIEXPORT jintArray JNICALL Java_org_me_renderscene_Billboard_NativeSetAlphaWhereWhite
(JNIEnv *envptr, jclass jClass, jintArray pixels, jint length)
{
int *mPixels = (*int)malloc(length * 4);
static int currentcolor;
static int writecolor;
static int red, green, blue;
for(int x = 0; x < length; x++)
{
currentcolor = pixels[x];
red = currentcolor << 16;
green = currentcolor << 8;
blue = currentcolor;
if((red == 0) && (green == 0) && (blue == 0))
{
mPixels[x] = 0x00000000;
}
else
{
mPixels[x] = currentcolor;
}
}
return mPixels;
}
the auto-generated stub for which is:
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class org_me_renderscene_Billboard */
#ifndef _Included_org_me_renderscene_Billboard
#define _Included_org_me_renderscene_Billboard
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_me_renderscene_Billboard
* Method: NativeSetAlphaWhereWhite
* Signature: ([II)[I
*/
JNIEXPORT jintArray JNICALL Java_org_me_renderscene_Billboard_NativeSetAlphaWhereWhite
(JNIEnv *, jclass, jintArray, jint);
#ifdef __cplusplus
}
#endif
#endif
i get these errors:
thomas#THOMASDESKLINUX:~/Documents/LinuxProgramming/EclipseWorkspace/RenderScene$ /home/thomas/Documents/LinuxProgramming/AndroidSDKs/android-ndk-r4b/ndk-build
Compile thumb : Billboardlib <= /home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/jni/org_me_renderscene_Billboard.c
/home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/jni/org_me_renderscene_Billboard.c: In function 'Java_org_me_renderscene_Billboard_NativeSetAlphaWhereWhite':
/home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/jni/org_me_renderscene_Billboard.c:9: error: expected expression before 'int'
/home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/jni/org_me_renderscene_Billboard.c:9: error: expected ',' or ';' before 'malloc'
/home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/jni/org_me_renderscene_Billboard.c:15: error: 'for' loop initial declarations are only allowed in C99 mode
/home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/jni/org_me_renderscene_Billboard.c:15: note: use option -std=c99 or -std=gnu99 to compile your code
/home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/jni/org_me_renderscene_Billboard.c:17: warning: dereferencing 'void *' pointer
/home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/jni/org_me_renderscene_Billboard.c:17: error: void value not ignored as it ought to be
make: *** [/home/thomas/Documents/LinuxProgramming/EclipseWorkspace/RenderScene/obj/local/armeabi/objs/Billboardlib/org_me_renderscene_Billboard.o] Error 1
why is this happening? my C code should be fine, and these errors dont make much sense.

Try (int*) instead of (*int)

what flags are you using in you Android.mk ?
did you set LOCAL_CFLAGS := -std=c99
also
you need to change to this
int *mPixels = (int*)malloc(length * 4);

int *mPixels = (*int)malloc(length * 4);
should be
int *mPixels = (int*)malloc(length * 4);
or even better
int *mPixels = (int*)malloc(length * sizeof(int));
and also note that this will not properly separate red, green, and blue:
red = currentcolor << 16;
green = currentcolor << 8;
blue = currentcolor;
Given that you're just checking for zero, and you don't really care about the individual RGB values, you can probably just get away with:
if ( (currentcolor & 0x00FFFFFF) == 0)
This will zero out the Alpha from the pixel, leaving behind just the RGB portion. If that whole thing is zero, each color must be zero, so there's no need to check each color individually.
Final thought:
I haven't done much with Android specifically, but isn't 0x000000 black and 0xFFFFFF white? So you're actually matching on black instead of white here.