Develop Reference

Xamarin tf.lite input objects

Im trying to reproduce tensorflow object detection on xamarin.
private MappedByteBuffer LoadModelFile()
{
AssetFileDescriptor fileDescriptor = Assets.OpenFd("detect.tflite");
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
return fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
}
View view = (View) sender;
MappedByteBuffer buffer = LoadModelFile();
Interpreter interpreter = new Interpreter(buffer);
var sr = new StreamReader(Assets.Open("labels.txt"));
var labels = sr.ReadToEnd()
.Split('\n')
.Select(s => s.Trim())
.Where(s => !string.IsNullOrEmpty(s))
.ToList();
var bitmap = BitmapFactory.DecodeResource(Resources, 2130837608);
var resizedBitmap = Bitmap.CreateScaledBitmap(bitmap, 1000, 750, false)
.Copy(Bitmap.Config.Argb8888, false);
float[][][][] imgData = null;
imgData = new float[1][][][];
imgData[0] = new float[1000][][];
for (int i = 0; i < imgData[0].Length; i++)
{
imgData[0][i] = new float[750][];
for (int j = 0; j < imgData[0][i].Length; j++)
{
imgData[0][i][j] = new float[3];
}
}
var intValuess = new int[1000 * 750];
resizedBitmap.GetPixels(intValuess, 0, 1000, 0, 0, 1000, 750);
int pixels = 0;
for (int i = 0; i < imgData[0].Length; i++)
{
for (int j = 0; j < imgData[0][i].Length; j++)
{
var val = intValuess[pixels++];
imgData[0][i][j][0] = (float)((val >> 16) & 0xFF);
imgData[0][i][j][1] = (float)((val >> 8) & 0xFF);
imgData[0][i][j][2] = (float)(val & 0xFF);
}
}
var outputs = new float[labels.Count];
interpreter.Run(imgData, outputs);
but i have error "cannot convert float[][][][] to Java.Lang.Object in line interpreter.Run(imgData, outputs);
How i can convert float[][][][] to Java.Lang.Object or where i can find tensorflow lite with xamarin examples.

I know it has been a while since you asked this question but maybe my response can be useful to someone.
I am also trying to use Xamarin with tflite, to run a simple CNN.
Here is my code:
private MappedByteBuffer LoadModelFile()
{
var assets = Application.Context.Assets;
AssetFileDescriptor fileDescriptor = assets.OpenFd("seed_model_no_qt.tflite");
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
return fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
}
private string Classify(MediaFile mediaFile)
{
var assets = Application.Context.Assets;
Bitmap bp = BitmapFactory.DecodeStream(mediaFile.GetStream());
var resizedBitmap = Bitmap.CreateScaledBitmap(bp, 1280, 1280, false).Copy(Bitmap.Config.Argb8888, false);
var bufint = new int[1280 * 1280];
resizedBitmap.GetPixels(bufint, 0, 1280, 0, 0, 1280, 1280);
int pixels = 0;
var input_buffer = new byte[4 * 1280 * 1280 * 3];
for(int i = 0; i < 1280; i++)
{
for(int k = 0; k < 1280; k++)
{
int val = bufint[pixels++];
Array.Copy(BitConverter.GetBytes(((val >> 16) & 0xFF) * (1f / 255f)), 0, input_buffer, (i * 1280 + k) * 12, 4);
Array.Copy(BitConverter.GetBytes(((val >> 8) & 0xFF) * (1f / 255f)), 0, input_buffer, (i * 1280 + k) * 12 + 4, 4);
Array.Copy(BitConverter.GetBytes((val & 0xFF) * (1f / 255f)), 0, input_buffer, (i * 1280 + k) * 12 + 8, 4);
}
}
var bytebuffer = Java.Nio.ByteBuffer.Wrap(input_buffer);
var output = Java.Nio.ByteBuffer.AllocateDirect(4*160*160);
interpreter.Run(bytebuffer, output);
var buffer = new byte[4 * 160 * 160];
Marshal.Copy(output.GetDirectBufferAddress(), buffer, 0, 4 * 160 * 160);
float sum = 0.0f;
for(int i = 0; i < 160*160; i++)
{
sum += BitConverter.ToSingle(buffer, i * 4);
}
return "Count : " + ((int)(sum/255)).ToString();
}
I reused your LoadModelFile() function as it is. The code takes an image from a mediaFile (coming from the phone camera), then resizes it to 1280x1280 rgb image before feeding it to a CNN as an array of float32 values.
Your float[][][][] to Java.Lang.Object issue came from the interpreter.Run() method expecting a Java Object. Some people online solve it by giving a Java.Nio.ByteBuffer as a parameter, instead of an array. It implies some bitwise manipulations but the Run method does accept the ByteBuffer object.
When filling the ByteBuffer, I advise you not to use its methods such as PutFloat(), but to fill a byte[] buffer and then use the Java.Nio.ByteBuffer.Wrap() method as I did. Using ByteBuffer's methods seemed to imply large performance issues in my case.
Same thing happens when manipulating the output of my CNN (a 160x160 heatmap of float32 values). Using ByteBuffer.Get() method to access the values was very slow. Instead, use Marshal.Copy to store the values into a byte array, then get back the float values with BitConverter.ToSingle.

Android byte array to Bitmap How to

How can I convert byte array received using socket.
The C++ client send image data which is of type uchar.
At the android side I am receiving this uchar array as byte[] which is ranges from -128 to +127.
What I wanted to do is that receives this data and display it. For that I was trying to convert to Bitmap using BitmapFactory.decodeByteArray(), but no luck I am getting null Bitmap. Am I doing right or any other method available.
Thanks in advance....

From the comments to the answers above, it seems like you want to create a Bitmap object from a stream of RGB values, not from any image format like PNG or JPEG.
This probably means that you know the image size already. In this case, you could do something like this:
byte[] rgbData = ... // From your server
int nrOfPixels = rgbData.length / 3; // Three bytes per pixel.
int pixels[] = new int[nrOfPixels];
for(int i = 0; i < nrOfPixels; i++) {
int r = data[3*i];
int g = data[3*i + 1];
int b = data[3*i + 2];
pixels[i] = Color.rgb(r,g,b);
}
Bitmap bitmap = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888);

I've been using it like below in one of my projects and so far it's been pretty solid. I'm not sure how picky it is as far as it not being compressed as a PNG though.
byte[] bytesImage;
Bitmap bmpOld; // Contains original Bitmap
Bitmap bmpNew;
ByteArrayOutputStream baoStream = new ByteArrayOutputStream();
bmpOld.compress(Bitmap.CompressFormat.PNG, 100, baoStream);
bytesImage = baoStream.toByteArray();
bmpNew = BitmapFactory.decodeByteArray(bytesImage, 0, bytesImage.length);
edit: I've adapted the code from this post to use RGB, so the code below should work for you. I haven't had a chance to test it yet so it may need some adjusting.
Byte[] bytesImage = {0,1,2, 0,1,2, 0,1,2, 0,1,2};
int intByteCount = bytesImage.length;
int[] intColors = new int[intByteCount / 3];
int intWidth = 2;
int intHeight = 2;
final int intAlpha = 255;
if ((intByteCount / 3) != (intWidth * intHeight)) {
throw new ArrayStoreException();
}
for (int intIndex = 0; intIndex < intByteCount - 2; intIndex = intIndex + 3) {
intColors[intIndex / 3] = (intAlpha << 24) | (bytesImage[intIndex] << 16) | (bytesImage[intIndex + 1] << 8) | bytesImage[intIndex + 2];
}
Bitmap bmpImage = Bitmap.createBitmap(intColors, intWidth, intHeight, Bitmap.Config.ARGB_8888);

InputStream is = new java.net.URL(urldisplay).openStream();
byte[] colors = IOUtils.toByteArray(is);
int nrOfPixels = colors.length / 3; // Three bytes per pixel.
int pixels[] = new int[nrOfPixels];
for(int i = 0; i < nrOfPixels; i++) {
int r = (int)(0xFF & colors[3*i]);
int g = (int)(0xFF & colors[3*i+1]);
int b = (int)(0xFF & colors[3*i+2]);
pixels[i] = Color.rgb(r,g,b);
}
imageBitmap = Bitmap.createBitmap(pixels, width, height,Bitmap.Config.ARGB_4444);
bmImage.setImageBitmap(imageBitmap );

Generated Bitmap has only zero values

I'm trying to decode a byte array into a bitmap to be used in android.
The byte array that i use for decoding is generated by an OpenGl conmmand named GlReadPixels and the data inside is correct.
MainActivity.dataInputStream.readFully(Image, 0,256 * 256 * 4);
//converting from rgba to argb
for (int i = 0; i < Image.length - 1; i = i + 4) {
aux = (Image[i + 3] & 0xFF);
IntImage[i + 3] = (int) (Image[i + 2] & 0xFF);
IntImage[i + 2] = (int) (Image[i + 1] & 0xFF);
IntImage[i + 1] = (int) (Image[i] & 0xFF);
IntImage[i] = aux;
}
if i do this :bmp = Bitmap.createBitmap(IntImage, 256, 256,Config.ARGB_8888); or this:
bmp = Bitmap.createBitmap(256, 256, Config.ARGB_8888);
bmp.setPixels(IntImage, 0, 256, 0, 0, 256, 256); ,
the resulted bitmap will have only 0 values.
Can somebody please tell me why that is ?

Bitmap.createBitmap is expecting array of 256*256 size, and you are giving it 256*256*4, and therefore you are giving one part only and it looks like this, 0x000000FF .. 0x000000RR, 0x000000GG, if you understand what I mean.
change your argb function in something like this
int[] imageARGB = new int[256 * 256];
for(int i = 0; i < imageARGB.length, i++){
imageARGB[i] = Color.argb(yourAlpha, RR, GG, BB);
}
Hope this helps and enjoy your work

Creating a new Bitmap with Pixel Data in JNI?

I've the below code to create a BitMap (Just a Black / Gray Image) in the JNI with 'ARGB_8888' configuration. But when I dump the content of the Bitmap in the Java code, I'm able to see only the configurations, but not the Pixel Data in the Bitmap.
JNI Code
// Image Details
int imgWidth = 128;
int imgHeight = 128;
int numPix = imgWidth * imgHeight;
// Creaing Bitmap Config Class
jclass bmpCfgCls = env->FindClass("android/graphics/Bitmap$Config");
jmethodID bmpClsValueOfMid = env->GetStaticMethodID(bmpCfgCls, "valueOf", "(Ljava/lang/String;)Landroid/graphics/Bitmap$Config;");
jobject jBmpCfg = env->CallStaticObjectMethod(bmpCfgCls, bmpClsValueOfMid, env->NewStringUTF("ARGB_8888"));
// Creating a Bitmap Class
jclass bmpCls = env->FindClass("android/graphics/Bitmap");
jmethodID createBitmapMid = env->GetStaticMethodID(bmpCls, "createBitmap", "(IILandroid/graphics/Bitmap$Config;)Landroid/graphics/Bitmap;");
jBmpObj = env->CallStaticObjectMethod(bmpCls, createBitmapMid, imgWidth, imgHeight, jBmpCfg);
// Creating Pixel Data
int triplicateLen = numPix * 4;
char *tripPixData = (char*)malloc(triplicateLen);
for (int lc = 0; lc < triplicateLen; lc++)
{
// Gray / Black Image
if (0 == (lc%4))
tripPixData[lc] = 0x7F; // Alpha
else
tripPixData[lc] = 0x00; // RGB
}
// Setting Pixels in Bitmap
jByteArr = env->NewByteArray(triplicateLen);
env->SetByteArrayRegion(jByteArr, 0, triplicateLen, (jbyte*)tripPixData);
jmethodID setPixelsMid = env->GetMethodID(bmpCls, "setPixels", "([IIIIIII)V");
env->CallVoidMethod(jBmpObj, setPixelsMid, (jintArray)jByteArr, 0, imgWidth, 0, 0, imgWidth, imgHeight);
free(tripPixData);
// Return BitMap Object
return jBmpObj;
In JAVA (Output)
// Checking the Configuration / Image Details
jBmpObj.getWidth() - 128
jBmpObj.getHeight() - 128
jBmpObj.getRowBytes() - 512
jBmpObj.getConfig() - ARGB 8888
// Getting Pixel Data
imgPixs = new int[jBmpObj.getWidth() * jBmpObj.getHeight()];
jBmpObj.getPixels(imgPixs, 0, jBmpObj.getWidth(), 0, 0, jBmpObj.getWidth(), jBmpObj.getHeight());
// Running a Loop on the imgPixs
imgPixs[<0 - imgPixs.lenght>] - 0 (Every Pixel Data)
I used the same concept to create a Bitmap in the Java Code, and it works fine (Even I'm able to see the image). But I want the logic to be in the JNI part and not in Java Code. So I tried the above logic and it failed in setting the Pixel Data.
Any input in fixing this issue will be really helpful,..

Full working example:
jclass bitmapConfig = jniEnv->FindClass("android/graphics/Bitmap$Config");
jfieldID rgba8888FieldID = jniEnv->GetStaticFieldID(bitmapConfig, "ARGB_8888", "Landroid/graphics/Bitmap$Config;");
jobject rgba8888Obj = jniEnv->GetStaticObjectField(bitmapConfig, rgba8888FieldID);
jclass bitmapClass = jniEnv->FindClass("android/graphics/Bitmap");
jmethodID createBitmapMethodID = jniEnv->GetStaticMethodID(bitmapClass,"createBitmap", "(IILandroid/graphics/Bitmap$Config;)Landroid/graphics/Bitmap;");
jobject bitmapObj = jniEnv->CallStaticObjectMethod(bitmapClass, createBitmapMethodID, _width, _height, rgba8888Obj);
jintArray pixels = jniEnv->NewIntArray(_width * _height);
for (int i = 0; i < _width * _height; i++)
{
unsigned char red = bitmap[i*4];
unsigned char green = bitmap[i*4 + 1];
unsigned char blue = bitmap[i*4 + 2];
unsigned char alpha = bitmap[i*4 + 3];
int currentPixel = (alpha << 24) | (red << 16) | (green << 8) | (blue);
jniEnv->SetIntArrayRegion(pixels, i, 1, &currentPixel);
}
jmethodID setPixelsMid = jniEnv->GetMethodID(bitmapClass, "setPixels", "([IIIIIII)V");
jniEnv->CallVoidMethod(bitmapObj, setPixelsMid, pixels, 0, _width, 0, 0, _width, _height);
where bitmap is unsigned char*.

You cannot cast byte[] to int[] in Java, therefore you cannot cast it in JNI. But you can cast char* to int*, so you can simply use your tripPixData to fill a new jjintArray.

IN Android each pixel represented as 0xFFFFFFFF ie ARGB.
0xFF referes most significamt 8 bits of given data.
From your snippet, where you are getting soure image data? But i have solved this issue
by using following code base.i hope this ll help you.
// Creating Pixel Data
unsigned char* rawData = //your raw data
**Note**: here you have get each r,g & b component as 8 bit data //If it is rgb image,if it
is monochrome you can use raw data
int triplicateLen = imgheight * imgwidth;
int *tripPixData = (int*) malloc(triplicateLen * sizeof(int));
if(rgb){
for (int lc = 0; lc < triplicateLen ; lc++){
tripPixData [lc] = (0xFF << 24) | (r[lc] << 16) | (g[lc] << 8) | b[lc];
}
}else{
for (int lc = 0; lc < triplicateLen ; lc++){
tripPixData [lc] = (0xFF << 24) | (rawData [lc] << 16) | (rawData [lc] << 8) | rawData [lc];
}
}

Android Screen capturing or make video from images

I want to make video of my android screen(what i am doing on the android screen) programatically.
Is there any best tutorial or help regarding this.
I have searched a lot but i found that thing...(Capture the android screen picture programtically).
Ok fine if i capture a lot of images after each milliseconds than how i make video with a lot of captured images in android programmatic.

you can use following code for screen capturing in Android.
Please view this url.....
http://android-coding.blogspot.in/2011/05/create-custom-dialog-with-dynamic.html

As long as you have bitmaps you can flip it into video using JCodec ( http://jcodec.org ).
Here's a sample image sequence encoder: https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/api/SequenceEncoder.java . You can modify it for your purposes by replacing BufferedImage with Bitmap.
Use these helper methods:
public static Picture fromBitmap(Bitmap src) {
Picture dst = Picture.create((int)src.getWidth(), (int)src.getHeight(), RGB);
fromBitmap(src, dst);
return dst;
}
public static void fromBitmap(Bitmap src, Picture dst) {
int[] dstData = dst.getPlaneData(0);
int[] packed = new int[src.getWidth() * src.getHeight()];
src.getPixels(packed, 0, src.getWidth(), 0, 0, src.getWidth(), src.getHeight());
for (int i = 0, srcOff = 0, dstOff = 0; i < src.getHeight(); i++) {
for (int j = 0; j < src.getWidth(); j++, srcOff++, dstOff += 3) {
int rgb = packed[srcOff];
dstData[dstOff] = (rgb >> 16) & 0xff;
dstData[dstOff + 1] = (rgb >> 8) & 0xff;
dstData[dstOff + 2] = rgb & 0xff;
}
}
}
public static Bitmap toBitmap(Picture src) {
Bitmap dst = Bitmap.create(pic.getWidth(), pic.getHeight(), ARGB_8888);
toBitmap(src, dst);
return dst;
}
public static void toBitmap(Picture src, Bitmap dst) {
int[] srcData = src.getPlaneData(0);
int[] packed = new int[src.getWidth() * src.getHeight()];
for (int i = 0, dstOff = 0, srcOff = 0; i < src.getHeight(); i++) {
for (int j = 0; j < src.getWidth(); j++, dstOff++, srcOff += 3) {
packed[dstOff] = (srcData[srcOff] << 16) | (srcData[srcOff + 1] << 8) | srcData[srcOff + 2];
}
}
dst.setPixels(packed, 0, src.getWidth(), 0, 0, src.getWidth(), src.getHeight());
}
You can as well wait for JCodec team to implement full Android support, they are working on it according to this: http://jcodec.org/news/no_deps.html

you can use following code for screen capturing in Android.
ImageView v1 = (ImageView)findViewById(R.id.mImage);
v1.setDrawingCacheEnabled(true);
Bitmap bm = v1.getDrawingCache();
For Creating Video from Images visit this link.