Im trying to reproduce tensorflow object detection on xamarin.
private MappedByteBuffer LoadModelFile()
{
AssetFileDescriptor fileDescriptor = Assets.OpenFd("detect.tflite");
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
return fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
}
View view = (View) sender;
MappedByteBuffer buffer = LoadModelFile();
Interpreter interpreter = new Interpreter(buffer);
var sr = new StreamReader(Assets.Open("labels.txt"));
var labels = sr.ReadToEnd()
.Split('\n')
.Select(s => s.Trim())
.Where(s => !string.IsNullOrEmpty(s))
.ToList();
var bitmap = BitmapFactory.DecodeResource(Resources, 2130837608);
var resizedBitmap = Bitmap.CreateScaledBitmap(bitmap, 1000, 750, false)
.Copy(Bitmap.Config.Argb8888, false);
float[][][][] imgData = null;
imgData = new float[1][][][];
imgData[0] = new float[1000][][];
for (int i = 0; i < imgData[0].Length; i++)
{
imgData[0][i] = new float[750][];
for (int j = 0; j < imgData[0][i].Length; j++)
{
imgData[0][i][j] = new float[3];
}
}
var intValuess = new int[1000 * 750];
resizedBitmap.GetPixels(intValuess, 0, 1000, 0, 0, 1000, 750);
int pixels = 0;
for (int i = 0; i < imgData[0].Length; i++)
{
for (int j = 0; j < imgData[0][i].Length; j++)
{
var val = intValuess[pixels++];
imgData[0][i][j][0] = (float)((val >> 16) & 0xFF);
imgData[0][i][j][1] = (float)((val >> 8) & 0xFF);
imgData[0][i][j][2] = (float)(val & 0xFF);
}
}
var outputs = new float[labels.Count];
interpreter.Run(imgData, outputs);
but i have error "cannot convert float[][][][] to Java.Lang.Object in line interpreter.Run(imgData, outputs);
How i can convert float[][][][] to Java.Lang.Object or where i can find tensorflow lite with xamarin examples.
I know it has been a while since you asked this question but maybe my response can be useful to someone.
I am also trying to use Xamarin with tflite, to run a simple CNN.
Here is my code:
private MappedByteBuffer LoadModelFile()
{
var assets = Application.Context.Assets;
AssetFileDescriptor fileDescriptor = assets.OpenFd("seed_model_no_qt.tflite");
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
return fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
}
private string Classify(MediaFile mediaFile)
{
var assets = Application.Context.Assets;
Bitmap bp = BitmapFactory.DecodeStream(mediaFile.GetStream());
var resizedBitmap = Bitmap.CreateScaledBitmap(bp, 1280, 1280, false).Copy(Bitmap.Config.Argb8888, false);
var bufint = new int[1280 * 1280];
resizedBitmap.GetPixels(bufint, 0, 1280, 0, 0, 1280, 1280);
int pixels = 0;
var input_buffer = new byte[4 * 1280 * 1280 * 3];
for(int i = 0; i < 1280; i++)
{
for(int k = 0; k < 1280; k++)
{
int val = bufint[pixels++];
Array.Copy(BitConverter.GetBytes(((val >> 16) & 0xFF) * (1f / 255f)), 0, input_buffer, (i * 1280 + k) * 12, 4);
Array.Copy(BitConverter.GetBytes(((val >> 8) & 0xFF) * (1f / 255f)), 0, input_buffer, (i * 1280 + k) * 12 + 4, 4);
Array.Copy(BitConverter.GetBytes((val & 0xFF) * (1f / 255f)), 0, input_buffer, (i * 1280 + k) * 12 + 8, 4);
}
}
var bytebuffer = Java.Nio.ByteBuffer.Wrap(input_buffer);
var output = Java.Nio.ByteBuffer.AllocateDirect(4*160*160);
interpreter.Run(bytebuffer, output);
var buffer = new byte[4 * 160 * 160];
Marshal.Copy(output.GetDirectBufferAddress(), buffer, 0, 4 * 160 * 160);
float sum = 0.0f;
for(int i = 0; i < 160*160; i++)
{
sum += BitConverter.ToSingle(buffer, i * 4);
}
return "Count : " + ((int)(sum/255)).ToString();
}
I reused your LoadModelFile() function as it is. The code takes an image from a mediaFile (coming from the phone camera), then resizes it to 1280x1280 rgb image before feeding it to a CNN as an array of float32 values.
Your float[][][][] to Java.Lang.Object issue came from the interpreter.Run() method expecting a Java Object. Some people online solve it by giving a Java.Nio.ByteBuffer as a parameter, instead of an array. It implies some bitwise manipulations but the Run method does accept the ByteBuffer object.
When filling the ByteBuffer, I advise you not to use its methods such as PutFloat(), but to fill a byte[] buffer and then use the Java.Nio.ByteBuffer.Wrap() method as I did. Using ByteBuffer's methods seemed to imply large performance issues in my case.
Same thing happens when manipulating the output of my CNN (a 160x160 heatmap of float32 values). Using ByteBuffer.Get() method to access the values was very slow. Instead, use Marshal.Copy to store the values into a byte array, then get back the float values with BitConverter.ToSingle.
Related
I get this fatal error while I run a tensorflow model in my android app:
Caused by: java.nio.BufferOverflowException
at java.nio.HeapFloatBuffer.put(HeapFloatBuffer.java:179)
at org.tensorflow.Tensor.writeTo(Tensor.java:488)
at org.tensorflow.contrib.android.TensorFlowInferenceInterface.fetch(TensorFlowInferenceInterface.java:488)
at org.tensorflow.contrib.android.TensorFlowInferenceInterface.fetch(TensorFlowInferenceInterface.java:442)
Code is as follows:
//sample values: WANTED_WIDTH = 714, WANTED_HEIGHT = 438
int[] intValues = new int[WANTED_WIDTH * WANTED_HEIGHT];
float[] floatValues = new float[WANTED_WIDTH * WANTED_HEIGHT * 3];
float[] outputValues = new float[WANTED_WIDTH * WANTED_HEIGHT * 3];
Bitmap bitmap = activity.mainBitmap;
Bitmap scaledBitmap = Bitmap.createScaledBitmap(bitmap, WANTED_WIDTH, WANTED_HEIGHT, true);
scaledBitmap.getPixels(intValues, 0, scaledBitmap.getWidth(), 0, 0, scaledBitmap.getWidth(), scaledBitmap.getHeight());
for (int i = 0; i < intValues.length; i++) {
final int val = intValues[i];
floatValues[i*3] = ((val >> 16) & 0xFF);
floatValues[i*3+1] = ((val >> 8) & 0xFF);
floatValues[i*3+2] = (val & 0xFF);
}
AssetManager assetManager = getResources().getAssets();
mInferenceInterface = new TensorFlowInferenceInterface(assetManager, MODEL_FILE);
final float[] styleVals = new float[NUM_STYLES];
for (int i = 0; i < NUM_STYLES; ++i) {
styleVals[i] = 0.0f / NUM_STYLES;
}
styleVals[params[0]] = 1.5f;
mInferenceInterface.feed(INPUT_NODE, floatValues, 1, WANTED_HEIGHT, WANTED_WIDTH, 3);
mInferenceInterface.feed("style_num", styleVals, NUM_STYLES);
mInferenceInterface.run(new String[] {OUTPUT_NODE}, false);
mInferenceInterface.fetch(OUTPUT_NODE, outputValues);
Error is shown to be on the last line of code I have provided above, i.e.
"mInferenceInterface.fetch(OUTPUT_NODE, outputValues);"
Any idea as of how to resolve this issue, I have already gone through google search, but nothing touches on this particular issue.
Thanks in advance!
you need to verify that the dimension of OUTPUT_NODE is equal to dimension of outputValues?
in addition, you can refer to my code of project, url: https://github.com/tz28/Chinese-number-gestures-recognition/blob/master/DigitalGestureRecognition/app/src/main/java/com/example/hc/digitalgesturerecognition/Classifier.java
I'm porting ios app to android and have issue with bitmap colors while creating it from byte array. This code works perfectly on ios (C# - Xamarin):
const int bitsPerComponent = 8;
const int bytePerPixel = 4;
var bytesPerRow = bytePerPixel * BarcodeImageWidth;
var colorSpace = CGColorSpace.CreateDeviceRGB();
var context = new CGBitmapContext(byteArray,
BarcodeImageWidth, BarcodeImageHeight,
bitsPerComponent, bytesPerRow,
colorSpace, CGImageAlphaInfo.NoneSkipFirst);
BarcodeImageView.Image = new UIImage(context.ToImage());
And this code on android makes bitmap with wrong colors:
var bitmap = Bitmap.CreateBitmap(barcode.Width, barcode.Height, Bitmap.Config.Argb8888);
bitmap.CopyPixelsFromBuffer(ByteBuffer.Wrap(imageBytes));
barcode.SetImageBitmap(bitmap);
Fixed that by manually created pixel array with skipped first (alpha) byte.
int pixelsCount = imageBytes.Length / 4;
int[] pixels = new int[pixelsCount];
for (int i = 0; i < pixelsCount; i++)
{
var offset = 4 * i;
int r = imageBytes[offset + 1];
int g = imageBytes[offset + 2];
int b = imageBytes[offset + 3];
pixels[i] = Color.Rgb(r, g, b);
}
var bitmap = Bitmap.CreateBitmap(pixels, barcode.Width, barcode.Height, Bitmap.Config.Argb8888);
barcode.SetImageBitmap(bitmap);
I'm working on an audio based application for android platform.in this code first i record my voice in wav format and then i get fft(in fftbegir method) and now i want to filter my voice in 0-4khz with zeroing fft data and then perform ifft but when i play new wav file i can hear very bad quality sound with lots of noise .the fft class is here http://introcs.cs.princeton.edu/java/97data/FFT.java.html and here is my code:
private void filter(Complex[] x, int size) throws IOException {
double d, b;
String strI;
byte[] bytes = new byte[2];
int i = 0;
double k = -3.14159;
Complex[] f = new Complex[size];
Complex[] iff;
byte[] ddd;
double[] kkk = new double[size];
FFT q = new FFT();
short shor;
double data9[] = new double[size];
d = 2 * 3.14159 / size;
totalAudioLen = size;
totalDataLen = totalAudioLen + 36;
while (i < size) {//////to make its lenght length power of 2
data9[i] = k;
k = k + d;
i++;
}
i = 0;
while (i < (size / 2) - 2000) {
f[i] = new Complex(x[i].re(), x[i].im());
i++;
}
while (i < (size / 2) + 2000) { ///i want to remov 2000 sample of fft
f[i] = new Complex(0, 0);
i++;
}
while (i < size) {
f[i] = new Complex(x[i].re(), x[i].im());
i++;
}
iff = q.ifft(f);
try {
out9 = new FileOutputStream(getridemal());
out10 = new FileOutputStream(getwavfilter());
out11 = new FileOutputStream(getkhodesh());
WriteWaveFileHeader(out10, totalAudioLen, totalDataLen,
longSampleRate, channels, byteRate);
for (i = 0; i < size; i++) {
b = iff[i].re();
shor = (short) (b * 32768.0);
bytes = ByteConvert.convertToByteArray(shor);
out10.write(bytes, 0, 2);
}
} finally {
out9.close();
out10.close();
out11.close();
}
}
private void fftbegir(String input, String output) {
double[] data8;
int i = 0;
int r, k, l;
double b;
int m = 2;
try {
in5 = new FileInputStream(input);
out5 = new FileOutputStream(output);
AppLog.logString("File size: " + totalDataLen);
totalAudioLen = in5.getChannel().size();
data8 = SoundDataUtils.load16BitPCMRawDataFileAsDoubleArray();
l = data8.length;
while (l > m) {
m = m * 2;
}
Complex[] x = new Complex[m];
while (i < l) {
x[i] = new Complex(data8[i], 0);
i++;
}
in5.close();
i--;
for (i = l; i < m; i++) {
x[i] = new Complex(0, 0);
}
FFT f = new FFT();
Complex[] y = f.fft(x);
filter(y, m);
out5.close();
}
}
thanks:)
Filtering in the frequency domain as you are doing does not work well.
Whilst applying the inverse FFT to the results of a FFT yields the same samples (that is to say, it is invertible), this no longer holds true when coefficients are modified.
There are (at least) a few issues here:
The Gibbs Phenomemum which results from the sharp transition from pass-band to stop-band
The fact that the FFT is a fairly lousy band-pass filter in the first place. Components of the frequencies in the stop-band appear in several adjacent bands, and therefore remain in the signal.
Each FFT bin contains a real and imaginary component. A complex value of (0,0) has magnitude of 0 but also loses the phase information in the process.
You'd be better off with an IIR band-stop filter, which operates in the time domain. Besides working as expected, it is far cheaper to compute too.
I've a c++ websocket server, and I want to ti send an openCV image (cv::Mat) to my Android client.
I understood that I should use base64 string, but I can't find out how to do it from my openCV frames.
I don't know how to convert a cv::Mat to a bytearray.
Thank you
Hi you can use this below code which works form me
C++ Client
Here we will send BGR raw byte in to socket by accessing Mat data pointer.
Before sending make sure that Mat is continues otherwise make it continues.
int sendImage(Mat frame){
int imgSize = frame.total()*frame.elemSize();
int bytes=0;
int clientSock;
const char* server_ip=ANDROID_IP;
int server_port=2000;
struct sockaddr_in serverAddr;
socklen_t serverAddrLen = sizeof(serverAddr);
if ((clientSock = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
printf("\n--> socket() failed.");
return -1;
}
serverAddr.sin_family = PF_INET;
serverAddr.sin_addr.s_addr = inet_addr(server_ip);
serverAddr.sin_port = htons(server_port);
if (connect(clientSock, (sockaddr*)&serverAddr, serverAddrLen) < 0) {
printf("\n--> connect() failed.");
return -1;
}
frame = (frame.reshape(0,1)); // to make it continuous
/* start sending images */
if ((bytes = send(clientSock, frame.data, imgSize, 0)) < 0){
printf("\n--> send() failed");
return -1;
}
/* if something went wrong, restart the connection */
if (bytes != imgSize) {
cout << "\n--> Connection closed " << endl;
close(clientSock);
return -1;
}
return 0;
}
Java Server
You should know size of image going to receive.
Receives stream from socket and convert to byte array.
Convert byte array BGR and create Bitmap.
Code for Receiving byte array from C++ Server
public static byte imageByte[];
int imageSize=921600;//expected image size 640X480X3
InputStream in = server.getInputStream();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte buffer[] = new byte[1024];
int remainingBytes = imageSize; //
while (remainingBytes > 0) {
int bytesRead = in.read(buffer);
if (bytesRead < 0) {
throw new IOException("Unexpected end of data");
}
baos.write(buffer, 0, bytesRead);
remainingBytes -= bytesRead;
}
in.close();
imageByte = baos.toByteArray();
baos.close();
Code to Convert byte array to RGB bitmap image
int nrOfPixels = imageByte.length / 3; // Three bytes per pixel.
int pixels[] = new int[nrOfPixels];
for(int i = 0; i < nrOfPixels; i++) {
int r = imageByte[3*i];
int g = imageByte[3*i + 1];
int b = imageByte[3*i + 2];
if (r < 0)
r = r + 256; //Convert to positive
if (g < 0)
g = g + 256; //Convert to positive
if (b < 0)
b = b + 256; //Convert to positive
pixels[i] = Color.rgb(b,g,r);
}
Bitmap bitmap = Bitmap.createBitmap(pixels, 640, 480, itmap.Config.ARGB_8888);
Check this answer in question Serializing OpenCV Mat_ . If it is not a problem for you to use boost, it can solve your problem. Probably you will need some additional JNI magic on client side.
You can take into account which of the data are important for you: cols (number of columns), rows (number of columns), data (which contains the pixel information), type (data type and channel number).
You have to vectorize your matrix, because it is not neccessarrily continuous and take into account the variations of the size of a pixel in the memory.
Suppose:
cv::Mat m;
Then to allocate:
int depth; // measured in bytes
switch (m.depth())
{
// ... you might check for all of the possibilities
case CV_16U:
depth = 2;
}
char *array = new char[4 + 4 + 4 + m.cols * m.rows * m.channels() * depth]; // rows + cols + type + data
And than write the header information:
int *rows = array;
int *cols = &array[4];
int *type = &array[8];
*rows = m.rows;
*cols = m.cols;
*type = m.type;
And finally the data:
char *mPtr;
for (int i = 0; i < m.rows; i++)
{
mPtr = m.ptr<char>(i); // data type doesn't matter
for (int j = 0; j < m.cols; j++)
{
array[i * rows + j + 3 * 4] = mPtr[j];
}
}
Hopefully no bugs in the code.
How can I convert byte array received using socket.
The C++ client send image data which is of type uchar.
At the android side I am receiving this uchar array as byte[] which is ranges from -128 to +127.
What I wanted to do is that receives this data and display it. For that I was trying to convert to Bitmap using BitmapFactory.decodeByteArray(), but no luck I am getting null Bitmap. Am I doing right or any other method available.
Thanks in advance....
From the comments to the answers above, it seems like you want to create a Bitmap object from a stream of RGB values, not from any image format like PNG or JPEG.
This probably means that you know the image size already. In this case, you could do something like this:
byte[] rgbData = ... // From your server
int nrOfPixels = rgbData.length / 3; // Three bytes per pixel.
int pixels[] = new int[nrOfPixels];
for(int i = 0; i < nrOfPixels; i++) {
int r = data[3*i];
int g = data[3*i + 1];
int b = data[3*i + 2];
pixels[i] = Color.rgb(r,g,b);
}
Bitmap bitmap = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888);
I've been using it like below in one of my projects and so far it's been pretty solid. I'm not sure how picky it is as far as it not being compressed as a PNG though.
byte[] bytesImage;
Bitmap bmpOld; // Contains original Bitmap
Bitmap bmpNew;
ByteArrayOutputStream baoStream = new ByteArrayOutputStream();
bmpOld.compress(Bitmap.CompressFormat.PNG, 100, baoStream);
bytesImage = baoStream.toByteArray();
bmpNew = BitmapFactory.decodeByteArray(bytesImage, 0, bytesImage.length);
edit: I've adapted the code from this post to use RGB, so the code below should work for you. I haven't had a chance to test it yet so it may need some adjusting.
Byte[] bytesImage = {0,1,2, 0,1,2, 0,1,2, 0,1,2};
int intByteCount = bytesImage.length;
int[] intColors = new int[intByteCount / 3];
int intWidth = 2;
int intHeight = 2;
final int intAlpha = 255;
if ((intByteCount / 3) != (intWidth * intHeight)) {
throw new ArrayStoreException();
}
for (int intIndex = 0; intIndex < intByteCount - 2; intIndex = intIndex + 3) {
intColors[intIndex / 3] = (intAlpha << 24) | (bytesImage[intIndex] << 16) | (bytesImage[intIndex + 1] << 8) | bytesImage[intIndex + 2];
}
Bitmap bmpImage = Bitmap.createBitmap(intColors, intWidth, intHeight, Bitmap.Config.ARGB_8888);
InputStream is = new java.net.URL(urldisplay).openStream();
byte[] colors = IOUtils.toByteArray(is);
int nrOfPixels = colors.length / 3; // Three bytes per pixel.
int pixels[] = new int[nrOfPixels];
for(int i = 0; i < nrOfPixels; i++) {
int r = (int)(0xFF & colors[3*i]);
int g = (int)(0xFF & colors[3*i+1]);
int b = (int)(0xFF & colors[3*i+2]);
pixels[i] = Color.rgb(r,g,b);
}
imageBitmap = Bitmap.createBitmap(pixels, width, height,Bitmap.Config.ARGB_4444);
bmImage.setImageBitmap(imageBitmap );