Develop Reference

TFLite C++ Invoke causes seg fault on android

I'm trying to build an android c++ app using tflite. I failed to compile the .so file on my own so I got some pre-compiled one's from the internet. I've created a dummy project to test it and it works just fine. However when I implement it on my project everything falls apart. I've checked whether the inputs are right, they are, the size is right, initialization is fine, it's almost identical with my dummy project, so what could be the issue?
Here is my code:
tflite.cpp:
#include "tflite.hpp"
tflite::tflite(uint8_t *data, size_t size)
{
try
{
lib_tflite::ErrorReporter* error_reporter;
this->m_env = lib_tflite::FlatBufferModel::BuildFromBuffer((const char *)data, size, error_reporter);
lib_tflite::ops::builtin::BuiltinOpResolver resolver;
lib_tflite::InterpreterBuilder(*this->m_env, resolver)(&m_interpreter);
if (m_interpreter->AllocateTensors() != kTfLiteOk)
{
throw std::runtime_error("Failed to allocate tensor");
}
m_interpreter->SetNumThreads(2);
this->m_input_node_count = m_interpreter->inputs().size();
this->m_output_node_count = m_interpreter->outputs().size();
for (size_t idx = 0; idx < this->m_input_node_count; ++idx)
{
int input = m_interpreter->inputs()[idx];
auto height = m_interpreter->tensor(input)->dims->data[1];
auto width = m_interpreter->tensor(input)->dims->data[2];
auto channels = m_interpreter->tensor(input)->dims->data[3];
std::vector<int> res = {(int)this->m_input_node_count, channels, width, height};
this->m_inputDims.push_back(res);
const TfLiteTensor* input_tensor = m_interpreter->input_tensor(idx);
size_t element_count = 1;
for (int i = 0; i < input_tensor->dims->size; i++)
{
element_count *= input_tensor->dims->data[i];
}
this->m_input_elem_size.push_back(element_count);
}
for (size_t idx = 0; idx < this->m_output_node_count; ++idx)
{
int output = m_interpreter->outputs()[idx];
auto height = m_interpreter->tensor(output)->dims->data[1];
auto width = m_interpreter->tensor(output)->dims->data[2];
auto channels = m_interpreter->tensor(output)->dims->data[3];
std::vector<int> res = {(int)this->m_output_node_count, channels, width, height};
this->m_outputDims.push_back(res);
const TfLiteTensor* output_tensor = m_interpreter->output_tensor(idx);
int element_count = 1;
for (int i = 0; i < output_tensor->dims->size; i++) {
element_count *= output_tensor->dims->data[i];
}
this->m_output_elem_size.push_back(element_count);
}
for (size_t idx = 0; idx < this->m_input_node_count; ++idx)
{
this->m_input_buffer.emplace_back(this->m_input_elem_size[idx], 0
);
}
for (size_t idx = 0; idx < this->m_output_node_count; ++idx)
{
this->m_output_buffer.emplace_back(this->m_output_elem_size[idx], 0
);
}
}
bool tflite::run(std::vector<float> &t_out_buffer,
std::vector<float> &t_cls_buffer,
std::vector<float> &t_buffer,
size_t region_size) noexcept
{
for(size_t idx = 0; idx < this->m_input_node_count; idx++)
{
float* data_ptr = m_interpreter->typed_input_tensor<float>(idx);
memcpy(data_ptr, t_buffer.data(), this->m_input_elem_size[idx]);
}
// This is where it fails
if (kTfLiteOk != this->m_interpreter->Invoke())
{
log_error("Failed to invoke\n");
return false;
}
for(size_t idx = 0; idx < this->m_output_node_count; idx++)
{
float* output = this->m_interpreter->typed_output_tensor<float>(idx);
this->m_output_buffer[idx] = std::vector<float> (output,
output + this->m_output_elem_size[idx]);
}
t_cls_buffer = this->m_output_buffer[0];
t_out_buffer = m_output_buffer[1];
}
/* end_of_file */
tflite.hpp:
#ifndef TFLITE_DRIVER_HPP
#define TFLITE_DRIVER_HPP
#include <memory>
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/model.h"
#include "tensorflow/lite/optional_debug_tools.h"
namespace lib_tflite = ::tflite;
class tflite
{
public:
tflite() = delete;
virtual ~tflite() noexcept = default;
tflite(tflite &&) = delete;
tflite & operator=(tflite &&) = delete;
tflite(const tflite &) = delete;
tflite & operator=(tflite &) = delete;
tflite(uint8_t *data, size_t size);
bool run( std::vector<float> &t_out_buffer,
std::vector<float> &t_cls_buffer,
std::vector<float> &t_buffer,
size_t region_size) noexcept;
private:
lib_tflite::ErrorReporter* error_reporter;
lib_tflite::ops::builtin::BuiltinOpResolver resolver;
std::unique_ptr<lib_tflite::FlatBufferModel> m_env;
std::unique_ptr<lib_tflite::Interpreter> m_interpreter;
std::vector<const char *> m_input_names;
std::vector<const char *> m_output_names;
size_t m_input_node_count;
size_t m_output_node_count;
std::vector<lib_tflite::Tensor> m_inputTensors;
std::vector<lib_tflite::Tensor> m_outputTensors;
std::vector<size_t> m_input_elem_size;
std::vector<size_t> m_output_elem_size;
std::vector<std::vector<int>> m_inputDims;
std::vector<std::vector<int>> m_outputDims;
std::vector<std::vector<float>> m_input_buffer;
std::vector<std::vector<float>> m_output_buffer;
std::unique_ptr<lib_tflite::MemoryAllocation> m_memory_info;
};
#endif // TFLITE_DRIVER_HPP
/* end_of_file */
Note that my project runs on 2 threads, one get's the input and the other calls tflite. Like I've said the input's are indeed correct. The constructor is ran once, but the run function runs every frame.
Here is the output I get when I run it:
E/libc: Access denied finding property "ro.mediatek.platform"
E/libc: Access denied finding property "ro.chipname"
A/libc: Fatal signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0x13c6759bfc61 in tid 27936 (processing), pid 27893 (ample.nerveblox)
I've debugged and found out the line this error happens is where I call this->m_interpreter->Invoke() inside the run function.

access denied error seems like there is no property for "ro.mediatek.platform" & "ro.chipname" if you run the "adb shell getprop".
So, I think, "ro.mediatek.platform" & "ro.chipname" are not a property.

why getpidcon() error on AOSP 7.1.2?

I am working on making a customized bsp based on AOSP Nougat latest source.
Android service process ask service manager to find or add the service.
And service manager try to check mac permissions by calling svc_can_register() or svc_can_find() which calls check_mac_perms() which calls getpidcon().
Let's see svc_can_find()
static int svc_can_find(const uint16_t *name, size_t name_len, pid_t spid, uid_t uid)
{
const char *perm = "find";
return check_mac_perms_from_lookup(spid, uid, perm, str8(name, name_len)) ? 1 : 0;
}
check_mac_perms_from_lookup() is like this:
static bool check_mac_perms_from_lookup(pid_t spid, uid_t uid, const char *perm, const char *name)
{
bool allowed;
char *tctx = NULL;
if (selinux_enabled <= 0) {
return true;
}
if (!sehandle) {
ALOGE("SELinux: Failed to find sehandle. Aborting service_manager.\n");
abort();
}
if (selabel_lookup(sehandle, &tctx, name, 0) != 0) {
ALOGE("SELinux: No match for %s in service_contexts.\n", name);
return false;
}
allowed = check_mac_perms(spid, uid, tctx, perm, name);
freecon(tctx);
return allowed;
}
It calls check_mac_perms(). check_mac_perms() like this:
static bool check_mac_perms(pid_t spid, uid_t uid, const char *tctx, const char *perm, const char *name)
{
char *sctx = NULL;
const char *class = "service_manager";
bool allowed;
struct audit_data ad;
if (getpidcon(spid, &sctx) < 0) {
ALOGE("SELinux: getpidcon(pid=%d) failed to retrieve pid context.\n", spid);
return false;
}
ad.pid = spid;
ad.uid = uid;
ad.name = name;
int result = selinux_check_access(sctx, tctx, class, perm, (void *) &ad);
allowed = (result == 0);
freecon(sctx);
return allowed;
}
It calls getpidcon(). getpidcon() is defined in
external/selinux/libselinux/src/procattr.c
getpidcon() is defined like this:
#define getpidattr_def(fn, attr) \
int get##fn(pid_t pid, char **c) \
{ \
if (pid <= 0) { \
errno = EINVAL; \
return -1; \
} else { \
return getprocattrcon(c, pid, #attr); \
} \
}
...
...
getpidattr_def(pidcon, current)
"getpidattr_def(pidcon, current)" is expanded to getpidcon() function
definition and it calls getprocatrcon()
getprocattrcon() is like this:
static int getprocattrcon(char ** context,
pid_t pid, const char *attr)
{
char *buf;
size_t size;
int fd;
ssize_t ret;
int errno_hold;
fd = openattr(pid, attr, O_RDONLY);
if (fd < 0)
return -1;
size = selinux_page_size;
buf = malloc(size);
if (!buf) {
ret = -1;
goto out;
}
memset(buf, 0, size);
do {
ret = read(fd, buf, size - 1);
} while (ret < 0 && errno == EINTR);
if (ret < 0)
goto out2;
if (ret == 0) {
*context = NULL;
goto out2;
}
*context = strdup(buf);
if (!(*context)) {
ret = -1;
goto out2;
}
ret = 0;
out2:
free(buf);
out:
errno_hold = errno;
close(fd);
errno = errno_hold;
return ret;
}
Pretty simple huh? Just opening some files and reading the contents
and return it by function argument.
It fails at openattr(). I've confirmed this by inserting some log function in
openattr(). openattr() is also simple function.
static int openattr(pid_t pid, const char *attr, int flags)
{
int fd, rc;
char *path;
pid_t tid;
if (pid > 0) {
rc = asprintf(&path, "/proc/%d/attr/%s", pid, attr);
} else if (pid == 0) {
rc = asprintf(&path, "/proc/thread-self/attr/%s", attr);
if (rc < 0)
return -1;
fd = open(path, flags | O_CLOEXEC);
if (fd >= 0 || errno != ENOENT)
goto out;
free(path);
tid = gettid();
rc = asprintf(&path, "/proc/self/task/%d/attr/%s", tid, attr);
} else {
errno = EINVAL;
return -1;
}
if (rc < 0)
return -1;
fd = open(path, flags | O_CLOEXEC);
out:
free(path);
return fd;
}
The fail point is "fd = open(path, flags | O_CLOEXEC);"
Even if the file exists, almost always opening fails. I don't understand ths and want to know what caused the problem. I've confirmed the failure
by inserting some log printing codes, checking android log(adb logcat) and reading the file from android shell(adb shell), e.g. 'cat /proc/412/attr/current'. Reading by 'cat ...' succeeded but log shows the opening the
file fails. The odd thing is if 'pid' is 0, it succeeds.
If opening fails, services can't be launched so the system don't
boot properly. If I ignore the fails and return success from getpidcon()
the system boots properly but this is not the right thing to do obviously.
I'm testing the bsp as selinux permissive mode.
Can anyone have a experience like me? If anyone, please share the
experience and the solution of the problem.
Thank you.
Sangyong Lee.

Tracing a process using ptrace on Android ARM64

I'm experiencing an infinite loop trying to trace simple hello world using ptrace() on Android ARM64 emulator that emulates AARCH64. I'm not sure why it is not stopping. I'm trying to trace a hello world program and get all executed instructions but it seems that this condition never returns false: while (WIFSTOPPED(wait_status))
int main(int argc, char** argv)
{
pid_t child_pid;
if(argc < 2)
{
fprintf(stderr, "Expected a program name as argument\n");
return -1;
}
child_pid = fork();
if (child_pid == 0)
run_target(argv[1]);
else if (child_pid > 0)
run_debugger(child_pid);
else
{
perror("fork");
return -1;
}
return 0;
}
void run_target(const char* programname)
{
printf("target started. will run '%s'\n", programname);
/* Allow tracing of this process */
if (ptrace(PTRACE_TRACEME, 0, 0, 0) < 0)
{
perror("ptrace");
return;
}
/* Replace this process's image with the given program */
execl(programname, programname, 0);
}
void run_debugger(pid_t child_pid)
{
int wait_status;
unsigned icounter = 0;
printf("debugger started\n");
/* Wait for child to stop on its first instruction */
wait(&wait_status);
while (WIFSTOPPED(wait_status))
{
icounter++;
struct user_pt_regs regs;
struct iovec io;
io.iov_base = &regs;
io.iov_len = sizeof(regs);
if (ptrace(PTRACE_GETREGSET, child_pid, (void*)NT_PRSTATUS, (void*)&io) == -1)
printf("BAD REGISTER REQUEST\n");
unsigned instr = ptrace(PTRACE_PEEKTEXT, child_pid, regs.pc, 0);
printf("icounter = %u. PCP = 0x%08x. instr = 0x%08x\n", icounter, regs.pc, instr);
/* Make the child execute another instruction */
if (ptrace(PTRACE_SINGLESTEP, child_pid, 0, 0) < 0)
{
perror("ptrace");
return;
}
/* Wait for child to stop on its next instruction */
wait(&wait_status);
}
printf("the child executed %u instructions\n", icounter);
}

How to extract frames at 30 fps using FFMPEG APIs on Android?

We are working on a project that consumes FFMPEG library for video frame extraction on Android platform.
On Windows, we have observed:
Using CLI, ffmpeg is capable of extracting frames at 30 fps using command ffmpeg -i input.flv -vf fps=1 out%d.png.
Using Xuggler, we are able to extract frames at 30 fps.
Using FFMPEG APIs directly in code, we are getting frames at 30 fps.
But when we use FFMPEG APIs directly on Android (See Hardware Details), we are getting following results:
720p video (1280 x 720) - 16 fps (approx. 60 ms/frame)
1080p video (1920 x 1080) - 7 fps (approx. 140 ms/frame)
We haven't tested Xuggler/CLI on Android yet.
Ideally, we should be able to get the data in constant time (approx. 30 ms/frame).
How can we get 30 fps on Android?
Code being used on Android:
if (avformat_open_input(&pFormatCtx, pcVideoFile, NULL, NULL)) {
iError = -1; //Couldn't open file
}
if (!iError) {
//Retrieve stream information
if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
iError = -2; //Couldn't find stream information
}
//Find the first video stream
if (!iError) {
for (i = 0; i < pFormatCtx->nb_streams; i++) {
if (AVMEDIA_TYPE_VIDEO
== pFormatCtx->streams[i]->codec->codec_type) {
iFramesInVideo = pFormatCtx->streams[i]->nb_index_entries;
duration = pFormatCtx->streams[i]->duration;
begin = pFormatCtx->streams[i]->start_time;
time_base = (pFormatCtx->streams[i]->time_base.num * 1.0f)
/ pFormatCtx->streams[i]->time_base.den;
pCodecCtx = avcodec_alloc_context3(NULL);
if (!pCodecCtx) {
iError = -6;
break;
}
AVCodecParameters params = { 0 };
iReturn = avcodec_parameters_from_context(&params,
pFormatCtx->streams[i]->codec);
if (iReturn < 0) {
iError = -7;
break;
}
iReturn = avcodec_parameters_to_context(pCodecCtx, &params);
if (iReturn < 0) {
iError = -7;
break;
}
//pCodecCtx = pFormatCtx->streams[i]->codec;
iVideoStreamIndex = i;
break;
}
}
}
if (!iError) {
if (iVideoStreamIndex == -1) {
iError = -3; // Didn't find a video stream
}
}
if (!iError) {
// Find the decoder for the video stream
pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if (pCodec == NULL) {
iError = -4;
}
}
if (!iError) {
// Open codec
if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
iError = -5;
}
if (!iError) {
iNumBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB24, pCodecCtx->width,
pCodecCtx->height, 1);
// initialize SWS context for software scaling
sws_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height,
pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height,
AV_PIX_FMT_RGB24,
SWS_BILINEAR,
NULL,
NULL,
NULL);
if (!sws_ctx) {
iError = -7;
}
}
clock_gettime(CLOCK_MONOTONIC_RAW, &end);
delta_us = (end.tv_sec - start.tv_sec) * 1000000
+ (end.tv_nsec - start.tv_nsec) / 1000;
start = end;
//LOGI("Starting_Frame_Extraction: %lld", delta_us);
if (!iError) {
while (av_read_frame(pFormatCtx, &packet) == 0) {
// Is this a packet from the video stream?
if (packet.stream_index == iVideoStreamIndex) {
pFrame = av_frame_alloc();
if (NULL == pFrame) {
iError = -8;
break;
}
// Decode video frame
avcodec_decode_video2(pCodecCtx, pFrame, &iFrameFinished,
&packet);
if (iFrameFinished) {
//OUR CODE
}
av_frame_free(&pFrame);
pFrame = NULL;
}
av_packet_unref(&packet);
}
}

You need some structures and functions from libavfilter.
The vf option means "video filter". The command line ffmpeg -i input -vf fps=30 out%d.png will output video_length_in_seconds * 30 regardless the original video fps. That means if the video is of 25 fps, you'll get some duplicate frames. While if the video is more than 30 fps, you'll lose some frames.
To achieve this, you have to init some filter context. See filtering_video.c example from ffmpeg source.
AVFilter* buffersrc = avfilter_get_by_name("buffer");
AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* outputs = avfilter_inout_alloc();
AVFilterInOut* inputs = avfilter_inout_alloc();
AVRational time_base = p_format_ctx->streams[video_stream]->time_base;
enum AVPixelFormat pix_fmts[] = { p_codec_ctx->pix_fmt, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
// failed, goto cleanup
}
char args[512];
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
p_codec_ctx->width, p_codec_ctx->height, p_codec_ctx->pix_fmt,
time_base.num, time_base.den,
p_codec_ctx->sample_aspect_ratio.num, p_codec_ctx->sample_aspect_ratio.den);
int ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
LOG(ERROR) << "Cannot create buffer source";
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return false;
}
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
// failed... blabla
}
ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts,
AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
// failed... blabla
}
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
const char* filter_description[256] = "fps=fps=30";
if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr.c_str(),
&inputs, &outputs, NULL)) < 0) {
// failed...
}
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0) {
// failed...
}
Ok, this is all initialization needed.
And adding some codes to decoding part:
avcodec_decode_video2(p_codec_ctx, p_frame, &got_frame, &packet);
if (*got_frame) {
p_frame->pts = av_frame_get_best_effort_timestamp(p_frame);
if (av_buffersrc_add_frame_flags(buffersrc_ctx, p_frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
// failed... blabla
}
while (1) {
int ret = av_buffersink_get_frame(buffersink_ctx, p_frame_stage);
// p_frame_stage is a AVFrame struct. Same size as p_frame. Need to allocated before.
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
if (ret < 0) {
// something wrong. filter failed.
}
// Do something with p_frame_stage here.
}
}

Please take a look at https://gitter.im/mobile-ffmpeg/Lobby?at=5c5bb384f04ef00644f1bb4e A few lines below, they mention options to accelerate the process, such as ... -preset ultrafast, -threads 10, -tune zerolatency, -x264-params sliced-threads=1

Choppy audio decoding when using swr_convert in Android

I'm using the following C function to decode packets in Android (with JNI). When I play an mp3 file the code works fine however and wma file results in choppy audio. I suspect the issue may be with the "swr_convert" function and the data_size I'm using but I'm not sure. Does anyone know why this would be happening?
int decodeFrameFromPacket(AVPacket *aPacket) {
int n;
AVPacket *pkt = aPacket;
AVFrame *decoded_frame = NULL;
int got_frame = 0;
if (aPacket->stream_index == global_audio_state->audio_stream) {
if (!decoded_frame) {
if (!(decoded_frame = avcodec_alloc_frame())) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Could not allocate audio frame\n");
return -2;
}
}
if (avcodec_decode_audio4(global_audio_state->audio_st->codec, decoded_frame, &got_frame, aPacket) < 0) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Error while decoding\n");
return -2;
}
int data_size = 0;
if (got_frame) {
/* if a frame has been decoded, output it */
data_size = av_samples_get_buffer_size(NULL, global_audio_state->audio_st->codec->channels,
decoded_frame->nb_samples,
global_audio_state->audio_st->codec->sample_fmt, 1);
}
swr_convert(global_audio_state->swr, (uint8_t **) &gAudioFrameRefBuffer, decoded_frame->nb_samples, (uint8_t const **) decoded_frame->data, decoded_frame->nb_samples);
avcodec_free_frame(&decoded_frame);
gAudioFrameDataLengthRefBuffer[0] = data_size;
return AUDIO_DATA_ID;
}
return 0;
}