It basically boils down to: if I have 4000 files in a directory, the File.isDirectory() function takes 1ms to execute, so the directory takes 4s to compute (too slow [ 1 ]).
I haven't got the most complete knowledge of the filesystem, but I think that isDirectory() can be batched for all the elements in the directory (reading a chunk of data, and then separating the individual file's metadatas). C/C++ code is acceptable (it can be run with the JNI) but should be left as a last resource.
I have found FileVisitor, but it doesn't seem that it is the best solution to my problem, as I don't have to visit the entire file tree. I also found BasicFileAttributeView but it seems it has the same problem. This is a related question but there aren't answers that provide a significant solution.
[ 1 ]: Because it is not the only thing I do it ends up being like 17s.
Edit: Code:
internal fun workFrom(unit: ProcessUnit<D>) {
launch {
var somethingAddedToPreload = false
val file = File(unit.first)
....
//Load children folders
file.listFiles(FileFilter {
it.isDirectory
})?.forEach {
getPreloadMapMutex().withLock {
if (getPreloadMap()[it.path] == null) {
val subfiles = it.list() ?: arrayOf()
for (filename in subfiles) {
addToProcess(it.path, ProcessUnit(it.path + DIVIDER + filename, unit.second))
}
getPreloadMap()[it.path] = PreloadedFolder(subfiles.size)
if (getPreloadMap().size > PRELOADED_MAP_MAXIMUM) cleanOldEntries()
getDeleteQueue().add(it.path)
somethingAddedToPreload = somethingAddedToPreload || subfiles.isNotEmpty()
}
}
}
...
if(somethingAddedToPreload) {
work()
}
}
}
private fun addToProcess(path: String, unit: ProcessUnit<D>) {
val f: () -> Pair<String, FetcherFunction<D>> = { load(path, unit) }
preloadList.add(f)
}
private suspend fun work() {
preloadListMutex.withLock {
preloadList.forEach {
launch {
val (path, data) = it.invoke()
if (FilePreloader.DEBUG) {
Log.d("FilePreloader.Processor", "Loading from $path: $data")
}
val list = getPreloadMap()[path]
?: throw IllegalStateException("A list has been deleted before elements were added. We are VERY out of memory!")
list.add(data)
}
}
preloadList.clear()
}
}
PS: I will remove the coroutines in work before implementing an optimization, complete code is here.
You could run a ls -F and check in the output if the file is a directory by looking at the last character, directories will end with /. E.g.
val cmd = "ls -F ${myFile.absolutePath}"
val process = Runtime.getRuntime().exec(cmd)
val files = process.inputStream
.bufferedReader()
.use(BufferedReader::readText)
.lines()
for (fileName in files) {
val isDir = fileName.endsWith("/")
}
I run a quick test on an emulator, with 4000 files and 4000 directories it's taking ~150ms for the whole thing.
Years ago, I had to write a JNI interface to opendir()/readdir()/closedir()/rewinddir() to address a similar performance issue. It's a bit of a hack, as it uses a jlong to hold a DIR * pointer from opendir() and pass it to subsequent readdir() and closedir() calls, but it was probably several orders of magnitude faster than Java's listFiles() on large directories.
It does require a JNI library, but you may find it useful:
/*
* Class: path_to_jni_ReadDir
* Method: opendir
* Signature: (Ljava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_path_to_jni_ReadDir_opendir
(JNIEnv *env, jclass cl, jstring jdirname )
{
const char *cdirname;
jboolean copy;
jlong dirp;
if ( NULL == jdirname )
{
return( ( jlong ) NULL );
}
cdirname= ( env )->GetStringUTFChars( jdirname , © );
if ( NULL == cdirname )
{
return( ( jlong ) NULL );
}
if ( 0 == ::strlen( cdirname ) )
{
( env )->ReleaseStringUTFChars( jdirname , cdirname );
return( ( jlong ) NULL );
}
dirp = ( jlong ) ::opendir( cdirname );
( env )->ReleaseStringUTFChars( jdirname , cdirname );
return( dirp );
}
/*
* Class: path_to_jni_ReadDir
* Method: readdir
* Signature: (J)Ljava/lang/String;
*/
JNIEXPORT jstring JNICALL Java_path_to_jni_ReadDir_readdir
(JNIEnv *env, jclass cl, jlong dirp )
{
struct dirent *dentp;
struct dirent *dentbuffer;
char buffer[ 8192 ];
jstring jfilename;
int rc;
dentbuffer = ( struct dirent * ) buffer;
dentp = NULL;
rc = ::readdir_r( ( DIR * ) dirp, dentbuffer, &dentp );
if ( ( SUCCESS != rc ) || ( NULL == dentp ) )
{
return( NULL );
}
jfilename = env->newStringUTF( dentp->d_name );
return( jfilename );
}
/*
* Class: path_to_jni_ReadDir
* Method: closedir
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_path_to_jni_ReadDir_closedir
(JNIEnv *env, jclass cl, jlong dirp )
{
jint rc;
rc = ::closedir( ( DIR * ) dirp );
return( rc );
}
/*
* Class: path_to_jni_ReadDir
* Method: rewinddir
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_path_to_jni_ReadDir_rewinddir
(JNIEnv *env, jclass cl, jlong dirp )
{
::rewinddir( ( DIR * ) dirp );
return;
}
I've removed customer-identifying information from the code, so it's not exactly as delivered and may have some typos.
Given the Android dirent structure is
struct dirent {
uint64_t d_ino;
int64_t d_off;
unsigned short d_reclen;
unsigned char d_type;
char d_name[256];
};
You can modify the JNI readdir method to add filters based on the d_type field, which contains one of the following values:
#define DT_UNKNOWN 0
#define DT_FIFO 1
#define DT_CHR 2
#define DT_DIR 4
#define DT_BLK 6
#define DT_REG 8
#define DT_LNK 10
#define DT_SOCK 12
#define DT_WHT 14
For example, if you're looking for directories, you can add a loop to keep calling ::readdir_r() until it either returns NULL or the d_type field is DT_DIR:
for ( ;; )
{
rc = ::readdir_r( ( DIR * ) dirp, dentbuffer, &dentp );
if ( ( SUCCESS != rc ) || ( NULL == dentp ) )
{
return( NULL );
}
if ( dentp->d_type == DT_DIR )
{
break;
}
}
Related
I have been trying to read JSON data from a JSON string, but my app keeps crashing immediately after it starts. No errors are displayed.
The JSON string is passed from the Javascript code in React Native :
let myJSONString = JSON.stringify({foo: "bar"});
The CMakeLists.txt file that sets up boost :
set(my_boost_dir ${CMAKE_CURRENT_SOURCE_DIR}/libs/ndk_25_boost_1.79.0)
set(MY_BOOST_LIBS_DIR ${my_boost_dir}/libs)
set(MY_BOOST_INC_DIR ${my_boost_dir}/include)
#-----------------------------------------
add_library (libboost_container SHARED IMPORTED)
set_target_properties( libboost_container PROPERTIES IMPORTED_LOCATION
${MY_BOOST_LIBS_DIR}/${ANDROID_ABI}/libboost_container.so
)
#-----------------------------------------
#-----------------------------------------
add_library (libboost_json SHARED IMPORTED)
set_target_properties( libboost_json PROPERTIES IMPORTED_LOCATION
${MY_BOOST_LIBS_DIR}/${ANDROID_ABI}/libboost_json.so
)
#-----------------------------------------
#-----------------------------------------
add_library (libboost_system SHARED IMPORTED)
set_target_properties( libboost_system PROPERTIES IMPORTED_LOCATION
${MY_BOOST_LIBS_DIR}/${ANDROID_ABI}/libboost_system.so
)
#-----------------------------------------
add_library( libboost_chrono SHARED IMPORTED)
set_target_properties(libboost_chrono PROPERTIES IMPORTED_LOCATION
${MY_BOOST_LIBS_DIR}/${ANDROID_ABI}/libboost_chrono.so
)
#-----------------------------------------
include_directories( ${MY_BOOST_INC_DIR}
)
This is how I try reading the JSON string :
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/json.hpp>
JNIEXPORT jstring JNICALL
Java_com_ca_com_1gen_1lib_1pers_c_1libs_1core_MyStringToJSON_myStringToJSON(JNIEnv *env,
jobject instance,
jstring myJSONStringEntity_)
{
const char *myJSONStringEntity = env->GetStringUTFChars(myJSONStringEntity_, 0);
// Short alias for this namespace
namespace pt = boost::property_tree;
// Create a root
pt::ptree root;
// Load the json file in this ptree
pt::read_json(myJSONStringEntity, root);
// Read values
std::string returnVal = root.get<std::string>("foo", 0);
return returnVal;
}
Where am I doing it wrong?
Thank you all in advance.
You're mixing Boost JSON (a JSON library) and Boost Property (a Property Tree Library). Only use the first!
Live On Coliru
#include <boost/json.hpp>
#include <boost/json/src.hpp> // for COLIRU (header-only)
#include <iostream>
#define JNIEXPORT
#define JNICALL
using jstring = std::string;
struct jobject { };
struct JNIEnv { char const* GetStringUTFChars(jstring const& s, int) { return s.data(); } };
JNIEXPORT jstring JNICALL
Java_com_ca_com_1gen_1lib_1pers_c_1libs_1core_MyStringToJSON_myStringToJSON(JNIEnv *env,
jobject /*instance*/,
jstring myJSONStringEntity_)
{
const char *myJSONStringEntity = env->GetStringUTFChars(myJSONStringEntity_, 0);
// Short alias for this namespace
namespace bj = boost::json;
// Create a root
bj::value root = bj::parse(myJSONStringEntity);
// Read values
std::string returnVal{root.at("foo").as_string()};
return returnVal;
}
int main() {
auto f = Java_com_ca_com_1gen_1lib_1pers_c_1libs_1core_MyStringToJSON_myStringToJSON;
std::cout << "Example: " << f(nullptr, {}, R"({"foo":"bar", "qux": [1,2,3]})") << "\n";
}
Prints
Example: bar
I'm performing certificate pinning in flutter by securely storing the certificate in JNI and fetching it during run time. But I get BAD_PKCS12_DATA(pkcs8_x509.c:626), errno = 0) when I fetch the data from JNI. The pinning works if I set it directly in flutter code though like
List<int> _crt = <int>[45, 45, 45, 45, 45, 66, 69, 71, 73, 78, ...]
Here is the JNI method:
extern "C" JNIEXPORT jintArray JNICALL Java_com_package_android_MainActivity_getCert
(JNIEnv *env, jobject This)
{
int a[] ={45,45,45,45,45,...};
jintArray ret = env->NewIntArray(sizeof(a));
env->SetIntArrayRegion(ret, 0, 6, a);
return ret;
}
MainActivity.kt:
external fun getCert(): IntArray
override fun configureFlutterEngine(#NonNull flutterEngine: FlutterEngine) {
super.configureFlutterEngine(flutterEngine)
MethodChannel(flutterEngine.dartExecutor.binaryMessenger, CHANNEL).setMethodCallHandler {
call, result ->
if (call.method == "cert") {
result.success(getCert())
}
}
}
Flutter code:
List<int> _crt;
_crt = await _platform.invokeMethod("cert");
//print("CRT: " + _crt.length);
SecurityContext context = SecurityContext(withTrustedRoots: true);
context.setTrustedCertificatesBytes(_crt);
httpClient = new HttpClient(context: context);
I'm confused why the returned int array from JNI doesnt work but would have no problem if I set it directly in flutter?
sizeof(a) doesn't give you the number of elements in a. It gives you total size of all the elements, in bytes.
What you want is sizeof(a) / sizeof(int) or sizeof(a) / sizeof(a[0]).
I'm testing Rust with JNI async execution. I want to do execute requests in Rust and return the result to Android asynchronously with callback. I'm testing code to execute the request in the command line and it works fine.
That is how it works on command line:
Callback struck:
struct Processor {
pub(crate) callback: Box<dyn FnMut(String)>,
}
impl Processor {
fn set_callback(&mut self, c: impl FnMut(String) + 'static) {
self.callback = Box::new(c);
}
fn process_events(&mut self, result: String) {
(self.callback)(result);
}
}
Tokio/reqwest:
const DATA_URL: &str = "https://pokeapi.co/api/v2/pokemon/1/";
#[tokio::main]
pub async fn load_swapi_async_with_cb(callback: Box<dyn FnMut(String)>) -> Result<(), Box<dyn std::error::Error>> {
println!("load_swload_swapi_async_with_cbapi_async started");
let mut cb = Processor {
callback: Box::new(callback),
};
let body = reqwest::get(DATA_URL)
.await?
.json::<HashMap<String, String>>()
.await?;
//println!("{:#?}", body);
let name = match body.get("name") {
Some(name) => name,
None => "Failed to parse"
}.to_string();
println!("Name is: {} ", name);
cb.process_events(name);
Ok(())
}
And JNI part:
#[no_mangle]
#[allow(non_snake_case)]
pub extern "C" fn Java_com_omg_app_greetings_MainActivity_callback(env: JNIEnv,
_class: JClass,
callback: JObject) {
static callback: dyn FnMut(String) + 'static = |name| {
let response = env.new_string(&name).expect("Couldn't create java string!");
env.call_method(callback, "rustCallbackResult", "(Ljava/lang/String;)V",
&[JValue::from(JObject::from(response))]).unwrap();
};
pokemon_api(callback);
}
And pokemon API method:
#[no_mangle]
pub extern fn pokemon_api(callback: impl FnMut(String) + 'static) {
let cb_box = Box::new(callback);
swapi::load_swapi_async_with_cb(cb_box);
}
The error I'm facing:
JNI ENV env non-constant value:
let response = env.new_string(&name).expect("Couldn't create java string!");
| ^^^ non-constant value
callback - doesn't have a size known at compile-time:
static callback: dyn FnMut(String) + 'static = |name| {
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^ doesn't have a size known at compile-time
I was checking how this working, but example seems to be out of date:
* https://github.com/mozilla/rust-android-gradle/blob/master/samples/rust/src/lib.rs
I fixed with my problem with the usage of the https://github.com/Dushistov/rust_swig
After you integrate it, it will autogenerate code and you can check how it does it.
When I try to call JVM from thread in C++, debbuger says that JNIEnv pointer is gone. Code:
void thread_call(JNIEnv* env,jobject text_view,jmethodID setText,const char* str){
env->CallVoidMethod(text_view,setText,env->NewStringUTF(str));
}
JNIEXPORT jstring JNICALL
Java_com_example_project_1lenspath_MainActivity_initPullThread(
JNIEnv *env,
jobject, jobject text_view){
...
std::thread pull(thread_call,env,text_view,setText,std::to_string(t).c_str());
...
}
//Debugger:
error: use of undeclared identifier 'env'
Are there any ways to call JVM without JNIEnv or any other solution?
You can cache the jvm pointer inside JNI_OnLoad, e.g.
JavaVM* g_jvm;
jint JNI_OnLoad(JavaVM* vm, void* reserved)
{
g_jvm = vm;
...
return result;
}
Then use this g_jvm to do AttachCurrentThread and DetachCurrentThread, e.g.
void foo()
{
jint res = javaVM->GetEnv((void**)&env, JNI_VERSION_1_6);
if (res != JNI_OK) {
res = javaVM->AttachCurrentThread(&env, NULL);
if (JNI_OK != res) {
return NULL;
}
}
...
g_jvm->DetachCurrentThread();
}
See: https://github.com/russell-shizhen/JniExample/blob/master/app/src/main/cpp/native-lib.cpp
I'm building a C++ library with Visual Studio and using it in an Android Studio project that I run in an emulator. If I don't use the problem code below, the rest of the project works fine.
In my java class:
public native int androidmain();
public void aFunction() {
androidmain();
}
And in my C++ library:
#if defined( PLATFORM_ANDROID )
extern "C" {
JNIEXPORT jint JNICALL Java_com_TestsRunnerAndroid2015_TestsRunnerAndroid2015_androidmain( JNIEnv * env, jobject, jobjectArray argsObj ) {
OutputString( "androidmain called" ); // ALMODEBUG
jsize stringCount = 0;
if ( env == nullptr ) {
OutputString( "env is null" );
} else {
OutputString( "env is not null" );
}
if ( argsObj != nullptr ) {
OutputString( "argsObj was not null" );
stringCount = env->GetArrayLength( argsObj ); // CRASH HERE
OutputString( "Got argsObj length." );
} else {
OutputString( "argsObj was null" );
}
...
The output I get from this is:
D/TAG: androidmain called
D/TAG: env is not null
D/TAG: argsObj was not null
W/art: Suspending all threads took: 94.170ms
A/art: art/runtime/java_vm_ext.cc:470] JNI DETECTED ERROR IN APPLICATION: use of invalid jobject 0x12c62000
A/art: art/runtime/java_vm_ext.cc:470] from int com.TestsRunnerAndroid2015.TestsRunnerAndroid2015.androidmain()
Additional comment: I'm aware I'm sending no arguments, yet parsing an argument array. I'm expecting the count of objects to be zero.
In Java code you declare androidmain() as not taking any arguments, but in C code you declare Java_com_TestsRunnerAndroid2015_TestsRunnerAndroid2015_androidmain(...) as taking an array of objects as an argument.
In Java code, when calling androidmain() you are not supplying argsObj argument for Java_com_TestsRunnerAndroid2015_TestsRunnerAndroid2015_androidmain(...)
Try declaring androidmain as public native int androidmain(Object[] argsObj); and when you call it -- pass that argument.