Develop Reference

Large file upload fails on node.js Hapi

I have a node.js script which should handle file uploads, also multiple at once. Uploading pictures and voices work just fine. However, video files at around 10 MB or larger do not upload. Sometimes it doesn't work at all and sometimes it gets stuck in the fs.writeFile function. Maybe there is a better way general as I came up with many parts in the code on my own. I need the md5 hash before creating the file on disk because its path will be generated from the hash. Also I get a SocketTimeoutException on the Android side. Code is mainly focused on that part right now, so don't worry about the missing input validation and onProgress.
NodeJS:
server.route({
method: 'POST',
path: '/uploadFile',
config: {
payload: {
output: 'stream',
allow: 'multipart/form-data',
maxBytes: 100*1024*1024 //100 mb
}
}, handler: async function (request, reply)
{
await incoming_uploadFile(request, reply);
}
});
...........
var joi = require('joi');
import { Paths } from "../util/Paths";
import * as fs from 'fs';
let data;
let numOfFiles: number;
export async function incoming_uploadFile(request, reply) {
data = request.payload;
await Application.InitializeSocket(null, "UploadFile");
let userID = await Application.AuthUser(JSON.parse(data['auth']));
numOfFiles = parseInt(data['numOfFiles']);
if (numOfFiles > 0)
upload(0);
}
async function upload(i: number)
{
const file = data['file' + i];
const meta = data['fileMeta' + i];
const metaJson = Application.StringToJson(meta);
const fileType: string = metaJson['fileType'];
const extension: string = metaJson['extension'];
var crypto = require('crypto');
const md5 = crypto.createHash('md5');
var length = parseInt(file.hapi.headers["content-length"]);
let buffer: Buffer = new Buffer(length);
let bufPos : number = 0;
file.on('data', function (b : Uint8Array) {
for (var i = 0; i < b.byteLength; ++i)
buffer.writeUInt8(b[i], bufPos++);
});
file.on('end', function (err) {
var hash = md5.update(buffer.toString("base64")).digest("hex");
const filePath: string = Paths.getFilePath(hash, fileType, extension);
// Creates all dirs that are missing on the path
var shell = require('shelljs');
shell.mkdir('-p', require('path').dirname(filePath));
console.log("writing buffer to file...");
fs.writeFile(filePath, buffer,null);
if ((++i) < numOfFiles)
upload(i);
});
}
Android uploading the file(s):
public static void uploadAttachments(ArrayList<EventAttachment> attachments)
{
OkHttpClient client = new OkHttpClient();
// Add attachments
MultipartBody.Builder builder = new MultipartBody.Builder();
builder.setType(MultipartBody.FORM);
builder.addFormDataPart("numOfFiles",String.valueOf(attachments.size()));
for (int i = 0; i < attachments.size();++i) {
EventAttachment attachment = attachments.get(i);
File file = new File(attachment.getPath());
String extension = MimeTypeMap.getFileExtensionFromUrl(attachment.getPath());
String type = null;
if (extension != null) {
type = MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension);
} else {
Log.e("x", "Could not get extensions of file " + file.getAbsolutePath() + ". File upload aborted.");
return;
}
ProgressRequestBody p = new ProgressRequestBody(RequestBody.create(MediaType.parse(type), file), new ProgressRequestBody.Listener() {
#Override
public void onProgress(int progress) {
}
});
builder.addFormDataPart("file" + i, file.getName(), p);
HashMap<String, String> hm = new HashMap<>();
hm.put("extension", extension.replace(".", ""));
hm.put("fileType", String.valueOf(attachment.getType()));
builder.addFormDataPart("fileMeta" + i, new JSONObject(hm).toString());
}
JSONObject jObj = new JSONObject();
builder.addFormDataPart("auth", putDefaultHeader(jObj).toString());
MultipartBody mb = builder.build();
okhttp3.Request request = new Request.Builder().url(EndPoint+UPLOAD_FILE).post(mb).build();
client.newCall(request).enqueue(new Callback() {
#Override
public void onFailure(Call call, IOException e) {
Log.e("x", "Error uploading file");
}
#Override
public void onResponse(Call call, Response response) throws IOException {
}
});
}
Thanks for your help.

Added
timeout: false,
parse: true
on the config payload and works for now.

Thread Executor not running properly inside loop

I am trying to implement multiple concurrent file uploads using IntentService for which I am using ThreadExecutor. I am iterating through a map and one-by-one and I am fetching the information and instantiating a workerthread. Inside the loop I have executor which is executing those threads. Below is my code.
#Override
protected void onHandleIntent(Intent intent) {
Debug.print("Called handleIntent");
mFileMap = (HashMap<String, Integer>) intent.getSerializableExtra("pdf_info");
mMapSize = mFileMap.size();
Log.e("pdf", mFileMap.toString());
mExecutor = Executors.newFixedThreadPool(2);
mTotalFileSize = getTotalFileSize();
Log.e("total_file_size", mTotalFileSize + "");
ArrayList<UploadWorker> uploadWorkerArrayList = new ArrayList<>();
Iterator it = mFileMap.entrySet().iterator();
for(Map.Entry<String, Integer> entry : mFileMap.entrySet())
{
String filePath = (String) entry.getKey();
Log.e("map_path: ", filePath);
int categoryId = (int) entry.getValue();
Log.e("map_no: ", categoryId + "");
// uploadWorkerArrayList.add(new UploadWorker(filePath, categoryId));
UploadWorker worker = new UploadWorker(filePath, categoryId);
mExecutor.submit(worker);
}
mExecutor.shutdown();
Log.e("workerlist: ", uploadWorkerArrayList.toString());
}
Below is my UploadWorker code.
class UploadWorker implements Runnable {
String filePath;
int categoryId;
public UploadWorker(String filePath, int categoryId) {
this.filePath = filePath;
this.categoryId = categoryId;
}
#Override
public synchronized void run() {
mFile = new File(filePath);
Log.e("uploadworker", filePath);
mParams = new LinkedHashMap<>();
mParams.put("file_type", 2 + "");
mParams.put("file_name", mFile.getName());
mParams.put("category_id", categoryId + "");
notificationId++;
mNotificationMap.put(filePath, notificationId);
createNotification(notificationId, mFile.getName());
int response = uploadPDF(filePath);
if (response == 1) {
JSONObject jObject = null;
String status = null;
try {
jObject = new JSONObject(mServerResponse);
status = jObject.getString("status");
int id = mNotificationMap.get(filePath);
updateNotification(100, id);
} catch (JSONException e) {
e.printStackTrace();
}
Log.e("<><>", status);
} else {
Log.e("<><>", "Upload fail");
}
}
}
However it happens that if say my maplist has 2 entries, then the iterator is first fetching both the entries and the executor is uploading only the final entry, that too twice. Please help.

Call an AsyncTask subclass of activity from another class?

I know this kind of questions are maybe too old, but I got stock with this silly thing.
I have an AsyncTask class which is a subclass of an activity class, and right now I want to call it from another class: following codes shows what I mean:
public class STA extends Activity {
public class ListSpdFiles extends AsyncTask<Void, Void, String[]> {
private static final String TAG = "ListSpdFiles: ";
/**
* Status code returned by the SPD on operation success.
*/
private static final int SUCCESS = 4;
private String initiator;
private String path;
private SecureApp pcas;
private boolean isConnected = false; // connected to PCAS service?
private PcasConnection pcasConnection = new PcasConnection() {
#Override
public void onPcasServiceConnected() {
Log.d(TAG, "pcasServiceConnected");
latch.countDown();
}
#Override
public void onPcasServiceDisconnected() {
Log.d(TAG, "pcasServiceDisconnected");
}
};
private CountDownLatch latch = new CountDownLatch(1);
public ListSpdFiles(String initiator, String path) {
this.initiator = initiator;
this.path = path;
}
private void init() {
Log.d(TAG, "starting task");
pcas = new AndroidNode(getApplicationContext(), pcasConnection);
isConnected = pcas.connect();
}
private void term() {
Log.d(TAG, "terminating task");
if (pcas != null) {
pcas.disconnect();
pcas = null;
isConnected = false;
}
}
#Override
protected void onPreExecute() {
super.onPreExecute();
init();
}
#Override
protected String[] doInBackground(Void... params) {
// check if connected to PCAS Service
if (!isConnected) {
Log.v(TAG, "not connected, terminating task");
return null;
}
// wait until connection with SPD is up
try {
if (!latch.await(20, TimeUnit.SECONDS)) {
Log.v(TAG, "unable to connected within allotted time, terminating task");
return null;
}
} catch (InterruptedException e) {
Log.v(TAG, "interrupted while waiting for connection in lsdir task");
return null;
}
// perform operation (this is where the actual operation is called)
try {
return lsdir();
} catch (DeadServiceException e) {
Log.i(TAG, "service boom", e);
return null;
} catch (DeadDeviceException e) {
Log.i(TAG, "device boom", e);
return null;
}
}
#Override
protected void onPostExecute(String[] listOfFiles) {
super.onPostExecute(listOfFiles);
if (listOfFiles == null) {
Log.i(TAG, "task concluded with null list of files");
// tv.setText("task concluded with a null list of files");
} else {
Log.i(TAG, "task concluded with the following list of files: "
+ Arrays.toString(listOfFiles));
//tv.setText("List of files received is:\n" + Arrays.toString(listOfFiles));
}
term();
}
#Override
protected void onCancelled(String[] listOfFiles) {
super.onCancelled(listOfFiles);
Log.i(TAG, "lsdir was canceled");
term();
}
/**
* Returns an array of strings containing the files available at the given path, or
* {#code null} on failure.
*/
private String[] lsdir() throws DeadDeviceException, DeadServiceException {
Result<List<String>> result = pcas.lsdir(initiator, path); // the lsdir call to the
final Global globalVariable = (Global) getApplicationContext();
if (globalVariable.getPasswordButt() == false) {
// Calling Application class (see application tag in AndroidManifest.xml)
// Get name and email from global/application context
final boolean isusername = globalVariable.getIsUsername();
if (isusername == true) {
String username = "/" + getLastAccessedBrowserPage() + ".username" + ".txt";
//String password = "/" + CurrentURL + "password" + ".txt";
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pcas.readFile(initiator, username, baos);
Log.i(TAG, "OutputStreampassword: "
+ new String(baos.toByteArray()));
String name = new String(baos.toByteArray());
if (!name.equalsIgnoreCase("")) {
globalVariable.setUsername(name);
// getCurrentInputConnection().setComposingText(name, 1);
// updateCandidates();
}
globalVariable.setIsUsername(false);
} else if (isusername == false)
Log.i(TAG, "Wrong Input Type For Username.");
// globalVariable.setUsernameButt(false);
} else if (globalVariable.getPasswordButt() == true) {
// Calling Application class (see application tag in AndroidManifest.xml)
// final Global globalVariable = (Global) getApplicationContext();
// Get name and email from global/application context
final boolean ispassword = globalVariable.getIsPassword();
if (ispassword == true) {
// String username = "/" + CurrentURL + "username" + ".txt";
String password = "/" + getLastAccessedBrowserPage() + ".password" + ".txt";
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pcas.readFile(initiator, password, baos);
Log.i(TAG, "OutputStreampassword: "
+ new String(baos.toByteArray()));
String name = new String(baos.toByteArray());
if (!name.equalsIgnoreCase("")) {
globalVariable.setPassword(name);
//getCurrentInputConnection().setComposingText(name, 1);
// updateCandidates();
}
globalVariable.setIsPassword(false);
} else if (ispassword == false)
Log.i(TAG, "Wrong Input Type For Password.");
globalVariable.setPasswordButt(false);
// boolpassword=false;
}
//}
if (result.getState() != SUCCESS) {
Log.v(TAG, "operation failed");
return null;
}
if (result.getValue() == null) {
Log.v(TAG, "operation succeeded but operation returned null list");
return null;
}
return result.getValue().toArray(new String[0]);
}
}
public String getLastAccessedBrowserPage() {
String Domain = null;
Cursor webLinksCursor = getContentResolver().query(Browser.BOOKMARKS_URI, Browser.HISTORY_PROJECTION, null, null, Browser.BookmarkColumns.DATE + " DESC");
int row_count = webLinksCursor.getCount();
int title_column_index = webLinksCursor.getColumnIndexOrThrow(Browser.BookmarkColumns.TITLE);
int url_column_index = webLinksCursor.getColumnIndexOrThrow(Browser.BookmarkColumns.URL);
if ((title_column_index > -1) && (url_column_index > -1) && (row_count > 0)) {
webLinksCursor.moveToFirst();
while (webLinksCursor.isAfterLast() == false) {
if (webLinksCursor.getInt(Browser.HISTORY_PROJECTION_BOOKMARK_INDEX) != 1) {
if (!webLinksCursor.isNull(url_column_index)) {
Log.i("History", "Last page browsed " + webLinksCursor.getString(url_column_index));
try {
Domain = getDomainName(webLinksCursor.getString(url_column_index));
Log.i("Domain", "Last page browsed " + Domain);
return Domain;
} catch (URISyntaxException e) {
e.printStackTrace();
}
break;
}
}
webLinksCursor.moveToNext();
}
}
webLinksCursor.close();
return null;
}
public String getDomainName(String url) throws URISyntaxException {
URI uri = new URI(url);
String domain = uri.getHost();
return domain.startsWith("www.") ? domain.substring(4) : domain;
}}
Would you please tell me what should I do to fix this code?

Looking over the code I did not see anywhere you referenced anything from the Activity itself besides the application context so you can move the ListSpdFiles class to its own java file and pass it a context into the constructor when you make a new instance of it.
Put this class in a ListSpdFiles.java file so it is no longer an inner class.
public class ListSpdFiles extends AsyncTask<Void, Void, String[]> {
Context applicationContext;
public ListSpdFiles(Context context, String initiator, String path) {
this.initiator = initiator;
this.path = path;
applicationContext = context.getApplicationContext();
}
// The rest of your code still goes here. Replace other calls to
// getApplicationContext() with the new applicationContext field
}
You can now use this class anywhere a Context is available. You create a new instance by doing:
ListSpdFiles listSpdFilesTask = new ListSpdFiles(context, "someInitiator", "somePath");
listSpdFilesTask.execute();

Android Pusher Invalid Signature - Client Side

Working with Pusher, issues with invalid signature following error:
Invalid signature: Expected HMAC SHA256 hex digest of socketID:channel, but got hash.
Here is my code:
public static PusherOptions pusherChannelAuthorise(){
Log.v(TAG, "pusherChannelAuthorise");
mMobileToken = Handler_Login.fetchmobile();
PUSHER_MOBILETOKEN_MAP.put(PUSHER_MOBILETOKEN_LABEL, mMobileToken);
HttpAuthorizer authoriser = new HttpAuthorizer(url);
authoriser.setHeaders(PUSHER_MOBILETOKEN_MAP);
options = new PusherOptions().setAuthorizer(authoriser);
return options;
}
public static void connect(){
Log.v(TAG, "connect" + "::CONNECTED::");
PusherOptions presenceChannelOptions = pusherChannelAuthorise();
presenceChannelConnected = true;
pusher = new Pusher(pusherHash, presenceChannelOptions);
pusher.connect(new ConnectionEventListener() {
#Override
public void onConnectionStateChange(ConnectionStateChange change) {
socketId = pusher.getConnection().getSocketId();
Log.v(TAG, "The socketId is: " + socketId);
}
#Override
public void onError(String message, String code, Exception e) {
Log.v(TAG, "There was a problem connecting!");
}
}, ConnectionState.CONNECTED);
presencechannel = pusher.subscribePresence(PUSHER_PUBLIC_CHANNEL_NAME, listener);
String myUuid = Handler_Login.getMyUuid();
privatechannel = pusher.subscribePrivate(DOCSYNC_CHANNEL + myUuid, docSyncListener);
privatechannel.bind("client-init", docSyncListener = new PrivateChannelEventListener() {
#Override
public void onEvent(String channelName, String eventName, String data) {
Log.v(TAG, "Private Test onEvent: " + channelName + " " + eventName + " " + data);
}
#Override
public void onSubscriptionSucceeded(String channelName) {
Log.v(TAG, "Private Channel onSubscriptionSucceeded: " + channelName);
}
#Override
public void onAuthenticationFailure(String message, Exception e) {
Log.v(TAG, "Private Channel onAuthenticationFailure: " + message + ":::" + e);
}
});
}
What am I doing wrong? Do I have to set additional parameters in the headers for the authoriser? #leggetter
Authentication to the presence channel works fine, but the private one fails.
EDIT:
Server Code (PHP, Laravel):
public function postMobilePusher(Request $request)
{
if (null !== $request->header('mobileToken')) {
$currentUser = User::where('mobileToken', '=', $request->header('mobileToken'))->first();
if (null !== $currentUser) {
define('APP_ID', ID);
define('APP_KEY', KEY);
define('APP_SECRET', SECRET);
$pusher = new Pusher(APP_KEY, APP_SECRET, APP_ID);
$socket_id = $request->get('socket_id');
$channel_name = $request->get('channel_name');
$user_id = $currentUser->id;
$user_info = [
'firstName' => $currentUser->firstName
];
return $auth = $pusher->presence_auth( $channel_name, $socket_id, $user_id, $user_info );
}
}
return abort(403, 'Unauthorized action.');
}

Update the server-side code to use socket_auth for private channels and presence_auth for presence channels:
public function postMobilePusher(Request $request)
{
if (null !== $request->header('mobileToken')) {
$currentUser = User::where('mobileToken', '=', $request->header('mobileToken'))->first();
if (null !== $currentUser) {
define('APP_ID', ID);
define('APP_KEY', KEY);
define('APP_SECRET', SECRET);
$pusher = new Pusher(APP_KEY, APP_SECRET, APP_ID);
$socket_id = $request->get('socket_id');
$channel_name = $request->get('channel_name');
$auth = null;
if(starts_with($channel_name, 'private-')) {
// TODO: check user has permission to access channel
$auth = $pusher->socket_auth($channel_name, $socket_id);
}
else {
// presence
// TODO: check user has permission to access channel
$user_id = $currentUser->id;
$user_info = [
'firstName' => $currentUser->firstName
];
$auth = $pusher->presence_auth( $channel_name, $socket_id, $user_id, $user_info );
}
}
}
return abort(403, 'Unauthorized action.');
}

android - format json string in adb logcat output

For logging purpose, I print out json response string and can see them in android "adb logcat" command. Is there a way to nicely format the json string in adb logcat output so that it looks like this?
{ "code" : "0",
"text" : "hello world"
}

You can use the JSONObject.toString() method to pretty print the JSON on logcat.
Log.d("tag", jsonObject.toString(4));
Output
(29124): {
(29124): "text": "hello world",
(29124): "code": "0"
(29124): }

I don't pretty print the JSON message in code. I just double click on the message in LogRabbit on Mac and it will pretty print it or highlight Base64 to decode it.
Full disclosure, I am the creator of LogRabbit for Mac.

You can format json in log with original form by use code of my logger:
Logger.dd("Section label", json);
Link on Github: https://github.com/scijoker/logger
Source:
import android.os.Build;
import android.util.Log;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
/**
* Created by scijoker on 08.10.15.
*/
public class Logger {
private static boolean DEBUG = true;
public static void d(String tag, String arg) {
if (isEnable()) {
log(tag, arg);
}
}
public static void d(String logMsg) {
if (isEnable()) {
log(getCurrentClassName(), getCurrentMethodName() + "(): " + logMsg);
}
}
public static void dd(String tag, Object source) {
if (isEnable()) {
Object o = getJsonObjFromStr(source);
if (o != null) {
try {
if (o instanceof JSONObject) {
format(tag, ((JSONObject) o).toString(2));
} else if (o instanceof JSONArray) {
format(tag, ((JSONArray) o).toString(2));
} else {
format(tag, source);
}
} catch (JSONException e) {
format(tag, source);
}
} else {
format(tag, source);
}
}
}
private static void log(String tag, String msg) {
Log.d(tag, msg);
}
private static String getSplitter(int length) {
StringBuilder builder = new StringBuilder();
for (int i = 0; i < length; i++) {
builder.append("-");
}
return builder.toString();
}
private static void format(String tag, Object source) {
tag = " " + tag + " ";
log(" ", " ");
log(" ", getSplitter(50) + tag + getSplitter(50));
log(" ", "" + source);
log(" ", getSplitter(100 + tag.length()));
log(" ", " ");
}
private static String getCurrentMethodName() {
return Thread.currentThread().getStackTrace()[4].getMethodName();
}
private static String getCurrentClassName() {
String className = Thread.currentThread().getStackTrace()[4].getClassName();
String[] temp = className.split("[\\.]");
className = temp[temp.length - 1];
return className;
}
private static Object getJsonObjFromStr(Object test) {
Object o = null;
try {
o = new JSONObject(test.toString());
} catch (JSONException ex) {
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) {
o = new JSONArray(test);
}
} catch (JSONException ex1) {
return null;
}
}
return o;
}
public static boolean isEnable() {
return DEBUG;
}
public static void setEnable(boolean flag) {
Logger.DEBUG = flag;
}
}

fastJson provide a method:
//serialize a prettyFormat json string
public static final String toJSONString(Object object, boolean prettyFormat);
eg:Log.d(TAG,JSON.toJSONString(object, true));