Jsoup with big XML - android

I'm using Jsoup on my Android application, to read xml file get by a webservice restful.
The Jsoup library works perfectly until the xml file contains few number of records.
But when I get a xml with 50k or 60k of records, I observed that the Jsoup allocate memory until 230MB 240MB. This is a problem because with
android:largeHeap="true"
I have 256MB of memory allocable.
This is a saple code, try it yourself
public class MainActivity extends ActionBarActivity {
private static Handler mHandler = new Handler();
private static Context context;
static TextView textView;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
textView = (TextView) findViewById(R.id.textView);
textView.setText("5 seconds to start task");
context = this.getApplicationContext();
mHandler.postDelayed(new Runnable() {
public void run() {
new syncDataWS(context).execute();
}
}, 5000);
}
#Override
public boolean onCreateOptionsMenu(Menu menu) {
getMenuInflater().inflate(R.menu.menu_main, menu);
return true;
}
#Override
public boolean onOptionsItemSelected(MenuItem item) {
int id = item.getItemId();
if (id == R.id.action_settings) {
return true;
}
return super.onOptionsItemSelected(item);
}
private static class syncDataWS extends AsyncTask<Void, String, Void> {
Context mContext;
public syncDataWS(Context context) {
this.mContext = context;
}
#Override
public void onPreExecute() {
super.onPreExecute();
textView.append("\nStart task");
}
#Override
protected Void doInBackground(Void... params) {
try {
publishProgress("Start call XML page");
Document WS_Document = Jsoup.connect("XML_EXAMPLE").maxBodySize(0).timeout(10 * 100000).get();
publishProgress("End call XML page");
publishProgress("Get rows of document");
Elements XML_RESULT_WS = WS_Document.select("row");
publishProgress("Record number : " + Integer.toString(XML_RESULT_WS.size()));
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
#Override
protected void onProgressUpdate(String... values) {
super.onProgressUpdate(values);
if (values != null && values.length > 0) {
textView.append("\n" + values[0]);
}
}
#Override
protected void onPostExecute(Void result) {
textView.append("\nEnd task");
}
}
}
and this is a example of xml
<XMLDataResponse xmlns="www.example.net"><XMLDataResult><root xmlns=""><row ID="1" ID2="2" ID3="3" ID4="4" F1="0.000000000000000e+000" F2="0.000000000000000e+000" F3="0.000000000000000e+000" F4="" F5="0.000000000000000e+000" F6="0.000000000000000e+000"/></root></XMLDataResult></XMLDataResponse>
Take the row tag and copy + paste it until you have 60 thousand records in your XML EXAMPLE. Put it wherever you want, provided it can be achievable with via http call (URL). Copy the url in the code
Jsoup.connect("COPY URL OF XML HERE")
And you can see what I mean.
I need a solution to solve this allocation issue, because sometimes, not always, the allocation arrives at 256MB and my app crashes.

You won't be able to solve this problem with Jsoup because Jsoup creates a complete DOM tree from the parsed XML which grows bigger and bigger inside your memory.
Jsoup btw. is meant as an HTML parser in the first place.
I'd use an event based XML parser like XMLPullParser.

Related

Dynamic android form from XML

I want to generate a form into my activity_main.xml ScrollView. XML is loaded and parsed correctly but when I'm trying to addView(LinearLayout) then it throws exception e. My application gets url of a XML file via push notification and then parses it. According to XML it then should generate a form and display it to the user. I used this as an example: https://www.ibm.com/developerworks/xml/tutorials/x-andddyntut/#l1
Here is my main activity:
public class MainActivity extends Activity {
// label to display gcm messages
TextView lblMessage;
Controller aController;
public ScrollView sv;
Button execute;
// Asyntask
AsyncTask<Void, Void, Void> mRegisterTask;
public static String name;
public static String email;
final Context context = this;
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
ScrollView sv = (ScrollView) findViewById(R.id.sv);
...
}
// Create a broadcast receiver to get message and show on screen
private final BroadcastReceiver mHandleMessageReceiver = new BroadcastReceiver() {
#Override
public void onReceive(Context context, Intent intent) {
String newMessage = intent.getExtras().getString(Config.EXTRA_MESSAGE);
// Waking up mobile if it is sleeping
aController.acquireWakeLock(getApplicationContext());
ScrollView sv = (ScrollView) findViewById(R.id.sv);
new DoInBackground(getApplicationContext(), sv).execute(newMessage);
// Releasing wake lock
aController.releaseWakeLock();
}
};
and here is my async class:
public class DoInBackground extends AsyncTask<String, Void, Void> {
Context mContext;
ScrollView mSv;
String tag = "DynamicFormXML";
XmlGuiForm theForm;
ProgressDialog progressDialog;
Handler progressHandler;
public DoInBackground(Context context, ScrollView sv) {
this.mContext = context;
this.mSv = sv;
}
#Override
protected void onPreExecute() {
super.onPreExecute();
}
#Override
protected Void doInBackground(String... params) {
if (GetFormData(params[0])) {
DisplayForm();
}
else
{
Log.e(tag,"Couldn't parse the Form.");
AlertDialog.Builder bd = new AlertDialog.Builder(mContext);
AlertDialog ad = bd.create();
ad.setTitle("Error");
ad.setMessage("Could not parse the Form data");
ad.show();
}
return null;
}
protected void onPostExecute() {
}
private boolean DisplayForm()
{
try
{
final LinearLayout ll = new LinearLayout(mContext);
mSv.addView(ll); //Here it fails
ll.setOrientation(android.widget.LinearLayout.VERTICAL);
...
} catch (Exception e) { // Goes to here
Log.e(tag,"Error Displaying Form");
return false;
}
}
I think the context of the main activity and also the empty Scrollview in main activity are forwarded correctly (they are not null) but i'm not 100% sure. Any help/hints are appreciated! :)
You can not touch the GUI from a background thread (e.g. the one running the doInBackground method).
In an AsynTask, you can put the UI code in onPostExecute, wich is invoked on the UI thread with the result of doInBackground.
If you have intermediate results you can call publishProgress from doInBackground, this will trigger the invocation of onProgressUpdate on the UI thread, where you can update the UI.
See AsyncTask API for an example and more details on what must be done on which thread.
Solution
Reordering code (so GUI stuff would be done onPostExecute) worked. Also i had a problem with not getting to onPostExecute() but i had to change it to onPostExecute(Void result).
Now my code looks like this and works like a charm:
public class DoInBackground extends AsyncTask<String, Void, Void> {
Context mContext;
LinearLayout mLl;
String tag = "DynamicFormXML";
XmlGuiForm theForm;
ProgressDialog progressDialog;
Handler progressHandler;
public DoInBackground(Context context, LinearLayout ll) {
this.mContext = context;
this.mLl = ll;
}
#Override
protected void onPreExecute() {
super.onPreExecute();
}
#Override
protected Void doInBackground(String... params) {
getFormData(params[0]);
return null;
}
protected void onPostExecute(Void result) {
DisplayForm();
}
I also added ScrollView and LinearLayout to my activity_main.xml so DisplayForm() looks like that (if you want to follow the example i mentioned before ):
private void DisplayForm() {
try
{
// walk thru our form elements and dynamically create them, leveraging our mini library of tools.
int i;
for (i=0;i<theForm.fields.size();i++) {
if (theForm.fields.elementAt(i).getType().equals("text")) {
theForm.fields.elementAt(i).obj = new XmlGuiEditBox(mContext,(theForm.fields.elementAt(i).isRequired() ? "*" : "") + theForm.fields.elementAt(i).getLabel(),"");
mLl.addView((View) theForm.fields.elementAt(i).obj);
}
...

Simple Jsoup code for Android

I recently started developing programs in Android, and have a small problem with this simple code. I am trying to parse the title of a website and stored in a string, but so far unsuccessful. Is it because I am not doing it in Async ? Or can it be a different issues all together?
public class MainActivity extends Activity implements OnClickListener {
private Button btnSearch;
private EditText editTextCarReg;
protected void onCreate(Bundle savedInstanceState)
{
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
//-------------------------------------------------
btnSearch = (Button) findViewById(R.id.btnSearch);
btnSearch.setOnClickListener(this);
//------------------------------------------------
editTextCarReg = (EditText) findViewById(R.id.editTextRegistration);
editTextCarReg.setOnClickListener(this);
}
public boolean onCreateOptionsMenu(Menu menu)
{
getMenuInflater().inflate(R.menu.main, menu);
return true;
}
public void example() throws IOException
{
Document doc = Jsoup.connect("http://http://9gag.com/").get();
String title = doc.title();
}
public void onClick(View v)
{
if(v.getId() == btnSearch.getId())
{
try {
example();
} catch (IOException e)
{
e.printStackTrace();
}
}
}
}
From your logs:
android.os.NetworkOnMainThreadException
This exception is thrown when an application attempts to perform a networking operation on its main thread.
Run Jsoup logic in AsyncTask.
(be sure that you added <uses-permission android:name="android.permission.INTERNET"/> as well)
You can write something like:
class JsoupTask extends AsyncTask<String, Void, Void> {
private Exception exception;
protected void doInBackground(String... url) {
Document doc = Jsoup.connect(url).get();
String title = doc.title();
...
}
protected void onPostExecute(RSSFeed feed) {
// here you can update your UI thread through Handler, for example
}
}
And call, like:
new JsoupTask().execute(url);
public static int SDK_INT = android.os.Build.VERSION.SDK_INT;
write this code before your request
if (SDK_INT >= 10) {
ThreadPolicy tp = ThreadPolicy.LAX;
StrictMode.setThreadPolicy(tp);
}
Using asynctask is better practice.

Android - progressdialog not displaying in AsyncTask

I have an android app that I am having trouble with.
Basically the ProgressDialog is not showing at all. I believe this to be a threading issue of some sort but I don't know how to fix it.
I am using ActionBarSherlock with some Fragments. I am also using the new Android DrawerLayout where I have my options on the drawer, which replace a fragment when clicked.
On first load of my app, I want to check the database to see if the inital data has been downloaded. If not, then I go off and begin an AsyncTask to download the data. This SHOULD have a ProgressDialog display during this, but it doesnt.
Can someone see where I am going wrong? Thanks.
MainScreen - The default landing page/fragment when the app opens
public class MainScreen extends SherlockFragment {
public static final String TAG = "MainScreen";
#Override
public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
View rootView = inflater.inflate(R.layout.activity_main, container, false);
setHasOptionsMenu(false);
ImageView imgLogo = (ImageView) rootView.findViewById(R.id.imgMainScreen);
imgLogo.setOnClickListener(new ButtonHandler(getActivity()));
checkDatabase();
return rootView;
}
private void checkDatabase() {
//Ensure there is data in the database
DBHelper db = new DBHelper(this.getSherlockActivity());
db.checkDatabase();
}
...
}
DBHelper.checkDatabase() - The method that initiates the download
public void checkDatabase() {
if (isEmpty()) {
//Connect to net and download data
NetworkManager nm = new NetworkManager(activity);
if (!nm.downloadData()) {
Toast.makeText(activity, R.string.internetCheck, Toast.LENGTH_SHORT).show();
}
}
}
and finally
NetworkManager.downloadData() - The method that kicks off the AsyncTask:
public boolean downloadData() {
try {
return new HttpConnection(activity).execute().get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
return false;
}
public class HttpConnection extends AsyncTask<Void, Void, Boolean> {
private ProgressDialog progressDialog;
private Activity m_activity;
protected HttpConnection(Activity activity) {
m_activity = activity;
}
#Override
protected void onPreExecute() {
progressDialog = new ProgressDialog(m_activity);
progressDialog.setMessage("Wait ...");
progressDialog.setCancelable(false);
progressDialog.setMax(100);
progressDialog.setProgressStyle(ProgressDialog.STYLE_SPINNER);
progressDialog.show();
super.onPreExecute();
}
#Override
protected Boolean doInBackground(Void... params) {
String[] types = new String[]{"type1", "type2", "type3", "type4", };
StringBuilder sb = new StringBuilder();
for(String type : types) {
sb = new StringBuilder();
if(DBHelper.TYPE4_TABLE.equals(type)) {
InputStream is = activity.getResources().openRawResource(R.raw.dbdata);
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
try {
sb.append(reader.readLine());
} catch (IOException e) {
Toast.makeText(activity.getApplicationContext(), "Error retriveving data", Toast.LENGTH_SHORT).show();
Log.e(Constants.TAG, "Error reading data");
e.printStackTrace();
}
} else {
sb = fetchURLData(Constants.ALL_URL+type);
}
cleanDataAndStore(sb, type);
}
return true;
}
#Override
protected void onPostExecute(Boolean result){
progressDialog.hide();
}
}
Using the above code, all I get is a white screen as the app tries to load, and sometimes an ANR. When the download is done, the fragment loads. So it works fine except for the missing ProgressDialog.
PS, Notice I'm setting the activity in each constructor.
Thanks.
Remove .get() from return new HttpConnection(activity).execute().get(); You are basically locking your UI thread. Once removed it should work as AsyncTasks are expected to work.
The purpose is to be Asynchronous so boolean downloadData() should have a return type of void. If you need to do something with the data then you should implement an interface "listener" and pass it to the AsyncTask.
Example Listener:
class TaskConnect extends AsyncTask<Void, Void, ConnectionResponse> {
private final AsyncTaskListener mListener;
/**
*
*/
public TaskConnect(AsyncTaskListener listener) {
...
mListener = listener;
}
#Override
protected void onPreExecute() {
if (mListener != null) {
mListener.onPreExecute(mId);
}
}
#Override
protected ConnectionResponse doInBackground(Void... cData) {
...
return responseData;
}
#Override
protected void onPostExecute(ConnectionResponse response) {
if (mListener != null) {
mListener.onComplete(response);
} else {
LOG.w("No AsyncTaskListener!", new Throwable());
}
}
}
public interface AsyncTaskListener {
public abstract void onPreExecute(int id);
public abstract void onComplete(ConnectionResponse response);
}
My issue was not the common issue of others where they were calling get() method after execute() method. My issue was the Context I was passing to my AsyncTask method. I have a settingsActivity and I have a ReadMeActivity that calls the asynctask task. Instead of using the context in which is was being called (ReadMeActivity.this) I used the settingsActivity which prevented it from being seen. Once I switched it and passed it the context in which the activity was being called it worked.
Hope it helps someone else.

AsyncTask as Inner class and static field issue

I have a method searchPlace() that updates a static Arrays of custom Place Object in a class A (FindItOnMap) with a google map, and a method updateMap() that updates the various geopoints .
I invoke these methods Button.onClick and all works properly.
Since these methods use internet data this operation could take a while, I have been looking for the implementation of an inner class B(YourCustomAsyncTask) inside the class A that extends AsyncTask to show a waiting dialog during the processing of these two methods
An user suggested a solution in this form (that apparently seems valid):
public class FindItOnMap extends MapActivity {
static Place[] foundResults;
private ProgressDialog dialog;
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.ricerca_condominio);
mapView = (MapView)findViewById(R.id.mapView);
...........
((ImageButton) findViewById(R.id.btSearch)).setOnClickListener(mSearchListenerListener);
}
OnClickListener mSearchListener = new OnClickListener() {
public void onClick(View v) {
String location=editorLocation.getText().toString();
String name=editorName.getText().toString();
//Call the AsyncTask here
new YourCustomAsyncTask().execute(new String[] {name, location});
}
};
private class YourCustomAsyncTask extends AsyncTask <String, Void, Void> {
#Override
protected void onPreExecute() {
dialog = new ProgressDialog(Main.this);
dialog.setMessage("Loading....");
dialog.setIndeterminate(true);
dialog.setCancelable(true);
dialog.show(); //Maybe you should call it in ruinOnUIThread in doInBackGround as suggested from a previous answer
}
#Override
protected Void doInBackground(String... strings) {
try {
search(strings[0], string[1]);
return null;
} catch(Exception e) {
}
}
#Override
protected void onPostExecute(Void params) {
updateMapWithResult();
dialog.dismiss();
//result
}
.....
}
The waiting dialog is showed and the methods are invoked in background,
However for some strange reason the static list foundResults results filled with various null items...
How is this possible?
If I invoke the method search(location, name) outside the inner class all works properly and updateMapWithResult(); updates all geopoint, so these two methods are ok. Only if I try to invoke this in the inner class the json calls seem to be working but the static variable foundResults is filled with null elements and the program doesn't work properly.
Any suggestion?
I have understand where is the problem.
You have to run the search method on the UI thread.
So change this code block:
#Override
protected Void doInBackground(String... strings) {
try {
search(strings[0], string[1]);
return null;
} catch(Exception e) {
}
}
with this
#Override
protected Void doInBackground(final String... strings) {
try {
runOnUiThread(new Runnable() {
public void run() {
search(strings[0], string[1]);
return null;
}
});
} catch(Exception e) {
e.printStackTrace();
}
}
And all should works correctly.
Here is one problem:
OnClickListener mSearchListener = new OnClickListener() {
public void onClick(View v) {
String Location=editorLocation.getText().toString();
String name=editorName.getText().toString();
//Call the AsyncTask here
new YourCustomAsyncTask().execute(new String[] {name, location});
}
Your Location should be location.
Also here:
#Override
protected Void doInBackground(String... strings) {
try {
search(strings[0], string[1]);
} catch(Exception e) {
}
}
#Override
protected void onPostExecute(Void params) {
updateMapWithResult();
dialog.dismiss();
//result
}
In doInBackground you don't assign a value after you search. You might try this:
#Override
protected Void doInBackground(String... strings) {
try {
search(strings[0], string[1]);
String name = string[0];
String location = string[1]
} catch(Exception e) {
}
}
Or something else that will assign value while it runs. As it is, it appears that you just search, and then nothing else.
The reason foundResults is null is because you don't ever assign it a value.
There is nothing wrong with your AsyncTask. Please include the search() method.

AsyncTask with ProgressDialog vs orientation change

after few desperate days I have finally created almost working example.
The goal: in onCreate I want to download and parseXML files in AsyncTask, show progress dialog, update UI and close the dialog.
The problem: When orientation changes Activity is restarted and AsyncTask loses reference to it. There are a lot of questions and blogs about it. But I cant find out why this particular solution doesnt work. Or how android handles dialogs in this case.
The state: When I start an app everything is ok. I can rotate the device and I can manually start the task again via menu. BUT after task finishes and I change the orientation again dialogs pops up (as expected) and nothing else happens. No progress change, no dialog dismiss. AsyncTask finishes normally.
The code:
package com.test;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.util.*;
public class Test extends TabActivity {
DownloadFileAsync task;
ProgressDialog progressDialog;
static final int PROGRESS_DIALOG = 0;
private static Data data;
/** Called when the activity is first created. */
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.main);
/* - Run from different locations bug - */
//http://code.google.com/p/android/issues/detail?id=2373
if (!isTaskRoot()) {
final Intent intent = getIntent();
final String intentAction = intent.getAction();
if (intent.hasCategory(Intent.CATEGORY_LAUNCHER) &&
intentAction != null && intentAction.equals(Intent.ACTION_MAIN)) {
finish();
}
}
/* - /Run from different locations bug */
/* -------------- Tabs ---------------- */
Resources res = getResources();
TabHost mTabHost = getTabHost();
mTabHost.addTab(mTabHost.newTabSpec("overview").setIndicator("MYTAB1",res.getDrawable(R.drawable.ic_tab_home)).setContent(R.id.tab1));
mTabHost.setCurrentTab(0);
/* -------------- /Tabs --------------- */
/* -------------- /Data --------------- */
task = (DownloadFileAsync)getLastNonConfigurationInstance();
if(task!= null) {
task.setActivity(this);
} else {
if(data == null) {
File datafile = this.getFileStreamPath("data.dat");
if(datafile.exists()){
//Log.d("log", "File exists!");
try {
long time = System.currentTimeMillis();
ObjectInputStream obj = new ObjectInputStream(new FileInputStream(datafile));
data = (Data)obj.readObject();
obj.close();
Log.d("time", "loaded in:"+(System.currentTimeMillis()- time));
if(data.isUpToDate() || !isOnline()){
update();
}
} catch (Exception e) {
e.printStackTrace();
datafile.delete();
data = null;
}
//Log.d("log", "Passed?");
}
}
/* DEBUG if(data == null || !data.isUpToDate())*/ this.synchronize();
}
/* -------------- /Data --------------- */
}
#Override
public boolean onCreateOptionsMenu(Menu menu) {
menu.add("Synchronize").setIcon(R.drawable.ic_menu_refresh);
return true;
}
#Override
public boolean onOptionsItemSelected(MenuItem item) {
synchronize();
return super.onOptionsItemSelected(item);
}
#Override
public Object onRetainNonConfigurationInstance() {
if(task != null) task.setActivity(null);
return(task);
}
protected Dialog onCreateDialog(int id) {
switch (id) {
case PROGRESS_DIALOG:
progressDialog = new ProgressDialog(this);
progressDialog.setMessage("Aktualizuji ...");
progressDialog.setProgressStyle(ProgressDialog.STYLE_HORIZONTAL);
progressDialog.setCancelable(false);
//progressDialog.show();
return progressDialog;
default:
return null;
}
}
public void update() {
}
private void onTaskCompleted() {
task = null;
dismissDialog(PROGRESS_DIALOG);
Log.d("tok","Task.onComplete");
update();
}
public void synchronize(){
if(isOnline()) {
showDialog(PROGRESS_DIALOG);
progressDialog.setProgress(0); // <-- this is the last time progressDialog updates
task = new DownloadFileAsync(this);
task.execute();
}
}
public boolean isOnline() {
ConnectivityManager cm =
(ConnectivityManager) getSystemService(WaspActivity.CONNECTIVITY_SERVICE);
NetworkInfo netInfo = cm.getActiveNetworkInfo();
if (netInfo != null && netInfo.isConnectedOrConnecting()) {
return true;
}
return false;
}
private static class DownloadFileAsync extends AsyncTask<String, String, String> {
private Data tempData;
private Test activity;
private int progress = 0;
private File metafile;
private File tempDir;
private FileOutputStream fos;
public DownloadFileAsync(Test activity) {
this.setActivity(activity);
... some more init ...
}
public void setActivity(Test activity) {
this.activity = activity;
}
#Override
protected void onPreExecute() {
super.onPreExecute();
tempData = new Data();
}
#Override
protected String doInBackground(String... aurl) {
try {
... some heavy load ...
//this.progress = someValue;
} catch (Exception e) {
Log.d("Error", "Error while processing files. Code:"+e.getMessage());
e.printStackTrace();
}
//Log.d("time","Task "+(System.currentTimeMillis() - time));
return null;
}
protected void onProgressUpdate(String... progress) {
if(activity != null) activity.progressDialog.setProgress(this.progress);
}
#Override
protected void onPostExecute(String unused) {
data = tempData;
tempData = null;
if(activity != null) {
activity.onTaskCompleted();
activity = null;
}
}
}
}
Yesterday, I wrote a blog post which describes handling configuration changes using retained Fragments.
The TL;DR is to use host your AsyncTask inside a Fragment, call setRetainInstance(true) on the Fragment, and report the AsyncTask's progress/results back to it's Activity through the retained Fragment.
add this to Test activity in manifest
android:configChanges="orientation|keyboardHidden"></activity>
and put this in your test class
#Override
public void onConfigurationChanged(Configuration newConfig) {
super.onConfigurationChanged(newConfig);
}
Put android:screenOrientation="portrait" in your <activity /> tag of manifest file.
After doing that, put these lines in protected void onPreExecute() method of asyncTask:
dialog.setMessage("Please wait.....");
dialog.setIndeterminate(true);
dialog.setCancelable(false);
dialog.show();

Categories

Resources