Develop Reference

How can i add custom header fields while uploading file into Amazon s3

I need to store multimedia files in Amazons3.
I used the following code for uploading a the file.
Method 1:
import android.content.Context;
import android.content.SharedPreferences;
import android.content.SharedPreferences.Editor;
import android.os.StrictMode;
import android.util.Log;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.ProgressEvent;
import com.amazonaws.services.s3.model.ProgressListener;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
public class AmazonUploader {
private static final long MIN_DEFAULT_PART_SIZE = 5 * 1024 * 1024;
private static final String TAG = "AmazonUploader";
private static final String PREFS_NAME = "preferences_simpl3r";
private static final String PREFS_UPLOAD_ID = "_uploadId";
private static final String PREFS_ETAGS = "_etags";
private static final String PREFS_ETAG_SEP = "~~";
private AmazonS3Client s3Client;
private String s3bucketName;
private String s3key;
private File file;
private SharedPreferences prefs;
private long partSize = MIN_DEFAULT_PART_SIZE;
private UploadProgressListener progressListener;
private long bytesUploaded = 0;
private boolean userInterrupted = false;
private boolean userAborted = false;
public AmazonUploader(Context context, AmazonS3Client s3Client, String s3bucketName, String s3key, File file) {
if (android.os.Build.VERSION.SDK_INT > 9) {
StrictMode.ThreadPolicy policy = new StrictMode.ThreadPolicy.Builder().permitAll().build();
StrictMode.setThreadPolicy(policy);
}
this.s3Client = s3Client;
this.s3key = s3key;
this.s3bucketName = s3bucketName;
this.file = file;
prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE);
}
/**
* Initiate a multipart file upload to Amazon S3
*
* #return the URL of a successfully uploaded file
*/
public String start() {
// initialize
List<PartETag> partETags = new ArrayList<PartETag>();
final long contentLength = file.length();
long filePosition = 0;
int startPartNumber = 1;
userInterrupted = false;
userAborted = false;
bytesUploaded = 0;
// check if we can resume an incomplete download
String uploadId = getCachedUploadId();
if (uploadId != null) {
// we can resume the download
Log.i(TAG, "resuming upload for " + uploadId);
// get the cached etags
List<PartETag> cachedEtags = getCachedPartEtags();
partETags.addAll(cachedEtags);
// calculate the start position for resume
startPartNumber = cachedEtags.size() + 1;
filePosition = (startPartNumber - 1) * partSize;
bytesUploaded = filePosition;
Log.i(TAG, "resuming at part " + startPartNumber + " position " + filePosition);
} else {
// initiate a new multi part upload
Log.i(TAG, "initiating new upload");
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(s3bucketName, s3key);
// ObjectMetadata obj = new ObjectMetadata();
// obj.setContentType("image/jpeg");
// obj.setHeader(Constants.APP_HEADER_REFERER, Constants.APP_REFERER_URL);
// initRequest.setObjectMetadata(obj);
configureInitiateRequest(initRequest);
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
uploadId = initResponse.getUploadId();
}
final AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(s3bucketName, s3key, uploadId);
for (int k = startPartNumber; filePosition < contentLength; k++) {
long thisPartSize = Math.min(partSize, (contentLength - filePosition));
Log.i(TAG, "starting file part " + k + " with size " + thisPartSize);
UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(s3bucketName)
.withKey(s3key).withUploadId(uploadId)
.withPartNumber(k).withFileOffset(filePosition).withFile(file)
.withPartSize(thisPartSize);
ProgressListener s3progressListener = new ProgressListener() {
public void progressChanged(ProgressEvent progressEvent) {
// bail out if user cancelled
// TODO calling shutdown too brute force?
if (userInterrupted) {
s3Client.shutdown();
throw new UploadIterruptedException("User interrupted");
} else if (userAborted) {
// aborted requests cannot be resumed, so clear any cached etags
clearProgressCache();
s3Client.abortMultipartUpload(abortRequest);
s3Client.shutdown();
}
bytesUploaded += progressEvent.getBytesTransfered();
//Log.d(TAG, "bytesUploaded=" + bytesUploaded);
// broadcast progress
float fpercent = ((bytesUploaded * 100) / contentLength);
int percent = Math.round(fpercent);
if (progressListener != null) {
progressListener.progressChanged(progressEvent, bytesUploaded, percent);
}
}
};
uploadRequest.setProgressListener(s3progressListener);
UploadPartResult result = s3Client.uploadPart(uploadRequest);
partETags.add(result.getPartETag());
// cache the part progress for this upload
if (k == 1) {
initProgressCache(uploadId);
}
// store part etag
cachePartEtag(result);
filePosition += thisPartSize;
}
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(
s3bucketName, s3key, uploadId,
partETags);
CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(compRequest);
bytesUploaded = 0;
Log.i(TAG, "upload complete for " + uploadId);
clearProgressCache();
return result.getLocation();
}
private String getCachedUploadId() {
return prefs.getString(s3key + PREFS_UPLOAD_ID, null);
}
private List<PartETag> getCachedPartEtags() {
List<PartETag> result = new ArrayList<PartETag>();
// get the cached etags
ArrayList<String> etags = SharedPreferencesUtils.getStringArrayPref(prefs, s3key + PREFS_ETAGS);
for (String etagString : etags) {
String partNum = etagString.substring(0, etagString.indexOf(PREFS_ETAG_SEP));
String partTag = etagString.substring(etagString.indexOf(PREFS_ETAG_SEP) + 2, etagString.length());
PartETag etag = new PartETag(Integer.parseInt(partNum), partTag);
result.add(etag);
}
return result;
}
private void cachePartEtag(UploadPartResult result) {
String serialEtag = result.getPartETag().getPartNumber() + PREFS_ETAG_SEP + result.getPartETag().getETag();
ArrayList<String> etags = SharedPreferencesUtils.getStringArrayPref(prefs, s3key + PREFS_ETAGS);
etags.add(serialEtag);
SharedPreferencesUtils.setStringArrayPref(prefs, s3key + PREFS_ETAGS, etags);
}
private void initProgressCache(String uploadId) {
// store uploadID
Editor edit = prefs.edit().putString(s3key + PREFS_UPLOAD_ID, uploadId);
AmazonSharedPreferencesCompact.apply(edit);
// create empty etag array
ArrayList<String> etags = new ArrayList<String>();
SharedPreferencesUtils.setStringArrayPref(prefs, s3key + PREFS_ETAGS, etags);
}
private void clearProgressCache() {
// clear the cached uploadId and etags
Editor edit = prefs.edit();
edit.remove(s3key + PREFS_UPLOAD_ID);
edit.remove(s3key + PREFS_ETAGS);
AmazonSharedPreferencesCompact.apply(edit);
}
public void interrupt() {
userInterrupted = true;
}
public void abort() {
userAborted = true;
}
/**
* Override to configure the multipart upload request.
* <p/>
* By default uploaded files are publicly readable.
*
* #param initRequest S3 request object for the file to be uploaded
*/
protected void configureInitiateRequest(InitiateMultipartUploadRequest initRequest) {
initRequest.setCannedACL(CannedAccessControlList.PublicRead);
ObjectMetadata obj = new ObjectMetadata();
obj.setContentType("image/jpeg");
obj.setHeader(Constants.APP_HEADER_REFERER, Constants.APP_REFERER_URL);
initRequest.withObjectMetadata(obj);
}
public void setPrefs(SharedPreferences prefs) {
this.prefs = prefs;
}
public long getPartSize() {
return partSize;
}
public void setPartSize(long partSize) {
if (partSize < MIN_DEFAULT_PART_SIZE) {
throw new IllegalStateException("Part size is less than S3 minimum of " + MIN_DEFAULT_PART_SIZE);
} else {
this.partSize = partSize;
}
}
public void setProgressListener(UploadProgressListener progressListener) {
this.progressListener = progressListener;
}
public interface UploadProgressListener {
public void progressChanged(ProgressEvent progressEvent, long bytesUploaded, int percentUploaded);
}
}
Method 2:
TransferObserver transferObserver = transferUtility.upload(
Constants.S3_BUCKET_NAME, /* The bucket to upload to */
fileName, /* The key for the uploaded object */
new File(imagePath), /* The file where the data to upload exists */
objectMetadata);
transferObserverListener(transferObserver);
in both method i got the following error
com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request...
Here i must pass customer header parameters, so i add like following
ObjectMetadata objectMetadata = new ObjectMetadata();
HashMap<String, String> mMetaMap = new HashMap<String, String>();
mMetaMap.put("content-type", "image/jpeg");
mMetaMap.put(Constants.APP_HEADER_REFERER, Constants.APP_REFERER_URL);
objectMetadata.setUserMetadata(mMetaMap);
But still i got the above error.
Is i'm passing the header parameters in correct way either i need to do changes. Kindly advise on this. Thanks

Filter HTML Code out of Website (cache it) and load it into WebView (RegEx)

I am relativly new to programming in java and android, so I wanted to ask you guys for a simple and understandable way of filtering two tables and their h3 headings of this website, possibly even cache it, and load it into a transparent WebView, so it doesnt look like a website. I thought of RegEx.. I do this to keep it up to date without having to service that thing.
With "simple and understandable" I mean comments, and possibly show what are just var names, method names or other custom names. And many explanations, comments and other things... Of course you can also just bomb the code in there, that would also work but I probably could not understand all of it.. ;)
Here's some code I tried:
package com.mrousavy.gemeindemuckendorfwipfing;
import android.os.AsyncTask;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by Marc on 15.10.2015.
*/
public class Table {
// found on stackoverflow
public static boolean exists2(String url) {
try {
URL u = new URL(url);
HttpURLConnection connection = (HttpURLConnection) u.openConnection();
connection.setRequestMethod("HEAD");
connection.connect();
return connection.getResponseCode() == HttpURLConnection.HTTP_OK;
} catch (Exception ex) {
return false;
}
}
/**
* must NOT be called in main thread!!!
*/
public static String getHTML2(String url) throws Exception {
try {
URL u = new URL(url);
BufferedReader in = new BufferedReader(new InputStreamReader(u.openStream()));
String tmp, html = "";
while ((tmp = in.readLine()) != null) {
html += tmp;
try {
Thread.sleep(10);
} catch (Exception e) {
}
}
return html;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* must NOT be called in main thread!!!
*/
public static List<String> getUrlsFromHTML2(String html) throws Exception {
List<String> urls = new ArrayList();
//init Patterns
Pattern divsPattern = Pattern.compile("<h3>.</table>");
//Pattern urlPattern = Pattern.compile("<a href=\"\\./files/(.*?)\"");
//search for right divs
Matcher divs = divsPattern.matcher(html);
while (divs.find()) {
//search for links
String innerDiv = divs.group(1);
Matcher url = urlPattern.matcher(innerDiv);
if (url.find()) {
if (!urls.contains(url.group(1)))
urls.add(url.group(1));
}
try {
Thread.sleep(10);
} catch (Exception e) {
}
}
return urls;
}
public static List<News> getNewsFromHTML(String html) {
List<News> ret = new ArrayList();
Pattern firstNewsPattern = Pattern.compile("<h3><strong>Aktuelle Meldungen</strong></h3>(.*?)<hr />");
Pattern newsPattern = Pattern.compile("<hr />(.*?)<hr />");
Pattern newsHeaderPattern = Pattern.compile("<h4>(.*?)</h4>");
Pattern hrefPattern = Pattern.compile("href=\"(.*?)\"");
Matcher newsHeader = null;
Matcher href = null;
Matcher firstNews = firstNewsPattern.matcher(html);
if(firstNews.find()) {
String content = firstNews.group(1).replace("./", "http://www.muckendorf-wipfing.at/");
href = hrefPattern.matcher(content);
while(href.find()) {
String url = href.group(1);
if(!url.contains("/")) {
content = content.replace("href=\"" + url + "\"", "href=\"" + "http://www.muckendorf-wipfing.at/" + url + "\"");
}
}
newsHeader = newsHeaderPattern.matcher(content);
if(newsHeader.find())
ret.add(new News(newsHeader.group(1).replaceAll("<(.*?)>", "").replaceAll("&#\\d{4};", ""), content));
}
Matcher news = newsPattern.matcher(html);
while(news.find()) {
String content = news.group(1).replace("./", "http://www.muckendorf-wipfing.at/");
href = hrefPattern.matcher(content);
while(href.find()) {
String url = href.group(1);
if(!url.contains("/")) {
content = content.replace("href=\"" + url + "\"", "href=\"" + "http://www.muckendorf-wipfing.at/" + url + "\"");
}
}
newsHeader = newsHeaderPattern.matcher(content);
if(newsHeader.find())
ret.add(new News(newsHeader.group(1).replaceAll("<(.*?)>", "").replaceAll("&#\\d{4};", ""), content));
}
return ret;
}
public static String listToString(List<String> list) {
String ret = "";
for(String str : list) {
ret += str + "§";
}
ret = ret.substring(0, ret.length()-1);
return ret;
}
public static List<String> stringToList(String str) {
String[] arr = str.split("§");
List <String> ret = new ArrayList();
for(String s : arr) {
if(!s.trim().equals(""))
ret.add(s);
}
return ret;
}
public static String extractContentFromHTML(String html) {
Pattern regex = Pattern.compile("<div id=\"content\">((.*?(<div.*?<\\/div>)*.*?)*)<\\/div>");
Pattern hrefPattern = Pattern.compile("href=\"(.*?)\"");
Matcher match = regex.matcher(html);
if(match.find()) {
String content = match.group(1).replace("./", "http://www.muckendorf-wipfing.at/");
Matcher href = hrefPattern.matcher(content);
while(href.find()) {
String url = href.group(1);
if(!url.contains("/")) {
content = content.replace("href=\"" + url + "\"", "href=\"" + "http://www.muckendorf-wipfing.at/" + url + "\"");
}
}
return content;
}
return "";
}
}
I hope someone can help me out! :)
Thank you! ^^

Don't use regex to parse html/xml, it's error prone. Try using specialized lib as the excellent Jsoup:
import org.jsoup.Jsoup;
import org.jsoup.Connection;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
[...]
final String url = "http://www.muckendorf-wipfing.at/25-0-Wirtschaft+und+Gastronomie.html";
String tablesHtml = parseHTML(url);
[...]
String parseHTML(String url) {
//Retrieve html of {url} via GET
Connection.Response response = Jsoup.connect(url).method(Connection.Method.GET).execute();
//Parse html
Document doc = response.parse();
//Select the div with id="content", where both tables are stored
Element contentDiv = doc.select("div#content").first();
//return the inner html of <div id="content"> selected above
return contentDiv.html();
}
The syntax of the select function can be found here
UPDATE: i've updated the code to parse the content of div too, creating a Table class that store <h3> text and table as 'html' and also as a bidimensional String array. It has a nice toString() method too useful to see what you get.
NB: The trick is in the jsoup select statement "h3:contains(" + h3Text + ") ~ table": it selects the tables after and h3 tag with h3Text (the title of the table) inside. Later we're taking only the first table of the list so we can be sure we're selecting the table coupled with the h3 title.
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
//[...]
/* //CASE 1: if you have to download the html
* String url = "http://www.muckendorf-wipfing.at/25-0-Wirtschaft+und+Gastronomie.html";
* //Retrieve html of "url" via GET
* Connection.Response response = Jsoup.connect(url).method(Connection.Method.GET).execute();
* //Parse html
*Document doc = response.parse();
*/
//CASE 2: If you already have the html in a String called htmlString
Document doc = Jsoup.parse(htmlString);
//Select the div with id="content", where both tables are stored
Element contentDiv = doc.select("div#content").first();
//Create a list for the data
List tables = new ArrayList<Table>();
//Loop on h3 titles and get the coupled table below
for ( Element h3 : contentDiv.select("h3") )
{
//get the text inside <h3> tag
String h3Text = h3.text();
//jsoup select statement to get the table
//immediately after the <h3></h3>
String select = "h3:contains(" + h3Text + ") ~ table";
//Actually get the jsoup table element jTable
Element jTable = contentDiv.select(select).first();
//Load the data on the list
tables.add(new Table(h3Text,jTable));
}
//print them
for ( Table t : tables )
System.out.println(t);
//[...]
class Table
{
String h3Title;
String htmlTable;
String[][] splittedTable;
Table(String h3Title, Element jTable)
{
this.h3Title = h3Title;
this.htmlTable = jTable.html();
this.splittedTable = getSplittedTable(jTable);
}
String[][] getSplittedTable(Element jTable)
{
//Get all the rows of the jTable
Elements trs = jTable.select("tr");
//Get the number of rows
int rows = trs.size();
//Get the columns of the first row (the same of all the rows)
int columns = trs.first().select("td").size();
//Allocate new bidimensional array table
String[][] table = new String[rows][columns];
int i = 0; int j = 0;
for ( Element tr : trs ) {
for ( Element td : tr.select("td") ) {
table[i][j++] = td.text();
}
j = 0; //reset column cursor
i++; //increment row cursor
}
return table;
}
#Override
String toString() {
StringBuilder sb = new StringBuilder();
String ln = System.lineSeparator();
sb.append(h3Title + ln);
sb.append("--" + ln);
sb.append(this.htmlTable + ln);
sb.append("--" + ln);
for (int i = 0; i < splittedTable.length; i++) {
for (int j = 0; j < splittedTable[i].length; j++) {
sb.append(splittedTable[i][j] + " | ")
}
sb.append(ln + "--" + ln);
}
return sb.toString();
}
}

Poor Performance of Android app

I had written a crawler program in C# which used to crawl on a given url or url postfixed with page number and download all the image files from it.It worked fine.
Now I am a newbie in android programming and thought to write the same thing for my android device so that I can also use it on my phone.
The algorithm I followed was...
1) Take the base url,start page no(in case the url is postfixed by page number in query string),end page no,and location to store the images on sdcard.
2) If end page no is less than start page no(means if i only want to crawl a single page) pass it to getHtml method. or else make a loop from start to end page and pass each url to getHtml method.
3) In getHtml method I download web page source and broke it in pieces to find link to image files in it.
4) for each image url found, download the image file to the given save location.
The algo seems easy but when I made the whole program I had some pretty big performance issues. It is so so heavy that while running it in emulator all i could see was gc clearing objects in logcat.Another very common issue is that the UI hangs.But as the program is only for personal use i can do with it because I doesnt know multithreading in android. But atleast the program should be fast. Is there any way I can decrease the number of objects or destroy them myself.
Is there anything I can do to improve this. I wonder how those GBs worth of games works flawlessly and my 20KB app is so so slow.
The whole code is.
package com.dwnldr;
import java.io.*;
import java.net.*;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import android.app.Activity;
import android.os.*;
import android.util.Log;
import android.view.View;
import android.widget.*;
public class IMGDwnldrActivity extends Activity {
EditText res;
String str;
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.main);
Button btn = (Button) findViewById(R.id.button1);
File SDCardRoot = Environment.getExternalStorageDirectory();
EditText saveat = (EditText) findViewById(R.id.editText4);
saveat.setText(SDCardRoot.getAbsolutePath());
btn.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View v) {
try {
//Getting URL,Start pageno,End pageno and the save location on sdcard
EditText baseurl = (EditText) findViewById(R.id.editText1);
String url = baseurl.getText().toString();
EditText startpage = (EditText) findViewById(R.id.editText2);
int start = Integer.parseInt(startpage.getText().toString());
EditText endpage = (EditText) findViewById(R.id.editText3);
int end = Integer.parseInt(endpage.getText().toString());
EditText saveat = (EditText) findViewById(R.id.editText4);
String save = saveat.getText().toString();
if (start <= end) {
for (int i = start; i <= end; i++) {
str = "\n--------------------";
str += "\nPage No" + String.valueOf(i);
writemsg(str);
getHtml(url + String.valueOf(i), save);
}
} else
getHtml(url, save);
writemsg("Done");
} catch (Exception ee) {
writemsg("\nException fired::" + ee.getMessage());
}
}
});
}
//method to get the source of a particular url
public void getHtml(String url, String save) throws ClientProtocolException, IOException {
try {
HttpClient httpClient = new DefaultHttpClient();
HttpContext localContext = new BasicHttpContext();
HttpGet httpGet = new HttpGet(url);
HttpResponse response = httpClient.execute(httpGet, localContext);
String result = "";
str = "\nDownloading Page....";
writemsg(str);
BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().
getContent()));
String line = null;
while ((line = reader.readLine()) != null) {
result += line + "\n";
}
str = "\nPage Downloaded...";
writemsg(str);
String[] pieces;
if (result.contains(".jpg") || result.contains(".jpeg")) {
pieces = result.split("\"");
Log.d("Events", String.valueOf(pieces.length));
for (int i = 0; i < pieces.length; i++) {
if (pieces[i].contains(".jpg") || pieces[i].contains(".jpeg")) {
if (pieces[i].contains("http")) {
Log.d("Events", pieces[i]);
downloadme(pieces[i], save);
} else {
URL u = new URL(url);
if (pieces[i].startsWith("."));
pieces[i] = pieces[i].substring(pieces[i].indexOf("/"), pieces[i].length());
writemsg(u.getProtocol() + "://" + u.getHost() + pieces[i]);
if (pieces[i].startsWith("/"))
downloadme(u.getProtocol() + "://" + u.getHost() + pieces[i], save);
else
downloadme(u.getProtocol() + "://" + u.getHost() + "/" + pieces[i], save);
}
}
}
}
} catch (Exception ee) {
writemsg("\nException fired::" + ee.getMessage());
}
}
//download each image url given
private void downloadme(String url1, String save) {
try {
str = "\nDownloading Image " + url1;
writemsg(str);
URL url = new URL(url1);
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("GET");
urlConnection.setDoOutput(true);
urlConnection.connect();
File f = new File(save);
if (f.isDirectory() && !f.exists())
f.mkdirs();
String fileName = url1.substring(url1.lastIndexOf('/') + 1, url1.length());
File file = new File(save, fileName);
FileOutputStream fileOutput = new FileOutputStream(file);
InputStream inputStream = urlConnection.getInputStream();
int totalSize = urlConnection.getContentLength();
str = "\nImage Size " + String.valueOf(totalSize / 1024);
writemsg(str);
byte[] buffer = new byte[1024];
int bufferLength = 0; //used to store a temporary size of the buffer
while ((bufferLength = inputStream.read(buffer)) > 0) {
fileOutput.write(buffer, 0, bufferLength);
}
fileOutput.close();
str = "\nDownloaded Image " + url1;
writemsg(str);
catch some possible
errors // ...
} catch (MalformedURLException e) {
writemsg("\nException fired::" + e.getMessage());
} catch (IOException e) {
writemsg("\nException fired::" + e.getMessage());
} catch (Exception ee) {
writemsg("\nException fired::" + ee.getMessage());
}
}
//write certain text to the Result textbox
private void writemsg(String msg) {
res = (EditText) findViewById(R.id.result);
String str = res.getText().toString();
str += msg;
res.setText(str);
}
}

Android: ProgressBar for HttpURLConnection String data

I am getting JSON data from a web service and would like to display a progress bar while the data is downloading. All the examples I have seen use a StringBuilder like so:
//Set up the initial connection
HttpURLConnection connection = (HttpURLConnection)url.openConnection();
connection.setRequestMethod("GET");
connection.setDoOutput(true);
connection.setReadTimeout(10000);
connection.connect();
InputStream stream = connection.getInputStream();
//read the result from the server
reader = new BufferedReader(new InputStreamReader(stream));
StringBuilder builder = new StringBuilder();
String line = "";
while ((line = reader.readLine()) != null) {
builder.append(line + '\n');
}
result = builder.toString();
I got the ProgressBar to work by downloading the data as a byte array, then converting the byte array to a String, but I'm wondering if there is a 'more correct' way to do this. Since I've found no other way of doing this, the following class can also serve as a working example, seems a bit of a hack, but it does work well.
package com.royaldigit.newsreader.services;
import android.os.AsyncTask;
import android.util.Log;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.royaldigit.newsreader.controller.commands.CommandInterface;
import com.royaldigit.newsreader.model.data.SearchResultDO;
import com.royaldigit.newsreader.model.data.SearchTermDO;
/**
* Gets news results from Feedzilla based on the search term currently stored in model.searchTermDO
*
* Sends progress update and returns results to the CommandInterface command reference:
* * command.onProgressUpdate(progress);
* * command.serviceComplete(results);
*
*
*/
public class FeedzillaSearchService {
private static final String TAG = "FeedzillaSearchService";
private static final String SERVICE_URI = "http://api.feedzilla.com/v1/categories/26/articles/search.json?q=";
private static final int STREAM_DIVISIONS = 10;
private CommandInterface command;
private SearchTermDO currentSearchTermDO;
private Integer maximumResults;
private DownloadTask task;
private ArrayList<SearchResultDO> results;
public Boolean isCanceled = false;
public void getData(CommandInterface cmd, SearchTermDO termDO, Integer maxResults){
command = cmd;
currentSearchTermDO = termDO;
//Feedzilla only allows count to be 100 or less, anything over throws an error
maximumResults = (maxResults > 100)? 100 : maxResults;
results = new ArrayList<SearchResultDO>();
task = new DownloadTask();
task.execute();
}
public void cancel() {
isCanceled = true;
if(task != null) task.cancel(true);
}
/**
* Handle GET request
*
*/
private class DownloadTask extends AsyncTask<Void, Integer, String> {
#Override
protected String doInBackground(Void...voids) {
String result = "";
if(currentSearchTermDO == null || currentSearchTermDO.term.equals("")) return result;
BufferedReader reader = null;
publishProgress(0);
try {
String path = SERVICE_URI + URLEncoder.encode(currentSearchTermDO.term, "UTF-8") + "&count=" + maximumResults;
Log.d(TAG, "path = "+path);
URL url = new URL(path);
//Set up the initial connection
HttpURLConnection connection = (HttpURLConnection)url.openConnection();
connection.setRequestMethod("GET");
connection.setDoOutput(true);
connection.setReadTimeout(10000);
connection.connect();
int length = connection.getContentLength();
InputStream stream = connection.getInputStream();
byte[] data = new byte[length];
int bufferSize = (int) Math.ceil(length / STREAM_DIVISIONS);
int progress = 0;
for(int i = 1; i < STREAM_DIVISIONS; i++){
int read = stream.read(data, progress, bufferSize);
progress += read;
publishProgress(i);
}
stream.read(data, progress, length - progress);
publishProgress(STREAM_DIVISIONS);
result = new String(data);
} catch (Exception e) {
Log.e(TAG, "Exception "+e.toString());
} finally {
if(reader != null){
try {
reader.close();
} catch(IOException ioe) {
ioe.printStackTrace();
}
}
}
return result;
}
protected void onProgressUpdate(Integer... progress) {
int currentProgress = progress[0] * 100/STREAM_DIVISIONS;
if(!this.isCancelled()) command.onProgressUpdate(currentProgress);
}
#Override
protected void onPostExecute(String result){
if(!this.isCancelled()) downloadTaskComplete(result);
}
}
/**
*
* #param data
*/
private void downloadTaskComplete(Object data){
if(!isCanceled){
try {
Log.d(TAG, data.toString());
JSONObject obj = new JSONObject(data.toString());
JSONArray array = obj.getJSONArray("articles");
for(int i = 0; i < array.length(); i++){
SearchResultDO dataObj = new SearchResultDO();
dataObj.title = array.getJSONObject(i).getString("title");
dataObj.url = array.getJSONObject(i).getString("url");
dataObj.snippet = array.getJSONObject(i).getString("summary");
dataObj.source = array.getJSONObject(i).getString("source");
dataObj.date = array.getJSONObject(i).getString("publish_date");
dataObj.termId = currentSearchTermDO.id;
//Reformat date
SimpleDateFormat format1 = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z");
try {
Date date = format1.parse(dataObj.date);
SimpleDateFormat format2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
dataObj.date = format2.format(date);
} catch(ParseException pe) {
Log.e(TAG, pe.getMessage());
}
results.add(dataObj);
}
command.serviceComplete(results);
} catch(JSONException e){
Log.e(TAG, e.toString());
command.serviceComplete(results);
}
}
}
}
UPDATE: Here is the finished version of the class using the suggestions from Nikolay. I ended up using the StringBuilder after all. The previous version would break because some times connection.getContentLength() returns -1. This version degrades gracefully for that case. Tested this implementation quite a bit and it seems bulletproof.
package com.royaldigit.newsreader.services;
import android.os.AsyncTask;
import android.util.Log;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.royaldigit.newsreader.controller.commands.CommandInterface;
import com.royaldigit.newsreader.model.data.SearchResultDO;
import com.royaldigit.newsreader.model.data.SearchTermDO;
/**
* Gets news results from Feedzilla based on the search term currently stored in model.searchTermDO
*
* Sends progress update and returns results to the CommandInterface command reference:
* * command.onProgressUpdate(progress);
* * command.serviceComplete(results);
*
*/
public class FeedzillaSearchService implements SearchServiceInterface {
private static final String TAG = "FeedzillaSearchService";
private static final String SERVICE_URI = "http://api.feedzilla.com/v1/categories/26/articles/search.json?q=";
private CommandInterface command;
private SearchTermDO currentSearchTermDO;
private Integer maximumResults;
private DownloadTask task;
private ArrayList<SearchResultDO> results;
private Boolean isCanceled = false;
public void getData(CommandInterface cmd, SearchTermDO termDO, Integer maxResults){
command = cmd;
currentSearchTermDO = termDO;
//Feedzilla only allows count to be 100 or less, anything over throws an error
maximumResults = (maxResults > 100)? 100 : maxResults;
results = new ArrayList<SearchResultDO>();
task = new DownloadTask();
task.execute();
}
public void cancel() {
isCanceled = true;
if(task != null) task.cancel(true);
}
/**
* Handle GET request
*
*/
private class DownloadTask extends AsyncTask<Void, Integer, String> {
#Override
protected String doInBackground(Void...voids) {
String result = "";
if(currentSearchTermDO == null || currentSearchTermDO.term.equals("")) return result;
BufferedReader reader = null;
publishProgress(0);
try {
String path = SERVICE_URI + URLEncoder.encode(currentSearchTermDO.term, "UTF-8") + "&count=" + maximumResults;
Log.d(TAG, "path = "+path);
URL url = new URL(path);
//Set up the initial connection
HttpURLConnection connection = (HttpURLConnection)url.openConnection();
connection.setRequestMethod("GET");
connection.setDoOutput(true);
connection.setReadTimeout(20000);
connection.connect();
//connection.getContentType() should return something like "application/json; charset=utf-8"
String[] values = connection.getContentType().toString().split(";");
String charset = "";
for (String value : values) {
value = value.trim();
if (value.toLowerCase().startsWith("charset=")) {
charset = value.substring("charset=".length());
break;
}
}
//Set default value if charset not set
if(charset.equals("")) charset = "utf-8";
int contentLength = connection.getContentLength();
InputStream stream = connection.getInputStream();
reader = new BufferedReader(new InputStreamReader(stream));
StringBuilder builder = new StringBuilder();
/**
* connection.getContentLength() can return -1 on some connections.
* If we have the content length calculate progress, else just set progress to 100 and build the string all at once.
*
*/
if(contentLength>-1){
//Odd byte array sizes don't always work, tried 512, 1024, 2048; 1024 is the magic number because it seems to work best.
byte[] data = new byte[1024];
int totalRead = 0;
int bytesRead = 0;
while ((bytesRead = stream.read(data)) > 0) {
try {
builder.append(new String(data, 0, bytesRead, charset));
} catch (UnsupportedEncodingException e) {
Log.e(TAG, "Invalid charset: " + e.getMessage());
//Append without charset (uses system's default charset)
builder.append(new String(data, 0, bytesRead));
}
totalRead += bytesRead;
int progress = (int) (totalRead * (100/(double) contentLength));
//Log.d(TAG, "length = " + contentLength + " bytesRead = " + bytesRead + " totalRead = " + totalRead + " progress = " + progress);
publishProgress(progress);
}
} else {
String line = "";
while ((line = reader.readLine()) != null) {
builder.append(line + '\n');
publishProgress(100);
}
}
result = builder.toString();
} catch (Exception e) {
Log.e(TAG, "Exception "+e.toString());
} finally {
if(reader != null){
try {
reader.close();
} catch(IOException ioe) {
ioe.printStackTrace();
}
}
}
return result;
}
protected void onProgressUpdate(Integer... progress) {
if(!this.isCancelled()) command.onProgressUpdate(progress[0]);
}
#Override
protected void onPostExecute(String result){
if(!this.isCancelled()) downloadTaskComplete(result);
}
}
/**
*
* #param data
*/
private void downloadTaskComplete(Object data){
if(!isCanceled){
try {
Log.d(TAG, data.toString());
JSONObject obj = new JSONObject(data.toString());
JSONArray array = obj.getJSONArray("articles");
for(int i = 0; i < array.length(); i++){
SearchResultDO dataObj = new SearchResultDO();
dataObj.title = array.getJSONObject(i).getString("title");
dataObj.url = array.getJSONObject(i).getString("url");
dataObj.snippet = array.getJSONObject(i).getString("summary");
dataObj.source = array.getJSONObject(i).getString("source");
dataObj.date = array.getJSONObject(i).getString("publish_date");
dataObj.termId = currentSearchTermDO.id;
//Reformat date
SimpleDateFormat format1 = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z");
try {
Date date = format1.parse(dataObj.date);
SimpleDateFormat format2 = new SimpleDateFormat(SearchResultDO.DATE_FORMAT_STRING);
dataObj.date = format2.format(date);
} catch(ParseException pe) {
Log.e(TAG, pe.getMessage());
}
results.add(dataObj);
}
} catch(JSONException e){
Log.e(TAG, e.toString());
}
command.serviceComplete(results);
}
}
}

Well, since content length is reported in bytes, there is really no other way. If you want to use a StringReader you could take the length of each line you read and calculate the total bytes read to achieve the same thing. Also, the regular idiom is to check the return value of read() to check if you have reached the end of the stream. If, for some reason, the content length is wrong, your code may read more/less data then available. Finally, when converting a byte blob to a string, you should explicitly specify the encoding. When dealing with HTTP, you can get that from the 'charset' parameter of the 'Content-Type' header.

I had similar problem. I tried the solution of Jeremy C, but it was inaccurate, because value of "Content-Length" from header can be very different than real data.
My solution is:
Send HTTP header from server (PHP):
$string = json_encode($data, JSON_PRETTY_PRINT | JSON_FORCE_OBJECT);
header("X-Size: ".strlen($string)); //for example with name: "X-Size"
print($string);
Read correct value "X-size" for contentLength variable from HTTP header before read from stream:
protected String doInBackground(URL... urls) {
if (General.DEBUG) Log.i(TAG, "WebAsyncTask(doInBackground)");
String result = "";
BufferedReader reader = null;
try {
HttpURLConnection conn = (HttpURLConnection) urls[0].openConnection();
conn.setConnectTimeout(General.TIMEOUT_CONNECTION);
conn.setReadTimeout(General.TIMEOUT_SOCKET);
conn.setRequestMethod("GET");
conn.connect();
if (General.DEBUG) Log.i(TAG, "X-Size: "+conn.getHeaderField("X-Size"));
if (General.DEBUG) Log.i(TAG, "getHeaderField: "+conn.getHeaderFields());
if(conn.getResponseCode() != General.HTTP_STATUS_200)
return General.ERR_HTTP;
int contentLength = -1;
try {
contentLength = Integer.parseInt(conn.getHeaderField("X-Size"));
} catch (Exception e){
e.printStackTrace();
}
InputStream stream = conn.getInputStream();
reader = new BufferedReader(new InputStreamReader(stream));
StringBuilder builder = new StringBuilder();
//Pokud delku zname:
if(contentLength > -1){
byte[] data = new byte[16]; //TODO
int totalRead = 0;
int bytesRead = 0;
while ((bytesRead = stream.read(data)) > 0){
Thread.sleep(100); //DEBUG TODO
try {
builder.append(new String(data, 0, bytesRead, "UTF-8"));
} catch (UnsupportedEncodingException e) {
Log.i(TAG, "Invalid charset: " + e.getMessage());
//Append without charset (uses system's default charset)
builder.append(new String(data, 0, bytesRead));
}
totalRead += bytesRead;
int progress = (int) (totalRead * (100/(double) contentLength));
Log.i(TAG, "length = " + contentLength + " bytesRead = " + bytesRead + " totalRead = " + totalRead + " progress = " + progress);
publishProgress(progress);
}
} else {
String line = "";
while ((line = reader.readLine()) != null) {
builder.append(line + '\n');
publishProgress(100);
}
}
result = builder.toString();
} catch (SocketException | SocketTimeoutException e){
if (General.DEBUG) Log.i(TAG, "SocketException or SocketTimeoutException");
e.printStackTrace();
return General.HTTP_TIMEOUT;
} catch (Exception e){
e.printStackTrace();
return General.ERR_HTTP;
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}

Using a progress Bar while I load a webpage in android

I'm currently loading a webpage using a thread and I need a progress bar to display while the page loads up. Whats the best way to implement this? My code to load the webpage is this
private Thread checkUpdate = new Thread() {
#Override
public void run() {
try {
/**
* establish a URL connection
*/
URL updateURL = new URL("http://www.mtsu.edu/alertupdates/");
URLConnection conn = updateURL.openConnection();
/**
* create an Input stream and buffered array to
* prepare for parsing.
*/
InputStream is = conn.getInputStream();
BufferedInputStream bis = new BufferedInputStream(is);
ByteArrayBuffer baf = new ByteArrayBuffer(50);
/**
* read in the html and parse it to bytes.
*/
int current = 0;
while((current = bis.read()) != -1){
baf.append((byte)current);
}
/**
* Convert the Bytes read to a String.
*/
html = new String(baf.toByteArray());
int position = html.indexOf("<h1>");
int position2 = html.indexOf("<!--",position);
html = html.substring(position, position2);
mHandler.post(showUpdate);
} catch (Exception e){}
}
};
Here is my attempt at Using AsyncTask and the entire code of my project.
package com.MTSUAndroid;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Calendar;
import org.apache.http.util.ByteArrayBuffer;
import android.app.Activity;
import android.app.ProgressDialog;
import android.graphics.Color;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.Handler;
import android.widget.Button;
import android.widget.ProgressBar;
import android.widget.TextView;
import android.widget.Toast;
import android.text.Html;
import android.view.Gravity;
import android.view.View;
import java.lang.String;
import android.content.Context;
public class Alerts extends Activity {
/**
* private variables to hold the html strings for parsing.
*/
private String html = "";
private Handler mHandler;
private TextView text1;
private TextView timestamp;
private Button home;
private Button refresh;
private ProgressDialog myProgress;
private int myProgressStatus = 0;
private Handler myHandler = new Handler();
/**
* overriding on create with a new handler for threading
*/
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.alerts);
initialControls();
//mHandler = new Handler();
//checkUpdate.start();
}
public void connectivityMessage(String msg){
Context context = getApplicationContext();
Toast toast = Toast.makeText(context, "", Toast.LENGTH_LONG);
toast.setGravity(Gravity.CENTER, 0, 0);
toast.setText(msg);
toast.show();
}
/**
* InitialControls function to set up all the initial controls for the GUI
* Such as buttons, etc...
*/
private void initialControls(){
text1 = (TextView)findViewById(R.id.textView1);
home = (Button)findViewById(R.id.home_button);
//myProgress = (ProgressBar)findViewById(R.id.progressBar1);
home.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View view) {
finish();
}
});
/**
* TimeStamp for the alerts refresh button
*/
timestamp = (TextView)findViewById(R.id.timestamp);
refresh = (Button)findViewById(R.id.update);
/**
* implementing the refresh button/loading the website
*/
refresh.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View v) {
try {
/**
* establish a URL connection
*/
URL updateURL = new URL("http://www.mtsu.edu/alertupdates/");
URLConnection conn = updateURL.openConnection();
/**
* create an Input stream and buffered array to
* prepare for parsing.
*/
InputStream is = conn.getInputStream();
BufferedInputStream bis = new BufferedInputStream(is);
ByteArrayBuffer baf = new ByteArrayBuffer(50);
/**
* read in the html and parse it to bytes.
*/
int current = 0;
while((current = bis.read()) != -1){
baf.append((byte)current);
}
/**
* Convert the Bytes read to a String.
*/
html = new String(baf.toByteArray());
int position = html.indexOf("<h1>");
int position2 = html.indexOf("<!--",position);
html = html.substring(position, position2);
mHandler.post(showUpdate);
/**
* using calendar class for the refresh button
*/
Calendar c = Calendar.getInstance();
int seconds = c.get(Calendar.SECOND);
int minutes = c.get(Calendar.MINUTE);
int hours = c.get(Calendar.HOUR);
int years = c.get(Calendar.YEAR);
int months = 1 + c.get(Calendar.MONTH);
int days = c.get(Calendar.DAY_OF_MONTH);
try{
if (c.get(Calendar.AM_PM) == 0)
{
String AM = "";
AM = "AM";
if (hours == 0)
{
hours = 12;
}
timestamp.setText("Refreshed on " + days + "-"
+ months + "-" + years + " " + hours + ":" + minutes + ":" + seconds + " " + AM);
timestamp.setTextSize(17f);
timestamp.setTextColor(Color.GREEN);
}
else
{
String PM = "";
PM = "PM";
timestamp.setText("Refreshed on " + days + "-"
+ months + "-" + years + " " + hours + ":" + minutes + ":" + seconds + " " + PM);
timestamp.setTextSize(17f);
timestamp.setTextColor(Color.GREEN);
}
}
catch (Exception e){}
}
/**
* Catch exception E to catch all errors.
*/
catch (Exception e) {}
}
}
);}
/**
* creating a new thread to run the URL.
*/
private Thread checkUpdate = new Thread() {
#Override
public void run() {
try {
/**
* establish a URL connection
*/
URL updateURL = new URL("http://www.mtsu.edu/alertupdates/");
URLConnection conn = updateURL.openConnection();
/**
* create an Input stream and buffered array to
* prepare for parsing.
*/
InputStream is = conn.getInputStream();
BufferedInputStream bis = new BufferedInputStream(is);
ByteArrayBuffer baf = new ByteArrayBuffer(50);
/**
* read in the html and parse it to bytes.
*/
int current = 0;
while((current = bis.read()) != -1){
baf.append((byte)current);
}
/**
* Convert the Bytes read to a String.
*/
html = new String(baf.toByteArray());
int position = html.indexOf("<h1>");
int position2 = html.indexOf("<!--",position);
html = html.substring(position, position2);
mHandler.post(showUpdate);
} catch (Exception e){}
}
};
/**
* set the textView to the freshly parsed html for viewing
*/
private Runnable showUpdate = new Runnable(){
#Override
public void run(){
text1.setText(Html.fromHtml(html));
}
};
public class myTask extends AsyncTask<Void,Void,Void>{
ProgressDialog progress;
public myTask(ProgressDialog progress) {
this.progress = progress;
}
#Override
protected void onPreExecute() {
progress = ProgressDialog.show(Alerts.this, "Loading data..", "Please Wait");
super.onPreExecute();
}
#Override
protected Void doInBackground(Void... params) {
try {
/**
* establish a URL connection
*/
URL updateURL = new URL("http://www.mtsu.edu/alertupdates/");
URLConnection conn = updateURL.openConnection();
/**
* create an Input stream and buffered array to
* prepare for parsing.
*/
InputStream is = conn.getInputStream();
BufferedInputStream bis = new BufferedInputStream(is);
ByteArrayBuffer baf = new ByteArrayBuffer(50);
/**
* read in the html and parse it to bytes.
*/
int current = 0;
while((current = bis.read()) != -1){
baf.append((byte)current);
}
/**
* Convert the Bytes read to a String.
*/
html = new String(baf.toByteArray());
int position = html.indexOf("<h1>");
int position2 = html.indexOf("<!--",position);
html = html.substring(position, position2);
mHandler.post(showUpdate);
} catch (Exception e){}
return null;
}
protected void onPostExecute() {
progress.dismiss();
}
}
}

An AsyncTask will help handle progress for you. Read here: http://developer.android.com/reference/android/os/AsyncTask.html