Develop Reference

Agora Raw Audio on Android

My end goal is to visualize the remote audio in a one on one call using the Agora Api. The agora api and available examples are quite vast but I did not find a example that allows me to access the audio as it is streamed in so that I can then get the samples max amplitude and send to a visualizer.The byte array would do just fine.
I have looked through the examples provide at https://github.com/AgoraIO/API-Examples, which seemed promising but I have not been able to solve this. Any help is appreciated.
(Within the API_Example on Github, I have attempted to implement ProcessRawData and AudioRecordService)
Update: The APIExample allows me to grab the raw data as it flows through and that is what I am looking for. The issue arises when I try to duplicate the "ProcessRawData" class in a new project. The call back for the audio observer is never called. I have gone through my code and it matches everything in the example. The only thing I can think of is that the method to import the "lib-raw-data" folder was incorrect. I simply copied the entrire folder 'lib-raw-data' from the example api project and into my own. I then added the library directory to the gradle.settings file as well as the gradle(app) file. Outisde of that, I simply made sure the code matches the example provided.
Below is the most basic form of my application with the imported library "lib-raw-data" as described. I have no errors within Android Studio so I don't know where to look. The example in the github above works, but the same code below does not.
MainActivity.java
public class MainActivity extends AppCompatActivity {
private SessionVideoCall videoCall;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
initVideoCall();
}
private void initVideoCall(){
videoCall = new SessionVideoCall(this);
videoCall.setChannelName(#CHANNEL_NAME);
videoCall.attachView();
videoCall.startCall();
}
}
SessionVideoCall.java
public class SessionVideoCall implements MediaDataAudioObserver {
private Handler handler;
private final String TAG = this.getClass().getSimpleName();
private static final int PERMISSION_REQ_ID = 44;
private static final String[] REQUESTED_PERMISSIONS = {
Manifest.permission.RECORD_AUDIO,
Manifest.permission.CAMERA,
android.Manifest.permission.READ_EXTERNAL_STORAGE,
Manifest.permission.WRITE_EXTERNAL_STORAGE
};
private final Activity ACTIVITTY;
private MediaDataObserverPlugin mediaDataObserverPlugin;
private String channelName;
private FrameLayout mLocalContainer;
private FrameLayout mRemoteContainer;
private VideoCanvas mLocalVideo;
private VideoCanvas mRemoteVideo;
private Timer timer;
boolean isVisualizerAttached = true;
public static RtcEngine engine;
// set up engine
public SessionVideoCall(Activity activity){
this.ACTIVITTY = activity;
handler = new Handler(Looper.getMainLooper());
}
// set channel name
public void setChannelName(String channelName){
this.channelName = channelName;
}
public void attachView(){
mLocalContainer = ACTIVITTY.findViewById(R.id.local_video_view_container);
mRemoteContainer = ACTIVITTY.findViewById(R.id.remote_video_view_container);
}
// start call
public void startCall(){
if(hasPermissions())
initEngineAndJoinChannel();
}
// end call
public void endCall(){
removeFromParent(mLocalVideo);
mLocalVideo = null;
removeFromParent(mRemoteVideo);
mRemoteVideo = null;
if (mediaDataObserverPlugin != null) {
mediaDataObserverPlugin.removeAudioObserver(this);
mediaDataObserverPlugin.removeAllBuffer();
}
MediaPreProcessing.releasePoint();
leaveChannel();
}
private void initEngineAndJoinChannel() {
initializeEngine();
setupObserver();
setupAudioConfig();
setupVideoConfig();
setupLocalVideo();
joinChannel();
}
private void initializeEngine() {
try {
engine = RtcEngine.create(ACTIVITTY.getApplicationContext(), ACTIVITTY.getString(R.string.agora_app_id), mRtcEventHandler);
} catch (Exception e) {
Log.e(TAG, Log.getStackTraceString(e));
throw new RuntimeException("NEED TO check rtc sdk init fatal error\n" + Log.getStackTraceString(e));
}
}
private void setupObserver(){
mediaDataObserverPlugin = MediaDataObserverPlugin.the();
MediaPreProcessing.setCallback(mediaDataObserverPlugin);
MediaPreProcessing.setAudioPlayByteBuffer(mediaDataObserverPlugin.byteBufferAudioPlay);
mediaDataObserverPlugin.addAudioObserver(this);
}
private void setupAudioConfig(){
engine.setChannelProfile(Constants.CHANNEL_PROFILE_LIVE_BROADCASTING);
engine.setClientRole(IRtcEngineEventHandler.ClientRole.CLIENT_ROLE_BROADCASTER);
engine.setDefaultAudioRoutetoSpeakerphone(false);
engine.setEnableSpeakerphone(false);
engine.setPlaybackAudioFrameParameters(4000, 1, RAW_AUDIO_FRAME_OP_MODE_READ_ONLY, 1024);
}
private void setupVideoConfig() {
engine.enableVideo();
engine.setVideoEncoderConfiguration(new VideoEncoderConfiguration(
VideoEncoderConfiguration.VD_640x360,
VideoEncoderConfiguration.FRAME_RATE.FRAME_RATE_FPS_15,
VideoEncoderConfiguration.STANDARD_BITRATE,
VideoEncoderConfiguration.ORIENTATION_MODE.ORIENTATION_MODE_FIXED_PORTRAIT));
}
private void setupLocalVideo() {
SurfaceView view = RtcEngine.CreateRendererView(ACTIVITTY);
view.setZOrderMediaOverlay(true);
mLocalContainer.addView(view);
mLocalVideo = new VideoCanvas(view, VideoCanvas.RENDER_MODE_HIDDEN, 0);
engine.setupLocalVideo(mLocalVideo);
}
private void joinChannel() {
String token = ACTIVITTY.getString(R.string.agora_access_token);
if (TextUtils.isEmpty(token) || TextUtils.equals(token, "#YOUR ACCESS TOKEN#")) {
token = null; // default, no token
}
engine.joinChannel(token, channelName, "Extra Optional Data", 0);
}
private void leaveChannel(){
if (mediaDataObserverPlugin != null) {
mediaDataObserverPlugin.removeAudioObserver(this);
mediaDataObserverPlugin.removeAllBuffer();
}
MediaPreProcessing.releasePoint();
engine.leaveChannel();
if(timer != null)
timer.cancel();
}
private void removeFromParent(VideoCanvas canvas) {
if (canvas != null) {
ViewParent parent = canvas.view.getParent();
if (parent != null) {
ViewGroup group = (ViewGroup) parent;
group.removeView(canvas.view);
//return group;
}
}
//return null;
}
private void setupRemoteVideo(int uid) {
ViewGroup parent = mRemoteContainer;
if (parent.indexOfChild(mLocalVideo.view) > -1) {
parent = mLocalContainer;
}
if (mRemoteVideo != null) {
return;
}
SurfaceView view = RtcEngine.CreateRendererView(ACTIVITTY);
view.setZOrderMediaOverlay(parent == mLocalContainer);
parent.addView(view);
mRemoteVideo = new VideoCanvas(view, VideoCanvas.RENDER_MODE_HIDDEN, uid);
// Initializes the video view of a remote user.
engine.setupRemoteVideo(mRemoteVideo);
}
private final IRtcEngineEventHandler mRtcEventHandler = new IRtcEngineEventHandler() {
#Override
public void onJoinChannelSuccess(String channel, int uid, int elapsed) {
super.onJoinChannelSuccess(channel, uid, elapsed);
setupTimer();
Log.d(TAG,"onJoinChannelSuccess: ");
}
#Override
public void onFirstRemoteVideoDecoded(final int uid, int width, int height, int elapsed) {
ACTIVITTY.runOnUiThread(new Runnable() {
#Override
public void run() {
Log.d(TAG,"First remote video decoded, uid: " + (uid & 0xFFFFFFFFL));
setupRemoteVideo(uid);
}
});
}
#Override
public void onUserOffline(final int uid, int reason) {
super.onUserOffline(uid, reason);
// when remote user logs off
}
#Override
public void onUserJoined(int uid, int elapsed) {
super.onUserJoined(uid, elapsed);
Log.i(TAG, "onUserJoined->" + uid);
Log.d(TAG, "user has joined call: " + uid);
handler.post(() ->
{
if (mediaDataObserverPlugin != null) {
mediaDataObserverPlugin.addDecodeBuffer(uid);
}
});
}
#Override
public void onRemoteAudioStateChanged(int uid, int state, int reason, int elapsed)
{
super.onRemoteAudioStateChanged(uid, state, reason, elapsed);
Log.i(TAG, "onRemoteAudioStateChanged->" + uid + ", state->" + state + ", reason->" + reason);
}
};
private boolean hasPermissions(){
return (checkSelfPermission(REQUESTED_PERMISSIONS[0]) &&
checkSelfPermission(REQUESTED_PERMISSIONS[1]) &&
checkSelfPermission(REQUESTED_PERMISSIONS[2]) &&
checkSelfPermission(REQUESTED_PERMISSIONS[3]));
}
private boolean checkSelfPermission(String permission) {
if (ContextCompat.checkSelfPermission(ACTIVITTY, permission) !=
PackageManager.PERMISSION_GRANTED) {
ActivityCompat.requestPermissions(ACTIVITTY, REQUESTED_PERMISSIONS, PERMISSION_REQ_ID);
return false;
}
return true;
}
private void setupTimer(){
timer = new Timer();
timer.schedule(new TimerTask() {
#Override
public void run() {
if(maxAmplitude > 0)
Log.e(TAG, "Amplitude Greater than 0: " + maxAmplitude);
}
},0,50);
}
#Override
public void onRecordAudioFrame(byte[] data, int audioFrameType, int samples, int bytesPerSample, int channels, int samplesPerSec, long renderTimeMs, int bufferLength) {
Log.e(TAG, "onRecordAudioFrame: ");
}
private int maxAmplitude = 0;
#Override
public void onPlaybackAudioFrame(byte[] data, int audioFrameType, int samples, int bytesPerSample, int channels, int samplesPerSec, long renderTimeMs, int bufferLength) {
if(isVisualizerAttached) {
short[] rawAudio = new short[data.length/2];
ByteBuffer.wrap(data).order(ByteOrder.BIG_ENDIAN).asShortBuffer().get(rawAudio);
short amplitude = 0;
for(short num: rawAudio){
if(num > amplitude)
amplitude = num;
}
Log.e(TAG, "onPlaybackAudioFrame: Supposedly we have data -> max: " + amplitude);
}
Log.e(TAG, "onPlaybackAudioFrame:");
}
#Override
public void onPlaybackAudioFrameBeforeMixing(int uid, byte[] data, int audioFrameType, int samples, int bytesPerSample, int channels, int samplesPerSec, long renderTimeMs, int bufferLength) {
Log.e(TAG, "onPlaybackAudioFrameBeforeMixing: ");
}
#Override
public void onMixedAudioFrame(byte[] data, int audioFrameType, int samples, int bytesPerSample, int channels, int samplesPerSec, long renderTimeMs, int bufferLength) {
Log.e(TAG, "onMixedAudioFrame: ");
}
}

Did you give/add permissions in the manifest? Because Agora does not show any runtime error when you don't give/add permissions; instead, your voice call goes blank, and the microphone and speaker buttons, if added, won't do anything. This is what I went through one day dealing with some code on GitHub by Agora (that is, it didn't look up the manifest file).
According to the Agora documentation, the following permissions are required:
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
<uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
<uses-permission android:name="android.permission.BLUETOOTH" />

I have discovered where in the API Example (provided by the github repo link above), where to interact with the remote audio data. In the CustomAudioSource class, within the AsyncTask object at the bottom of the file. Had to run on physical device to take advantage of it.
Now I am need of finding the max amplitude of the audio byte array. The formula given within this example is "lengthInByte = sampleRate/1000 × 2 × channels × audio duration (ms)"

Android main thread UI not responding while implementing Google Speech-to-text. How to solve?

Currently, I am implementing google Speech to Text in my project. The sample code referred is this: Click Here.
I have used the SpeechService and Voice Recorder class from this project.
public class SpeechService extends Service {
public static final List<String> SCOPE =
Collections.singletonList("https://www.googleapis.com/auth/cloud-platform");
private static final String TAG = "SpeechService";
private static final String PREFS = "SpeechService";
private static final String PREF_ACCESS_TOKEN_VALUE = "access_token_value";
private static final String PREF_ACCESS_TOKEN_EXPIRATION_TIME = "access_token_expiration_time";
/**
* We reuse an access token if its expiration time is longer than this.
*/
private static final int ACCESS_TOKEN_EXPIRATION_TOLERANCE = 30 * 60 * 1000; // thirty minutes
/**
* We refresh the current access token before it expires.
*/
private static final int ACCESS_TOKEN_FETCH_MARGIN = 60 * 1000; // one minute
private static final String HOSTNAME = "speech.googleapis.com";
private static final int PORT = 443;
private static Handler mHandler;
private final SpeechBinder mBinder = new SpeechBinder();
private final ArrayList<Listener> mListeners = new ArrayList<>();
private final StreamObserver<StreamingRecognizeResponse> mResponseObserver
= new StreamObserver<StreamingRecognizeResponse>() {
#Override
public void onNext(StreamingRecognizeResponse response) {
Log.e("Speech", "Recognized");
String text = null;
boolean isFinal = false;
if (response.getResultsCount() > 0) {
System.out.println("result count....."+String.valueOf(response.getResultsCount()));
final StreamingRecognitionResult result = response.getResults(0);
isFinal = result.getIsFinal();
if (result.getAlternativesCount() > 0) {
final SpeechRecognitionAlternative alternative = result.getAlternatives(0);
text = alternative.getTranscript();
}
}
if (text != null && isFinal) {
for (Listener listener : mListeners) {
listener.onSpeechRecognized(text, isFinal);
}
} else {
for (Listener listener : mListeners) {
listener.onRandomStupidity();
}
}
}
#Override
public void onError(Throwable t) {
Log.e(TAG, "Error calling the API.", t);
for(Listener listener : mListeners){
listener.onErrorRecognizing();
}
}
#Override
public void onCompleted() {
Log.i(TAG, "API completed.");
}
};
private volatile AccessTokenTask mAccessTokenTask;
private final Runnable mFetchAccessTokenRunnable = new Runnable() {
#Override
public void run() {
fetchAccessToken();
}
};
private SpeechGrpc.SpeechStub mApi;
private StreamObserver<StreamingRecognizeRequest> mRequestObserver;
public static SpeechService from(IBinder binder) {
return ((SpeechBinder) binder).getService();
}
#Override
public void onCreate() {
super.onCreate();
mHandler = new Handler();
fetchAccessToken();
}
#Override
public void onDestroy() {
super.onDestroy();
mHandler.removeCallbacks(mFetchAccessTokenRunnable);
mHandler = null;
// Release the gRPC channel.
if (mApi != null) {
final ManagedChannel channel = (ManagedChannel) mApi.getChannel();
if (channel != null && !channel.isShutdown()) {
try {
channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Log.e(TAG, "Error shutting down the gRPC channel.", e);
}
}
mApi = null;
}
}
private void fetchAccessToken() {
if (mAccessTokenTask != null) {
return;
}
mAccessTokenTask = new AccessTokenTask();
mAccessTokenTask.execute();
}
private String getDefaultLanguageCode() {
final LangInnerResponse languageToLearn = MemoryCache.getLanguageToLearn();
if(languageToLearn != null) {
Log.e("Test Lang", languageToLearn.getCode());
return languageToLearn.getCode();
} else {
final Locale locale = Locale.getDefault();
final StringBuilder language = new StringBuilder(locale.getLanguage());
final String country = locale.getCountry();
if (!TextUtils.isEmpty(country)) {
language.append("-");
language.append(country);
}
return language.toString();
}
}
#Nullable
#Override
public IBinder onBind(Intent intent) {
return mBinder;
}
public void addListener(#NonNull Listener listener) {
mListeners.add(listener);
}
public void removeListener(#NonNull Listener listener) {
mListeners.remove(listener);
}
/**
** Starts recognizing speech audio.
*
* #param sampleRate The sample rate of the audio.
*/
public void startRecognizing(int sampleRate) {
if (mApi == null) {
Log.w(TAG, "API not ready. Ignoring the request.");
return;
}
System.out.println("calling api....");
// Configure the API
mRequestObserver = mApi.streamingRecognize(mResponseObserver);
mRequestObserver.onNext(StreamingRecognizeRequest.newBuilder()
.setStreamingConfig(StreamingRecognitionConfig.newBuilder()
.setConfig(RecognitionConfig.newBuilder()
.setLanguageCode(getDefaultLanguageCode())
.setEncoding(RecognitionConfig.AudioEncoding.LINEAR16)
.setSampleRateHertz(sampleRate)
.build())
.setInterimResults(true)
.setSingleUtterance(true)
.build())
.build());
}
/**
* Recognizes the speech audio. This method should be called every time a chunk of byte buffer
* is ready.
*
* #param data The audio data.
* #param size The number of elements that are actually relevant in the {#code data}.
*/
public void recognize(byte[] data, int size) {
if (mRequestObserver == null) {
return;
}
// Call the streaming recognition API
mRequestObserver.onNext(StreamingRecognizeRequest.newBuilder()
.setAudioContent(ByteString.copyFrom(data, 0, size))
.build());
}
/**
* Finishes recognizing speech audio.
*/
public void finishRecognizing() {
if (mRequestObserver == null) {
return;
}
mRequestObserver.onCompleted();
mRequestObserver = null;
}
public interface Listener {
/**
* Called when a new piece of text was recognized by the Speech API.
*
* #param text The text.
* #param isFinal {#code true} when the API finished processing audio.
*/
void onSpeechRecognized(String text, boolean isFinal);
void onErrorRecognizing();
void onRandomStupidity();
}
/**
* Authenticates the gRPC channel using the specified {#link GoogleCredentials}.
*/
private static class GoogleCredentialsInterceptor implements ClientInterceptor {
private final Credentials mCredentials;
private Metadata mCached;
private Map<String, List<String>> mLastMetadata;
GoogleCredentialsInterceptor(Credentials credentials) {
mCredentials = credentials;
}
private static Metadata toHeaders(Map<String, List<String>> metadata) {
Metadata headers = new Metadata();
if (metadata != null) {
for (String key : metadata.keySet()) {
Metadata.Key<String> headerKey = Metadata.Key.of(
key, Metadata.ASCII_STRING_MARSHALLER);
for (String value : metadata.get(key)) {
headers.put(headerKey, value);
}
}
}
return headers;
}
#Override
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
final MethodDescriptor<ReqT, RespT> method, CallOptions callOptions,
final Channel next) {
return new ClientInterceptors.CheckedForwardingClientCall<ReqT, RespT>(
next.newCall(method, callOptions)) {
#Override
protected void checkedStart(Listener<RespT> responseListener, Metadata headers)
throws StatusException {
Metadata cachedSaved;
URI uri = serviceUri(next, method);
synchronized (this) {
Map<String, List<String>> latestMetadata = getRequestMetadata(uri);
if (mLastMetadata == null || mLastMetadata != latestMetadata) {
mLastMetadata = latestMetadata;
mCached = toHeaders(mLastMetadata);
}
cachedSaved = mCached;
}
headers.merge(cachedSaved);
delegate().start(responseListener, headers);
}
};
}
/**
* Generate a JWT-specific service URI. The URI is simply an identifier with enough
* information for a service to know that the JWT was intended for it. The URI will
* commonly be verified with a simple string equality check.
*/
private URI serviceUri(Channel channel, MethodDescriptor<?, ?> method)
throws StatusException {
String authority = channel.authority();
if (authority == null) {
throw Status.UNAUTHENTICATED
.withDescription("Channel has no authority")
.asException();
}
// Always use HTTPS, by definition.
final String scheme = "https";
final int defaultPort = 443;
String path = "/" + MethodDescriptor.extractFullServiceName(method.getFullMethodName());
URI uri;
try {
uri = new URI(scheme, authority, path, null, null);
} catch (URISyntaxException e) {
throw Status.UNAUTHENTICATED
.withDescription("Unable to construct service URI for auth")
.withCause(e).asException();
}
// The default port must not be present. Alternative ports should be present.
if (uri.getPort() == defaultPort) {
uri = removePort(uri);
}
return uri;
}
private URI removePort(URI uri) throws StatusException {
try {
return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), -1 /* port */,
uri.getPath(), uri.getQuery(), uri.getFragment());
} catch (URISyntaxException e) {
throw Status.UNAUTHENTICATED
.withDescription("Unable to construct service URI after removing port")
.withCause(e).asException();
}
}
private Map<String, List<String>> getRequestMetadata(URI uri) throws StatusException {
try {
return mCredentials.getRequestMetadata(uri);
} catch (IOException e) {
throw Status.UNAUTHENTICATED.withCause(e).asException();
}
}
}
private class SpeechBinder extends Binder {
SpeechService getService() {
return SpeechService.this;
}
}
private class CreateApiSingle implements SingleOnSubscribe<SpeechGrpc.SpeechStub> {
#Override
public void subscribe(SingleEmitter<SpeechGrpc.SpeechStub> emitter) throws Exception {
final AccessToken accessToken = generateCredentials();
final SpeechGrpc.SpeechStub api = generateApi(accessToken);
emitter.onSuccess(api);
}
private AccessToken generateCredentials() throws IOException {
final SharedPreferences prefs =
getSharedPreferences(PREFS, Context.MODE_PRIVATE);
String tokenValue = prefs.getString(PREF_ACCESS_TOKEN_VALUE, null);
long expirationTime = prefs.getLong(PREF_ACCESS_TOKEN_EXPIRATION_TIME, -1);
// Check if the current token is still valid for a while
if (tokenValue != null && expirationTime > 0) {
if (expirationTime
> System.currentTimeMillis() + ACCESS_TOKEN_EXPIRATION_TOLERANCE) {
return new AccessToken(tokenValue, new Date(expirationTime));
}
}
// ***** WARNING *****
// In this sample, we load the credential from a JSON file stored in a raw resource
// folder of this client app. You should never do this in your app. Instead, store
// the file in your server and obtain an access token from there.
// *******************
final InputStream stream = getResources().openRawResource(R.raw.credential);
final GoogleCredentials credentials = GoogleCredentials.fromStream(stream)
.createScoped(SCOPE);
final AccessToken token = credentials.refreshAccessToken();
prefs.edit()
.putString(PREF_ACCESS_TOKEN_VALUE, token.getTokenValue())
.putLong(PREF_ACCESS_TOKEN_EXPIRATION_TIME,
token.getExpirationTime().getTime())
.apply();
stream.close();
return token;
}
private SpeechGrpc.SpeechStub generateApi(AccessToken accessToken) {
final ManagedChannel channel = new OkHttpChannelProvider()
.builderForAddress(HOSTNAME, PORT)
.nameResolverFactory(new DnsNameResolverProvider())
.intercept(new GoogleCredentialsInterceptor(new GoogleCredentials(accessToken)
.createScoped(SCOPE)))
.build();
return SpeechGrpc.newStub(channel);
}
}
private class AccessTokenTask extends AsyncTask<Void, Void, AccessToken> {
#Override
protected AccessToken doInBackground(Void... voids) {
final SharedPreferences prefs =
getSharedPreferences(PREFS, Context.MODE_PRIVATE);
String tokenValue = prefs.getString(PREF_ACCESS_TOKEN_VALUE, null);
long expirationTime = prefs.getLong(PREF_ACCESS_TOKEN_EXPIRATION_TIME, -1);
// Check if the current token is still valid for a while
if (tokenValue != null && expirationTime > 0) {
if (expirationTime
> System.currentTimeMillis() + ACCESS_TOKEN_EXPIRATION_TOLERANCE) {
return new AccessToken(tokenValue, new Date(expirationTime));
}
}
// ***** WARNING *****
// In this sample, we load the credential from a JSON file stored in a raw resource
// folder of this client app. You should never do this in your app. Instead, store
// the file in your server and obtain an access token from there.
// *******************
final InputStream stream = getResources().openRawResource(R.raw.credential);
try {
final GoogleCredentials credentials = GoogleCredentials.fromStream(stream)
.createScoped(SCOPE);
final AccessToken token = credentials.refreshAccessToken();
prefs.edit()
.putString(PREF_ACCESS_TOKEN_VALUE, token.getTokenValue())
.putLong(PREF_ACCESS_TOKEN_EXPIRATION_TIME,
token.getExpirationTime().getTime())
.apply();
return token;
} catch (IOException e) {
Log.e(TAG, "Failed to obtain access token.", e);
}
return null;
}
#Override
protected void onPostExecute(AccessToken accessToken) {
mAccessTokenTask = null;
final ManagedChannel channel = new OkHttpChannelProvider()
.builderForAddress(HOSTNAME, PORT)
.nameResolverFactory(new DnsNameResolverProvider())
.intercept(new GoogleCredentialsInterceptor(new GoogleCredentials(accessToken)
.createScoped(SCOPE)))
.build();
mApi = SpeechGrpc.newStub(channel);
// Schedule access token refresh before it expires
if (mHandler != null) {
mHandler.postDelayed(mFetchAccessTokenRunnable,
Math.max(accessToken.getExpirationTime().getTime()
- System.currentTimeMillis()
- ACCESS_TOKEN_FETCH_MARGIN, ACCESS_TOKEN_EXPIRATION_TOLERANCE));
}
}
}}
public class VoiceRecorder {
private static final int[] SAMPLE_RATE_CANDIDATES = new int[]{48000, 44100};
private static final int CHANNEL = AudioFormat.CHANNEL_IN_MONO;
private static final int ENCODING = AudioFormat.ENCODING_PCM_16BIT;
private static final int AMPLITUDE_THRESHOLD = 1500;
private static final int SPEECH_TIMEOUT_MILLIS = 2000;
private static final int MAX_SPEECH_LENGTH_MILLIS = 30 * 1000;
public static abstract class Callback {
/**
* Called when the recorder starts hearing voice.
*/
public void onVoiceStart() {
}
/**
* Called when the recorder is hearing voice.
*
* #param data The audio data in {#link AudioFormat#ENCODING_PCM_16BIT}.
* #param size The size of the actual data in {#code data}.
*/
public void onVoice(byte[] data, int size) {
}
/**
* Called when the recorder stops hearing voice.
*/
public void onVoiceEnd() {
}
}
private final Callback mCallback;
private AudioRecord mAudioRecord;
private Thread mThread;
private byte[] mBuffer;
private final Object mLock = new Object();
/** The timestamp of the last time that voice is heard. */
private long mLastVoiceHeardMillis = Long.MAX_VALUE;
/** The timestamp when the current voice is started. */
private long mVoiceStartedMillis;
public VoiceRecorder(#NonNull Callback callback) {
mCallback = callback;
}
/**
* Starts recording audio.
*
* <p>The caller is responsible for calling {#link #stop()} later.</p>
*/
public void start() {
// Stop recording if it is currently ongoing.
stop();
// Try to create a new recording session.
mAudioRecord = createAudioRecord();
if (mAudioRecord == null) {
throw new RuntimeException("Cannot instantiate VoiceRecorder");
}
// Start recording.
mAudioRecord.startRecording();
// Start processing the captured audio.
mThread = new Thread(new ProcessVoice());
mThread.start();
}
/**
* Stops recording audio.
*/
public void stop() {
synchronized (mLock) {
System.out.println("stop audio record....");
dismiss();
if (mThread != null) {
mThread.interrupt();
mThread = null;
}
if (mAudioRecord != null) {
mAudioRecord.stop();
mAudioRecord.release();
mAudioRecord = null;
}
mBuffer = null;
System.out.println("stop audio record....2");
}
}
/**
* Dismisses the currently ongoing utterance.
*/
public void dismiss() {
if (mLastVoiceHeardMillis != Long.MAX_VALUE) {
mLastVoiceHeardMillis = Long.MAX_VALUE;
mCallback.onVoiceEnd();
}
}
/**
* Retrieves the sample rate currently used to record audio.
*
* #return The sample rate of recorded audio.
*/
public int getSampleRate() {
if (mAudioRecord != null) {
return mAudioRecord.getSampleRate();
}
return 0;
}
/**
* Creates a new {#link AudioRecord}.
*
* #return A newly created {#link AudioRecord}, or null if it cannot be created (missing
* permissions?).
*/
private AudioRecord createAudioRecord() {
for (int sampleRate : SAMPLE_RATE_CANDIDATES) {
final int sizeInBytes = AudioRecord.getMinBufferSize(sampleRate, CHANNEL, ENCODING);
if (sizeInBytes == AudioRecord.ERROR_BAD_VALUE) {
continue;
}
final AudioRecord audioRecord = new AudioRecord(MediaRecorder.AudioSource.MIC,
sampleRate, CHANNEL, ENCODING, sizeInBytes);
if (audioRecord.getState() == AudioRecord.STATE_INITIALIZED) {
mBuffer = new byte[sizeInBytes];
return audioRecord;
} else {
audioRecord.release();
}
}
return null;
}
/**
* Continuously processes the captured audio and notifies {#link #mCallback} of corresponding
* events.
*/
private class ProcessVoice implements Runnable {
#Override
public void run() {
while (true) {
synchronized (mLock) {
if (Thread.currentThread().isInterrupted()) {
break;
}
final int size = mAudioRecord.read(mBuffer, 0, mBuffer.length);
final long now = System.currentTimeMillis();
if (isHearingVoice(mBuffer, size)) {
if (mLastVoiceHeardMillis == Long.MAX_VALUE) {
mVoiceStartedMillis = now;
mCallback.onVoiceStart();
}
mCallback.onVoice(mBuffer, size);
mLastVoiceHeardMillis = now;
if (now - mVoiceStartedMillis > MAX_SPEECH_LENGTH_MILLIS) {
end();
}
} else if (mLastVoiceHeardMillis != Long.MAX_VALUE) {
mCallback.onVoice(mBuffer, size);
if (now - mLastVoiceHeardMillis > SPEECH_TIMEOUT_MILLIS) {
end();
}
}
}
}
}
private void end() {
mLastVoiceHeardMillis = Long.MAX_VALUE;
mCallback.onVoiceEnd();
System.out.println("end...");
}
private boolean isHearingVoice(byte[] buffer, int size) {
for (int i = 0; i < size - 1; i += 2) {
// The buffer has LINEAR16 in little endian.
int s = buffer[i + 1];
if (s < 0) s *= -1;
s <<= 8;
s += Math.abs(buffer[i]);
if (s > AMPLITUDE_THRESHOLD) {
return true;
}
}
return false;
}
}}
Then I implemented the Speech Service & Voice Recorder callback as follows:
private VoiceRecorder voiceRecorder;
private final SpeechService.Listener speechServiceListener = new SpeechService.Listener() {
#Override
public void onSpeechRecognized(final String text, final boolean isFinal) {
if (isFinal) {
System.out.println("ui thread...");
if (!TextUtils.isEmpty(text)) {
runOnUiThread(() -> {
showMessage(text);
flingAnswer(text);
});
}
}
}
#Override
public void onErrorRecognizing() {
showMessage("Please try again. Could not detect.");
}
#Override
public void onRandomStupidity() {
}
};
private SpeechService speechService;
private final VoiceRecorder.Callback voiceCallback = new VoiceRecorder.Callback() {
#Override
public void onVoiceStart() {
if (speechService != null) {
System.out.println("voice start....");
speechService.startRecognizing(voiceRecorder.getSampleRate());
}
}
#Override
public void onVoice(byte[] data, int size) {
if (speechService != null) {
speechService.recognize(data, size);
}
}
#Override
public void onVoiceEnd() {
if (speechService != null) {
speechService.finishRecognizing();
}
}
};
private final ServiceConnection serviceConnection = new ServiceConnection() {
#Override
public void onServiceConnected(ComponentName componentName, IBinder binder) {
speechService = SpeechService.from(binder);
speechService.addListener(speechServiceListener);
}
#Override
public void onServiceDisconnected(ComponentName componentName) {
speechService = null;
}
};
For voice input this is the code:
#Override
public void stopRecognizing() {
stopVoiceRecorder();
Log.e("Recording", "Stopped");
}
#Override
public void startRecognizing() {
if (permissionManager != null && permissionManager.askForPermissions()) {
startVoiceRecorder();
vibrate.vibrate(50);//Providing haptic feedback to user on press.
}
Log.e("Recording", "Started");
}
binding.imgVoice.setOnTouchListener((v, event) -> {
switch (event.getAction()) {
case MotionEvent.ACTION_UP:
System.out.println("up...");
mCallback.stopRecognizing();
binding.imgVoice
.animate()
.scaleX(1.0f)
.scaleY(1.0f);
binding.imgVoice.setVisibility(View.GONE);
binding.progressBar.setVisibility(View.VISIBLE);
break;
case MotionEvent.ACTION_DOWN:
System.out.println("down...");
binding.imgVoice
.animate()
.scaleX(1.8f)
.scaleY(1.8f);
mCallback.startRecognizing();
break;
}
return true;
});
}
When I press the mic, event registered as Action_Down, I start the voice recorder and on releasing the mic , voice recorder is stopped. Also, with the Action_Down I am scaling up the mic icon which needs to be scaled down on Action_Up . But the ui freezes as a whole most of the times. I find that the onNext() callback for StreamObserver is continuously being invoked before the isFinal becomes true.
private void startVoiceRecorder() {
if (voiceRecorder != null) {
voiceRecorder.stop();
}
voiceRecorder = new VoiceRecorder(voiceCallback);
voiceRecorder.start();
}
private void stopVoiceRecorder() {
if (voiceRecorder != null) {
voiceRecorder.stop();
voiceRecorder = null;
}
}
But I want the mic to scale down as soon as I release the mic(on Action up event) which is not happening.
So if anyone can help me over this?
Thanks in Advance.

Rxandroid ble Guidelines for handling re-connection

[Update]
I am able to reconnect to the ble device after disconnect but can't seem to read or write any characteristics. Logcat spews this out after reconnect, is this my app doing something wrong or is it because of the ble device?
08-09 15:05:45.109 9601-10364/com.project.app D/BluetoothGatt: setCharacteristicNotification() - uuid: cb67d1e1-cfb5-45f5-9123-3f07d9189f1b enable: false
08-09 15:05:45.111 9601-10352/com.project.app D/RxBle#ConnectionOperationQueue: STARTED DescriptorWriteOperation(54881118)
08-09 15:05:45.114 9601-10364/com.project.app D/RxBle#ConnectionOperationQueue: QUEUED DescriptorWriteOperation(191754430)
08-09 15:05:45.116 9601-9601/com.project.app D/BtleConnManager: RETRY 2/-1 :::: com.polidea.rxandroidble.exceptions.BleCannotSetCharacteristicNotificationException
08-09 15:06:15.117 9601-10352/com.project.app D/RxBle#ConnectionOperationQueue: FINISHED DescriptorWriteOperation(54881118)
08-09 15:06:15.118 9601-10365/com.project.app D/BluetoothGatt: setCharacteristicNotification() - uuid: cb67d0e1-cfb5-45f5-9123-3f07d9189f1b enable: false
08-09 15:06:15.120 9601-10352/com.project.app D/RxBle#ConnectionOperationQueue: STARTED DescriptorWriteOperation(88995281)
08-09 15:06:15.124 9601-10365/com.project.app D/RxBle#ConnectionOperationQueue: QUEUED DescriptorWriteOperation(108601267)
08-09 15:06:15.124 9601-10366/com.project.app D/BluetoothGatt: setCharacteristicNotification() - uuid: cb67d1e1-cfb5-45f5-9123-3f07d9189f1b enable: true
08-09 15:06:15.126 9601-9601/com.project.app D/BtleConnManager: RETRY 2/-1 :::: com.polidea.rxandroidble.exceptions.BleCannotSetCharacteristicNotificationException
08-09 15:06:15.131 9601-10366/com.project.app D/RxBle#ConnectionOperationQueue: QUEUED DescriptorWriteOperation(98838561)
08-09 15:06:45.126 9601-10352/com.project.app D/RxBle#ConnectionOperationQueue: FINISHED DescriptorWriteOperation(88995281)
[Update]
Using rxandroidble1 and rxjava1
Hi I am new to the concept of rxjava and ble connections but I've been put on a existing project with very little documentation and I'm having problems handling the re-connection after a connection is lost.
I've checked out the sample app of rxandroidble but it only handles connection and no the re-connection if it looses it. Or is the library supposed to handle it by it's own or am I missing something.
The general problem can be described as such :
I connect the phone-app to my ble device. Everything works as expected i get notifications from my subsriptions when temperature is changing on my ble device.
I loose connection either by turning of the ble chip on the device or turning off Bluetooth on my phone or walking out of range.
I turn on Bluetooth again either on my phone or the ble device.
I manage to reconnect but my subscriptions aren't re subscribed so i don't get any notifications back to my phone when temperature or other values are changing.
According to my employer this code should have worked fine in the past but I can't seem to get it to work after it looses the connection. So can any of you guys see any errors in the logic of the code. Or might there be a problem with the ble device? Or is this a common bug or problem with the RxBleConnectionSharingAdapter or what? I've tried everything but nothing seems to work.
Or am I missing something in like the onUnsibcribeMethod or something?
I guess the establish connection method is the most relevant part of the code. Iv'e tried re-subscribing via to a characteristic after reconnect via the test method but the app just crashes then.
This is my connection manager class :
private static final String TAG = "HELLOP";
private static RxBleClient rxBleClient;
private RxBleConnection rxBleConnection;
private static final int MAX_RETRIES = 10;
private static final int SHORT_RETRY_DELAY_MS = 1000;
private static final int LONG_RETRY_DELAY_MS = 30000;
private final Context mContext;
private final String mMacAddress;
private final String gatewayName;
private final RxBleDevice mBleDevice;
private PublishSubject<Void> mDisconnectTriggerSubject = PublishSubject.create();
private Observable<RxBleConnection> mConnectionObservable;
private final ProjectDeviceManager mProjectDeviceManager;
private BehaviorSubject<Boolean> connectionStatusSubject = BehaviorSubject.create();
private boolean isAutoSignIn = false;
private BondStateReceiver bondStateReceiver;
private boolean isBonded = false;
//gets the client
public static RxBleClient getRxBleClient(Context context) {
if (rxBleClient == null) {
// rxBleClient = /*MockedClient.getClient();*/RxBleClient.create(this);
// RxBleClient.setLogLevel(RxBleLog.DEBUG);
// super.onCreate();
rxBleClient = RxBleClient.create(context);
RxBleClient.setLogLevel(RxBleLog.DEBUG);
}
return rxBleClient;
}
public BtleConnectionManager(final Context context, final String macAddress, String name) {
mContext = context;
mMacAddress = macAddress;
gatewayName = name;
mBleDevice = getRxBleClient(context).getBleDevice(macAddress);
mProjectDeviceManager = new ProjectDeviceManager(this);
}
#Override
public final Context getContext() {
return mContext;
}
#Override
public final ProjectDeviceManager getProjectDeviceManager() {
return mProjectDeviceManager;
}
#Override
public final boolean isConnected() {
return mBleDevice.getConnectionState() == RxBleConnection.RxBleConnectionState.CONNECTED;
}
#Override
public String getConnectionName() {
if (gatewayName != null && !gatewayName.isEmpty()) {
return gatewayName;
} else {
return mMacAddress;
}
}
final RxBleDevice getBleDevice() {
return mBleDevice;
}
public final synchronized Observable<RxBleConnection> getConnection() {
if (mConnectionObservable == null || mBleDevice.getConnectionState() == RxBleConnection.RxBleConnectionState.DISCONNECTED
|| mBleDevice.getConnectionState() == RxBleConnection.RxBleConnectionState.DISCONNECTING) {
establishConnection();
}
return mConnectionObservable;
}
public void goBack() {
Intent intent = null;
try {
intent = new Intent(mContext,
Class.forName("com.Project.dcpapp.BluetoothActivity"));
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK | Intent.FLAG_ACTIVITY_CLEAR_TASK);
((Activity) mContext).startActivity(intent);
((Activity) mContext).finish();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
public void setAutoSignIn(boolean value) {
this.isAutoSignIn = value;
}
public boolean getAutoSignIn() {
return this.isAutoSignIn;
}
#Override
public void pause() {
}
#Override
public void resume() {
}
#Override
public Observable<Boolean> observeConnectionStatus() {
return connectionStatusSubject;
}
#Override
public Calendar getLastConnectionTime() {
return mProjectDeviceManager.getLastUpdateTime();
}
public void disconnect() {
BluetoothDevice bluetoothDevice = getBleDevice().getBluetoothDevice();
Log.d("BtleConnManager", " disconnect " + bluetoothDevice.getBondState());
Handler handler = new Handler(Looper.getMainLooper());
handler.postDelayed(new Runnable() {
#Override
public void run() {
mDisconnectTriggerSubject.onNext(null);
mConnectionObservable = null;
}
}, 700);
}
public void removeBond() {
Method m = null;
BluetoothDevice bluetoothDevice = getBleDevice().getBluetoothDevice();
Log.d("BtleConnManager", " removeBond " + bluetoothDevice.getBondState());
if (bluetoothDevice.getBondState() == BluetoothDevice.BOND_BONDED) {
try {
m = bluetoothDevice.getClass().getMethod("removeBond", (Class[]) null);
m.invoke(bluetoothDevice, (Object[]) null);
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
}
}
public void bond() {
BluetoothDevice bluetoothDevice = getBleDevice().getBluetoothDevice();
Log.d("BtleConnManager ", "bond state " + bluetoothDevice.getBondState());
if (bluetoothDevice.getBondState() == BluetoothDevice.BOND_NONE
&& Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) {
bondStateReceiver = new BondStateReceiver();
final IntentFilter filter = new IntentFilter(BluetoothDevice.ACTION_BOND_STATE_CHANGED);
getContext().registerReceiver(bondStateReceiver, filter);
bluetoothDevice.createBond();
}
}
public void setBonded(boolean value) {
this.isBonded = value;
}
public boolean isBonded() {
return this.isBonded;
}
private class BondStateReceiver extends BroadcastReceiver {
#Override
public void onReceive(Context context, Intent intent) {
final String action = intent.getAction();
if (action.equals(BluetoothDevice.ACTION_BOND_STATE_CHANGED)) {
final int state = intent.getIntExtra(BluetoothDevice.EXTRA_BOND_STATE, BluetoothDevice.ERROR);
switch (state) {
case BluetoothDevice.BOND_BONDED:
setBonded(true);
Log.d("BtleConManager", "Bonded ");
break;
case BluetoothDevice.BOND_BONDING:
Log.d("BtleConManager", "Bonding ");
break;
case BluetoothDevice.BOND_NONE:
Log.d("BtleConManager", "unbonded ");
setBonded(false);
final int prevState = intent.getIntExtra(BluetoothDevice.EXTRA_PREVIOUS_BOND_STATE, BluetoothDevice.ERROR);
if (prevState == BluetoothDevice.BOND_BONDING) {
Toast.makeText(getContext(), R.string.error_bluetooth_bonding_failed, Toast.LENGTH_LONG).show();
}
break;
}
}
}
}
private void establishConnection() {
Log.d("BtleConnManager", " establishConnection");
mConnectionObservable = mBleDevice
.establishConnection(false)
.observeOn(AndroidSchedulers.mainThread())
.doOnNext(rxBleConnection -> {
// Save connection to use if retry is done when already connected
this.rxBleConnection = rxBleConnection;
// Notify observers that connection is established
connectionStatusSubject.onNext(true);
})
.onErrorResumeNext(error -> {
// Return the saved connection if already connected
if (error instanceof BleAlreadyConnectedException && rxBleConnection != null) {
return Observable.just(rxBleConnection);
} else {
return Observable.error(error);
}
})
//.retryWhen(getRetryRule()) Do not retry connect here - retry when using getConnection instead (otherwise a double retry connection will be done)
.takeUntil(mDisconnectTriggerSubject)
.doOnError(throwable -> {
this.rxBleConnection = null;
if (!isConnected()) {
// Notify observers that connection has failed
connectionStatusSubject.onNext(false);
}
}).doOnUnsubscribe(() -> {
Log.d("BtleConnManager", "establishConnection Unsubscribe ");
connectionStatusSubject.onNext(false);
}).doOnCompleted(() -> Log.d("BtleConnManager", "establishConnection completed"))
.doOnSubscribe(() -> {
})
//.subscribeOn(AndroidSchedulers.mainThread())
//.compose(bindUntilEvent(PAUSE))
.compose(new ConnectionSharingAdapter());
}
public void test(){
mConnectionObservable
.flatMap(rxBleConnection -> rxBleConnection.setupNotification(UUID.fromString("cb67d1c1-cfb5-45f5-9123-3f07d9189f1b")))
.flatMap(notificationObservable -> notificationObservable)
.observeOn(AndroidSchedulers.mainThread())
.retryWhen(errors -> errors.flatMap(error -> {
if (error instanceof BleDisconnectedException) {
Log.d("Retry", "Retrying");
return Observable.just(null);
}
return Observable.error(error);
}))
.doOnError(throwable -> {
Log.d(TAG, "establishConnection: " + throwable.getMessage());
})
.subscribe(bytes -> {
Log.d(TAG, "establishConnection: characteristics changed" + new String(bytes));
// React on characteristic changes
});
}
public RetryWithDelay getRetryRule() {
return new RetryWithDelay(MAX_RETRIES, SHORT_RETRY_DELAY_MS);
}
public RetryWithDelay getInfiniteRetryRule() {
return new RetryWithDelay(RetryWithDelay.NO_MAX, LONG_RETRY_DELAY_MS);
}
public class RetryWithDelay implements
Func1<Observable<? extends Throwable>, Observable<?>> {
public static final int NO_MAX = -1;
private final int maxRetries;
private final int retryDelayMs;
private int retryCount;
public RetryWithDelay(final int maxRetries, final int retryDelayMs) {
this.maxRetries = maxRetries;
this.retryDelayMs = retryDelayMs;
this.retryCount = 0;
}
#Override
public Observable<?> call(Observable<? extends Throwable> attempts) {
return attempts
.flatMap(new Func1<Throwable, Observable<?>>() {
#Override
public Observable<?> call(Throwable throwable) {
++retryCount;
if (mConnectionObservable == null) {
// If manually disconnected return empty observable
return Observable.empty();
} else if (throwable instanceof BleAlreadyConnectedException) {
return Observable.error(throwable);
} else if (retryCount < maxRetries || maxRetries == NO_MAX) {
Log.d("BtleConnManager", " RETRY " + retryCount + "/" + maxRetries + " :::: " + throwable.getClass().getName());
// When this Observable calls onNext, the original
// Observable will be retried (i.e. re-subscribed).
return Observable.timer(retryDelayMs, TimeUnit.MILLISECONDS);
} else {
//Last try
Log.d("BtleConnManager", " LAST RETRY " + retryCount + "/" + maxRetries + " :::: " + throwable.getClass().getName());
return Observable.error(throwable);
}
}
});
}
}

In establishConnection you set the autoConnect parameter to false, which will prevent automatic reconnection. If you set it to true instead, it should automatically reconnect. See https://stackoverflow.com/a/40187086/556495 and http://polidea.github.io/RxAndroidBle/ under Auto connect.
Note that this will not work if Bluetooth is turned off/restarted on the phone/tablet. So you'll probably also need a Bluetooth state change broadcast listener to restart everything when that happens.

waitingInMainSignalCatcherLoop,Thread*=0x72c22ee000,peer=0x12d00280,"Signal Catcher"]: reacting to signal 3

Good day everyone, I would like to ask, hat is the cause of that ANR?. In my project I have service which is binded in a activity. Now when I exit in that activity the app is hang for a moment. My thought is that the service is still running though I unbind it in onStop() of the activity.
Here is my service class
public class SpeechService extends Service {
public interface Listener {
/**
* Called when a new piece of text was recognized by the Speech API.
*
* #param text The text.
* #param isFinal {#code true} when the API finished processing audio.
*/
void onSpeechRecognized(String text, boolean isFinal);
}
private static final String TAG = "SpeechService";
private static final String PREFS = "SpeechService";
private static final String PREF_ACCESS_TOKEN_VALUE = "access_token_value";
private static final String PREF_ACCESS_TOKEN_EXPIRATION_TIME = "access_token_expiration_time";
/** We reuse an access token if its expiration time is longer than this. */
private static final int ACCESS_TOKEN_EXPIRATION_TOLERANCE = 30 * 60 * 1000; // thirty minutes
/** We refresh the current access token before it expires. */
private static final int ACCESS_TOKEN_FETCH_MARGIN = 60 * 1000; // one minute
public static final List<String> SCOPE =
Collections.singletonList("https://www.googleapis.com/auth/cloud-platform");
private static final String HOSTNAME = "speech.googleapis.com";
private static final int PORT = 443;
private final SpeechBinder mBinder = new SpeechBinder();
private final ArrayList<Listener> mListeners = new ArrayList<>();
private volatile AccessTokenTask mAccessTokenTask;
private SpeechGrpc.SpeechStub mApi;
private static Handler mHandler;
private final StreamObserver<StreamingRecognizeResponse> mResponseObserver
= new StreamObserver<StreamingRecognizeResponse>() {
#Override
public void onNext(StreamingRecognizeResponse response) {
String text = null;
boolean isFinal = false;
if (response.getResultsCount() > 0) {
final StreamingRecognitionResult result = response.getResults(0);
isFinal = result.getIsFinal();
if (result.getAlternativesCount() > 0) {
final SpeechRecognitionAlternative alternative = result.getAlternatives(0);
text = alternative.getTranscript();
}
}
if (text != null) {
for (Listener listener : mListeners) {
listener.onSpeechRecognized(text, isFinal);
}
}
}
#Override
public void onError(Throwable t) {
Log.e(TAG, "Error calling the API.", t);
}
#Override
public void onCompleted() {
Log.i(TAG, "API completed.");
}
};
private final StreamObserver<RecognizeResponse> mFileResponseObserver
= new StreamObserver<RecognizeResponse>() {
#Override
public void onNext(RecognizeResponse response) {
String text = null;
if (response.getResultsCount() > 0) {
final SpeechRecognitionResult result = response.getResults(0);
if (result.getAlternativesCount() > 0) {
final SpeechRecognitionAlternative alternative = result.getAlternatives(0);
text = alternative.getTranscript();
}
}
if (text != null) {
for (Listener listener : mListeners) {
listener.onSpeechRecognized(text, true);
}
}
}
#Override
public void onError(Throwable t) {
Log.e(TAG, "Error calling the API.", t);
}
#Override
public void onCompleted() {
Log.i(TAG, "API completed.");
}
};
private StreamObserver<StreamingRecognizeRequest> mRequestObserver;
public static SpeechService from(IBinder binder) {
return ((SpeechBinder) binder).getService();
}
#Override
public void onCreate() {
super.onCreate();
mHandler = new Handler();
fetchAccessToken();
}
#Override
public void onDestroy() {
super.onDestroy();
mHandler.removeCallbacks(mFetchAccessTokenRunnable);
mHandler = null;
// Release the gRPC channel.
if (mApi != null) {
final ManagedChannel channel = (ManagedChannel) mApi.getChannel();
if (channel != null && !channel.isShutdown()) {
try {
channel.shutdown().awaitTermination(1, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Log.e(TAG, "Error shutting down the gRPC channel.", e);
}
}
mApi = null;
}
}
private void fetchAccessToken() {
if (mAccessTokenTask != null) {
return;
}
mAccessTokenTask = new AccessTokenTask();
mAccessTokenTask.execute();
}
private String getDefaultLanguageCode() {
final Locale locale = Locale.getDefault();
final StringBuilder language = new StringBuilder(locale.getLanguage());
final String country = locale.getCountry();
if (!TextUtils.isEmpty(country)) {
language.append("-");
language.append(country);
}
return language.toString();
}
#Nullable
#Override
public IBinder onBind(Intent intent) {
return mBinder;
}
public void addListener(#NonNull Listener listener) {
mListeners.add(listener);
}
public void removeListener(#NonNull Listener listener) {
mListeners.remove(listener);
}
/**
* Starts recognizing speech audio.
*
* #param sampleRate The sample rate of the audio.
*/
public void startRecognizing(int sampleRate) {
if (mApi == null) {
Log.w(TAG, "API not ready. Ignoring the request.");
return;
}
// Configure the API
mRequestObserver = mApi.streamingRecognize(mResponseObserver);
mRequestObserver.onNext(StreamingRecognizeRequest.newBuilder()
.setStreamingConfig(StreamingRecognitionConfig.newBuilder()
.setConfig(RecognitionConfig.newBuilder()
.setLanguageCode(getDefaultLanguageCode())
.setEncoding(RecognitionConfig.AudioEncoding.LINEAR16)
.setSampleRateHertz(sampleRate)
.build())
.setInterimResults(true)
.setSingleUtterance(true)
.build())
.build());
}
/**
* Recognizes the speech audio. This method should be called every time a chunk of byte buffer
* is ready.
*
* #param data The audio data.
* #param size The number of elements that are actually relevant in the {#code data}.
*/
public void recognize(byte[] data, int size) {
if (mRequestObserver == null) {
return;
}
// Call the streaming recognition API
mRequestObserver.onNext(StreamingRecognizeRequest.newBuilder()
.setAudioContent(ByteString.copyFrom(data, 0, size))
.build());
}
/**
* Finishes recognizing speech audio.
*/
public void finishRecognizing() {
if (mRequestObserver == null) {
return;
}
mRequestObserver.onCompleted();
mRequestObserver = null;
}
/**
* Recognize all data from the specified {#link InputStream}.
*
* #param stream The audio data.
*/
public void recognizeInputStream(InputStream stream) {
try {
mApi.recognize(
RecognizeRequest.newBuilder()
.setConfig(RecognitionConfig.newBuilder()
.setEncoding(RecognitionConfig.AudioEncoding.LINEAR16)
.setLanguageCode("en-US")
.setSampleRateHertz(16000)
.build())
.setAudio(RecognitionAudio.newBuilder()
.setContent(ByteString.readFrom(stream))
.build())
.build(),
mFileResponseObserver);
} catch (IOException e) {
Log.e(TAG, "Error loading the input", e);
}
}
private class SpeechBinder extends Binder {
SpeechService getService() {
return SpeechService.this;
}
}
private final Runnable mFetchAccessTokenRunnable = new Runnable() {
#Override
public void run() {
fetchAccessToken();
}
};
private class AccessTokenTask extends AsyncTask<Void, Void, AccessToken> {
#Override
protected AccessToken doInBackground(Void... voids) {
final SharedPreferences prefs =
getSharedPreferences(PREFS, Context.MODE_PRIVATE);
String tokenValue = prefs.getString(PREF_ACCESS_TOKEN_VALUE, null);
long expirationTime = prefs.getLong(PREF_ACCESS_TOKEN_EXPIRATION_TIME, -1);
// Check if the current token is still valid for a while
if (tokenValue != null && expirationTime > 0) {
if (expirationTime
> System.currentTimeMillis() + ACCESS_TOKEN_EXPIRATION_TOLERANCE) {
return new AccessToken(tokenValue, new Date(expirationTime));
}
}
// ***** WARNING *****
// In this sample, we load the credential from a JSON file stored in a raw resource
// folder of this client app. You should never do this in your app. Instead, store
// the file in your server and obtain an access token from there.
// *******************
final InputStream stream = getResources().openRawResource(R.raw.credential);
try {
final GoogleCredentials credentials = GoogleCredentials.fromStream(stream)
.createScoped(SCOPE);
final AccessToken token = credentials.refreshAccessToken();
prefs.edit()
.putString(PREF_ACCESS_TOKEN_VALUE, token.getTokenValue())
.putLong(PREF_ACCESS_TOKEN_EXPIRATION_TIME,
token.getExpirationTime().getTime())
.apply();
return token;
} catch (IOException e) {
Log.e(TAG, "Failed to obtain access token.", e);
}
return null;
}
#Override
protected void onPostExecute(AccessToken accessToken) {
mAccessTokenTask = null;
final ManagedChannel channel = new OkHttpChannelProvider()
.builderForAddress(HOSTNAME, PORT)
.nameResolverFactory(new DnsNameResolverProvider())
.intercept(new GoogleCredentialsInterceptor(new GoogleCredentials(accessToken)
.createScoped(SCOPE)))
.build();
mApi = SpeechGrpc.newStub(channel);
// Schedule access token refresh before it expires
if (mHandler != null) {
mHandler.postDelayed(mFetchAccessTokenRunnable,
Math.max(accessToken.getExpirationTime().getTime()
- System.currentTimeMillis()
- ACCESS_TOKEN_FETCH_MARGIN, ACCESS_TOKEN_EXPIRATION_TOLERANCE));
}
}
}
/**
* Authenticates the gRPC channel using the specified {#link GoogleCredentials}.
*/
private static class GoogleCredentialsInterceptor implements ClientInterceptor {
private final Credentials mCredentials;
private Metadata mCached;
private Map<String, List<String>> mLastMetadata;
GoogleCredentialsInterceptor(Credentials credentials) {
mCredentials = credentials;
}
#Override
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
final MethodDescriptor<ReqT, RespT> method, CallOptions callOptions,
final Channel next) {
return new ClientInterceptors.CheckedForwardingClientCall<ReqT, RespT>(
next.newCall(method, callOptions)) {
#Override
protected void checkedStart(Listener<RespT> responseListener, Metadata headers)
throws StatusException {
Metadata cachedSaved;
URI uri = serviceUri(next, method);
synchronized (this) {
Map<String, List<String>> latestMetadata = getRequestMetadata(uri);
if (mLastMetadata == null || mLastMetadata != latestMetadata) {
mLastMetadata = latestMetadata;
mCached = toHeaders(mLastMetadata);
}
cachedSaved = mCached;
}
headers.merge(cachedSaved);
delegate().start(responseListener, headers);
}
};
}
/**
* Generate a JWT-specific service URI. The URI is simply an identifier with enough
* information for a service to know that the JWT was intended for it. The URI will
* commonly be verified with a simple string equality check.
*/
private URI serviceUri(Channel channel, MethodDescriptor<?, ?> method)
throws StatusException {
String authority = channel.authority();
if (authority == null) {
throw Status.UNAUTHENTICATED
.withDescription("Channel has no authority")
.asException();
}
// Always use HTTPS, by definition.
final String scheme = "https";
final int defaultPort = 443;
String path = "/" + MethodDescriptor.extractFullServiceName(method.getFullMethodName());
URI uri;
try {
uri = new URI(scheme, authority, path, null, null);
} catch (URISyntaxException e) {
throw Status.UNAUTHENTICATED
.withDescription("Unable to construct service URI for auth")
.withCause(e).asException();
}
// The default port must not be present. Alternative ports should be present.
if (uri.getPort() == defaultPort) {
uri = removePort(uri);
}
return uri;
}
private URI removePort(URI uri) throws StatusException {
try {
return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), -1 /* port */,
uri.getPath(), uri.getQuery(), uri.getFragment());
} catch (URISyntaxException e) {
throw Status.UNAUTHENTICATED
.withDescription("Unable to construct service URI after removing port")
.withCause(e).asException();
}
}
private Map<String, List<String>> getRequestMetadata(URI uri) throws StatusException {
try {
return mCredentials.getRequestMetadata(uri);
} catch (IOException e) {
throw Status.UNAUTHENTICATED.withCause(e).asException();
}
}
private static Metadata toHeaders(Map<String, List<String>> metadata) {
Metadata headers = new Metadata();
if (metadata != null) {
for (String key : metadata.keySet()) {
Metadata.Key<String> headerKey = Metadata.Key.of(
key, Metadata.ASCII_STRING_MARSHALLER);
for (String value : metadata.get(key)) {
headers.put(headerKey, value);
}
}
}
return headers;
}
}
}
and here is my activity class
public class MainActivity extends AppCompatActivity implements MessageDialogFragment.Listener {
private static final String FRAGMENT_MESSAGE_DIALOG = "message_dialog";
private static final String STATE_RESULTS = "results";
private static final int REQUEST_RECORD_AUDIO_PERMISSION = 1;
private SpeechService mSpeechService;
private VoiceRecorder mVoiceRecorder;
private final VoiceRecorder.Callback mVoiceCallback = new VoiceRecorder.Callback() {
#Override
public void onVoiceStart() {
showStatus(true);
if (mSpeechService != null) {
mSpeechService.startRecognizing(mVoiceRecorder.getSampleRate());
}
}
#Override
public void onVoice(byte[] data, int size) {
if (mSpeechService != null) {
mSpeechService.recognize(data, size);
}
}
#Override
public void onVoiceEnd() {
showStatus(false);
if (mSpeechService != null) {
mSpeechService.finishRecognizing();
}
}
};
// Resource caches
private int mColorHearing;
private int mColorNotHearing;
// View references
private TextView mStatus;
private TextView mText, mResult;
private Button editButton, clearButton;
private SharedPreferences settings;
private final ServiceConnection mServiceConnection = new ServiceConnection() {
#Override
public void onServiceConnected(ComponentName componentName, IBinder binder) {
mSpeechService = SpeechService.from(binder);
mSpeechService.addListener(mSpeechServiceListener);
mStatus.setVisibility(View.VISIBLE);
}
#Override
public void onServiceDisconnected(ComponentName componentName) {
mSpeechService = null;
}
};
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
getSupportActionBar().setDisplayHomeAsUpEnabled(true);
getSupportActionBar().setDisplayShowHomeEnabled(true);
final Resources resources = getResources();
final Resources.Theme theme = getTheme();
mColorHearing = ResourcesCompat.getColor(resources, R.color.status_hearing, theme);
mColorNotHearing = ResourcesCompat.getColor(resources, R.color.status_not_hearing, theme);
mStatus = (TextView) findViewById(R.id.status);
mText = (TextView) findViewById(R.id.text);
mResult = (TextView) findViewById(R.id.resultText);
editButton = (Button)findViewById(R.id.button1);
clearButton = (Button)findViewById(R.id.button2);
settings = getSharedPreferences("MyPreference", Context.MODE_PRIVATE);
clearButton.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View v) {
Sounds sounds = new Sounds(getApplicationContext());
if(settings.getBoolean("muteAble", false ) == true){
sounds.playSound();
}
mResult.setText("");
}
});
editButton.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View v) {
Sounds sounds = new Sounds(getApplicationContext());
if(settings.getBoolean("muteAble", false ) == true){
sounds.playSound();
}
Intent editIntent = new Intent(MainActivity.this, EditorActivity.class);
String forEditText = mResult.getText().toString();
editIntent.putExtra("forEdit", forEditText);
startActivity(editIntent);
}
});
}
#Override
public boolean onOptionsItemSelected(MenuItem item) {
int id = item.getItemId();
if(id == android.R.id.home){
this.finish();
}
return super.onOptionsItemSelected(item);
}
#Override
protected void onStart() {
super.onStart();
// Prepare Cloud Speech API
bindService(new Intent(this, SpeechService.class), mServiceConnection, BIND_AUTO_CREATE);
// Start listening to voices
if (ActivityCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO)
== PackageManager.PERMISSION_GRANTED) {
startVoiceRecorder();
} else if (ActivityCompat.shouldShowRequestPermissionRationale(this,
Manifest.permission.RECORD_AUDIO)) {
showPermissionMessageDialog();
} else {
ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.RECORD_AUDIO},
REQUEST_RECORD_AUDIO_PERMISSION);
}
}
#Override
protected void onStop() {
// Stop listening to voice
stopVoiceRecorder();
// Stop Cloud Speech API
mSpeechService.removeListener(mSpeechServiceListener);
unbindService(mServiceConnection);
mSpeechService = null;
super.onStop();
}
#Override
public void onRequestPermissionsResult(int requestCode, #NonNull String[] permissions,
#NonNull int[] grantResults) {
if (requestCode == REQUEST_RECORD_AUDIO_PERMISSION) {
if (permissions.length == 1 && grantResults.length == 1
&& grantResults[0] == PackageManager.PERMISSION_GRANTED) {
startVoiceRecorder();
} else {
showPermissionMessageDialog();
}
} else {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
}
}
private void startVoiceRecorder() {
if (mVoiceRecorder != null) {
mVoiceRecorder.stop();
}
mVoiceRecorder = new VoiceRecorder(mVoiceCallback);
mVoiceRecorder.start();
}
private void stopVoiceRecorder() {
if (mVoiceRecorder != null) {
mVoiceRecorder.stop();
mVoiceRecorder = null;
}
}
private void showPermissionMessageDialog() {
MessageDialogFragment
.newInstance(getString(R.string.permission_message))
.show(getSupportFragmentManager(), FRAGMENT_MESSAGE_DIALOG);
}
private void showStatus(final boolean hearingVoice) {
runOnUiThread(new Runnable() {
#Override
public void run() {
mStatus.setTextColor(hearingVoice ? mColorHearing : mColorNotHearing);
}
});
}
#Override
public void onMessageDialogDismissed() {
ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.RECORD_AUDIO},
REQUEST_RECORD_AUDIO_PERMISSION);
}
private final SpeechService.Listener mSpeechServiceListener =
new SpeechService.Listener() {
#Override
public void onSpeechRecognized(final String text, final boolean isFinal) {
if (isFinal) {
mVoiceRecorder.dismiss();
}
if (mText != null && !TextUtils.isEmpty(text)) {
runOnUiThread(new Runnable() {
#Override
public void run() {
if (isFinal) {
mText.setText(null);
mResult.append(" "+text.toString());
} else {
mText.setText(text);
}
}
});
}
}
};
}
Thank in advance for your help

Reusable AsyncTask by any activity

My goal is to have an AsyncTask that
can execute multiple times (one task at a time of course)
its current task can be cancelled
can be used by any activity
can execute many different tasks
does not have any problem with screen rotation (or phonecalls etc)
To achieve that i have created the classes shown below. But my experience with (and understanding of) threads is very limited. And since i don't know of any way to debug multiple threads, there is no way (for me) of knowing if this is going to work or not. So what i'm really asking is: Is this code ok?
And since there is no code that it is currently using this, here's an example use for it:
Data2Get d2g = new Data2Get(this, Data2Get.OpCountNumbers);
d2g.setParam("up2Num", String.valueOf(800));
LongOpsRunner.getLongOpsRunner().runOp(d2g);
So, here we go. This is the interface that every activity that wants to execute a long task (operation - op) should implement:
public interface LongOpsActivity {
public void onTaskCompleted(OpResult result);
}
This is a class to enclose any result of any task:
public class OpResult {
public LongOpsActivity forActivity;
public int opType;
public Object result;
public OpResult(LongOpsActivity forActivity, int opType, Object result){
this.forActivity = forActivity;
this.opType = opType;
this.result = result;
}
}
And finally the big part, the singleton async task class:
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import android.os.AsyncTask;
public class LongOpsRunner extends AsyncTask<Void, OpResult, Void> {
public class Data2Get implements Cloneable {
// one id for each operation
public static final int OpCountNumbers = 1;
public static final int OpCountLetters = 2;
public LongOpsActivity forActivity;
public int opType;
private HashMap<String, String> params = new HashMap<String, String>();
public Data2Get(LongOpsActivity forActivity, int opType) {
this.forActivity = forActivity;
this.opType = opType;
}
public void setParam(String key, String value) {
params.put(key, value);
}
public String getParam(String key) {
return params.get(key);
}
public void clearParams() {
params.clear();
}
#Override
protected Object clone() throws CloneNotSupportedException {
// deep clone
Data2Get myClone = (Data2Get) super.clone();
myClone.clearParams();
for (Entry<String, String> entry : params.entrySet()) {
myClone.setParam(new String(entry.getKey()), new String(entry.getValue()));
}
return myClone;
}
}
private class IntermediateResult extends OpResult {
public IntermediateResult(LongOpsActivity forActivity, int opType, Object result) {
super(forActivity, opType, result);
}
}
// not really needed
private class FinalResult extends OpResult {
public FinalResult(LongOpsActivity forActivity, int opType, Object result) {
super(forActivity, opType, result);
}
}
private final ReentrantLock lock = new ReentrantLock();
private final Condition executeOp = lock.newCondition();
private volatile boolean finished = false;
private volatile boolean waiting = true;
private volatile boolean shouldCancel = false;
private volatile boolean activityHasBeenNotified = true;
private Data2Get startingOpParams = null;
private Data2Get currentOpParams = null;
private FinalResult currentOpResult;
protected Void doInBackground(Void... nothing) {
try {
lock.lockInterruptibly();
do {
waiting = true;
while (waiting) {
executeOp.await();
}
shouldCancel = false;
activityHasBeenNotified = false;
boolean opCancelled = false;
try {
currentOpParams = (Data2Get) startingOpParams.clone();
} catch (CloneNotSupportedException cns) {
// do nothing
}
switch (currentOpParams.opType) {
case Data2Get.OpCountNumbers:
int numberCounter = 0;
int numLoopCount = 0;
while ((!opCancelled) & (numLoopCount <= 5000000)) {
if (!shouldCancel) {
numberCounter = (numberCounter + 1)
% Integer.parseInt(currentOpParams.getParam("up2Num"));
if (numberCounter == 0) {
numLoopCount++;
publishProgress(new IntermediateResult(
currentOpParams.forActivity,
currentOpParams.opType,
"Numbers loop count:" + numLoopCount));
}
} else {
opCancelled = true;
activityHasBeenNotified = true;
}
if (!opCancelled) {
currentOpResult = new FinalResult(
currentOpParams.forActivity,
currentOpParams.opType,
"Numbers loop completed.");
publishProgress(currentOpResult);
}
}
break;
case Data2Get.OpCountLetters:
int letterLoopCount = 0;
char ch = 'a';
while (!opCancelled & (letterLoopCount <= 5000000)) {
if (!shouldCancel) {
ch++;
if (Character.toString(ch).equals(currentOpParams.getParam("up2Letter"))) {
ch = 'a';
letterLoopCount++;
publishProgress(new IntermediateResult(
currentOpParams.forActivity,
currentOpParams.opType,
"Letters loop count:" + letterLoopCount));
}
} else {
opCancelled = true;
activityHasBeenNotified = true;
}
if (!opCancelled) {
currentOpResult = new FinalResult(
currentOpParams.forActivity,
currentOpParams.opType,
"Letters loop completed.");
publishProgress(currentOpResult);
}
}
break;
default:
}
} while (!finished);
lock.unlock();
} catch (InterruptedException e) {
// do nothing
}
return null;
}
public void cancelCurrentOp() {
shouldCancel = true;
}
#Override
protected void onProgressUpdate(OpResult... res) {
OpResult result = res[0];
if (result instanceof IntermediateResult) {
// normal progress update
// use result.forActivity to show something in the activity
} else {
notifyActivityOpCompleted(result);
}
}
public boolean currentOpIsFinished() {
return waiting;
}
public void runOp(Data2Get d2g) {
// Call this to run an operation
// Should check first currentOpIsFinished() most of the times
startingOpParams = d2g;
waiting = false;
executeOp.signal();
}
public void terminateAsyncTask() {
// The task will only finish when we call this method
finished = true;
lock.unlock(); // won't this throw an exception?
}
protected void onCancelled() {
// Make sure we clean up if the task is killed
terminateAsyncTask();
}
// if phone is rotated, use setActivity(null) inside
// onRetainNonConfigurationInstance()
// and setActivity(this) inside the constructor
// and all that only if there is an operation still running
public void setActivity(LongOpsActivity activity) {
currentOpParams.forActivity = activity;
if (currentOpIsFinished() & (!activityHasBeenNotified)) {
notifyActivityOpCompleted(currentOpResult);
}
}
private void notifyActivityOpCompleted(OpResult result) {
if (currentOpParams.forActivity != null) {
currentOpParams.forActivity.onTaskCompleted(result);
activityHasBeenNotified = true;
}
}
private static LongOpsRunner ref;
private LongOpsRunner() {
this.execute();
}
public static synchronized LongOpsRunner getLongOpsRunner() {
if (ref == null)
ref = new LongOpsRunner();
return ref;
}
public Object clone() throws CloneNotSupportedException {
throw new CloneNotSupportedException();
}
}
I hope someone helps with making this work, as it would be very useful not only for me, but many other people out there. Thank you.

Try Loaders. I switched from simple AsyncTasks to AsyncTaskLoaders and they solve lots of problems. If you implement a Loader as a standalone class, it would meet all of your requirements, especially when it comes to rotation which is the biggest issue with old AsyncTask.