I am developing a messaging application for Android and I am using an Azure Web API as my backend which connects to an Azure SQL DB. I am trying to get the service to allow users to log in and proceed through the application with an identity. I want to use Individual User Accounts to do this but I do not know how.
I have been through every piece of documentation on the internet and yet I am still at a loss on how to receive a token from the service and use it to access auhtorised resources.
If someone could please highlight the appropriate methods I need to call from the generated classes as well as methods I should write myself, that would be very helpful!
This is the Account Controller that was generated:
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Security.Claims;
using System.Security.Cryptography;
using System.Threading.Tasks;
using System.Web;
using System.Web.Http;
using Microsoft.AspNet.Identity;
using Microsoft.AspNet.Identity.EntityFramework;
using Microsoft.AspNet.Identity.Owin;
using Microsoft.Owin.Security;
using Microsoft.Owin.Security.Cookies;
using Microsoft.Owin.Security.OAuth;
using AcademicAssistant.Models;
using AcademicAssistant.Providers;
using AcademicAssistant.Results;
namespace AcademicAssistant.Controllers
{
[Authorize]
[RoutePrefix("api/Account")]
public class AccountController : ApiController
{
private const string LocalLoginProvider = "Local";
private ApplicationUserManager _userManager;
public AccountController()
{
}
public AccountController(ApplicationUserManager userManager,
ISecureDataFormat<AuthenticationTicket> accessTokenFormat)
{
UserManager = userManager;
AccessTokenFormat = accessTokenFormat;
}
public ApplicationUserManager UserManager
{
get
{
return _userManager ?? Request.GetOwinContext().GetUserManager<ApplicationUserManager>();
}
private set
{
_userManager = value;
}
}
public ISecureDataFormat<AuthenticationTicket> AccessTokenFormat { get; private set; }
// GET api/Account/UserInfo
[HostAuthentication(DefaultAuthenticationTypes.ExternalBearer)]
[Route("UserInfo")]
public UserInfoViewModel GetUserInfo()
{
ExternalLoginData externalLogin = ExternalLoginData.FromIdentity(User.Identity as ClaimsIdentity);
return new UserInfoViewModel
{
Email = User.Identity.GetUserName(),
HasRegistered = externalLogin == null,
LoginProvider = externalLogin != null ? externalLogin.LoginProvider : null
};
}
// POST api/Account/Logout
[Route("Logout")]
public IHttpActionResult Logout()
{
Authentication.SignOut(CookieAuthenticationDefaults.AuthenticationType);
return Ok();
}
// GET api/Account/ManageInfo?returnUrl=%2F&generateState=true
[Route("ManageInfo")]
public async Task<ManageInfoViewModel> GetManageInfo(string returnUrl, bool generateState = false)
{
IdentityUser user = await UserManager.FindByIdAsync(User.Identity.GetUserId());
if (user == null)
{
return null;
}
List<UserLoginInfoViewModel> logins = new List<UserLoginInfoViewModel>();
foreach (IdentityUserLogin linkedAccount in user.Logins)
{
logins.Add(new UserLoginInfoViewModel
{
LoginProvider = linkedAccount.LoginProvider,
ProviderKey = linkedAccount.ProviderKey
});
}
if (user.PasswordHash != null)
{
logins.Add(new UserLoginInfoViewModel
{
LoginProvider = LocalLoginProvider,
ProviderKey = user.UserName,
});
}
return new ManageInfoViewModel
{
LocalLoginProvider = LocalLoginProvider,
Email = user.UserName,
Logins = logins,
ExternalLoginProviders = GetExternalLogins(returnUrl, generateState)
};
}
// POST api/Account/ChangePassword
[Route("ChangePassword")]
public async Task<IHttpActionResult> ChangePassword(ChangePasswordBindingModel model)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
IdentityResult result = await UserManager.ChangePasswordAsync(User.Identity.GetUserId(), model.OldPassword,
model.NewPassword);
if (!result.Succeeded)
{
return GetErrorResult(result);
}
return Ok();
}
// POST api/Account/SetPassword
[Route("SetPassword")]
public async Task<IHttpActionResult> SetPassword(SetPasswordBindingModel model)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
IdentityResult result = await UserManager.AddPasswordAsync(User.Identity.GetUserId(), model.NewPassword);
if (!result.Succeeded)
{
return GetErrorResult(result);
}
return Ok();
}
// POST api/Account/AddExternalLogin
[Route("AddExternalLogin")]
public async Task<IHttpActionResult> AddExternalLogin(AddExternalLoginBindingModel model)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
Authentication.SignOut(DefaultAuthenticationTypes.ExternalCookie);
AuthenticationTicket ticket = AccessTokenFormat.Unprotect(model.ExternalAccessToken);
if (ticket == null || ticket.Identity == null || (ticket.Properties != null
&& ticket.Properties.ExpiresUtc.HasValue
&& ticket.Properties.ExpiresUtc.Value < DateTimeOffset.UtcNow))
{
return BadRequest("External login failure.");
}
ExternalLoginData externalData = ExternalLoginData.FromIdentity(ticket.Identity);
if (externalData == null)
{
return BadRequest("The external login is already associated with an account.");
}
IdentityResult result = await UserManager.AddLoginAsync(User.Identity.GetUserId(),
new UserLoginInfo(externalData.LoginProvider, externalData.ProviderKey));
if (!result.Succeeded)
{
return GetErrorResult(result);
}
return Ok();
}
// POST api/Account/RemoveLogin
[Route("RemoveLogin")]
public async Task<IHttpActionResult> RemoveLogin(RemoveLoginBindingModel model)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
IdentityResult result;
if (model.LoginProvider == LocalLoginProvider)
{
result = await UserManager.RemovePasswordAsync(User.Identity.GetUserId());
}
else
{
result = await UserManager.RemoveLoginAsync(User.Identity.GetUserId(),
new UserLoginInfo(model.LoginProvider, model.ProviderKey));
}
if (!result.Succeeded)
{
return GetErrorResult(result);
}
return Ok();
}
// GET api/Account/ExternalLogin
[OverrideAuthentication]
[HostAuthentication(DefaultAuthenticationTypes.ExternalCookie)]
[AllowAnonymous]
[Route("ExternalLogin", Name = "ExternalLogin")]
public async Task<IHttpActionResult> GetExternalLogin(string provider, string error = null)
{
if (error != null)
{
return Redirect(Url.Content("~/") + "#error=" + Uri.EscapeDataString(error));
}
if (!User.Identity.IsAuthenticated)
{
return new ChallengeResult(provider, this);
}
ExternalLoginData externalLogin = ExternalLoginData.FromIdentity(User.Identity as ClaimsIdentity);
if (externalLogin == null)
{
return InternalServerError();
}
if (externalLogin.LoginProvider != provider)
{
Authentication.SignOut(DefaultAuthenticationTypes.ExternalCookie);
return new ChallengeResult(provider, this);
}
ApplicationUser user = await UserManager.FindAsync(new UserLoginInfo(externalLogin.LoginProvider,
externalLogin.ProviderKey));
bool hasRegistered = user != null;
if (hasRegistered)
{
Authentication.SignOut(DefaultAuthenticationTypes.ExternalCookie);
ClaimsIdentity oAuthIdentity = await user.GenerateUserIdentityAsync(UserManager,
OAuthDefaults.AuthenticationType);
ClaimsIdentity cookieIdentity = await user.GenerateUserIdentityAsync(UserManager,
CookieAuthenticationDefaults.AuthenticationType);
AuthenticationProperties properties = ApplicationOAuthProvider.CreateProperties(user.UserName);
Authentication.SignIn(properties, oAuthIdentity, cookieIdentity);
}
else
{
IEnumerable<Claim> claims = externalLogin.GetClaims();
ClaimsIdentity identity = new ClaimsIdentity(claims, OAuthDefaults.AuthenticationType);
Authentication.SignIn(identity);
}
return Ok();
}
// GET api/Account/ExternalLogins?returnUrl=%2F&generateState=true
[AllowAnonymous]
[Route("ExternalLogins")]
public IEnumerable<ExternalLoginViewModel> GetExternalLogins(string returnUrl, bool generateState = false)
{
IEnumerable<AuthenticationDescription> descriptions = Authentication.GetExternalAuthenticationTypes();
List<ExternalLoginViewModel> logins = new List<ExternalLoginViewModel>();
string state;
if (generateState)
{
const int strengthInBits = 256;
state = RandomOAuthStateGenerator.Generate(strengthInBits);
}
else
{
state = null;
}
foreach (AuthenticationDescription description in descriptions)
{
ExternalLoginViewModel login = new ExternalLoginViewModel
{
Name = description.Caption,
Url = Url.Route("ExternalLogin", new
{
provider = description.AuthenticationType,
response_type = "token",
client_id = Startup.PublicClientId,
redirect_uri = new Uri(Request.RequestUri, returnUrl).AbsoluteUri,
state = state
}),
State = state
};
logins.Add(login);
}
return logins;
}
// POST api/Account/Register
[AllowAnonymous]
public async Task<IHttpActionResult> Register(RegisterBindingModel model)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
var user = new ApplicationUser() { UserName = model.Email, Email = model.Email };
IdentityResult result = await UserManager.CreateAsync(user, model.Password);
if (!result.Succeeded)
{
return GetErrorResult(result);
}
return Ok();
}
// POST api/Account/RegisterExternal
[OverrideAuthentication]
[HostAuthentication(DefaultAuthenticationTypes.ExternalBearer)]
[Route("RegisterExternal")]
public async Task<IHttpActionResult> RegisterExternal(RegisterExternalBindingModel model)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
var info = await Authentication.GetExternalLoginInfoAsync();
if (info == null)
{
return InternalServerError();
}
var user = new ApplicationUser() { UserName = model.Email, Email = model.Email };
IdentityResult result = await UserManager.CreateAsync(user);
if (!result.Succeeded)
{
return GetErrorResult(result);
}
result = await UserManager.AddLoginAsync(user.Id, info.Login);
if (!result.Succeeded)
{
return GetErrorResult(result);
}
return Ok();
}
protected override void Dispose(bool disposing)
{
if (disposing && _userManager != null)
{
_userManager.Dispose();
_userManager = null;
}
base.Dispose(disposing);
}
#region Helpers
private IAuthenticationManager Authentication
{
get { return Request.GetOwinContext().Authentication; }
}
private IHttpActionResult GetErrorResult(IdentityResult result)
{
if (result == null)
{
return InternalServerError();
}
if (!result.Succeeded)
{
if (result.Errors != null)
{
foreach (string error in result.Errors)
{
ModelState.AddModelError("", error);
}
}
if (ModelState.IsValid)
{
// No ModelState errors are available to send, so just return an empty BadRequest.
return BadRequest();
}
return BadRequest(ModelState);
}
return null;
}
private class ExternalLoginData
{
public string LoginProvider { get; set; }
public string ProviderKey { get; set; }
public string UserName { get; set; }
public IList<Claim> GetClaims()
{
IList<Claim> claims = new List<Claim>();
claims.Add(new Claim(ClaimTypes.NameIdentifier, ProviderKey, null, LoginProvider));
if (UserName != null)
{
claims.Add(new Claim(ClaimTypes.Name, UserName, null, LoginProvider));
}
return claims;
}
public static ExternalLoginData FromIdentity(ClaimsIdentity identity)
{
if (identity == null)
{
return null;
}
Claim providerKeyClaim = identity.FindFirst(ClaimTypes.NameIdentifier);
if (providerKeyClaim == null || String.IsNullOrEmpty(providerKeyClaim.Issuer)
|| String.IsNullOrEmpty(providerKeyClaim.Value))
{
return null;
}
if (providerKeyClaim.Issuer == ClaimsIdentity.DefaultIssuer)
{
return null;
}
return new ExternalLoginData
{
LoginProvider = providerKeyClaim.Issuer,
ProviderKey = providerKeyClaim.Value,
UserName = identity.FindFirstValue(ClaimTypes.Name)
};
}
}
private static class RandomOAuthStateGenerator
{
private static RandomNumberGenerator _random = new RNGCryptoServiceProvider();
public static string Generate(int strengthInBits)
{
const int bitsPerByte = 8;
if (strengthInBits % bitsPerByte != 0)
{
throw new ArgumentException("strengthInBits must be evenly divisible by 8.", "strengthInBits");
}
int strengthInBytes = strengthInBits / bitsPerByte;
byte[] data = new byte[strengthInBytes];
_random.GetBytes(data);
return HttpServerUtility.UrlTokenEncode(data);
}
}
#endregion
}
}
This is the message controller I wrote:
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.Entity;
using System.Data.Entity.Infrastructure;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using System.Web.Http;
using System.Web.Http.Description;
using AcademicAssistant.Models;
namespace AcademicAssistant.Controllers
{
[Authorize]
public class MessagesController : ApiController
{
private AcademicAssistantContext db = new AcademicAssistantContext();
// GET: api/Messages
[Authorize]
public IQueryable<Message> GetMessages()
{
return db.Messages;
}
// GET: api/Messages/5
[Authorize]
[ResponseType(typeof(Message))]
public async Task<IHttpActionResult> GetMessage(int id)
{
Message message = await db.Messages.FindAsync(id);
if (message == null)
{
return NotFound();
}
return Ok(message);
}
// PUT: api/Messages/5
[Authorize]
[ResponseType(typeof(void))]
public async Task<IHttpActionResult> PutMessage(int id, Message message)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
if (id != message.MessageID)
{
return BadRequest();
}
db.Entry(message).State = EntityState.Modified;
try
{
await db.SaveChangesAsync();
}
catch (DbUpdateConcurrencyException)
{
if (!MessageExists(id))
{
return NotFound();
}
else
{
throw;
}
}
return StatusCode(HttpStatusCode.NoContent);
}
// POST: api/Messages
[Authorize]
[ResponseType(typeof(Message))]
public async Task<IHttpActionResult> PostMessage(Message message)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
db.Messages.Add(message);
await db.SaveChangesAsync();
return CreatedAtRoute("DefaultApi", new { id = message.MessageID }, message);
}
// DELETE: api/Messages/5
[Authorize]
[ResponseType(typeof(Message))]
public async Task<IHttpActionResult> DeleteMessage(int id)
{
Message message = await db.Messages.FindAsync(id);
if (message == null)
{
return NotFound();
}
db.Messages.Remove(message);
await db.SaveChangesAsync();
return Ok(message);
}
[Authorize]
protected override void Dispose(bool disposing)
{
if (disposing)
{
db.Dispose();
}
base.Dispose(disposing);
}
[Authorize]
private bool MessageExists(int id)
{
return db.Messages.Count(e => e.MessageID == id) > 0;
}
}
}
Related
I am trying connect React Native with Mailcore2. Use mailcore2-android-4.aar (already build): http://d.etpan.org/mailcore2-deps/mailcore2-android/
I create my class and connect it with React Native App, as a result I can use debug in Android Studio 2021.3.1 (Dolphin)
My code:
public class Mail extends ReactContextBaseJavaModule {
private ReactContext mReactContext;
private IMAPFetchMessagesOperation fetchMessagesOp;
private java.util.List\<IMAPMessage\> messages;
private IMAPSession session;
private String status = "Failed";
public interface Callbacks {}
public Mail(ReactApplicationContext reactContext) {
super(reactContext);
this.mReactContext = reactContext;
session = new IMAPSession();
session.setUsername("my-email");
session.setPassword("my-client-secret");
session.setHostname("imap.yandex.ru");
session.setPort(993);
session.setConnectionType(ConnectionType.ConnectionTypeTLS);
}
#Override
public String getName() {
return "Mail";
}
#ReactMethod
public void updateMessages(Callback cb) {
fetchMessagesOp = session.fetchMessagesByNumberOperation(
"INBOX",
IMAPMessagesRequestKind.IMAPMessagesRequestKindHeaders |
IMAPMessagesRequestKind.IMAPMessagesRequestKindStructure,
IndexSet.indexSetWithRange(new Range(1, Range.RangeMax))
);
fetchMessagesOp.start(new OperationCallback() {
#Override
public void succeeded() {
status = "OK";
messages = fetchMessagesOp.messages();
cb.invoke(null, "");
}
#Override
public void failed(MailException ex) {
cb.invoke(ex.toString(), null);
}
});
}
#ReactMethod
public void getStatus(Callback cb) {
try {
if (Objects.equals(status, "OK")) {
cb.invoke(null, "OK");
} else {
cb.invoke("Failed", null);
}
} catch (Exception ex) {
cb.invoke(ex.toString(), null);
}
}
#ReactMethod
public void getMessages(Callback cb) {
if (messages != null) {
String tempString = "";
for (int indexMsg = 0; indexMsg < messages.size(); ++indexMsg) {
tempString += messages.get(indexMsg).toString();
}
cb.invoke(null, tempString);
} else {
cb.invoke("No messages", null);
}
}
}
My code for calling this class from React Native App:
let status = false;
function checkStatus() {
console.log('checkStatus');
NativeModules.Mail.getStatus((err: any, s: string) =\> {
if (err !== null) {
console.log(s);
} else {
status = true;
}
});
if (status === false) setTimeout(checkStatus, 5000);
}
function getMessages() {
NativeModules.Mail.updateMessages((err: any, s: string) =\> {
if (err === null) {
console.log(s);
} else {
console.log(err);
}
});
When I clicked on the button with included getMessages(), App crashed
I need to combine these two Data. They both have their own Fragment,Dao, Model and Repository. And both return different data from different tables.
ItemFavourite table stores id of the tables aboves Item and ItemMoto.
public LiveData<Resource<List<Item>>> getItemFavouriteData() {
return itemFavouriteData;
}
//Moto
public LiveData<Resource<List<ItemMoto>>> getItemFavouriteDataMoto() {
return itemFavouriteDataMoto;
}
This is how I tried it.
public class FavouriteViewModel extends PSViewModel {
private final LiveData<Resource<List<Item>>> itemFavouriteData;
private final LiveData<Resource<List<ItemMoto>>> itemFavouriteDataMoto;
private MutableLiveData<FavouriteViewModel.TmpDataHolder> itemFavouriteListObj = new
MutableLiveData<>();
private MutableLiveData<FavouriteMotoViewModel.TmpDataHolder> itemFavouriteListObjMoto = new
MutableLiveData<>();
#Inject
FavouriteViewModel(ItemRepository itemRepository, ItemMotoRepository itemMotoRepository) {
itemFavouriteData = Transformations.switchMap(itemFavouriteListObj, obj -> {
if (obj == null) {
return AbsentLiveData.create();
}
Utils.psLog("itemFavouriteData");
return itemRepository.getFavouriteList(Config.API_KEY, obj.userId, obj.offset);
});
itemFavouriteDataMoto = Transformations.switchMap(itemFavouriteListObjMoto, obj -> {
if (obj == null) {
return AbsentLiveData.create();
}
Utils.psLog("itemFavouriteData");
return itemMotoRepository.getFavouriteList(Config.API_KEY, obj.userId, obj.offset);
});
}
public LiveData<Resource<List<Item>>> getItemFavouriteData() {
return itemFavouriteData;
}
public LiveData<Resource<List<ItemMoto>>> getItemFavouriteDataMoto() {
return itemFavouriteDataMoto;
}
private static LiveData<Resource<List<Item>>> mergeDataSources(LiveData... sources) {
MediatorLiveData<Resource<List<Item>>> mergedSources = new MediatorLiveData();
for (LiveData source : sources) {
mergedSources.addSource(source, mergedSources::setValue);
}
return mergedSources;
}
public LiveData<Resource<List<Item>>> getFavourites() {
return mergeDataSources(
getItemFavouriteDataMoto(),
getItemFavouriteData());
}
}
From Fragment I observe the data like this:
LiveData<Resource<List<Item>>> news = favouriteViewModel.getFavourites();
if (news != null) {
news.observe(this, listResource -> {
if (listResource != null) {
switch (listResource.status) {
case LOADING:
// Loading State
// Data are from Local DB
if (listResource.data != null) {
//fadeIn Animation
fadeIn(binding.get().getRoot());
// Update the data
replaceData(listResource.data);
}
break;
case SUCCESS:
// Success State
// Data are from Server
if (listResource.data != null) {
// Update the data
replaceData(listResource.data);
}
favouriteViewModel.setLoadingState(false);
break;
case ERROR:
// Error State
favouriteViewModel.setLoadingState(false);
favouriteViewModel.forceEndLoading = true;
break;
default:
// Default
break;
}
} else {
// Init Object or Empty Data
if (favouriteViewModel.offset > 1) {
// No more data for this list
// So, Block all future loading
favouriteViewModel.forceEndLoading = true;
}
}
});
}
The only data I am getting are from Item table only.
Using mediator live data we can observe the 2 livedata.
val groupChatFeed: LiveData<List<Feed<*>>> = MediatorLiveData<List<Feed<*>>>().apply {
fun prepareDataAndSetStates(): List<Feed<*>> {
val data: MutableList<Feed<*>> = mutableListOf()
if (postList.value?.data?.isNullOrEmpty() == false) {
data.addAll(postList.value?.data ?: emptyList())
}
if (connectionRecommendations.value?.data?.isNullOrEmpty() == false) {
val recommendations = connectionRecommendations.value?.data?.toFeedItem()
data.add(recommendations)
}
return data
}
addSource(postList) {
value = prepareDataAndSetStates()
}
addSource(connectionRecommendations) {
value = prepareDataAndSetStates()
}
}
We are observing 2 different livedata postList and connectionRecommendations.
You can use MediatorLiveData and tuples, but you can technically also use this library I wrote for this specific purpose which does it for you, and solve it like this
import static com.zhuinden.livedatacombineutiljava.LiveDataCombineUtil.*;
private final LiveData<Pair<Resource<List<Item>>, Resource<List<ItemMoto>>>> favorites = combine(itemFavouriteData, itemFavouriteDataMoto, (favorites, favoriteMoto) -> {
return Pair.create(favorites, favoriteMoto);
});
public LiveData<Pair<Resource<List<Item>>, Resource<List<ItemMoto>>>> getFavorites() {
return favorites;
}
I have gone through many tutorials with API.AI But didn't get the exact solution. My requirement is simply:- user will send some command using voice or text and get that commands in my application and execute some method.
API.AI
Actions on Google
Tutorial of Google Assistant
First of all you need to train your model on API.AI to respond upon some text given to the model.
Some code with API.AI FYI:
//Initialize Service
private void initService(final LanguageConfig selectedLanguage) {
try {
final AIConfiguration.SupportedLanguages lang = AIConfiguration.SupportedLanguages.fromLanguageTag(selectedLanguage.getLanguageCode());
final AIConfiguration config = new AIConfiguration(selectedLanguage.getAccessToken(),
lang,
AIConfiguration.RecognitionEngine.System);
aiDataService = new AIDataService(this, config);
} catch (Exception e) {
e.printStackTrace();
}
}
//Send request method where you can put user typed text to get the result from API.AI
private void sendRequest(final String textToSend, final int flag) {
Log.w(TAG, "Sending" + textToSend);
final AsyncTask<String, Void, AIResponse> task = new AsyncTask<String, Void, AIResponse>() {
private AIError aiError;
#Override
protected void onPreExecute() {
super.onPreExecute();
showHideProgressBar(true);
if (mVoiceRecorder != null) {
mVoiceRecorder.pauseRecording();
}
}
#Override
protected AIResponse doInBackground(final String... params) {
final AIRequest request = new AIRequest();
String query = params[0];
String event = params[1];
if (!TextUtils.isEmpty(query))
request.setQuery(query);
if (!TextUtils.isEmpty(event)) {
request.setEvent(new AIEvent(event));
}
final String contextString = params[2];
RequestExtras requestExtras = null;
if (!TextUtils.isEmpty(contextString)) {
final List<AIContext> contexts = Collections.singletonList(new AIContext(contextString));
requestExtras = new RequestExtras(contexts, null);
}
try {
Log.i("API AI Request", "" + request.toString());
return aiDataService.request(request, requestExtras);
} catch (final AIServiceException e) {
aiError = new AIError(e);
return null;
}
}
#Override
protected void onPostExecute(final AIResponse response) {
showHideProgressBar(false);
speechSentStatus = false;
okSentStatus = false;
if (response != null) {
onResult(response, flag, textToSend);
} else {
onError(aiError);
}
}
};
if (flag == OPEN_COMPLAIN_CODE) {
task.execute("", Config.Events[0], Config.Events[0]);
} else if (flag == OPEN_DIAGNOSIS_CODE) {
task.execute("", Config.Events[1], Config.Events[1]);
} else if (flag == Constants.OPEN_MEDICATION_CODE) {
task.execute("", Config.Events[2], Config.Events[2]);
} else if (flag == Constants.OPEN_LABTEST_CODE) {
task.execute("", Config.Events[3], Config.Events[3]);
} else if (flag == Constants.COMPLAINTS_ADDED) {
task.execute("", Config.Events[0], Config.Events[0]);
} else if (flag == Constants.DIAGNOSIS_ADDED) {
task.execute("", Config.Events[1], Config.Events[1]);
} else {
task.execute(textToSend, null, "");
}
}
//Based on result you can handle the business logic
private void onResult(final AIResponse response, final int flag, final String textToSend) {
runOnUiThread(new Runnable() {
#Override
public void run() {
apiAiResponseCounter = apiAiResponseCounter + 1;
isLast = false;
final Result result = response.getResult();
Log.w(TAG, "" + result.getFulfillment().getSpeech());
if (flag == Constants.COMPLAINTS_ADDED) {
//method you want to execute on receiving certain text from model
send(textToSend.toLowerCase(), DONTTEXT);
} else if (flag == Constants.DIAGNOSIS_ADDED) {
send(textToSend.toLowerCase(), DONTTEXT);
} else {
String error = "";
final String speech = result.getFulfillment().getSpeech();
if (speech.contains("?")) {
if (!result.getAction().equalsIgnoreCase("input.unknown")) {
if (result.getAction().equalsIgnoreCase(Config.Actions[5]) && result.isActionIncomplete() == false) {
//DONOTHING
} else {
digiMessage(speech, YESNO);
}
} else {
digiMessage(speech, ChatMessageAdapter.OTHER_MESSAGE);
}
} else {
if (speech.equalsIgnoreCase("Please help me the intake duration of the medication")) {
digiMessage(speech, ChatMessageAdapter.DURATION);
} else if (speech.equalsIgnoreCase("Please provide the daily routine for the medication intake")) {
digiMessage(speech, ChatMessageAdapter.FREQUENCY);
} else {
digiMessage(speech, ChatMessageAdapter.OTHER_MESSAGE);
}
}
if (result.getAction().equalsIgnoreCase(Config.Actions[4]) || result.getAction().equalsIgnoreCase(Config.Actions[5])) {
if (result.isActionIncomplete() == true) {
playSpeech(speech);
} else {
speechBuffer = "";
speechBuffer = speech;
}
} else {
if (result.getAction().equalsIgnoreCase(Config.Actions[11])) {
isLast = true;
if (mVoiceRecorder != null) {
stopVoiceRecording();
}
} else {
playSpeech(speech);
}
}
}
}
});
if (flag == Constants.COMPLAINTS_ADDED || flag == Constants.DIAGNOSIS_ADDED) {
Log.w(TAG, "Skipped");
} else {
inflateUI(response.getResult());
}
}
Guys I'm using Unity and Watson to make and vr app that accepts voice commands. I'm using the ExampleStreaming.cs provided with the asset but the recognition is terrible. I change the language model to Portuguese on the code. I'm using Samsung galaxy s7 on gear vr with the mic plugged in. I guess the app is using the cellphone mic and not the plugged one. Any help here.
Here's the code
private int _recordingRoutine = 0;
private string _microphoneID = null;
private AudioClip _recording = null;
private int _recordingBufferSize = 1;
private int _recordingHZ = 22050;
private SpeechToText _service;
public GolemController GolemControllerObj;
void Start()
{
LogSystem.InstallDefaultReactors();
Runnable.Run(CreateService());
}
private IEnumerator CreateService()
{
// Create credential and instantiate service
Credentials credentials = null;
if (!string.IsNullOrEmpty(_username) && !string.IsNullOrEmpty(_password))
{
// Authenticate using username and password
credentials = new Credentials(_username, _password, _serviceUrl);
}
else if (!string.IsNullOrEmpty(_iamApikey))
{
// Authenticate using iamApikey
TokenOptions tokenOptions = new TokenOptions()
{
IamApiKey = _iamApikey,
IamUrl = _iamUrl
};
credentials = new Credentials(tokenOptions, _serviceUrl);
// Wait for tokendata
while (!credentials.HasIamTokenData())
yield return null;
}
else
{
throw new WatsonException("Please provide either username and password or IAM apikey to authenticate the service.");
}
_service = new SpeechToText(credentials);
_service.StreamMultipart = true;
_service.RecognizeModel="pt-BR_BroadbandModel";
Active = true;
StartRecording();
}
public bool Active
{
get { return _service.IsListening; }
set
{
if (value && !_service.IsListening)
{
_service.DetectSilence = true;
_service.EnableWordConfidence = true;
_service.EnableTimestamps = true;
_service.SilenceThreshold = 0.01f;
_service.MaxAlternatives = 0;
_service.EnableInterimResults = true;
_service.OnError = OnError;
_service.InactivityTimeout = -1;
_service.ProfanityFilter = false;
_service.SmartFormatting = true;
_service.SpeakerLabels = false;
_service.WordAlternativesThreshold = null;
_service.StartListening(OnRecognize, OnRecognizeSpeaker);
}
else if (!value && _service.IsListening)
{
_service.StopListening();
}
}
}
private void StartRecording()
{
if (_recordingRoutine == 0)
{
UnityObjectUtil.StartDestroyQueue();
_recordingRoutine = Runnable.Run(RecordingHandler());
}
}
private void StopRecording()
{
if (_recordingRoutine != 0)
{
Microphone.End(_microphoneID);
Runnable.Stop(_recordingRoutine);
_recordingRoutine = 0;
}
}
private void OnError(string error)
{
Active = false;
Log.Debug("ExampleStreaming.OnError()", "Error! {0}", error);
}
private IEnumerator RecordingHandler()
{
Log.Debug("ExampleStreaming.RecordingHandler()", "devices: {0}", Microphone.devices);
_recording = Microphone.Start(_microphoneID, true, _recordingBufferSize, _recordingHZ);
yield return null; // let _recordingRoutine get set..
if (_recording == null)
{
StopRecording();
yield break;
}
bool bFirstBlock = true;
int midPoint = _recording.samples / 2;
float[] samples = null;
while (_recordingRoutine != 0 && _recording != null)
{
int writePos = Microphone.GetPosition(_microphoneID);
if (writePos > _recording.samples || !Microphone.IsRecording(_microphoneID))
{
Log.Error("ExampleStreaming.RecordingHandler()", "Microphone disconnected.");
StopRecording();
yield break;
}
if ((bFirstBlock && writePos >= midPoint)
|| (!bFirstBlock && writePos < midPoint))
{
// front block is recorded, make a RecordClip and pass it onto our callback.
samples = new float[midPoint];
_recording.GetData(samples, bFirstBlock ? 0 : midPoint);
AudioData record = new AudioData();
record.MaxLevel = Mathf.Max(Mathf.Abs(Mathf.Min(samples)), Mathf.Max(samples));
record.Clip = AudioClip.Create("Recording", midPoint, _recording.channels, _recordingHZ, false);
record.Clip.SetData(samples, 0);
_service.OnListen(record);
bFirstBlock = !bFirstBlock;
}
else
{
// calculate the number of samples remaining until we ready for a block of audio,
// and wait that amount of time it will take to record.
int remaining = bFirstBlock ? (midPoint - writePos) : (_recording.samples - writePos);
float timeRemaining = (float)remaining / (float)_recordingHZ;
yield return new WaitForSeconds(timeRemaining);
}
}
yield break;
}
private void OnRecognize(SpeechRecognitionEvent result, Dictionary<string, object> customData)
{
if (result != null && result.results.Length > 0)
{
foreach (var res in result.results)
{
foreach (var alt in res.alternatives)
{
string text = string.Format("{0} ({1}, {2:0.00})\n", alt.transcript, res.final ? "Final" : "Interim", alt.confidence);
Log.Debug("ExampleStreaming.OnRecognize()", text);
ResultsField.text = text;
if(res.final == true)
{
GolemControllerObj.GolemActions(alt.transcript);
}
}
if (res.keywords_result != null && res.keywords_result.keyword != null)
{
foreach (var keyword in res.keywords_result.keyword)
{
Log.Debug("ExampleStreaming.OnRecognize()", "keyword: {0}, confidence: {1}, start time: {2}, end time: {3}", keyword.normalized_text, keyword.confidence, keyword.start_time, keyword.end_time);
}
}
if (res.word_alternatives != null)
{
foreach (var wordAlternative in res.word_alternatives)
{
Log.Debug("ExampleStreaming.OnRecognize()", "Word alternatives found. Start time: {0} | EndTime: {1}", wordAlternative.start_time, wordAlternative.end_time);
foreach(var alternative in wordAlternative.alternatives)
Log.Debug("ExampleStreaming.OnRecognize()", "\t word: {0} | confidence: {1}", alternative.word, alternative.confidence);
}
}
}
}
}
private void OnRecognizeSpeaker(SpeakerRecognitionEvent result, Dictionary<string, object> customData)
{
if (result != null)
{
foreach (SpeakerLabelsResult labelResult in result.speaker_labels)
{
Log.Debug("ExampleStreaming.OnRecognize()", string.Format("speaker result: {0} | confidence: {3} | from: {1} | to: {2}", labelResult.speaker, labelResult.from, labelResult.to, labelResult.confidence));
}
}
}
I need use SPP over Bluetooth on Xamarin.Android/iOS/Mono. This is the code I'm trying for Xamarin.Android but the behavior is the same if I go to iOS and Mono on Linux/Mac:
using System;
using System.Collections.Generic;
using System.Text;
using Android.Bluetooth;
using Java.Util;
using System.IO;
using System.Threading;
using PI.SDK.Devices.BC.Responses;
using System.Threading.Tasks;
namespace PI.SDK.Devices.BC
{
public class BluetoothDeviceConnectionChannel : IBCDeviceConnectionChannel
{
private Queue<ResponseBase> _dispatcher;
private bool _abort = false;
private BluetoothAdapter _adapter;
private BluetoothSocket _socket;
private BluetoothDevice _device;
private static UUID _uuid = UUID.FromString("00001101-0000-1000-8000-00805F9B34FB");
private StreamReader _reader;
private StreamWriter _writer;
private string _deviceAddress;
public event Action<string> Notify;
public bool IsOpen { get { return _socket.IsConnected; } }
public BluetoothDeviceConnectionChannel(string deviceAddress)
{
_adapter = BluetoothAdapter.DefaultAdapter;
if (_adapter == null)
throw new PIDeviceManagerException("Bluetooth is not supported on this Android device");
_deviceAddress = deviceAddress;
}
public BluetoothDeviceConnectionChannel(BluetoothDevice device) : this(device.Address) { }
public void Close()
{
_socket.Close();
}
public bool Open()
{
if (!_adapter.IsEnabled)
{
throw new PIDeviceManagerException("Bluetooth is not enabled");
}
_adapter.CancelDiscovery();
_device = _adapter.GetRemoteDevice(_deviceAddress);
_socket = _device.CreateRfcommSocketToServiceRecord(_uuid);
_socket.Connect();
if (_socket.IsConnected)
{
_reader = new StreamReader(_socket.InputStream, Encoding.GetEncoding("Windows-1252"));
_writer = new StreamWriter(_socket.OutputStream, Encoding.GetEncoding("Windows-1252"));
_dispatcher = new Queue<ResponseBase>();
Task.Factory.StartNew(() => ReceiveData());
return true;
}
return false;
}
public void ReceiveData()
{
while (_socket != null && _socket.IsConnected)
{
var data = _reader.ReadToEnd();
if (string.IsNullOrWhiteSpace(data))
continue;
var dataBuffer = data.ToCharArray();
var synBuilder = new StringBuilder();
foreach (var c in dataBuffer)
{
switch (c)
{
case ControlChars.NACK:
case ControlChars.EOT:
#if DEBUG
System.Diagnostics.Debug.WriteLine($"[PINPAD -> APP] {c.ToString().Dump()}");
#endif
_abort = true;
return;
case ControlChars.ACK:
#if DEBUG
System.Diagnostics.Debug.WriteLine($"[PINPAD -> APP] {c.ToString().Dump()}");
#endif
continue;
case ControlChars.SYN:
synBuilder.Append(c);
break;
case ControlChars.ETB:
synBuilder.Append(c);
var cmdResponse = synBuilder.ToString();
#if DEBUG
System.Diagnostics.Debug.WriteLine($"[PINPAD -> APP] {cmdResponse.Dump()}");
#endif
var response = CommandResponseParser.Parse(cmdResponse);
if (response != null)
{
_dispatcher.Enqueue(response);
}
return;
default:
synBuilder.Append(c);
break;
}
}
}
}
public ResponseBase SendData(string data)
{
_abort = false;
try
{
_writer.Write(data);
}
catch
{
throw new PIException("Unable to send data to device");
}
#if DEBUG
System.Diagnostics.Debug.WriteLine($"[APP -> PINPAD] {data.Dump()}");
#endif
if (data[0] == ControlChars.CAN)
{
Thread.Sleep(100);
return null;
}
while (!_abort)
{
if (_dispatcher.Count > 0)
{
var response = _dispatcher.Dequeue();
if (response != null)
{
if (response is PPNotifyResponse)
{
if (Notify != null && Notify.GetInvocationList().Length > 0)
Notify(response.Message);
continue;
}
return response;
}
}
}
throw new InvalidOperationException("invalidData");
}
public ResponseBase SendData(CommandBase data)
{
var cmd = data.ToBCCommandString();
return SendData(cmd);
}
}
}
I want to achieve he same behavior of the code bellow for Windows using SerialPort class and a COMxxx port where this port, is nothing more than a Serial-Over-Bluetooth COM with the target device.
using PI.SDK.Devices.BC.Responses;
using System;
using System.Collections.Generic;
using System.IO.Ports;
using System.Text;
using System.Threading;
namespace PI.SDK.Devices.BC
{
public class SerialDeviceConnectionChannel : IBCDeviceConnectionChannel
{
private SerialPort _port;
private Queue<ResponseBase> _dispatcher;
private bool _abort = false;
public event Action<string> Notify;
public bool IsOpen { get { return _port.IsOpen; } }
public SerialDeviceConnectionChannel(string port)
{
_port = new SerialPort(port, 19200, Parity.None, 8, StopBits.One);
_port.ReadTimeout = 3 * 1000;
_port.WriteTimeout = 3 * 1000;
_port.Encoding = Encoding.GetEncoding(1252);
_port.DataReceived += DataReceived;
}
public void Close()
{
_port.Close();
}
public bool Open()
{
while (true)
{
try
{
_port.Open();
_port.DiscardInBuffer();
_port.DiscardInBuffer();
break;
}
catch { Console.WriteLine($"Trying to connect to {_port}"); }
}
_dispatcher = new Queue<ResponseBase>();
return _port.IsOpen;
}
private void DataReceived(object sender, SerialDataReceivedEventArgs e)
{
var data = _port.ReadExisting();
var dataBuffer = data.ToCharArray();
var synBuilder = new StringBuilder();
foreach (var c in dataBuffer)
{
switch (c)
{
case ControlChars.NACK:
case ControlChars.EOT:
#if DEBUG
Console.WriteLine($"[PINPAD -> APP] {c.ToString().Dump()}");
#endif
_abort = true;
return;
case ControlChars.ACK:
#if DEBUG
Console.WriteLine($"[PINPAD -> APP] {c.ToString().Dump()}");
#endif
continue;
case ControlChars.SYN:
synBuilder.Append(c);
break;
case ControlChars.ETB:
synBuilder.Append(c);
var cmdResponse = synBuilder.ToString();
#if DEBUG
Console.WriteLine($"[PINPAD -> APP] {cmdResponse.Dump()}");
#endif
var response = CommandResponseParser.Parse(cmdResponse);
if (response != null)
{
_dispatcher.Enqueue(response);
}
return;
default:
synBuilder.Append(c);
break;
}
}
}
public ResponseBase SendData(string data)
{
_abort = false;
try
{
_port.Write(data);
}
catch
{
throw new PIException("Unable to send data to device");
}
#if DEBUG
Console.WriteLine($"[APP -> PINPAD] {data.Dump()}");
#endif
if (data[0] == ControlChars.CAN)
{
Thread.Sleep(100);
return null;
}
while (!_abort)
{
if (_dispatcher.Count > 0)
{
var response = _dispatcher.Dequeue();
if (response != null)
{
if (response is PPNotifyResponse)
{
if (Notify != null && Notify.GetInvocationList().Length > 0)
Notify(response.Message);
continue;
}
return response;
}
}
}
throw new InvalidOperationException("invalidData");
}
public ResponseBase SendData(CommandBase data)
{
var cmd = data.ToBCCommandString();
return SendData(cmd);
}
}
}
This code hangs when calling _reader.ReadToEnd(); on all other platforms except on Windows. Looks like I'm not getting the response back somehow.
Note that the Android/iOS/Mono version, must respect the serial connection configuration as stated on the ctor of the Windows classe, and the Encoding for the messages and the Serial communication, must be Windows-1252.
Any help pointing the mistakes or how to get it work the same way as on Windows would be appreciated, since there is no SerialPort class, I'm kinda lost on those devices and looks like bluetooth comm is something obscure when talking about xamarin/mobile devices.
Thanks!
Best regards,
Gutemberg
Found the problem. While in Windows the calls to serialPort.Read() can be async and on other thread, in Android/iOS/Mono, it can't.
If I start reading just after the _writer.Write(), in other words, on the same thread, I can get it working just fine.