Develop Reference

Android WebRTC remote stream not displaying on SurfaceView, getting 0 frames

I'm attempting to stream my webcam locally from a webpage on my computer to an Android app (native WebRTC). I'm using WebRTC for the peer connection and NodeJS with Socket.io for signaling. When I initiate the video stream all sdp appears to be set correctly, but no track plays on my Andorid surface view. All I get is a black screen and this console output:
2020-08-19 13:23:19.667 30492-31575/com.example.demowebrtcclient I/org.webrtc.Logging: EglRenderer: video_viewDuration: 4050 ms. Frames received: 0. Dropped: 0. Rendered: 0. Render fps: .0. Average render time: NA. Average swapBuffer time: NA.
Here's my code, sorry there's so much I just don't know where the root of the problem is.
Peer 1 (Webpage, JS)
navigator.mediaDevices.getUserMedia({audio: true, video: true})
.then(function(s) {
stream = s;
video.srcObject = stream;
video.play();
});
function startStream() {
socket.emit('broadcaster');
}
socket.on('answer', function(id, description) {
let RTCDescription = description;
if (isAndroid) {
RTCDescription = new RTCSessionDescription();
RTCDescription.sdp = description;
RTCDescription.type = "answer";
console.log("Answer received");
}
console.log(RTCDescription);
peerConnection.setRemoteDescription(RTCDescription);
});
socket.on('watcher', function(id) {
peerConnection = new RTCPeerConnection(config);
// peerConnections[id] = peerConnection;
stream.getTracks().forEach(track => peerConnection.addTrack(track, stream));
peerConnection.createOffer()
.then(function(sdp) {
peerConnection.setLocalDescription(sdp);
})
.then(function() {
socket.emit('offer', id, peerConnection.localDescription);
})
peerConnection.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('candidate', id, event.candidate);
}
};
});
socket.on('candidate', function(id, candidate) {
console.log("Candidate recieved");
peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
});
socket.on('close', function(id) {
peerConnection.close();
delete peerConnection;
});
Peer 2 (Android, Java)
private void initializePeerConnectionFactory() {
PeerConnectionFactory.InitializationOptions initOptions = PeerConnectionFactory.InitializationOptions.builder(getApplicationContext())
.setEnableInternalTracer(true)
.setFieldTrials("WebRTC-H264HighProfile/Enabled/")
.createInitializationOptions();
PeerConnectionFactory.initialize(initOptions);
VideoEncoderFactory defaultVideoEncoderFactory = new DefaultVideoEncoderFactory(rootEglBaseContext, /* enableIntelVp8Encoder */true, /* enableH264HighProfile */true);
VideoDecoderFactory defaultVideoDecoderFactory = new DefaultVideoDecoderFactory(rootEglBaseContext);
PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
options.disableEncryption = true;
options.disableNetworkMonitor = true;
factory = PeerConnectionFactory.builder()
.setVideoEncoderFactory(defaultVideoEncoderFactory)
.setVideoDecoderFactory(defaultVideoDecoderFactory)
.setOptions(options)
.createPeerConnectionFactory();
}
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
rootEglBase = EglBase.create();
rootEglBaseContext = rootEglBase.getEglBaseContext();
videoView = findViewById(R.id.video_view);
videoView.setMirror(true);
videoView.setEnableHardwareScaler(true);
videoView.init(rootEglBaseContext, null);
videoSink = new ProxyVideoSink();
iceServers = new ArrayList<>();
stunServer = (PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer());
iceServers.add(stunServer);
executor = Executors.newSingleThreadScheduledExecutor();
mediaConstraints = new MediaConstraints();
mSocket.on(Socket.EVENT_CONNECT, onConnect);
mSocket.on("offer", handleOffer);
mSocket.on("broadcaster", onBroadcast);
mSocket.on("candidate", onCandidate);
mSocket.connect();
initializePeerConnectionFactory();
}
private Emitter.Listener handleOffer = new Emitter.Listener() {
#Override
public void call(final Object... args) {
Log.d("socket", "Offer recieved");
peerConnection = factory.createPeerConnection(iceServers, observer);
String id = (String) args[0];
JSONObject data = (JSONObject) args[1];
peerConnection.setRemoteDescription(new SdpAdapter("setremote") {
#Override
public void onSetSuccess() {
peerConnection.createAnswer(new SdpAdapter("createanswer") {
#Override
public void onCreateSuccess(final SessionDescription sdp) {
super.onCreateSuccess(sdp);
Log.d("socket", "Session description " + sdp.toString() + " created");
SdpAdapter local = new SdpAdapter("setlocal") {
#Override
public void onSetSuccess() {
super.onSetSuccess();
mSocket.emit("answer" , id, peerConnection.getLocalDescription().description);
}
};
peerConnection.setLocalDescription(local, sdp);
}
}, mediaConstraints);
}
}, new SessionDescription(SessionDescription.Type.OFFER, data.optString("sdp")));
}
};
private Emitter.Listener onConnect = new Emitter.Listener() {
#Override
public void call(final Object... args) {
Log.d("socket", "Socket connected");
mSocket.emit("watcher");
}
};
private Emitter.Listener onBroadcast = new Emitter.Listener() {
#Override
public void call(final Object... args) {
Log.d("socket", "Received broadcast request");
mSocket.emit("watcher");
}
};
private Emitter.Listener onCandidate = new Emitter.Listener() {
#Override
public void call(final Object... args) {
Log.d("ice", "New ice candidate recieved");
JSONObject data = (JSONObject) args[1];
IceCandidate candidate = new IceCandidate(data.optString("sdpMid"), Integer.parseInt(data.optString("sdpMLineIndex")), data.optString("candidate"));
peerConnection.addIceCandidate(candidate);
}
};
public class SdpAdapter implements SdpObserver {
private String name;
public SdpAdapter(String name) {
this.name = name;
}
#Override
public void onCreateSuccess(SessionDescription sessionDescription) {
Log.d("socket", "SDP create success for " + name );
}
#Override
public void onSetSuccess() {
Log.d("socket", "SDP set success for " + name);
}
#Override
public void onCreateFailure(String s) {
Log.d("socket", "SDP create failure for" + s);
}
#Override
public void onSetFailure(String s) {
Log.d("socket", "SDP set failure for" + s);
}
};
public static class ProxyVideoSink implements VideoSink {
private VideoSink mTarget;
#Override
synchronized public void onFrame(VideoFrame frame) {
if (mTarget == null) {
Log.d("socket", "Dropping frame in proxy because target is null.");
return;
}
mTarget.onFrame(frame);
}
synchronized void setTarget(VideoSink target) {
this.mTarget = target;
}
}
Observer observer = new Observer() {
#Override
public void onIceCandidate(IceCandidate iceCandidate) {
Log.d("ice", iceCandidate.toString());
mSocket.emit("candidate", iceCandidate);
}
#Override
public void onAddStream(MediaStream stream) {
executor.execute(new Runnable() {
#Override
public void run() {
Log.d("socket", "Stream added");
Log.d("tracks", "Getting tracks");
VideoTrack remoteVideoTrack = (VideoTrack)stream.videoTracks.get(0);
AudioTrack remoteAudioTrack = (AudioTrack)stream.audioTracks.get(0);
Log.d("tracks", "Enabling tracks");
remoteAudioTrack.setEnabled(true);
remoteVideoTrack.setEnabled(true);
Log.d("tracks", "Adding sink");
videoSink.setTarget(videoView);
remoteVideoTrack.addSink(videoSink);
peerConnection.getStats(reports -> {
for (StatsReport report : reports) {
Log.d("Stats", "Stats: " + report.toString());
}
}, null);
}
});
}
};
}

Disabling encryption in Android will break interoperability with the browser, unless you start them in a special (insecure) mode.
options.disableEncryption = true;
A good point to get started with the Android native WebRTC API is by looking at the example App code:
https://chromium.googlesource.com/external/webrtc/+/refs/heads/master/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java
I'll recommend to add more logs and extend the description / question.
Recommended procedure when debugging a WebRTC issue:
Did ICE succeed? (iceConnectionState is connected) If not, look at the gathered and received candidates. Check involved firewall configurations, NAT and TURN.
Do you receive packets but no frames can be rendered? Check your codecs and encoders / decoders.

Socket.IO-client Java with Header

I have socket server that need user-id in order to connect. Therefore, I need to set extraHeader to connection constructor. I tried the following code but there's no luck. Please help me get through, thanks.
Add extra header 'x-user-id': userId to connection constructor
private Socket mSocket;
{
try {
IO.Options opts = new IO.Options();
mSocket = IO.socket(socket_url, opts);
mSocket.io().on(Manager.EVENT_TRANSPORT, new Emitter.Listener() {
#Override
public void call(Object... args) {
Transport transport = (Transport)args[0];
transport.on(Transport.EVENT_REQUEST_HEADERS, new Emitter.Listener() {
#Override
public void call(Object... args) {
#SuppressWarnings("unchecked")
Map<String, List<String>> headers = (Map<String, List<String>>)args[0];
// modify request headers
headers.put("x-user-id", Arrays.asList("5B59F68B7B7811E88C3E52964BF487E4"));
}
}).on(Transport.EVENT_RESPONSE_HEADERS, new Emitter.Listener() {
#Override
public void call(Object... args) {
}
});
}
});
} catch (URISyntaxException e) {}
}
This is my swift code that is works.
private init(){
self.socketManager = SocketManager(socketURL: URL(string:"\(MAIN_URL)/message")!, config: [.log(true),.extraHeaders(["x-user-id":AppManager.instance.userId])])
self.socket = socketManager?.defaultSocket
}

You can use also stomp client provided by Spring framework.
To compile and run the following method you should add 'spring-messaging' and 'spring-websocket' maven dependencies to your project.
public ListenableFuture<StompSession> connect() {
List<Transport> transports = new ArrayList<>(2);
transports.add(new WebSocketTransport(new StandardWebSocketClient()));
transports.add(new RestTemplateXhrTransport());
SockJsClient sockJsClient = new SockJsClient(transports);
sockJsClient.setMessageCodec(new Jackson2SockJsMessageCodec());
WebSocketStompClient stompClient = new WebSocketStompClient(sockJsClient);
stompClient.setMessageConverter(new MappingJackson2MessageConverter());
WebSocketHttpHeaders headers = new WebSocketHttpHeaders();
headers.add("x-user-id", "5B59F68B7B7811E88C3E52964BF487E4");
return stompClient.connect(socket_url, headers, new StompSessionHandlerAdapter() {});
}

The problem is fixed on server side. Here is my configuration.
Add this >>>
var server = app.listen(3000, {// options can go here
transports: ['xhr-polling']
});
io.use((socket, next) => {
let clientId = socket.handshake.headers['x-user-id'];
});

How to get Socket Id in Socket.io on Android?

I want to get the id of my socket running in Android. Everything is create normally and is able to send data back and forth but I need to get the Androids socket id for usage throughout the entire application. I have tried to find a solution but either they were outdated or does not work. Below is my latest attempt at solving this issue. With this attempt it errors saying that there are no elements in args[0]. I have also tried mSocket.id(); but that returns nothing.
public class GameSocket {
private Socket mSocket = null;
private static GameSocket mInstance;
public synchronized static GameSocket getInstance() {
if (mInstance == null) {
mInstance = new GameSocket();
}
return mInstance;
}
public void initialize() {
mSocket = IO.socket(URI.create(SERVER_URI));
mSocket.off();
mSocket.on(Socket.EVENT_CONNECT, new Emitter.Listener() {
#Override
public void call(Object... args) {
Log.d(GameSocket.class.getSimpleName(), "Event connection...");
mSocketId = args[0].toString();
Log.d("CONNECT ID", mSocketId);
}
})
.on(Socket.EVENT_ERROR, new Emitter.Listener() {
#Override
public void call(Object... args) {
if (args != null) {
Log.e(GameSocket.class.getSimpleName(), "Socket Event Error: " + args[0].toString());
}
}
});
}
}

Server response Timeout function in parse.com android

I am working on parse.com android application. Like in normal http request if we don't get the response from the server in a certain time then it will show time out functionality. For example my following code will works for http request time out:
HttpsURLConnection connection = null;
connection = (HttpsURLConnection) serverAddress.openConnection();
connection.setRequestMethod("GET");
connection.setDoOutput(true);
connection.setHostnameVerifier(DO_NOT_VERIFY);
connection.setReadTimeout(3*1000);
Now i want the same functionality in parse.com, if i can't get the response from the parse server because of any reason like internet connectivity issue, then it will gives me timeout.
Thanks in advance.

Parse.com SDK does not have such functionality which you have described at your question.
But you can do some trick, for example, at ParseQuery exists method cancel(). So if you want timeout less then standard one, you can run query in background and and wait until or query callback will delivery result of it, or your implementation of timeout will cancel query running.
Update:
public class TimeoutQuery<T extends ParseObject> {
private ParseQuery<T> mQuery;
private final long mTimeout;
private FindCallback<T> mCallback;
private final Object mLock = new Object();
private final Thread mThread;
public TimeoutQuery(ParseQuery<T> query, long timeout) {
mQuery = query;
mTimeout = timeout;
mThread = new Thread() {
#Override
public void run() {
if (isInterrupted()) return;
try {
Thread.sleep(mTimeout);
} catch (InterruptedException e) {
return;
}
cancelQuery();
}
};
}
private void cancelQuery() {
synchronized (mLock) {
if (null == mQuery) return; // it's already canceled
mQuery.cancel();
new Handler(Looper.getMainLooper()).post(new Runnable() {
#Override
public void run() {
mCallback.done(Collections.<T>emptyList(), new ParseException(ParseException.TIMEOUT, ""));
}
});
}
}
public void findInBackground(final FindCallback<T> callback) {
synchronized (mLock) {
mCallback = callback;
mQuery.findInBackground(new FindCallback<T>() {
#Override
public void done(List<T> ts, ParseException e) {
synchronized (mLock) {
mThread.interrupt();
mQuery = null;
mCallback.done(ts, e);
}
}
});
mThread.start();
}
}
}
Usage:
ParseQuery<ParseObject> query = ParseQuery.getQuery("Test");
new TimeoutQuery<>(query, 1000).findInBackground(new FindCallback<ParseObject>() {
...
});

Servlet WebSocket Acessed by Android

I'm trying learn how to use the websocket and make a simple servlet for being connected with Android but I don't get it.
The index.jsp :
var ws = new WebSocket("ws://" + document.location.host + "/myws/ServletWS");
ws.onopen = function() { };
ws.onclose = function() { };
ws.onerror = function() { log("ERROR"); };
ws.onmessage = function(data) { var message = data.data; };
function sendMessage(msg) { ws.send(msg); }
How or where I receive the data from client?
Now on the servlet:
#Override protected StreamInbound createWebSocketInbound(String subProtocol, HttpServletRequest request) {
return new ConnectionWS();
}
class ConnectionWS extends MessageInbound {
private WsOutbound outbound;
#Override protected void onOpen(WsOutbound outbound) {
this.outbound = outbound;
}
#Override protected void onTextMessage(CharBuffer msg) throws IOException {
String message = msg.toString();
ServletWS.processData(message);
}
public void sendMessage(String message) {
CharBuffer cb = CharBuffer.wrap(message);
try {
outbound.writeTextMessage(cb);
} catch (IOException e) {}
}
}
public void processData(String message){
here I have to call the sendMessage with the answer to the client
}
I have saw a lot of examples on web but all of then about chat.
Thanks a lot for any help.

I understand that, you have a basic knowledge about tomcat configuration as well as java Servlet programming. As WekSocket is newly introduced in Tomcat, you may need to use latest tomcat version to implement WebSocket over it. I have used Apache Tomcat 7.0.42 for it.
So here we go. First, create a Servlet which will just create a new WebSocket for the request. You may need to modify it, if you want to go by session rather than request. Here is sample code.
import org.apache.catalina.websocket.StreamInbound;
import org.apache.catalina.websocket.WebSocketServlet;
public class WsChatServlet extends WebSocketServlet {
private static final long serialVersionUID = 1456546233L;
#Override
protected StreamInbound createWebSocketInbound(String protocol,
HttpServletRequest request) {
return new IncomingMessageHandler();
}
}
Now, create a Message Handler class which will handle each WebSocket stream independently. and that's it !
public class IncomingMessageHandler extends MessageInbound {
private WsOutbound myoutbound;
public IncomingMessageHandler() {
}
#Override
public void onOpen(WsOutbound outbound) {
logger.info("Open Client.");
this.myoutbound = outbound;
}
#Override
public void onClose(int status) {
logger.info("Close Client.");
}
/**
* Called when received plain Text Message
*/
#Override
public void onTextMessage(CharBuffer cb) throws IOException {
}
/**
* We can use this method to pass image binary data, eventually !
*/
#Override
public void onBinaryMessage(ByteBuffer bb) throws IOException {
}
public synchronized void sendTextMessage(String message) {
try {
CharBuffer buffer = CharBuffer.wrap(message);
this.getMyoutbound().writeTextMessage(buffer);
this.getMyoutbound().flush();
} catch (IOException e) {
}
}
/**
* Set websocket connection timeout in milliseconds,
* -1 means never
*/
#Override
public int getReadTimeout() {
return -1;
}
public WsOutbound getMyoutbound() {
return myoutbound;
}
public void setMyoutbound(WsOutbound myoutbound) {
this.myoutbound = myoutbound;
}
}

If not misunderstood and you want to use web sockets on Android then recommended API for you is jWebSocket.
Get it here, hopefully it already provides you APIs for a lot of the work that you need to do or even more.
http://jwebsocket.org/