Develop Reference

Android WebRTC remote stream not displaying on SurfaceView, getting 0 frames

I'm attempting to stream my webcam locally from a webpage on my computer to an Android app (native WebRTC). I'm using WebRTC for the peer connection and NodeJS with Socket.io for signaling. When I initiate the video stream all sdp appears to be set correctly, but no track plays on my Andorid surface view. All I get is a black screen and this console output:
2020-08-19 13:23:19.667 30492-31575/com.example.demowebrtcclient I/org.webrtc.Logging: EglRenderer: video_viewDuration: 4050 ms. Frames received: 0. Dropped: 0. Rendered: 0. Render fps: .0. Average render time: NA. Average swapBuffer time: NA.
Here's my code, sorry there's so much I just don't know where the root of the problem is.
Peer 1 (Webpage, JS)
navigator.mediaDevices.getUserMedia({audio: true, video: true})
.then(function(s) {
stream = s;
video.srcObject = stream;
video.play();
});
function startStream() {
socket.emit('broadcaster');
}
socket.on('answer', function(id, description) {
let RTCDescription = description;
if (isAndroid) {
RTCDescription = new RTCSessionDescription();
RTCDescription.sdp = description;
RTCDescription.type = "answer";
console.log("Answer received");
}
console.log(RTCDescription);
peerConnection.setRemoteDescription(RTCDescription);
});
socket.on('watcher', function(id) {
peerConnection = new RTCPeerConnection(config);
// peerConnections[id] = peerConnection;
stream.getTracks().forEach(track => peerConnection.addTrack(track, stream));
peerConnection.createOffer()
.then(function(sdp) {
peerConnection.setLocalDescription(sdp);
})
.then(function() {
socket.emit('offer', id, peerConnection.localDescription);
})
peerConnection.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('candidate', id, event.candidate);
}
};
});
socket.on('candidate', function(id, candidate) {
console.log("Candidate recieved");
peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
});
socket.on('close', function(id) {
peerConnection.close();
delete peerConnection;
});
Peer 2 (Android, Java)
private void initializePeerConnectionFactory() {
PeerConnectionFactory.InitializationOptions initOptions = PeerConnectionFactory.InitializationOptions.builder(getApplicationContext())
.setEnableInternalTracer(true)
.setFieldTrials("WebRTC-H264HighProfile/Enabled/")
.createInitializationOptions();
PeerConnectionFactory.initialize(initOptions);
VideoEncoderFactory defaultVideoEncoderFactory = new DefaultVideoEncoderFactory(rootEglBaseContext, /* enableIntelVp8Encoder */true, /* enableH264HighProfile */true);
VideoDecoderFactory defaultVideoDecoderFactory = new DefaultVideoDecoderFactory(rootEglBaseContext);
PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
options.disableEncryption = true;
options.disableNetworkMonitor = true;
factory = PeerConnectionFactory.builder()
.setVideoEncoderFactory(defaultVideoEncoderFactory)
.setVideoDecoderFactory(defaultVideoDecoderFactory)
.setOptions(options)
.createPeerConnectionFactory();
}
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
rootEglBase = EglBase.create();
rootEglBaseContext = rootEglBase.getEglBaseContext();
videoView = findViewById(R.id.video_view);
videoView.setMirror(true);
videoView.setEnableHardwareScaler(true);
videoView.init(rootEglBaseContext, null);
videoSink = new ProxyVideoSink();
iceServers = new ArrayList<>();
stunServer = (PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer());
iceServers.add(stunServer);
executor = Executors.newSingleThreadScheduledExecutor();
mediaConstraints = new MediaConstraints();
mSocket.on(Socket.EVENT_CONNECT, onConnect);
mSocket.on("offer", handleOffer);
mSocket.on("broadcaster", onBroadcast);
mSocket.on("candidate", onCandidate);
mSocket.connect();
initializePeerConnectionFactory();
}
private Emitter.Listener handleOffer = new Emitter.Listener() {
#Override
public void call(final Object... args) {
Log.d("socket", "Offer recieved");
peerConnection = factory.createPeerConnection(iceServers, observer);
String id = (String) args[0];
JSONObject data = (JSONObject) args[1];
peerConnection.setRemoteDescription(new SdpAdapter("setremote") {
#Override
public void onSetSuccess() {
peerConnection.createAnswer(new SdpAdapter("createanswer") {
#Override
public void onCreateSuccess(final SessionDescription sdp) {
super.onCreateSuccess(sdp);
Log.d("socket", "Session description " + sdp.toString() + " created");
SdpAdapter local = new SdpAdapter("setlocal") {
#Override
public void onSetSuccess() {
super.onSetSuccess();
mSocket.emit("answer" , id, peerConnection.getLocalDescription().description);
}
};
peerConnection.setLocalDescription(local, sdp);
}
}, mediaConstraints);
}
}, new SessionDescription(SessionDescription.Type.OFFER, data.optString("sdp")));
}
};
private Emitter.Listener onConnect = new Emitter.Listener() {
#Override
public void call(final Object... args) {
Log.d("socket", "Socket connected");
mSocket.emit("watcher");
}
};
private Emitter.Listener onBroadcast = new Emitter.Listener() {
#Override
public void call(final Object... args) {
Log.d("socket", "Received broadcast request");
mSocket.emit("watcher");
}
};
private Emitter.Listener onCandidate = new Emitter.Listener() {
#Override
public void call(final Object... args) {
Log.d("ice", "New ice candidate recieved");
JSONObject data = (JSONObject) args[1];
IceCandidate candidate = new IceCandidate(data.optString("sdpMid"), Integer.parseInt(data.optString("sdpMLineIndex")), data.optString("candidate"));
peerConnection.addIceCandidate(candidate);
}
};
public class SdpAdapter implements SdpObserver {
private String name;
public SdpAdapter(String name) {
this.name = name;
}
#Override
public void onCreateSuccess(SessionDescription sessionDescription) {
Log.d("socket", "SDP create success for " + name );
}
#Override
public void onSetSuccess() {
Log.d("socket", "SDP set success for " + name);
}
#Override
public void onCreateFailure(String s) {
Log.d("socket", "SDP create failure for" + s);
}
#Override
public void onSetFailure(String s) {
Log.d("socket", "SDP set failure for" + s);
}
};
public static class ProxyVideoSink implements VideoSink {
private VideoSink mTarget;
#Override
synchronized public void onFrame(VideoFrame frame) {
if (mTarget == null) {
Log.d("socket", "Dropping frame in proxy because target is null.");
return;
}
mTarget.onFrame(frame);
}
synchronized void setTarget(VideoSink target) {
this.mTarget = target;
}
}
Observer observer = new Observer() {
#Override
public void onIceCandidate(IceCandidate iceCandidate) {
Log.d("ice", iceCandidate.toString());
mSocket.emit("candidate", iceCandidate);
}
#Override
public void onAddStream(MediaStream stream) {
executor.execute(new Runnable() {
#Override
public void run() {
Log.d("socket", "Stream added");
Log.d("tracks", "Getting tracks");
VideoTrack remoteVideoTrack = (VideoTrack)stream.videoTracks.get(0);
AudioTrack remoteAudioTrack = (AudioTrack)stream.audioTracks.get(0);
Log.d("tracks", "Enabling tracks");
remoteAudioTrack.setEnabled(true);
remoteVideoTrack.setEnabled(true);
Log.d("tracks", "Adding sink");
videoSink.setTarget(videoView);
remoteVideoTrack.addSink(videoSink);
peerConnection.getStats(reports -> {
for (StatsReport report : reports) {
Log.d("Stats", "Stats: " + report.toString());
}
}, null);
}
});
}
};
}

Disabling encryption in Android will break interoperability with the browser, unless you start them in a special (insecure) mode.
options.disableEncryption = true;
A good point to get started with the Android native WebRTC API is by looking at the example App code:
https://chromium.googlesource.com/external/webrtc/+/refs/heads/master/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java
I'll recommend to add more logs and extend the description / question.
Recommended procedure when debugging a WebRTC issue:
Did ICE succeed? (iceConnectionState is connected) If not, look at the gathered and received candidates. Check involved firewall configurations, NAT and TURN.
Do you receive packets but no frames can be rendered? Check your codecs and encoders / decoders.

Action cable client for android

I want to subscribe to a specific channel of action cable.
Any library or client of actioncable for android?
I am using one library https://github.com/hosopy/actioncable-client-java but not able to connect with action cable.

1.connect with the rails action cable path
private void start() {
try {
Request request = new Request.Builder().url("ws://"+Constants.HOST_ADDRESS +"/cable").addHeader("auth-token", AUTH_TOKEN).build();
EchoWebSocketListener listener = new EchoWebSocketListener();
WebSocket ws = client.newWebSocket(request, listener);
client.dispatcher().executorService().shutdown();
client = new OkHttpClient();
} catch (Exception e) {
e.printStackTrace();
}
}
2.Subscribe to the channel
private final class EchoWebSocketListener extends WebSocketListener {
#Override
public void onOpen(WebSocket webSocket, Response response) {
webSocket.send("{\"command\":\"subscribe\", \"identifier\":\"{\\\"channel\\\":\\\"CommunicationsChannel\\\"}\"}");
#Override
public void onMessage(WebSocket webSocket, String text) {
try {
JSONObject jsonBot = new JSONObject(text);
}
}
}
The WebSocket listener can receive all the message send from rails application via ‘CommunicationsChannel’, you will be able to recive it in ‘onMessage’ function
check this resource for other actionable android implementation :
https://www.xploralab.com/?s=action+cable

WebRTC Android - createPeerConnection() always returns null

I'm new to Android and WebRTC, and I have been struggling to get a p2p video connection working between a website and my android device. I have already gotten this working between two websites, so I'm fairly sure my issue lies in the Android side.
When I try to create a new PeerConnection object using PeerConnectionFactory.createPeerConnection(), the result is always null. I am using release vesion M58 of WebRTC.
PeerConnection.IceServer iceServer = new PeerConnection.IceServer("http://stun.stun.l.google.com:19302/");
List<PeerConnection.IceServer> servers = new ArrayList<PeerConnection.IceServer>();
servers.add(iceServer);
// Media Constraints
final MediaConstraints mediaConstraints = new MediaConstraints();
mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true"));
mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true"));
mediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true"));
// Set up remote rendering stuff
rootEGLBase = EglBase.create();
svr = (SurfaceViewRenderer) findViewById(R.id.remote_video);
svr.init(rootEGLBase.getEglBaseContext(), null);
svr.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT);
svr.setZOrderMediaOverlay(true);
svr.setEnableHardwareScaler(true);
remoteProxyRenderer.setTarget(svr);
remoteRender = remoteProxyRenderer;
// Create a peer connection factory
PeerConnectionFactory.initializeAndroidGlobals(this.ctx, true);
PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
options.networkIgnoreMask = 0;
PeerConnectionFactory peerConnectionFactory = new PeerConnectionFactory(options);
// wait until my websocket has connected
System.out.println("waiting for websocket to be inited...");
while (this.ws == null);
System.out.println("done wait");
final WebSocket in_scope_ws = this.ws;
// create peer connection observer
PeerConnection.Observer pcObserver = new PeerConnection.Observer() {/* not included - mostly just prints */};
// Create peerconnection
final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(servers, mediaConstraints, pcObserver);
Here is the PeerConnection.Observer anonymous implementation that I removed from the above code. I'm sure that I'm doing a lot of other stuff wrong in here, but I figured I'd wait to solve this until I got the offer/answer exchange working first.
PeerConnection.Observer pcObserver = new PeerConnection.Observer() {
final String TAG = "PEER_CONNECTION_FACTORY";
#Override
public void onSignalingChange(PeerConnection.SignalingState signalingState) {
Log.d(TAG, "onSignalingChange");
}
#Override
public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
Log.d(TAG, "onIceConnectionChange");
}
#Override
public void onIceConnectionReceivingChange(boolean b) {
Log.d(TAG, "onIceConnectionReceivingChange");
}
#Override
public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
Log.d(TAG, "onIceGatheringChange");
}
// not sure what to put here
#Override
public void onIceCandidate(IceCandidate iceCandidate) {
System.out.println("AN ICE CANDIDATE HAS BEEN DISCOVERED");
if (iceCandidate != null && in_scope_ws != null) {
try {
JSONObject payload = new JSONObject();
payload.put("sdpMLineIndex", iceCandidate.sdpMLineIndex);
payload.put("sdpMid", iceCandidate.sdpMid);
payload.put("candidate", iceCandidate.sdp);
JSONObject candidateObject = new JSONObject();
candidateObject.put("ice", iceCandidate.toString());
in_scope_ws.sendText(candidateObject.toString());
} catch (JSONException e) {
e.printStackTrace();
}
//send ice candidate to server - probably need to convert to string and put in JSON object
// in_scope_ws.send("")
}
}
#Override
public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
Log.d(TAG, "onIceCandidatesRemoved");
}
// pretty sure this is the only garbage I need right now
#Override
public void onAddStream(MediaStream mediaStream) {
System.out.println("IN ADD STREAM");
if(mediaStream.videoTracks.size()==0) {
Log.d("onAddStream", "NO REMOTE STREAM");
System.out.println("NO REMOTE STREAM (PRINTLN)");
}
mediaStream.videoTracks.get(0).addRenderer(new VideoRenderer (remoteRender));
// VideoRendererGui.update(remoteRender, 0, 0, 100, 100, RendererCommon.ScalingType.SCALE_ASPECT_FILL, false);
}
#Override
public void onRemoveStream(MediaStream mediaStream) {
Log.d(TAG, "onRemoveStream");
}
#Override
public void onDataChannel(DataChannel dataChannel) {
Log.d(TAG, "onDataChannel");
}
#Override
public void onRenegotiationNeeded() {
Log.d(TAG, "onRenegotiationNeeded");
}
#Override
public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) { Log.d(TAG, "onAddTrack"); }
};

You can enable verbose logging to trace the problem.
org.webrtc.Logging.enableLogToDebugOutput(Logging.Severity.LS_VERBOSE);
Also make sure you have granted required runtime permissions (CAMERA, RECORD_AUTDIO)

you need to createLocalMediaStream and set videoHeight videoWidth,createVideoSource,createAudioSource
before you createPeerConnection.

Another problem that createPeerConnection() returning null could be with wrong configuration of IceServer, make sure you set correct IceServer and correct credentials.

I had the same issue. The peerConnection object was always returned as null.
In my case, the username for one of the turn servers was not set. After setting that, peerConnection object was created successfully.
val servers = listOf(
...,
PeerConnection.IceServer.builder("server_url")
.setUsername("user_name")
.setPassword("password")
.createIceServer()
...
) as MutableList<PeerConnection.IceServer>

How to get Socket Id in Socket.io on Android?

I want to get the id of my socket running in Android. Everything is create normally and is able to send data back and forth but I need to get the Androids socket id for usage throughout the entire application. I have tried to find a solution but either they were outdated or does not work. Below is my latest attempt at solving this issue. With this attempt it errors saying that there are no elements in args[0]. I have also tried mSocket.id(); but that returns nothing.
public class GameSocket {
private Socket mSocket = null;
private static GameSocket mInstance;
public synchronized static GameSocket getInstance() {
if (mInstance == null) {
mInstance = new GameSocket();
}
return mInstance;
}
public void initialize() {
mSocket = IO.socket(URI.create(SERVER_URI));
mSocket.off();
mSocket.on(Socket.EVENT_CONNECT, new Emitter.Listener() {
#Override
public void call(Object... args) {
Log.d(GameSocket.class.getSimpleName(), "Event connection...");
mSocketId = args[0].toString();
Log.d("CONNECT ID", mSocketId);
}
})
.on(Socket.EVENT_ERROR, new Emitter.Listener() {
#Override
public void call(Object... args) {
if (args != null) {
Log.e(GameSocket.class.getSimpleName(), "Socket Event Error: " + args[0].toString());
}
}
});
}
}

How to transfer data from a socket.io Service to an Activity

Currently, I have a Service that is supposed to keep a socket.io connection open for multiple different Activitys to use. One of the biggest functions of the application is to update an album-art gallery whenever a song is added to a playlist. The Node.js server broadcasts if a song is added, and my goal is for the SocketService to update the UI when .on("song added"... is triggered. However, currently, I cannot figure out how to transfer the incoming data to the UI from the SocketService. How would I go about doing this?
For reference:
The "song add" command from the server emits a JSONObject.
public class SocketService extends IntentService {
Socket socket;
String roomID;
String userID;
ArrayList<JSONObject> queuedSongs;
ArrayList<JSONObject> suggestionSongs;
int users;
public SocketService() {
super("SocketService");
}
#Override
protected void onHandleIntent(Intent intent) {
userID = intent.getExtras().getString("userid", "0");
Log.i("SocketService", "userID=" + userID);
roomID = intent.getExtras().getString("code");
Log.i("SocketService", "roomID=" + roomID);
super.onCreate();
try {
socket = IO.socket("server address");
socket.on(Socket.EVENT_CONNECT, new Emitter.Listener() {
#Override
public void call(Object... args) {
Log.i("Internet-Task", "Connected to "server address"");
}
}).on("song add", new Emitter.Listener() {
#Override
public void call(Object... args) {
try {
suggestionSongs.add(new JSONObject(args[0].toString()));
} catch (JSONException e) {
e.printStackTrace();
}
}
});
//**//
socket.io().on(Manager.EVENT_TRANSPORT, new Emitter.Listener() {
#Override
public void call(Object... args) {
Transport transport = (Transport) args[0];
transport.on(Transport.EVENT_REQUEST_HEADERS, new Emitter.Listener() {
#Override
public void call(Object... args) {
#SuppressWarnings("unchecked")
Map<String, List<String>> headers = (Map<String, List<String>>) args[0];
// modify request headers
headers.put("userid", Arrays.asList(userID));
headers.put("roomid", Arrays.asList(roomID));
}
});
socket.connect();
} catch (URISyntaxException e) {
Log.e(getClass().getCanonicalName(), e.toString());
}
Additionally, here is the call to the service made in the Activity onCreate(). No new thread is made prior.
protected void onCreate(Bundle savedInstanceState) {
...
Intent serviceIntent = new Intent(getBaseContext(), SocketService.class);
Bundle serviceBundle = new Bundle();
serviceBundle.putString("code", roomID);;
serviceBundle.putString("userid", userID);
serviceIntent.putExtras(mBundle);
startService(serviceIntent);
Again, my question is: How do I transfer data from my SocketService to the UI of my Activity?