From ac9a1a773e98cc23112c458448532cb2ee4c39a9 Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Mon, 6 Apr 2020 10:26:29 +0200 Subject: [PATCH] receive candidates/transport-info --- .../xmpp/jingle/JingleRtpConnection.java | 61 +++++++++++++++++-- .../xmpp/jingle/RtpContentMap.java | 15 ++++- .../jingle/stanzas/IceUdpTransportInfo.java | 47 ++++++++++++++ 3 files changed, 117 insertions(+), 6 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index ad2b9fdad..ef459cb88 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -16,16 +16,19 @@ import org.webrtc.PeerConnectionFactory; import org.webrtc.RtpReceiver; import org.webrtc.SdpObserver; +import java.util.ArrayDeque; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import eu.siacs.conversations.Config; import eu.siacs.conversations.xml.Element; import eu.siacs.conversations.xml.Namespace; +import eu.siacs.conversations.xmpp.jingle.stanzas.Group; import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; -import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; import eu.siacs.conversations.xmpp.stanzas.MessagePacket; import rocks.xmpp.addr.Jid; @@ -45,6 +48,8 @@ public class JingleRtpConnection extends AbstractJingleConnection { private RtpContentMap initialRtpContentMap; private PeerConnection peerConnection; + private final ArrayDeque pendingIceCandidates = new ArrayDeque<>(); + public JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) { super(jingleConnectionManager, id, initiator); @@ -57,14 +62,51 @@ public class JingleRtpConnection extends AbstractJingleConnection { case SESSION_INITIATE: receiveSessionInitiate(jinglePacket); break; + case TRANSPORT_INFO: + receiveTransportInfo(jinglePacket); + break; default: Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction())); break; } } + private void receiveTransportInfo(final JinglePacket jinglePacket) { + if (isInState(State.SESSION_INITIALIZED, State.SESSION_ACCEPTED)) { + final RtpContentMap contentMap; + try { + contentMap = RtpContentMap.of(jinglePacket); + } catch (IllegalArgumentException | NullPointerException e) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e); + return; + } + final Group originalGroup = this.initialRtpContentMap != null ? this.initialRtpContentMap.group : null; + final List identificationTags = originalGroup == null ? Collections.emptyList() : originalGroup.getIdentificationTags(); + if (identificationTags.size() == 0) { + Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": no identification tags found in initial offer. we won't be able to calculate mLineIndices"); + } + for(final Map.Entry content : contentMap.contents.entrySet()) { + final String ufrag = content.getValue().transport.getAttribute("ufrag"); + for(final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) { + final String sdp = candidate.toSdpAttribute(ufrag); + final String sdpMid = content.getKey(); + final int mLineIndex = identificationTags.indexOf(sdpMid); + final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp); + Log.d(Config.LOGTAG,"received candidate: "+iceCandidate); + if (isInState(State.SESSION_ACCEPTED)) { + this.peerConnection.addIceCandidate(iceCandidate); + } else { + this.pendingIceCandidates.push(iceCandidate); + } + } + } + } else { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state); + } + } + private void receiveSessionInitiate(final JinglePacket jinglePacket) { - Log.d(Config.LOGTAG,jinglePacket.toString()); + Log.d(Config.LOGTAG, jinglePacket.toString()); if (isInitiator()) { Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid())); //TODO respond with out-of-order @@ -73,13 +115,15 @@ public class JingleRtpConnection extends AbstractJingleConnection { final RtpContentMap contentMap; try { contentMap = RtpContentMap.of(jinglePacket); - } catch (IllegalArgumentException | NullPointerException e) { + contentMap.requireContentDescriptions(); + } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e); return; } Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents"); final State oldState = this.state; if (transition(State.SESSION_INITIALIZED)) { + this.initialRtpContentMap = contentMap; if (oldState == State.PROCEED) { processContents(contentMap); sendSessionAccept(); @@ -225,7 +269,10 @@ public class JingleRtpConnection extends AbstractJingleConnection { stream.addTrack(audioTrack); - this.peerConnection = peerConnectionFactory.createPeerConnection(Collections.emptyList(), new PeerConnection.Observer() { + final List iceServers = ImmutableList.of( + PeerConnection.IceServer.builder("stun:xmpp.conversations.im:3478").createIceServer() + ); + this.peerConnection = peerConnectionFactory.createPeerConnection(iceServers, new PeerConnection.Observer() { @Override public void onSignalingChange(PeerConnection.SignalingState signalingState) { @@ -249,7 +296,7 @@ public class JingleRtpConnection extends AbstractJingleConnection { @Override public void onIceCandidate(IceCandidate iceCandidate) { IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp); - Log.d(Config.LOGTAG, "onIceCandidate: " + iceCandidate.sdp); + Log.d(Config.LOGTAG, "onIceCandidate: " + iceCandidate.sdp + " mLineIndex=" + iceCandidate.sdpMLineIndex); sendTransportInfo(iceCandidate.sdpMid, candidate); } @@ -352,6 +399,10 @@ public class JingleRtpConnection extends AbstractJingleConnection { } + private synchronized boolean isInState(State... state) { + return Arrays.asList(state).contains(this.state); + } + private synchronized boolean transition(final State target) { final Collection validTransitions = VALID_TRANSITIONS.get(this.state); if (validTransitions != null && validTransitions.contains(target)) { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java index 445d34dad..1ebd810b7 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java @@ -47,6 +47,17 @@ public class RtpContentMap { return new RtpContentMap(group, contentMapBuilder.build()); } + public void requireContentDescriptions() { + if (this.contents.size() == 0) { + throw new IllegalStateException("No contents available"); + } + for(Map.Entry entry : this.contents.entrySet()) { + if (entry.getValue().description == null) { + throw new IllegalStateException(String.format("%s is lacking content description", entry.getKey())); + } + } + } + public JinglePacket toJinglePacket(final JinglePacket.Action action, final String sessionId) { final JinglePacket jinglePacket = new JinglePacket(action, sessionId); if (this.group != null) { @@ -89,7 +100,9 @@ public class RtpContentMap { final GenericTransportInfo transportInfo = content.getTransport(); final RtpDescription rtpDescription; final IceUdpTransportInfo iceUdpTransportInfo; - if (description instanceof RtpDescription) { + if (description == null) { + rtpDescription = null; + } else if (description instanceof RtpDescription) { rtpDescription = (RtpDescription) description; } else { Log.d(Config.LOGTAG, "description was " + description); diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java index e4d953c83..1b3fe18f2 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java @@ -2,14 +2,22 @@ package eu.siacs.conversations.xmpp.jingle.stanzas; import android.util.Log; +import com.google.common.base.Function; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.checkerframework.checker.nullness.compatqual.NullableDecl; + +import java.util.Collection; import java.util.HashMap; import java.util.Hashtable; import java.util.List; +import java.util.Map; import eu.siacs.conversations.Config; import eu.siacs.conversations.xml.Element; @@ -144,6 +152,43 @@ public class IceUdpTransportInfo extends GenericTransportInfo { return candidate; } + public String toSdpAttribute(final String ufrag) { + final String foundation = this.getAttribute("foundation"); + final String component = this.getAttribute("component"); + final String transport = this.getAttribute("protocol"); + final String priority = this.getAttribute("priority"); + final String connectionAddress = this.getAttribute("ip"); + final String port = this.getAttribute("port"); + final Map additionalParameter = new HashMap<>(); + final String relAddr = this.getAttribute("rel-addr"); + if (relAddr != null) { + additionalParameter.put("raddr",relAddr); + } + final String relPort = this.getAttribute("rel-port"); + if (relPort != null) { + additionalParameter.put("rport", relPort); + } + final String generation = this.getAttribute("generation"); + if (generation != null) { + additionalParameter.put("generation", generation); + } + if (ufrag != null) { + additionalParameter.put("ufrag", ufrag); + } + final String parametersString = Joiner.on(' ').join(Collections2.transform(additionalParameter.entrySet(), input -> String.format("%s %s",input.getKey(),input.getValue()))); + return String.format( + "candidate:%s %s %s %s %s %s %s", + foundation, + component, + transport, + priority, + connectionAddress, + port, + parametersString + + ); + } + // https://tools.ietf.org/html/draft-ietf-mmusic-ice-sip-sdp-39#section-5.1 public static Candidate fromSdpAttribute(final String attribute) { final String[] pair = attribute.split(":", 2); @@ -164,6 +209,8 @@ public class IceUdpTransportInfo extends GenericTransportInfo { candidate.setAttribute("component", component); candidate.setAttribute("foundation", foundation); candidate.setAttribute("generation", additional.get("generation")); + candidate.setAttribute("rel-addr", additional.get("raddr")); + candidate.setAttribute("rel-port", additional.get("rport")); candidate.setAttribute("ip", connectionAddress); candidate.setAttribute("port", port); candidate.setAttribute("priority", priority);