diff --git a/src/main/java/eu/siacs/conversations/xml/Element.java b/src/main/java/eu/siacs/conversations/xml/Element.java index c0ece7f4c..4d53a17b7 100644 --- a/src/main/java/eu/siacs/conversations/xml/Element.java +++ b/src/main/java/eu/siacs/conversations/xml/Element.java @@ -1,5 +1,7 @@ package eu.siacs.conversations.xml; +import org.jetbrains.annotations.NotNull; + import java.util.ArrayList; import java.util.Hashtable; import java.util.List; @@ -165,8 +167,9 @@ public class Element { return this.attributes; } + @NotNull public String toString() { - StringBuilder elementOutput = new StringBuilder(); + final StringBuilder elementOutput = new StringBuilder(); if ((content == null) && (children.size() == 0)) { Tag emptyTag = Tag.empty(name); emptyTag.setAtttributes(this.attributes); diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index 4796d0c58..71cdb02c4 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -25,7 +25,6 @@ import org.webrtc.IceCandidate; import org.webrtc.PeerConnection; import org.webrtc.VideoTrack; -import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -142,7 +141,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this); - private final ArrayDeque>> pendingIceCandidates = new ArrayDeque<>(); + //TODO convert to Queue>? + private final Queue> pendingIceCandidates = new LinkedList<>(); private final OmemoVerification omemoVerification = new OmemoVerification(); private final Message message; private State state = State.NULL; @@ -193,7 +193,6 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web @Override synchronized void deliverPacket(final JinglePacket jinglePacket) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection"); switch (jinglePacket.getAction()) { case SESSION_INITIATE: receiveSessionInitiate(jinglePacket); @@ -254,23 +253,29 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web private void receiveTransportInfo(final JinglePacket jinglePacket) { //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) { - respondOk(jinglePacket); final RtpContentMap contentMap; try { contentMap = RtpContentMap.of(jinglePacket); - } catch (IllegalArgumentException | NullPointerException e) { + } catch (final IllegalArgumentException | NullPointerException e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e); + respondOk(jinglePacket); return; } final Set> candidates = contentMap.contents.entrySet(); if (this.state == State.SESSION_ACCEPTED) { + //zero candidates + modified credentials are an ICE restart offer + if (checkForIceRestart(contentMap, jinglePacket)) { + return; + } + respondOk(jinglePacket); try { processCandidates(candidates); } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored"); } } else { - pendingIceCandidates.push(candidates); + respondOk(jinglePacket); + pendingIceCandidates.addAll(candidates); } } else { if (isTerminated()) { @@ -283,37 +288,106 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } } + private boolean checkForIceRestart(final RtpContentMap rtpContentMap, final JinglePacket jinglePacket) { + final RtpContentMap existing = getRemoteContentMap(); + final Map existingCredentials = existing.getCredentials(); + final Map newCredentials = rtpContentMap.getCredentials(); + if (!existingCredentials.keySet().equals(newCredentials.keySet())) { + return false; + } + if (existingCredentials.equals(newCredentials)) { + return false; + } + final boolean isOffer = rtpContentMap.emptyCandidates(); + Log.d(Config.LOGTAG, "detected ICE restart. offer=" + isOffer + " " + jinglePacket); + //TODO reset to 'actpass'? + final RtpContentMap restartContentMap = existing.modifiedCredentials(newCredentials); + try { + if (applyIceRestart(isOffer, restartContentMap)) { + return false; + } else { + Log.d(Config.LOGTAG, "responding with tie break"); + //TODO respond with conflict + return true; + } + } catch (Exception e) { + Log.d(Config.LOGTAG, "failure to apply ICE restart. sending error", e); + //TODO send some kind of error + return true; + } + } + + private boolean applyIceRestart(final boolean isOffer, final RtpContentMap restartContentMap) throws ExecutionException, InterruptedException { + final SessionDescription sessionDescription = SessionDescription.of(restartContentMap); + final org.webrtc.SessionDescription.Type type = isOffer ? org.webrtc.SessionDescription.Type.OFFER : org.webrtc.SessionDescription.Type.ANSWER; + org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(type, sessionDescription.toString()); + if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) { + if (isInitiator()) { + //We ignore the offer and respond with tie-break. This will clause the responder not to apply the content map + return false; + } + //rollback our own local description. should happen automatically but doesn't + webRTCWrapper.rollbackLocalDescription().get(); + } + webRTCWrapper.setRemoteDescription(sdp).get(); + if (isInitiator()) { + this.responderRtpContentMap = restartContentMap; + } else { + this.initiatorRtpContentMap = restartContentMap; + } + if (isOffer) { + webRTCWrapper.setIsReadyToReceiveIceCandidates(false); + final SessionDescription localSessionDescription = setLocalSessionDescription(); + setLocalContentMap(RtpContentMap.of(localSessionDescription)); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } + return true; + } + private void processCandidates(final Set> contents) { - final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap; + for (final Map.Entry content : contents) { + processCandidate(content); + } + } + + private void processCandidate(final Map.Entry content) { + final RtpContentMap rtpContentMap = getRemoteContentMap(); + final List indices = toIdentificationTags(rtpContentMap); + final String sdpMid = content.getKey(); //aka content name + final IceUdpTransportInfo transport = content.getValue().transport; + final IceUdpTransportInfo.Credentials credentials = transport.getCredentials(); + + //TODO check that credentials remained the same + + for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) { + final String sdp; + try { + sdp = candidate.toSdpAttribute(credentials.ufrag); + } catch (final IllegalArgumentException e) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage()); + continue; + } + final int mLineIndex = indices.indexOf(sdpMid); + if (mLineIndex < 0) { + Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices); + } + final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp); + Log.d(Config.LOGTAG, "received candidate: " + iceCandidate); + this.webRTCWrapper.addIceCandidate(iceCandidate); + } + } + + private RtpContentMap getRemoteContentMap() { + return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap; + } + + private List toIdentificationTags(final RtpContentMap rtpContentMap) { final Group originalGroup = rtpContentMap.group; final List identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags(); if (identificationTags.size() == 0) { Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices"); } - processCandidates(identificationTags, contents); - } - - private void processCandidates(final List indices, final Set> contents) { - for (final Map.Entry content : contents) { - final String ufrag = content.getValue().transport.getAttribute("ufrag"); - for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) { - final String sdp; - try { - sdp = candidate.toSdpAttribute(ufrag); - } catch (IllegalArgumentException e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage()); - continue; - } - final String sdpMid = content.getKey(); - final int mLineIndex = indices.indexOf(sdpMid); - if (mLineIndex < 0) { - Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices); - } - final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp); - Log.d(Config.LOGTAG, "received candidate: " + iceCandidate); - this.webRTCWrapper.addIceCandidate(iceCandidate); - } - } + return identificationTags; } private ListenableFuture receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) { @@ -401,11 +475,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) { respondOk(jinglePacket); - - final Set> candidates = contentMap.contents.entrySet(); - if (candidates.size() > 0) { - pendingIceCandidates.push(candidates); - } + pendingIceCandidates.addAll(contentMap.contents.entrySet()); if (target == State.SESSION_INITIALIZED_PRE_APPROVED) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate"); sendSessionAccept(); @@ -495,8 +565,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web sendSessionTerminate(Reason.FAILED_APPLICATION); return; } - final List identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags(); - processCandidates(identificationTags, contentMap.contents.entrySet()); + processCandidates(contentMap.contents.entrySet()); } private void sendSessionAccept() { @@ -558,9 +627,10 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } private void addIceCandidatesFromBlackLog() { - while (!this.pendingIceCandidates.isEmpty()) { - processCandidates(this.pendingIceCandidates.poll()); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log"); + Map.Entry foo; + while ((foo = this.pendingIceCandidates.poll()) != null) { + processCandidate(foo); + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidate from back log"); } } @@ -1335,7 +1405,13 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web @Override public void onIceCandidate(final IceCandidate iceCandidate) { - final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp); + final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; + final Collection currentUfrags = Collections2.transform(rtpContentMap.getCredentials().values(), c -> c.ufrag); + final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, currentUfrags); + if (candidate == null) { + Log.d(Config.LOGTAG,"ignoring (not sending) candidate: "+iceCandidate.toString()); + return; + } Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString()); sendTransportInfo(iceCandidate.sdpMid, candidate); } @@ -1373,23 +1449,42 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web @Override public void onRenegotiationNeeded() { Log.d(Config.LOGTAG, "onRenegotiationNeeded()"); - this.webRTCWrapper.execute(this::renegotiate); + this.webRTCWrapper.execute(this::initiateIceRestart); } - private void renegotiate() { + private void initiateIceRestart() { + PeerConnection.SignalingState signalingState = webRTCWrapper.getSignalingState(); + Log.d(Config.LOGTAG, "initiateIceRestart() - " + signalingState); + if (signalingState != PeerConnection.SignalingState.STABLE) { + return; + } this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false); + final SessionDescription sessionDescription; try { - final SessionDescription sessionDescription = setLocalSessionDescription(); - final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription); - setRenegotiatedContentMap(rtpContentMap); - this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + sessionDescription = setLocalSessionDescription(); } catch (final Exception e) { Log.d(Config.LOGTAG, "failed to renegotiate", e); //TODO send some sort of failure (comparable to when initiating) + return; } + final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription); + final RtpContentMap transportInfo = rtpContentMap.transportInfo(); + final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId); + Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket); + jinglePacket.setTo(id.with); + xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> { + if (response.getType() == IqPacket.TYPE.RESULT) { + Log.d(Config.LOGTAG, "received success to our ice restart"); + setLocalContentMap(rtpContentMap); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } else { + Log.d(Config.LOGTAG, "received failure to our ice restart"); + //TODO handle tie-break. Rollback? + } + }); } - private void setRenegotiatedContentMap(final RtpContentMap rtpContentMap) { + private void setLocalContentMap(final RtpContentMap rtpContentMap) { if (isInitiator()) { this.initiatorRtpContentMap = rtpContentMap; } else { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java index 9baffcf81..99db8bd34 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java @@ -1,7 +1,5 @@ package eu.siacs.conversations.xmpp.jingle; -import android.util.Log; - import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -17,9 +15,9 @@ import org.checkerframework.checker.nullness.compatqual.NullableDecl; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; -import eu.siacs.conversations.Config; import eu.siacs.conversations.xmpp.jingle.stanzas.Content; import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription; import eu.siacs.conversations.xmpp.jingle.stanzas.GenericTransportInfo; @@ -137,7 +135,37 @@ public class RtpContentMap { final IceUdpTransportInfo newTransportInfo = transportInfo.cloneWrapper(); newTransportInfo.addChild(candidate); return new RtpContentMap(null, ImmutableMap.of(contentName, new DescriptionTransport(null, newTransportInfo))); + } + RtpContentMap transportInfo() { + return new RtpContentMap( + null, + Maps.transformValues(contents, dt -> new DescriptionTransport(null, dt.transport.cloneWrapper())) + ); + } + + public Map getCredentials() { + return Maps.transformValues(contents, dt -> dt.transport.getCredentials()); + } + + public boolean emptyCandidates() { + int count = 0; + for (DescriptionTransport descriptionTransport : contents.values()) { + count += descriptionTransport.transport.getCandidates().size(); + } + return count == 0; + } + + public RtpContentMap modifiedCredentials(Map credentialsMap) { + final ImmutableMap.Builder contentMapBuilder = new ImmutableMap.Builder<>(); + for (final Map.Entry content : contents.entrySet()) { + final RtpDescription rtpDescription = content.getValue().description; + IceUdpTransportInfo transportInfo = content.getValue().transport; + final IceUdpTransportInfo.Credentials credentials = Objects.requireNonNull(credentialsMap.get(content.getKey())); + final IceUdpTransportInfo modifiedTransportInfo = transportInfo.modifyCredentials(credentials); + contentMapBuilder.put(content.getKey(), new DescriptionTransport(rtpDescription, modifiedTransportInfo)); + } + return new RtpContentMap(this.group, contentMapBuilder.build()); } public static class DescriptionTransport { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index 9ea4cd389..401121b86 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -17,7 +17,6 @@ import com.google.common.util.concurrent.SettableFuture; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; -import org.webrtc.Camera1Enumerator; import org.webrtc.Camera2Enumerator; import org.webrtc.CameraEnumerationAndroid; import org.webrtc.CameraEnumerator; @@ -87,6 +86,7 @@ public class WebRTCWrapper { private final EventCallback eventCallback; private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false); + private final AtomicBoolean ignoreOnRenegotiationNeeded = new AtomicBoolean(false); private final Queue iceCandidates = new LinkedList<>(); private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() { @Override @@ -163,6 +163,10 @@ public class WebRTCWrapper { @Override public void onRenegotiationNeeded() { + if (ignoreOnRenegotiationNeeded.get()) { + Log.d(EXTENDED_LOGGING_TAG, "ignoring onRenegotiationNeeded()"); + return; + } Log.d(EXTENDED_LOGGING_TAG, "onRenegotiationNeeded()"); final PeerConnection.PeerConnectionState currentState = peerConnection == null ? null : peerConnection.connectionState(); if (currentState != null && currentState != PeerConnection.PeerConnectionState.NEW) { @@ -307,12 +311,12 @@ public class WebRTCWrapper { } void restartIce() { - executorService.execute(()-> requirePeerConnection().restartIce()); + executorService.execute(() -> requirePeerConnection().restartIce()); } public void setIsReadyToReceiveIceCandidates(final boolean ready) { readyToReceivedIceCandidates.set(ready); - while(ready && iceCandidates.peek() != null) { + while (ready && iceCandidates.peek() != null) { eventCallback.onIceCandidate(iceCandidates.poll()); } } @@ -452,6 +456,26 @@ public class WebRTCWrapper { }, MoreExecutors.directExecutor()); } + public ListenableFuture rollbackLocalDescription() { + final SettableFuture future = SettableFuture.create(); + final SessionDescription rollback = new SessionDescription(SessionDescription.Type.ROLLBACK, ""); + ignoreOnRenegotiationNeeded.set(true); + requirePeerConnection().setLocalDescription(new SetSdpObserver() { + @Override + public void onSetSuccess() { + future.set(null); + ignoreOnRenegotiationNeeded.set(false); + } + + @Override + public void onSetFailure(final String message) { + future.setException(new FailureToSetDescriptionException(message)); + } + }, rollback); + return future; + } + + private static void logDescription(final SessionDescription sessionDescription) { for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) { Log.d(EXTENDED_LOGGING_TAG, line); @@ -552,6 +576,10 @@ public class WebRTCWrapper { executorService.execute(command); } + public PeerConnection.SignalingState getSignalingState() { + return requirePeerConnection().signalingState(); + } + public interface EventCallback { void onIceCandidate(IceCandidate iceCandidate); diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java index 022c4d2dd..2b8770578 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java @@ -1,6 +1,7 @@ package eu.siacs.conversations.xmpp.jingle.stanzas; import com.google.common.base.Joiner; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ArrayListMultimap; @@ -8,6 +9,7 @@ import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import java.util.Collection; import java.util.HashMap; import java.util.Hashtable; import java.util.LinkedHashMap; @@ -58,6 +60,12 @@ public class IceUdpTransportInfo extends GenericTransportInfo { return fingerprint == null ? null : Fingerprint.upgrade(fingerprint); } + public Credentials getCredentials() { + final String ufrag = this.getAttribute("ufrag"); + final String password = this.getAttribute("pwd"); + return new Credentials(ufrag, password); + } + public List getCandidates() { final ImmutableList.Builder builder = new ImmutableList.Builder<>(); for (final Element child : getChildren()) { @@ -74,6 +82,37 @@ public class IceUdpTransportInfo extends GenericTransportInfo { return transportInfo; } + public IceUdpTransportInfo modifyCredentials(Credentials credentials) { + final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); + transportInfo.setAttribute("ufrag", credentials.ufrag); + transportInfo.setAttribute("pwd", credentials.password); + transportInfo.setChildren(getChildren()); + return transportInfo; + } + + public static class Credentials { + public final String ufrag; + public final String password; + + public Credentials(String ufrag, String password) { + this.ufrag = ufrag; + this.password = password; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Credentials that = (Credentials) o; + return Objects.equal(ufrag, that.ufrag) && Objects.equal(password, that.password); + } + + @Override + public int hashCode() { + return Objects.hashCode(ufrag, password); + } + } + public static class Candidate extends Element { private Candidate() { @@ -89,7 +128,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo { } // https://tools.ietf.org/html/draft-ietf-mmusic-ice-sip-sdp-39#section-5.1 - public static Candidate fromSdpAttribute(final String attribute) { + public static Candidate fromSdpAttribute(final String attribute, Collection currentUfrags) { final String[] pair = attribute.split(":", 2); if (pair.length == 2 && "candidate".equals(pair[0])) { final String[] segments = pair[1].split(" "); @@ -105,6 +144,10 @@ public class IceUdpTransportInfo extends GenericTransportInfo { for (int i = 6; i < segments.length - 1; i = i + 2) { additional.put(segments[i], segments[i + 1]); } + final String ufrag = additional.get("ufrag"); + if (ufrag != null && !currentUfrags.contains(ufrag)) { + return null; + } final Candidate candidate = new Candidate(); candidate.setAttribute("component", component); candidate.setAttribute("foundation", foundation);