From d057ae3439ca83d4c1a00dc0f8d8941e3fd83723 Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Wed, 15 Apr 2020 12:07:19 +0200 Subject: [PATCH] transmit media from proposal to actual session --- .../xmpp/jingle/JingleConnectionManager.java | 17 +++--- .../xmpp/jingle/JingleRtpConnection.java | 57 ++++++++++++------- .../xmpp/jingle/WebRTCWrapper.java | 30 ++++++---- 3 files changed, 66 insertions(+), 38 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java index e56105c2e..a50dcd70f 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java @@ -7,6 +7,7 @@ import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.compatqual.NullableDecl; @@ -163,6 +164,7 @@ public class JingleConnectionManager extends AbstractConnectionManager { if (fromSelf) { Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": ignore jingle message from self"); + //TODO proceed from self should maybe dedup/change the busy that we set earlier return; } @@ -176,16 +178,17 @@ public class JingleConnectionManager extends AbstractConnectionManager { if (rtpDescriptions.size() > 0 && rtpDescriptions.size() == descriptions.size() && !usesTor(account)) { final Collection media = Collections2.transform(rtpDescriptions, RtpDescription::getMedia); if (media.contains(Media.UNKNOWN)) { - Log.d(Config.LOGTAG,account.getJid().asBareJid()+": encountered unknown media in session proposal. "+propose); + Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": encountered unknown media in session proposal. " + propose); return; } if (isBusy()) { //TODO only if no other devices are active - //TODO create + //TODO create busy final MessagePacket reject = mXmppConnectionService.getMessageGenerator().sessionReject(from, sessionId); mXmppConnectionService.sendMessagePacket(account, reject); } else { final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, from); this.connections.put(id, rtpConnection); + rtpConnection.setProposedMedia(ImmutableSet.copyOf(media)); rtpConnection.deliveryMessage(from, message, serverMsgId, timestamp); } } else { @@ -193,7 +196,7 @@ public class JingleConnectionManager extends AbstractConnectionManager { } } else if ("proceed".equals(message.getName())) { synchronized (rtpSessionProposals) { - final RtpSessionProposal proposal = getRtpSessionProposal(account,from.asBareJid(),sessionId); + final RtpSessionProposal proposal = getRtpSessionProposal(account, from.asBareJid(), sessionId); if (proposal != null) { rtpSessionProposals.remove(proposal); final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, account.getJid()); @@ -222,7 +225,7 @@ public class JingleConnectionManager extends AbstractConnectionManager { } private RtpSessionProposal getRtpSessionProposal(final Account account, Jid from, String sessionId) { - for(RtpSessionProposal rtpSessionProposal : rtpSessionProposals.keySet()) { + for (RtpSessionProposal rtpSessionProposal : rtpSessionProposals.keySet()) { if (rtpSessionProposal.sessionId.equals(sessionId) && rtpSessionProposal.with.equals(from) && rtpSessionProposal.account.getJid().equals(account.getJid())) { return rtpSessionProposal; } @@ -424,9 +427,9 @@ public class JingleConnectionManager extends AbstractConnectionManager { } public void updateProposedSessionDiscovered(Account account, Jid from, String sessionId, final DeviceDiscoveryState target) { - final RtpSessionProposal sessionProposal = new RtpSessionProposal(account, from.asBareJid(), sessionId); synchronized (this.rtpSessionProposals) { - final DeviceDiscoveryState currentState = rtpSessionProposals.get(sessionProposal); + final RtpSessionProposal sessionProposal = getRtpSessionProposal(account, from.asBareJid(), sessionId); + final DeviceDiscoveryState currentState = sessionProposal == null ? null : rtpSessionProposals.get(sessionProposal); if (currentState == null) { Log.d(Config.LOGTAG, "unable to find session proposal for session id " + sessionId); return; @@ -491,7 +494,7 @@ public class JingleConnectionManager extends AbstractConnectionManager { public final Set media; private RtpSessionProposal(Account account, Jid with, String sessionId) { - this(account,with,sessionId, Collections.emptySet()); + this(account, with, sessionId, Collections.emptySet()); } private RtpSessionProposal(Account account, Jid with, String sessionId, Set media) { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index 7daabe163..e4ef46b60 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -33,7 +33,6 @@ import eu.siacs.conversations.entities.RtpSessionStatus; import eu.siacs.conversations.services.AppRTCAudioManager; import eu.siacs.conversations.xml.Element; import eu.siacs.conversations.xml.Namespace; -import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription; import eu.siacs.conversations.xmpp.jingle.stanzas.Group; import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; @@ -147,6 +146,9 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web case TIMEOUT: return State.TERMINATED_CANCEL_OR_TIMEOUT; case FAILED_APPLICATION: + case SECURITY_ERROR: + case UNSUPPORTED_TRANSPORTS: + case UNSUPPORTED_APPLICATIONS: return State.TERMINATED_APPLICATION_FAILURE; default: return State.TERMINATED_CONNECTIVITY_ERROR; @@ -182,7 +184,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web return; } webRTCWrapper.close(); - if (!isInitiator() && isInState(State.PROPOSED,State.SESSION_INITIALIZED)) { + if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) { xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); } if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) { @@ -271,6 +273,18 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents"); final State target; if (this.state == State.PROCEED) { + Preconditions.checkState( + proposedMedia != null && proposedMedia.size() > 0, + "proposed media must be set when processing pre-approved session-initiate" + ); + if (!this.proposedMedia.equals(contentMap.getMedia())) { + sendSessionTerminate(Reason.SECURITY_ERROR,String.format( + "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s", + this.proposedMedia, + contentMap.getMedia() + )); + return; + } target = State.SESSION_INITIALIZED_PRE_APPROVED; } else { target = State.SESSION_INITIALIZED; @@ -357,20 +371,20 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); return; } - sendSessionAccept(offer); + sendSessionAccept(rtpContentMap.getMedia(), offer); } - private void sendSessionAccept(final SessionDescription offer) { - discoverIceServers(iceServers -> sendSessionAccept(offer,iceServers)); + private void sendSessionAccept(final Set media, final SessionDescription offer) { + discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers)); } - private synchronized void sendSessionAccept(final SessionDescription offer, final List iceServers) { + private synchronized void sendSessionAccept(final Set media, final SessionDescription offer, final List iceServers) { if (TERMINATED.contains(this.state)) { - Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do."); + Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do."); return; } try { - setupWebRTC(iceServers); + setupWebRTC(media, iceServers); } catch (WebRTCWrapper.InitializationException e) { sendSessionTerminate(Reason.FAILED_APPLICATION); return; @@ -492,8 +506,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web input -> (RtpDescription) input ); final Collection media = Collections2.transform(descriptions, RtpDescription::getMedia); - Preconditions.checkState(!media.contains(Media.UNKNOWN),"RTP descriptions contain unknown media"); - Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": received session proposal from "+from+" for "+media); + Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media"); + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media); this.proposedMedia = Sets.newHashSet(media); if (serverMsgId != null) { this.message.setServerMsgId(serverMsgId); @@ -511,6 +525,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } private void receiveProceed(final Jid from, final String serverMsgId, final long timestamp) { + final Set media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed"); + Preconditions.checkState(media.size() > 0, "Proposed media should not be empty"); if (from.equals(id.with)) { if (isInitiator()) { if (transition(State.PROCEED)) { @@ -518,7 +534,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web this.message.setServerMsgId(serverMsgId); } this.message.setTime(timestamp); - this.sendSessionInitiate(State.SESSION_INITIALIZED_PRE_APPROVED); + this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED); } else { Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state)); } @@ -555,18 +571,18 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } } - private void sendSessionInitiate(final State targetState) { + private void sendSessionInitiate(final Set media, final State targetState) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate"); - discoverIceServers(iceServers -> sendSessionInitiate(targetState, iceServers)); + discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers)); } - private synchronized void sendSessionInitiate(final State targetState, final List iceServers) { + private synchronized void sendSessionInitiate(final Set media, final State targetState, final List iceServers) { if (TERMINATED.contains(this.state)) { - Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do."); + Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do."); return; } try { - setupWebRTC(iceServers); + setupWebRTC(media, iceServers); } catch (WebRTCWrapper.InitializationException e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize webrtc"); transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE); @@ -607,6 +623,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web writeLogMessage(target); final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId); jinglePacket.setReason(reason, text); + Log.d(Config.LOGTAG,jinglePacket.toString()); send(jinglePacket); jingleConnectionManager.finishConnection(this); } @@ -756,7 +773,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web public synchronized void endCall() { if (TERMINATED.contains(this.state)) { - Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": received endCall() when session has already been terminated. nothing to do"); + Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do"); return; } if (isInState(State.PROPOSED) && !isInitiator()) { @@ -791,9 +808,9 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator()); } - private void setupWebRTC(final List iceServers) throws WebRTCWrapper.InitializationException { + private void setupWebRTC(final Set media, final List iceServers) throws WebRTCWrapper.InitializationException { this.webRTCWrapper.setup(this.xmppConnectionService); - this.webRTCWrapper.initializePeerConnection(iceServers); + this.webRTCWrapper.initializePeerConnection(media, iceServers); } private void acceptCallFromProposed() { @@ -1018,7 +1035,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } public void setProposedMedia(final Set media) { - + this.proposedMedia = media; } private interface OnIceServersDiscovered { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index 4b207dcc5..8562f2caa 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -158,8 +158,10 @@ public class WebRTCWrapper { }); } - public void initializePeerConnection(final List iceServers) throws InitializationException { + public void initializePeerConnection(final Set media, final List iceServers) throws InitializationException { Preconditions.checkState(this.eglBase != null); + Preconditions.checkNotNull(media); + Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection"); PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder() .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext())) .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true)) @@ -168,7 +170,7 @@ public class WebRTCWrapper { final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream"); - this.optionalCapturer = getVideoCapturer(); + this.optionalCapturer = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent(); if (this.optionalCapturer.isPresent()) { final CameraVideoCapturer capturer = this.optionalCapturer.get(); @@ -183,10 +185,12 @@ public class WebRTCWrapper { } - //set up audio track - final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints()); - this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource); - stream.addTrack(this.localAudioTrack); + if (media.contains(Media.AUDIO)) { + //set up audio track + final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints()); + this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource); + stream.addTrack(this.localAudioTrack); + } final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver); @@ -201,23 +205,27 @@ public class WebRTCWrapper { public void close() { final PeerConnection peerConnection = this.peerConnection; + final Optional optionalCapturer = this.optionalCapturer; + final AppRTCAudioManager audioManager = this.appRTCAudioManager; + final EglBase eglBase = this.eglBase; if (peerConnection != null) { peerConnection.dispose(); } - final AppRTCAudioManager audioManager = this.appRTCAudioManager; if (audioManager != null) { mainHandler.post(audioManager::stop); } this.localVideoTrack = null; this.remoteVideoTrack = null; - if (this.optionalCapturer.isPresent()) { + if (optionalCapturer != null && optionalCapturer.isPresent()) { try { - this.optionalCapturer.get().stopCapture(); + optionalCapturer.get().stopCapture(); } catch (InterruptedException e) { - Log.e(Config.LOGTAG,"unable to stop capturing"); + Log.e(Config.LOGTAG, "unable to stop capturing"); } } - eglBase.release(); + if (eglBase != null) { + eglBase.release(); + } } public boolean isMicrophoneEnabled() {