transmit media from proposal to actual session

This commit is contained in:
Daniel Gultsch 2020-04-15 12:07:19 +02:00
parent 8c273e7eee
commit d057ae3439
3 changed files with 66 additions and 38 deletions

View File

@ -7,6 +7,7 @@ import com.google.common.base.Function;
import com.google.common.base.Objects; import com.google.common.base.Objects;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.Collections2; import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableSet;
import org.checkerframework.checker.nullness.compatqual.NullableDecl; import org.checkerframework.checker.nullness.compatqual.NullableDecl;
@ -163,6 +164,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
if (fromSelf) { if (fromSelf) {
Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": ignore jingle message from self"); Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": ignore jingle message from self");
//TODO proceed from self should maybe dedup/change the busy that we set earlier
return; return;
} }
@ -176,16 +178,17 @@ public class JingleConnectionManager extends AbstractConnectionManager {
if (rtpDescriptions.size() > 0 && rtpDescriptions.size() == descriptions.size() && !usesTor(account)) { if (rtpDescriptions.size() > 0 && rtpDescriptions.size() == descriptions.size() && !usesTor(account)) {
final Collection<Media> media = Collections2.transform(rtpDescriptions, RtpDescription::getMedia); final Collection<Media> media = Collections2.transform(rtpDescriptions, RtpDescription::getMedia);
if (media.contains(Media.UNKNOWN)) { if (media.contains(Media.UNKNOWN)) {
Log.d(Config.LOGTAG,account.getJid().asBareJid()+": encountered unknown media in session proposal. "+propose); Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": encountered unknown media in session proposal. " + propose);
return; return;
} }
if (isBusy()) { //TODO only if no other devices are active if (isBusy()) { //TODO only if no other devices are active
//TODO create //TODO create busy
final MessagePacket reject = mXmppConnectionService.getMessageGenerator().sessionReject(from, sessionId); final MessagePacket reject = mXmppConnectionService.getMessageGenerator().sessionReject(from, sessionId);
mXmppConnectionService.sendMessagePacket(account, reject); mXmppConnectionService.sendMessagePacket(account, reject);
} else { } else {
final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, from); final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, from);
this.connections.put(id, rtpConnection); this.connections.put(id, rtpConnection);
rtpConnection.setProposedMedia(ImmutableSet.copyOf(media));
rtpConnection.deliveryMessage(from, message, serverMsgId, timestamp); rtpConnection.deliveryMessage(from, message, serverMsgId, timestamp);
} }
} else { } else {
@ -193,7 +196,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
} }
} else if ("proceed".equals(message.getName())) { } else if ("proceed".equals(message.getName())) {
synchronized (rtpSessionProposals) { synchronized (rtpSessionProposals) {
final RtpSessionProposal proposal = getRtpSessionProposal(account,from.asBareJid(),sessionId); final RtpSessionProposal proposal = getRtpSessionProposal(account, from.asBareJid(), sessionId);
if (proposal != null) { if (proposal != null) {
rtpSessionProposals.remove(proposal); rtpSessionProposals.remove(proposal);
final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, account.getJid()); final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, account.getJid());
@ -222,7 +225,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
} }
private RtpSessionProposal getRtpSessionProposal(final Account account, Jid from, String sessionId) { private RtpSessionProposal getRtpSessionProposal(final Account account, Jid from, String sessionId) {
for(RtpSessionProposal rtpSessionProposal : rtpSessionProposals.keySet()) { for (RtpSessionProposal rtpSessionProposal : rtpSessionProposals.keySet()) {
if (rtpSessionProposal.sessionId.equals(sessionId) && rtpSessionProposal.with.equals(from) && rtpSessionProposal.account.getJid().equals(account.getJid())) { if (rtpSessionProposal.sessionId.equals(sessionId) && rtpSessionProposal.with.equals(from) && rtpSessionProposal.account.getJid().equals(account.getJid())) {
return rtpSessionProposal; return rtpSessionProposal;
} }
@ -424,9 +427,9 @@ public class JingleConnectionManager extends AbstractConnectionManager {
} }
public void updateProposedSessionDiscovered(Account account, Jid from, String sessionId, final DeviceDiscoveryState target) { public void updateProposedSessionDiscovered(Account account, Jid from, String sessionId, final DeviceDiscoveryState target) {
final RtpSessionProposal sessionProposal = new RtpSessionProposal(account, from.asBareJid(), sessionId);
synchronized (this.rtpSessionProposals) { synchronized (this.rtpSessionProposals) {
final DeviceDiscoveryState currentState = rtpSessionProposals.get(sessionProposal); final RtpSessionProposal sessionProposal = getRtpSessionProposal(account, from.asBareJid(), sessionId);
final DeviceDiscoveryState currentState = sessionProposal == null ? null : rtpSessionProposals.get(sessionProposal);
if (currentState == null) { if (currentState == null) {
Log.d(Config.LOGTAG, "unable to find session proposal for session id " + sessionId); Log.d(Config.LOGTAG, "unable to find session proposal for session id " + sessionId);
return; return;
@ -491,7 +494,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
public final Set<Media> media; public final Set<Media> media;
private RtpSessionProposal(Account account, Jid with, String sessionId) { private RtpSessionProposal(Account account, Jid with, String sessionId) {
this(account,with,sessionId, Collections.emptySet()); this(account, with, sessionId, Collections.emptySet());
} }
private RtpSessionProposal(Account account, Jid with, String sessionId, Set<Media> media) { private RtpSessionProposal(Account account, Jid with, String sessionId, Set<Media> media) {

View File

@ -33,7 +33,6 @@ import eu.siacs.conversations.entities.RtpSessionStatus;
import eu.siacs.conversations.services.AppRTCAudioManager; import eu.siacs.conversations.services.AppRTCAudioManager;
import eu.siacs.conversations.xml.Element; import eu.siacs.conversations.xml.Element;
import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xml.Namespace;
import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription;
import eu.siacs.conversations.xmpp.jingle.stanzas.Group; import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
@ -147,6 +146,9 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
case TIMEOUT: case TIMEOUT:
return State.TERMINATED_CANCEL_OR_TIMEOUT; return State.TERMINATED_CANCEL_OR_TIMEOUT;
case FAILED_APPLICATION: case FAILED_APPLICATION:
case SECURITY_ERROR:
case UNSUPPORTED_TRANSPORTS:
case UNSUPPORTED_APPLICATIONS:
return State.TERMINATED_APPLICATION_FAILURE; return State.TERMINATED_APPLICATION_FAILURE;
default: default:
return State.TERMINATED_CONNECTIVITY_ERROR; return State.TERMINATED_CONNECTIVITY_ERROR;
@ -182,7 +184,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
return; return;
} }
webRTCWrapper.close(); webRTCWrapper.close();
if (!isInitiator() && isInState(State.PROPOSED,State.SESSION_INITIALIZED)) { if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
} }
if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) { if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
@ -271,6 +273,18 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents"); Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
final State target; final State target;
if (this.state == State.PROCEED) { if (this.state == State.PROCEED) {
Preconditions.checkState(
proposedMedia != null && proposedMedia.size() > 0,
"proposed media must be set when processing pre-approved session-initiate"
);
if (!this.proposedMedia.equals(contentMap.getMedia())) {
sendSessionTerminate(Reason.SECURITY_ERROR,String.format(
"Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
this.proposedMedia,
contentMap.getMedia()
));
return;
}
target = State.SESSION_INITIALIZED_PRE_APPROVED; target = State.SESSION_INITIALIZED_PRE_APPROVED;
} else { } else {
target = State.SESSION_INITIALIZED; target = State.SESSION_INITIALIZED;
@ -357,20 +371,20 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
return; return;
} }
sendSessionAccept(offer); sendSessionAccept(rtpContentMap.getMedia(), offer);
} }
private void sendSessionAccept(final SessionDescription offer) { private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
discoverIceServers(iceServers -> sendSessionAccept(offer,iceServers)); discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
} }
private synchronized void sendSessionAccept(final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) { private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
if (TERMINATED.contains(this.state)) { if (TERMINATED.contains(this.state)) {
Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do."); Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
return; return;
} }
try { try {
setupWebRTC(iceServers); setupWebRTC(media, iceServers);
} catch (WebRTCWrapper.InitializationException e) { } catch (WebRTCWrapper.InitializationException e) {
sendSessionTerminate(Reason.FAILED_APPLICATION); sendSessionTerminate(Reason.FAILED_APPLICATION);
return; return;
@ -492,8 +506,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
input -> (RtpDescription) input input -> (RtpDescription) input
); );
final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia); final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
Preconditions.checkState(!media.contains(Media.UNKNOWN),"RTP descriptions contain unknown media"); Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": received session proposal from "+from+" for "+media); Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
this.proposedMedia = Sets.newHashSet(media); this.proposedMedia = Sets.newHashSet(media);
if (serverMsgId != null) { if (serverMsgId != null) {
this.message.setServerMsgId(serverMsgId); this.message.setServerMsgId(serverMsgId);
@ -511,6 +525,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
} }
private void receiveProceed(final Jid from, final String serverMsgId, final long timestamp) { private void receiveProceed(final Jid from, final String serverMsgId, final long timestamp) {
final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
if (from.equals(id.with)) { if (from.equals(id.with)) {
if (isInitiator()) { if (isInitiator()) {
if (transition(State.PROCEED)) { if (transition(State.PROCEED)) {
@ -518,7 +534,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
this.message.setServerMsgId(serverMsgId); this.message.setServerMsgId(serverMsgId);
} }
this.message.setTime(timestamp); this.message.setTime(timestamp);
this.sendSessionInitiate(State.SESSION_INITIALIZED_PRE_APPROVED); this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
} else { } else {
Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state)); Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
} }
@ -555,18 +571,18 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
} }
} }
private void sendSessionInitiate(final State targetState) { private void sendSessionInitiate(final Set<Media> media, final State targetState) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate"); Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
discoverIceServers(iceServers -> sendSessionInitiate(targetState, iceServers)); discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
} }
private synchronized void sendSessionInitiate(final State targetState, final List<PeerConnection.IceServer> iceServers) { private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
if (TERMINATED.contains(this.state)) { if (TERMINATED.contains(this.state)) {
Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do."); Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
return; return;
} }
try { try {
setupWebRTC(iceServers); setupWebRTC(media, iceServers);
} catch (WebRTCWrapper.InitializationException e) { } catch (WebRTCWrapper.InitializationException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize webrtc"); Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize webrtc");
transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE); transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
@ -607,6 +623,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
writeLogMessage(target); writeLogMessage(target);
final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId); final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
jinglePacket.setReason(reason, text); jinglePacket.setReason(reason, text);
Log.d(Config.LOGTAG,jinglePacket.toString());
send(jinglePacket); send(jinglePacket);
jingleConnectionManager.finishConnection(this); jingleConnectionManager.finishConnection(this);
} }
@ -756,7 +773,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
public synchronized void endCall() { public synchronized void endCall() {
if (TERMINATED.contains(this.state)) { if (TERMINATED.contains(this.state)) {
Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": received endCall() when session has already been terminated. nothing to do"); Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
return; return;
} }
if (isInState(State.PROPOSED) && !isInitiator()) { if (isInState(State.PROPOSED) && !isInitiator()) {
@ -791,9 +808,9 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator()); throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
} }
private void setupWebRTC(final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException { private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
this.webRTCWrapper.setup(this.xmppConnectionService); this.webRTCWrapper.setup(this.xmppConnectionService);
this.webRTCWrapper.initializePeerConnection(iceServers); this.webRTCWrapper.initializePeerConnection(media, iceServers);
} }
private void acceptCallFromProposed() { private void acceptCallFromProposed() {
@ -1018,7 +1035,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
} }
public void setProposedMedia(final Set<Media> media) { public void setProposedMedia(final Set<Media> media) {
this.proposedMedia = media;
} }
private interface OnIceServersDiscovered { private interface OnIceServersDiscovered {

View File

@ -158,8 +158,10 @@ public class WebRTCWrapper {
}); });
} }
public void initializePeerConnection(final List<PeerConnection.IceServer> iceServers) throws InitializationException { public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
Preconditions.checkState(this.eglBase != null); Preconditions.checkState(this.eglBase != null);
Preconditions.checkNotNull(media);
Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder() PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
.setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext())) .setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
.setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true)) .setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
@ -168,7 +170,7 @@ public class WebRTCWrapper {
final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream"); final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
this.optionalCapturer = getVideoCapturer(); this.optionalCapturer = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
if (this.optionalCapturer.isPresent()) { if (this.optionalCapturer.isPresent()) {
final CameraVideoCapturer capturer = this.optionalCapturer.get(); final CameraVideoCapturer capturer = this.optionalCapturer.get();
@ -183,10 +185,12 @@ public class WebRTCWrapper {
} }
if (media.contains(Media.AUDIO)) {
//set up audio track //set up audio track
final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints()); final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource); this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
stream.addTrack(this.localAudioTrack); stream.addTrack(this.localAudioTrack);
}
final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver); final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
@ -201,24 +205,28 @@ public class WebRTCWrapper {
public void close() { public void close() {
final PeerConnection peerConnection = this.peerConnection; final PeerConnection peerConnection = this.peerConnection;
final Optional<CameraVideoCapturer> optionalCapturer = this.optionalCapturer;
final AppRTCAudioManager audioManager = this.appRTCAudioManager;
final EglBase eglBase = this.eglBase;
if (peerConnection != null) { if (peerConnection != null) {
peerConnection.dispose(); peerConnection.dispose();
} }
final AppRTCAudioManager audioManager = this.appRTCAudioManager;
if (audioManager != null) { if (audioManager != null) {
mainHandler.post(audioManager::stop); mainHandler.post(audioManager::stop);
} }
this.localVideoTrack = null; this.localVideoTrack = null;
this.remoteVideoTrack = null; this.remoteVideoTrack = null;
if (this.optionalCapturer.isPresent()) { if (optionalCapturer != null && optionalCapturer.isPresent()) {
try { try {
this.optionalCapturer.get().stopCapture(); optionalCapturer.get().stopCapture();
} catch (InterruptedException e) { } catch (InterruptedException e) {
Log.e(Config.LOGTAG,"unable to stop capturing"); Log.e(Config.LOGTAG, "unable to stop capturing");
} }
} }
if (eglBase != null) {
eglBase.release(); eglBase.release();
} }
}
public boolean isMicrophoneEnabled() { public boolean isMicrophoneEnabled() {
final AudioTrack audioTrack = this.localAudioTrack; final AudioTrack audioTrack = this.localAudioTrack;