sum7/Conversations
sum7
/
Conversations
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

transmit media from proposal to actual session

This commit is contained in:
Daniel Gultsch 2020-04-15 12:07:19 +02:00
parent 8c273e7eee
commit d057ae3439
3 changed files with 66 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java
View File

@ -7,6 +7,7 @@ import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableSet;
import org.checkerframework.checker.nullness.compatqual.NullableDecl;
@ -163,6 +164,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
if (fromSelf) {
Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": ignore jingle message from self");
//TODO proceed from self should maybe dedup/change the busy that we set earlier
return;
}
@ -176,16 +178,17 @@ public class JingleConnectionManager extends AbstractConnectionManager {
if (rtpDescriptions.size() > 0 && rtpDescriptions.size() == descriptions.size() && !usesTor(account)) {
final Collection<Media> media = Collections2.transform(rtpDescriptions, RtpDescription::getMedia);
if (media.contains(Media.UNKNOWN)) {
Log.d(Config.LOGTAG,account.getJid().asBareJid()+": encountered unknown media in session proposal. "+propose);
Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": encountered unknown media in session proposal. " + propose);
return;
}
if (isBusy()) { //TODO only if no other devices are active
//TODO create
//TODO create busy
final MessagePacket reject = mXmppConnectionService.getMessageGenerator().sessionReject(from, sessionId);
mXmppConnectionService.sendMessagePacket(account, reject);
} else {
final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, from);
this.connections.put(id, rtpConnection);
rtpConnection.setProposedMedia(ImmutableSet.copyOf(media));
rtpConnection.deliveryMessage(from, message, serverMsgId, timestamp);
}
} else {
@ -193,7 +196,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
}
} else if ("proceed".equals(message.getName())) {
synchronized (rtpSessionProposals) {
final RtpSessionProposal proposal = getRtpSessionProposal(account,from.asBareJid(),sessionId);
final RtpSessionProposal proposal = getRtpSessionProposal(account, from.asBareJid(), sessionId);
if (proposal != null) {
rtpSessionProposals.remove(proposal);
final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, account.getJid());
@ -222,7 +225,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
}
private RtpSessionProposal getRtpSessionProposal(final Account account, Jid from, String sessionId) {
for(RtpSessionProposal rtpSessionProposal : rtpSessionProposals.keySet()) {
for (RtpSessionProposal rtpSessionProposal : rtpSessionProposals.keySet()) {
if (rtpSessionProposal.sessionId.equals(sessionId) && rtpSessionProposal.with.equals(from) && rtpSessionProposal.account.getJid().equals(account.getJid())) {
return rtpSessionProposal;
}
@ -424,9 +427,9 @@ public class JingleConnectionManager extends AbstractConnectionManager {
}
public void updateProposedSessionDiscovered(Account account, Jid from, String sessionId, final DeviceDiscoveryState target) {
final RtpSessionProposal sessionProposal = new RtpSessionProposal(account, from.asBareJid(), sessionId);
synchronized (this.rtpSessionProposals) {
final DeviceDiscoveryState currentState = rtpSessionProposals.get(sessionProposal);
final RtpSessionProposal sessionProposal = getRtpSessionProposal(account, from.asBareJid(), sessionId);
final DeviceDiscoveryState currentState = sessionProposal == null ? null : rtpSessionProposals.get(sessionProposal);
if (currentState == null) {
Log.d(Config.LOGTAG, "unable to find session proposal for session id " + sessionId);
return;
@ -491,7 +494,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
public final Set<Media> media;
private RtpSessionProposal(Account account, Jid with, String sessionId) {
this(account,with,sessionId, Collections.emptySet());
this(account, with, sessionId, Collections.emptySet());
}
private RtpSessionProposal(Account account, Jid with, String sessionId, Set<Media> media) {

57
src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java
View File

@ -33,7 +33,6 @@ import eu.siacs.conversations.entities.RtpSessionStatus;
import eu.siacs.conversations.services.AppRTCAudioManager;
import eu.siacs.conversations.xml.Element;
import eu.siacs.conversations.xml.Namespace;
import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription;
import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
@ -147,6 +146,9 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
case TIMEOUT:
return State.TERMINATED_CANCEL_OR_TIMEOUT;
case FAILED_APPLICATION:
case SECURITY_ERROR:
case UNSUPPORTED_TRANSPORTS:
case UNSUPPORTED_APPLICATIONS:
return State.TERMINATED_APPLICATION_FAILURE;
default:
return State.TERMINATED_CONNECTIVITY_ERROR;
@ -182,7 +184,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
return;
}
webRTCWrapper.close();
if (!isInitiator() && isInState(State.PROPOSED,State.SESSION_INITIALIZED)) {
if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
}
if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
@ -271,6 +273,18 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
final State target;
if (this.state == State.PROCEED) {
Preconditions.checkState(
proposedMedia != null && proposedMedia.size() > 0,
"proposed media must be set when processing pre-approved session-initiate"
);
if (!this.proposedMedia.equals(contentMap.getMedia())) {
sendSessionTerminate(Reason.SECURITY_ERROR,String.format(
"Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
this.proposedMedia,
contentMap.getMedia()
));
return;
}
target = State.SESSION_INITIALIZED_PRE_APPROVED;
} else {
target = State.SESSION_INITIALIZED;
@ -357,20 +371,20 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
return;
}
sendSessionAccept(offer);
sendSessionAccept(rtpContentMap.getMedia(), offer);
}
private void sendSessionAccept(final SessionDescription offer) {
discoverIceServers(iceServers -> sendSessionAccept(offer,iceServers));
private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
}
private synchronized void sendSessionAccept(final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
if (TERMINATED.contains(this.state)) {
Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do.");
Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
return;
}
try {
setupWebRTC(iceServers);
setupWebRTC(media, iceServers);
} catch (WebRTCWrapper.InitializationException e) {
sendSessionTerminate(Reason.FAILED_APPLICATION);
return;
@ -492,8 +506,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
input -> (RtpDescription) input
);
final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
Preconditions.checkState(!media.contains(Media.UNKNOWN),"RTP descriptions contain unknown media");
Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": received session proposal from "+from+" for "+media);
Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
this.proposedMedia = Sets.newHashSet(media);
if (serverMsgId != null) {
this.message.setServerMsgId(serverMsgId);
@ -511,6 +525,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
private void receiveProceed(final Jid from, final String serverMsgId, final long timestamp) {
final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
if (from.equals(id.with)) {
if (isInitiator()) {
if (transition(State.PROCEED)) {
@ -518,7 +534,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
this.message.setServerMsgId(serverMsgId);
}
this.message.setTime(timestamp);
this.sendSessionInitiate(State.SESSION_INITIALIZED_PRE_APPROVED);
this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
} else {
Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
}
@ -555,18 +571,18 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
}
private void sendSessionInitiate(final State targetState) {
private void sendSessionInitiate(final Set<Media> media, final State targetState) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
discoverIceServers(iceServers -> sendSessionInitiate(targetState, iceServers));
discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
}
private synchronized void sendSessionInitiate(final State targetState, final List<PeerConnection.IceServer> iceServers) {
private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
if (TERMINATED.contains(this.state)) {
Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": ICE servers got discovered when session was already terminated. nothing to do.");
Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
return;
}
try {
setupWebRTC(iceServers);
setupWebRTC(media, iceServers);
} catch (WebRTCWrapper.InitializationException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize webrtc");
transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
@ -607,6 +623,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
writeLogMessage(target);
final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
jinglePacket.setReason(reason, text);
Log.d(Config.LOGTAG,jinglePacket.toString());
send(jinglePacket);
jingleConnectionManager.finishConnection(this);
}
@ -756,7 +773,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
public synchronized void endCall() {
if (TERMINATED.contains(this.state)) {
Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": received endCall() when session has already been terminated. nothing to do");
Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
return;
}
if (isInState(State.PROPOSED) && !isInitiator()) {
@ -791,9 +808,9 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
}
private void setupWebRTC(final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
this.webRTCWrapper.setup(this.xmppConnectionService);
this.webRTCWrapper.initializePeerConnection(iceServers);
this.webRTCWrapper.initializePeerConnection(media, iceServers);
}
private void acceptCallFromProposed() {
@ -1018,7 +1035,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
public void setProposedMedia(final Set<Media> media) {
this.proposedMedia = media;
}
private interface OnIceServersDiscovered {

30
src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java
View File

@ -158,8 +158,10 @@ public class WebRTCWrapper {
});
}
public void initializePeerConnection(final List<PeerConnection.IceServer> iceServers) throws InitializationException {
public void initializePeerConnection(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws InitializationException {
Preconditions.checkState(this.eglBase != null);
Preconditions.checkNotNull(media);
Preconditions.checkArgument(media.size() > 0, "media can not be empty when initializing peer connection");
PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder()
.setVideoDecoderFactory(new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()))
.setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
@ -168,7 +170,7 @@ public class WebRTCWrapper {
final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
this.optionalCapturer = getVideoCapturer();
this.optionalCapturer = media.contains(Media.VIDEO) ? getVideoCapturer() : Optional.absent();
if (this.optionalCapturer.isPresent()) {
final CameraVideoCapturer capturer = this.optionalCapturer.get();
@ -183,10 +185,12 @@ public class WebRTCWrapper {
}
//set up audio track
final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
stream.addTrack(this.localAudioTrack);
if (media.contains(Media.AUDIO)) {
//set up audio track
final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
this.localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
stream.addTrack(this.localAudioTrack);
}
final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
@ -201,23 +205,27 @@ public class WebRTCWrapper {
public void close() {
final PeerConnection peerConnection = this.peerConnection;
final Optional<CameraVideoCapturer> optionalCapturer = this.optionalCapturer;
final AppRTCAudioManager audioManager = this.appRTCAudioManager;
final EglBase eglBase = this.eglBase;
if (peerConnection != null) {
peerConnection.dispose();
}
final AppRTCAudioManager audioManager = this.appRTCAudioManager;
if (audioManager != null) {
mainHandler.post(audioManager::stop);
}
this.localVideoTrack = null;
this.remoteVideoTrack = null;
if (this.optionalCapturer.isPresent()) {
if (optionalCapturer != null && optionalCapturer.isPresent()) {
try {
this.optionalCapturer.get().stopCapture();
optionalCapturer.get().stopCapture();
} catch (InterruptedException e) {
Log.e(Config.LOGTAG,"unable to stop capturing");
Log.e(Config.LOGTAG, "unable to stop capturing");
}
}
eglBase.release();
if (eglBase != null) {
eglBase.release();
}
}
public boolean isMicrophoneEnabled() {