From a9a35fb74bda76f68480e77dbb690dd03d7f35e5 Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Tue, 7 Apr 2020 13:15:24 +0200 Subject: [PATCH] show status in RtpSessionActivity --- .../services/XmppConnectionService.java | 43 +++++++++- .../conversations/ui/RtpSessionActivity.java | 80 ++++++++++++++++++- .../siacs/conversations/ui/XmppActivity.java | 15 +++- .../xmpp/jingle/JingleConnectionManager.java | 10 +++ .../xmpp/jingle/JingleRtpConnection.java | 57 +++++++++++-- .../xmpp/jingle/RtpEndUserState.java | 12 +++ .../xmpp/jingle/WebRTCWrapper.java | 3 +- src/main/res/values/strings.xml | 4 + 8 files changed, 209 insertions(+), 15 deletions(-) create mode 100644 src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java diff --git a/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java b/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java index d68981bf6..5951b7116 100644 --- a/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java +++ b/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java @@ -143,6 +143,7 @@ import eu.siacs.conversations.xmpp.chatstate.ChatState; import eu.siacs.conversations.xmpp.forms.Data; import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager; import eu.siacs.conversations.xmpp.jingle.OnJinglePacketReceived; +import eu.siacs.conversations.xmpp.jingle.RtpEndUserState; import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; import eu.siacs.conversations.xmpp.mam.MamReference; import eu.siacs.conversations.xmpp.pep.Avatar; @@ -267,6 +268,7 @@ public class XmppConnectionService extends Service { private final Set mOnUpdateBlocklist = Collections.newSetFromMap(new WeakHashMap()); private final Set mOnMucRosterUpdate = Collections.newSetFromMap(new WeakHashMap()); private final Set mOnKeyStatusUpdated = Collections.newSetFromMap(new WeakHashMap()); + private final Set onJingleRtpConnectionUpdate = Collections.newSetFromMap(new WeakHashMap()); private final Object LISTENER_LOCK = new Object(); @@ -2467,6 +2469,30 @@ public class XmppConnectionService extends Service { } } + public void setOnRtpConnectionUpdateListener(final OnJingleRtpConnectionUpdate listener) { + final boolean remainingListeners; + synchronized (LISTENER_LOCK) { + remainingListeners = checkListeners(); + if (!this.onJingleRtpConnectionUpdate.add(listener)) { + Log.w(Config.LOGTAG, listener.getClass().getName() + " is already registered as OnJingleRtpConnectionUpdate"); + } + } + if (remainingListeners) { + switchToForeground(); + } + } + + public void removeRtpConnectionUpdateListener(final OnJingleRtpConnectionUpdate listener) { + final boolean remainingListeners; + synchronized (LISTENER_LOCK) { + this.onJingleRtpConnectionUpdate.remove(listener); + remainingListeners = checkListeners(); + } + if (remainingListeners) { + switchToBackground(); + } + } + public void setOnMucRosterUpdateListener(OnMucRosterUpdate listener) { final boolean remainingListeners; synchronized (LISTENER_LOCK) { @@ -2499,6 +2525,7 @@ public class XmppConnectionService extends Service { && this.mOnMucRosterUpdate.size() == 0 && this.mOnUpdateBlocklist.size() == 0 && this.mOnShowErrorToasts.size() == 0 + && this.onJingleRtpConnectionUpdate.size() == 0 && this.mOnKeyStatusUpdated.size() == 0); } @@ -3943,6 +3970,12 @@ public class XmppConnectionService extends Service { } } + public void notifyJingleRtpConnectionUpdate(final Account account, final Jid with, final RtpEndUserState state) { + for(OnJingleRtpConnectionUpdate listener : threadSafeList(this.onJingleRtpConnectionUpdate)) { + listener.onJingleRtpConnectionUpdate(account, with, state); + } + } + public void updateAccountUi() { for (OnAccountUpdate listener : threadSafeList(this.mOnAccountUpdates)) { listener.onAccountUpdate(); @@ -3986,9 +4019,9 @@ public class XmppConnectionService extends Service { } } - public Account findAccountByJid(final Jid accountJid) { - for (Account account : this.accounts) { - if (account.getJid().asBareJid().equals(accountJid.asBareJid())) { + public Account findAccountByJid(final Jid jid) { + for (final Account account : this.accounts) { + if (account.getJid().asBareJid().equals(jid.asBareJid())) { return account; } } @@ -4620,6 +4653,10 @@ public class XmppConnectionService extends Service { void onConversationUpdate(); } + public interface OnJingleRtpConnectionUpdate { + void onJingleRtpConnectionUpdate(final Account account, final Jid with, final RtpEndUserState state); + } + public interface OnAccountUpdate { void onAccountUpdate(); } diff --git a/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java b/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java index 2fdf2ac65..b1cc321e0 100644 --- a/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java +++ b/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java @@ -1,16 +1,31 @@ package eu.siacs.conversations.ui; +import android.content.Intent; import android.databinding.DataBindingUtil; import android.os.Bundle; +import android.util.Log; import android.view.View; import android.view.WindowManager; +import java.lang.ref.WeakReference; + +import eu.siacs.conversations.Config; import eu.siacs.conversations.R; import eu.siacs.conversations.databinding.ActivityRtpSessionBinding; +import eu.siacs.conversations.entities.Account; +import eu.siacs.conversations.entities.Contact; +import eu.siacs.conversations.services.XmppConnectionService; +import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection; +import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection; +import eu.siacs.conversations.xmpp.jingle.RtpEndUserState; +import rocks.xmpp.addr.Jid; -public class RtpSessionActivity extends XmppActivity { +public class RtpSessionActivity extends XmppActivity implements XmppConnectionService.OnJingleRtpConnectionUpdate { public static final String EXTRA_WITH = "with"; + public static final String EXTRA_SESSION_ID = "session_id"; + + private WeakReference rtpConnectionReference; private ActivityRtpSessionBinding binding; @@ -27,11 +42,12 @@ public class RtpSessionActivity extends XmppActivity { } private void rejectCall(View view) { - + requireRtpConnection().rejectCall(); + finish(); } private void acceptCall(View view) { - + requireRtpConnection().acceptCall(); } @Override @@ -41,6 +57,64 @@ public class RtpSessionActivity extends XmppActivity { @Override void onBackendConnected() { + final Intent intent = getIntent(); + final Account account = extractAccount(intent); + final String with = intent.getStringExtra(EXTRA_WITH); + final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID); + if (with != null && sessionId != null) { + final WeakReference reference = xmppConnectionService.getJingleConnectionManager() + .findJingleRtpConnection(account, Jid.ofEscaped(with), sessionId); + if (reference == null || reference.get() == null) { + finish(); + return; + } + this.rtpConnectionReference = reference; + binding.with.setText(getWith().getDisplayName()); + showState(requireRtpConnection().getEndUserState()); + } + } + + private void showState(final RtpEndUserState state) { + switch (state) { + case INCOMING_CALL: + binding.status.setText(R.string.rtp_state_incoming_call); + break; + case CONNECTING: + binding.status.setText(R.string.rtp_state_connecting); + break; + case CONNECTED: + binding.status.setText(R.string.rtp_state_connected); + break; + case ACCEPTING_CALL: + binding.status.setText(R.string.rtp_state_accepting_call); + break; + } + } + + private Contact getWith() { + final AbstractJingleConnection.Id id = requireRtpConnection().getId(); + final Account account = id.account; + return account.getRoster().getContact(id.with); + } + + private JingleRtpConnection requireRtpConnection() { + final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + if (connection == null) { + throw new IllegalStateException("No RTP connection found"); + } + return connection; + } + + @Override + public void onJingleRtpConnectionUpdate(Account account, Jid with, RtpEndUserState state) { + final AbstractJingleConnection.Id id = requireRtpConnection().getId(); + if (account == id.account && id.with.equals(with)) { + runOnUiThread(()->{ + showState(state); + }); + } else { + Log.d(Config.LOGTAG,"received update for other rtp session"); + } } } diff --git a/src/main/java/eu/siacs/conversations/ui/XmppActivity.java b/src/main/java/eu/siacs/conversations/ui/XmppActivity.java index f476736fa..9ed35190b 100644 --- a/src/main/java/eu/siacs/conversations/ui/XmppActivity.java +++ b/src/main/java/eu/siacs/conversations/ui/XmppActivity.java @@ -305,6 +305,9 @@ public abstract class XmppActivity extends ActionBarActivity { if (this instanceof OnKeyStatusUpdated) { this.xmppConnectionService.setOnKeyStatusUpdatedListener((OnKeyStatusUpdated) this); } + if (this instanceof XmppConnectionService.OnJingleRtpConnectionUpdate) { + this.xmppConnectionService.setOnRtpConnectionUpdateListener((XmppConnectionService.OnJingleRtpConnectionUpdate) this); + } } protected void unregisterListeners() { @@ -332,6 +335,9 @@ public abstract class XmppActivity extends ActionBarActivity { if (this instanceof OnKeyStatusUpdated) { this.xmppConnectionService.removeOnNewKeysAvailableListener((OnKeyStatusUpdated) this); } + if (this instanceof XmppConnectionService.OnJingleRtpConnectionUpdate) { + this.xmppConnectionService.removeRtpConnectionUpdateListener((XmppConnectionService.OnJingleRtpConnectionUpdate) this); + } } @Override @@ -388,13 +394,18 @@ public abstract class XmppActivity extends ActionBarActivity { } } + @SuppressLint("UnsupportedChromeOsCameraSystemFeature") @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); metrics = getResources().getDisplayMetrics(); ExceptionHelper.init(getApplicationContext()); new EmojiService(this).init(); - this.isCameraFeatureAvailable = getPackageManager().hasSystemFeature(PackageManager.FEATURE_CAMERA); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) { + this.isCameraFeatureAvailable = getPackageManager().hasSystemFeature(PackageManager.FEATURE_CAMERA_ANY); + } else { + this.isCameraFeatureAvailable = getPackageManager().hasSystemFeature(PackageManager.FEATURE_CAMERA); + } this.mTheme = findTheme(); setTheme(this.mTheme); @@ -851,7 +862,7 @@ public abstract class XmppActivity extends ActionBarActivity { } protected Account extractAccount(Intent intent) { - String jid = intent != null ? intent.getStringExtra(EXTRA_ACCOUNT) : null; + final String jid = intent != null ? intent.getStringExtra(EXTRA_ACCOUNT) : null; try { return jid != null ? xmppConnectionService.findAccountByJid(Jid.of(jid)) : null; } catch (IllegalArgumentException e) { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java index bc9e25f2e..982dc00b4 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java @@ -5,6 +5,7 @@ import android.util.Log; import com.google.common.base.Objects; import com.google.common.base.Preconditions; +import java.lang.ref.WeakReference; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -244,6 +245,15 @@ public class JingleConnectionManager extends AbstractConnectionManager { } } + public WeakReference findJingleRtpConnection(Account account, Jid with, String sessionId) { + final AbstractJingleConnection.Id id = AbstractJingleConnection.Id.of(account, Jid.ofEscaped(with), sessionId); + final AbstractJingleConnection connection = connections.get(id); + if (connection instanceof JingleRtpConnection) { + return new WeakReference<>((JingleRtpConnection) connection); + } + return null; + } + public void updateProposedSessionDiscovered(Account account, Jid from, String sessionId, final DeviceDiscoveryState target) { final RtpSessionProposal sessionProposal = new RtpSessionProposal(account,from.asBareJid(),sessionId); synchronized (this.rtpSessionProposals) { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index f18212df8..8ea923804 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.webrtc.IceCandidate; +import org.webrtc.PeerConnection; import java.util.ArrayDeque; import java.util.Arrays; @@ -230,6 +231,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class); intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().asBareJid().toEscapedString()); intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString()); + intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId); intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); xmppConnectionService.startActivity(intent); } @@ -293,26 +295,59 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web xmppConnectionService.sendIqPacket(id.account, jinglePacket, null); } - - public void pickUpCall() { + public RtpEndUserState getEndUserState() { switch (this.state) { case PROPOSED: - pickupCallFromProposed(); + if (isInitiator()) { + return RtpEndUserState.RINGING; + } else { + return RtpEndUserState.INCOMING_CALL; + } + case PROCEED: + if (isInitiator()) { + return RtpEndUserState.CONNECTING; + } else { + return RtpEndUserState.ACCEPTING_CALL; + } + case SESSION_INITIALIZED: + return RtpEndUserState.CONNECTING; + case SESSION_ACCEPTED: + final PeerConnection.PeerConnectionState state = webRTCWrapper.getState(); + if (state == PeerConnection.PeerConnectionState.CONNECTED) { + return RtpEndUserState.CONNECTED; + } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) { + return RtpEndUserState.CONNECTING; + } else { + return RtpEndUserState.FAILED; + } + } + return RtpEndUserState.FAILED; + } + + + public void acceptCall() { + switch (this.state) { + case PROPOSED: + acceptCallFromProposed(); break; case SESSION_INITIALIZED: - pickupCallFromSessionInitialized(); + acceptCallFromSessionInitialized(); break; default: throw new IllegalStateException("Can not pick up call from " + this.state); } } + public void rejectCall() { + Log.d(Config.LOGTAG, "todo rejecting call"); + } + private void setupWebRTC() { this.webRTCWrapper.setup(this.xmppConnectionService); this.webRTCWrapper.initializePeerConnection(); } - private void pickupCallFromProposed() { + private void acceptCallFromProposed() { transitionOrThrow(State.PROCEED); final MessagePacket messagePacket = new MessagePacket(); messagePacket.setTo(id.with); @@ -322,7 +357,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web xmppConnectionService.sendMessagePacket(id.account, messagePacket); } - private void pickupCallFromSessionInitialized() { + private void acceptCallFromSessionInitialized() { } @@ -335,6 +370,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web if (validTransitions != null && validTransitions.contains(target)) { this.state = target; Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target); + updateEndUserState(); return true; } else { return false; @@ -353,4 +389,13 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString()); sendTransportInfo(iceCandidate.sdpMid, candidate); } + + @Override + public void onConnectionChange(PeerConnection.PeerConnectionState newState) { + updateEndUserState(); + } + + private void updateEndUserState() { + xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, getEndUserState()); + } } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java new file mode 100644 index 000000000..bfae8e53b --- /dev/null +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java @@ -0,0 +1,12 @@ +package eu.siacs.conversations.xmpp.jingle; + +public enum RtpEndUserState { + INCOMING_CALL, //received a 'propose' message + CONNECTING, //session-initiate or session-accepted but no webrtc peer connection yet + CONNECTED, //session-accepted and webrtc peer connection is connected + FINDING_DEVICE, //'propose' has been sent out; no 184 ack yet + RINGING, //'propose' has been sent out and it has been 184 acked + ACCEPTED_ON_OTHER_DEVICE, //received 'accept' from one of our own devices + ACCEPTING_CALL, //'proceed' message has been sent; but no session-initiate has been received + FAILED //something went wrong. TODO needs more concrete error states +} diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index f4b03a062..2fdd99003 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -50,7 +50,7 @@ public class WebRTCWrapper { @Override public void onConnectionChange(PeerConnection.PeerConnectionState newState) { - Log.d(Config.LOGTAG, "onConnectionChange(" + newState + ")"); + eventCallback.onConnectionChange(newState); } @Override @@ -329,5 +329,6 @@ public class WebRTCWrapper { public interface EventCallback { void onIceCandidate(IceCandidate iceCandidate); + void onConnectionChange(PeerConnection.PeerConnectionState newState); } } diff --git a/src/main/res/values/strings.xml b/src/main/res/values/strings.xml index d91374ade..b7de99684 100644 --- a/src/main/res/values/strings.xml +++ b/src/main/res/values/strings.xml @@ -885,6 +885,10 @@ About Please enable an account Make call + Incoming call + Connecting + Connected + Accepting call View %1$d Participant View %1$d Participants