support picture in picture for video calls

This commit is contained in:
Daniel Gultsch 2020-04-16 17:38:05 +02:00
parent 21e412ef6f
commit ea2ed85ed7
8 changed files with 95 additions and 37 deletions

View File

@ -292,7 +292,9 @@
android:name=".ui.ChannelDiscoveryActivity" android:name=".ui.ChannelDiscoveryActivity"
android:label="@string/discover_channels" /> android:label="@string/discover_channels" />
<activity android:name=".ui.RtpSessionActivity" <activity android:name=".ui.RtpSessionActivity"
android:launchMode="singleTop"/> android:supportsPictureInPicture="true"
android:launchMode="singleTask"
android:autoRemoveFromRecents="true"/>
</application> </application>
</manifest> </manifest>

View File

@ -2,6 +2,7 @@ package eu.siacs.conversations.ui;
import android.Manifest; import android.Manifest;
import android.annotation.SuppressLint; import android.annotation.SuppressLint;
import android.app.PictureInPictureParams;
import android.content.Context; import android.content.Context;
import android.content.Intent; import android.content.Intent;
import android.databinding.DataBindingUtil; import android.databinding.DataBindingUtil;
@ -11,6 +12,7 @@ import android.os.PowerManager;
import android.support.annotation.NonNull; import android.support.annotation.NonNull;
import android.support.annotation.StringRes; import android.support.annotation.StringRes;
import android.util.Log; import android.util.Log;
import android.util.Rational;
import android.view.View; import android.view.View;
import android.view.WindowManager; import android.view.WindowManager;
import android.widget.Toast; import android.widget.Toast;
@ -19,6 +21,7 @@ import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import org.webrtc.PeerConnection;
import org.webrtc.RendererCommon; import org.webrtc.RendererCommon;
import org.webrtc.SurfaceViewRenderer; import org.webrtc.SurfaceViewRenderer;
import org.webrtc.VideoTrack; import org.webrtc.VideoTrack;
@ -47,31 +50,33 @@ import static java.util.Arrays.asList;
public class RtpSessionActivity extends XmppActivity implements XmppConnectionService.OnJingleRtpConnectionUpdate { public class RtpSessionActivity extends XmppActivity implements XmppConnectionService.OnJingleRtpConnectionUpdate {
public static final String EXTRA_WITH = "with";
public static final String EXTRA_SESSION_ID = "session_id";
public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
public static final String EXTRA_LAST_ACTION = "last_action";
public static final String ACTION_ACCEPT_CALL = "action_accept_call";
public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
private static final List<RtpEndUserState> END_CARD = Arrays.asList( private static final List<RtpEndUserState> END_CARD = Arrays.asList(
RtpEndUserState.APPLICATION_ERROR, RtpEndUserState.APPLICATION_ERROR,
RtpEndUserState.DECLINED_OR_BUSY, RtpEndUserState.DECLINED_OR_BUSY,
RtpEndUserState.CONNECTIVITY_ERROR RtpEndUserState.CONNECTIVITY_ERROR
); );
private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session"; private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session";
private static final int REQUEST_ACCEPT_CALL = 0x1111; private static final int REQUEST_ACCEPT_CALL = 0x1111;
public static final String EXTRA_WITH = "with";
public static final String EXTRA_SESSION_ID = "session_id";
public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state";
public static final String EXTRA_LAST_ACTION = "last_action";
public static final String ACTION_ACCEPT_CALL = "action_accept_call";
public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call";
public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call";
private WeakReference<JingleRtpConnection> rtpConnectionReference; private WeakReference<JingleRtpConnection> rtpConnectionReference;
private ActivityRtpSessionBinding binding; private ActivityRtpSessionBinding binding;
private PowerManager.WakeLock mProximityWakeLock; private PowerManager.WakeLock mProximityWakeLock;
private static Set<Media> actionToMedia(final String action) {
if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
} else {
return ImmutableSet.of(Media.AUDIO);
}
}
@Override @Override
public void onCreate(Bundle savedInstanceState) { public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState); super.onCreate(savedInstanceState);
@ -235,14 +240,6 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
} }
} }
private static Set<Media> actionToMedia(final String action) {
if (ACTION_MAKE_VIDEO_CALL.equals(action)) {
return ImmutableSet.of(Media.AUDIO, Media.VIDEO);
} else {
return ImmutableSet.of(Media.AUDIO);
}
}
private void proposeJingleRtpSession(final Account account, final Jid with, final Set<Media> media) { private void proposeJingleRtpSession(final Account account, final Jid with, final Set<Media> media) {
xmppConnectionService.getJingleConnectionManager().proposeJingleRtpSession(account, with, media); xmppConnectionService.getJingleConnectionManager().proposeJingleRtpSession(account, with, media);
putScreenInCallMode(media); putScreenInCallMode(media);
@ -284,6 +281,29 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
super.onBackPressed(); super.onBackPressed();
} }
@Override
public void onUserLeaveHint() {
Log.d(Config.LOGTAG, "user leave hint");
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
if (shouldBePictureInPicture()) {
enterPictureInPictureMode(
new PictureInPictureParams.Builder()
.setAspectRatio(new Rational(10, 16))
.build()
);
}
}
}
private boolean shouldBePictureInPicture() {
try {
final JingleRtpConnection rtpConnection = requireRtpConnection();
return rtpConnection.getMedia().contains(Media.VIDEO) && rtpConnection.getEndUserState() == RtpEndUserState.CONNECTED;
} catch (IllegalStateException e) {
return false;
}
}
private void initializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) { private void initializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) {
final WeakReference<JingleRtpConnection> reference = xmppConnectionService.getJingleConnectionManager() final WeakReference<JingleRtpConnection> reference = xmppConnectionService.getJingleConnectionManager()
@ -380,7 +400,11 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
@SuppressLint("RestrictedApi") @SuppressLint("RestrictedApi")
private void updateButtonConfiguration(final RtpEndUserState state) { private void updateButtonConfiguration(final RtpEndUserState state) {
if (state == RtpEndUserState.INCOMING_CALL) { if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) {
this.binding.rejectCall.setVisibility(View.INVISIBLE);
this.binding.endCall.setVisibility(View.INVISIBLE);
this.binding.acceptCall.setVisibility(View.INVISIBLE);
} else if (state == RtpEndUserState.INCOMING_CALL) {
this.binding.rejectCall.setOnClickListener(this::rejectCall); this.binding.rejectCall.setOnClickListener(this::rejectCall);
this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp); this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_white_48dp);
this.binding.rejectCall.setVisibility(View.VISIBLE); this.binding.rejectCall.setVisibility(View.VISIBLE);
@ -388,10 +412,6 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
this.binding.acceptCall.setOnClickListener(this::acceptCall); this.binding.acceptCall.setOnClickListener(this::acceptCall);
this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp); this.binding.acceptCall.setImageResource(R.drawable.ic_call_white_48dp);
this.binding.acceptCall.setVisibility(View.VISIBLE); this.binding.acceptCall.setVisibility(View.VISIBLE);
} else if (state == RtpEndUserState.ENDING_CALL) {
this.binding.rejectCall.setVisibility(View.INVISIBLE);
this.binding.endCall.setVisibility(View.INVISIBLE);
this.binding.acceptCall.setVisibility(View.INVISIBLE);
} else if (state == RtpEndUserState.DECLINED_OR_BUSY) { } else if (state == RtpEndUserState.DECLINED_OR_BUSY) {
this.binding.rejectCall.setVisibility(View.INVISIBLE); this.binding.rejectCall.setVisibility(View.INVISIBLE);
this.binding.endCall.setOnClickListener(this::exit); this.binding.endCall.setOnClickListener(this::exit);
@ -416,13 +436,21 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
updateInCallButtonConfiguration(state); updateInCallButtonConfiguration(state);
} }
private boolean isPictureInPicture() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
return isInPictureInPictureMode();
} else {
return false;
}
}
private void updateInCallButtonConfiguration() { private void updateInCallButtonConfiguration() {
updateInCallButtonConfiguration(requireRtpConnection().getEndUserState()); updateInCallButtonConfiguration(requireRtpConnection().getEndUserState());
} }
@SuppressLint("RestrictedApi") @SuppressLint("RestrictedApi")
private void updateInCallButtonConfiguration(final RtpEndUserState state) { private void updateInCallButtonConfiguration(final RtpEndUserState state) {
if (state == RtpEndUserState.CONNECTED) { if (state == RtpEndUserState.CONNECTED && !isPictureInPicture()) {
if (getMedia().contains(Media.VIDEO)) { if (getMedia().contains(Media.VIDEO)) {
updateInCallButtonConfigurationVideo(requireRtpConnection().isVideoEnabled()); updateInCallButtonConfigurationVideo(requireRtpConnection().isVideoEnabled());
} else { } else {
@ -513,12 +541,23 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) { if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) {
binding.localVideo.setVisibility(View.GONE); binding.localVideo.setVisibility(View.GONE);
binding.remoteVideo.setVisibility(View.GONE); binding.remoteVideo.setVisibility(View.GONE);
if (isPictureInPicture()) {
binding.appBarLayout.setVisibility(View.GONE);
binding.pipPlaceholder.setVisibility(View.VISIBLE);
if (state == RtpEndUserState.APPLICATION_ERROR || state == RtpEndUserState.CONNECTIVITY_ERROR) {
binding.pipWarning.setVisibility(View.VISIBLE);
} else {
binding.pipWarning.setVisibility(View.GONE);
}
} else {
binding.appBarLayout.setVisibility(View.VISIBLE); binding.appBarLayout.setVisibility(View.VISIBLE);
binding.pipPlaceholder.setVisibility(View.GONE);
}
getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN);
return; return;
} }
final Optional<VideoTrack> localVideoTrack = requireRtpConnection().geLocalVideoTrack(); final Optional<VideoTrack> localVideoTrack = requireRtpConnection().geLocalVideoTrack();
if (localVideoTrack.isPresent()) { if (localVideoTrack.isPresent() && !isPictureInPicture()) {
ensureSurfaceViewRendererIsSetup(binding.localVideo); ensureSurfaceViewRendererIsSetup(binding.localVideo);
//paint local view over remote view //paint local view over remote view
binding.localVideo.setZOrderMediaOverlay(true); binding.localVideo.setZOrderMediaOverlay(true);
@ -600,9 +639,9 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
public void onJingleRtpConnectionUpdate(Account account, Jid with, final String sessionId, RtpEndUserState state) { public void onJingleRtpConnectionUpdate(Account account, Jid with, final String sessionId, RtpEndUserState state) {
Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")"); Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")");
if (END_CARD.contains(state)) { if (END_CARD.contains(state)) {
Log.d(Config.LOGTAG,"end card reached"); Log.d(Config.LOGTAG, "end card reached");
releaseProximityWakeLock(); releaseProximityWakeLock();
runOnUiThread(()-> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)); runOnUiThread(() -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON));
} }
if (with.isBareJid()) { if (with.isBareJid()) {
updateRtpSessionProposalState(account, with, state); updateRtpSessionProposalState(account, with, state);
@ -610,7 +649,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
} }
if (this.rtpConnectionReference == null) { if (this.rtpConnectionReference == null) {
if (END_CARD.contains(state)) { if (END_CARD.contains(state)) {
Log.d(Config.LOGTAG,"not reinitializing session"); Log.d(Config.LOGTAG, "not reinitializing session");
return; return;
} }
//this happens when going from proposed session to actual session //this happens when going from proposed session to actual session
@ -620,6 +659,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
final AbstractJingleConnection.Id id = requireRtpConnection().getId(); final AbstractJingleConnection.Id id = requireRtpConnection().getId();
if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) { if (account == id.account && id.with.equals(with) && id.sessionId.equals(sessionId)) {
if (state == RtpEndUserState.ENDED) { if (state == RtpEndUserState.ENDED) {
resetIntent(account, with, state, requireRtpConnection().getMedia());
finish(); finish();
return; return;
} else if (END_CARD.contains(state)) { } else if (END_CARD.contains(state)) {
@ -641,7 +681,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
Log.d(Config.LOGTAG, "onAudioDeviceChanged in activity: selected:" + selectedAudioDevice + ", available:" + availableAudioDevices); Log.d(Config.LOGTAG, "onAudioDeviceChanged in activity: selected:" + selectedAudioDevice + ", available:" + availableAudioDevices);
try { try {
if (getMedia().contains(Media.VIDEO)) { if (getMedia().contains(Media.VIDEO)) {
Log.d(Config.LOGTAG,"nothing to do; in video mode"); Log.d(Config.LOGTAG, "nothing to do; in video mode");
return; return;
} }
final RtpEndUserState endUserState = requireRtpConnection().getEndUserState(); final RtpEndUserState endUserState = requireRtpConnection().getEndUserState();
@ -652,7 +692,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe
audioManager.getAudioDevices().size() audioManager.getAudioDevices().size()
); );
} else if (END_CARD.contains(endUserState)) { } else if (END_CARD.contains(endUserState)) {
Log.d(Config.LOGTAG,"onAudioDeviceChanged() nothing to do because end card has been reached"); Log.d(Config.LOGTAG, "onAudioDeviceChanged() nothing to do because end card has been reached");
} else { } else {
putProximityWakeLockInProperState(); putProximityWakeLockInProperState();
} }

Binary file not shown.

After

Width:  |  Height:  |  Size: 714 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 364 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 590 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 843 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.0 KiB

View File

@ -8,6 +8,22 @@
android:layout_height="match_parent" android:layout_height="match_parent"
android:background="?color_background_secondary"> android:background="?color_background_secondary">
<LinearLayout
android:id="@+id/pip_placeholder"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="@color/black"
android:gravity="center"
android:orientation="horizontal"
android:visibility="gone">
<ImageView
android:id="@+id/pip_warning"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:src="@drawable/ic_warning_white_48dp" />
</LinearLayout>
<android.support.design.widget.AppBarLayout <android.support.design.widget.AppBarLayout
android:id="@+id/app_bar_layout" android:id="@+id/app_bar_layout"
android:layout_width="match_parent" android:layout_width="match_parent"
@ -37,7 +53,6 @@
</android.support.design.widget.AppBarLayout> </android.support.design.widget.AppBarLayout>
<org.webrtc.SurfaceViewRenderer <org.webrtc.SurfaceViewRenderer
app:elevation="4dp"
android:id="@+id/local_video" android:id="@+id/local_video"
android:layout_width="@dimen/local_video_preview_width" android:layout_width="@dimen/local_video_preview_width"
android:layout_height="@dimen/local_video_preview_height" android:layout_height="@dimen/local_video_preview_height"
@ -47,7 +62,8 @@
android:layout_marginTop="24dp" android:layout_marginTop="24dp"
android:layout_marginEnd="24dp" android:layout_marginEnd="24dp"
android:layout_marginRight="24dp" android:layout_marginRight="24dp"
android:visibility="gone" /> android:visibility="gone"
app:elevation="4dp" />
<org.webrtc.SurfaceViewRenderer <org.webrtc.SurfaceViewRenderer
android:id="@+id/remote_video" android:id="@+id/remote_video"