diff --git a/app/build.gradle b/app/build.gradle index a5a1b0d96..fd2b088ff 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -109,6 +109,10 @@ dependencies { // XMPP Address library implementation 'org.jxmpp:jxmpp-jid:1.0.3' + // WebRTC + implementation 'im.conversations.webrtc:webrtc-android:104.0.0' + + // Consistent Color Generation implementation 'org.hsluv:hsluv:0.2' @@ -116,15 +120,19 @@ dependencies { // DNS library (XMPP needs to resolve SRV records) implementation 'de.measite.minidns:minidns-hla:0.2.4' + // Guava implementation 'com.google.guava:guava:31.1-android' + // HTTP library implementation "com.squareup.okhttp3:okhttp:4.10.0" + // JSON parser implementation 'com.google.code.gson:gson:2.10.1' + // logging framework + logging api implementation 'org.slf4j:slf4j-api:1.7.36' implementation 'com.github.tony19:logback-android:2.0.1' diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index dc4cbfa33..7075be30e 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -106,6 +106,11 @@ + \ No newline at end of file diff --git a/app/src/main/java/eu/siacs/conversations/Config.java b/app/src/main/java/eu/siacs/conversations/Config.java new file mode 100644 index 000000000..f0f3020f3 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/Config.java @@ -0,0 +1,10 @@ +package eu.siacs.conversations; + +import android.net.Uri; + +public class Config { + public static final String LOGTAG = "conversations"; + public static final Uri HELP = Uri.parse("https://help.conversations.im"); + public static final boolean REQUIRE_RTP_VERIFICATION = + false; // require a/v calls to be verified with OMEMO +} diff --git a/app/src/main/java/eu/siacs/conversations/generator/MessageGenerator.java b/app/src/main/java/eu/siacs/conversations/generator/MessageGenerator.java new file mode 100644 index 000000000..87d62ba59 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/generator/MessageGenerator.java @@ -0,0 +1,54 @@ +package eu.siacs.conversations.generator; + +import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection; +import eu.siacs.conversations.xmpp.jingle.Media; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import im.conversations.android.xmpp.manager.JingleConnectionManager; +import im.conversations.android.xmpp.model.stanza.Message; +import org.jxmpp.jid.Jid; + +public final class MessageGenerator { + + private MessageGenerator() { + throw new IllegalStateException("Do not instantiate me"); + } + + public static Message sessionProposal( + final JingleConnectionManager.RtpSessionProposal proposal) { + final Message packet = new Message(Message.Type.CHAT); // we want to carbon copy those + packet.setTo(proposal.with); + packet.setId(JingleRtpConnection.JINGLE_MESSAGE_PROPOSE_ID_PREFIX + proposal.sessionId); + final Element propose = packet.addChild("propose", Namespace.JINGLE_MESSAGE); + propose.setAttribute("id", proposal.sessionId); + for (final Media media : proposal.media) { + propose.addChild("description", Namespace.JINGLE_APPS_RTP) + .setAttribute("media", media.toString()); + } + + packet.addChild("request", "urn:xmpp:receipts"); + packet.addChild("store", "urn:xmpp:hints"); + return packet; + } + + public static Message sessionRetract( + final JingleConnectionManager.RtpSessionProposal proposal) { + final Message packet = new Message(Message.Type.CHAT); // we want to carbon copy those + packet.setTo(proposal.with); + final Element propose = packet.addChild("retract", Namespace.JINGLE_MESSAGE); + propose.setAttribute("id", proposal.sessionId); + propose.addChild("description", Namespace.JINGLE_APPS_RTP); + packet.addChild("store", "urn:xmpp:hints"); + return packet; + } + + public static Message sessionReject(final Jid with, final String sessionId) { + final Message packet = new Message(Message.Type.CHAT); // we want to carbon copy those + packet.setTo(with); + final Element propose = packet.addChild("reject", Namespace.JINGLE_MESSAGE); + propose.setAttribute("id", sessionId); + propose.addChild("description", Namespace.JINGLE_APPS_RTP); + packet.addChild("store", "urn:xmpp:hints"); + return packet; + } +} diff --git a/app/src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java b/app/src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java new file mode 100644 index 000000000..12d0a1e16 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java @@ -0,0 +1,660 @@ +/* + * Copyright 2014 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +package eu.siacs.conversations.services; + +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.content.IntentFilter; +import android.content.pm.PackageManager; +import android.media.AudioDeviceInfo; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioRecord; +import android.media.MediaRecorder; +import android.os.Build; +import android.util.Log; +import androidx.annotation.Nullable; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.utils.AppRTCUtils; +import eu.siacs.conversations.xmpp.jingle.Media; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import org.webrtc.ThreadUtils; + +/** AppRTCAudioManager manages all audio related parts of the AppRTC demo. */ +public class AppRTCAudioManager { + + private static CountDownLatch microphoneLatch; + + private final Context apprtcContext; + // Contains speakerphone setting: auto, true or false + @Nullable private SpeakerPhonePreference speakerPhonePreference; + // Handles all tasks related to Bluetooth headset devices. + private final AppRTCBluetoothManager bluetoothManager; + @Nullable private final AudioManager audioManager; + @Nullable private AudioManagerEvents audioManagerEvents; + private AudioManagerState amState; + private boolean savedIsSpeakerPhoneOn; + private boolean savedIsMicrophoneMute; + private boolean hasWiredHeadset; + // Default audio device; speaker phone for video calls or earpiece for audio + // only calls. + private AudioDevice defaultAudioDevice; + // Contains the currently selected audio device. + // This device is changed automatically using a certain scheme where e.g. + // a wired headset "wins" over speaker phone. It is also possible for a + // user to explicitly select a device (and overrid any predefined scheme). + // See |userSelectedAudioDevice| for details. + private AudioDevice selectedAudioDevice; + // Contains the user-selected audio device which overrides the predefined + // selection scheme. + // TODO(henrika): always set to AudioDevice.NONE today. Add support for + // explicit selection based on choice by userSelectedAudioDevice. + private AudioDevice userSelectedAudioDevice; + // Proximity sensor object. It measures the proximity of an object in cm + // relative to the view screen of a device and can therefore be used to + // assist device switching (close to ear <=> use headset earpiece if + // available, far from ear <=> use speaker phone). + @Nullable private AppRTCProximitySensor proximitySensor; + // Contains a list of available audio devices. A Set collection is used to + // avoid duplicate elements. + private Set audioDevices = new HashSet<>(); + // Broadcast receiver for wired headset intent broadcasts. + private final BroadcastReceiver wiredHeadsetReceiver; + // Callback method for changes in audio focus. + @Nullable private AudioManager.OnAudioFocusChangeListener audioFocusChangeListener; + + private AppRTCAudioManager( + Context context, final SpeakerPhonePreference speakerPhonePreference) { + Log.d(Config.LOGTAG, "ctor"); + ThreadUtils.checkIsOnMainThread(); + apprtcContext = context; + audioManager = ((AudioManager) context.getSystemService(Context.AUDIO_SERVICE)); + bluetoothManager = AppRTCBluetoothManager.create(context, this); + wiredHeadsetReceiver = new WiredHeadsetReceiver(); + amState = AudioManagerState.UNINITIALIZED; + this.speakerPhonePreference = speakerPhonePreference; + if (speakerPhonePreference == SpeakerPhonePreference.EARPIECE && hasEarpiece()) { + defaultAudioDevice = AudioDevice.EARPIECE; + } else { + defaultAudioDevice = AudioDevice.SPEAKER_PHONE; + } + // Create and initialize the proximity sensor. + // Tablet devices (e.g. Nexus 7) does not support proximity sensors. + // Note that, the sensor will not be active until start() has been called. + proximitySensor = + AppRTCProximitySensor.create( + context, + // This method will be called each time a state change is detected. + // Example: user holds his hand over the device (closer than ~5 cm), + // or removes his hand from the device. + this::onProximitySensorChangedState); + Log.d(Config.LOGTAG, "defaultAudioDevice: " + defaultAudioDevice); + AppRTCUtils.logDeviceInfo(Config.LOGTAG); + } + + public void switchSpeakerPhonePreference(final SpeakerPhonePreference speakerPhonePreference) { + this.speakerPhonePreference = speakerPhonePreference; + if (speakerPhonePreference == SpeakerPhonePreference.EARPIECE && hasEarpiece()) { + defaultAudioDevice = AudioDevice.EARPIECE; + } else { + defaultAudioDevice = AudioDevice.SPEAKER_PHONE; + } + updateAudioDeviceState(); + } + + /** Construction. */ + public static AppRTCAudioManager create( + Context context, SpeakerPhonePreference speakerPhonePreference) { + return new AppRTCAudioManager(context, speakerPhonePreference); + } + + public static boolean isMicrophoneAvailable() { + microphoneLatch = new CountDownLatch(1); + AudioRecord audioRecord = null; + boolean available = true; + try { + final int sampleRate = 44100; + final int channel = AudioFormat.CHANNEL_IN_MONO; + final int format = AudioFormat.ENCODING_PCM_16BIT; + final int bufferSize = AudioRecord.getMinBufferSize(sampleRate, channel, format); + audioRecord = + new AudioRecord( + MediaRecorder.AudioSource.MIC, sampleRate, channel, format, bufferSize); + audioRecord.startRecording(); + final short[] buffer = new short[bufferSize]; + final int audioStatus = audioRecord.read(buffer, 0, bufferSize); + if (audioStatus == AudioRecord.ERROR_INVALID_OPERATION + || audioStatus == AudioRecord.STATE_UNINITIALIZED) available = false; + } catch (Exception e) { + available = false; + } finally { + release(audioRecord); + } + microphoneLatch.countDown(); + return available; + } + + private static void release(final AudioRecord audioRecord) { + if (audioRecord == null) { + return; + } + try { + audioRecord.release(); + } catch (Exception e) { + // ignore + } + } + + /** + * This method is called when the proximity sensor reports a state change, e.g. from "NEAR to + * FAR" or from "FAR to NEAR". + */ + private void onProximitySensorChangedState() { + if (speakerPhonePreference != SpeakerPhonePreference.AUTO) { + return; + } + // The proximity sensor should only be activated when there are exactly two + // available audio devices. + if (audioDevices.size() == 2 + && audioDevices.contains(AppRTCAudioManager.AudioDevice.EARPIECE) + && audioDevices.contains(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE)) { + if (proximitySensor.sensorReportsNearState()) { + // Sensor reports that a "handset is being held up to a person's ear", + // or "something is covering the light sensor". + setAudioDeviceInternal(AppRTCAudioManager.AudioDevice.EARPIECE); + } else { + // Sensor reports that a "handset is removed from a person's ear", or + // "the light sensor is no longer covered". + setAudioDeviceInternal(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE); + } + } + } + + @SuppressWarnings("deprecation") + public void start(AudioManagerEvents audioManagerEvents) { + Log.d(Config.LOGTAG, AppRTCAudioManager.class.getName() + ".start()"); + ThreadUtils.checkIsOnMainThread(); + if (amState == AudioManagerState.RUNNING) { + Log.e(Config.LOGTAG, "AudioManager is already active"); + return; + } + awaitMicrophoneLatch(); + this.audioManagerEvents = audioManagerEvents; + amState = AudioManagerState.RUNNING; + // Store current audio state so we can restore it when stop() is called. + savedIsSpeakerPhoneOn = audioManager.isSpeakerphoneOn(); + savedIsMicrophoneMute = audioManager.isMicrophoneMute(); + hasWiredHeadset = hasWiredHeadset(); + // Create an AudioManager.OnAudioFocusChangeListener instance. + audioFocusChangeListener = + new AudioManager.OnAudioFocusChangeListener() { + // Called on the listener to notify if the audio focus for this listener has + // been changed. + // The |focusChange| value indicates whether the focus was gained, whether the + // focus was lost, + // and whether that loss is transient, or whether the new focus holder will hold + // it for an + // unknown amount of time. + // TODO(henrika): possibly extend support of handling audio-focus changes. Only + // contains + // logging for now. + @Override + public void onAudioFocusChange(int focusChange) { + final String typeOfChange; + switch (focusChange) { + case AudioManager.AUDIOFOCUS_GAIN: + typeOfChange = "AUDIOFOCUS_GAIN"; + break; + case AudioManager.AUDIOFOCUS_GAIN_TRANSIENT: + typeOfChange = "AUDIOFOCUS_GAIN_TRANSIENT"; + break; + case AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE: + typeOfChange = "AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE"; + break; + case AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK: + typeOfChange = "AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK"; + break; + case AudioManager.AUDIOFOCUS_LOSS: + typeOfChange = "AUDIOFOCUS_LOSS"; + break; + case AudioManager.AUDIOFOCUS_LOSS_TRANSIENT: + typeOfChange = "AUDIOFOCUS_LOSS_TRANSIENT"; + break; + case AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK: + typeOfChange = "AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK"; + break; + default: + typeOfChange = "AUDIOFOCUS_INVALID"; + break; + } + Log.d(Config.LOGTAG, "onAudioFocusChange: " + typeOfChange); + } + }; + // Request audio playout focus (without ducking) and install listener for changes in focus. + int result = + audioManager.requestAudioFocus( + audioFocusChangeListener, + AudioManager.STREAM_VOICE_CALL, + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT); + if (result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + Log.d(Config.LOGTAG, "Audio focus request granted for VOICE_CALL streams"); + } else { + Log.e(Config.LOGTAG, "Audio focus request failed"); + } + // Start by setting MODE_IN_COMMUNICATION as default audio mode. It is + // required to be in this mode when playout and/or recording starts for + // best possible VoIP performance. + audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION); + // Always disable microphone mute during a WebRTC call. + setMicrophoneMute(false); + // Set initial device states. + userSelectedAudioDevice = AudioDevice.NONE; + selectedAudioDevice = AudioDevice.NONE; + audioDevices.clear(); + // Initialize and start Bluetooth if a BT device is available or initiate + // detection of new (enabled) BT devices. + bluetoothManager.start(); + // Do initial selection of audio device. This setting can later be changed + // either by adding/removing a BT or wired headset or by covering/uncovering + // the proximity sensor. + updateAudioDeviceState(); + // Register receiver for broadcast intents related to adding/removing a + // wired headset. + registerReceiver(wiredHeadsetReceiver, new IntentFilter(Intent.ACTION_HEADSET_PLUG)); + Log.d(Config.LOGTAG, "AudioManager started"); + } + + private void awaitMicrophoneLatch() { + final CountDownLatch latch = microphoneLatch; + if (latch == null) { + return; + } + try { + latch.await(); + } catch (InterruptedException e) { + // ignore + } + } + + @SuppressWarnings("deprecation") + public void stop() { + Log.d(Config.LOGTAG, AppRTCAudioManager.class.getName() + ".stop()"); + ThreadUtils.checkIsOnMainThread(); + if (amState != AudioManagerState.RUNNING) { + Log.e(Config.LOGTAG, "Trying to stop AudioManager in incorrect state: " + amState); + return; + } + amState = AudioManagerState.UNINITIALIZED; + unregisterReceiver(wiredHeadsetReceiver); + bluetoothManager.stop(); + // Restore previously stored audio states. + setSpeakerphoneOn(savedIsSpeakerPhoneOn); + setMicrophoneMute(savedIsMicrophoneMute); + audioManager.setMode(AudioManager.MODE_NORMAL); + // Abandon audio focus. Gives the previous focus owner, if any, focus. + audioManager.abandonAudioFocus(audioFocusChangeListener); + audioFocusChangeListener = null; + Log.d(Config.LOGTAG, "Abandoned audio focus for VOICE_CALL streams"); + if (proximitySensor != null) { + proximitySensor.stop(); + proximitySensor = null; + } + audioManagerEvents = null; + } + + /** Changes selection of the currently active audio device. */ + private void setAudioDeviceInternal(AudioDevice device) { + Log.d(Config.LOGTAG, "setAudioDeviceInternal(device=" + device + ")"); + AppRTCUtils.assertIsTrue(audioDevices.contains(device)); + switch (device) { + case SPEAKER_PHONE: + setSpeakerphoneOn(true); + break; + case EARPIECE: + case WIRED_HEADSET: + case BLUETOOTH: + setSpeakerphoneOn(false); + break; + default: + Log.e(Config.LOGTAG, "Invalid audio device selection"); + break; + } + selectedAudioDevice = device; + } + + /** + * Changes default audio device. TODO(henrika): add usage of this method in the AppRTCMobile + * client. + */ + public void setDefaultAudioDevice(AudioDevice defaultDevice) { + ThreadUtils.checkIsOnMainThread(); + switch (defaultDevice) { + case SPEAKER_PHONE: + defaultAudioDevice = defaultDevice; + break; + case EARPIECE: + if (hasEarpiece()) { + defaultAudioDevice = defaultDevice; + } else { + defaultAudioDevice = AudioDevice.SPEAKER_PHONE; + } + break; + default: + Log.e(Config.LOGTAG, "Invalid default audio device selection"); + break; + } + Log.d(Config.LOGTAG, "setDefaultAudioDevice(device=" + defaultAudioDevice + ")"); + updateAudioDeviceState(); + } + + /** Changes selection of the currently active audio device. */ + public void selectAudioDevice(AudioDevice device) { + ThreadUtils.checkIsOnMainThread(); + if (!audioDevices.contains(device)) { + Log.e(Config.LOGTAG, "Can not select " + device + " from available " + audioDevices); + } + userSelectedAudioDevice = device; + updateAudioDeviceState(); + } + + /** Returns current set of available/selectable audio devices. */ + public Set getAudioDevices() { + ThreadUtils.checkIsOnMainThread(); + return Collections.unmodifiableSet(new HashSet<>(audioDevices)); + } + + /** Returns the currently selected audio device. */ + public AudioDevice getSelectedAudioDevice() { + ThreadUtils.checkIsOnMainThread(); + return selectedAudioDevice; + } + + /** Helper method for receiver registration. */ + private void registerReceiver(BroadcastReceiver receiver, IntentFilter filter) { + apprtcContext.registerReceiver(receiver, filter); + } + + /** Helper method for unregistration of an existing receiver. */ + private void unregisterReceiver(BroadcastReceiver receiver) { + apprtcContext.unregisterReceiver(receiver); + } + + /** Sets the speaker phone mode. */ + private void setSpeakerphoneOn(boolean on) { + boolean wasOn = audioManager.isSpeakerphoneOn(); + if (wasOn == on) { + return; + } + audioManager.setSpeakerphoneOn(on); + } + + /** Sets the microphone mute state. */ + private void setMicrophoneMute(boolean on) { + boolean wasMuted = audioManager.isMicrophoneMute(); + if (wasMuted == on) { + return; + } + audioManager.setMicrophoneMute(on); + } + + /** Gets the current earpiece state. */ + private boolean hasEarpiece() { + return apprtcContext.getPackageManager().hasSystemFeature(PackageManager.FEATURE_TELEPHONY); + } + + /** + * Checks whether a wired headset is connected or not. This is not a valid indication that audio + * playback is actually over the wired headset as audio routing depends on other conditions. We + * only use it as an early indicator (during initialization) of an attached wired headset. + */ + @Deprecated + private boolean hasWiredHeadset() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + return audioManager.isWiredHeadsetOn(); + } else { + final AudioDeviceInfo[] devices = audioManager.getDevices(AudioManager.GET_DEVICES_ALL); + for (AudioDeviceInfo device : devices) { + final int type = device.getType(); + if (type == AudioDeviceInfo.TYPE_WIRED_HEADSET) { + Log.d(Config.LOGTAG, "hasWiredHeadset: found wired headset"); + return true; + } else if (type == AudioDeviceInfo.TYPE_USB_DEVICE) { + Log.d(Config.LOGTAG, "hasWiredHeadset: found USB audio device"); + return true; + } + } + return false; + } + } + + /** + * Updates list of possible audio devices and make new device selection. TODO(henrika): add unit + * test to verify all state transitions. + */ + public void updateAudioDeviceState() { + ThreadUtils.checkIsOnMainThread(); + Log.d( + Config.LOGTAG, + "--- updateAudioDeviceState: " + + "wired headset=" + + hasWiredHeadset + + ", " + + "BT state=" + + bluetoothManager.getState()); + Log.d( + Config.LOGTAG, + "Device status: " + + "available=" + + audioDevices + + ", " + + "selected=" + + selectedAudioDevice + + ", " + + "user selected=" + + userSelectedAudioDevice); + // Check if any Bluetooth headset is connected. The internal BT state will + // change accordingly. + // TODO(henrika): perhaps wrap required state into BT manager. + if (bluetoothManager.getState() == AppRTCBluetoothManager.State.HEADSET_AVAILABLE + || bluetoothManager.getState() == AppRTCBluetoothManager.State.HEADSET_UNAVAILABLE + || bluetoothManager.getState() == AppRTCBluetoothManager.State.SCO_DISCONNECTING) { + bluetoothManager.updateDevice(); + } + // Update the set of available audio devices. + Set newAudioDevices = new HashSet<>(); + if (bluetoothManager.getState() == AppRTCBluetoothManager.State.SCO_CONNECTED + || bluetoothManager.getState() == AppRTCBluetoothManager.State.SCO_CONNECTING + || bluetoothManager.getState() == AppRTCBluetoothManager.State.HEADSET_AVAILABLE) { + newAudioDevices.add(AudioDevice.BLUETOOTH); + } + if (hasWiredHeadset) { + // If a wired headset is connected, then it is the only possible option. + newAudioDevices.add(AudioDevice.WIRED_HEADSET); + } else { + // No wired headset, hence the audio-device list can contain speaker + // phone (on a tablet), or speaker phone and earpiece (on mobile phone). + newAudioDevices.add(AudioDevice.SPEAKER_PHONE); + if (hasEarpiece()) { + newAudioDevices.add(AudioDevice.EARPIECE); + } + } + // Store state which is set to true if the device list has changed. + boolean audioDeviceSetUpdated = !audioDevices.equals(newAudioDevices); + // Update the existing audio device set. + audioDevices = newAudioDevices; + // Correct user selected audio devices if needed. + if (bluetoothManager.getState() == AppRTCBluetoothManager.State.HEADSET_UNAVAILABLE + && userSelectedAudioDevice == AudioDevice.BLUETOOTH) { + // If BT is not available, it can't be the user selection. + userSelectedAudioDevice = AudioDevice.NONE; + } + if (hasWiredHeadset && userSelectedAudioDevice == AudioDevice.SPEAKER_PHONE) { + // If user selected speaker phone, but then plugged wired headset then make + // wired headset as user selected device. + userSelectedAudioDevice = AudioDevice.WIRED_HEADSET; + } + if (!hasWiredHeadset && userSelectedAudioDevice == AudioDevice.WIRED_HEADSET) { + // If user selected wired headset, but then unplugged wired headset then make + // speaker phone as user selected device. + userSelectedAudioDevice = AudioDevice.SPEAKER_PHONE; + } + // Need to start Bluetooth if it is available and user either selected it explicitly or + // user did not select any output device. + boolean needBluetoothAudioStart = + bluetoothManager.getState() == AppRTCBluetoothManager.State.HEADSET_AVAILABLE + && (userSelectedAudioDevice == AudioDevice.NONE + || userSelectedAudioDevice == AudioDevice.BLUETOOTH); + // Need to stop Bluetooth audio if user selected different device and + // Bluetooth SCO connection is established or in the process. + boolean needBluetoothAudioStop = + (bluetoothManager.getState() == AppRTCBluetoothManager.State.SCO_CONNECTED + || bluetoothManager.getState() + == AppRTCBluetoothManager.State.SCO_CONNECTING) + && (userSelectedAudioDevice != AudioDevice.NONE + && userSelectedAudioDevice != AudioDevice.BLUETOOTH); + if (bluetoothManager.getState() == AppRTCBluetoothManager.State.HEADSET_AVAILABLE + || bluetoothManager.getState() == AppRTCBluetoothManager.State.SCO_CONNECTING + || bluetoothManager.getState() == AppRTCBluetoothManager.State.SCO_CONNECTED) { + Log.d( + Config.LOGTAG, + "Need BT audio: start=" + + needBluetoothAudioStart + + ", " + + "stop=" + + needBluetoothAudioStop + + ", " + + "BT state=" + + bluetoothManager.getState()); + } + // Start or stop Bluetooth SCO connection given states set earlier. + if (needBluetoothAudioStop) { + bluetoothManager.stopScoAudio(); + bluetoothManager.updateDevice(); + } + if (needBluetoothAudioStart && !needBluetoothAudioStop) { + // Attempt to start Bluetooth SCO audio (takes a few second to start). + if (!bluetoothManager.startScoAudio()) { + // Remove BLUETOOTH from list of available devices since SCO failed. + audioDevices.remove(AudioDevice.BLUETOOTH); + audioDeviceSetUpdated = true; + } + } + // Update selected audio device. + final AudioDevice newAudioDevice; + if (bluetoothManager.getState() == AppRTCBluetoothManager.State.SCO_CONNECTED) { + // If a Bluetooth is connected, then it should be used as output audio + // device. Note that it is not sufficient that a headset is available; + // an active SCO channel must also be up and running. + newAudioDevice = AudioDevice.BLUETOOTH; + } else if (hasWiredHeadset) { + // If a wired headset is connected, but Bluetooth is not, then wired headset is used as + // audio device. + newAudioDevice = AudioDevice.WIRED_HEADSET; + } else { + // No wired headset and no Bluetooth, hence the audio-device list can contain speaker + // phone (on a tablet), or speaker phone and earpiece (on mobile phone). + // |defaultAudioDevice| contains either AudioDevice.SPEAKER_PHONE or + // AudioDevice.EARPIECE + // depending on the user's selection. + newAudioDevice = defaultAudioDevice; + } + // Switch to new device but only if there has been any changes. + if (newAudioDevice != selectedAudioDevice || audioDeviceSetUpdated) { + // Do the required device switch. + setAudioDeviceInternal(newAudioDevice); + Log.d( + Config.LOGTAG, + "New device status: " + + "available=" + + audioDevices + + ", " + + "selected=" + + newAudioDevice); + if (audioManagerEvents != null) { + // Notify a listening client that audio device has been changed. + audioManagerEvents.onAudioDeviceChanged(selectedAudioDevice, audioDevices); + } + } + Log.d(Config.LOGTAG, "--- updateAudioDeviceState done"); + } + + /** AudioDevice is the names of possible audio devices that we currently support. */ + public enum AudioDevice { + SPEAKER_PHONE, + WIRED_HEADSET, + EARPIECE, + BLUETOOTH, + NONE + } + + /** AudioManager state. */ + public enum AudioManagerState { + UNINITIALIZED, + PREINITIALIZED, + RUNNING, + } + + public enum SpeakerPhonePreference { + AUTO, + EARPIECE, + SPEAKER; + + public static SpeakerPhonePreference of(final Set media) { + if (media.contains(Media.VIDEO)) { + return SPEAKER; + } else { + return EARPIECE; + } + } + } + + /** Selected audio device change event. */ + public interface AudioManagerEvents { + // Callback fired once audio device is changed or list of available audio devices changed. + void onAudioDeviceChanged( + AudioDevice selectedAudioDevice, Set availableAudioDevices); + } + + /* Receiver which handles changes in wired headset availability. */ + private class WiredHeadsetReceiver extends BroadcastReceiver { + private static final int STATE_UNPLUGGED = 0; + private static final int STATE_PLUGGED = 1; + private static final int HAS_NO_MIC = 0; + private static final int HAS_MIC = 1; + + @Override + public void onReceive(Context context, Intent intent) { + int state = intent.getIntExtra("state", STATE_UNPLUGGED); + int microphone = intent.getIntExtra("microphone", HAS_NO_MIC); + String name = intent.getStringExtra("name"); + Log.d( + Config.LOGTAG, + "WiredHeadsetReceiver.onReceive" + + AppRTCUtils.getThreadInfo() + + ": " + + "a=" + + intent.getAction() + + ", s=" + + (state == STATE_UNPLUGGED ? "unplugged" : "plugged") + + ", m=" + + (microphone == HAS_MIC ? "mic" : "no mic") + + ", n=" + + name + + ", sb=" + + isInitialStickyBroadcast()); + hasWiredHeadset = (state == STATE_PLUGGED); + updateAudioDeviceState(); + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/services/AppRTCBluetoothManager.java b/app/src/main/java/eu/siacs/conversations/services/AppRTCBluetoothManager.java new file mode 100644 index 000000000..26e2fa2a3 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/services/AppRTCBluetoothManager.java @@ -0,0 +1,570 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +package eu.siacs.conversations.services; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.bluetooth.BluetoothAdapter; +import android.bluetooth.BluetoothDevice; +import android.bluetooth.BluetoothHeadset; +import android.bluetooth.BluetoothProfile; +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.content.IntentFilter; +import android.content.pm.PackageManager; +import android.media.AudioManager; +import android.os.Build; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; +import androidx.annotation.Nullable; +import androidx.core.app.ActivityCompat; +import com.google.common.collect.ImmutableList; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.utils.AppRTCUtils; +import java.util.Collections; +import java.util.List; +import org.webrtc.ThreadUtils; + +/** AppRTCProximitySensor manages functions related to Bluetoth devices in the AppRTC demo. */ +public class AppRTCBluetoothManager { + // Timeout interval for starting or stopping audio to a Bluetooth SCO device. + private static final int BLUETOOTH_SCO_TIMEOUT_MS = 4000; + // Maximum number of SCO connection attempts. + private static final int MAX_SCO_CONNECTION_ATTEMPTS = 2; + private final Context apprtcContext; + private final AppRTCAudioManager apprtcAudioManager; + @Nullable private final AudioManager audioManager; + private final Handler handler; + private final BluetoothProfile.ServiceListener bluetoothServiceListener; + private final BroadcastReceiver bluetoothHeadsetReceiver; + int scoConnectionAttempts; + private State bluetoothState; + @Nullable private BluetoothAdapter bluetoothAdapter; + @Nullable private BluetoothHeadset bluetoothHeadset; + @Nullable private BluetoothDevice bluetoothDevice; + // Runs when the Bluetooth timeout expires. We use that timeout after calling + // startScoAudio() or stopScoAudio() because we're not guaranteed to get a + // callback after those calls. + private final Runnable bluetoothTimeoutRunnable = + new Runnable() { + @Override + public void run() { + bluetoothTimeout(); + } + }; + + protected AppRTCBluetoothManager(Context context, AppRTCAudioManager audioManager) { + Log.d(Config.LOGTAG, "ctor"); + ThreadUtils.checkIsOnMainThread(); + apprtcContext = context; + apprtcAudioManager = audioManager; + this.audioManager = getAudioManager(context); + bluetoothState = State.UNINITIALIZED; + bluetoothServiceListener = new BluetoothServiceListener(); + bluetoothHeadsetReceiver = new BluetoothHeadsetBroadcastReceiver(); + handler = new Handler(Looper.getMainLooper()); + } + + /** Construction. */ + static AppRTCBluetoothManager create(Context context, AppRTCAudioManager audioManager) { + Log.d(Config.LOGTAG, "create" + AppRTCUtils.getThreadInfo()); + return new AppRTCBluetoothManager(context, audioManager); + } + + /** Returns the internal state. */ + public State getState() { + ThreadUtils.checkIsOnMainThread(); + return bluetoothState; + } + + /** + * Activates components required to detect Bluetooth devices and to enable BT SCO (audio is + * routed via BT SCO) for the headset profile. The end state will be HEADSET_UNAVAILABLE but a + * state machine has started which will start a state change sequence where the final outcome + * depends on if/when the BT headset is enabled. Example of state change sequence when start() + * is called while BT device is connected and enabled: UNINITIALIZED --> HEADSET_UNAVAILABLE --> + * HEADSET_AVAILABLE --> SCO_CONNECTING --> SCO_CONNECTED <==> audio is now routed via BT SCO. + * Note that the AppRTCAudioManager is also involved in driving this state change. + */ + public void start() { + ThreadUtils.checkIsOnMainThread(); + if (bluetoothState != State.UNINITIALIZED) { + Log.w(Config.LOGTAG, "Invalid BT state"); + return; + } + bluetoothHeadset = null; + bluetoothDevice = null; + scoConnectionAttempts = 0; + // Get a handle to the default local Bluetooth adapter. + bluetoothAdapter = BluetoothAdapter.getDefaultAdapter(); + if (bluetoothAdapter == null) { + Log.w(Config.LOGTAG, "Device does not support Bluetooth"); + return; + } + // Ensure that the device supports use of BT SCO audio for off call use cases. + if (this.audioManager == null || !audioManager.isBluetoothScoAvailableOffCall()) { + Log.e(Config.LOGTAG, "Bluetooth SCO audio is not available off call"); + return; + } + // Establish a connection to the HEADSET profile (includes both Bluetooth Headset and + // Hands-Free) proxy object and install a listener. + if (!getBluetoothProfileProxy( + apprtcContext, bluetoothServiceListener, BluetoothProfile.HEADSET)) { + Log.e(Config.LOGTAG, "BluetoothAdapter.getProfileProxy(HEADSET) failed"); + return; + } + // Register receivers for BluetoothHeadset change notifications. + IntentFilter bluetoothHeadsetFilter = new IntentFilter(); + // Register receiver for change in connection state of the Headset profile. + bluetoothHeadsetFilter.addAction(BluetoothHeadset.ACTION_CONNECTION_STATE_CHANGED); + // Register receiver for change in audio connection state of the Headset profile. + bluetoothHeadsetFilter.addAction(BluetoothHeadset.ACTION_AUDIO_STATE_CHANGED); + registerReceiver(bluetoothHeadsetReceiver, bluetoothHeadsetFilter); + if (hasBluetoothConnectPermission()) { + Log.d( + Config.LOGTAG, + "HEADSET profile state: " + + stateToString( + bluetoothAdapter.getProfileConnectionState( + BluetoothProfile.HEADSET))); + } + Log.d(Config.LOGTAG, "Bluetooth proxy for headset profile has started"); + bluetoothState = State.HEADSET_UNAVAILABLE; + Log.d(Config.LOGTAG, "start done: BT state=" + bluetoothState); + } + + /** Stops and closes all components related to Bluetooth audio. */ + public void stop() { + ThreadUtils.checkIsOnMainThread(); + Log.d(Config.LOGTAG, "stop: BT state=" + bluetoothState); + if (bluetoothAdapter == null) { + return; + } + // Stop BT SCO connection with remote device if needed. + stopScoAudio(); + // Close down remaining BT resources. + if (bluetoothState == State.UNINITIALIZED) { + return; + } + unregisterReceiver(bluetoothHeadsetReceiver); + cancelTimer(); + if (bluetoothHeadset != null) { + bluetoothAdapter.closeProfileProxy(BluetoothProfile.HEADSET, bluetoothHeadset); + bluetoothHeadset = null; + } + bluetoothAdapter = null; + bluetoothDevice = null; + bluetoothState = State.UNINITIALIZED; + Log.d(Config.LOGTAG, "stop done: BT state=" + bluetoothState); + } + + /** + * Starts Bluetooth SCO connection with remote device. Note that the phone application always + * has the priority on the usage of the SCO connection for telephony. If this method is called + * while the phone is in call it will be ignored. Similarly, if a call is received or sent while + * an application is using the SCO connection, the connection will be lost for the application + * and NOT returned automatically when the call ends. Also note that: up to and including API + * version JELLY_BEAN_MR1, this method initiates a virtual voice call to the Bluetooth headset. + * After API version JELLY_BEAN_MR2 only a raw SCO audio connection is established. + * TODO(henrika): should we add support for virtual voice call to BT headset also for JBMR2 and + * higher. It might be required to initiates a virtual voice call since many devices do not + * accept SCO audio without a "call". + */ + public boolean startScoAudio() { + ThreadUtils.checkIsOnMainThread(); + Log.d( + Config.LOGTAG, + "startSco: BT state=" + + bluetoothState + + ", " + + "attempts: " + + scoConnectionAttempts + + ", " + + "SCO is on: " + + isScoOn()); + if (scoConnectionAttempts >= MAX_SCO_CONNECTION_ATTEMPTS) { + Log.e(Config.LOGTAG, "BT SCO connection fails - no more attempts"); + return false; + } + if (bluetoothState != State.HEADSET_AVAILABLE) { + Log.e(Config.LOGTAG, "BT SCO connection fails - no headset available"); + return false; + } + // Start BT SCO channel and wait for ACTION_AUDIO_STATE_CHANGED. + Log.d(Config.LOGTAG, "Starting Bluetooth SCO and waits for ACTION_AUDIO_STATE_CHANGED..."); + // The SCO connection establishment can take several seconds, hence we cannot rely on the + // connection to be available when the method returns but instead register to receive the + // intent ACTION_SCO_AUDIO_STATE_UPDATED and wait for the state to be + // SCO_AUDIO_STATE_CONNECTED. + bluetoothState = State.SCO_CONNECTING; + audioManager.startBluetoothSco(); + audioManager.setBluetoothScoOn(true); + scoConnectionAttempts++; + startTimer(); + Log.d( + Config.LOGTAG, + "startScoAudio done: BT state=" + + bluetoothState + + ", " + + "SCO is on: " + + isScoOn()); + return true; + } + + /** Stops Bluetooth SCO connection with remote device. */ + public void stopScoAudio() { + ThreadUtils.checkIsOnMainThread(); + Log.d( + Config.LOGTAG, + "stopScoAudio: BT state=" + bluetoothState + ", " + "SCO is on: " + isScoOn()); + if (bluetoothState != State.SCO_CONNECTING && bluetoothState != State.SCO_CONNECTED) { + return; + } + cancelTimer(); + audioManager.stopBluetoothSco(); + audioManager.setBluetoothScoOn(false); + bluetoothState = State.SCO_DISCONNECTING; + Log.d( + Config.LOGTAG, + "stopScoAudio done: BT state=" + bluetoothState + ", " + "SCO is on: " + isScoOn()); + } + + /** + * Use the BluetoothHeadset proxy object (controls the Bluetooth Headset Service via IPC) to + * update the list of connected devices for the HEADSET profile. The internal state will change + * to HEADSET_UNAVAILABLE or to HEADSET_AVAILABLE and |bluetoothDevice| will be mapped to the + * connected device if available. + */ + @SuppressLint("MissingPermission") + public void updateDevice() { + if (bluetoothState == State.UNINITIALIZED || bluetoothHeadset == null) { + return; + } + Log.d(Config.LOGTAG, "updateDevice"); + // Get connected devices for the headset profile. Returns the set of + // devices which are in state STATE_CONNECTED. The BluetoothDevice class + // is just a thin wrapper for a Bluetooth hardware address. + final List devices; + if (hasBluetoothConnectPermission()) { + devices = bluetoothHeadset.getConnectedDevices(); + } else { + devices = ImmutableList.of(); + } + if (devices.isEmpty()) { + bluetoothDevice = null; + bluetoothState = State.HEADSET_UNAVAILABLE; + Log.d(Config.LOGTAG, "No connected bluetooth headset"); + } else { + // Always use first device in list. Android only supports one device. + bluetoothDevice = devices.get(0); + bluetoothState = State.HEADSET_AVAILABLE; + Log.d( + Config.LOGTAG, + "Connected bluetooth headset: " + + "name=" + + bluetoothDevice.getName() + + ", " + + "state=" + + stateToString(bluetoothHeadset.getConnectionState(bluetoothDevice)) + + ", SCO audio=" + + bluetoothHeadset.isAudioConnected(bluetoothDevice)); + } + Log.d(Config.LOGTAG, "updateDevice done: BT state=" + bluetoothState); + } + + /** Stubs for test mocks. */ + @Nullable + protected AudioManager getAudioManager(Context context) { + return (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + } + + protected void registerReceiver(BroadcastReceiver receiver, IntentFilter filter) { + apprtcContext.registerReceiver(receiver, filter); + } + + protected void unregisterReceiver(BroadcastReceiver receiver) { + apprtcContext.unregisterReceiver(receiver); + } + + protected boolean getBluetoothProfileProxy( + Context context, BluetoothProfile.ServiceListener listener, int profile) { + return bluetoothAdapter.getProfileProxy(context, listener, profile); + } + + protected boolean hasBluetoothConnectPermission() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + return ActivityCompat.checkSelfPermission( + apprtcContext, Manifest.permission.BLUETOOTH_CONNECT) + == PackageManager.PERMISSION_GRANTED; + } else { + return true; + } + } + + /** Ensures that the audio manager updates its list of available audio devices. */ + private void updateAudioDeviceState() { + ThreadUtils.checkIsOnMainThread(); + Log.d(Config.LOGTAG, "updateAudioDeviceState"); + apprtcAudioManager.updateAudioDeviceState(); + } + + /** Starts timer which times out after BLUETOOTH_SCO_TIMEOUT_MS milliseconds. */ + private void startTimer() { + ThreadUtils.checkIsOnMainThread(); + Log.d(Config.LOGTAG, "startTimer"); + handler.postDelayed(bluetoothTimeoutRunnable, BLUETOOTH_SCO_TIMEOUT_MS); + } + + /** Cancels any outstanding timer tasks. */ + private void cancelTimer() { + ThreadUtils.checkIsOnMainThread(); + Log.d(Config.LOGTAG, "cancelTimer"); + handler.removeCallbacks(bluetoothTimeoutRunnable); + } + + /** + * Called when start of the BT SCO channel takes too long time. Usually happens when the BT + * device has been turned on during an ongoing call. + */ + @SuppressLint("MissingPermission") + private void bluetoothTimeout() { + ThreadUtils.checkIsOnMainThread(); + if (bluetoothState == State.UNINITIALIZED || bluetoothHeadset == null) { + return; + } + Log.d( + Config.LOGTAG, + "bluetoothTimeout: BT state=" + + bluetoothState + + ", " + + "attempts: " + + scoConnectionAttempts + + ", " + + "SCO is on: " + + isScoOn()); + if (bluetoothState != State.SCO_CONNECTING) { + return; + } + // Bluetooth SCO should be connecting; check the latest result. + boolean scoConnected = false; + final List devices; + if (hasBluetoothConnectPermission()) { + devices = bluetoothHeadset.getConnectedDevices(); + } else { + devices = Collections.emptyList(); + } + if (devices.size() > 0) { + bluetoothDevice = devices.get(0); + if (bluetoothHeadset.isAudioConnected(bluetoothDevice)) { + Log.d(Config.LOGTAG, "SCO connected with " + bluetoothDevice.getName()); + scoConnected = true; + } else { + Log.d(Config.LOGTAG, "SCO is not connected with " + bluetoothDevice.getName()); + } + } + if (scoConnected) { + // We thought BT had timed out, but it's actually on; updating state. + bluetoothState = State.SCO_CONNECTED; + scoConnectionAttempts = 0; + } else { + // Give up and "cancel" our request by calling stopBluetoothSco(). + Log.w(Config.LOGTAG, "BT failed to connect after timeout"); + stopScoAudio(); + } + updateAudioDeviceState(); + Log.d(Config.LOGTAG, "bluetoothTimeout done: BT state=" + bluetoothState); + } + + /** Checks whether audio uses Bluetooth SCO. */ + private boolean isScoOn() { + return audioManager.isBluetoothScoOn(); + } + + /** Converts BluetoothAdapter states into local string representations. */ + private String stateToString(int state) { + switch (state) { + case BluetoothAdapter.STATE_DISCONNECTED: + return "DISCONNECTED"; + case BluetoothAdapter.STATE_CONNECTED: + return "CONNECTED"; + case BluetoothAdapter.STATE_CONNECTING: + return "CONNECTING"; + case BluetoothAdapter.STATE_DISCONNECTING: + return "DISCONNECTING"; + case BluetoothAdapter.STATE_OFF: + return "OFF"; + case BluetoothAdapter.STATE_ON: + return "ON"; + case BluetoothAdapter.STATE_TURNING_OFF: + // Indicates the local Bluetooth adapter is turning off. Local clients should + // immediately + // attempt graceful disconnection of any remote links. + return "TURNING_OFF"; + case BluetoothAdapter.STATE_TURNING_ON: + // Indicates the local Bluetooth adapter is turning on. However local clients should + // wait + // for STATE_ON before attempting to use the adapter. + return "TURNING_ON"; + default: + return "INVALID"; + } + } + + // Bluetooth connection state. + public enum State { + // Bluetooth is not available; no adapter or Bluetooth is off. + UNINITIALIZED, + // Bluetooth error happened when trying to start Bluetooth. + ERROR, + // Bluetooth proxy object for the Headset profile exists, but no connected headset devices, + // SCO is not started or disconnected. + HEADSET_UNAVAILABLE, + // Bluetooth proxy object for the Headset profile connected, connected Bluetooth headset + // present, but SCO is not started or disconnected. + HEADSET_AVAILABLE, + // Bluetooth audio SCO connection with remote device is closing. + SCO_DISCONNECTING, + // Bluetooth audio SCO connection with remote device is initiated. + SCO_CONNECTING, + // Bluetooth audio SCO connection with remote device is established. + SCO_CONNECTED + } + + /** + * Implementation of an interface that notifies BluetoothProfile IPC clients when they have been + * connected to or disconnected from the service. + */ + private class BluetoothServiceListener implements BluetoothProfile.ServiceListener { + @Override + // Called to notify the client when the proxy object has been connected to the service. + // Once we have the profile proxy object, we can use it to monitor the state of the + // connection and perform other operations that are relevant to the headset profile. + public void onServiceConnected(int profile, BluetoothProfile proxy) { + if (profile != BluetoothProfile.HEADSET || bluetoothState == State.UNINITIALIZED) { + return; + } + Log.d( + Config.LOGTAG, + "BluetoothServiceListener.onServiceConnected: BT state=" + bluetoothState); + // Android only supports one connected Bluetooth Headset at a time. + bluetoothHeadset = (BluetoothHeadset) proxy; + updateAudioDeviceState(); + Log.d(Config.LOGTAG, "onServiceConnected done: BT state=" + bluetoothState); + } + + @Override + /** Notifies the client when the proxy object has been disconnected from the service. */ + public void onServiceDisconnected(int profile) { + if (profile != BluetoothProfile.HEADSET || bluetoothState == State.UNINITIALIZED) { + return; + } + Log.d( + Config.LOGTAG, + "BluetoothServiceListener.onServiceDisconnected: BT state=" + bluetoothState); + stopScoAudio(); + bluetoothHeadset = null; + bluetoothDevice = null; + bluetoothState = State.HEADSET_UNAVAILABLE; + updateAudioDeviceState(); + Log.d(Config.LOGTAG, "onServiceDisconnected done: BT state=" + bluetoothState); + } + } + + // Intent broadcast receiver which handles changes in Bluetooth device availability. + // Detects headset changes and Bluetooth SCO state changes. + private class BluetoothHeadsetBroadcastReceiver extends BroadcastReceiver { + @Override + public void onReceive(Context context, Intent intent) { + if (bluetoothState == State.UNINITIALIZED) { + return; + } + final String action = intent.getAction(); + // Change in connection state of the Headset profile. Note that the + // change does not tell us anything about whether we're streaming + // audio to BT over SCO. Typically received when user turns on a BT + // headset while audio is active using another audio device. + if (action.equals(BluetoothHeadset.ACTION_CONNECTION_STATE_CHANGED)) { + final int state = + intent.getIntExtra( + BluetoothHeadset.EXTRA_STATE, BluetoothHeadset.STATE_DISCONNECTED); + Log.d( + Config.LOGTAG, + "BluetoothHeadsetBroadcastReceiver.onReceive: " + + "a=ACTION_CONNECTION_STATE_CHANGED, " + + "s=" + + stateToString(state) + + ", " + + "sb=" + + isInitialStickyBroadcast() + + ", " + + "BT state: " + + bluetoothState); + if (state == BluetoothHeadset.STATE_CONNECTED) { + scoConnectionAttempts = 0; + updateAudioDeviceState(); + } else if (state == BluetoothHeadset.STATE_CONNECTING) { + // No action needed. + } else if (state == BluetoothHeadset.STATE_DISCONNECTING) { + // No action needed. + } else if (state == BluetoothHeadset.STATE_DISCONNECTED) { + // Bluetooth is probably powered off during the call. + stopScoAudio(); + updateAudioDeviceState(); + } + // Change in the audio (SCO) connection state of the Headset profile. + // Typically received after call to startScoAudio() has finalized. + } else if (action.equals(BluetoothHeadset.ACTION_AUDIO_STATE_CHANGED)) { + final int state = + intent.getIntExtra( + BluetoothHeadset.EXTRA_STATE, + BluetoothHeadset.STATE_AUDIO_DISCONNECTED); + Log.d( + Config.LOGTAG, + "BluetoothHeadsetBroadcastReceiver.onReceive: " + + "a=ACTION_AUDIO_STATE_CHANGED, " + + "s=" + + stateToString(state) + + ", " + + "sb=" + + isInitialStickyBroadcast() + + ", " + + "BT state: " + + bluetoothState); + if (state == BluetoothHeadset.STATE_AUDIO_CONNECTED) { + cancelTimer(); + if (bluetoothState == State.SCO_CONNECTING) { + Log.d(Config.LOGTAG, "+++ Bluetooth audio SCO is now connected"); + bluetoothState = State.SCO_CONNECTED; + scoConnectionAttempts = 0; + updateAudioDeviceState(); + } else { + Log.w( + Config.LOGTAG, + "Unexpected state BluetoothHeadset.STATE_AUDIO_CONNECTED"); + } + } else if (state == BluetoothHeadset.STATE_AUDIO_CONNECTING) { + Log.d(Config.LOGTAG, "+++ Bluetooth audio SCO is now connecting..."); + } else if (state == BluetoothHeadset.STATE_AUDIO_DISCONNECTED) { + Log.d(Config.LOGTAG, "+++ Bluetooth audio SCO is now disconnected"); + if (isInitialStickyBroadcast()) { + Log.d( + Config.LOGTAG, + "Ignore STATE_AUDIO_DISCONNECTED initial sticky broadcast."); + return; + } + updateAudioDeviceState(); + } + } + Log.d(Config.LOGTAG, "onReceive done: BT state=" + bluetoothState); + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/services/AppRTCProximitySensor.java b/app/src/main/java/eu/siacs/conversations/services/AppRTCProximitySensor.java new file mode 100644 index 000000000..818a6e85a --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/services/AppRTCProximitySensor.java @@ -0,0 +1,161 @@ +/* + * Copyright 2014 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +package eu.siacs.conversations.services; + +import android.content.Context; +import android.hardware.Sensor; +import android.hardware.SensorEvent; +import android.hardware.SensorEventListener; +import android.hardware.SensorManager; +import android.os.Build; +import android.util.Log; +import androidx.annotation.Nullable; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.utils.AppRTCUtils; +import org.webrtc.ThreadUtils; + +/** + * AppRTCProximitySensor manages functions related to the proximity sensor in the AppRTC demo. On + * most device, the proximity sensor is implemented as a boolean-sensor. It returns just two values + * "NEAR" or "FAR". Thresholding is done on the LUX value i.e. the LUX value of the light sensor is + * compared with a threshold. A LUX-value more than the threshold means the proximity sensor returns + * "FAR". Anything less than the threshold value and the sensor returns "NEAR". + */ +public class AppRTCProximitySensor implements SensorEventListener { + // This class should be created, started and stopped on one thread + // (e.g. the main thread). We use |nonThreadSafe| to ensure that this is + // the case. Only active when |DEBUG| is set to true. + private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); + private final Runnable onSensorStateListener; + private final SensorManager sensorManager; + @Nullable private Sensor proximitySensor; + private boolean lastStateReportIsNear; + + private AppRTCProximitySensor(Context context, Runnable sensorStateListener) { + Log.d(Config.LOGTAG, "AppRTCProximitySensor" + AppRTCUtils.getThreadInfo()); + onSensorStateListener = sensorStateListener; + sensorManager = ((SensorManager) context.getSystemService(Context.SENSOR_SERVICE)); + } + + /** Construction */ + static AppRTCProximitySensor create(Context context, Runnable sensorStateListener) { + return new AppRTCProximitySensor(context, sensorStateListener); + } + + /** Activate the proximity sensor. Also do initialization if called for the first time. */ + public boolean start() { + threadChecker.checkIsOnValidThread(); + Log.d(Config.LOGTAG, "start" + AppRTCUtils.getThreadInfo()); + if (!initDefaultSensor()) { + // Proximity sensor is not supported on this device. + return false; + } + sensorManager.registerListener(this, proximitySensor, SensorManager.SENSOR_DELAY_NORMAL); + return true; + } + + /** Deactivate the proximity sensor. */ + public void stop() { + threadChecker.checkIsOnValidThread(); + Log.d(Config.LOGTAG, "stop" + AppRTCUtils.getThreadInfo()); + if (proximitySensor == null) { + return; + } + sensorManager.unregisterListener(this, proximitySensor); + } + + /** Getter for last reported state. Set to true if "near" is reported. */ + public boolean sensorReportsNearState() { + threadChecker.checkIsOnValidThread(); + return lastStateReportIsNear; + } + + @Override + public final void onAccuracyChanged(Sensor sensor, int accuracy) { + threadChecker.checkIsOnValidThread(); + AppRTCUtils.assertIsTrue(sensor.getType() == Sensor.TYPE_PROXIMITY); + if (accuracy == SensorManager.SENSOR_STATUS_UNRELIABLE) { + Log.e(Config.LOGTAG, "The values returned by this sensor cannot be trusted"); + } + } + + @Override + public final void onSensorChanged(SensorEvent event) { + threadChecker.checkIsOnValidThread(); + AppRTCUtils.assertIsTrue(event.sensor.getType() == Sensor.TYPE_PROXIMITY); + // As a best practice; do as little as possible within this method and + // avoid blocking. + float distanceInCentimeters = event.values[0]; + if (distanceInCentimeters < proximitySensor.getMaximumRange()) { + Log.d(Config.LOGTAG, "Proximity sensor => NEAR state"); + lastStateReportIsNear = true; + } else { + Log.d(Config.LOGTAG, "Proximity sensor => FAR state"); + lastStateReportIsNear = false; + } + // Report about new state to listening client. Client can then call + // sensorReportsNearState() to query the current state (NEAR or FAR). + if (onSensorStateListener != null) { + onSensorStateListener.run(); + } + Log.d( + Config.LOGTAG, + "onSensorChanged" + + AppRTCUtils.getThreadInfo() + + ": " + + "accuracy=" + + event.accuracy + + ", timestamp=" + + event.timestamp + + ", distance=" + + event.values[0]); + } + + /** + * Get default proximity sensor if it exists. Tablet devices (e.g. Nexus 7) does not support + * this type of sensor and false will be returned in such cases. + */ + private boolean initDefaultSensor() { + if (proximitySensor != null) { + return true; + } + proximitySensor = sensorManager.getDefaultSensor(Sensor.TYPE_PROXIMITY); + if (proximitySensor == null) { + return false; + } + logProximitySensorInfo(); + return true; + } + + /** Helper method for logging information about the proximity sensor. */ + private void logProximitySensorInfo() { + if (proximitySensor == null) { + return; + } + StringBuilder info = new StringBuilder("Proximity sensor: "); + info.append("name=").append(proximitySensor.getName()); + info.append(", vendor: ").append(proximitySensor.getVendor()); + info.append(", power: ").append(proximitySensor.getPower()); + info.append(", resolution: ").append(proximitySensor.getResolution()); + info.append(", max range: ").append(proximitySensor.getMaximumRange()); + info.append(", min delay: ").append(proximitySensor.getMinDelay()); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT_WATCH) { + // Added in API level 20. + info.append(", type: ").append(proximitySensor.getStringType()); + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + // Added in API level 21. + info.append(", max delay: ").append(proximitySensor.getMaxDelay()); + info.append(", reporting mode: ").append(proximitySensor.getReportingMode()); + info.append(", isWakeUpSensor: ").append(proximitySensor.isWakeUpSensor()); + } + Log.d(Config.LOGTAG, info.toString()); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/utils/AppRTCUtils.java b/app/src/main/java/eu/siacs/conversations/utils/AppRTCUtils.java new file mode 100644 index 000000000..a628d3ac2 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/utils/AppRTCUtils.java @@ -0,0 +1,66 @@ +/* + * Copyright 2014 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +package eu.siacs.conversations.utils; + +import android.os.Build; +import android.util.Log; + +/** AppRTCUtils provides helper functions for managing thread safety. */ +public final class AppRTCUtils { + private AppRTCUtils() {} + + /** Helper method which throws an exception when an assertion has failed. */ + public static void assertIsTrue(boolean condition) { + if (!condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + /** Helper method for building a string of thread information. */ + public static String getThreadInfo() { + return "@[name=" + + Thread.currentThread().getName() + + ", id=" + + Thread.currentThread().getId() + + "]"; + } + + /** Information about the current build, taken from system properties. */ + public static void logDeviceInfo(String tag) { + Log.d( + tag, + "Android SDK: " + + Build.VERSION.SDK_INT + + ", " + + "Release: " + + Build.VERSION.RELEASE + + ", " + + "Brand: " + + Build.BRAND + + ", " + + "Device: " + + Build.DEVICE + + ", " + + "Id: " + + Build.ID + + ", " + + "Hardware: " + + Build.HARDWARE + + ", " + + "Manufacturer: " + + Build.MANUFACTURER + + ", " + + "Model: " + + Build.MODEL + + ", " + + "Product: " + + Build.PRODUCT); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/AbstractJingleConnection.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/AbstractJingleConnection.java new file mode 100644 index 000000000..04cea807d --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/AbstractJingleConnection.java @@ -0,0 +1,110 @@ +package eu.siacs.conversations.xmpp.jingle; + +import android.content.Context; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; +import im.conversations.android.IDs; +import im.conversations.android.xmpp.XmppConnection; +import im.conversations.android.xmpp.model.stanza.Iq; +import org.jxmpp.jid.Jid; + +public abstract class AbstractJingleConnection extends XmppConnection.Delegate { + + public static final String JINGLE_MESSAGE_PROPOSE_ID_PREFIX = "jm-propose-"; + public static final String JINGLE_MESSAGE_PROCEED_ID_PREFIX = "jm-proceed-"; + + protected final Id id; + private final Jid initiator; + + AbstractJingleConnection( + final Context context, + final XmppConnection connection, + final Id id, + final Jid initiator) { + super(context, connection); + this.id = id; + this.initiator = initiator; + } + + boolean isInitiator() { + return initiator.equals(connection.getBoundAddress()); + } + + public abstract void deliverPacket(Iq jinglePacket); + + public Id getId() { + return id; + } + + public abstract void notifyRebound(); + + public static class Id implements OngoingRtpSession { + public final Jid with; + public final String sessionId; + + private Id(final Jid with, final String sessionId) { + Preconditions.checkNotNull(with); + Preconditions.checkNotNull(sessionId); + this.with = with; + this.sessionId = sessionId; + } + + public static Id of(final JinglePacket jinglePacket) { + return new Id(jinglePacket.getFrom(), jinglePacket.getSessionId()); + } + + public static Id of(Jid with, final String sessionId) { + return new Id(with, sessionId); + } + + public static Id of(Jid with) { + return new Id(with, IDs.medium()); + } + + @Override + public Jid getWith() { + return with; + } + + @Override + public String getSessionId() { + return sessionId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Id id = (Id) o; + return Objects.equal(with, id.with) && Objects.equal(sessionId, id.sessionId); + } + + @Override + public int hashCode() { + return Objects.hashCode(with, sessionId); + } + } + + public enum State { + NULL, // default value; nothing has been sent or received yet + PROPOSED, + ACCEPTED, + PROCEED, + REJECTED, + REJECTED_RACED, // used when we want to reject but haven’t received session init yet + RETRACTED, + RETRACTED_RACED, // used when receiving a retract after we already asked to proceed + SESSION_INITIALIZED, // equal to 'PENDING' + SESSION_INITIALIZED_PRE_APPROVED, + SESSION_ACCEPTED, // equal to 'ACTIVE' + TERMINATED_SUCCESS, // equal to 'ENDED' (after successful call) ui will just close + TERMINATED_DECLINED_OR_BUSY, // equal to 'ENDED' (after other party declined the call) + TERMINATED_CONNECTIVITY_ERROR, // equal to 'ENDED' (but after network failures; ui will + // display retry button) + TERMINATED_CANCEL_OR_TIMEOUT, // more or less the same as retracted; caller pressed end call + // before session was accepted + TERMINATED_APPLICATION_FAILURE, + TERMINATED_SECURITY_ERROR + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java new file mode 100644 index 000000000..c3b4f4d59 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java @@ -0,0 +1,86 @@ +package eu.siacs.conversations.xmpp.jingle; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableSet; +import eu.siacs.conversations.xmpp.jingle.stanzas.Content; +import java.util.Set; + +public final class ContentAddition { + + public final Direction direction; + public final Set summary; + + private ContentAddition(Direction direction, Set summary) { + this.direction = direction; + this.summary = summary; + } + + public Set media() { + return ImmutableSet.copyOf(Collections2.transform(summary, s -> s.media)); + } + + public static ContentAddition of(final Direction direction, final RtpContentMap rtpContentMap) { + return new ContentAddition(direction, summary(rtpContentMap)); + } + + public static Set summary(final RtpContentMap rtpContentMap) { + return ImmutableSet.copyOf( + Collections2.transform( + rtpContentMap.contents.entrySet(), + e -> { + final RtpContentMap.DescriptionTransport dt = e.getValue(); + return new Summary(e.getKey(), dt.description.getMedia(), dt.senders); + })); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("direction", direction) + .add("summary", summary) + .toString(); + } + + public enum Direction { + OUTGOING, + INCOMING + } + + public static final class Summary { + public final String name; + public final Media media; + public final Content.Senders senders; + + private Summary(final String name, final Media media, final Content.Senders senders) { + this.name = name; + this.media = media; + this.senders = senders; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Summary summary = (Summary) o; + return Objects.equal(name, summary.name) + && media == summary.media + && senders == summary.senders; + } + + @Override + public int hashCode() { + return Objects.hashCode(name, media, senders); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("name", name) + .add("media", media) + .add("senders", senders) + .toString(); + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/DirectConnectionUtils.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/DirectConnectionUtils.java new file mode 100644 index 000000000..b6b88f50b --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/DirectConnectionUtils.java @@ -0,0 +1,64 @@ +package eu.siacs.conversations.xmpp.jingle; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.UUID; +import org.jxmpp.jid.Jid; + +public class DirectConnectionUtils { + + private static List getLocalAddresses() { + final List addresses = new ArrayList<>(); + final Enumeration interfaces; + try { + interfaces = NetworkInterface.getNetworkInterfaces(); + } catch (SocketException e) { + return addresses; + } + while (interfaces.hasMoreElements()) { + NetworkInterface networkInterface = interfaces.nextElement(); + final Enumeration inetAddressEnumeration = + networkInterface.getInetAddresses(); + while (inetAddressEnumeration.hasMoreElements()) { + final InetAddress inetAddress = inetAddressEnumeration.nextElement(); + if (inetAddress.isLoopbackAddress() || inetAddress.isLinkLocalAddress()) { + continue; + } + if (inetAddress instanceof Inet6Address) { + // let's get rid of scope + try { + addresses.add(Inet6Address.getByAddress(inetAddress.getAddress())); + } catch (UnknownHostException e) { + // ignored + } + } else { + addresses.add(inetAddress); + } + } + } + return addresses; + } + + public static List getLocalCandidates(Jid jid) { + SecureRandom random = new SecureRandom(); + ArrayList candidates = new ArrayList<>(); + for (InetAddress inetAddress : getLocalAddresses()) { + final JingleCandidate candidate = + new JingleCandidate(UUID.randomUUID().toString(), true); + candidate.setHost(inetAddress.getHostAddress()); + candidate.setPort(random.nextInt(60000) + 1024); + candidate.setType(JingleCandidate.TYPE_DIRECT); + candidate.setJid(jid); + candidate.setPriority(8257536 + candidates.size()); + candidates.add(candidate); + } + return candidates; + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleCandidate.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleCandidate.java new file mode 100644 index 000000000..6a4b334be --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleCandidate.java @@ -0,0 +1,153 @@ +package eu.siacs.conversations.xmpp.jingle; + +import im.conversations.android.xml.Element; +import java.util.ArrayList; +import java.util.List; +import org.jxmpp.jid.Jid; + +public class JingleCandidate { + + public static int TYPE_UNKNOWN; + public static int TYPE_DIRECT = 0; + public static int TYPE_PROXY = 1; + + private final boolean ours; + private boolean usedByCounterpart = false; + private final String cid; + private String host; + private int port; + private int type; + private Jid jid; + private int priority; + + public JingleCandidate(String cid, boolean ours) { + this.ours = ours; + this.cid = cid; + } + + public String getCid() { + return cid; + } + + public void setHost(String host) { + this.host = host; + } + + public String getHost() { + return this.host; + } + + public void setJid(final Jid jid) { + this.jid = jid; + } + + public Jid getJid() { + return this.jid; + } + + public void setPort(int port) { + this.port = port; + } + + public int getPort() { + return this.port; + } + + public void setType(int type) { + this.type = type; + } + + public void setType(String type) { + if (type == null) { + this.type = TYPE_UNKNOWN; + return; + } + switch (type) { + case "proxy": + this.type = TYPE_PROXY; + break; + case "direct": + this.type = TYPE_DIRECT; + break; + default: + this.type = TYPE_UNKNOWN; + break; + } + } + + public void setPriority(int i) { + this.priority = i; + } + + public int getPriority() { + return this.priority; + } + + public boolean equals(JingleCandidate other) { + return this.getCid().equals(other.getCid()); + } + + public boolean equalValues(JingleCandidate other) { + return other != null + && other.getHost().equals(this.getHost()) + && (other.getPort() == this.getPort()); + } + + public boolean isOurs() { + return ours; + } + + public int getType() { + return this.type; + } + + public static List parse(final List elements) { + final List candidates = new ArrayList<>(); + for (final Element element : elements) { + if ("candidate".equals(element.getName())) { + candidates.add(JingleCandidate.parse(element)); + } + } + return candidates; + } + + public static JingleCandidate parse(Element element) { + final JingleCandidate candidate = new JingleCandidate(element.getAttribute("cid"), false); + candidate.setHost(element.getAttribute("host")); + candidate.setJid(element.getAttributeAsJid("jid")); + candidate.setType(element.getAttribute("type")); + candidate.setPriority(Integer.parseInt(element.getAttribute("priority"))); + candidate.setPort(Integer.parseInt(element.getAttribute("port"))); + return candidate; + } + + public Element toElement() { + Element element = new Element("candidate"); + element.setAttribute("cid", this.getCid()); + element.setAttribute("host", this.getHost()); + element.setAttribute("port", Integer.toString(this.getPort())); + if (jid != null) { + element.setAttribute("jid", jid); + } + element.setAttribute("priority", Integer.toString(this.getPriority())); + if (this.getType() == TYPE_DIRECT) { + element.setAttribute("type", "direct"); + } else if (this.getType() == TYPE_PROXY) { + element.setAttribute("type", "proxy"); + } + return element; + } + + public void flagAsUsedByCounterpart() { + this.usedByCounterpart = true; + } + + public boolean isUsedByCounterpart() { + return this.usedByCounterpart; + } + + public String toString() { + return String.format( + "%s:%s (priority=%s,ours=%s)", getHost(), getPort(), getPriority(), isOurs()); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java new file mode 100644 index 000000000..1130469a0 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -0,0 +1,2672 @@ +package eu.siacs.conversations.xmpp.jingle; + +import android.content.Context; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.services.AppRTCAudioManager; +import eu.siacs.conversations.xmpp.jingle.stanzas.Content; +import eu.siacs.conversations.xmpp.jingle.stanzas.Group; +import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; +import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; +import eu.siacs.conversations.xmpp.jingle.stanzas.Reason; +import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; +import im.conversations.android.BuildConfig; +import im.conversations.android.axolotl.AxolotlEncryptionException; +import im.conversations.android.axolotl.AxolotlService; +import im.conversations.android.dns.IP; +import im.conversations.android.notification.RtpSessionNotification; +import im.conversations.android.transformer.CallLogEntry; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import im.conversations.android.xmpp.Entity; +import im.conversations.android.xmpp.XmppConnection; +import im.conversations.android.xmpp.manager.AxolotlManager; +import im.conversations.android.xmpp.manager.DiscoManager; +import im.conversations.android.xmpp.manager.ExternalDiscoManager; +import im.conversations.android.xmpp.manager.JingleConnectionManager; +import im.conversations.android.xmpp.model.disco.external.Service; +import im.conversations.android.xmpp.model.error.Condition; +import im.conversations.android.xmpp.model.error.Error; +import im.conversations.android.xmpp.model.jmi.Accept; +import im.conversations.android.xmpp.model.jmi.JingleMessage; +import im.conversations.android.xmpp.model.jmi.Proceed; +import im.conversations.android.xmpp.model.jmi.Propose; +import im.conversations.android.xmpp.model.jmi.Reject; +import im.conversations.android.xmpp.model.jmi.Retract; +import im.conversations.android.xmpp.model.stanza.Iq; +import im.conversations.android.xmpp.model.stanza.Message; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import org.jxmpp.jid.Jid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.webrtc.EglBase; +import org.webrtc.IceCandidate; +import org.webrtc.PeerConnection; +import org.webrtc.VideoTrack; +import org.whispersystems.libsignal.IdentityKey; + +public class JingleRtpConnection extends AbstractJingleConnection + implements WebRTCWrapper.EventCallback { + + private static final Logger LOGGER = LoggerFactory.getLogger(JingleRtpConnection.class); + public static final List STATES_SHOWING_ONGOING_CALL = + Arrays.asList( + State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED); + private static final long BUSY_TIME_OUT = 30; + private static final List TERMINATED = + Arrays.asList( + State.ACCEPTED, + State.REJECTED, + State.REJECTED_RACED, + State.RETRACTED, + State.RETRACTED_RACED, + State.TERMINATED_SUCCESS, + State.TERMINATED_DECLINED_OR_BUSY, + State.TERMINATED_CONNECTIVITY_ERROR, + State.TERMINATED_CANCEL_OR_TIMEOUT, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR); + + private static final Map> VALID_TRANSITIONS; + + static { + final ImmutableMap.Builder> transitionBuilder = + new ImmutableMap.Builder<>(); + transitionBuilder.put( + State.NULL, + ImmutableList.of( + State.PROPOSED, + State.SESSION_INITIALIZED, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR)); + transitionBuilder.put( + State.PROPOSED, + ImmutableList.of( + State.ACCEPTED, + State.PROCEED, + State.REJECTED, + State.RETRACTED, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR, + State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection + // rebinds + )); + transitionBuilder.put( + State.PROCEED, + ImmutableList.of( + State.REJECTED_RACED, + State.RETRACTED_RACED, + State.SESSION_INITIALIZED_PRE_APPROVED, + State.TERMINATED_SUCCESS, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR, + State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error + // bounces of the proceed message + )); + transitionBuilder.put( + State.SESSION_INITIALIZED, + ImmutableList.of( + State.SESSION_ACCEPTED, + State.TERMINATED_SUCCESS, + State.TERMINATED_DECLINED_OR_BUSY, + State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors + // and IQ timeouts + State.TERMINATED_CANCEL_OR_TIMEOUT, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR)); + transitionBuilder.put( + State.SESSION_INITIALIZED_PRE_APPROVED, + ImmutableList.of( + State.SESSION_ACCEPTED, + State.TERMINATED_SUCCESS, + State.TERMINATED_DECLINED_OR_BUSY, + State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors + // and IQ timeouts + State.TERMINATED_CANCEL_OR_TIMEOUT, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR)); + transitionBuilder.put( + State.SESSION_ACCEPTED, + ImmutableList.of( + State.TERMINATED_SUCCESS, + State.TERMINATED_DECLINED_OR_BUSY, + State.TERMINATED_CONNECTIVITY_ERROR, + State.TERMINATED_CANCEL_OR_TIMEOUT, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR)); + VALID_TRANSITIONS = transitionBuilder.build(); + } + + private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this); + private final Queue> + pendingIceCandidates = new LinkedList<>(); + private final OmemoVerification omemoVerification = new OmemoVerification(); + private State state = State.NULL; + private Set proposedMedia; + private RtpContentMap initiatorRtpContentMap; + private RtpContentMap responderRtpContentMap; + private RtpContentMap incomingContentAdd; + private RtpContentMap outgoingContentAdd; + private IceUdpTransportInfo.Setup peerDtlsSetup; + private final Stopwatch sessionDuration = Stopwatch.createUnstarted(); + private final Queue stateHistory = new LinkedList<>(); + private final RtpSessionNotification rtpSessionNotification; + private ScheduledFuture ringingTimeoutFuture; + private CallLogEntry message = null; + + public JingleRtpConnection( + final Context context, + final XmppConnection connection, + final Id id, + final Jid initiator) { + super(context, connection, id, initiator); + this.rtpSessionNotification = new RtpSessionNotification(context); + } + + private static State reasonToState(Reason reason) { + switch (reason) { + case SUCCESS: + return State.TERMINATED_SUCCESS; + case DECLINE: + case BUSY: + return State.TERMINATED_DECLINED_OR_BUSY; + case CANCEL: + case TIMEOUT: + return State.TERMINATED_CANCEL_OR_TIMEOUT; + case SECURITY_ERROR: + return State.TERMINATED_SECURITY_ERROR; + case FAILED_APPLICATION: + case UNSUPPORTED_TRANSPORTS: + case UNSUPPORTED_APPLICATIONS: + return State.TERMINATED_APPLICATION_FAILURE; + default: + return State.TERMINATED_CONNECTIVITY_ERROR; + } + } + + @Override + public synchronized void deliverPacket(final Iq iq) { + final var jinglePacket = JinglePacket.upgrade(iq); + switch (jinglePacket.getAction()) { + case SESSION_INITIATE: + receiveSessionInitiate(jinglePacket); + break; + case TRANSPORT_INFO: + receiveTransportInfo(jinglePacket); + break; + case SESSION_ACCEPT: + receiveSessionAccept(jinglePacket); + break; + case SESSION_TERMINATE: + receiveSessionTerminate(jinglePacket); + break; + case CONTENT_ADD: + receiveContentAdd(jinglePacket); + break; + case CONTENT_ACCEPT: + receiveContentAccept(jinglePacket); + break; + case CONTENT_REJECT: + receiveContentReject(jinglePacket); + break; + case CONTENT_REMOVE: + receiveContentRemove(jinglePacket); + break; + default: + respondOk(jinglePacket); + LOGGER.debug( + String.format( + "%s: received unhandled jingle action %s", + connection.getAccount().address, jinglePacket.getAction())); + break; + } + } + + @Override + public synchronized void notifyRebound() { + if (isTerminated()) { + return; + } + webRTCWrapper.close(); + if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) { + this.rtpSessionNotification.cancelIncomingCallNotification(); + } + if (isInState( + State.SESSION_INITIALIZED, + State.SESSION_INITIALIZED_PRE_APPROVED, + State.SESSION_ACCEPTED)) { + // we might have already changed resources (full jid) at this point; so this might not + // even reach the other party + sendSessionTerminate(Reason.CONNECTIVITY_ERROR); + } else { + transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR); + finish(); + } + } + + private void receiveSessionTerminate(final JinglePacket jinglePacket) { + respondOk(jinglePacket); + final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason(); + final State previous = this.state; + LOGGER.debug( + connection.getAccount().address + + ": received session terminate reason=" + + wrapper.reason + + "(" + + Strings.nullToEmpty(wrapper.text) + + ") while in state " + + previous); + if (TERMINATED.contains(previous)) { + LOGGER.debug( + connection.getAccount().address + + ": ignoring session terminate because already in " + + previous); + return; + } + webRTCWrapper.close(); + final State target = reasonToState(wrapper.reason); + transitionOrThrow(target); + writeLogMessage(target); + if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) { + this.rtpSessionNotification.cancelIncomingCallNotification(); + } + finish(); + } + + private void receiveTransportInfo(final JinglePacket jinglePacket) { + // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to + // INITIALIZED only after transport-info has been received + if (isInState( + State.NULL, + State.PROCEED, + State.SESSION_INITIALIZED, + State.SESSION_INITIALIZED_PRE_APPROVED, + State.SESSION_ACCEPTED)) { + final RtpContentMap contentMap; + try { + contentMap = RtpContentMap.of(jinglePacket); + } catch (final IllegalArgumentException | NullPointerException e) { + LOGGER.debug( + connection.getAccount().address + + ": improperly formatted contents; ignoring", + e); + respondOk(jinglePacket); + return; + } + receiveTransportInfo(jinglePacket, contentMap); + } else { + if (isTerminated()) { + respondOk(jinglePacket); + LOGGER.debug( + connection.getAccount().address + + ": ignoring out-of-order transport info; we where already" + + " terminated"); + } else { + LOGGER.debug( + connection.getAccount().address + + ": received transport info while in state=" + + this.state); + terminateWithOutOfOrder(jinglePacket); + } + } + } + + private void receiveTransportInfo( + final JinglePacket jinglePacket, final RtpContentMap contentMap) { + final Set> candidates = + contentMap.contents.entrySet(); + if (this.state == State.SESSION_ACCEPTED) { + // zero candidates + modified credentials are an ICE restart offer + if (checkForIceRestart(jinglePacket, contentMap)) { + return; + } + respondOk(jinglePacket); + try { + processCandidates(candidates); + } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { + LOGGER.warn( + connection.getAccount().address + + ": PeerConnection was not initialized when processing transport" + + " info. this usually indicates a race condition that can be" + + " ignored"); + } + } else { + respondOk(jinglePacket); + pendingIceCandidates.addAll(candidates); + } + } + + private void receiveContentAdd(final JinglePacket jinglePacket) { + final RtpContentMap modification; + try { + modification = RtpContentMap.of(jinglePacket); + modification.requireContentDescriptions(); + } catch (final RuntimeException e) { + LOGGER.debug( + connection.getAccount().address + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + if (isInState(State.SESSION_ACCEPTED)) { + receiveContentAdd(jinglePacket, modification); + } else { + terminateWithOutOfOrder(jinglePacket); + } + } + + private void receiveContentAdd( + final JinglePacket jinglePacket, final RtpContentMap modification) { + final RtpContentMap remote = getRemoteContentMap(); + if (!Collections.disjoint(modification.getNames(), remote.getNames())) { + respondOk(jinglePacket); + this.webRTCWrapper.close(); + sendSessionTerminate( + Reason.FAILED_APPLICATION, + String.format( + "contents with names %s already exists", + Joiner.on(", ").join(modification.getNames()))); + return; + } + final ContentAddition contentAddition = + ContentAddition.of(ContentAddition.Direction.INCOMING, modification); + + final RtpContentMap outgoing = this.outgoingContentAdd; + final Set outgoingContentAddSummary = + outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing); + + if (outgoingContentAddSummary.equals(contentAddition.summary)) { + if (isInitiator()) { + LOGGER.debug( + connection.getAccount().address + + ": respond with tie break to matching content-add offer"); + respondWithTieBreak(jinglePacket); + } else { + LOGGER.debug( + connection.getAccount().address + + ": automatically accept matching content-add offer"); + acceptContentAdd(contentAddition.summary, modification); + } + return; + } + + // once we can display multiple video tracks we can be more loose with this condition + // theoretically it should also be fine to automatically accept audio only contents + if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) { + LOGGER.debug(connection.getAccount().address + ": received " + contentAddition); + this.incomingContentAdd = modification; + respondOk(jinglePacket); + updateEndUserState(); + } else { + respondOk(jinglePacket); + // TODO do we want to add a reason? + rejectContentAdd(modification); + } + } + + private void receiveContentAccept(final JinglePacket jinglePacket) { + final RtpContentMap receivedContentAccept; + try { + receivedContentAccept = RtpContentMap.of(jinglePacket); + receivedContentAccept.requireContentDescriptions(); + } catch (final RuntimeException e) { + LOGGER.debug( + connection.getAccount().address + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + + final RtpContentMap outgoingContentAdd = this.outgoingContentAdd; + if (outgoingContentAdd == null) { + LOGGER.debug("received content-accept when we had no outgoing content add"); + terminateWithOutOfOrder(jinglePacket); + return; + } + final Set ourSummary = ContentAddition.summary(outgoingContentAdd); + if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) { + this.outgoingContentAdd = null; + respondOk(jinglePacket); + receiveContentAccept(receivedContentAccept); + } else { + LOGGER.debug("received content-accept did not match our outgoing content-add"); + terminateWithOutOfOrder(jinglePacket); + } + } + + private void receiveContentAccept(final RtpContentMap receivedContentAccept) { + final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup(); + final RtpContentMap modifiedContentMap = + getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup); + + setRemoteContentMap(modifiedContentMap); + + final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator()); + + final org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.ANSWER, answer.toString()); + + try { + this.webRTCWrapper.setRemoteDescription(sdp).get(); + } catch (final Exception e) { + final Throwable cause = Throwables.getRootCause(e); + LOGGER.debug( + connection.getAccount().address + + ": unable to set remote description after receiving content-accept", + cause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); + return; + } + updateEndUserState(); + LOGGER.debug( + connection.getAccount().address + + ": remote has accepted content-add " + + ContentAddition.summary(receivedContentAccept)); + } + + private void receiveContentReject(final JinglePacket jinglePacket) { + final RtpContentMap receivedContentReject; + try { + receivedContentReject = RtpContentMap.of(jinglePacket); + } catch (final RuntimeException e) { + LOGGER.debug( + connection.getAccount().address + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + this.webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + + final RtpContentMap outgoingContentAdd = this.outgoingContentAdd; + if (outgoingContentAdd == null) { + LOGGER.debug("received content-reject when we had no outgoing content add"); + terminateWithOutOfOrder(jinglePacket); + return; + } + final Set ourSummary = ContentAddition.summary(outgoingContentAdd); + if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) { + this.outgoingContentAdd = null; + respondOk(jinglePacket); + LOGGER.debug(jinglePacket.toString()); + receiveContentReject(ourSummary); + } else { + LOGGER.debug("received content-reject did not match our outgoing content-add"); + terminateWithOutOfOrder(jinglePacket); + } + } + + private void receiveContentReject(final Set summary) { + try { + this.webRTCWrapper.removeTrack(Media.VIDEO); + final RtpContentMap localContentMap = customRollback(); + modifyLocalContentMap(localContentMap); + } catch (final Exception e) { + final Throwable cause = Throwables.getRootCause(e); + LOGGER.debug( + connection.getAccount().address + + ": unable to rollback local description after receiving" + + " content-reject", + cause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); + return; + } + LOGGER.debug( + connection.getAccount().address + + ": remote has rejected our content-add " + + summary); + } + + private void receiveContentRemove(final JinglePacket jinglePacket) { + final RtpContentMap receivedContentRemove; + try { + receivedContentRemove = RtpContentMap.of(jinglePacket); + receivedContentRemove.requireContentDescriptions(); + } catch (final RuntimeException e) { + LOGGER.debug( + connection.getAccount().address + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + this.webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + respondOk(jinglePacket); + receiveContentRemove(receivedContentRemove); + } + + private void receiveContentRemove(final RtpContentMap receivedContentRemove) { + final RtpContentMap incomingContentAdd = this.incomingContentAdd; + final Set contentAddSummary = + incomingContentAdd == null + ? Collections.emptySet() + : ContentAddition.summary(incomingContentAdd); + final Set removeSummary = + ContentAddition.summary(receivedContentRemove); + if (contentAddSummary.equals(removeSummary)) { + this.incomingContentAdd = null; + updateEndUserState(); + } else { + webRTCWrapper.close(); + sendSessionTerminate( + Reason.FAILED_APPLICATION, + String.format( + "%s only supports %s as a means to retract a not yet accepted %s", + BuildConfig.APP_NAME, + JinglePacket.Action.CONTENT_REMOVE, + JinglePacket.Action.CONTENT_ACCEPT)); + } + } + + public synchronized void retractContentAdd() { + final RtpContentMap outgoingContentAdd = this.outgoingContentAdd; + if (outgoingContentAdd == null) { + throw new IllegalStateException("Not outgoing content add"); + } + try { + webRTCWrapper.removeTrack(Media.VIDEO); + final RtpContentMap localContentMap = customRollback(); + modifyLocalContentMap(localContentMap); + } catch (final Exception e) { + final Throwable cause = Throwables.getRootCause(e); + LOGGER.debug( + connection.getAccount().address + + ": unable to rollback local description after trying to retract" + + " content-add", + cause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); + return; + } + this.outgoingContentAdd = null; + final JinglePacket retract = + outgoingContentAdd + .toStub() + .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId); + this.send(retract); + LOGGER.debug( + connection.getAccount().address + + ": retract content-add " + + ContentAddition.summary(outgoingContentAdd)); + } + + private RtpContentMap customRollback() throws ExecutionException, InterruptedException { + final SessionDescription sdp = setLocalSessionDescription(); + final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator()); + final SessionDescription answer = generateFakeResponse(localRtpContentMap); + this.webRTCWrapper + .setRemoteDescription( + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.ANSWER, answer.toString())) + .get(); + return localRtpContentMap; + } + + private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) { + final RtpContentMap currentRemote = getRemoteContentMap(); + final RtpContentMap.Diff diff = currentRemote.diff(localContentMap); + if (diff.isEmpty()) { + throw new IllegalStateException( + "Unexpected rollback condition. No difference between local and remote"); + } + final RtpContentMap patch = localContentMap.toContentModification(diff.added); + if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) { + final RtpContentMap nextRemote = + currentRemote.addContent( + patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup()); + return SessionDescription.of(nextRemote, !isInitiator()); + } + throw new IllegalStateException( + "Unexpected rollback condition. Senders were not uniformly none"); + } + + public synchronized void acceptContentAdd( + @NonNull final Set contentAddition) { + final RtpContentMap incomingContentAdd = this.incomingContentAdd; + if (incomingContentAdd == null) { + throw new IllegalStateException("No incoming content add"); + } + + if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) { + this.incomingContentAdd = null; + acceptContentAdd(contentAddition, incomingContentAdd); + } else { + throw new IllegalStateException( + "Accepted content add does not match pending content-add"); + } + } + + private void acceptContentAdd( + @NonNull final Set contentAddition, + final RtpContentMap incomingContentAdd) { + final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup(); + final RtpContentMap modifiedContentMap = + getRemoteContentMap().addContent(incomingContentAdd, setup); + this.setRemoteContentMap(modifiedContentMap); + + final SessionDescription offer; + try { + offer = SessionDescription.of(modifiedContentMap, !isInitiator()); + } catch (final IllegalArgumentException | NullPointerException e) { + LOGGER.debug( + connection.getAccount().address + + ": unable convert offer from content-add to SDP", + e); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); + return; + } + this.incomingContentAdd = null; + acceptContentAdd(contentAddition, offer); + } + + private void acceptContentAdd( + final Set contentAddition, final SessionDescription offer) { + final org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.OFFER, offer.toString()); + try { + this.webRTCWrapper.setRemoteDescription(sdp).get(); + + // TODO add tracks for 'media' where contentAddition.senders matches + + // TODO if senders.sending(isInitiator()) + + this.webRTCWrapper.addTrack(Media.VIDEO); + + // TODO add additional transceivers for recv only cases + + final SessionDescription answer = setLocalSessionDescription(); + final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator()); + + final RtpContentMap contentAcceptMap = + rtpContentMap.toContentModification( + Collections2.transform(contentAddition, ca -> ca.name)); + LOGGER.debug( + connection.getAccount().address + + ": sending content-accept " + + ContentAddition.summary(contentAcceptMap)); + modifyLocalContentMap(rtpContentMap); + sendContentAccept(contentAcceptMap); + } catch (final Exception e) { + LOGGER.debug("unable to accept content add", Throwables.getRootCause(e)); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION); + } + } + + private void sendContentAccept(final RtpContentMap contentAcceptMap) { + final JinglePacket jinglePacket = + contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId); + send(jinglePacket); + } + + public synchronized void rejectContentAdd() { + final RtpContentMap incomingContentAdd = this.incomingContentAdd; + if (incomingContentAdd == null) { + throw new IllegalStateException("No incoming content add"); + } + this.incomingContentAdd = null; + updateEndUserState(); + rejectContentAdd(incomingContentAdd); + } + + private void rejectContentAdd(final RtpContentMap contentMap) { + final JinglePacket jinglePacket = + contentMap + .toStub() + .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId); + LOGGER.debug( + connection.getAccount().address + + ": rejecting content " + + ContentAddition.summary(contentMap)); + send(jinglePacket); + } + + private boolean checkForIceRestart(final Iq jinglePacket, final RtpContentMap rtpContentMap) { + final RtpContentMap existing = getRemoteContentMap(); + final Set existingCredentials; + final IceUdpTransportInfo.Credentials newCredentials; + try { + existingCredentials = existing.getCredentials(); + newCredentials = rtpContentMap.getDistinctCredentials(); + } catch (final IllegalStateException e) { + LOGGER.debug("unable to gather credentials for comparison", e); + return false; + } + if (existingCredentials.contains(newCredentials)) { + return false; + } + // TODO an alternative approach is to check if we already got an iq result to our + // ICE-restart + // and if that's the case we are seeing an answer. + // This might be more spec compliant but also more error prone potentially + final boolean isOffer = rtpContentMap.emptyCandidates(); + final RtpContentMap restartContentMap; + try { + if (isOffer) { + LOGGER.debug("received offer to restart ICE " + newCredentials); + restartContentMap = + existing.modifiedCredentials( + newCredentials, IceUdpTransportInfo.Setup.ACTPASS); + } else { + final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup(); + LOGGER.debug( + "received confirmation of ICE restart" + + newCredentials + + " peer_setup=" + + setup); + // DTLS setup attribute needs to be rewritten to reflect current peer state + // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM + restartContentMap = existing.modifiedCredentials(newCredentials, setup); + } + if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) { + return isOffer; + } else { + LOGGER.debug("ignoring ICE restart. sending tie-break"); + respondWithTieBreak(jinglePacket); + return true; + } + } catch (final Exception exception) { + respondOk(jinglePacket); + final Throwable rootCause = Throwables.getRootCause(exception); + if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) { + // If this happens a termination is already in progress + LOGGER.debug("ignoring PeerConnectionNotInitialized on ICE restart"); + return true; + } + LOGGER.debug("failure to apply ICE restart", rootCause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage()); + return true; + } + } + + private IceUdpTransportInfo.Setup getPeerDtlsSetup() { + final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup; + if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) { + throw new IllegalStateException("Invalid peer setup"); + } + return peerSetup; + } + + private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) { + if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) { + throw new IllegalArgumentException("Trying to store invalid peer dtls setup"); + } + this.peerDtlsSetup = setup; + } + + private boolean applyIceRestart( + final Iq jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer) + throws ExecutionException, InterruptedException { + final SessionDescription sessionDescription = + SessionDescription.of(restartContentMap, !isInitiator()); + final org.webrtc.SessionDescription.Type type = + isOffer + ? org.webrtc.SessionDescription.Type.OFFER + : org.webrtc.SessionDescription.Type.ANSWER; + org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription(type, sessionDescription.toString()); + if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) { + if (isInitiator()) { + // We ignore the offer and respond with tie-break. This will clause the responder + // not to apply the content map + return false; + } + } + webRTCWrapper.setRemoteDescription(sdp).get(); + setRemoteContentMap(restartContentMap); + if (isOffer) { + webRTCWrapper.setIsReadyToReceiveIceCandidates(false); + final SessionDescription localSessionDescription = setLocalSessionDescription(); + setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator())); + // We need to respond OK before sending any candidates + respondOk(jinglePacket); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } else { + storePeerDtlsSetup(restartContentMap.getDtlsSetup()); + } + return true; + } + + private void processCandidates( + final Set> contents) { + for (final Map.Entry content : contents) { + processCandidate(content); + } + } + + private void processCandidate( + final Map.Entry content) { + final RtpContentMap rtpContentMap = getRemoteContentMap(); + final List indices = toIdentificationTags(rtpContentMap); + final String sdpMid = content.getKey(); // aka content name + final IceUdpTransportInfo transport = content.getValue().transport; + final IceUdpTransportInfo.Credentials credentials = transport.getCredentials(); + + // TODO check that credentials remained the same + + for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) { + final String sdp; + try { + sdp = candidate.toSdpAttribute(credentials.ufrag); + } catch (final IllegalArgumentException e) { + LOGGER.debug( + connection.getAccount().address + + ": ignoring invalid ICE candidate " + + e.getMessage()); + continue; + } + final int mLineIndex = indices.indexOf(sdpMid); + if (mLineIndex < 0) { + LOGGER.warn( + "mLineIndex not found for " + sdpMid + ". available indices " + indices); + } + final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp); + LOGGER.debug("received candidate: " + iceCandidate); + this.webRTCWrapper.addIceCandidate(iceCandidate); + } + } + + private RtpContentMap getRemoteContentMap() { + return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap; + } + + private RtpContentMap getLocalContentMap() { + return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; + } + + private List toIdentificationTags(final RtpContentMap rtpContentMap) { + final Group originalGroup = rtpContentMap.group; + final List identificationTags = + originalGroup == null + ? rtpContentMap.getNames() + : originalGroup.getIdentificationTags(); + if (identificationTags.size() == 0) { + LOGGER.warn( + connection.getAccount().address + + ": no identification tags found in initial offer. we won't be able" + + " to calculate mLineIndices"); + } + return identificationTags; + } + + private ListenableFuture receiveRtpContentMap( + final JinglePacket jinglePacket, final boolean expectVerification) { + final RtpContentMap receivedContentMap; + try { + receivedContentMap = RtpContentMap.of(jinglePacket); + } catch (final Exception e) { + return Futures.immediateFailedFuture(e); + } + if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) { + final ListenableFuture> future = + getManager(AxolotlManager.class) + .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with); + return Futures.transform( + future, + omemoVerifiedPayload -> { + // TODO test if an exception here triggers a correct abort + omemoVerification.setOrEnsureEqual(omemoVerifiedPayload); + LOGGER.debug( + connection.getAccount().address + + ": received verifiable DTLS fingerprint via " + + omemoVerification); + return omemoVerifiedPayload.getPayload(); + }, + MoreExecutors.directExecutor()); + } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) { + return Futures.immediateFailedFuture( + new SecurityException("DTLS fingerprint was unexpectedly not verifiable")); + } else { + return Futures.immediateFuture(receivedContentMap); + } + } + + private void receiveSessionInitiate(final JinglePacket jinglePacket) { + if (isInitiator()) { + LOGGER.debug( + String.format( + "%s: received session-initiate even though we were initiating", + connection.getAccount().address)); + if (isTerminated()) { + LOGGER.debug( + String.format( + "%s: got a reason to terminate with out-of-order. but already in" + + " state %s", + connection.getAccount().address, getState())); + respondWithOutOfOrder(jinglePacket); + } else { + terminateWithOutOfOrder(jinglePacket); + } + return; + } + final ListenableFuture future = receiveRtpContentMap(jinglePacket, false); + Futures.addCallback( + future, + new FutureCallback() { + @Override + public void onSuccess(@Nullable RtpContentMap rtpContentMap) { + receiveSessionInitiate(jinglePacket, rtpContentMap); + } + + @Override + public void onFailure(@NonNull final Throwable throwable) { + respondOk(jinglePacket); + sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage()); + } + }, + MoreExecutors.directExecutor()); + } + + private void receiveSessionInitiate( + final JinglePacket jinglePacket, final RtpContentMap contentMap) { + try { + contentMap.requireContentDescriptions(); + contentMap.requireDTLSFingerprint(true); + } catch (final RuntimeException e) { + LOGGER.debug( + connection.getAccount().address + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + LOGGER.debug("processing session-init with " + contentMap.contents.size() + " contents"); + final State target; + if (this.state == State.PROCEED) { + Preconditions.checkState( + proposedMedia != null && proposedMedia.size() > 0, + "proposed media must be set when processing pre-approved session-initiate"); + if (!this.proposedMedia.equals(contentMap.getMedia())) { + sendSessionTerminate( + Reason.SECURITY_ERROR, + String.format( + "Your session proposal (Jingle Message Initiation) included media" + + " %s but your session-initiate was %s", + this.proposedMedia, contentMap.getMedia())); + return; + } + target = State.SESSION_INITIALIZED_PRE_APPROVED; + } else { + target = State.SESSION_INITIALIZED; + } + if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) { + respondOk(jinglePacket); + pendingIceCandidates.addAll(contentMap.contents.entrySet()); + if (target == State.SESSION_INITIALIZED_PRE_APPROVED) { + LOGGER.debug( + connection.getAccount().address + + ": automatically accepting session-initiate"); + sendSessionAccept(); + } else { + LOGGER.debug( + connection.getAccount().address + + ": received not pre-approved session-initiate. start ringing"); + startRinging(); + } + } else { + LOGGER.debug( + String.format( + "%s: received session-initiate while in state %s", + connection.getAccount().address, state)); + terminateWithOutOfOrder(jinglePacket); + } + } + + private void receiveSessionAccept(final JinglePacket jinglePacket) { + if (!isInitiator()) { + LOGGER.debug( + String.format( + "%s: received session-accept even though we were responding", + connection.getAccount().address)); + terminateWithOutOfOrder(jinglePacket); + return; + } + final ListenableFuture future = + receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint()); + Futures.addCallback( + future, + new FutureCallback() { + @Override + public void onSuccess(@Nullable RtpContentMap rtpContentMap) { + receiveSessionAccept(jinglePacket, rtpContentMap); + } + + @Override + public void onFailure(@NonNull final Throwable throwable) { + respondOk(jinglePacket); + LOGGER.debug( + connection.getAccount().address + + ": improperly formatted contents in session-accept", + throwable); + webRTCWrapper.close(); + sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage()); + } + }, + MoreExecutors.directExecutor()); + } + + private void receiveSessionAccept(final Iq jinglePacket, final RtpContentMap contentMap) { + try { + contentMap.requireContentDescriptions(); + contentMap.requireDTLSFingerprint(); + } catch (final RuntimeException e) { + respondOk(jinglePacket); + LOGGER.debug( + connection.getAccount().address + + ": improperly formatted contents in session-accept", + e); + webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + final Set initiatorMedia = this.initiatorRtpContentMap.getMedia(); + if (!initiatorMedia.equals(contentMap.getMedia())) { + sendSessionTerminate( + Reason.SECURITY_ERROR, + String.format( + "Your session-included included media %s but our session-initiate was" + + " %s", + this.proposedMedia, contentMap.getMedia())); + return; + } + LOGGER.debug("processing session-accept with " + contentMap.contents.size() + " contents"); + if (transition(State.SESSION_ACCEPTED)) { + respondOk(jinglePacket); + receiveSessionAccept(contentMap); + } else { + LOGGER.debug( + String.format( + "%s: received session-accept while in state %s", + connection.getAccount().address, state)); + respondOk(jinglePacket); + } + } + + private void receiveSessionAccept(final RtpContentMap contentMap) { + this.responderRtpContentMap = contentMap; + this.storePeerDtlsSetup(contentMap.getDtlsSetup()); + final SessionDescription sessionDescription; + try { + sessionDescription = SessionDescription.of(contentMap, false); + } catch (final IllegalArgumentException | NullPointerException e) { + LOGGER.debug( + connection.getAccount().address + + ": unable convert offer from session-accept to SDP", + e); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); + return; + } + final org.webrtc.SessionDescription answer = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString()); + try { + this.webRTCWrapper.setRemoteDescription(answer).get(); + } catch (final Exception e) { + LOGGER.debug( + connection.getAccount().address + + ": unable to set remote description after receiving session-accept", + Throwables.getRootCause(e)); + webRTCWrapper.close(); + sendSessionTerminate( + Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage()); + return; + } + processCandidates(contentMap.contents.entrySet()); + } + + private void sendSessionAccept() { + final RtpContentMap rtpContentMap = this.initiatorRtpContentMap; + if (rtpContentMap == null) { + throw new IllegalStateException("initiator RTP Content Map has not been set"); + } + final SessionDescription offer; + try { + offer = SessionDescription.of(rtpContentMap, true); + } catch (final IllegalArgumentException | NullPointerException e) { + LOGGER.debug( + connection.getAccount().address + + ": unable convert offer from session-initiate to SDP", + e); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); + return; + } + sendSessionAccept(rtpContentMap.getMedia(), offer); + } + + private void sendSessionAccept(final Set media, final SessionDescription offer) { + discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers)); + } + + private synchronized void sendSessionAccept( + final Set media, + final SessionDescription offer, + final List iceServers) { + if (isTerminated()) { + LOGGER.warn( + connection.getAccount().address + + ": ICE servers got discovered when session was already terminated." + + " nothing to do."); + return; + } + try { + setupWebRTC(media, iceServers); + } catch (final WebRTCWrapper.InitializationException e) { + LOGGER.debug(connection.getAccount().address + ": unable to initialize WebRTC"); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); + return; + } + final org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.OFFER, offer.toString()); + try { + this.webRTCWrapper.setRemoteDescription(sdp).get(); + addIceCandidatesFromBlackLog(); + org.webrtc.SessionDescription webRTCSessionDescription = + this.webRTCWrapper.setLocalDescription().get(); + prepareSessionAccept(webRTCSessionDescription); + } catch (final Exception e) { + failureToAcceptSession(e); + } + } + + private void failureToAcceptSession(final Throwable throwable) { + if (isTerminated()) { + return; + } + final Throwable rootCause = Throwables.getRootCause(throwable); + LOGGER.debug("unable to send session accept", rootCause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage()); + } + + private void addIceCandidatesFromBlackLog() { + Map.Entry foo; + while ((foo = this.pendingIceCandidates.poll()) != null) { + processCandidate(foo); + LOGGER.debug(connection.getAccount().address + ": added candidate from back log"); + } + } + + private void prepareSessionAccept( + final org.webrtc.SessionDescription webRTCSessionDescription) { + final SessionDescription sessionDescription = + SessionDescription.parse(webRTCSessionDescription.description); + final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false); + this.responderRtpContentMap = respondingRtpContentMap; + storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip()); + final ListenableFuture outgoingContentMapFuture = + prepareOutgoingContentMap(respondingRtpContentMap); + Futures.addCallback( + outgoingContentMapFuture, + new FutureCallback() { + @Override + public void onSuccess(final RtpContentMap outgoingContentMap) { + sendSessionAccept(outgoingContentMap); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } + + @Override + public void onFailure(@NonNull Throwable throwable) { + failureToAcceptSession(throwable); + } + }, + MoreExecutors.directExecutor()); + } + + private void sendSessionAccept(final RtpContentMap rtpContentMap) { + if (isTerminated()) { + LOGGER.warn( + connection.getAccount().address + + ": preparing session accept was too slow. already terminated." + + " nothing to do."); + return; + } + transitionOrThrow(State.SESSION_ACCEPTED); + final JinglePacket sessionAccept = + rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId); + send(sessionAccept); + } + + private ListenableFuture prepareOutgoingContentMap( + final RtpContentMap rtpContentMap) { + if (this.omemoVerification.hasDeviceId()) { + ListenableFuture> + verifiedPayloadFuture = + getManager(AxolotlManager.class) + .encrypt( + rtpContentMap, + id.with, + omemoVerification.getDeviceId()); + return Futures.transform( + verifiedPayloadFuture, + verifiedPayload -> { + omemoVerification.setOrEnsureEqual(verifiedPayload); + return verifiedPayload.getPayload(); + }, + MoreExecutors.directExecutor()); + } else { + return Futures.immediateFuture(rtpContentMap); + } + } + + public synchronized void deliveryMessage( + final Jid from, final JingleMessage message, final String serverMessageId) { + LOGGER.debug( + connection.getAccount().address + + ": delivered message to JingleRtpConnection " + + message); + if (message instanceof Propose) { + receivePropose(from, (Propose) message, serverMessageId); + } else if (message instanceof Proceed) { + receiveProceed(from, (Proceed) message, serverMessageId); + } else if (message instanceof Retract) { + receiveRetract(from, serverMessageId); + } else if (message instanceof Reject) { + receiveReject(from, serverMessageId); + } else if (message instanceof Accept) { + receiveAccept(from, serverMessageId); + } + } + + public void deliverFailedProceed(final String message) { + LOGGER.debug( + connection.getAccount().address + + ": receive message error for proceed message (" + + Strings.nullToEmpty(message) + + ")"); + if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) { + webRTCWrapper.close(); + LOGGER.debug( + connection.getAccount().address + ": transitioned into connectivity error"); + this.finish(); + } + } + + private void receiveAccept(final Jid from, final String serverMsgId) { + final boolean originatedFromMyself = + from.asBareJid().equals(connection.getAccount().address); + if (originatedFromMyself) { + if (transition(State.ACCEPTED)) { + acceptedOnOtherDevice(serverMsgId); + } else { + LOGGER.debug( + connection.getAccount().address + + ": unable to transition to accept because already in state=" + + this.state); + } + } else { + LOGGER.debug(connection.getAccount().address + ": ignoring 'accept' from " + from); + } + } + + private void acceptedOnOtherDevice(final String serverMsgId) { + if (serverMsgId != null) { + this.message.setServerMsgId(serverMsgId); + } + this.message.setCarbon(true); // indicate that call was accepted on other device + this.writeLogMessageSuccess(0); + this.rtpSessionNotification.cancelIncomingCallNotification(); + this.finish(); + } + + private void receiveReject(final Jid from, final String serverMsgId) { + final boolean originatedFromMyself = + from.asBareJid().equals(connection.getAccount().address); + // reject from another one of my clients + if (originatedFromMyself) { + receiveRejectFromMyself(serverMsgId); + } else if (isInitiator()) { + if (from.equals(id.with)) { + receiveRejectFromResponder(); + } else { + LOGGER.debug( + connection.getAccount().address + + ": ignoring reject from " + + from + + " for session with " + + id.with); + } + } else { + LOGGER.debug( + connection.getAccount().address + + ": ignoring reject from " + + from + + " for session with " + + id.with); + } + } + + private void receiveRejectFromMyself(final String serverMsgId) { + if (transition(State.REJECTED)) { + this.rtpSessionNotification.cancelIncomingCallNotification(); + this.finish(); + if (serverMsgId != null) { + this.message.setServerMsgId(serverMsgId); + } + this.message.setCarbon(true); // indicate that call was rejected on other device + writeLogMessageMissed(); + } else { + LOGGER.debug("not able to transition into REJECTED because already in " + this.state); + } + } + + private void receiveRejectFromResponder() { + if (isInState(State.PROCEED)) { + LOGGER.debug( + connection.getAccount().address + + ": received reject while still in proceed. callee reconsidered"); + closeTransitionLogFinish(State.REJECTED_RACED); + return; + } + if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) { + LOGGER.debug( + connection.getAccount().address + + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee" + + " reconsidered before receiving session-init"); + closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY); + return; + } + LOGGER.debug( + connection.getAccount().address + + ": ignoring reject from responder because already in state " + + this.state); + } + + private void receivePropose(final Jid from, final Propose propose, final String serverMsgId) { + final boolean originatedFromMyself = + from.asBareJid().equals(connection.getAccount().address); + if (originatedFromMyself) { + LOGGER.debug(connection.getAccount().address + ": saw proposal from myself. ignoring"); + } else if (transition( + State.PROPOSED, + () -> { + final Collection descriptions = + Collections2.transform( + Collections2.filter( + propose.getDescriptions(), + d -> d instanceof RtpDescription), + input -> (RtpDescription) input); + final Collection media = + Collections2.transform(descriptions, RtpDescription::getMedia); + Preconditions.checkState( + !media.contains(Media.UNKNOWN), + "RTP descriptions contain unknown media"); + LOGGER.debug( + connection.getAccount().address + + ": received session proposal from " + + from + + " for " + + media); + this.proposedMedia = Sets.newHashSet(media); + })) { + if (serverMsgId != null) { + this.message.setServerMsgId(serverMsgId); + } + startRinging(); + } else { + LOGGER.debug( + connection.getAccount().address + + ": ignoring session proposal because already in " + + state); + } + } + + private void startRinging() { + LOGGER.debug( + connection.getAccount().address + + ": received call from " + + id.with + + ". start ringing"); + ringingTimeoutFuture = + getManager(JingleConnectionManager.class) + .schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS); + rtpSessionNotification.startRinging(getAccount(), id, getMedia()); + } + + private synchronized void ringingTimeout() { + LOGGER.debug(connection.getAccount().address + ": timeout reached for ringing"); + switch (this.state) { + case PROPOSED: + message.markUnread(); + rejectCallFromProposed(); + break; + case SESSION_INITIALIZED: + message.markUnread(); + rejectCallFromSessionInitiate(); + break; + } + rtpSessionNotification.pushMissedCallNow(message); + } + + private void cancelRingingTimeout() { + final ScheduledFuture future = this.ringingTimeoutFuture; + if (future != null && !future.isCancelled()) { + future.cancel(false); + } + } + + private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId) { + final Set media = + Preconditions.checkNotNull( + this.proposedMedia, "Proposed media has to be set before handling proceed"); + Preconditions.checkState(media.size() > 0, "Proposed media should not be empty"); + if (from.equals(id.with)) { + if (isInitiator()) { + if (transition(State.PROCEED)) { + if (serverMsgId != null) { + this.message.setServerMsgId(serverMsgId); + } + final Integer remoteDeviceId = proceed.getDeviceId(); + if (isOmemoEnabled()) { + this.omemoVerification.setDeviceId(remoteDeviceId); + } else { + if (remoteDeviceId != null) { + LOGGER.debug( + connection.getAccount().address + + ": remote party signaled support for OMEMO" + + " verification but we have OMEMO disabled"); + } + this.omemoVerification.setDeviceId(null); + } + this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED); + } else { + LOGGER.debug( + String.format( + "%s: ignoring proceed because already in %s", + connection.getAccount().address, this.state)); + } + } else { + LOGGER.debug( + String.format( + "%s: ignoring proceed because we were not initializing", + connection.getAccount().address)); + } + } else if (from.asBareJid().equals(connection.getAccount().address)) { + if (transition(State.ACCEPTED)) { + LOGGER.debug( + connection.getAccount().address + + ": moved session with " + + id.with + + " into state accepted after received carbon copied proceed"); + acceptedOnOtherDevice(serverMsgId); + } + } else { + LOGGER.debug( + String.format( + "%s: ignoring proceed from %s. was expected from %s", + connection.getAccount().address, from, id.with)); + } + } + + private void receiveRetract(final Jid from, final String serverMsgId) { + if (from.equals(id.with)) { + final State target = + this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED; + if (transition(target)) { + rtpSessionNotification.cancelIncomingCallNotification(); + rtpSessionNotification.pushMissedCallNow(message); + LOGGER.debug( + connection.getAccount().address + + ": session with " + + id.with + + " has been retracted (serverMsgId=" + + serverMsgId + + ")"); + if (serverMsgId != null) { + this.message.setServerMsgId(serverMsgId); + } + if (target == State.RETRACTED) { + this.message.markUnread(); + } + writeLogMessageMissed(); + finish(); + } else { + LOGGER.debug("ignoring retract because already in " + this.state); + } + } else { + // TODO parse retract from self + LOGGER.debug( + connection.getAccount().address + + ": received retract from " + + from + + ". expected retract from" + + id.with + + ". ignoring"); + } + } + + public void sendSessionInitiate() { + sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED); + } + + private void sendSessionInitiate(final Set media, final State targetState) { + LOGGER.debug(connection.getAccount().address + ": prepare session-initiate"); + discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers)); + } + + private synchronized void sendSessionInitiate( + final Set media, + final State targetState, + final List iceServers) { + if (isTerminated()) { + LOGGER.warn( + connection.getAccount().address + + ": ICE servers got discovered when session was already terminated." + + " nothing to do."); + return; + } + try { + setupWebRTC(media, iceServers); + } catch (final WebRTCWrapper.InitializationException e) { + LOGGER.debug(connection.getAccount().address + ": unable to initialize WebRTC"); + webRTCWrapper.close(); + sendRetract(Reason.ofThrowable(e)); + return; + } + try { + org.webrtc.SessionDescription webRTCSessionDescription = + this.webRTCWrapper.setLocalDescription().get(); + prepareSessionInitiate(webRTCSessionDescription, targetState); + } catch (final Exception e) { + // TODO sending the error text is worthwhile as well. Especially for FailureToSet + // exceptions + failureToInitiateSession(e, targetState); + } + } + + private void failureToInitiateSession(final Throwable throwable, final State targetState) { + if (isTerminated()) { + return; + } + LOGGER.debug( + connection.getAccount().address + ": unable to sendSessionInitiate", + Throwables.getRootCause(throwable)); + webRTCWrapper.close(); + final Reason reason = Reason.ofThrowable(throwable); + if (isInState(targetState)) { + sendSessionTerminate(reason, throwable.getMessage()); + } else { + sendRetract(reason); + } + } + + private void sendRetract(final Reason reason) { + // TODO embed reason into retract + sendJingleMessage("retract", id.with.asBareJid()); + transitionOrThrow(reasonToState(reason)); + this.finish(); + } + + private void prepareSessionInitiate( + final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) { + final SessionDescription sessionDescription = + SessionDescription.parse(webRTCSessionDescription.description); + final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true); + this.initiatorRtpContentMap = rtpContentMap; + final ListenableFuture outgoingContentMapFuture = + encryptSessionInitiate(rtpContentMap); + Futures.addCallback( + outgoingContentMapFuture, + new FutureCallback() { + @Override + public void onSuccess(final RtpContentMap outgoingContentMap) { + sendSessionInitiate(outgoingContentMap, targetState); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } + + @Override + public void onFailure(@NonNull final Throwable throwable) { + failureToInitiateSession(throwable, targetState); + } + }, + MoreExecutors.directExecutor()); + } + + private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) { + if (isTerminated()) { + LOGGER.warn( + connection.getAccount().address + + ": preparing session was too slow. already terminated. nothing to" + + " do."); + return; + } + this.transitionOrThrow(targetState); + final JinglePacket sessionInitiate = + rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId); + send(sessionInitiate); + } + + private ListenableFuture encryptSessionInitiate( + final RtpContentMap rtpContentMap) { + if (this.omemoVerification.hasDeviceId()) { + final ListenableFuture> + verifiedPayloadFuture = + getManager(AxolotlManager.class) + .encrypt( + rtpContentMap, + id.with, + omemoVerification.getDeviceId()); + final ListenableFuture future = + Futures.transform( + verifiedPayloadFuture, + verifiedPayload -> { + omemoVerification.setSessionFingerprint( + verifiedPayload.getFingerprint()); + return verifiedPayload.getPayload(); + }, + MoreExecutors.directExecutor()); + if (Config.REQUIRE_RTP_VERIFICATION) { + return future; + } + return Futures.catching( + future, + AxolotlEncryptionException.class, + e -> { + LOGGER.warn( + connection.getAccount().address + + ": unable to use OMEMO DTLS verification on outgoing" + + " session initiate. falling back", + e); + return rtpContentMap; + }, + MoreExecutors.directExecutor()); + } else { + return Futures.immediateFuture(rtpContentMap); + } + } + + private void sendSessionTerminate(final Reason reason) { + sendSessionTerminate(reason, null); + } + + private void sendSessionTerminate(final Reason reason, final String text) { + final State previous = this.state; + final State target = reasonToState(reason); + transitionOrThrow(target); + if (previous != State.NULL) { + writeLogMessage(target); + } + final JinglePacket jinglePacket = + new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId); + jinglePacket.setReason(reason, text); + LOGGER.debug(jinglePacket.toString()); + send(jinglePacket); + finish(); + } + + private void sendTransportInfo( + final String contentName, IceUdpTransportInfo.Candidate candidate) { + final RtpContentMap transportInfo; + try { + final RtpContentMap rtpContentMap = + isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; + transportInfo = rtpContentMap.transportInfo(contentName, candidate); + } catch (final Exception e) { + LOGGER.debug( + connection.getAccount().address + + ": unable to prepare transport-info from candidate for content=" + + contentName); + return; + } + final JinglePacket jinglePacket = + transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId); + send(jinglePacket); + } + + private void send(final JinglePacket jinglePacket) { + jinglePacket.setTo(id.with); + connection.sendIqPacket(jinglePacket, this::handleIqResponse); + connection.sendIqPacket(jinglePacket, this::handleIqResponse); + } + + private synchronized void handleIqResponse(final Iq response) { + if (response.getType() == Iq.Type.ERROR) { + handleIqErrorResponse(response); + return; + } + if (response.getType() == Iq.Type.TIMEOUT) { + handleIqTimeoutResponse(response); + } + } + + private void handleIqErrorResponse(final Iq response) { + Preconditions.checkArgument(response.getType() == Iq.Type.ERROR); + final var error = response.getError(); + final var errorCondition = error == null ? null : error.getCondition(); + LOGGER.debug( + connection.getAccount().address + + ": received IQ-error from " + + response.getFrom() + + " in RTP session. " + + errorCondition); + if (isTerminated()) { + LOGGER.info( + connection.getAccount().address + + ": ignoring error because session was already terminated"); + return; + } + this.webRTCWrapper.close(); + final State target; + if (Arrays.asList( + "service-unavailable", + "recipient-unavailable", + "remote-server-not-found", + "remote-server-timeout") + .contains(errorCondition.getName())) { + target = State.TERMINATED_CONNECTIVITY_ERROR; + } else { + target = State.TERMINATED_APPLICATION_FAILURE; + } + transitionOrThrow(target); + this.finish(); + } + + private void handleIqTimeoutResponse(final Iq response) { + Preconditions.checkArgument(response.getType() == Iq.Type.TIMEOUT); + LOGGER.debug( + connection.getAccount().address + + ": received IQ timeout in RTP session with " + + id.with + + ". terminating with connectivity error"); + if (isTerminated()) { + LOGGER.info( + connection.getAccount().address + + ": ignoring error because session was already terminated"); + return; + } + this.webRTCWrapper.close(); + transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR); + this.finish(); + } + + private void terminateWithOutOfOrder(final Iq jinglePacket) { + LOGGER.debug(connection.getAccount().address + ": terminating session with out-of-order"); + this.webRTCWrapper.close(); + transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE); + respondWithOutOfOrder(jinglePacket); + this.finish(); + } + + private void respondWithTieBreak(final Iq jinglePacket) { + respondWithJingleError( + jinglePacket, "tie-break", Error.Type.CANCEL, new Condition.Conflict()); + } + + private void respondWithOutOfOrder(final Iq jinglePacket) { + respondWithJingleError( + jinglePacket, "out-of-order", Error.Type.WAIT, new Condition.UnexpectedRequest()); + } + + private void respondWithJingleError( + final Iq original, String jingleCondition, final Error.Type type, Condition condition) { + // TODO add jingle condition + connection.sendErrorFor(original, type, condition); + } + + private void respondOk(final Iq jinglePacket) { + connection.sendResultFor(jinglePacket); + } + + public RtpEndUserState getEndUserState() { + switch (this.state) { + case NULL: + case PROPOSED: + case SESSION_INITIALIZED: + if (isInitiator()) { + return RtpEndUserState.RINGING; + } else { + return RtpEndUserState.INCOMING_CALL; + } + case PROCEED: + if (isInitiator()) { + return RtpEndUserState.RINGING; + } else { + return RtpEndUserState.ACCEPTING_CALL; + } + case SESSION_INITIALIZED_PRE_APPROVED: + if (isInitiator()) { + return RtpEndUserState.RINGING; + } else { + return RtpEndUserState.CONNECTING; + } + case SESSION_ACCEPTED: + final ContentAddition ca = getPendingContentAddition(); + if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) { + return RtpEndUserState.INCOMING_CONTENT_ADD; + } + return getPeerConnectionStateAsEndUserState(); + case REJECTED: + case REJECTED_RACED: + case TERMINATED_DECLINED_OR_BUSY: + if (isInitiator()) { + return RtpEndUserState.DECLINED_OR_BUSY; + } else { + return RtpEndUserState.ENDED; + } + case TERMINATED_SUCCESS: + case ACCEPTED: + case RETRACTED: + case TERMINATED_CANCEL_OR_TIMEOUT: + return RtpEndUserState.ENDED; + case RETRACTED_RACED: + if (isInitiator()) { + return RtpEndUserState.ENDED; + } else { + return RtpEndUserState.RETRACTED; + } + case TERMINATED_CONNECTIVITY_ERROR: + return zeroDuration() + ? RtpEndUserState.CONNECTIVITY_ERROR + : RtpEndUserState.CONNECTIVITY_LOST_ERROR; + case TERMINATED_APPLICATION_FAILURE: + return RtpEndUserState.APPLICATION_ERROR; + case TERMINATED_SECURITY_ERROR: + return RtpEndUserState.SECURITY_ERROR; + } + throw new IllegalStateException( + String.format("%s has no equivalent EndUserState", this.state)); + } + + private RtpEndUserState getPeerConnectionStateAsEndUserState() { + final PeerConnection.PeerConnectionState state; + try { + state = webRTCWrapper.getState(); + } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { + // We usually close the WebRTCWrapper *before* transitioning so we might still + // be in SESSION_ACCEPTED even though the peerConnection has been torn down + return RtpEndUserState.ENDING_CALL; + } + switch (state) { + case CONNECTED: + return RtpEndUserState.CONNECTED; + case NEW: + case CONNECTING: + return RtpEndUserState.CONNECTING; + case CLOSED: + return RtpEndUserState.ENDING_CALL; + default: + return zeroDuration() + ? RtpEndUserState.CONNECTIVITY_ERROR + : RtpEndUserState.RECONNECTING; + } + } + + public ContentAddition getPendingContentAddition() { + final RtpContentMap in = this.incomingContentAdd; + final RtpContentMap out = this.outgoingContentAdd; + if (out != null) { + return ContentAddition.of(ContentAddition.Direction.OUTGOING, out); + } else if (in != null) { + return ContentAddition.of(ContentAddition.Direction.INCOMING, in); + } else { + return null; + } + } + + public Set getMedia() { + final State current = getState(); + if (current == State.NULL) { + if (isInitiator()) { + return Preconditions.checkNotNull( + this.proposedMedia, "RTP connection has not been initialized properly"); + } + throw new IllegalStateException("RTP connection has not been initialized yet"); + } + if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) { + return Preconditions.checkNotNull( + this.proposedMedia, "RTP connection has not been initialized properly"); + } + final RtpContentMap localContentMap = getLocalContentMap(); + final RtpContentMap initiatorContentMap = initiatorRtpContentMap; + if (localContentMap != null) { + return localContentMap.getMedia(); + } else if (initiatorContentMap != null) { + return initiatorContentMap.getMedia(); + } else if (isTerminated()) { + return Collections.emptySet(); // we might fail before we ever got a chance to set media + } else { + return Preconditions.checkNotNull( + this.proposedMedia, "RTP connection has not been initialized properly"); + } + } + + public boolean isVerified() { + final IdentityKey fingerprint = this.omemoVerification.getFingerprint(); + if (fingerprint == null) { + return false; + } + // TODO look up fingerprint trust status; + return false; + } + + public boolean addMedia(final Media media) { + final Set currentMedia = getMedia(); + if (currentMedia.contains(media)) { + throw new IllegalStateException(String.format("%s has already been proposed", media)); + } + // TODO add state protection - can only add while ACCEPTED or so + LOGGER.debug("adding media: " + media); + return webRTCWrapper.addTrack(media); + } + + public synchronized void acceptCall() { + switch (this.state) { + case PROPOSED: + cancelRingingTimeout(); + acceptCallFromProposed(); + break; + case SESSION_INITIALIZED: + cancelRingingTimeout(); + acceptCallFromSessionInitialized(); + break; + case ACCEPTED: + LOGGER.warn( + connection.getAccount().address + + ": the call has already been accepted with another client. UI" + + " was just lagging behind"); + break; + case PROCEED: + case SESSION_ACCEPTED: + LOGGER.warn( + connection.getAccount().address + + ": the call has already been accepted. user probably double" + + " tapped the UI"); + break; + default: + throw new IllegalStateException("Can not accept call from " + this.state); + } + } + + public void notifyPhoneCall() { + LOGGER.debug("a phone call has just been started. killing jingle rtp connections"); + if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) { + rejectCall(); + } else { + endCall(); + } + } + + public synchronized void rejectCall() { + if (isTerminated()) { + LOGGER.warn( + connection.getAccount().address + + ": received rejectCall() when session has already been terminated." + + " nothing to do"); + return; + } + switch (this.state) { + case PROPOSED: + rejectCallFromProposed(); + break; + case SESSION_INITIALIZED: + rejectCallFromSessionInitiate(); + break; + default: + throw new IllegalStateException("Can not reject call from " + this.state); + } + } + + public synchronized void endCall() { + if (isTerminated()) { + LOGGER.warn( + connection.getAccount().address + + ": received endCall() when session has already been terminated." + + " nothing to do"); + return; + } + if (isInState(State.PROPOSED) && !isInitiator()) { + rejectCallFromProposed(); + return; + } + if (isInState(State.PROCEED)) { + if (isInitiator()) { + retractFromProceed(); + } else { + rejectCallFromProceed(); + } + return; + } + if (isInitiator() + && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) { + this.webRTCWrapper.close(); + sendSessionTerminate(Reason.CANCEL); + return; + } + if (isInState(State.SESSION_INITIALIZED)) { + rejectCallFromSessionInitiate(); + return; + } + if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) { + this.webRTCWrapper.close(); + sendSessionTerminate(Reason.SUCCESS); + return; + } + if (isInState( + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_CONNECTIVITY_ERROR, + State.TERMINATED_DECLINED_OR_BUSY)) { + LOGGER.debug("ignoring request to end call because already in state " + this.state); + return; + } + throw new IllegalStateException( + "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator()); + } + + private void retractFromProceed() { + LOGGER.debug("retract from proceed"); + this.sendJingleMessage("retract"); + closeTransitionLogFinish(State.RETRACTED_RACED); + } + + private void closeTransitionLogFinish(final State state) { + this.webRTCWrapper.close(); + transitionOrThrow(state); + writeLogMessage(state); + finish(); + } + + private void setupWebRTC( + final Set media, final List iceServers) + throws WebRTCWrapper.InitializationException { + getManager(JingleConnectionManager.class).ensureConnectionIsRegistered(this); + this.webRTCWrapper.setup(this.context, AppRTCAudioManager.SpeakerPhonePreference.of(media)); + this.webRTCWrapper.initializePeerConnection(media, iceServers); + } + + private void acceptCallFromProposed() { + transitionOrThrow(State.PROCEED); + rtpSessionNotification.cancelIncomingCallNotification(); + this.sendJingleMessage("accept", connection.getAccount().address); + this.sendJingleMessage("proceed"); + } + + private void rejectCallFromProposed() { + transitionOrThrow(State.REJECTED); + writeLogMessageMissed(); + rtpSessionNotification.cancelIncomingCallNotification(); + this.sendJingleMessage("reject"); + finish(); + } + + private void rejectCallFromProceed() { + this.sendJingleMessage("reject"); + closeTransitionLogFinish(State.REJECTED_RACED); + } + + private void rejectCallFromSessionInitiate() { + webRTCWrapper.close(); + sendSessionTerminate(Reason.DECLINE); + rtpSessionNotification.cancelIncomingCallNotification(); + } + + private void sendJingleMessage(final String action) { + sendJingleMessage(action, id.with); + } + + private void sendJingleMessage(final String action, final Jid to) { + final Message messagePacket = new Message(); + messagePacket.setType(Message.Type.CHAT); // we want to carbon copy those + messagePacket.setTo(to); + final Element intent = + messagePacket + .addChild(action, Namespace.JINGLE_MESSAGE) + .setAttribute("id", id.sessionId); + if ("proceed".equals(action)) { + messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId); + if (isOmemoEnabled()) { + final int deviceId = getAccount().getPublicDeviceIdInt(); + final Element device = + intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION); + device.setAttribute("id", deviceId); + } + } + messagePacket.addChild("store", "urn:xmpp:hints"); + connection.sendMessagePacket(messagePacket); + } + + private boolean isOmemoEnabled() { + // TODO look up if omemo is enabled for this chat + return false; + } + + private void acceptCallFromSessionInitialized() { + rtpSessionNotification.cancelIncomingCallNotification(); + sendSessionAccept(); + } + + private synchronized boolean isInState(State... state) { + return Arrays.asList(state).contains(this.state); + } + + private boolean transition(final State target) { + return transition(target, null); + } + + private synchronized boolean transition(final State target, final Runnable runnable) { + final Collection validTransitions = VALID_TRANSITIONS.get(this.state); + if (validTransitions != null && validTransitions.contains(target)) { + this.state = target; + if (runnable != null) { + runnable.run(); + } + LOGGER.debug(connection.getAccount().address + ": transitioned into " + target); + updateEndUserState(); + updateOngoingCallNotification(); + return true; + } else { + return false; + } + } + + public void transitionOrThrow(final State target) { + if (!transition(target)) { + throw new IllegalStateException( + String.format("Unable to transition from %s to %s", this.state, target)); + } + } + + @Override + public void onIceCandidate(final IceCandidate iceCandidate) { + final RtpContentMap rtpContentMap = + isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; + final IceUdpTransportInfo.Credentials credentials; + try { + credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid); + } catch (final IllegalArgumentException e) { + LOGGER.debug("ignoring (not sending) candidate: " + iceCandidate, e); + return; + } + final String uFrag = credentials.ufrag; + final IceUdpTransportInfo.Candidate candidate = + IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag); + if (candidate == null) { + LOGGER.debug("ignoring (not sending) candidate: " + iceCandidate); + return; + } + LOGGER.debug("sending candidate: " + iceCandidate); + sendTransportInfo(iceCandidate.sdpMid, candidate); + } + + @Override + public void onConnectionChange(final PeerConnection.PeerConnectionState newState) { + LOGGER.debug( + connection.getAccount().address + ": PeerConnectionState changed to " + newState); + this.stateHistory.add(newState); + if (newState == PeerConnection.PeerConnectionState.CONNECTED) { + this.sessionDuration.start(); + updateOngoingCallNotification(); + } else if (this.sessionDuration.isRunning()) { + this.sessionDuration.stop(); + updateOngoingCallNotification(); + } + + final boolean neverConnected = + !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED); + + if (newState == PeerConnection.PeerConnectionState.FAILED) { + if (neverConnected) { + if (isTerminated()) { + LOGGER.debug( + connection.getAccount().address + + ": not sending session-terminate after connectivity error" + + " because session is already in state " + + this.state); + return; + } + webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection); + return; + } else { + this.restartIce(); + } + } + updateEndUserState(); + } + + private void restartIce() { + this.stateHistory.clear(); + this.webRTCWrapper.restartIce(); + } + + @Override + public void onRenegotiationNeeded() { + this.webRTCWrapper.execute(this::renegotiate); + } + + private void renegotiate() { + final SessionDescription sessionDescription; + try { + sessionDescription = setLocalSessionDescription(); + } catch (final Exception e) { + final Throwable cause = Throwables.getRootCause(e); + LOGGER.debug("failed to renegotiate", cause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); + return; + } + final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator()); + final RtpContentMap currentContentMap = getLocalContentMap(); + final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap); + final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap); + + LOGGER.debug( + connection.getAccount().address + + ": renegotiate. iceRestart=" + + iceRestart + + " content id diff=" + + diff); + + if (diff.hasModifications() && iceRestart) { + webRTCWrapper.close(); + sendSessionTerminate( + Reason.FAILED_APPLICATION, + "WebRTC unexpectedly tried to modify content and transport at once"); + return; + } + + if (iceRestart) { + initiateIceRestart(rtpContentMap); + return; + } else if (diff.isEmpty()) { + LOGGER.debug( + "renegotiation. nothing to do. SignalingState=" + + this.webRTCWrapper.getSignalingState()); + } + + if (diff.added.size() > 0) { + modifyLocalContentMap(rtpContentMap); + sendContentAdd(rtpContentMap, diff.added); + } + } + + private void initiateIceRestart(final RtpContentMap rtpContentMap) { + final RtpContentMap transportInfo = rtpContentMap.transportInfo(); + final Iq jinglePacket = + transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId); + LOGGER.debug("initiating ice restart: " + jinglePacket); + jinglePacket.setTo(id.with); + connection.sendIqPacket( + jinglePacket, + (response) -> { + if (response.getType() == Iq.Type.RESULT) { + LOGGER.debug("received success to our ice restart"); + setLocalContentMap(rtpContentMap); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + return; + } + if (response.getType() == Iq.Type.ERROR) { + if (isTieBreak(response)) { + LOGGER.debug("received tie-break as result of ice restart"); + return; + } + handleIqErrorResponse(response); + } + if (response.getType() == Iq.Type.TIMEOUT) { + handleIqTimeoutResponse(response); + } + }); + } + + private boolean isTieBreak(final Iq response) { + final Element error = response.findChild("error"); + return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS); + } + + private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection added) { + final RtpContentMap contentAdd = rtpContentMap.toContentModification(added); + this.outgoingContentAdd = contentAdd; + final Iq jinglePacket = + contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId); + jinglePacket.setTo(id.with); + connection.sendIqPacket( + jinglePacket, + (response) -> { + if (response.getType() == Iq.Type.RESULT) { + LOGGER.debug( + connection.getAccount().address + + ": received ACK to our content-add"); + return; + } + if (response.getType() == Iq.Type.ERROR) { + if (isTieBreak(response)) { + this.outgoingContentAdd = null; + LOGGER.debug("received tie-break as result of our content-add"); + return; + } + handleIqErrorResponse(response); + } + if (response.getType() == Iq.Type.TIMEOUT) { + handleIqTimeoutResponse(response); + } + }); + } + + private void setLocalContentMap(final RtpContentMap rtpContentMap) { + if (isInitiator()) { + this.initiatorRtpContentMap = rtpContentMap; + } else { + this.responderRtpContentMap = rtpContentMap; + } + } + + private void setRemoteContentMap(final RtpContentMap rtpContentMap) { + if (isInitiator()) { + this.responderRtpContentMap = rtpContentMap; + } else { + this.initiatorRtpContentMap = rtpContentMap; + } + } + + // this method is to be used for content map modifications that modify media + private void modifyLocalContentMap(final RtpContentMap rtpContentMap) { + final RtpContentMap activeContents = rtpContentMap.activeContents(); + setLocalContentMap(activeContents); + this.webRTCWrapper.switchSpeakerPhonePreference( + AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia())); + updateEndUserState(); + } + + private SessionDescription setLocalSessionDescription() + throws ExecutionException, InterruptedException { + final org.webrtc.SessionDescription sessionDescription = + this.webRTCWrapper.setLocalDescription().get(); + return SessionDescription.parse(sessionDescription.description); + } + + private void closeWebRTCSessionAfterFailedConnection() { + this.webRTCWrapper.close(); + synchronized (this) { + if (isTerminated()) { + LOGGER.debug( + connection.getAccount().address + + ": no need to send session-terminate after failed connection." + + " Other party already did"); + return; + } + sendSessionTerminate(Reason.CONNECTIVITY_ERROR); + } + } + + public boolean zeroDuration() { + return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0; + } + + public long getCallDuration() { + return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS); + } + + public AppRTCAudioManager getAudioManager() { + return webRTCWrapper.getAudioManager(); + } + + public boolean isMicrophoneEnabled() { + return webRTCWrapper.isMicrophoneEnabled(); + } + + public boolean setMicrophoneEnabled(final boolean enabled) { + return webRTCWrapper.setMicrophoneEnabled(enabled); + } + + public boolean isVideoEnabled() { + return webRTCWrapper.isVideoEnabled(); + } + + public void setVideoEnabled(final boolean enabled) { + webRTCWrapper.setVideoEnabled(enabled); + } + + public boolean isCameraSwitchable() { + return webRTCWrapper.isCameraSwitchable(); + } + + public boolean isFrontCamera() { + return webRTCWrapper.isFrontCamera(); + } + + public ListenableFuture switchCamera() { + return webRTCWrapper.switchCamera(); + } + + @Override + public void onAudioDeviceChanged( + AppRTCAudioManager.AudioDevice selectedAudioDevice, + Set availableAudioDevices) { + getManager(JingleConnectionManager.class) + .notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices); + } + + private void updateEndUserState() { + final RtpEndUserState endUserState = getEndUserState(); + ToneManager.getInstance(context).transition(isInitiator(), endUserState, getMedia()); + getManager(JingleConnectionManager.class) + .notifyJingleRtpConnectionUpdate(id.with, id.sessionId, endUserState); + } + + private void updateOngoingCallNotification() { + final State state = this.state; + if (STATES_SHOWING_ONGOING_CALL.contains(state)) { + final boolean reconnecting; + if (state == State.SESSION_ACCEPTED) { + reconnecting = + getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING; + } else { + reconnecting = false; + } + + // TODO decide what we want to do with ongoing call? create a foreground service of + // RtpSessionService? + + // xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting); + } else { + // xmppConnectionService.removeOngoingCall(); + } + } + + private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) { + final var externalServicesFuture = getManager(ExternalDiscoManager.class).getServices(); + Futures.addCallback( + externalServicesFuture, + new FutureCallback<>() { + @Override + public void onSuccess(final Collection services) { + final ImmutableList.Builder listBuilder = + ImmutableList.builder(); + + for (final Service service : services) { + final var optionalIceServer = toIceServer(service); + if (optionalIceServer.isPresent()) { + final var iceServer = optionalIceServer.get(); + LOGGER.debug("discovered ICE Server: {}", iceServer); + listBuilder.add(iceServer); + } + } + + final var iceServers = listBuilder.build(); + if (iceServers.size() == 0) { + LOGGER.warn("No ICE server discovered"); + } + onIceServersDiscovered.onIceServersDiscovered(iceServers); + } + + @Override + public void onFailure(@NonNull final Throwable throwable) { + LOGGER.warn("External services discovery failed", throwable); + onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList()); + } + }, + MoreExecutors.directExecutor()); + } + + private static Optional toIceServer(final Service service) { + final String type = service.getAttribute("type"); + final String host = service.getAttribute("host"); + final Optional portOptional = service.getOptionalIntAttribute("port"); + final String transport = service.getAttribute("transport"); + final String username = service.getAttribute("username"); + final String password = service.getAttribute("password"); + if (Strings.isNullOrEmpty(host) || !portOptional.isPresent()) { + return Optional.absent(); + } + final int port = portOptional.get(); + if (port < 0 || port > 65535) { + return Optional.absent(); + } + if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) + && Arrays.asList("udp", "tcp").contains(transport)) { + if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) { + LOGGER.debug("skipping invalid combination of udp/tls in external services"); + return Optional.absent(); + } + // TODO Starting on milestone 110, Chromium will perform + // stricter validation of TURN and STUN URLs passed to the + // constructor of an RTCPeerConnection. More specifically, + // STUN URLs will not support a query section, and TURN URLs + // will support only a transport parameter in their query + // section. + final PeerConnection.IceServer.Builder iceServerBuilder = + PeerConnection.IceServer.builder( + String.format( + "%s:%s:%s?transport=%s", + type, IP.wrapIPv6(host), port, transport)); + iceServerBuilder.setTlsCertPolicy( + PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK); + if (username != null && password != null) { + iceServerBuilder.setUsername(username); + iceServerBuilder.setPassword(password); + } else if (Arrays.asList("turn", "turns").contains(type)) { + // The WebRTC spec requires throwing an + // InvalidAccessError when username (from libwebrtc + // source coder) + // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc + LOGGER.debug("skipping {}/{} without username and password", type, transport); + return Optional.absent(); + } + return Optional.of(iceServerBuilder.createIceServer()); + } else { + return Optional.absent(); + } + } + + private void finish() { + if (isTerminated()) { + this.cancelRingingTimeout(); + this.webRTCWrapper.verifyClosed(); + getManager(JingleConnectionManager.class) + .setTerminalSessionState(id, getEndUserState(), getMedia()); + getManager(JingleConnectionManager.class).finishConnectionOrThrow(this); + } else { + throw new IllegalStateException( + String.format("Unable to call finish from %s", this.state)); + } + } + + private void writeLogMessage(final State state) { + final long duration = getCallDuration(); + if (state == State.TERMINATED_SUCCESS + || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) { + writeLogMessageSuccess(duration); + } else { + writeLogMessageMissed(); + } + } + + private void writeLogMessageSuccess(final long duration) { + this.message.setDuration(duration); + // this.message.setBody(new RtpSessionStatus(true, duration).toString()); + this.writeMessage(); + } + + private void writeLogMessageMissed() { + // this.message.setBody(new RtpSessionStatus(false, 0).toString()); + this.writeMessage(); + } + + private void writeMessage() { + // TODO write CallLogEntry to DB + } + + public State getState() { + return this.state; + } + + public boolean isTerminated() { + return TERMINATED.contains(this.state); + } + + public Optional getLocalVideoTrack() { + return webRTCWrapper.getLocalVideoTrack(); + } + + public Optional getRemoteVideoTrack() { + return webRTCWrapper.getRemoteVideoTrack(); + } + + public EglBase.Context getEglBaseContext() { + return webRTCWrapper.getEglBaseContext(); + } + + public void setProposedMedia(final Set media) { + this.proposedMedia = media; + } + + public void fireStateUpdate() { + final RtpEndUserState endUserState = getEndUserState(); + getManager(JingleConnectionManager.class) + .notifyJingleRtpConnectionUpdate(id.with, id.sessionId, endUserState); + } + + public boolean isSwitchToVideoAvailable() { + final boolean prerequisite = + Media.audioOnly(getMedia()) + && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING) + .contains(getEndUserState()); + return prerequisite && remoteHasVideoFeature(); + } + + private boolean remoteHasVideoFeature() { + return getManager(DiscoManager.class) + .hasFeature(Entity.presence(id.with), Namespace.JINGLE_FEATURE_VIDEO); + } + + private interface OnIceServersDiscovered { + void onIceServersDiscovered(List iceServers); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java new file mode 100644 index 000000000..036b5bf88 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java @@ -0,0 +1,34 @@ +package eu.siacs.conversations.xmpp.jingle; + +import com.google.common.collect.ImmutableSet; +import java.util.Locale; +import java.util.Set; +import javax.annotation.Nonnull; + +public enum Media { + VIDEO, + AUDIO, + UNKNOWN; + + @Override + @Nonnull + public String toString() { + return super.toString().toLowerCase(Locale.ROOT); + } + + public static Media of(String value) { + try { + return value == null ? UNKNOWN : Media.valueOf(value.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + return UNKNOWN; + } + } + + public static boolean audioOnly(Set media) { + return ImmutableSet.of(AUDIO).equals(media); + } + + public static boolean videoOnly(Set media) { + return ImmutableSet.of(VIDEO).equals(media); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/MediaBuilder.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/MediaBuilder.java new file mode 100644 index 000000000..a1614bb06 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/MediaBuilder.java @@ -0,0 +1,48 @@ +package eu.siacs.conversations.xmpp.jingle; + +import com.google.common.collect.ArrayListMultimap; +import java.util.List; + +public class MediaBuilder { + private String media; + private int port; + private String protocol; + private List formats; + private String connectionData; + private ArrayListMultimap attributes; + + public MediaBuilder setMedia(String media) { + this.media = media; + return this; + } + + public MediaBuilder setPort(int port) { + this.port = port; + return this; + } + + public MediaBuilder setProtocol(String protocol) { + this.protocol = protocol; + return this; + } + + public MediaBuilder setFormats(List formats) { + this.formats = formats; + return this; + } + + public MediaBuilder setConnectionData(String connectionData) { + this.connectionData = connectionData; + return this; + } + + public MediaBuilder setAttributes(ArrayListMultimap attributes) { + this.attributes = attributes; + return this; + } + + public SessionDescription.Media createMedia() { + return new SessionDescription.Media( + media, port, protocol, formats, connectionData, attributes); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OmemoVerification.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OmemoVerification.java new file mode 100644 index 000000000..3e8f77775 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OmemoVerification.java @@ -0,0 +1,83 @@ +package eu.siacs.conversations.xmpp.jingle; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; +import im.conversations.android.axolotl.AxolotlService; +import java.util.concurrent.atomic.AtomicBoolean; +import org.whispersystems.libsignal.IdentityKey; + +public class OmemoVerification { + + private final AtomicBoolean deviceIdWritten = new AtomicBoolean(false); + private final AtomicBoolean identityKeyWritten = new AtomicBoolean(false); + private Integer deviceId; + private IdentityKey identityKey; + + public void setDeviceId(final Integer id) { + if (deviceIdWritten.compareAndSet(false, true)) { + this.deviceId = id; + return; + } + throw new IllegalStateException("Device Id has already been set"); + } + + public int getDeviceId() { + Preconditions.checkNotNull(this.deviceId, "Device ID is null"); + return this.deviceId; + } + + public boolean hasDeviceId() { + return this.deviceId != null; + } + + public void setSessionFingerprint(final IdentityKey identityKey) { + Preconditions.checkNotNull(identityKey, "IdentityKey must not be null"); + if (identityKeyWritten.compareAndSet(false, true)) { + this.identityKey = identityKey; + return; + } + throw new IllegalStateException("Identity Key has already been set"); + } + + public IdentityKey getFingerprint() { + return this.identityKey; + } + + public void setOrEnsureEqual(AxolotlService.OmemoVerifiedPayload omemoVerifiedPayload) { + setOrEnsureEqual(omemoVerifiedPayload.getDeviceId(), omemoVerifiedPayload.getFingerprint()); + } + + public void setOrEnsureEqual(final int deviceId, final IdentityKey identityKey) { + Preconditions.checkNotNull(identityKey, "IdentityKey must not be null"); + if (this.deviceIdWritten.get() || this.identityKeyWritten.get()) { + if (this.identityKey == null) { + throw new IllegalStateException( + "No session fingerprint has been previously provided"); + } + if (!identityKey.equals(this.identityKey)) { + throw new SecurityException("IdentityKeys did not match"); + } + if (this.deviceId == null) { + throw new IllegalStateException("No Device Id has been previously provided"); + } + if (this.deviceId != deviceId) { + throw new IllegalStateException("Device Ids did not match"); + } + } else { + this.setSessionFingerprint(identityKey); + this.setDeviceId(deviceId); + } + } + + public boolean hasFingerprint() { + return this.identityKey != null; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("deviceId", deviceId) + .add("fingerprint", identityKey) + .toString(); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OmemoVerifiedRtpContentMap.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OmemoVerifiedRtpContentMap.java new file mode 100644 index 000000000..827dc97b4 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OmemoVerifiedRtpContentMap.java @@ -0,0 +1,20 @@ +package eu.siacs.conversations.xmpp.jingle; + +import eu.siacs.conversations.xmpp.jingle.stanzas.Group; +import eu.siacs.conversations.xmpp.jingle.stanzas.OmemoVerifiedIceUdpTransportInfo; +import java.util.Map; + +public class OmemoVerifiedRtpContentMap extends RtpContentMap { + public OmemoVerifiedRtpContentMap(Group group, Map contents) { + super(group, contents); + for (final DescriptionTransport descriptionTransport : contents.values()) { + if (descriptionTransport.transport instanceof OmemoVerifiedIceUdpTransportInfo) { + ((OmemoVerifiedIceUdpTransportInfo) descriptionTransport.transport) + .ensureNoPlaintextFingerprint(); + continue; + } + throw new IllegalStateException( + "OmemoVerifiedRtpContentMap contains non-verified transport info"); + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OnPrimaryCandidateFound.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OnPrimaryCandidateFound.java new file mode 100644 index 000000000..16c61444c --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OnPrimaryCandidateFound.java @@ -0,0 +1,5 @@ +package eu.siacs.conversations.xmpp.jingle; + +public interface OnPrimaryCandidateFound { + void onPrimaryCandidateFound(boolean success, JingleCandidate canditate); +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OnTransportConnected.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OnTransportConnected.java new file mode 100644 index 000000000..8352b8c00 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OnTransportConnected.java @@ -0,0 +1,7 @@ +package eu.siacs.conversations.xmpp.jingle; + +public interface OnTransportConnected { + void failed(); + + void established(); +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OngoingRtpSession.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OngoingRtpSession.java new file mode 100644 index 000000000..edf4064af --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/OngoingRtpSession.java @@ -0,0 +1,9 @@ +package eu.siacs.conversations.xmpp.jingle; + +import org.jxmpp.jid.Jid; + +public interface OngoingRtpSession { + Jid getWith(); + + String getSessionId(); +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java new file mode 100644 index 000000000..0fca5c0cb --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java @@ -0,0 +1,489 @@ +package eu.siacs.conversations.xmpp.jingle; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.base.Strings; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import eu.siacs.conversations.xmpp.jingle.stanzas.Content; +import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription; +import eu.siacs.conversations.xmpp.jingle.stanzas.GenericTransportInfo; +import eu.siacs.conversations.xmpp.jingle.stanzas.Group; +import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; +import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; +import eu.siacs.conversations.xmpp.jingle.stanzas.OmemoVerifiedIceUdpTransportInfo; +import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; + +public class RtpContentMap { + + public final Group group; + public final Map contents; + + public RtpContentMap(Group group, Map contents) { + this.group = group; + this.contents = contents; + } + + public static RtpContentMap of(final JinglePacket jinglePacket) { + final Map contents = + DescriptionTransport.of(jinglePacket.getJingleContents()); + if (isOmemoVerified(contents)) { + return new OmemoVerifiedRtpContentMap(jinglePacket.getGroup(), contents); + } else { + return new RtpContentMap(jinglePacket.getGroup(), contents); + } + } + + private static boolean isOmemoVerified(Map contents) { + final Collection values = contents.values(); + if (values.size() == 0) { + return false; + } + for (final DescriptionTransport descriptionTransport : values) { + if (descriptionTransport.transport instanceof OmemoVerifiedIceUdpTransportInfo) { + continue; + } + return false; + } + return true; + } + + public static RtpContentMap of( + final SessionDescription sessionDescription, final boolean isInitiator) { + final ImmutableMap.Builder contentMapBuilder = + new ImmutableMap.Builder<>(); + for (SessionDescription.Media media : sessionDescription.media) { + final String id = Iterables.getFirst(media.attributes.get("mid"), null); + Preconditions.checkNotNull(id, "media has no mid"); + contentMapBuilder.put( + id, DescriptionTransport.of(sessionDescription, isInitiator, media)); + } + final String groupAttribute = + Iterables.getFirst(sessionDescription.attributes.get("group"), null); + final Group group = groupAttribute == null ? null : Group.ofSdpString(groupAttribute); + return new RtpContentMap(group, contentMapBuilder.build()); + } + + public Set getMedia() { + return Sets.newHashSet( + Collections2.transform( + contents.values(), + input -> { + final RtpDescription rtpDescription = + input == null ? null : input.description; + return rtpDescription == null + ? Media.UNKNOWN + : input.description.getMedia(); + })); + } + + public Set getSenders() { + return ImmutableSet.copyOf(Collections2.transform(contents.values(), dt -> dt.senders)); + } + + public List getNames() { + return ImmutableList.copyOf(contents.keySet()); + } + + void requireContentDescriptions() { + if (this.contents.size() == 0) { + throw new IllegalStateException("No contents available"); + } + for (Map.Entry entry : this.contents.entrySet()) { + if (entry.getValue().description == null) { + throw new IllegalStateException( + String.format("%s is lacking content description", entry.getKey())); + } + } + } + + void requireDTLSFingerprint() { + requireDTLSFingerprint(false); + } + + void requireDTLSFingerprint(final boolean requireActPass) { + if (this.contents.size() == 0) { + throw new IllegalStateException("No contents available"); + } + for (Map.Entry entry : this.contents.entrySet()) { + final IceUdpTransportInfo transport = entry.getValue().transport; + final IceUdpTransportInfo.Fingerprint fingerprint = transport.getFingerprint(); + if (fingerprint == null + || Strings.isNullOrEmpty(fingerprint.getContent()) + || Strings.isNullOrEmpty(fingerprint.getHash())) { + throw new SecurityException( + String.format( + "Use of DTLS-SRTP (XEP-0320) is required for content %s", + entry.getKey())); + } + final IceUdpTransportInfo.Setup setup = fingerprint.getSetup(); + if (setup == null) { + throw new SecurityException( + String.format( + "Use of DTLS-SRTP (XEP-0320) is required for content %s but" + + " missing setup attribute", + entry.getKey())); + } + if (requireActPass && setup != IceUdpTransportInfo.Setup.ACTPASS) { + throw new SecurityException( + "Initiator needs to offer ACTPASS as setup for DTLS-SRTP (XEP-0320)"); + } + } + } + + JinglePacket toJinglePacket(final JinglePacket.Action action, final String sessionId) { + final JinglePacket jinglePacket = new JinglePacket(action, sessionId); + if (this.group != null) { + jinglePacket.addGroup(this.group); + } + for (Map.Entry entry : this.contents.entrySet()) { + final DescriptionTransport descriptionTransport = entry.getValue(); + final Content content = + new Content( + Content.Creator.INITIATOR, + descriptionTransport.senders, + entry.getKey()); + if (descriptionTransport.description != null) { + content.addChild(descriptionTransport.description); + } + content.addChild(descriptionTransport.transport); + jinglePacket.addJingleContent(content); + } + return jinglePacket; + } + + RtpContentMap transportInfo( + final String contentName, final IceUdpTransportInfo.Candidate candidate) { + final RtpContentMap.DescriptionTransport descriptionTransport = contents.get(contentName); + final IceUdpTransportInfo transportInfo = + descriptionTransport == null ? null : descriptionTransport.transport; + if (transportInfo == null) { + throw new IllegalArgumentException( + "Unable to find transport info for content name " + contentName); + } + final IceUdpTransportInfo newTransportInfo = transportInfo.cloneWrapper(); + newTransportInfo.addChild(candidate); + return new RtpContentMap( + null, + ImmutableMap.of( + contentName, + new DescriptionTransport( + descriptionTransport.senders, null, newTransportInfo))); + } + + RtpContentMap transportInfo() { + return new RtpContentMap( + null, + Maps.transformValues( + contents, + dt -> + new DescriptionTransport( + dt.senders, null, dt.transport.cloneWrapper()))); + } + + public IceUdpTransportInfo.Credentials getDistinctCredentials() { + final Set allCredentials = getCredentials(); + final IceUdpTransportInfo.Credentials credentials = + Iterables.getFirst(allCredentials, null); + if (allCredentials.size() == 1 && credentials != null) { + if (Strings.isNullOrEmpty(credentials.password) + || Strings.isNullOrEmpty(credentials.ufrag)) { + throw new IllegalStateException("Credentials are missing password or ufrag"); + } + return credentials; + } + throw new IllegalStateException("Content map does not have distinct credentials"); + } + + public Set getCredentials() { + final Set credentials = + ImmutableSet.copyOf( + Collections2.transform( + contents.values(), dt -> dt.transport.getCredentials())); + if (credentials.isEmpty()) { + throw new IllegalStateException("Content map does not have any credentials"); + } + return credentials; + } + + public IceUdpTransportInfo.Credentials getCredentials(final String contentName) { + final DescriptionTransport descriptionTransport = this.contents.get(contentName); + if (descriptionTransport == null) { + throw new IllegalArgumentException( + String.format( + "Unable to find transport info for content name %s", contentName)); + } + return descriptionTransport.transport.getCredentials(); + } + + public IceUdpTransportInfo.Setup getDtlsSetup() { + final Set setups = + ImmutableSet.copyOf( + Collections2.transform( + contents.values(), dt -> dt.transport.getFingerprint().getSetup())); + final IceUdpTransportInfo.Setup setup = Iterables.getFirst(setups, null); + if (setups.size() == 1 && setup != null) { + return setup; + } + throw new IllegalStateException("Content map doesn't have distinct DTLS setup"); + } + + private DTLS getDistinctDtls() { + final Set dtlsSet = + ImmutableSet.copyOf( + Collections2.transform( + contents.values(), + dt -> { + final IceUdpTransportInfo.Fingerprint fp = + dt.transport.getFingerprint(); + return new DTLS(fp.getHash(), fp.getSetup(), fp.getContent()); + })); + final DTLS dtls = Iterables.getFirst(dtlsSet, null); + if (dtlsSet.size() == 1 && dtls != null) { + return dtls; + } + throw new IllegalStateException("Content map doesn't have distinct DTLS setup"); + } + + public boolean emptyCandidates() { + int count = 0; + for (DescriptionTransport descriptionTransport : contents.values()) { + count += descriptionTransport.transport.getCandidates().size(); + } + return count == 0; + } + + public RtpContentMap modifiedCredentials( + IceUdpTransportInfo.Credentials credentials, final IceUdpTransportInfo.Setup setup) { + final ImmutableMap.Builder contentMapBuilder = + new ImmutableMap.Builder<>(); + for (final Map.Entry content : contents.entrySet()) { + final DescriptionTransport descriptionTransport = content.getValue(); + final RtpDescription rtpDescription = descriptionTransport.description; + final IceUdpTransportInfo transportInfo = descriptionTransport.transport; + final IceUdpTransportInfo modifiedTransportInfo = + transportInfo.modifyCredentials(credentials, setup); + contentMapBuilder.put( + content.getKey(), + new DescriptionTransport( + descriptionTransport.senders, rtpDescription, modifiedTransportInfo)); + } + return new RtpContentMap(this.group, contentMapBuilder.build()); + } + + public RtpContentMap modifiedSenders(final Content.Senders senders) { + return new RtpContentMap( + this.group, + Maps.transformValues( + contents, + dt -> new DescriptionTransport(senders, dt.description, dt.transport))); + } + + public RtpContentMap toContentModification(final Collection modifications) { + return new RtpContentMap( + this.group, + Maps.transformValues( + Maps.filterKeys(contents, Predicates.in(modifications)), + dt -> + new DescriptionTransport( + dt.senders, dt.description, IceUdpTransportInfo.STUB))); + } + + public RtpContentMap toStub() { + return new RtpContentMap( + null, + Maps.transformValues( + this.contents, + dt -> + new DescriptionTransport( + dt.senders, + RtpDescription.stub(dt.description.getMedia()), + IceUdpTransportInfo.STUB))); + } + + public RtpContentMap activeContents() { + return new RtpContentMap( + group, Maps.filterValues(this.contents, dt -> dt.senders != Content.Senders.NONE)); + } + + public Diff diff(final RtpContentMap rtpContentMap) { + final Set existingContentIds = this.contents.keySet(); + final Set newContentIds = rtpContentMap.contents.keySet(); + return new Diff( + ImmutableSet.copyOf(Sets.difference(newContentIds, existingContentIds)), + ImmutableSet.copyOf(Sets.difference(existingContentIds, newContentIds))); + } + + public boolean iceRestart(final RtpContentMap rtpContentMap) { + try { + return !getDistinctCredentials().equals(rtpContentMap.getDistinctCredentials()); + } catch (final IllegalStateException e) { + return false; + } + } + + public RtpContentMap addContent( + final RtpContentMap modification, final IceUdpTransportInfo.Setup setup) { + final IceUdpTransportInfo.Credentials credentials = getDistinctCredentials(); + final DTLS dtls = getDistinctDtls(); + final IceUdpTransportInfo iceUdpTransportInfo = + IceUdpTransportInfo.of(credentials, setup, dtls.hash, dtls.fingerprint); + final Map combined = merge(contents, modification.contents); + /*new ImmutableMap.Builder() + .putAll(contents) + .putAll(modification.contents) + .build();*/ + final Map combinedFixedTransport = + Maps.transformValues( + combined, + dt -> + new DescriptionTransport( + dt.senders, dt.description, iceUdpTransportInfo)); + return new RtpContentMap(modification.group, combinedFixedTransport); + } + + private static Map merge( + final Map a, final Map b) { + final Map combined = new HashMap<>(); + combined.putAll(a); + combined.putAll(b); + return ImmutableMap.copyOf(combined); + } + + public static class DescriptionTransport { + public final Content.Senders senders; + public final RtpDescription description; + public final IceUdpTransportInfo transport; + + public DescriptionTransport( + final Content.Senders senders, + final RtpDescription description, + final IceUdpTransportInfo transport) { + this.senders = senders; + this.description = description; + this.transport = transport; + } + + public static DescriptionTransport of(final Content content) { + final GenericDescription description = content.getDescription(); + final GenericTransportInfo transportInfo = content.getTransport(); + final Content.Senders senders = content.getSenders(); + final RtpDescription rtpDescription; + final IceUdpTransportInfo iceUdpTransportInfo; + if (description == null) { + rtpDescription = null; + } else if (description instanceof RtpDescription) { + rtpDescription = (RtpDescription) description; + } else { + throw new UnsupportedApplicationException( + "Content does not contain rtp description"); + } + if (transportInfo instanceof IceUdpTransportInfo) { + iceUdpTransportInfo = (IceUdpTransportInfo) transportInfo; + } else { + throw new UnsupportedTransportException( + "Content does not contain ICE-UDP transport"); + } + return new DescriptionTransport( + senders, + rtpDescription, + OmemoVerifiedIceUdpTransportInfo.upgrade(iceUdpTransportInfo)); + } + + private static DescriptionTransport of( + final SessionDescription sessionDescription, + final boolean isInitiator, + final SessionDescription.Media media) { + final Content.Senders senders = Content.Senders.of(media, isInitiator); + final RtpDescription rtpDescription = RtpDescription.of(sessionDescription, media); + final IceUdpTransportInfo transportInfo = + IceUdpTransportInfo.of(sessionDescription, media); + return new DescriptionTransport(senders, rtpDescription, transportInfo); + } + + public static Map of(final Map contents) { + return ImmutableMap.copyOf( + Maps.transformValues( + contents, content -> content == null ? null : of(content))); + } + } + + public static class UnsupportedApplicationException extends IllegalArgumentException { + UnsupportedApplicationException(String message) { + super(message); + } + } + + public static class UnsupportedTransportException extends IllegalArgumentException { + UnsupportedTransportException(String message) { + super(message); + } + } + + public static final class Diff { + public final Set added; + public final Set removed; + + private Diff(final Set added, final Set removed) { + this.added = added; + this.removed = removed; + } + + public boolean hasModifications() { + return !this.added.isEmpty() || !this.removed.isEmpty(); + } + + public boolean isEmpty() { + return this.added.isEmpty() && this.removed.isEmpty(); + } + + @Override + @Nonnull + public String toString() { + return MoreObjects.toStringHelper(this) + .add("added", added) + .add("removed", removed) + .toString(); + } + } + + public static final class DTLS { + public final String hash; + public final IceUdpTransportInfo.Setup setup; + public final String fingerprint; + + private DTLS(String hash, IceUdpTransportInfo.Setup setup, String fingerprint) { + this.hash = hash; + this.setup = setup; + this.fingerprint = fingerprint; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DTLS dtls = (DTLS) o; + return Objects.equal(hash, dtls.hash) + && setup == dtls.setup + && Objects.equal(fingerprint, dtls.fingerprint); + } + + @Override + public int hashCode() { + return Objects.hashCode(hash, setup, fingerprint); + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java new file mode 100644 index 000000000..99cdc633f --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java @@ -0,0 +1,21 @@ +package eu.siacs.conversations.xmpp.jingle; + +public enum RtpEndUserState { + INCOMING_CALL, // received a 'propose' message + CONNECTING, // session-initiate or session-accepted but no webrtc peer connection yet + CONNECTED, // session-accepted and webrtc peer connection is connected + RECONNECTING, // session-accepted and webrtc peer connection was connected once but is currently + // disconnected or failed + INCOMING_CONTENT_ADD, // session-accepted with a pending, incoming content-add + FINDING_DEVICE, // 'propose' has been sent out; no 184 ack yet + RINGING, // 'propose' has been sent out and it has been 184 acked + ACCEPTING_CALL, // 'proceed' message has been sent; but no session-initiate has been received + ENDING_CALL, // libwebrt says 'closed' but session-terminate hasnt gone through + ENDED, // close UI + DECLINED_OR_BUSY, // other party declined; no retry button + CONNECTIVITY_ERROR, // network error; retry button + CONNECTIVITY_LOST_ERROR, // network error but for call duration > 0 + RETRACTED, // user pressed home or power button during 'ringing' - shows retry button + APPLICATION_ERROR, // something rather bad happened; libwebrtc failed or we got in IQ-error + SECURITY_ERROR // problem with DTLS (missing) or verification +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java new file mode 100644 index 000000000..ec9bbbbb4 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java @@ -0,0 +1,415 @@ +package eu.siacs.conversations.xmpp.jingle; + +import android.util.Log; +import android.util.Pair; +import androidx.annotation.NonNull; +import com.google.common.base.CharMatcher; +import com.google.common.base.Joiner; +import com.google.common.base.Strings; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.xmpp.jingle.stanzas.Group; +import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; +import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; +import im.conversations.android.xml.Namespace; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +public class SessionDescription { + + public static final String LINE_DIVIDER = "\r\n"; + private static final String HARDCODED_MEDIA_PROTOCOL = + "UDP/TLS/RTP/SAVPF"; // probably only true for DTLS-SRTP aka when we have a fingerprint + private static final int HARDCODED_MEDIA_PORT = 9; + private static final String HARDCODED_ICE_OPTIONS = "trickle"; + private static final String HARDCODED_CONNECTION = "IN IP4 0.0.0.0"; + + public final int version; + public final String name; + public final String connectionData; + public final ArrayListMultimap attributes; + public final List media; + + public SessionDescription( + int version, + String name, + String connectionData, + ArrayListMultimap attributes, + List media) { + this.version = version; + this.name = name; + this.connectionData = connectionData; + this.attributes = attributes; + this.media = media; + } + + private static void appendAttributes( + StringBuilder s, ArrayListMultimap attributes) { + for (Map.Entry attribute : attributes.entries()) { + final String key = attribute.getKey(); + final String value = attribute.getValue(); + s.append("a=").append(key); + if (!Strings.isNullOrEmpty(value)) { + s.append(':').append(value); + } + s.append(LINE_DIVIDER); + } + } + + public static SessionDescription parse(final String input) { + final SessionDescriptionBuilder sessionDescriptionBuilder = new SessionDescriptionBuilder(); + MediaBuilder currentMediaBuilder = null; + ArrayListMultimap attributeMap = ArrayListMultimap.create(); + ImmutableList.Builder mediaBuilder = new ImmutableList.Builder<>(); + for (final String line : input.split(LINE_DIVIDER)) { + final String[] pair = line.trim().split("=", 2); + if (pair.length < 2 || pair[0].length() != 1) { + Log.d(Config.LOGTAG, "skipping sdp parsing on line " + line); + continue; + } + final char key = pair[0].charAt(0); + final String value = pair[1]; + switch (key) { + case 'v': + sessionDescriptionBuilder.setVersion(ignorantIntParser(value)); + break; + case 'c': + if (currentMediaBuilder != null) { + currentMediaBuilder.setConnectionData(value); + } else { + sessionDescriptionBuilder.setConnectionData(value); + } + break; + case 's': + sessionDescriptionBuilder.setName(value); + break; + case 'a': + final Pair attribute = parseAttribute(value); + attributeMap.put(attribute.first, attribute.second); + break; + case 'm': + if (currentMediaBuilder == null) { + sessionDescriptionBuilder.setAttributes(attributeMap); + } else { + currentMediaBuilder.setAttributes(attributeMap); + mediaBuilder.add(currentMediaBuilder.createMedia()); + } + attributeMap = ArrayListMultimap.create(); + currentMediaBuilder = new MediaBuilder(); + final String[] parts = value.split(" "); + if (parts.length >= 3) { + currentMediaBuilder.setMedia(parts[0]); + currentMediaBuilder.setPort(ignorantIntParser(parts[1])); + currentMediaBuilder.setProtocol(parts[2]); + ImmutableList.Builder formats = new ImmutableList.Builder<>(); + for (int i = 3; i < parts.length; ++i) { + formats.add(ignorantIntParser(parts[i])); + } + currentMediaBuilder.setFormats(formats.build()); + } else { + Log.d(Config.LOGTAG, "skipping media line " + line); + } + break; + } + } + if (currentMediaBuilder != null) { + currentMediaBuilder.setAttributes(attributeMap); + mediaBuilder.add(currentMediaBuilder.createMedia()); + } else { + sessionDescriptionBuilder.setAttributes(attributeMap); + } + sessionDescriptionBuilder.setMedia(mediaBuilder.build()); + return sessionDescriptionBuilder.createSessionDescription(); + } + + public static SessionDescription of( + final RtpContentMap contentMap, final boolean isInitiatorContentMap) { + final SessionDescriptionBuilder sessionDescriptionBuilder = new SessionDescriptionBuilder(); + final ArrayListMultimap attributeMap = ArrayListMultimap.create(); + final ImmutableList.Builder mediaListBuilder = new ImmutableList.Builder<>(); + final Group group = contentMap.group; + if (group != null) { + final String semantics = group.getSemantics(); + checkNoWhitespace(semantics, "group semantics value must not contain any whitespace"); + attributeMap.put( + "group", + group.getSemantics() + + " " + + Joiner.on(' ').join(group.getIdentificationTags())); + } + + attributeMap.put("msid-semantic", " WMS my-media-stream"); + + for (final Map.Entry entry : + contentMap.contents.entrySet()) { + final String name = entry.getKey(); + RtpContentMap.DescriptionTransport descriptionTransport = entry.getValue(); + RtpDescription description = descriptionTransport.description; + IceUdpTransportInfo transport = descriptionTransport.transport; + final ArrayListMultimap mediaAttributes = ArrayListMultimap.create(); + final String ufrag = transport.getAttribute("ufrag"); + final String pwd = transport.getAttribute("pwd"); + if (Strings.isNullOrEmpty(ufrag)) { + throw new IllegalArgumentException( + "Transport element is missing required ufrag attribute"); + } + checkNoWhitespace(ufrag, "ufrag value must not contain any whitespaces"); + mediaAttributes.put("ice-ufrag", ufrag); + if (Strings.isNullOrEmpty(pwd)) { + throw new IllegalArgumentException( + "Transport element is missing required pwd attribute"); + } + checkNoWhitespace(pwd, "pwd value must not contain any whitespaces"); + mediaAttributes.put("ice-pwd", pwd); + mediaAttributes.put("ice-options", HARDCODED_ICE_OPTIONS); + final IceUdpTransportInfo.Fingerprint fingerprint = transport.getFingerprint(); + if (fingerprint != null) { + mediaAttributes.put( + "fingerprint", fingerprint.getHash() + " " + fingerprint.getContent()); + final IceUdpTransportInfo.Setup setup = fingerprint.getSetup(); + if (setup != null) { + mediaAttributes.put("setup", setup.toString().toLowerCase(Locale.ROOT)); + } + } + final ImmutableList.Builder formatBuilder = new ImmutableList.Builder<>(); + for (RtpDescription.PayloadType payloadType : description.getPayloadTypes()) { + final String id = payloadType.getId(); + if (Strings.isNullOrEmpty(id)) { + throw new IllegalArgumentException("Payload type is missing id"); + } + if (!isInt(id)) { + throw new IllegalArgumentException("Payload id is not numeric"); + } + formatBuilder.add(payloadType.getIntId()); + mediaAttributes.put("rtpmap", payloadType.toSdpAttribute()); + final List parameters = payloadType.getParameters(); + if (parameters.size() == 1) { + mediaAttributes.put( + "fmtp", RtpDescription.Parameter.toSdpString(id, parameters.get(0))); + } else if (parameters.size() > 0) { + mediaAttributes.put( + "fmtp", RtpDescription.Parameter.toSdpString(id, parameters)); + } + for (RtpDescription.FeedbackNegotiation feedbackNegotiation : + payloadType.getFeedbackNegotiations()) { + final String type = feedbackNegotiation.getType(); + final String subtype = feedbackNegotiation.getSubType(); + if (Strings.isNullOrEmpty(type)) { + throw new IllegalArgumentException( + "a feedback for payload-type " + + id + + " negotiation is missing type"); + } + checkNoWhitespace( + type, "feedback negotiation type must not contain whitespace"); + mediaAttributes.put( + "rtcp-fb", + id + + " " + + type + + (Strings.isNullOrEmpty(subtype) ? "" : " " + subtype)); + } + for (RtpDescription.FeedbackNegotiationTrrInt feedbackNegotiationTrrInt : + payloadType.feedbackNegotiationTrrInts()) { + mediaAttributes.put( + "rtcp-fb", id + " trr-int " + feedbackNegotiationTrrInt.getValue()); + } + } + + for (RtpDescription.FeedbackNegotiation feedbackNegotiation : + description.getFeedbackNegotiations()) { + final String type = feedbackNegotiation.getType(); + final String subtype = feedbackNegotiation.getSubType(); + if (Strings.isNullOrEmpty(type)) { + throw new IllegalArgumentException("a feedback negotiation is missing type"); + } + checkNoWhitespace(type, "feedback negotiation type must not contain whitespace"); + mediaAttributes.put( + "rtcp-fb", + "* " + type + (Strings.isNullOrEmpty(subtype) ? "" : " " + subtype)); + } + for (final RtpDescription.FeedbackNegotiationTrrInt feedbackNegotiationTrrInt : + description.feedbackNegotiationTrrInts()) { + mediaAttributes.put("rtcp-fb", "* trr-int " + feedbackNegotiationTrrInt.getValue()); + } + for (final RtpDescription.RtpHeaderExtension extension : + description.getHeaderExtensions()) { + final String id = extension.getId(); + final String uri = extension.getUri(); + if (Strings.isNullOrEmpty(id)) { + throw new IllegalArgumentException("A header extension is missing id"); + } + checkNoWhitespace(id, "header extension id must not contain whitespace"); + if (Strings.isNullOrEmpty(uri)) { + throw new IllegalArgumentException("A header extension is missing uri"); + } + checkNoWhitespace(uri, "feedback negotiation uri must not contain whitespace"); + mediaAttributes.put("extmap", id + " " + uri); + } + + if (description.hasChild( + "extmap-allow-mixed", Namespace.JINGLE_RTP_HEADER_EXTENSIONS)) { + mediaAttributes.put("extmap-allow-mixed", ""); + } + + for (final RtpDescription.SourceGroup sourceGroup : description.getSourceGroups()) { + final String semantics = sourceGroup.getSemantics(); + final List groups = sourceGroup.getSsrcs(); + if (Strings.isNullOrEmpty(semantics)) { + throw new IllegalArgumentException( + "A SSRC group is missing semantics attribute"); + } + checkNoWhitespace(semantics, "source group semantics must not contain whitespace"); + if (groups.size() == 0) { + throw new IllegalArgumentException("A SSRC group is missing SSRC ids"); + } + mediaAttributes.put( + "ssrc-group", + String.format("%s %s", semantics, Joiner.on(' ').join(groups))); + } + for (final RtpDescription.Source source : description.getSources()) { + for (final RtpDescription.Source.Parameter parameter : source.getParameters()) { + final String id = source.getSsrcId(); + final String parameterName = parameter.getParameterName(); + final String parameterValue = parameter.getParameterValue(); + if (Strings.isNullOrEmpty(id)) { + throw new IllegalArgumentException( + "A source specific media attribute is missing the id"); + } + checkNoWhitespace( + id, "A source specific media attributes must not contain whitespaces"); + if (Strings.isNullOrEmpty(parameterName)) { + throw new IllegalArgumentException( + "A source specific media attribute is missing its name"); + } + if (Strings.isNullOrEmpty(parameterValue)) { + throw new IllegalArgumentException( + "A source specific media attribute is missing its value"); + } + mediaAttributes.put("ssrc", id + " " + parameterName + ":" + parameterValue); + } + } + + mediaAttributes.put("mid", name); + + mediaAttributes.put( + descriptionTransport.senders.asMediaAttribute(isInitiatorContentMap), ""); + if (description.hasChild("rtcp-mux", Namespace.JINGLE_APPS_RTP) || group != null) { + mediaAttributes.put("rtcp-mux", ""); + } + + // random additional attributes + mediaAttributes.put("rtcp", "9 IN IP4 0.0.0.0"); + + final MediaBuilder mediaBuilder = new MediaBuilder(); + mediaBuilder.setMedia(description.getMedia().toString().toLowerCase(Locale.ROOT)); + mediaBuilder.setConnectionData(HARDCODED_CONNECTION); + mediaBuilder.setPort(HARDCODED_MEDIA_PORT); + mediaBuilder.setProtocol(HARDCODED_MEDIA_PROTOCOL); + mediaBuilder.setAttributes(mediaAttributes); + mediaBuilder.setFormats(formatBuilder.build()); + mediaListBuilder.add(mediaBuilder.createMedia()); + } + sessionDescriptionBuilder.setVersion(0); + sessionDescriptionBuilder.setName("-"); + sessionDescriptionBuilder.setMedia(mediaListBuilder.build()); + sessionDescriptionBuilder.setAttributes(attributeMap); + + return sessionDescriptionBuilder.createSessionDescription(); + } + + public static String checkNoWhitespace(final String input, final String message) { + if (CharMatcher.whitespace().matchesAnyOf(input)) { + throw new IllegalArgumentException(message); + } + return input; + } + + public static int ignorantIntParser(final String input) { + try { + return Integer.parseInt(input); + } catch (NumberFormatException e) { + return 0; + } + } + + public static boolean isInt(final String input) { + if (input == null) { + return false; + } + try { + Integer.parseInt(input); + return true; + } catch (NumberFormatException e) { + return false; + } + } + + public static Pair parseAttribute(final String input) { + final String[] pair = input.split(":", 2); + if (pair.length == 2) { + return new Pair<>(pair[0], pair[1]); + } else { + return new Pair<>(pair[0], ""); + } + } + + @NonNull + @Override + public String toString() { + final StringBuilder s = + new StringBuilder() + .append("v=") + .append(version) + .append(LINE_DIVIDER) + // TODO randomize or static + .append("o=- 8770656990916039506 2 IN IP4 127.0.0.1") + .append(LINE_DIVIDER) // what ever that means + .append("s=") + .append(name) + .append(LINE_DIVIDER) + .append("t=0 0") + .append(LINE_DIVIDER); + appendAttributes(s, attributes); + for (Media media : this.media) { + s.append("m=") + .append(media.media) + .append(' ') + .append(media.port) + .append(' ') + .append(media.protocol) + .append(' ') + .append(Joiner.on(' ').join(media.formats)) + .append(LINE_DIVIDER); + s.append("c=").append(media.connectionData).append(LINE_DIVIDER); + appendAttributes(s, media.attributes); + } + return s.toString(); + } + + public static class Media { + public final String media; + public final int port; + public final String protocol; + public final List formats; + public final String connectionData; + public final ArrayListMultimap attributes; + + public Media( + String media, + int port, + String protocol, + List formats, + String connectionData, + ArrayListMultimap attributes) { + this.media = media; + this.port = port; + this.protocol = protocol; + this.formats = formats; + this.connectionData = connectionData; + this.attributes = attributes; + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescriptionBuilder.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescriptionBuilder.java new file mode 100644 index 000000000..c7da602e0 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescriptionBuilder.java @@ -0,0 +1,41 @@ +package eu.siacs.conversations.xmpp.jingle; + +import com.google.common.collect.ArrayListMultimap; +import java.util.List; + +public class SessionDescriptionBuilder { + private int version; + private String name; + private String connectionData; + private ArrayListMultimap attributes; + private List media; + + public SessionDescriptionBuilder setVersion(int version) { + this.version = version; + return this; + } + + public SessionDescriptionBuilder setName(String name) { + this.name = name; + return this; + } + + public SessionDescriptionBuilder setConnectionData(String connectionData) { + this.connectionData = connectionData; + return this; + } + + public SessionDescriptionBuilder setAttributes(ArrayListMultimap attributes) { + this.attributes = attributes; + return this; + } + + public SessionDescriptionBuilder setMedia(List media) { + this.media = media; + return this; + } + + public SessionDescription createSessionDescription() { + return new SessionDescription(version, name, connectionData, attributes, media); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java new file mode 100644 index 000000000..da44a7ae4 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java @@ -0,0 +1,274 @@ +package eu.siacs.conversations.xmpp.jingle; + +import static java.util.Arrays.asList; + +import android.content.Context; +import android.media.AudioManager; +import android.media.ToneGenerator; +import android.util.Log; +import eu.siacs.conversations.Config; +import im.conversations.android.xmpp.manager.JingleConnectionManager; +import java.util.Arrays; +import java.util.Set; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public class ToneManager { + + private final ToneGenerator toneGenerator; + private final Context context; + + private ToneState state = null; + private RtpEndUserState endUserState = null; + private ScheduledFuture currentTone; + private ScheduledFuture currentResetFuture; + private boolean appRtcAudioManagerHasControl = false; + + private static volatile ToneManager INSTANCE; + + private ToneManager(final Context context) { + ToneGenerator toneGenerator; + try { + toneGenerator = new ToneGenerator(AudioManager.STREAM_VOICE_CALL, 60); + } catch (final RuntimeException e) { + Log.e(Config.LOGTAG, "unable to instantiate ToneGenerator", e); + toneGenerator = null; + } + this.toneGenerator = toneGenerator; + this.context = context.getApplicationContext(); + } + + private static ToneState of( + final boolean isInitiator, final RtpEndUserState state, final Set media) { + if (isInitiator) { + if (asList( + RtpEndUserState.FINDING_DEVICE, + RtpEndUserState.RINGING, + RtpEndUserState.CONNECTING) + .contains(state)) { + return ToneState.RINGING; + } + if (state == RtpEndUserState.DECLINED_OR_BUSY) { + return ToneState.BUSY; + } + } + if (state == RtpEndUserState.ENDING_CALL) { + if (media.contains(Media.VIDEO)) { + return ToneState.NULL; + } else { + return ToneState.ENDING_CALL; + } + } + if (Arrays.asList( + RtpEndUserState.CONNECTED, + RtpEndUserState.RECONNECTING, + RtpEndUserState.INCOMING_CONTENT_ADD) + .contains(state)) { + if (media.contains(Media.VIDEO)) { + return ToneState.NULL; + } else { + return ToneState.CONNECTED; + } + } + return ToneState.NULL; + } + + public void transition(final RtpEndUserState state, final Set media) { + transition(state, of(true, state, media), media); + } + + void transition( + final boolean isInitiator, final RtpEndUserState state, final Set media) { + transition(state, of(isInitiator, state, media), media); + } + + private synchronized void transition( + final RtpEndUserState endUserState, final ToneState state, final Set media) { + final RtpEndUserState normalizeEndUserState = normalize(endUserState); + if (this.endUserState == normalizeEndUserState) { + return; + } + this.endUserState = normalizeEndUserState; + if (this.state == state) { + return; + } + if (state == ToneState.NULL && this.state == ToneState.ENDING_CALL) { + return; + } + cancelCurrentTone(); + Log.d(Config.LOGTAG, getClass().getName() + ".transition(" + state + ")"); + if (state != ToneState.NULL) { + configureAudioManagerForCall(media); + } + switch (state) { + case RINGING: + scheduleWaitingTone(); + break; + case CONNECTED: + scheduleConnected(); + break; + case BUSY: + scheduleBusy(); + break; + case ENDING_CALL: + scheduleEnding(); + break; + case NULL: + if (noResetScheduled()) { + resetAudioManager(); + } + break; + default: + throw new IllegalStateException("Unable to handle transition to " + state); + } + this.state = state; + } + + private static RtpEndUserState normalize(final RtpEndUserState endUserState) { + if (Arrays.asList( + RtpEndUserState.CONNECTED, + RtpEndUserState.RECONNECTING, + RtpEndUserState.INCOMING_CONTENT_ADD) + .contains(endUserState)) { + return RtpEndUserState.CONNECTED; + } else { + return endUserState; + } + } + + void setAppRtcAudioManagerHasControl(final boolean appRtcAudioManagerHasControl) { + this.appRtcAudioManagerHasControl = appRtcAudioManagerHasControl; + } + + private void scheduleConnected() { + this.currentTone = + JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE.schedule( + () -> { + startTone(ToneGenerator.TONE_PROP_PROMPT, 200); + }, + 0, + TimeUnit.SECONDS); + } + + private void scheduleEnding() { + this.currentTone = + JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE.schedule( + () -> { + startTone(ToneGenerator.TONE_CDMA_CALLDROP_LITE, 375); + }, + 0, + TimeUnit.SECONDS); + this.currentResetFuture = + JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE.schedule( + this::resetAudioManager, 375, TimeUnit.MILLISECONDS); + } + + private void scheduleBusy() { + this.currentTone = + JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE.schedule( + () -> { + startTone(ToneGenerator.TONE_CDMA_NETWORK_BUSY, 2500); + }, + 0, + TimeUnit.SECONDS); + this.currentResetFuture = + JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE.schedule( + this::resetAudioManager, 2500, TimeUnit.MILLISECONDS); + } + + private void scheduleWaitingTone() { + this.currentTone = + JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE.scheduleAtFixedRate( + () -> { + startTone(ToneGenerator.TONE_CDMA_DIAL_TONE_LITE, 750); + }, + 0, + 3, + TimeUnit.SECONDS); + } + + private boolean noResetScheduled() { + return this.currentResetFuture == null || this.currentResetFuture.isDone(); + } + + private void cancelCurrentTone() { + if (currentTone != null) { + currentTone.cancel(true); + } + if (toneGenerator != null) { + toneGenerator.stopTone(); + } + } + + private void startTone(final int toneType, final int durationMs) { + if (toneGenerator != null) { + this.toneGenerator.startTone(toneType, durationMs); + } else { + Log.e(Config.LOGTAG, "failed to start tone. ToneGenerator doesn't exist"); + } + } + + private void configureAudioManagerForCall(final Set media) { + if (appRtcAudioManagerHasControl) { + Log.d( + Config.LOGTAG, + ToneManager.class.getName() + + ": do not configure audio manager because RTC has control"); + return; + } + final AudioManager audioManager = + (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + if (audioManager == null) { + return; + } + final boolean isSpeakerPhone = media.contains(Media.VIDEO); + Log.d( + Config.LOGTAG, + ToneManager.class.getName() + + ": putting AudioManager into communication mode. speaker=" + + isSpeakerPhone); + audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION); + audioManager.setSpeakerphoneOn(isSpeakerPhone); + } + + private void resetAudioManager() { + if (appRtcAudioManagerHasControl) { + Log.d( + Config.LOGTAG, + ToneManager.class.getName() + + ": do not reset audio manager because RTC has control"); + return; + } + final AudioManager audioManager = + (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + if (audioManager == null) { + return; + } + Log.d( + Config.LOGTAG, + ToneManager.class.getName() + ": putting AudioManager back into normal mode"); + audioManager.setMode(AudioManager.MODE_NORMAL); + audioManager.setSpeakerphoneOn(false); + } + + public static ToneManager getInstance(final Context context) { + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (ToneManager.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new ToneManager(context); + return INSTANCE; + } + } + + private enum ToneState { + NULL, + RINGING, + CONNECTED, + BUSY, + ENDING_CALL + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java new file mode 100644 index 000000000..061214048 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java @@ -0,0 +1,71 @@ +package eu.siacs.conversations.xmpp.jingle; + +import android.util.Log; +import com.google.common.base.CaseFormat; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import eu.siacs.conversations.Config; +import java.util.UUID; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.webrtc.MediaStreamTrack; +import org.webrtc.PeerConnection; +import org.webrtc.RtpSender; +import org.webrtc.RtpTransceiver; + +class TrackWrapper { + public final T track; + public final RtpSender rtpSender; + + private TrackWrapper(final T track, final RtpSender rtpSender) { + Preconditions.checkNotNull(track); + Preconditions.checkNotNull(rtpSender); + this.track = track; + this.rtpSender = rtpSender; + } + + public static TrackWrapper addTrack( + final PeerConnection peerConnection, final T mediaStreamTrack) { + final RtpSender rtpSender = peerConnection.addTrack(mediaStreamTrack); + return new TrackWrapper<>(mediaStreamTrack, rtpSender); + } + + public static Optional get( + @Nullable final PeerConnection peerConnection, final TrackWrapper trackWrapper) { + if (trackWrapper == null) { + return Optional.absent(); + } + final RtpTransceiver transceiver = + peerConnection == null ? null : getTransceiver(peerConnection, trackWrapper); + if (transceiver == null) { + Log.w(Config.LOGTAG, "unable to detect transceiver for " + trackWrapper.rtpSender.id()); + return Optional.of(trackWrapper.track); + } + final RtpTransceiver.RtpTransceiverDirection direction = transceiver.getDirection(); + if (direction == RtpTransceiver.RtpTransceiverDirection.SEND_ONLY + || direction == RtpTransceiver.RtpTransceiverDirection.SEND_RECV) { + return Optional.of(trackWrapper.track); + } else { + Log.d(Config.LOGTAG, "withholding track because transceiver is " + direction); + return Optional.absent(); + } + } + + public static RtpTransceiver getTransceiver( + @Nonnull final PeerConnection peerConnection, final TrackWrapper trackWrapper) { + final RtpSender rtpSender = trackWrapper.rtpSender; + for (final RtpTransceiver transceiver : peerConnection.getTransceivers()) { + if (transceiver.getSender().id().equals(rtpSender.id())) { + return transceiver; + } + } + return null; + } + + public static String id(final Class clazz) { + return String.format( + "%s-%s", + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, clazz.getSimpleName()), + UUID.randomUUID().toString()); + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/VideoSourceWrapper.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/VideoSourceWrapper.java new file mode 100644 index 000000000..988a013ab --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/VideoSourceWrapper.java @@ -0,0 +1,182 @@ +package eu.siacs.conversations.xmpp.jingle; + +import android.content.Context; +import android.util.Log; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import eu.siacs.conversations.Config; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; +import javax.annotation.Nullable; +import org.webrtc.Camera2Enumerator; +import org.webrtc.CameraEnumerationAndroid; +import org.webrtc.CameraEnumerator; +import org.webrtc.CameraVideoCapturer; +import org.webrtc.EglBase; +import org.webrtc.PeerConnectionFactory; +import org.webrtc.SurfaceTextureHelper; +import org.webrtc.VideoSource; + +class VideoSourceWrapper { + + private static final int CAPTURING_RESOLUTION = 1920; + private static final int CAPTURING_MAX_FRAME_RATE = 30; + + private final CameraVideoCapturer cameraVideoCapturer; + private final CameraEnumerationAndroid.CaptureFormat captureFormat; + private final Set availableCameras; + private boolean isFrontCamera = false; + private VideoSource videoSource; + + VideoSourceWrapper( + CameraVideoCapturer cameraVideoCapturer, + CameraEnumerationAndroid.CaptureFormat captureFormat, + Set cameras) { + this.cameraVideoCapturer = cameraVideoCapturer; + this.captureFormat = captureFormat; + this.availableCameras = cameras; + } + + private int getFrameRate() { + return Math.max( + captureFormat.framerate.min, + Math.min(CAPTURING_MAX_FRAME_RATE, captureFormat.framerate.max)); + } + + public void initialize( + final PeerConnectionFactory peerConnectionFactory, + final Context context, + final EglBase.Context eglBaseContext) { + final SurfaceTextureHelper surfaceTextureHelper = + SurfaceTextureHelper.create("webrtc", eglBaseContext); + this.videoSource = peerConnectionFactory.createVideoSource(false); + this.cameraVideoCapturer.initialize( + surfaceTextureHelper, context, this.videoSource.getCapturerObserver()); + } + + public VideoSource getVideoSource() { + final VideoSource videoSource = this.videoSource; + if (videoSource == null) { + throw new IllegalStateException("VideoSourceWrapper was not initialized"); + } + return videoSource; + } + + public void startCapture() { + final int frameRate = getFrameRate(); + Log.d( + Config.LOGTAG, + String.format( + "start capturing at %dx%d@%d", + captureFormat.width, captureFormat.height, frameRate)); + this.cameraVideoCapturer.startCapture(captureFormat.width, captureFormat.height, frameRate); + } + + public void stopCapture() throws InterruptedException { + this.cameraVideoCapturer.stopCapture(); + } + + public void dispose() { + this.cameraVideoCapturer.dispose(); + if (this.videoSource != null) { + dispose(this.videoSource); + } + } + + private static void dispose(final VideoSource videoSource) { + try { + videoSource.dispose(); + } catch (final IllegalStateException e) { + Log.e(Config.LOGTAG, "unable to dispose video source", e); + } + } + + public ListenableFuture switchCamera() { + final SettableFuture future = SettableFuture.create(); + this.cameraVideoCapturer.switchCamera( + new CameraVideoCapturer.CameraSwitchHandler() { + @Override + public void onCameraSwitchDone(final boolean isFrontCamera) { + VideoSourceWrapper.this.isFrontCamera = isFrontCamera; + future.set(isFrontCamera); + } + + @Override + public void onCameraSwitchError(final String message) { + future.setException( + new IllegalStateException( + String.format("Unable to switch camera %s", message))); + } + }); + return future; + } + + public boolean isFrontCamera() { + return this.isFrontCamera; + } + + public boolean isCameraSwitchable() { + return this.availableCameras.size() > 1; + } + + public static class Factory { + final Context context; + + public Factory(final Context context) { + this.context = context; + } + + public VideoSourceWrapper create() { + final CameraEnumerator enumerator = new Camera2Enumerator(context); + final Set deviceNames = ImmutableSet.copyOf(enumerator.getDeviceNames()); + for (final String deviceName : deviceNames) { + if (isFrontFacing(enumerator, deviceName)) { + final VideoSourceWrapper videoSourceWrapper = + of(enumerator, deviceName, deviceNames); + if (videoSourceWrapper == null) { + return null; + } + videoSourceWrapper.isFrontCamera = true; + return videoSourceWrapper; + } + } + if (deviceNames.size() == 0) { + return null; + } else { + return of(enumerator, Iterables.get(deviceNames, 0), deviceNames); + } + } + + @Nullable + private VideoSourceWrapper of( + final CameraEnumerator enumerator, + final String deviceName, + final Set availableCameras) { + final CameraVideoCapturer capturer = enumerator.createCapturer(deviceName, null); + if (capturer == null) { + return null; + } + final ArrayList choices = + new ArrayList<>(enumerator.getSupportedFormats(deviceName)); + Collections.sort(choices, (a, b) -> b.width - a.width); + for (final CameraEnumerationAndroid.CaptureFormat captureFormat : choices) { + if (captureFormat.width <= CAPTURING_RESOLUTION) { + return new VideoSourceWrapper(capturer, captureFormat, availableCameras); + } + } + return null; + } + + private static boolean isFrontFacing( + final CameraEnumerator cameraEnumerator, final String deviceName) { + try { + return cameraEnumerator.isFrontFacing(deviceName); + } catch (final NullPointerException e) { + return false; + } + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java new file mode 100644 index 000000000..ec05e6b71 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -0,0 +1,750 @@ +package eu.siacs.conversations.xmpp.jingle; + +import android.content.Context; +import android.os.Build; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.services.AppRTCAudioManager; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.webrtc.AudioSource; +import org.webrtc.AudioTrack; +import org.webrtc.CandidatePairChangeEvent; +import org.webrtc.DataChannel; +import org.webrtc.DefaultVideoDecoderFactory; +import org.webrtc.DefaultVideoEncoderFactory; +import org.webrtc.EglBase; +import org.webrtc.IceCandidate; +import org.webrtc.MediaConstraints; +import org.webrtc.MediaStream; +import org.webrtc.MediaStreamTrack; +import org.webrtc.PeerConnection; +import org.webrtc.PeerConnectionFactory; +import org.webrtc.RtpReceiver; +import org.webrtc.RtpTransceiver; +import org.webrtc.SdpObserver; +import org.webrtc.SessionDescription; +import org.webrtc.VideoTrack; +import org.webrtc.audio.JavaAudioDeviceModule; +import org.webrtc.voiceengine.WebRtcAudioEffects; + +@SuppressWarnings("UnstableApiUsage") +public class WebRTCWrapper { + + private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName(); + + private final ExecutorService executorService = Executors.newSingleThreadExecutor(); + + private static final Set HARDWARE_AEC_BLACKLIST = + new ImmutableSet.Builder() + .add("Pixel") + .add("Pixel XL") + .add("Moto G5") + .add("Moto G (5S) Plus") + .add("Moto G4") + .add("TA-1053") + .add("Mi A1") + .add("Mi A2") + .add("E5823") // Sony z5 compact + .add("Redmi Note 5") + .add("FP2") // Fairphone FP2 + .add("MI 5") + .add("GT-I9515") // Samsung Galaxy S4 Value Edition (jfvelte) + .add("GT-I9515L") // Samsung Galaxy S4 Value Edition (jfvelte) + .add("GT-I9505") // Samsung Galaxy S4 (jfltexx) + .build(); + + private final EventCallback eventCallback; + private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false); + private final Queue iceCandidates = new LinkedList<>(); + private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = + new AppRTCAudioManager.AudioManagerEvents() { + @Override + public void onAudioDeviceChanged( + AppRTCAudioManager.AudioDevice selectedAudioDevice, + Set availableAudioDevices) { + eventCallback.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices); + } + }; + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + private TrackWrapper localAudioTrack = null; + private TrackWrapper localVideoTrack = null; + private VideoTrack remoteVideoTrack = null; + private final PeerConnection.Observer peerConnectionObserver = + new PeerConnection.Observer() { + @Override + public void onSignalingChange(PeerConnection.SignalingState signalingState) { + Log.d(EXTENDED_LOGGING_TAG, "onSignalingChange(" + signalingState + ")"); + // this is called after removeTrack or addTrack + // and should then trigger a content-add or content-remove or something + // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/removeTrack + } + + @Override + public void onConnectionChange(final PeerConnection.PeerConnectionState newState) { + eventCallback.onConnectionChange(newState); + } + + @Override + public void onIceConnectionChange( + PeerConnection.IceConnectionState iceConnectionState) { + Log.d( + EXTENDED_LOGGING_TAG, + "onIceConnectionChange(" + iceConnectionState + ")"); + } + + @Override + public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) { + Log.d(Config.LOGTAG, "remote candidate selected: " + event.remote); + Log.d(Config.LOGTAG, "local candidate selected: " + event.local); + } + + @Override + public void onIceConnectionReceivingChange(boolean b) {} + + @Override + public void onIceGatheringChange( + PeerConnection.IceGatheringState iceGatheringState) { + Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")"); + } + + @Override + public void onIceCandidate(IceCandidate iceCandidate) { + if (readyToReceivedIceCandidates.get()) { + eventCallback.onIceCandidate(iceCandidate); + } else { + iceCandidates.add(iceCandidate); + } + } + + @Override + public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {} + + @Override + public void onAddStream(MediaStream mediaStream) { + Log.d( + EXTENDED_LOGGING_TAG, + "onAddStream(numAudioTracks=" + + mediaStream.audioTracks.size() + + ",numVideoTracks=" + + mediaStream.videoTracks.size() + + ")"); + } + + @Override + public void onRemoveStream(MediaStream mediaStream) {} + + @Override + public void onDataChannel(DataChannel dataChannel) {} + + @Override + public void onRenegotiationNeeded() { + Log.d(EXTENDED_LOGGING_TAG, "onRenegotiationNeeded()"); + final PeerConnection.PeerConnectionState currentState = + peerConnection == null ? null : peerConnection.connectionState(); + if (currentState != null + && currentState != PeerConnection.PeerConnectionState.NEW) { + eventCallback.onRenegotiationNeeded(); + } + } + + @Override + public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) { + final MediaStreamTrack track = rtpReceiver.track(); + Log.d( + EXTENDED_LOGGING_TAG, + "onAddTrack(kind=" + + (track == null ? "null" : track.kind()) + + ",numMediaStreams=" + + mediaStreams.length + + ")"); + if (track instanceof VideoTrack) { + remoteVideoTrack = (VideoTrack) track; + } + } + + @Override + public void onTrack(final RtpTransceiver transceiver) { + Log.d( + EXTENDED_LOGGING_TAG, + "onTrack(mid=" + + transceiver.getMid() + + ",media=" + + transceiver.getMediaType() + + ",direction=" + + transceiver.getDirection() + + ")"); + } + + @Override + public void onRemoveTrack(final RtpReceiver receiver) { + Log.d(EXTENDED_LOGGING_TAG, "onRemoveTrack(" + receiver.id() + ")"); + } + }; + @Nullable private PeerConnectionFactory peerConnectionFactory = null; + @Nullable private PeerConnection peerConnection = null; + private AppRTCAudioManager appRTCAudioManager = null; + private Context context = null; + private EglBase eglBase = null; + private VideoSourceWrapper videoSourceWrapper; + + WebRTCWrapper(final EventCallback eventCallback) { + this.eventCallback = eventCallback; + } + + private static void dispose(final PeerConnection peerConnection) { + try { + peerConnection.dispose(); + } catch (final IllegalStateException e) { + Log.e(Config.LOGTAG, "unable to dispose of peer connection", e); + } + } + + public void setup( + final Context service, + @Nonnull final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) + throws InitializationException { + try { + PeerConnectionFactory.initialize( + PeerConnectionFactory.InitializationOptions.builder(service) + .setFieldTrials("WebRTC-BindUsingInterfaceName/Enabled/") + .createInitializationOptions()); + } catch (final UnsatisfiedLinkError e) { + throw new InitializationException("Unable to initialize PeerConnectionFactory", e); + } + try { + this.eglBase = EglBase.create(); + } catch (final RuntimeException e) { + throw new InitializationException("Unable to create EGL base", e); + } + this.context = service; + mainHandler.post( + () -> { + appRTCAudioManager = AppRTCAudioManager.create(service, speakerPhonePreference); + ToneManager.getInstance(context).setAppRtcAudioManagerHasControl(true); + appRTCAudioManager.start(audioManagerEvents); + eventCallback.onAudioDeviceChanged( + appRTCAudioManager.getSelectedAudioDevice(), + appRTCAudioManager.getAudioDevices()); + }); + } + + synchronized void initializePeerConnection( + final Set media, final List iceServers) + throws InitializationException { + Preconditions.checkState(this.eglBase != null); + Preconditions.checkNotNull(media); + Preconditions.checkArgument( + media.size() > 0, "media can not be empty when initializing peer connection"); + final boolean setUseHardwareAcousticEchoCanceler = + WebRtcAudioEffects.canUseAcousticEchoCanceler() + && !HARDWARE_AEC_BLACKLIST.contains(Build.MODEL); + Log.d( + Config.LOGTAG, + String.format( + "setUseHardwareAcousticEchoCanceler(%s) model=%s", + setUseHardwareAcousticEchoCanceler, Build.MODEL)); + this.peerConnectionFactory = + PeerConnectionFactory.builder() + .setVideoDecoderFactory( + new DefaultVideoDecoderFactory(eglBase.getEglBaseContext())) + .setVideoEncoderFactory( + new DefaultVideoEncoderFactory( + eglBase.getEglBaseContext(), true, true)) + .setAudioDeviceModule( + JavaAudioDeviceModule.builder(requireContext()) + .setUseHardwareAcousticEchoCanceler( + setUseHardwareAcousticEchoCanceler) + .createAudioDeviceModule()) + .createPeerConnectionFactory(); + + final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers); + final PeerConnection peerConnection = + requirePeerConnectionFactory() + .createPeerConnection(rtcConfig, peerConnectionObserver); + if (peerConnection == null) { + throw new InitializationException("Unable to create PeerConnection"); + } + + if (media.contains(Media.VIDEO)) { + addVideoTrack(peerConnection); + } + + if (media.contains(Media.AUDIO)) { + addAudioTrack(peerConnection); + } + peerConnection.setAudioPlayout(true); + peerConnection.setAudioRecording(true); + + this.peerConnection = peerConnection; + } + + private VideoSourceWrapper initializeVideoSourceWrapper() { + final VideoSourceWrapper existingVideoSourceWrapper = this.videoSourceWrapper; + if (existingVideoSourceWrapper != null) { + existingVideoSourceWrapper.startCapture(); + return existingVideoSourceWrapper; + } + final VideoSourceWrapper videoSourceWrapper = + new VideoSourceWrapper.Factory(requireContext()).create(); + if (videoSourceWrapper == null) { + throw new IllegalStateException("Could not instantiate VideoSourceWrapper"); + } + videoSourceWrapper.initialize( + requirePeerConnectionFactory(), requireContext(), eglBase.getEglBaseContext()); + videoSourceWrapper.startCapture(); + this.videoSourceWrapper = videoSourceWrapper; + return videoSourceWrapper; + } + + public synchronized boolean addTrack(final Media media) { + if (media == Media.VIDEO) { + return addVideoTrack(requirePeerConnection()); + } else if (media == Media.AUDIO) { + return addAudioTrack(requirePeerConnection()); + } + throw new IllegalStateException(String.format("Could not add track for %s", media)); + } + + public synchronized void removeTrack(final Media media) { + if (media == Media.VIDEO) { + removeVideoTrack(requirePeerConnection()); + } + } + + private boolean addAudioTrack(final PeerConnection peerConnection) { + final AudioSource audioSource = + requirePeerConnectionFactory().createAudioSource(new MediaConstraints()); + final AudioTrack audioTrack = + requirePeerConnectionFactory() + .createAudioTrack(TrackWrapper.id(AudioTrack.class), audioSource); + this.localAudioTrack = TrackWrapper.addTrack(peerConnection, audioTrack); + return true; + } + + private boolean addVideoTrack(final PeerConnection peerConnection) { + final TrackWrapper existing = this.localVideoTrack; + if (existing != null) { + final RtpTransceiver transceiver = + TrackWrapper.getTransceiver(peerConnection, existing); + if (transceiver == null) { + Log.w(EXTENDED_LOGGING_TAG, "unable to restart video transceiver"); + return false; + } + transceiver.setDirection(RtpTransceiver.RtpTransceiverDirection.SEND_RECV); + this.videoSourceWrapper.startCapture(); + return true; + } + final VideoSourceWrapper videoSourceWrapper; + try { + videoSourceWrapper = initializeVideoSourceWrapper(); + } catch (final IllegalStateException e) { + Log.d(Config.LOGTAG, "could not add video track", e); + return false; + } + final VideoTrack videoTrack = + requirePeerConnectionFactory() + .createVideoTrack( + TrackWrapper.id(VideoTrack.class), + videoSourceWrapper.getVideoSource()); + this.localVideoTrack = TrackWrapper.addTrack(peerConnection, videoTrack); + return true; + } + + private void removeVideoTrack(final PeerConnection peerConnection) { + final TrackWrapper localVideoTrack = this.localVideoTrack; + if (localVideoTrack != null) { + + final RtpTransceiver exactTransceiver = + TrackWrapper.getTransceiver(peerConnection, localVideoTrack); + if (exactTransceiver == null) { + throw new IllegalStateException(); + } + exactTransceiver.setDirection(RtpTransceiver.RtpTransceiverDirection.INACTIVE); + } + final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper; + if (videoSourceWrapper != null) { + try { + videoSourceWrapper.stopCapture(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + private static PeerConnection.RTCConfiguration buildConfiguration( + final List iceServers) { + final PeerConnection.RTCConfiguration rtcConfig = + new PeerConnection.RTCConfiguration(iceServers); + rtcConfig.tcpCandidatePolicy = + PeerConnection.TcpCandidatePolicy.DISABLED; // XEP-0176 doesn't support tcp + rtcConfig.continualGatheringPolicy = + PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; + rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; + rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE; + rtcConfig.enableImplicitRollback = true; + return rtcConfig; + } + + void reconfigurePeerConnection(final List iceServers) { + requirePeerConnection().setConfiguration(buildConfiguration(iceServers)); + } + + void restartIce() { + executorService.execute( + () -> { + final PeerConnection peerConnection; + try { + peerConnection = requirePeerConnection(); + } catch (final PeerConnectionNotInitialized e) { + Log.w( + EXTENDED_LOGGING_TAG, + "PeerConnection vanished before we could execute restart"); + return; + } + setIsReadyToReceiveIceCandidates(false); + peerConnection.restartIce(); + }); + } + + public void setIsReadyToReceiveIceCandidates(final boolean ready) { + readyToReceivedIceCandidates.set(ready); + final int was = iceCandidates.size(); + while (ready && iceCandidates.peek() != null) { + eventCallback.onIceCandidate(iceCandidates.poll()); + } + final int is = iceCandidates.size(); + Log.d( + EXTENDED_LOGGING_TAG, + "setIsReadyToReceiveCandidates(" + ready + ") was=" + was + " is=" + is); + } + + synchronized void close() { + final PeerConnection peerConnection = this.peerConnection; + final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory; + final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper; + final AppRTCAudioManager audioManager = this.appRTCAudioManager; + final EglBase eglBase = this.eglBase; + if (peerConnection != null) { + this.peerConnection = null; + dispose(peerConnection); + } + if (audioManager != null) { + ToneManager.getInstance(context).setAppRtcAudioManagerHasControl(false); + mainHandler.post(audioManager::stop); + } + this.localVideoTrack = null; + this.remoteVideoTrack = null; + if (videoSourceWrapper != null) { + this.videoSourceWrapper = null; + try { + videoSourceWrapper.stopCapture(); + } catch (final InterruptedException e) { + Log.e(Config.LOGTAG, "unable to stop capturing"); + } + videoSourceWrapper.dispose(); + } + if (eglBase != null) { + eglBase.release(); + this.eglBase = null; + } + if (peerConnectionFactory != null) { + this.peerConnectionFactory = null; + peerConnectionFactory.dispose(); + } + } + + synchronized void verifyClosed() { + if (this.peerConnection != null + || this.eglBase != null + || this.localVideoTrack != null + || this.remoteVideoTrack != null) { + final IllegalStateException e = + new IllegalStateException("WebRTCWrapper hasn't been closed properly"); + Log.e(Config.LOGTAG, "verifyClosed() failed. Going to throw", e); + throw e; + } + } + + boolean isCameraSwitchable() { + final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper; + return videoSourceWrapper != null && videoSourceWrapper.isCameraSwitchable(); + } + + boolean isFrontCamera() { + final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper; + return videoSourceWrapper == null || videoSourceWrapper.isFrontCamera(); + } + + ListenableFuture switchCamera() { + final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper; + if (videoSourceWrapper == null) { + return Futures.immediateFailedFuture( + new IllegalStateException("VideoSourceWrapper has not been initialized")); + } + return videoSourceWrapper.switchCamera(); + } + + boolean isMicrophoneEnabled() { + final Optional audioTrack = + TrackWrapper.get(peerConnection, this.localAudioTrack); + if (audioTrack.isPresent()) { + try { + return audioTrack.get().enabled(); + } catch (final IllegalStateException e) { + // sometimes UI might still be rendering the buttons when a background thread has + // already ended the call + return false; + } + } else { + throw new IllegalStateException("Local audio track does not exist (yet)"); + } + } + + boolean setMicrophoneEnabled(final boolean enabled) { + final Optional audioTrack = + TrackWrapper.get(peerConnection, this.localAudioTrack); + if (audioTrack.isPresent()) { + try { + audioTrack.get().setEnabled(enabled); + return true; + } catch (final IllegalStateException e) { + Log.d(Config.LOGTAG, "unable to toggle microphone", e); + // ignoring race condition in case MediaStreamTrack has been disposed + return false; + } + } else { + throw new IllegalStateException("Local audio track does not exist (yet)"); + } + } + + boolean isVideoEnabled() { + final Optional videoTrack = + TrackWrapper.get(peerConnection, this.localVideoTrack); + if (videoTrack.isPresent()) { + return videoTrack.get().enabled(); + } + return false; + } + + void setVideoEnabled(final boolean enabled) { + final Optional videoTrack = + TrackWrapper.get(peerConnection, this.localVideoTrack); + if (videoTrack.isPresent()) { + videoTrack.get().setEnabled(enabled); + return; + } + throw new IllegalStateException("Local video track does not exist"); + } + + synchronized ListenableFuture setLocalDescription() { + return Futures.transformAsync( + getPeerConnectionFuture(), + peerConnection -> { + if (peerConnection == null) { + return Futures.immediateFailedFuture( + new IllegalStateException("PeerConnection was null")); + } + final SettableFuture future = SettableFuture.create(); + peerConnection.setLocalDescription( + new SetSdpObserver() { + @Override + public void onSetSuccess() { + final SessionDescription description = + peerConnection.getLocalDescription(); + Log.d(EXTENDED_LOGGING_TAG, "set local description:"); + logDescription(description); + future.set(description); + } + + @Override + public void onSetFailure(final String message) { + future.setException( + new FailureToSetDescriptionException(message)); + } + }); + return future; + }, + MoreExecutors.directExecutor()); + } + + public static void logDescription(final SessionDescription sessionDescription) { + for (final String line : + sessionDescription.description.split( + eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) { + Log.d(EXTENDED_LOGGING_TAG, line); + } + } + + synchronized ListenableFuture setRemoteDescription( + final SessionDescription sessionDescription) { + Log.d(EXTENDED_LOGGING_TAG, "setting remote description:"); + logDescription(sessionDescription); + return Futures.transformAsync( + getPeerConnectionFuture(), + peerConnection -> { + if (peerConnection == null) { + return Futures.immediateFailedFuture( + new IllegalStateException("PeerConnection was null")); + } + final SettableFuture future = SettableFuture.create(); + peerConnection.setRemoteDescription( + new SetSdpObserver() { + @Override + public void onSetSuccess() { + future.set(null); + } + + @Override + public void onSetFailure(final String message) { + future.setException( + new FailureToSetDescriptionException(message)); + } + }, + sessionDescription); + return future; + }, + MoreExecutors.directExecutor()); + } + + @Nonnull + private ListenableFuture getPeerConnectionFuture() { + final PeerConnection peerConnection = this.peerConnection; + if (peerConnection == null) { + return Futures.immediateFailedFuture(new PeerConnectionNotInitialized()); + } else { + return Futures.immediateFuture(peerConnection); + } + } + + @Nonnull + private PeerConnection requirePeerConnection() { + final PeerConnection peerConnection = this.peerConnection; + if (peerConnection == null) { + throw new PeerConnectionNotInitialized(); + } + return peerConnection; + } + + @Nonnull + private PeerConnectionFactory requirePeerConnectionFactory() { + final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory; + if (peerConnectionFactory == null) { + throw new IllegalStateException("Make sure PeerConnectionFactory is initialized"); + } + return peerConnectionFactory; + } + + void addIceCandidate(IceCandidate iceCandidate) { + requirePeerConnection().addIceCandidate(iceCandidate); + } + + PeerConnection.PeerConnectionState getState() { + return requirePeerConnection().connectionState(); + } + + public PeerConnection.SignalingState getSignalingState() { + try { + return requirePeerConnection().signalingState(); + } catch (final IllegalStateException e) { + return PeerConnection.SignalingState.CLOSED; + } + } + + EglBase.Context getEglBaseContext() { + return this.eglBase.getEglBaseContext(); + } + + Optional getLocalVideoTrack() { + return TrackWrapper.get(peerConnection, this.localVideoTrack); + } + + Optional getRemoteVideoTrack() { + return Optional.fromNullable(this.remoteVideoTrack); + } + + private Context requireContext() { + final Context context = this.context; + if (context == null) { + throw new IllegalStateException("call setup first"); + } + return context; + } + + AppRTCAudioManager getAudioManager() { + return appRTCAudioManager; + } + + void execute(final Runnable command) { + executorService.execute(command); + } + + public void switchSpeakerPhonePreference(AppRTCAudioManager.SpeakerPhonePreference preference) { + mainHandler.post(() -> appRTCAudioManager.switchSpeakerPhonePreference(preference)); + } + + public interface EventCallback { + void onIceCandidate(IceCandidate iceCandidate); + + void onConnectionChange(PeerConnection.PeerConnectionState newState); + + void onAudioDeviceChanged( + AppRTCAudioManager.AudioDevice selectedAudioDevice, + Set availableAudioDevices); + + void onRenegotiationNeeded(); + } + + private abstract static class SetSdpObserver implements SdpObserver { + + @Override + public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) { + throw new IllegalStateException("Not able to use SetSdpObserver"); + } + + @Override + public void onCreateFailure(String s) { + throw new IllegalStateException("Not able to use SetSdpObserver"); + } + } + + static class InitializationException extends Exception { + + private InitializationException(final String message, final Throwable throwable) { + super(message, throwable); + } + + private InitializationException(final String message) { + super(message); + } + } + + public static class PeerConnectionNotInitialized extends IllegalStateException { + + private PeerConnectionNotInitialized() { + super("initialize PeerConnection first"); + } + } + + private static class FailureToSetDescriptionException extends IllegalArgumentException { + public FailureToSetDescriptionException(String message) { + super(message); + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Content.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Content.java new file mode 100644 index 000000000..98ef9fd0a --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Content.java @@ -0,0 +1,170 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import android.util.Log; +import androidx.annotation.NonNull; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.xmpp.jingle.SessionDescription; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import java.util.Locale; +import java.util.Set; + +public class Content extends Element { + + public Content(final Creator creator, final Senders senders, final String name) { + super("content", Namespace.JINGLE); + this.setAttribute("creator", creator.toString()); + this.setAttribute("name", name); + this.setSenders(senders); + } + + private Content() { + super("content", Namespace.JINGLE); + } + + public static Content upgrade(final Element element) { + Preconditions.checkArgument("content".equals(element.getName())); + final Content content = new Content(); + content.setAttributes(element.getAttributes()); + content.setChildren(element.getChildren()); + return content; + } + + public String getContentName() { + return this.getAttribute("name"); + } + + public Creator getCreator() { + return Creator.of(getAttribute("creator")); + } + + public Senders getSenders() { + final String attribute = getAttribute("senders"); + if (Strings.isNullOrEmpty(attribute)) { + return Senders.BOTH; + } + return Senders.of(getAttribute("senders")); + } + + public void setSenders(final Senders senders) { + if (senders != null && senders != Senders.BOTH) { + this.setAttribute("senders", senders.toString()); + } + } + + public GenericDescription getDescription() { + final Element description = this.findChild("description"); + if (description == null) { + return null; + } + final String namespace = description.getNamespace(); + if (FileTransferDescription.NAMESPACES.contains(namespace)) { + return FileTransferDescription.upgrade(description); + } else if (Namespace.JINGLE_APPS_RTP.equals(namespace)) { + return RtpDescription.upgrade(description); + } else { + return GenericDescription.upgrade(description); + } + } + + public void setDescription(final GenericDescription description) { + Preconditions.checkNotNull(description); + this.addChild(description); + } + + public String getDescriptionNamespace() { + final Element description = this.findChild("description"); + return description == null ? null : description.getNamespace(); + } + + public GenericTransportInfo getTransport() { + final Element transport = this.findChild("transport"); + final String namespace = transport == null ? null : transport.getNamespace(); + if (Namespace.JINGLE_TRANSPORTS_IBB.equals(namespace)) { + return IbbTransportInfo.upgrade(transport); + } else if (Namespace.JINGLE_TRANSPORTS_S5B.equals(namespace)) { + return S5BTransportInfo.upgrade(transport); + } else if (Namespace.JINGLE_TRANSPORT_ICE_UDP.equals(namespace)) { + return IceUdpTransportInfo.upgrade(transport); + } else if (transport != null) { + return GenericTransportInfo.upgrade(transport); + } else { + return null; + } + } + + public void setTransport(GenericTransportInfo transportInfo) { + this.addChild(transportInfo); + } + + public enum Creator { + INITIATOR, + RESPONDER; + + public static Creator of(final String value) { + return Creator.valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + @NonNull + public String toString() { + return super.toString().toLowerCase(Locale.ROOT); + } + } + + public enum Senders { + BOTH, + INITIATOR, + NONE, + RESPONDER; + + public static Senders of(final String value) { + return Senders.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static Senders of(final SessionDescription.Media media, final boolean initiator) { + final Set attributes = media.attributes.keySet(); + if (attributes.contains("sendrecv")) { + return BOTH; + } else if (attributes.contains("inactive")) { + return NONE; + } else if (attributes.contains("sendonly")) { + return initiator ? INITIATOR : RESPONDER; + } else if (attributes.contains("recvonly")) { + return initiator ? RESPONDER : INITIATOR; + } + Log.w(Config.LOGTAG, "assuming default value for senders"); + // If none of the attributes "sendonly", "recvonly", "inactive", and "sendrecv" is + // present, "sendrecv" SHOULD be assumed as the default + // https://www.rfc-editor.org/rfc/rfc4566 + return BOTH; + } + + @Override + @NonNull + public String toString() { + return super.toString().toLowerCase(Locale.ROOT); + } + + public String asMediaAttribute(final boolean initiator) { + final boolean responder = !initiator; + if (this == Content.Senders.BOTH) { + return "sendrecv"; + } else if (this == Content.Senders.NONE) { + return "inactive"; + } else if ((initiator && this == Content.Senders.INITIATOR) + || (responder && this == Content.Senders.RESPONDER)) { + return "sendonly"; + } else if ((initiator && this == Content.Senders.RESPONDER) + || (responder && this == Content.Senders.INITIATOR)) { + return "recvonly"; + } else { + throw new IllegalStateException( + String.format( + "illegal combination of initiator=%s and %s", initiator, this)); + } + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/FileTransferDescription.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/FileTransferDescription.java new file mode 100644 index 000000000..2d0e2f257 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/FileTransferDescription.java @@ -0,0 +1,69 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import com.google.common.base.Preconditions; +import im.conversations.android.xml.Element; +import java.util.Arrays; +import java.util.List; + +public class FileTransferDescription extends GenericDescription { + + public static List NAMESPACES = + Arrays.asList(Version.FT_3.namespace, Version.FT_4.namespace, Version.FT_5.namespace); + + private FileTransferDescription(String name, String namespace) { + super(name, namespace); + } + + public Version getVersion() { + final String namespace = getNamespace(); + if (namespace.equals(Version.FT_3.namespace)) { + return Version.FT_3; + } else if (namespace.equals(Version.FT_4.namespace)) { + return Version.FT_4; + } else if (namespace.equals(Version.FT_5.namespace)) { + return Version.FT_5; + } else { + throw new IllegalStateException("Unknown namespace"); + } + } + + public Element getFileOffer() { + final Version version = getVersion(); + if (version == Version.FT_3) { + final Element offer = this.findChild("offer"); + return offer == null ? null : offer.findChild("file"); + } else { + return this.findChild("file"); + } + } + + public static FileTransferDescription upgrade(final Element element) { + Preconditions.checkArgument( + "description".equals(element.getName()), + "Name of provided element is not description"); + Preconditions.checkArgument( + NAMESPACES.contains(element.getNamespace()), + "Element does not match a file transfer namespace"); + final FileTransferDescription description = + new FileTransferDescription("description", element.getNamespace()); + description.setAttributes(element.getAttributes()); + description.setChildren(element.getChildren()); + return description; + } + + public enum Version { + FT_3("urn:xmpp:jingle:apps:file-transfer:3"), + FT_4("urn:xmpp:jingle:apps:file-transfer:4"), + FT_5("urn:xmpp:jingle:apps:file-transfer:5"); + + private final String namespace; + + Version(String namespace) { + this.namespace = namespace; + } + + public String getNamespace() { + return namespace; + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/GenericDescription.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/GenericDescription.java new file mode 100644 index 000000000..a7c511da9 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/GenericDescription.java @@ -0,0 +1,20 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import com.google.common.base.Preconditions; +import im.conversations.android.xml.Element; + +public class GenericDescription extends Element { + + GenericDescription(String name, final String namespace) { + super(name, namespace); + } + + public static GenericDescription upgrade(final Element element) { + Preconditions.checkArgument("description".equals(element.getName())); + final GenericDescription description = + new GenericDescription("description", element.getNamespace()); + description.setAttributes(element.getAttributes()); + description.setChildren(element.getChildren()); + return description; + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/GenericTransportInfo.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/GenericTransportInfo.java new file mode 100644 index 000000000..6666225f2 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/GenericTransportInfo.java @@ -0,0 +1,20 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import com.google.common.base.Preconditions; +import im.conversations.android.xml.Element; + +public class GenericTransportInfo extends Element { + + protected GenericTransportInfo(String name, String xmlns) { + super(name, xmlns); + } + + public static GenericTransportInfo upgrade(final Element element) { + Preconditions.checkArgument("transport".equals(element.getName())); + final GenericTransportInfo transport = + new GenericTransportInfo("transport", element.getNamespace()); + transport.setAttributes(element.getAttributes()); + transport.setChildren(element.getChildren()); + return transport; + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Group.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Group.java new file mode 100644 index 000000000..81d686163 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Group.java @@ -0,0 +1,62 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import java.util.Collection; +import java.util.List; + +public class Group extends Element { + + private Group() { + super("group", Namespace.JINGLE_APPS_GROUPING); + } + + public Group(final String semantics, final Collection identificationTags) { + super("group", Namespace.JINGLE_APPS_GROUPING); + this.setAttribute("semantics", semantics); + for (String tag : identificationTags) { + this.addChild(new Element("content").setAttribute("name", tag)); + } + } + + public String getSemantics() { + return this.getAttribute("semantics"); + } + + public List getIdentificationTags() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : this.children) { + if ("content".equals(child.getName())) { + final String name = child.getAttribute("name"); + if (name != null) { + builder.add(name); + } + } + } + return builder.build(); + } + + public static Group ofSdpString(final String input) { + ImmutableList.Builder tagBuilder = new ImmutableList.Builder<>(); + final String[] parts = input.split(" "); + if (parts.length >= 2) { + final String semantics = parts[0]; + for (int i = 1; i < parts.length; ++i) { + tagBuilder.add(parts[i]); + } + return new Group(semantics, tagBuilder.build()); + } + return null; + } + + public static Group upgrade(final Element element) { + Preconditions.checkArgument("group".equals(element.getName())); + Preconditions.checkArgument(Namespace.JINGLE_APPS_GROUPING.equals(element.getNamespace())); + final Group group = new Group(); + group.setAttributes(element.getAttributes()); + group.setChildren(element.getChildren()); + return group; + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IbbTransportInfo.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IbbTransportInfo.java new file mode 100644 index 000000000..2ee136be7 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IbbTransportInfo.java @@ -0,0 +1,49 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import com.google.common.base.Preconditions; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; + +public class IbbTransportInfo extends GenericTransportInfo { + + private IbbTransportInfo(final String name, final String xmlns) { + super(name, xmlns); + } + + public IbbTransportInfo(final String transportId, final int blockSize) { + super("transport", Namespace.JINGLE_TRANSPORTS_IBB); + Preconditions.checkNotNull(transportId, "Transport ID can not be null"); + Preconditions.checkArgument(blockSize > 0, "Block size must be larger than 0"); + this.setAttribute("block-size", blockSize); + this.setAttribute("sid", transportId); + } + + public String getTransportId() { + return this.getAttribute("sid"); + } + + public int getBlockSize() { + final String blockSize = this.getAttribute("block-size"); + if (blockSize == null) { + return 0; + } + try { + return Integer.parseInt(blockSize); + } catch (NumberFormatException e) { + return 0; + } + } + + public static IbbTransportInfo upgrade(final Element element) { + Preconditions.checkArgument( + "transport".equals(element.getName()), "Name of provided element is not transport"); + Preconditions.checkArgument( + Namespace.JINGLE_TRANSPORTS_IBB.equals(element.getNamespace()), + "Element does not match ibb transport namespace"); + final IbbTransportInfo transportInfo = + new IbbTransportInfo("transport", Namespace.JINGLE_TRANSPORTS_IBB); + transportInfo.setAttributes(element.getAttributes()); + transportInfo.setChildren(element.getChildren()); + return transportInfo; + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java new file mode 100644 index 000000000..abd4e83ff --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java @@ -0,0 +1,411 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import androidx.annotation.NonNull; +import com.google.common.base.Joiner; +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import eu.siacs.conversations.xmpp.jingle.SessionDescription; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; + +public class IceUdpTransportInfo extends GenericTransportInfo { + + public static final IceUdpTransportInfo STUB = new IceUdpTransportInfo(); + + public IceUdpTransportInfo() { + super("transport", Namespace.JINGLE_TRANSPORT_ICE_UDP); + } + + public static IceUdpTransportInfo upgrade(final Element element) { + Preconditions.checkArgument( + "transport".equals(element.getName()), "Name of provided element is not transport"); + Preconditions.checkArgument( + Namespace.JINGLE_TRANSPORT_ICE_UDP.equals(element.getNamespace()), + "Element does not match ice-udp transport namespace"); + final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); + transportInfo.setAttributes(element.getAttributes()); + transportInfo.setChildren(element.getChildren()); + return transportInfo; + } + + public static IceUdpTransportInfo of( + SessionDescription sessionDescription, SessionDescription.Media media) { + final String ufrag = Iterables.getFirst(media.attributes.get("ice-ufrag"), null); + final String pwd = Iterables.getFirst(media.attributes.get("ice-pwd"), null); + final IceUdpTransportInfo iceUdpTransportInfo = new IceUdpTransportInfo(); + if (ufrag != null) { + iceUdpTransportInfo.setAttribute("ufrag", ufrag); + } + if (pwd != null) { + iceUdpTransportInfo.setAttribute("pwd", pwd); + } + final Fingerprint fingerprint = Fingerprint.of(sessionDescription, media); + if (fingerprint != null) { + iceUdpTransportInfo.addChild(fingerprint); + } + return iceUdpTransportInfo; + } + + public static IceUdpTransportInfo of( + final Credentials credentials, + final Setup setup, + final String hash, + final String fingerprint) { + final IceUdpTransportInfo iceUdpTransportInfo = new IceUdpTransportInfo(); + iceUdpTransportInfo.addChild(Fingerprint.of(setup, hash, fingerprint)); + iceUdpTransportInfo.setAttribute("ufrag", credentials.ufrag); + iceUdpTransportInfo.setAttribute("pwd", credentials.password); + return iceUdpTransportInfo; + } + + public Fingerprint getFingerprint() { + final Element fingerprint = this.findChild("fingerprint", Namespace.JINGLE_APPS_DTLS); + return fingerprint == null ? null : Fingerprint.upgrade(fingerprint); + } + + public Credentials getCredentials() { + final String ufrag = this.getAttribute("ufrag"); + final String password = this.getAttribute("pwd"); + return new Credentials(ufrag, password); + } + + public List getCandidates() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : getChildren()) { + if ("candidate".equals(child.getName())) { + builder.add(Candidate.upgrade(child)); + } + } + return builder.build(); + } + + public IceUdpTransportInfo cloneWrapper() { + final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); + transportInfo.setAttributes(new Hashtable<>(getAttributes())); + return transportInfo; + } + + public IceUdpTransportInfo modifyCredentials(final Credentials credentials, final Setup setup) { + final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); + transportInfo.setAttribute("ufrag", credentials.ufrag); + transportInfo.setAttribute("pwd", credentials.password); + for (final Element child : getChildren()) { + if (child.getName().equals("fingerprint") + && Namespace.JINGLE_APPS_DTLS.equals(child.getNamespace())) { + final Fingerprint fingerprint = new Fingerprint(); + fingerprint.setAttributes(new Hashtable<>(child.getAttributes())); + fingerprint.setContent(child.getContent()); + fingerprint.setAttribute("setup", setup.toString().toLowerCase(Locale.ROOT)); + transportInfo.addChild(fingerprint); + } + } + return transportInfo; + } + + public static class Credentials { + public final String ufrag; + public final String password; + + public Credentials(String ufrag, String password) { + this.ufrag = ufrag; + this.password = password; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Credentials that = (Credentials) o; + return Objects.equal(ufrag, that.ufrag) && Objects.equal(password, that.password); + } + + @Override + public int hashCode() { + return Objects.hashCode(ufrag, password); + } + + @Override + @NonNull + public String toString() { + return MoreObjects.toStringHelper(this) + .add("ufrag", ufrag) + .add("password", password) + .toString(); + } + } + + public static class Candidate extends Element { + + private Candidate() { + super("candidate"); + } + + public static Candidate upgrade(final Element element) { + Preconditions.checkArgument("candidate".equals(element.getName())); + final Candidate candidate = new Candidate(); + candidate.setAttributes(element.getAttributes()); + candidate.setChildren(element.getChildren()); + return candidate; + } + + // https://tools.ietf.org/html/draft-ietf-mmusic-ice-sip-sdp-39#section-5.1 + public static Candidate fromSdpAttribute(final String attribute, String currentUfrag) { + final String[] pair = attribute.split(":", 2); + if (pair.length == 2 && "candidate".equals(pair[0])) { + final String[] segments = pair[1].split(" "); + if (segments.length >= 6) { + final String id = UUID.randomUUID().toString(); + final String foundation = segments[0]; + final String component = segments[1]; + final String transport = segments[2].toLowerCase(Locale.ROOT); + final String priority = segments[3]; + final String connectionAddress = segments[4]; + final String port = segments[5]; + final HashMap additional = new HashMap<>(); + for (int i = 6; i < segments.length - 1; i = i + 2) { + additional.put(segments[i], segments[i + 1]); + } + final String ufrag = additional.get("ufrag"); + if (ufrag != null && !ufrag.equals(currentUfrag)) { + return null; + } + final Candidate candidate = new Candidate(); + candidate.setAttribute("component", component); + candidate.setAttribute("foundation", foundation); + candidate.setAttribute("generation", additional.get("generation")); + candidate.setAttribute("rel-addr", additional.get("raddr")); + candidate.setAttribute("rel-port", additional.get("rport")); + candidate.setAttribute("id", id); + candidate.setAttribute("ip", connectionAddress); + candidate.setAttribute("port", port); + candidate.setAttribute("priority", priority); + candidate.setAttribute("protocol", transport); + candidate.setAttribute("type", additional.get("typ")); + return candidate; + } + } + return null; + } + + public int getComponent() { + return getAttributeAsInt("component"); + } + + public int getFoundation() { + return getAttributeAsInt("foundation"); + } + + public int getGeneration() { + return getAttributeAsInt("generation"); + } + + public String getId() { + return getAttribute("id"); + } + + public String getIp() { + return getAttribute("ip"); + } + + public int getNetwork() { + return getAttributeAsInt("network"); + } + + public int getPort() { + return getAttributeAsInt("port"); + } + + public int getPriority() { + return getAttributeAsInt("priority"); + } + + public String getProtocol() { + return getAttribute("protocol"); + } + + public String getRelAddr() { + return getAttribute("rel-addr"); + } + + public int getRelPort() { + return getAttributeAsInt("rel-port"); + } + + public String getType() { // TODO might be converted to enum + return getAttribute("type"); + } + + private int getAttributeAsInt(final String name) { + final String value = this.getAttribute(name); + if (value == null) { + return 0; + } + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return 0; + } + } + + public String toSdpAttribute(final String ufrag) { + final String foundation = this.getAttribute("foundation"); + checkNotNullNoWhitespace(foundation, "foundation"); + final String component = this.getAttribute("component"); + checkNotNullNoWhitespace(component, "component"); + final String protocol = this.getAttribute("protocol"); + checkNotNullNoWhitespace(protocol, "protocol"); + final String transport = protocol.toLowerCase(Locale.ROOT); + if (!"udp".equals(transport)) { + throw new IllegalArgumentException( + String.format("'%s' is not a supported protocol", transport)); + } + final String priority = this.getAttribute("priority"); + checkNotNullNoWhitespace(priority, "priority"); + final String connectionAddress = this.getAttribute("ip"); + checkNotNullNoWhitespace(connectionAddress, "ip"); + final String port = this.getAttribute("port"); + checkNotNullNoWhitespace(port, "port"); + final Map additionalParameter = new LinkedHashMap<>(); + final String relAddr = this.getAttribute("rel-addr"); + final String type = this.getAttribute("type"); + if (type != null) { + additionalParameter.put("typ", type); + } + if (relAddr != null) { + additionalParameter.put("raddr", relAddr); + } + final String relPort = this.getAttribute("rel-port"); + if (relPort != null) { + additionalParameter.put("rport", relPort); + } + final String generation = this.getAttribute("generation"); + if (generation != null) { + additionalParameter.put("generation", generation); + } + if (ufrag != null) { + additionalParameter.put("ufrag", ufrag); + } + final String parametersString = + Joiner.on(' ') + .join( + Collections2.transform( + additionalParameter.entrySet(), + input -> + String.format( + "%s %s", + input.getKey(), input.getValue()))); + return String.format( + "candidate:%s %s %s %s %s %s %s", + foundation, + component, + transport, + priority, + connectionAddress, + port, + parametersString); + } + } + + private static void checkNotNullNoWhitespace(final String value, final String name) { + if (Strings.isNullOrEmpty(value)) { + throw new IllegalArgumentException( + String.format("Parameter %s is missing or empty", name)); + } + SessionDescription.checkNoWhitespace( + value, String.format("Parameter %s contains white spaces", name)); + } + + public static class Fingerprint extends Element { + + private Fingerprint() { + super("fingerprint", Namespace.JINGLE_APPS_DTLS); + } + + public static Fingerprint upgrade(final Element element) { + Preconditions.checkArgument("fingerprint".equals(element.getName())); + Preconditions.checkArgument(Namespace.JINGLE_APPS_DTLS.equals(element.getNamespace())); + final Fingerprint fingerprint = new Fingerprint(); + fingerprint.setAttributes(element.getAttributes()); + fingerprint.setContent(element.getContent()); + return fingerprint; + } + + private static Fingerprint of(ArrayListMultimap attributes) { + final String fingerprint = Iterables.getFirst(attributes.get("fingerprint"), null); + final String setup = Iterables.getFirst(attributes.get("setup"), null); + if (setup != null && fingerprint != null) { + final String[] fingerprintParts = fingerprint.split(" ", 2); + if (fingerprintParts.length == 2) { + final String hash = fingerprintParts[0]; + final String actualFingerprint = fingerprintParts[1]; + final Fingerprint element = new Fingerprint(); + element.setAttribute("hash", hash); + element.setAttribute("setup", setup); + element.setContent(actualFingerprint); + return element; + } + } + return null; + } + + public static Fingerprint of( + final SessionDescription sessionDescription, final SessionDescription.Media media) { + final Fingerprint fingerprint = of(media.attributes); + return fingerprint == null ? of(sessionDescription.attributes) : fingerprint; + } + + private static Fingerprint of(final Setup setup, final String hash, final String content) { + final Fingerprint fingerprint = new Fingerprint(); + fingerprint.setContent(content); + fingerprint.setAttribute("hash", hash); + fingerprint.setAttribute("setup", setup.toString().toLowerCase(Locale.ROOT)); + return fingerprint; + } + + public String getHash() { + return this.getAttribute("hash"); + } + + public Setup getSetup() { + final String setup = this.getAttribute("setup"); + return setup == null ? null : Setup.of(setup); + } + } + + public enum Setup { + ACTPASS, + PASSIVE, + ACTIVE; + + public static Setup of(String setup) { + try { + return valueOf(setup.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + return null; + } + } + + public Setup flip() { + if (this == PASSIVE) { + return ACTIVE; + } + if (this == ACTIVE) { + return PASSIVE; + } + throw new IllegalStateException(this.name() + " can not be flipped"); + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/JinglePacket.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/JinglePacket.java new file mode 100644 index 000000000..8d2dfd8c9 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/JinglePacket.java @@ -0,0 +1,162 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import androidx.annotation.NonNull; +import com.google.common.base.CaseFormat; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import im.conversations.android.xmpp.model.stanza.Iq; +import java.util.Map; +import org.jxmpp.jid.Jid; + +public class JinglePacket extends Iq { + + private JinglePacket() { + super(); + } + + public JinglePacket(final Action action, final String sessionId) { + super(Iq.Type.SET); + final Element jingle = addChild("jingle", Namespace.JINGLE); + jingle.setAttribute("sid", sessionId); + jingle.setAttribute("action", action.toString()); + } + + public static JinglePacket upgrade(final Iq iqPacket) { + Preconditions.checkArgument(iqPacket.hasChild("jingle", Namespace.JINGLE)); + Preconditions.checkArgument(iqPacket.getType() == Iq.Type.SET); + final JinglePacket jinglePacket = new JinglePacket(); + jinglePacket.setAttributes(iqPacket.getAttributes()); + jinglePacket.setChildren(iqPacket.getChildren()); + return jinglePacket; + } + + // TODO deprecate this somehow and make file transfer fail if there are multiple (or something) + public Content getJingleContent() { + final Element content = getJingleChild("content"); + return content == null ? null : Content.upgrade(content); + } + + public Group getGroup() { + final Element jingle = findChild("jingle", Namespace.JINGLE); + final Element group = jingle.findChild("group", Namespace.JINGLE_APPS_GROUPING); + return group == null ? null : Group.upgrade(group); + } + + public void addGroup(final Group group) { + this.addJingleChild(group); + } + + public Map getJingleContents() { + final Element jingle = findChild("jingle", Namespace.JINGLE); + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + for (final Element child : jingle.getChildren()) { + if ("content".equals(child.getName())) { + final Content content = Content.upgrade(child); + builder.put(content.getContentName(), content); + } + } + return builder.build(); + } + + public void addJingleContent(final Content content) { // take content interface + addJingleChild(content); + } + + public ReasonWrapper getReason() { + final Element reasonElement = getJingleChild("reason"); + if (reasonElement == null) { + return new ReasonWrapper(Reason.UNKNOWN, null); + } + String text = null; + Reason reason = Reason.UNKNOWN; + for (Element child : reasonElement.getChildren()) { + if ("text".equals(child.getName())) { + text = child.getContent(); + } else { + reason = Reason.of(child.getName()); + } + } + return new ReasonWrapper(reason, text); + } + + public void setReason(final Reason reason, final String text) { + final Element jingle = findChild("jingle", Namespace.JINGLE); + final Element reasonElement = jingle.addChild("reason"); + reasonElement.addChild(reason.toString()); + if (!Strings.isNullOrEmpty(text)) { + reasonElement.addChild("text").setContent(text); + } + } + + // RECOMMENDED for session-initiate, NOT RECOMMENDED otherwise + public void setInitiator(final Jid initiator) { + Preconditions.checkArgument(initiator.isEntityFullJid(), "initiator should be a full JID"); + findChild("jingle", Namespace.JINGLE).setAttribute("initiator", initiator); + } + + // RECOMMENDED for session-accept, NOT RECOMMENDED otherwise + public void setResponder(Jid responder) { + Preconditions.checkArgument(responder.isEntityFullJid(), "responder should be a full JID"); + findChild("jingle", Namespace.JINGLE).setAttribute("responder", responder); + } + + public Element getJingleChild(final String name) { + final Element jingle = findChild("jingle", Namespace.JINGLE); + return jingle == null ? null : jingle.findChild(name); + } + + public void addJingleChild(final Element child) { + final Element jingle = findChild("jingle", Namespace.JINGLE); + jingle.addChild(child); + } + + public String getSessionId() { + return findChild("jingle", Namespace.JINGLE).getAttribute("sid"); + } + + public Action getAction() { + return Action.of(findChild("jingle", Namespace.JINGLE).getAttribute("action")); + } + + public enum Action { + CONTENT_ACCEPT, + CONTENT_ADD, + CONTENT_MODIFY, + CONTENT_REJECT, + CONTENT_REMOVE, + DESCRIPTION_INFO, + SECURITY_INFO, + SESSION_ACCEPT, + SESSION_INFO, + SESSION_INITIATE, + SESSION_TERMINATE, + TRANSPORT_ACCEPT, + TRANSPORT_INFO, + TRANSPORT_REJECT, + TRANSPORT_REPLACE; + + public static Action of(final String value) { + // TODO handle invalid + return Action.valueOf(CaseFormat.LOWER_HYPHEN.to(CaseFormat.UPPER_UNDERSCORE, value)); + } + + @Override + @NonNull + public String toString() { + return CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.LOWER_HYPHEN, super.toString()); + } + } + + public static class ReasonWrapper { + public final Reason reason; + public final String text; + + public ReasonWrapper(Reason reason, String text) { + this.reason = reason; + this.text = text; + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/OmemoVerifiedIceUdpTransportInfo.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/OmemoVerifiedIceUdpTransportInfo.java new file mode 100644 index 000000000..4d2ec20fc --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/OmemoVerifiedIceUdpTransportInfo.java @@ -0,0 +1,27 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import im.conversations.android.xml.Namespace; + +public class OmemoVerifiedIceUdpTransportInfo extends IceUdpTransportInfo { + + public void ensureNoPlaintextFingerprint() { + if (this.findChild("fingerprint", Namespace.JINGLE_APPS_DTLS) != null) { + throw new IllegalStateException( + "OmemoVerifiedIceUdpTransportInfo contains plaintext fingerprint"); + } + } + + public static IceUdpTransportInfo upgrade(final IceUdpTransportInfo transportInfo) { + if (transportInfo.hasChild("fingerprint", Namespace.JINGLE_APPS_DTLS)) { + return transportInfo; + } + if (transportInfo.hasChild("fingerprint", Namespace.OMEMO_DTLS_SRTP_VERIFICATION)) { + final OmemoVerifiedIceUdpTransportInfo omemoVerifiedIceUdpTransportInfo = + new OmemoVerifiedIceUdpTransportInfo(); + omemoVerifiedIceUdpTransportInfo.setAttributes(transportInfo.getAttributes()); + omemoVerifiedIceUdpTransportInfo.setChildren(transportInfo.getChildren()); + return omemoVerifiedIceUdpTransportInfo; + } + return transportInfo; + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Reason.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Reason.java new file mode 100644 index 000000000..cb61c4b91 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Reason.java @@ -0,0 +1,65 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import androidx.annotation.NonNull; +import com.google.common.base.CaseFormat; +import com.google.common.base.Throwables; +import eu.siacs.conversations.xmpp.jingle.RtpContentMap; +import im.conversations.android.axolotl.AxolotlEncryptionException; + +public enum Reason { + ALTERNATIVE_SESSION, + BUSY, + CANCEL, + CONNECTIVITY_ERROR, + DECLINE, + EXPIRED, + FAILED_APPLICATION, + FAILED_TRANSPORT, + GENERAL_ERROR, + GONE, + INCOMPATIBLE_PARAMETERS, + MEDIA_ERROR, + SECURITY_ERROR, + SUCCESS, + TIMEOUT, + UNSUPPORTED_APPLICATIONS, + UNSUPPORTED_TRANSPORTS, + UNKNOWN; + + public static Reason of(final String value) { + try { + return Reason.valueOf(CaseFormat.LOWER_HYPHEN.to(CaseFormat.UPPER_UNDERSCORE, value)); + } catch (Exception e) { + return UNKNOWN; + } + } + + @Override + @NonNull + public String toString() { + return CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.LOWER_HYPHEN, super.toString()); + } + + public static Reason of(final RuntimeException e) { + if (e instanceof SecurityException) { + return SECURITY_ERROR; + } else if (e instanceof RtpContentMap.UnsupportedTransportException) { + return UNSUPPORTED_TRANSPORTS; + } else if (e instanceof RtpContentMap.UnsupportedApplicationException) { + return UNSUPPORTED_APPLICATIONS; + } else { + return FAILED_APPLICATION; + } + } + + public static Reason ofThrowable(final Throwable throwable) { + final Throwable root = Throwables.getRootCause(throwable); + if (root instanceof RuntimeException) { + return of((RuntimeException) root); + } + if (root instanceof AxolotlEncryptionException) { + return SECURITY_ERROR; + } + return FAILED_APPLICATION; + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java new file mode 100644 index 000000000..9e1ca65a1 --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java @@ -0,0 +1,658 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import android.util.Pair; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import eu.siacs.conversations.xmpp.jingle.Media; +import eu.siacs.conversations.xmpp.jingle.SessionDescription; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RtpDescription extends GenericDescription { + + private RtpDescription(final String media) { + super("description", Namespace.JINGLE_APPS_RTP); + this.setAttribute("media", media); + } + + private RtpDescription() { + super("description", Namespace.JINGLE_APPS_RTP); + } + + public static RtpDescription stub(final Media media) { + return new RtpDescription(media.toString()); + } + + public Media getMedia() { + return Media.of(this.getAttribute("media")); + } + + public List getPayloadTypes() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (Element child : getChildren()) { + if ("payload-type".equals(child.getName())) { + builder.add(PayloadType.of(child)); + } + } + return builder.build(); + } + + public List getFeedbackNegotiations() { + return FeedbackNegotiation.fromChildren(this.getChildren()); + } + + public List feedbackNegotiationTrrInts() { + return FeedbackNegotiationTrrInt.fromChildren(this.getChildren()); + } + + public List getHeaderExtensions() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : getChildren()) { + if ("rtp-hdrext".equals(child.getName()) + && Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(child.getNamespace())) { + builder.add(RtpHeaderExtension.upgrade(child)); + } + } + return builder.build(); + } + + public List getSources() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : this.children) { + if ("source".equals(child.getName()) + && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals( + child.getNamespace())) { + builder.add(Source.upgrade(child)); + } + } + return builder.build(); + } + + public List getSourceGroups() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : this.children) { + if ("ssrc-group".equals(child.getName()) + && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals( + child.getNamespace())) { + builder.add(SourceGroup.upgrade(child)); + } + } + return builder.build(); + } + + public static RtpDescription upgrade(final Element element) { + Preconditions.checkArgument( + "description".equals(element.getName()), + "Name of provided element is not description"); + Preconditions.checkArgument( + Namespace.JINGLE_APPS_RTP.equals(element.getNamespace()), + "Element does not match the jingle rtp namespace"); + final RtpDescription description = new RtpDescription(); + description.setAttributes(element.getAttributes()); + description.setChildren(element.getChildren()); + return description; + } + + public static class FeedbackNegotiation extends Element { + private FeedbackNegotiation() { + super("rtcp-fb", Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION); + } + + public FeedbackNegotiation(String type, String subType) { + super("rtcp-fb", Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION); + this.setAttribute("type", type); + if (subType != null) { + this.setAttribute("subtype", subType); + } + } + + public String getType() { + return this.getAttribute("type"); + } + + public String getSubType() { + return this.getAttribute("subtype"); + } + + private static FeedbackNegotiation upgrade(final Element element) { + Preconditions.checkArgument("rtcp-fb".equals(element.getName())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace())); + final FeedbackNegotiation feedback = new FeedbackNegotiation(); + feedback.setAttributes(element.getAttributes()); + feedback.setChildren(element.getChildren()); + return feedback; + } + + public static List fromChildren(final List children) { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : children) { + if ("rtcp-fb".equals(child.getName()) + && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) { + builder.add(upgrade(child)); + } + } + return builder.build(); + } + } + + public static class FeedbackNegotiationTrrInt extends Element { + + private FeedbackNegotiationTrrInt(int value) { + super("rtcp-fb-trr-int", Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION); + this.setAttribute("value", value); + } + + private FeedbackNegotiationTrrInt() { + super("rtcp-fb-trr-int", Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION); + } + + public int getValue() { + final String value = getAttribute("value"); + return Integer.parseInt(value); + } + + private static FeedbackNegotiationTrrInt upgrade(final Element element) { + Preconditions.checkArgument("rtcp-fb-trr-int".equals(element.getName())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace())); + final FeedbackNegotiationTrrInt trr = new FeedbackNegotiationTrrInt(); + trr.setAttributes(element.getAttributes()); + trr.setChildren(element.getChildren()); + return trr; + } + + public static List fromChildren(final List children) { + ImmutableList.Builder builder = + new ImmutableList.Builder<>(); + for (final Element child : children) { + if ("rtcp-fb-trr-int".equals(child.getName()) + && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) { + builder.add(upgrade(child)); + } + } + return builder.build(); + } + } + + // XEP-0294: Jingle RTP Header Extensions Negotiation + // maps to `extmap:$id $uri` + public static class RtpHeaderExtension extends Element { + + private RtpHeaderExtension() { + super("rtp-hdrext", Namespace.JINGLE_RTP_HEADER_EXTENSIONS); + } + + public RtpHeaderExtension(String id, String uri) { + super("rtp-hdrext", Namespace.JINGLE_RTP_HEADER_EXTENSIONS); + this.setAttribute("id", id); + this.setAttribute("uri", uri); + } + + public String getId() { + return this.getAttribute("id"); + } + + public String getUri() { + return this.getAttribute("uri"); + } + + public static RtpHeaderExtension upgrade(final Element element) { + Preconditions.checkArgument("rtp-hdrext".equals(element.getName())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(element.getNamespace())); + final RtpHeaderExtension extension = new RtpHeaderExtension(); + extension.setAttributes(element.getAttributes()); + extension.setChildren(element.getChildren()); + return extension; + } + + public static RtpHeaderExtension ofSdpString(final String sdp) { + final String[] pair = sdp.split(" ", 2); + if (pair.length == 2) { + final String id = pair[0]; + final String uri = pair[1]; + return new RtpHeaderExtension(id, uri); + } else { + return null; + } + } + } + + // maps to `rtpmap:$id $name/$clockrate/$channels` + public static class PayloadType extends Element { + + private PayloadType() { + super("payload-type", Namespace.JINGLE_APPS_RTP); + } + + public PayloadType(String id, String name, int clockRate, int channels) { + super("payload-type", Namespace.JINGLE_APPS_RTP); + this.setAttribute("id", id); + this.setAttribute("name", name); + this.setAttribute("clockrate", clockRate); + if (channels != 1) { + this.setAttribute("channels", channels); + } + } + + public String toSdpAttribute() { + final int channels = getChannels(); + final String name = getPayloadTypeName(); + Preconditions.checkArgument(name != null, "Payload-type name must not be empty"); + SessionDescription.checkNoWhitespace( + name, "payload-type name must not contain whitespaces"); + return getId() + + " " + + name + + "/" + + getClockRate() + + (channels == 1 ? "" : "/" + channels); + } + + public int getIntId() { + final String id = this.getAttribute("id"); + return id == null ? 0 : SessionDescription.ignorantIntParser(id); + } + + public String getId() { + return this.getAttribute("id"); + } + + public String getPayloadTypeName() { + return this.getAttribute("name"); + } + + public int getClockRate() { + final String clockRate = this.getAttribute("clockrate"); + if (clockRate == null) { + return 0; + } + try { + return Integer.parseInt(clockRate); + } catch (NumberFormatException e) { + return 0; + } + } + + public int getChannels() { + final String channels = this.getAttribute("channels"); + if (channels == null) { + return 1; // The number of channels; if omitted, it MUST be assumed to contain one + // channel + } + try { + return Integer.parseInt(channels); + } catch (NumberFormatException e) { + return 1; + } + } + + public List getParameters() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (Element child : getChildren()) { + if ("parameter".equals(child.getName())) { + builder.add(Parameter.of(child)); + } + } + return builder.build(); + } + + public List getFeedbackNegotiations() { + return FeedbackNegotiation.fromChildren(this.getChildren()); + } + + public List feedbackNegotiationTrrInts() { + return FeedbackNegotiationTrrInt.fromChildren(this.getChildren()); + } + + public static PayloadType of(final Element element) { + Preconditions.checkArgument( + "payload-type".equals(element.getName()), + "element name must be called payload-type"); + PayloadType payloadType = new PayloadType(); + payloadType.setAttributes(element.getAttributes()); + payloadType.setChildren(element.getChildren()); + return payloadType; + } + + public static PayloadType ofSdpString(final String sdp) { + final String[] pair = sdp.split(" ", 2); + if (pair.length == 2) { + final String id = pair[0]; + final String[] parts = pair[1].split("/"); + if (parts.length >= 2) { + final String name = parts[0]; + final int clockRate = SessionDescription.ignorantIntParser(parts[1]); + final int channels; + if (parts.length >= 3) { + channels = SessionDescription.ignorantIntParser(parts[2]); + } else { + channels = 1; + } + return new PayloadType(id, name, clockRate, channels); + } + } + return null; + } + + public void addChildren(final List children) { + if (children != null) { + this.children.addAll(children); + } + } + + public void addParameters(List parameters) { + if (parameters != null) { + this.children.addAll(parameters); + } + } + } + + // map to `fmtp $id key=value;key=value + // where id is the id of the parent payload-type + public static class Parameter extends Element { + + private Parameter() { + super("parameter", Namespace.JINGLE_APPS_RTP); + } + + public Parameter(String name, String value) { + super("parameter", Namespace.JINGLE_APPS_RTP); + this.setAttribute("name", name); + this.setAttribute("value", value); + } + + public String getParameterName() { + return this.getAttribute("name"); + } + + public String getParameterValue() { + return this.getAttribute("value"); + } + + public static Parameter of(final Element element) { + Preconditions.checkArgument( + "parameter".equals(element.getName()), "element name must be called parameter"); + Parameter parameter = new Parameter(); + parameter.setAttributes(element.getAttributes()); + parameter.setChildren(element.getChildren()); + return parameter; + } + + public static String toSdpString(final String id, List parameters) { + final StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(id).append(' '); + for (int i = 0; i < parameters.size(); ++i) { + final Parameter p = parameters.get(i); + final String name = p.getParameterName(); + Preconditions.checkArgument( + name != null, String.format("parameter for %s must have a name", id)); + SessionDescription.checkNoWhitespace( + name, + String.format("parameter names for %s must not contain whitespaces", id)); + + final String value = p.getParameterValue(); + Preconditions.checkArgument( + value != null, String.format("parameter for %s must have a value", id)); + SessionDescription.checkNoWhitespace( + value, + String.format("parameter values for %s must not contain whitespaces", id)); + + stringBuilder.append(name).append('=').append(value); + if (i != parameters.size() - 1) { + stringBuilder.append(';'); + } + } + return stringBuilder.toString(); + } + + public static String toSdpString(final String id, final Parameter parameter) { + final String name = parameter.getParameterName(); + final String value = parameter.getParameterValue(); + Preconditions.checkArgument( + value != null, String.format("parameter for %s must have a value", id)); + SessionDescription.checkNoWhitespace( + value, + String.format("parameter values for %s must not contain whitespaces", id)); + if (Strings.isNullOrEmpty(name)) { + return String.format("%s %s", id, value); + } else { + return String.format("%s %s=%s", id, name, value); + } + } + + static Pair> ofSdpString(final String sdp) { + final String[] pair = sdp.split(" "); + if (pair.length == 2) { + final String id = pair[0]; + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final String parameter : pair[1].split(";")) { + final String[] parts = parameter.split("=", 2); + if (parts.length == 2) { + builder.add(new Parameter(parts[0], parts[1])); + } + } + return new Pair<>(id, builder.build()); + } else { + return null; + } + } + } + + // XEP-0339: Source-Specific Media Attributes in Jingle + // maps to `a=ssrc: :` + public static class Source extends Element { + + private Source() { + super("source", Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES); + } + + public Source(String ssrcId, Collection parameters) { + super("source", Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES); + this.setAttribute("ssrc", ssrcId); + for (Parameter parameter : parameters) { + this.addChild(parameter); + } + } + + public String getSsrcId() { + return this.getAttribute("ssrc"); + } + + public List getParameters() { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (Element child : this.children) { + if ("parameter".equals(child.getName())) { + builder.add(Parameter.upgrade(child)); + } + } + return builder.build(); + } + + public static Source upgrade(final Element element) { + Preconditions.checkArgument("source".equals(element.getName())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals( + element.getNamespace())); + final Source source = new Source(); + source.setChildren(element.getChildren()); + source.setAttributes(element.getAttributes()); + return source; + } + + public static class Parameter extends Element { + + public String getParameterName() { + return this.getAttribute("name"); + } + + public String getParameterValue() { + return this.getAttribute("value"); + } + + private Parameter() { + super("parameter"); + } + + public Parameter(final String attribute, final String value) { + super("parameter"); + this.setAttribute("name", attribute); + if (value != null) { + this.setAttribute("value", value); + } + } + + public static Parameter upgrade(final Element element) { + Preconditions.checkArgument("parameter".equals(element.getName())); + Parameter parameter = new Parameter(); + parameter.setAttributes(element.getAttributes()); + parameter.setChildren(element.getChildren()); + return parameter; + } + } + } + + public static class SourceGroup extends Element { + + public SourceGroup(final String semantics, List ssrcs) { + this(); + this.setAttribute("semantics", semantics); + for (String ssrc : ssrcs) { + this.addChild("source").setAttribute("ssrc", ssrc); + } + } + + private SourceGroup() { + super("ssrc-group", Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES); + } + + public String getSemantics() { + return this.getAttribute("semantics"); + } + + public List getSsrcs() { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (Element child : this.children) { + if ("source".equals(child.getName())) { + final String ssrc = child.getAttribute("ssrc"); + if (ssrc != null) { + builder.add(ssrc); + } + } + } + return builder.build(); + } + + public static SourceGroup upgrade(final Element element) { + Preconditions.checkArgument("ssrc-group".equals(element.getName())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals( + element.getNamespace())); + final SourceGroup group = new SourceGroup(); + group.setChildren(element.getChildren()); + group.setAttributes(element.getAttributes()); + return group; + } + } + + public static RtpDescription of( + final SessionDescription sessionDescription, final SessionDescription.Media media) { + final RtpDescription rtpDescription = new RtpDescription(media.media); + final Map> parameterMap = new HashMap<>(); + final ArrayListMultimap feedbackNegotiationMap = + ArrayListMultimap.create(); + final ArrayListMultimap sourceParameterMap = + ArrayListMultimap.create(); + final Set attributes = + Sets.newHashSet( + Iterables.concat( + sessionDescription.attributes.keySet(), media.attributes.keySet())); + for (final String rtcpFb : media.attributes.get("rtcp-fb")) { + final String[] parts = rtcpFb.split(" "); + if (parts.length >= 2) { + final String id = parts[0]; + final String type = parts[1]; + final String subType = parts.length >= 3 ? parts[2] : null; + if ("trr-int".equals(type)) { + if (subType != null) { + feedbackNegotiationMap.put( + id, + new FeedbackNegotiationTrrInt( + SessionDescription.ignorantIntParser(subType))); + } + } else { + feedbackNegotiationMap.put(id, new FeedbackNegotiation(type, subType)); + } + } + } + for (final String ssrc : media.attributes.get(("ssrc"))) { + final String[] parts = ssrc.split(" ", 2); + if (parts.length == 2) { + final String id = parts[0]; + final String[] subParts = parts[1].split(":", 2); + final String attribute = subParts[0]; + final String value = subParts.length == 2 ? subParts[1] : null; + sourceParameterMap.put(id, new Source.Parameter(attribute, value)); + } + } + for (final String fmtp : media.attributes.get("fmtp")) { + final Pair> pair = Parameter.ofSdpString(fmtp); + if (pair != null) { + parameterMap.put(pair.first, pair.second); + } + } + rtpDescription.addChildren(feedbackNegotiationMap.get("*")); + for (final String rtpmap : media.attributes.get("rtpmap")) { + final PayloadType payloadType = PayloadType.ofSdpString(rtpmap); + if (payloadType != null) { + payloadType.addParameters(parameterMap.get(payloadType.getId())); + payloadType.addChildren(feedbackNegotiationMap.get(payloadType.getId())); + rtpDescription.addChild(payloadType); + } + } + for (final String extmap : media.attributes.get("extmap")) { + final RtpHeaderExtension extension = RtpHeaderExtension.ofSdpString(extmap); + if (extension != null) { + rtpDescription.addChild(extension); + } + } + if (attributes.contains("extmap-allow-mixed")) { + rtpDescription.addChild("extmap-allow-mixed", Namespace.JINGLE_RTP_HEADER_EXTENSIONS); + } + for (final String ssrcGroup : media.attributes.get("ssrc-group")) { + final String[] parts = ssrcGroup.split(" "); + if (parts.length >= 2) { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + final String semantics = parts[0]; + for (int i = 1; i < parts.length; ++i) { + builder.add(parts[i]); + } + rtpDescription.addChild(new SourceGroup(semantics, builder.build())); + } + } + for (Map.Entry> source : + sourceParameterMap.asMap().entrySet()) { + rtpDescription.addChild(new Source(source.getKey(), source.getValue())); + } + if (media.attributes.containsKey("rtcp-mux")) { + rtpDescription.addChild("rtcp-mux"); + } + return rtpDescription; + } + + private void addChildren(List elements) { + if (elements != null) { + this.children.addAll(elements); + } + } +} diff --git a/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/S5BTransportInfo.java b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/S5BTransportInfo.java new file mode 100644 index 000000000..13fcb2f7e --- /dev/null +++ b/app/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/S5BTransportInfo.java @@ -0,0 +1,53 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import com.google.common.base.Preconditions; +import eu.siacs.conversations.xmpp.jingle.JingleCandidate; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import java.util.Collection; +import java.util.List; + +public class S5BTransportInfo extends GenericTransportInfo { + + private S5BTransportInfo(final String name, final String xmlns) { + super(name, xmlns); + } + + public String getTransportId() { + return this.getAttribute("sid"); + } + + public S5BTransportInfo( + final String transportId, final Collection candidates) { + super("transport", Namespace.JINGLE_TRANSPORTS_S5B); + Preconditions.checkNotNull(transportId, "transport id must not be null"); + for (JingleCandidate candidate : candidates) { + this.addChild(candidate.toElement()); + } + this.setAttribute("sid", transportId); + } + + public S5BTransportInfo(final String transportId, final Element child) { + super("transport", Namespace.JINGLE_TRANSPORTS_S5B); + Preconditions.checkNotNull(transportId, "transport id must not be null"); + this.addChild(child); + this.setAttribute("sid", transportId); + } + + public List getCandidates() { + return JingleCandidate.parse(this.getChildren()); + } + + public static S5BTransportInfo upgrade(final Element element) { + Preconditions.checkArgument( + "transport".equals(element.getName()), "Name of provided element is not transport"); + Preconditions.checkArgument( + Namespace.JINGLE_TRANSPORTS_S5B.equals(element.getNamespace()), + "Element does not match s5b transport namespace"); + final S5BTransportInfo transportInfo = + new S5BTransportInfo("transport", Namespace.JINGLE_TRANSPORTS_S5B); + transportInfo.setAttributes(element.getAttributes()); + transportInfo.setChildren(element.getChildren()); + return transportInfo; + } +} diff --git a/app/src/main/java/im/conversations/android/AbstractAccountService.java b/app/src/main/java/im/conversations/android/AbstractAccountService.java new file mode 100644 index 000000000..72b285537 --- /dev/null +++ b/app/src/main/java/im/conversations/android/AbstractAccountService.java @@ -0,0 +1,15 @@ +package im.conversations.android; + +import android.content.Context; +import im.conversations.android.database.model.Account; + +public abstract class AbstractAccountService { + + protected Context context; + protected Account account; + + protected AbstractAccountService(final Context context, final Account account) { + this.context = context; + this.account = account; + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/axolotl/AxolotlAddress.java b/app/src/main/java/im/conversations/android/axolotl/AxolotlAddress.java similarity index 94% rename from app/src/main/java/im/conversations/android/xmpp/axolotl/AxolotlAddress.java rename to app/src/main/java/im/conversations/android/axolotl/AxolotlAddress.java index 3bc246253..e325e748a 100644 --- a/app/src/main/java/im/conversations/android/xmpp/axolotl/AxolotlAddress.java +++ b/app/src/main/java/im/conversations/android/axolotl/AxolotlAddress.java @@ -1,4 +1,4 @@ -package im.conversations.android.xmpp.axolotl; +package im.conversations.android.axolotl; import org.jxmpp.jid.BareJid; import org.whispersystems.libsignal.SignalProtocolAddress; diff --git a/app/src/main/java/im/conversations/android/axolotl/AxolotlDecryptionException.java b/app/src/main/java/im/conversations/android/axolotl/AxolotlDecryptionException.java new file mode 100644 index 000000000..a8122d237 --- /dev/null +++ b/app/src/main/java/im/conversations/android/axolotl/AxolotlDecryptionException.java @@ -0,0 +1,12 @@ +package im.conversations.android.axolotl; + +public class AxolotlDecryptionException extends Exception { + + public AxolotlDecryptionException(final String message) { + super(message); + } + + public AxolotlDecryptionException(final Throwable throwable) { + super(throwable); + } +} diff --git a/app/src/main/java/im/conversations/android/axolotl/AxolotlEncryptionException.java b/app/src/main/java/im/conversations/android/axolotl/AxolotlEncryptionException.java new file mode 100644 index 000000000..f7e0901d4 --- /dev/null +++ b/app/src/main/java/im/conversations/android/axolotl/AxolotlEncryptionException.java @@ -0,0 +1,16 @@ +package im.conversations.android.axolotl; + +public class AxolotlEncryptionException extends Exception { + + public AxolotlEncryptionException(String msg) { + super(msg); + } + + public AxolotlEncryptionException(String msg, Exception e) { + super(msg, e); + } + + public AxolotlEncryptionException(Exception e) { + super(e); + } +} diff --git a/app/src/main/java/im/conversations/android/axolotl/AxolotlPayload.java b/app/src/main/java/im/conversations/android/axolotl/AxolotlPayload.java new file mode 100644 index 000000000..a3db568ed --- /dev/null +++ b/app/src/main/java/im/conversations/android/axolotl/AxolotlPayload.java @@ -0,0 +1,27 @@ +package im.conversations.android.axolotl; + +import java.nio.charset.StandardCharsets; +import org.whispersystems.libsignal.IdentityKey; + +public class AxolotlPayload { + + public final AxolotlAddress axolotlAddress; + public final IdentityKey identityKey; + public final boolean preKeyMessage; + public final byte[] payload; + + public AxolotlPayload( + AxolotlAddress axolotlAddress, + final IdentityKey identityKey, + final boolean preKeyMessage, + byte[] payload) { + this.axolotlAddress = axolotlAddress; + this.identityKey = identityKey; + this.preKeyMessage = preKeyMessage; + this.payload = payload; + } + + public String payloadAsString() { + return new String(payload, StandardCharsets.UTF_8); + } +} diff --git a/app/src/main/java/im/conversations/android/axolotl/AxolotlService.java b/app/src/main/java/im/conversations/android/axolotl/AxolotlService.java new file mode 100644 index 000000000..aaae7f52d --- /dev/null +++ b/app/src/main/java/im/conversations/android/axolotl/AxolotlService.java @@ -0,0 +1,206 @@ +package im.conversations.android.axolotl; + +import android.content.Context; +import android.os.Build; +import com.google.common.base.Optional; +import eu.siacs.conversations.xmpp.jingle.OmemoVerification; +import im.conversations.android.AbstractAccountService; +import im.conversations.android.database.AxolotlDatabaseStore; +import im.conversations.android.database.model.Account; +import im.conversations.android.xmpp.model.axolotl.Encrypted; +import im.conversations.android.xmpp.model.axolotl.Header; +import im.conversations.android.xmpp.model.axolotl.Key; +import im.conversations.android.xmpp.model.axolotl.Payload; +import java.security.InvalidAlgorithmParameterException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import org.jxmpp.jid.Jid; +import org.whispersystems.libsignal.DuplicateMessageException; +import org.whispersystems.libsignal.IdentityKey; +import org.whispersystems.libsignal.InvalidKeyException; +import org.whispersystems.libsignal.InvalidKeyIdException; +import org.whispersystems.libsignal.InvalidMessageException; +import org.whispersystems.libsignal.InvalidVersionException; +import org.whispersystems.libsignal.LegacyMessageException; +import org.whispersystems.libsignal.NoSessionException; +import org.whispersystems.libsignal.UntrustedIdentityException; +import org.whispersystems.libsignal.protocol.PreKeySignalMessage; +import org.whispersystems.libsignal.protocol.SignalMessage; +import org.whispersystems.libsignal.state.SessionRecord; +import org.whispersystems.libsignal.state.SignalProtocolStore; + +public class AxolotlService extends AbstractAccountService { + + public static final String KEY_TYPE = "AES"; + public static final String CIPHER_MODE = "AES/GCM/NoPadding"; + + public static final String BOUNCY_CASTLE_PROVIDER = "BC"; + + private final SignalProtocolStore signalProtocolStore; + + public AxolotlService(final Context context, final Account account) { + super(context, account); + this.signalProtocolStore = new AxolotlDatabaseStore(context, account); + } + + private AxolotlSession buildReceivingSession( + final Jid from, final IdentityKey identityKey, final Header header) { + final Optional sid = header.getSourceDevice(); + if (sid.isPresent()) { + return AxolotlSession.of( + signalProtocolStore, + identityKey, + new AxolotlAddress(from.asBareJid(), sid.get())); + } + throw new IllegalArgumentException("Header did not contain a source device id"); + } + + public AxolotlSession getExistingSession(final AxolotlAddress axolotlAddress) { + final SessionRecord sessionState = signalProtocolStore.loadSession(axolotlAddress); + if (sessionState == null) { + return null; + } + final IdentityKey identityKey = sessionState.getSessionState().getRemoteIdentityKey(); + return AxolotlSession.of(signalProtocolStore, identityKey, axolotlAddress); + } + + private AxolotlSession getExistingSessionOrThrow(final AxolotlAddress axolotlAddress) + throws NoSessionException { + final var session = getExistingSession(axolotlAddress); + if (session == null) { + throw new NoSessionException( + String.format("No session for %s", axolotlAddress.toString())); + } + return session; + } + + public AxolotlPayload decrypt(final Jid from, final Encrypted encrypted) + throws AxolotlDecryptionException { + try { + return decryptOrThrow(from, encrypted); + } catch (final IllegalArgumentException + | NotEncryptedForThisDeviceException + | InvalidMessageException + | InvalidVersionException + | UntrustedIdentityException + | DuplicateMessageException + | InvalidKeyIdException + | LegacyMessageException + | InvalidKeyException + | NoSessionException + | OutdatedSenderException + | NoSuchPaddingException + | NoSuchAlgorithmException + | NoSuchProviderException + | InvalidAlgorithmParameterException + | java.security.InvalidKeyException + | IllegalBlockSizeException + | BadPaddingException e) { + throw new AxolotlDecryptionException(e); + } + } + + private AxolotlPayload decryptOrThrow(final Jid from, final Encrypted encrypted) + throws NotEncryptedForThisDeviceException, InvalidMessageException, + InvalidVersionException, UntrustedIdentityException, DuplicateMessageException, + InvalidKeyIdException, LegacyMessageException, InvalidKeyException, + NoSessionException, OutdatedSenderException, NoSuchPaddingException, + NoSuchAlgorithmException, NoSuchProviderException, + InvalidAlgorithmParameterException, java.security.InvalidKeyException, + IllegalBlockSizeException, BadPaddingException { + final Payload payload = encrypted.getPayload(); + final Header header = encrypted.getHeader(); + final Key ourKey = header.getKey(signalProtocolStore.getLocalRegistrationId()); + if (ourKey == null) { + throw new NotEncryptedForThisDeviceException(); + } + final byte[] keyWithAuthTag; + final AxolotlSession session; + final boolean preKeyMessage; + if (ourKey.isPreKey()) { + final PreKeySignalMessage preKeySignalMessage = + new PreKeySignalMessage(ourKey.asBytes()); + preKeyMessage = true; + session = buildReceivingSession(from, preKeySignalMessage.getIdentityKey(), header); + keyWithAuthTag = session.sessionCipher.decrypt(preKeySignalMessage); + } else { + final SignalMessage signalMessage = new SignalMessage(ourKey.asBytes()); + preKeyMessage = false; + session = + getExistingSessionOrThrow( + new AxolotlAddress(from.asBareJid(), header.getSourceDevice().get())); + keyWithAuthTag = session.sessionCipher.decrypt(signalMessage); + } + if (keyWithAuthTag.length < 32) { + throw new OutdatedSenderException( + "Key did not contain auth tag. Sender needs to update their OMEMO client"); + } + if (payload == null) { + return new AxolotlPayload( + session.axolotlAddress, session.identityKey, preKeyMessage, null); + } + final byte[] key = new byte[16]; + final byte[] authTag = new byte[16]; + final byte[] iv = header.getIv(); + System.arraycopy(keyWithAuthTag, 0, key, 0, key.length); + System.arraycopy(keyWithAuthTag, key.length, authTag, 0, authTag.length); + final Cipher cipher; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + cipher = Cipher.getInstance(CIPHER_MODE); + } else { + cipher = Cipher.getInstance(CIPHER_MODE, BOUNCY_CASTLE_PROVIDER); + } + final SecretKey secretKey = new SecretKeySpec(key, KEY_TYPE); + final IvParameterSpec ivSpec = new IvParameterSpec(iv); + cipher.init(Cipher.DECRYPT_MODE, secretKey, ivSpec); + final byte[] payloadAsBytes = payload.asBytes(); + final byte[] payloadWithAuthTag = new byte[payloadAsBytes.length + 16]; + System.arraycopy(payloadAsBytes, 0, payloadWithAuthTag, 0, payloadAsBytes.length); + System.arraycopy(authTag, 0, payloadWithAuthTag, payloadAsBytes.length, authTag.length); + final byte[] decryptedPayload = cipher.doFinal(payloadWithAuthTag); + return new AxolotlPayload( + session.axolotlAddress, session.identityKey, preKeyMessage, decryptedPayload); + } + + public SignalProtocolStore getSignalProtocolStore() { + return this.signalProtocolStore; + } + + public static class OmemoVerifiedPayload { + private final int deviceId; + private final IdentityKey identityKey; + private final T payload; + + public OmemoVerifiedPayload(OmemoVerification omemoVerification, T payload) { + this.deviceId = omemoVerification.getDeviceId(); + this.identityKey = omemoVerification.getFingerprint(); + this.payload = payload; + } + + public int getDeviceId() { + return deviceId; + } + + public IdentityKey getFingerprint() { + return identityKey; + } + + public T getPayload() { + return payload; + } + } + + public static class NotVerifiedException extends SecurityException { + + public NotVerifiedException(String message) { + super(message); + } + } +} diff --git a/app/src/main/java/im/conversations/android/axolotl/AxolotlSession.java b/app/src/main/java/im/conversations/android/axolotl/AxolotlSession.java new file mode 100644 index 000000000..071ddecdd --- /dev/null +++ b/app/src/main/java/im/conversations/android/axolotl/AxolotlSession.java @@ -0,0 +1,29 @@ +package im.conversations.android.axolotl; + +import org.whispersystems.libsignal.IdentityKey; +import org.whispersystems.libsignal.SessionCipher; +import org.whispersystems.libsignal.state.SignalProtocolStore; + +public class AxolotlSession { + + public final AxolotlAddress axolotlAddress; + public final IdentityKey identityKey; + public final SessionCipher sessionCipher; + + private AxolotlSession( + AxolotlAddress axolotlAddress, + final IdentityKey identityKey, + SessionCipher sessionCipher) { + this.axolotlAddress = axolotlAddress; + this.identityKey = identityKey; + this.sessionCipher = sessionCipher; + } + + public static AxolotlSession of( + final SignalProtocolStore signalProtocolStore, + final IdentityKey identityKey, + final AxolotlAddress axolotlAddress) { + final var sessionCipher = new SessionCipher(signalProtocolStore, axolotlAddress); + return new AxolotlSession(axolotlAddress, identityKey, sessionCipher); + } +} diff --git a/app/src/main/java/im/conversations/android/axolotl/EncryptionBuilder.java b/app/src/main/java/im/conversations/android/axolotl/EncryptionBuilder.java new file mode 100644 index 000000000..d96027eb7 --- /dev/null +++ b/app/src/main/java/im/conversations/android/axolotl/EncryptionBuilder.java @@ -0,0 +1,157 @@ +package im.conversations.android.axolotl; + +import android.annotation.SuppressLint; +import android.os.Build; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import im.conversations.android.Conversations; +import im.conversations.android.xmpp.model.axolotl.Encrypted; +import im.conversations.android.xmpp.model.axolotl.Header; +import im.conversations.android.xmpp.model.axolotl.Key; +import im.conversations.android.xmpp.model.axolotl.Payload; +import java.nio.charset.StandardCharsets; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.util.ArrayList; +import java.util.List; +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.KeyGenerator; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import org.whispersystems.libsignal.UntrustedIdentityException; +import org.whispersystems.libsignal.protocol.CiphertextMessage; + +public class EncryptionBuilder { + + private Long sourceDeviceId; + + private ArrayList sessions; + + private byte[] payload; + + public Encrypted build() throws AxolotlEncryptionException { + try { + return buildOrThrow(); + } catch (final InvalidAlgorithmParameterException + | NoSuchPaddingException + | IllegalBlockSizeException + | NoSuchAlgorithmException + | BadPaddingException + | NoSuchProviderException + | InvalidKeyException + | UntrustedIdentityException e) { + throw new AxolotlEncryptionException(e); + } + } + + private Encrypted buildOrThrow() + throws InvalidAlgorithmParameterException, NoSuchPaddingException, + IllegalBlockSizeException, NoSuchAlgorithmException, BadPaddingException, + NoSuchProviderException, InvalidKeyException, UntrustedIdentityException { + final long sourceDeviceId = + Preconditions.checkNotNull(this.sourceDeviceId, "Specify a source device id"); + final var payloadCleartext = Preconditions.checkNotNull(this.payload, "Specify a payload"); + Preconditions.checkState(sessions.size() > 0, "Add at least on session"); + final var sessions = ImmutableList.copyOf(this.sessions); + final var key = generateKey(); + final var iv = generateIv(); + final var encryptedPayload = encrypt(payloadCleartext, key, iv); + final var keyWithAuthTag = new byte[32]; + System.arraycopy(key, 0, keyWithAuthTag, 0, key.length); + System.arraycopy( + encryptedPayload.authTag, 0, keyWithAuthTag, 16, encryptedPayload.authTag.length); + final var header = buildHeader(sessions, keyWithAuthTag); + header.addIv(iv); + header.setSourceDevice(sourceDeviceId); + final var encrypted = new Encrypted(); + encrypted.addExtension(header); + final var payload = encrypted.addExtension(new Payload()); + payload.setContent(encryptedPayload.encrypted); + return encrypted; + } + + public EncryptionBuilder payload(final String payload) { + this.payload = payload.getBytes(StandardCharsets.UTF_8); + return this; + } + + public EncryptionBuilder session(final AxolotlSession session) { + this.sessions.add(session); + return this; + } + + public EncryptionBuilder sourceDeviceId(final long sourceDeviceId) { + this.sourceDeviceId = sourceDeviceId; + return this; + } + + private Header buildHeader(List sessions, final byte[] keyWithAuthTag) + throws UntrustedIdentityException { + final var header = new Header(); + for (final AxolotlSession session : sessions) { + final var cipherMessage = session.sessionCipher.encrypt(keyWithAuthTag); + final var key = header.addExtension(new Key()); + key.setContent(cipherMessage.serialize()); + key.setIsPreKey(cipherMessage.getType() == CiphertextMessage.PREKEY_TYPE); + } + return header; + } + + @SuppressLint("DeprecatedProvider") + private static EncryptedPayload encrypt( + final byte[] payloadCleartext, final byte[] key, final byte[] iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, NoSuchProviderException, + IllegalBlockSizeException, BadPaddingException, + InvalidAlgorithmParameterException, InvalidKeyException { + final Cipher cipher; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + cipher = Cipher.getInstance(AxolotlService.CIPHER_MODE); + } else { + cipher = + Cipher.getInstance( + AxolotlService.CIPHER_MODE, AxolotlService.BOUNCY_CASTLE_PROVIDER); + } + final SecretKey secretKey = new SecretKeySpec(key, AxolotlService.KEY_TYPE); + final IvParameterSpec ivSpec = new IvParameterSpec(iv); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, ivSpec); + final var encryptedWithAuthTag = cipher.doFinal(payloadCleartext); + final var authTag = new byte[16]; + final var encrypted = new byte[encryptedWithAuthTag.length - authTag.length]; + + System.arraycopy(encryptedWithAuthTag, 0, encrypted, 0, encrypted.length); + System.arraycopy(encryptedWithAuthTag, encrypted.length, authTag, 0, authTag.length); + return new EncryptedPayload(encrypted, authTag); + } + + private static byte[] generateKey() { + try { + KeyGenerator generator = KeyGenerator.getInstance(AxolotlService.KEY_TYPE); + generator.init(128); + return generator.generateKey().getEncoded(); + } catch (final NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + private static final class EncryptedPayload { + public final byte[] encrypted; + public final byte[] authTag; + + private EncryptedPayload(byte[] encrypted, byte[] authTag) { + this.encrypted = encrypted; + this.authTag = authTag; + } + } + + private static byte[] generateIv() { + final byte[] iv = new byte[12]; + Conversations.SECURE_RANDOM.nextBytes(iv); + return iv; + } +} diff --git a/app/src/main/java/im/conversations/android/axolotl/NotEncryptedForThisDeviceException.java b/app/src/main/java/im/conversations/android/axolotl/NotEncryptedForThisDeviceException.java new file mode 100644 index 000000000..cfbaf5581 --- /dev/null +++ b/app/src/main/java/im/conversations/android/axolotl/NotEncryptedForThisDeviceException.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018, Daniel Gultsch All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package im.conversations.android.axolotl; + +public class NotEncryptedForThisDeviceException extends AxolotlEncryptionException { + public NotEncryptedForThisDeviceException() { + super("Message was not encrypted for this device"); + } +} diff --git a/app/src/main/java/im/conversations/android/axolotl/OutdatedSenderException.java b/app/src/main/java/im/conversations/android/axolotl/OutdatedSenderException.java new file mode 100644 index 000000000..41c5c1459 --- /dev/null +++ b/app/src/main/java/im/conversations/android/axolotl/OutdatedSenderException.java @@ -0,0 +1,8 @@ +package im.conversations.android.axolotl; + +public class OutdatedSenderException extends AxolotlDecryptionException { + + public OutdatedSenderException(final String message) { + super(message); + } +} diff --git a/app/src/main/java/im/conversations/android/database/AxolotlDatabaseStore.java b/app/src/main/java/im/conversations/android/database/AxolotlDatabaseStore.java index 8e783c9e4..6c7dd8e7b 100644 --- a/app/src/main/java/im/conversations/android/database/AxolotlDatabaseStore.java +++ b/app/src/main/java/im/conversations/android/database/AxolotlDatabaseStore.java @@ -1,9 +1,10 @@ package im.conversations.android.database; import android.content.Context; +import im.conversations.android.AbstractAccountService; +import im.conversations.android.axolotl.AxolotlAddress; import im.conversations.android.database.dao.AxolotlDao; import im.conversations.android.database.model.Account; -import im.conversations.android.xmpp.axolotl.AxolotlAddress; import java.util.List; import org.whispersystems.libsignal.IdentityKey; import org.whispersystems.libsignal.IdentityKeyPair; @@ -14,14 +15,10 @@ import org.whispersystems.libsignal.state.SessionRecord; import org.whispersystems.libsignal.state.SignalProtocolStore; import org.whispersystems.libsignal.state.SignedPreKeyRecord; -public class AxolotlDatabaseStore implements SignalProtocolStore { - - private final Context context; - private final Account account; +public class AxolotlDatabaseStore extends AbstractAccountService implements SignalProtocolStore { public AxolotlDatabaseStore(final Context context, final Account account) { - this.context = context; - this.account = account; + super(context, account); } private AxolotlDao axolotlDao() { diff --git a/app/src/main/java/im/conversations/android/database/dao/DiscoDao.java b/app/src/main/java/im/conversations/android/database/dao/DiscoDao.java index 1ed9dc575..0c91d3859 100644 --- a/app/src/main/java/im/conversations/android/database/dao/DiscoDao.java +++ b/app/src/main/java/im/conversations/android/database/dao/DiscoDao.java @@ -26,6 +26,7 @@ import im.conversations.android.xmpp.model.disco.info.Identity; import im.conversations.android.xmpp.model.disco.info.InfoQuery; import im.conversations.android.xmpp.model.disco.items.Item; import java.util.Collection; +import org.jxmpp.jid.BareJid; import org.jxmpp.jid.Jid; import org.jxmpp.jid.parts.Resourcepart; @@ -186,5 +187,44 @@ public abstract class DiscoDao { "SELECT EXISTS (SELECT disco_item.id FROM disco_item JOIN disco_feature on" + " disco_item.discoId=disco_feature.discoId WHERE accountId=:account AND" + " address=:entity AND feature=:feature)") - public abstract boolean hasFeature(final long account, final Jid entity, final String feature); + protected abstract boolean hasDiscoItemFeature( + final long account, final Jid entity, final String feature); + + @Query( + "SELECT EXISTS (SELECT presence.id FROM presence JOIN disco_feature on" + + " presence.discoId=disco_feature.discoId WHERE accountId=:account AND" + + " address=:address AND resource=:resource AND feature=:feature)") + protected abstract boolean hasPresenceFeature( + final long account, + final BareJid address, + final Resourcepart resource, + final String feature); + + @Query( + "SELECT count(presence.id) FROM presence JOIN disco_feature on" + + " presence.discoId=disco_feature.discoId WHERE accountId=:account AND" + + " address=:address AND feature=:feature") + public abstract int countPresencesWithFeature( + final long account, final BareJid address, final String feature); + + public int countPresencesWithFeature(final Account account, final String feature) { + return countPresencesWithFeature(account.id, account.address, feature); + } + + public boolean hasFeature(final long account, final Entity entity, final String feature) { + if (entity instanceof Entity.DiscoItem) { + return hasDiscoItemFeature(account, entity.address, feature); + } + if (entity instanceof Entity.Presence) { + return hasPresenceFeature( + account, + entity.address.asBareJid(), + entity.address.getResourceOrEmpty(), + feature); + } + throw new IllegalStateException( + String.format( + "Discovering features for %s is not implemented", + entity.getClass().getName())); + } } diff --git a/app/src/main/java/im/conversations/android/database/dao/RosterDao.java b/app/src/main/java/im/conversations/android/database/dao/RosterDao.java index 2f32b7090..ce59748e1 100644 --- a/app/src/main/java/im/conversations/android/database/dao/RosterDao.java +++ b/app/src/main/java/im/conversations/android/database/dao/RosterDao.java @@ -12,6 +12,7 @@ import im.conversations.android.database.entity.RosterItemGroupEntity; import im.conversations.android.database.model.Account; import im.conversations.android.xmpp.model.roster.Item; import java.util.Collection; +import org.jxmpp.jid.BareJid; import org.jxmpp.jid.Jid; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +34,9 @@ public abstract class RosterDao extends GroupDao { @Query("UPDATE account SET rosterVersion=:version WHERE id=:account") protected abstract void setRosterVersion(final long account, final String version); + @Query("SELECT EXISTS (SELECT id FROM roster WHERE accountId=:account AND address=:address)") + public abstract boolean isInRoster(final long account, final BareJid address); + @Transaction public void set( final Account account, final String version, final Collection rosterItems) { diff --git a/app/src/main/java/im/conversations/android/notification/AbstractNotification.java b/app/src/main/java/im/conversations/android/notification/AbstractNotification.java new file mode 100644 index 000000000..8e8b74fce --- /dev/null +++ b/app/src/main/java/im/conversations/android/notification/AbstractNotification.java @@ -0,0 +1,28 @@ +package im.conversations.android.notification; + +import android.content.Context; +import android.content.SharedPreferences; +import androidx.preference.PreferenceManager; +import im.conversations.android.R; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +public abstract class AbstractNotification { + + protected static final ScheduledExecutorService SCHEDULED_EXECUTOR_SERVICE = + Executors.newSingleThreadScheduledExecutor(); + + protected final Context context; + + protected AbstractNotification(final Context context) { + this.context = context; + } + + public boolean notificationsFromStrangers() { + final SharedPreferences preferences = + PreferenceManager.getDefaultSharedPreferences(context); + return preferences.getBoolean( + "notifications_from_strangers", + context.getResources().getBoolean(R.bool.notifications_from_strangers)); + } +} diff --git a/app/src/main/java/im/conversations/android/notification/Channels.java b/app/src/main/java/im/conversations/android/notification/Channels.java index af671a94b..0477838ba 100644 --- a/app/src/main/java/im/conversations/android/notification/Channels.java +++ b/app/src/main/java/im/conversations/android/notification/Channels.java @@ -11,10 +11,11 @@ import im.conversations.android.R; public final class Channels { - private final Application application; - - private static final String CHANNEL_GROUP_STATUS = "status"; static final String CHANNEL_FOREGROUND = "foreground"; + static final String INCOMING_CALLS_NOTIFICATION_CHANNEL = "incoming_calls_channel"; + static final String CHANNEL_GROUP_STATUS = "status"; + static final String CHANNEL_GROUP_CALLS = "calls"; + private final Application application; public Channels(final Application application) { this.application = application; @@ -29,6 +30,8 @@ public final class Channels { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { this.initializeGroups(notificationManager); this.initializeForegroundChannel(notificationManager); + + this.initializeIncomingCallChannel(notificationManager); } } @@ -38,6 +41,10 @@ public final class Channels { new NotificationChannelGroup( CHANNEL_GROUP_STATUS, application.getString(R.string.notification_group_status_information))); + notificationManager.createNotificationChannelGroup( + new NotificationChannelGroup( + CHANNEL_GROUP_CALLS, + application.getString(R.string.notification_group_calls))); } @RequiresApi(api = Build.VERSION_CODES.O) @@ -55,4 +62,21 @@ public final class Channels { foregroundServiceChannel.setGroup(CHANNEL_GROUP_STATUS); notificationManager.createNotificationChannel(foregroundServiceChannel); } + + @RequiresApi(api = Build.VERSION_CODES.O) + private void initializeIncomingCallChannel(final NotificationManager notificationManager) { + final NotificationChannel incomingCallsChannel = + new NotificationChannel( + INCOMING_CALLS_NOTIFICATION_CHANNEL, + application.getString(R.string.incoming_calls_channel_name), + NotificationManager.IMPORTANCE_HIGH); + incomingCallsChannel.setSound(null, null); + incomingCallsChannel.setShowBadge(false); + incomingCallsChannel.setLightColor(RtpSessionNotification.LED_COLOR); + incomingCallsChannel.enableLights(true); + incomingCallsChannel.setGroup(CHANNEL_GROUP_CALLS); + incomingCallsChannel.setBypassDnd(true); + incomingCallsChannel.enableVibration(false); + notificationManager.createNotificationChannel(incomingCallsChannel); + } } diff --git a/app/src/main/java/im/conversations/android/notification/ForegroundServiceNotification.java b/app/src/main/java/im/conversations/android/notification/ForegroundServiceNotification.java index 0fc32edc9..7bfdc6491 100644 --- a/app/src/main/java/im/conversations/android/notification/ForegroundServiceNotification.java +++ b/app/src/main/java/im/conversations/android/notification/ForegroundServiceNotification.java @@ -11,28 +11,26 @@ import im.conversations.android.R; import im.conversations.android.ui.activity.MainActivity; import im.conversations.android.xmpp.ConnectionPool; -public class ForegroundServiceNotification { +public class ForegroundServiceNotification extends AbstractNotification { public static final int ID = 1; - private final Service service; - public ForegroundServiceNotification(final Service service) { - this.service = service; + super(service); } public Notification build(final ConnectionPool.Summary summary) { - final Notification.Builder builder = new Notification.Builder(service); + final Notification.Builder builder = new Notification.Builder(context); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { // starting with Android 7 the app name is displayed as part of the notification // this means we do not have to repeat it in the 'content title' builder.setContentTitle( - service.getString( + context.getString( R.string.connected_accounts, summary.connected, summary.total)); } else { - builder.setContentTitle(service.getString(R.string.app_name)); + builder.setContentTitle(context.getString(R.string.app_name)); builder.setContentText( - service.getString( + context.getString( R.string.connected_accounts, summary.connected, summary.total)); } builder.setContentIntent(buildPendingIntent()); @@ -53,15 +51,15 @@ public class ForegroundServiceNotification { private PendingIntent buildPendingIntent() { return PendingIntent.getActivity( - service, + context, 0, - new Intent(service, MainActivity.class), + new Intent(context, MainActivity.class), PendingIntent.FLAG_IMMUTABLE | PendingIntent.FLAG_UPDATE_CURRENT); } public void update(final ConnectionPool.Summary summary) { final var notificationManager = - ContextCompat.getSystemService(service, NotificationManager.class); + ContextCompat.getSystemService(context, NotificationManager.class); if (notificationManager == null) { return; } diff --git a/app/src/main/java/im/conversations/android/notification/OngoingCall.java b/app/src/main/java/im/conversations/android/notification/OngoingCall.java new file mode 100644 index 000000000..9bb5b4b85 --- /dev/null +++ b/app/src/main/java/im/conversations/android/notification/OngoingCall.java @@ -0,0 +1,34 @@ +package im.conversations.android.notification; + +import com.google.common.base.Objects; +import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection; +import eu.siacs.conversations.xmpp.jingle.Media; +import java.util.Set; + +public class OngoingCall { + public final AbstractJingleConnection.Id id; + public final Set media; + public final boolean reconnecting; + + public OngoingCall( + AbstractJingleConnection.Id id, Set media, final boolean reconnecting) { + this.id = id; + this.media = media; + this.reconnecting = reconnecting; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OngoingCall that = (OngoingCall) o; + return reconnecting == that.reconnecting + && Objects.equal(id, that.id) + && Objects.equal(media, that.media); + } + + @Override + public int hashCode() { + return Objects.hashCode(id, media, reconnecting); + } +} diff --git a/app/src/main/java/im/conversations/android/notification/RtpSessionNotification.java b/app/src/main/java/im/conversations/android/notification/RtpSessionNotification.java new file mode 100644 index 000000000..e62638f12 --- /dev/null +++ b/app/src/main/java/im/conversations/android/notification/RtpSessionNotification.java @@ -0,0 +1,284 @@ +package im.conversations.android.notification; + +import android.Manifest; +import android.app.Notification; +import android.app.NotificationManager; +import android.app.PendingIntent; +import android.content.Context; +import android.content.Intent; +import android.content.SharedPreferences; +import android.content.pm.PackageManager; +import android.content.res.Resources; +import android.media.Ringtone; +import android.media.RingtoneManager; +import android.net.Uri; +import android.os.Build; +import android.os.Vibrator; +import android.preference.PreferenceManager; +import android.util.Log; +import androidx.core.app.ActivityCompat; +import androidx.core.app.NotificationCompat; +import androidx.core.app.NotificationManagerCompat; +import androidx.core.content.ContextCompat; +import com.google.common.base.Strings; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection; +import eu.siacs.conversations.xmpp.jingle.Media; +import im.conversations.android.R; +import im.conversations.android.database.model.Account; +import im.conversations.android.service.RtpSessionService; +import im.conversations.android.transformer.CallLogEntry; +import im.conversations.android.ui.activity.RtpSessionActivity; +import java.util.Set; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RtpSessionNotification extends AbstractNotification { + + private static final Logger LOGGER = LoggerFactory.getLogger(RtpSessionNotification.class); + + public static final int INCOMING_CALL_ID = 2; + + public static final int LED_COLOR = 0xff00ff00; + + private static final long[] CALL_PATTERN = {0, 500, 300, 600}; + + private Ringtone currentlyPlayingRingtone = null; + private ScheduledFuture vibrationFuture; + + public RtpSessionNotification(Context context) { + super(context); + } + + public void cancelIncomingCallNotification() { + stopSoundAndVibration(); + cancel(INCOMING_CALL_ID); + } + + public boolean stopSoundAndVibration() { + int stopped = 0; + if (this.currentlyPlayingRingtone != null) { + if (this.currentlyPlayingRingtone.isPlaying()) { + Log.d(Config.LOGTAG, "stop playing ring tone"); + ++stopped; + } + this.currentlyPlayingRingtone.stop(); + } + if (this.vibrationFuture != null && !this.vibrationFuture.isCancelled()) { + Log.d(Config.LOGTAG, "stop vibration"); + this.vibrationFuture.cancel(true); + ++stopped; + } + return stopped > 0; + } + + private void notify(int id, Notification notification) { + final var notificationManager = NotificationManagerCompat.from(context); + try { + if (ActivityCompat.checkSelfPermission(context, Manifest.permission.POST_NOTIFICATIONS) + != PackageManager.PERMISSION_GRANTED) { + LOGGER.warn("Lacking notification permission"); + return; + } + notificationManager.notify(id, notification); + } catch (final RuntimeException e) { + LOGGER.warn("Could not post notification", e); + } + } + + private void cancel(final int notificationId) { + final var notificationManager = NotificationManagerCompat.from(context); + notificationManager.cancel(notificationId); + } + + public synchronized void startRinging( + final Account account, final AbstractJingleConnection.Id id, final Set media) { + showIncomingCallNotification(account, id, media); + final NotificationManager notificationManager = + (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE); + final int currentInterruptionFilter; + if (notificationManager != null) { + currentInterruptionFilter = notificationManager.getCurrentInterruptionFilter(); + } else { + currentInterruptionFilter = 1; // INTERRUPTION_FILTER_ALL + } + if (currentInterruptionFilter != 1) { + Log.d( + Config.LOGTAG, + "do not ring or vibrate because interruption filter has been set to " + + currentInterruptionFilter); + return; + } + final ScheduledFuture currentVibrationFuture = this.vibrationFuture; + this.vibrationFuture = + SCHEDULED_EXECUTOR_SERVICE.scheduleAtFixedRate( + new VibrationRunnable(), 0, 3, TimeUnit.SECONDS); + if (currentVibrationFuture != null) { + currentVibrationFuture.cancel(true); + } + final SharedPreferences preferences = + PreferenceManager.getDefaultSharedPreferences(context); + final Resources resources = context.getResources(); + final String ringtonePreference = + preferences.getString( + "call_ringtone", resources.getString(R.string.incoming_call_ringtone)); + if (Strings.isNullOrEmpty(ringtonePreference)) { + Log.d(Config.LOGTAG, "ringtone has been set to none"); + return; + } + final Uri uri = Uri.parse(ringtonePreference); + this.currentlyPlayingRingtone = RingtoneManager.getRingtone(context, uri); + if (this.currentlyPlayingRingtone == null) { + Log.d(Config.LOGTAG, "unable to find ringtone for uri " + uri); + return; + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + this.currentlyPlayingRingtone.setLooping(true); + } + this.currentlyPlayingRingtone.play(); + } + + private void showIncomingCallNotification( + final Account account, final AbstractJingleConnection.Id id, final Set media) { + final Intent fullScreenIntent = new Intent(context, RtpSessionActivity.class); + fullScreenIntent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, account.id); + fullScreenIntent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toString()); + fullScreenIntent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId); + fullScreenIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + fullScreenIntent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP); + final NotificationCompat.Builder builder = + new NotificationCompat.Builder( + context, Channels.INCOMING_CALLS_NOTIFICATION_CHANNEL); + if (media.contains(Media.VIDEO)) { + builder.setSmallIcon(R.drawable.ic_videocam_24dp); + builder.setContentTitle(context.getString(R.string.rtp_state_incoming_video_call)); + } else { + builder.setSmallIcon(R.drawable.ic_call_24dp); + builder.setContentTitle(context.getString(R.string.rtp_state_incoming_call)); + } + // TODO fix me once we have a contact model + /*final Contact contact = id.getContact(); + builder.setLargeIcon( + mXmppConnectionService + .getAvatarService() + .get(contact, AvatarService.getSystemUiAvatarSize(mXmppConnectionService))); + final Uri systemAccount = contact.getSystemAccount(); + if (systemAccount != null) { + builder.addPerson(systemAccount.toString()); + } + builder.setContentText(id.account.getRoster().getContact(id.with).getDisplayName());*/ + builder.setVisibility(NotificationCompat.VISIBILITY_PUBLIC); + builder.setPriority(NotificationCompat.PRIORITY_HIGH); + builder.setCategory(NotificationCompat.CATEGORY_CALL); + PendingIntent pendingIntent = createPendingRtpSession(account, id, Intent.ACTION_VIEW, 101); + builder.setFullScreenIntent(pendingIntent, true); + builder.setContentIntent(pendingIntent); // old androids need this? + builder.setOngoing(true); + builder.addAction( + new NotificationCompat.Action.Builder( + R.drawable.ic_call_end_24dp, + context.getString(R.string.dismiss_call), + createCallAction( + id.sessionId, RtpSessionService.ACTION_DISMISS_CALL, 102)) + .build()); + builder.addAction( + new NotificationCompat.Action.Builder( + R.drawable.ic_call_24dp, + context.getString(R.string.answer_call), + createPendingRtpSession( + account, id, RtpSessionActivity.ACTION_ACCEPT_CALL, 103)) + .build()); + modifyIncomingCall(builder); + final Notification notification = builder.build(); + notification.flags = notification.flags | Notification.FLAG_INSISTENT; + notify(INCOMING_CALL_ID, notification); + } + + public Notification getOngoingCallNotification(final Account account, OngoingCall ongoingCall) { + final AbstractJingleConnection.Id id = ongoingCall.id; + final NotificationCompat.Builder builder = + new NotificationCompat.Builder(context, "ongoing_calls"); + if (ongoingCall.media.contains(Media.VIDEO)) { + builder.setSmallIcon(R.drawable.ic_videocam_24dp); + if (ongoingCall.reconnecting) { + builder.setContentTitle(context.getString(R.string.reconnecting_video_call)); + } else { + builder.setContentTitle(context.getString(R.string.ongoing_video_call)); + } + } else { + builder.setSmallIcon(R.drawable.ic_call_24dp); + if (ongoingCall.reconnecting) { + builder.setContentTitle(context.getString(R.string.reconnecting_call)); + } else { + builder.setContentTitle(context.getString(R.string.ongoing_call)); + } + } + // TODO fix me when we have a Contact model + // builder.setContentText(id.account.getRoster().getContact(id.with).getDisplayName()); + builder.setVisibility(NotificationCompat.VISIBILITY_PUBLIC); + builder.setPriority(NotificationCompat.PRIORITY_HIGH); + builder.setCategory(NotificationCompat.CATEGORY_CALL); + builder.setContentIntent(createPendingRtpSession(account, id, Intent.ACTION_VIEW, 101)); + builder.setOngoing(true); + builder.addAction( + new NotificationCompat.Action.Builder( + R.drawable.ic_call_end_24dp, + context.getString(R.string.hang_up), + createCallAction( + id.sessionId, RtpSessionService.ACTION_END_CALL, 104)) + .build()); + return builder.build(); + } + + private PendingIntent createPendingRtpSession( + final Account account, + final AbstractJingleConnection.Id id, + final String action, + final int requestCode) { + final Intent fullScreenIntent = new Intent(context, RtpSessionActivity.class); + fullScreenIntent.setAction(action); + fullScreenIntent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, account.id); + fullScreenIntent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toString()); + fullScreenIntent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId); + return PendingIntent.getActivity( + context, + requestCode, + fullScreenIntent, + PendingIntent.FLAG_IMMUTABLE | PendingIntent.FLAG_UPDATE_CURRENT); + } + + private PendingIntent createCallAction(String sessionId, final String action, int requestCode) { + final Intent intent = new Intent(context, RtpSessionService.class); + intent.setAction(action); + intent.setPackage(context.getPackageName()); + intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, sessionId); + return PendingIntent.getService( + context, + requestCode, + intent, + PendingIntent.FLAG_MUTABLE | PendingIntent.FLAG_UPDATE_CURRENT); + } + + private void modifyIncomingCall(final NotificationCompat.Builder mBuilder) { + mBuilder.setPriority(NotificationCompat.PRIORITY_HIGH); + setNotificationColor(mBuilder); + mBuilder.setLights(LED_COLOR, 2000, 3000); + } + + private void setNotificationColor(final NotificationCompat.Builder mBuilder) { + mBuilder.setColor(ContextCompat.getColor(context, R.color.seed)); + } + + public void pushMissedCallNow(CallLogEntry message) {} + + private class VibrationRunnable implements Runnable { + + @Override + public void run() { + final Vibrator vibrator = (Vibrator) context.getSystemService(Context.VIBRATOR_SERVICE); + vibrator.vibrate(CALL_PATTERN, -1); + } + } +} diff --git a/app/src/main/java/im/conversations/android/service/RtpSessionService.java b/app/src/main/java/im/conversations/android/service/RtpSessionService.java new file mode 100644 index 000000000..a341f6c93 --- /dev/null +++ b/app/src/main/java/im/conversations/android/service/RtpSessionService.java @@ -0,0 +1,16 @@ +package im.conversations.android.service; + +import android.content.Intent; +import androidx.lifecycle.LifecycleService; + +public class RtpSessionService extends LifecycleService { + + public static final String ACTION_DISMISS_CALL = "dismiss_call"; + public static final String ACTION_END_CALL = "end_call"; + + @Override + public int onStartCommand(final Intent intent, final int flags, final int startId) { + + return super.onStartCommand(intent, flags, startId); + } +} diff --git a/app/src/main/java/im/conversations/android/tls/TrustManager.java b/app/src/main/java/im/conversations/android/tls/TrustManager.java index 46fac9cb7..cf220cf5a 100644 --- a/app/src/main/java/im/conversations/android/tls/TrustManager.java +++ b/app/src/main/java/im/conversations/android/tls/TrustManager.java @@ -1,19 +1,16 @@ package im.conversations.android.tls; import android.content.Context; +import im.conversations.android.AbstractAccountService; import im.conversations.android.database.model.Account; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import javax.net.ssl.X509TrustManager; -public class TrustManager implements X509TrustManager { - - private final Context context; - private final Account account; +public class TrustManager extends AbstractAccountService implements X509TrustManager { public TrustManager(final Context context, final Account account) { - this.context = context; - this.account = account; + super(context, account); } @Override diff --git a/app/src/main/java/im/conversations/android/transformer/CallLogEntry.java b/app/src/main/java/im/conversations/android/transformer/CallLogEntry.java new file mode 100644 index 000000000..4a6b64c38 --- /dev/null +++ b/app/src/main/java/im/conversations/android/transformer/CallLogEntry.java @@ -0,0 +1,11 @@ +package im.conversations.android.transformer; + +public class CallLogEntry { + public void setServerMsgId(String serverMsgId) {} + + public void setCarbon(boolean b) {} + + public void markUnread() {} + + public void setDuration(long duration) {} +} diff --git a/app/src/main/java/im/conversations/android/transformer/TransformationFactory.java b/app/src/main/java/im/conversations/android/transformer/TransformationFactory.java index aa17fe2cf..617bc8605 100644 --- a/app/src/main/java/im/conversations/android/transformer/TransformationFactory.java +++ b/app/src/main/java/im/conversations/android/transformer/TransformationFactory.java @@ -2,6 +2,7 @@ package im.conversations.android.transformer; import android.content.Context; import im.conversations.android.xml.Namespace; +import im.conversations.android.xmpp.Entity; import im.conversations.android.xmpp.XmppConnection; import im.conversations.android.xmpp.manager.DiscoManager; import im.conversations.android.xmpp.model.occupant.OccupantId; @@ -34,7 +35,8 @@ public class TransformationFactory extends XmppConnection.Delegate { if (message.getType() == Message.Type.GROUPCHAT && message.hasExtension(OccupantId.class)) { if (from != null && getManager(DiscoManager.class) - .hasFeature(from.asBareJid(), Namespace.OCCUPANT_ID)) { + .hasFeature( + Entity.discoItem(from.asBareJid()), Namespace.OCCUPANT_ID)) { occupantId = message.getExtension(OccupantId.class).getId(); } else { occupantId = null; diff --git a/app/src/main/java/im/conversations/android/ui/activity/RtpSessionActivity.java b/app/src/main/java/im/conversations/android/ui/activity/RtpSessionActivity.java new file mode 100644 index 000000000..660d90eb6 --- /dev/null +++ b/app/src/main/java/im/conversations/android/ui/activity/RtpSessionActivity.java @@ -0,0 +1,1494 @@ +package im.conversations.android.ui.activity; + +import static java.util.Arrays.asList; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.Activity; +import android.app.PictureInPictureParams; +import android.content.ActivityNotFoundException; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.PowerManager; +import android.os.SystemClock; +import android.util.Log; +import android.util.Rational; +import android.view.KeyEvent; +import android.view.Menu; +import android.view.MenuItem; +import android.view.View; +import android.view.WindowManager; +import android.widget.Toast; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import androidx.annotation.StringRes; +import androidx.databinding.DataBindingUtil; +import androidx.lifecycle.ViewModelProvider; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.services.AppRTCAudioManager; +import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection; +import eu.siacs.conversations.xmpp.jingle.ContentAddition; +import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection; +import eu.siacs.conversations.xmpp.jingle.Media; +import eu.siacs.conversations.xmpp.jingle.RtpEndUserState; +import im.conversations.android.R; +import im.conversations.android.database.model.Account; +import im.conversations.android.databinding.ActivityRtpSessionBinding; +import im.conversations.android.ui.Activities; +import im.conversations.android.ui.model.RtpSessionViewModel; +import im.conversations.android.util.MainThreadExecutor; +import im.conversations.android.util.PermissionUtils; +import im.conversations.android.util.Rationals; +import im.conversations.android.util.TimeFrameUtils; +import im.conversations.android.xmpp.manager.JingleConnectionManager; +import java.lang.ref.WeakReference; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.jxmpp.jid.Jid; +import org.jxmpp.jid.impl.JidCreate; +import org.webrtc.RendererCommon; +import org.webrtc.SurfaceViewRenderer; +import org.webrtc.VideoTrack; + +public class RtpSessionActivity extends BaseActivity + implements JingleConnectionManager.OnJingleRtpConnectionUpdate, + im.conversations.android.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged { + + public static final String EXTRA_ACCOUNT = "account"; + public static final String EXTRA_WITH = "with"; + public static final String EXTRA_SESSION_ID = "session_id"; + public static final String EXTRA_LAST_REPORTED_STATE = "last_reported_state"; + public static final String EXTRA_LAST_ACTION = "last_action"; + public static final String ACTION_ACCEPT_CALL = "action_accept_call"; + public static final String ACTION_MAKE_VOICE_CALL = "action_make_voice_call"; + public static final String ACTION_MAKE_VIDEO_CALL = "action_make_video_call"; + + private static final int CALL_DURATION_UPDATE_INTERVAL = 333; + + private static final List END_CARD = + Arrays.asList( + RtpEndUserState.APPLICATION_ERROR, + RtpEndUserState.SECURITY_ERROR, + RtpEndUserState.DECLINED_OR_BUSY, + RtpEndUserState.CONNECTIVITY_ERROR, + RtpEndUserState.CONNECTIVITY_LOST_ERROR, + RtpEndUserState.RETRACTED); + private static final List STATES_SHOWING_HELP_BUTTON = + Arrays.asList( + RtpEndUserState.APPLICATION_ERROR, + RtpEndUserState.CONNECTIVITY_ERROR, + RtpEndUserState.SECURITY_ERROR); + private static final List STATES_SHOWING_SWITCH_TO_CHAT = + Arrays.asList( + RtpEndUserState.CONNECTING, + RtpEndUserState.CONNECTED, + RtpEndUserState.RECONNECTING, + RtpEndUserState.INCOMING_CONTENT_ADD); + private static final List STATES_CONSIDERED_CONNECTED = + Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING); + private static final List STATES_SHOWING_PIP_PLACEHOLDER = + Arrays.asList( + RtpEndUserState.ACCEPTING_CALL, + RtpEndUserState.CONNECTING, + RtpEndUserState.RECONNECTING); + private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session"; + private static final int REQUEST_ACCEPT_CALL = 0x1111; + private static final int REQUEST_ACCEPT_CONTENT = 0x1112; + private static final int REQUEST_ADD_CONTENT = 0x1113; + private WeakReference rtpConnectionReference; + private JingleConnectionManager jingleConnectionManager; + + private ActivityRtpSessionBinding binding; + private RtpSessionViewModel viewModel; + private PowerManager.WakeLock mProximityWakeLock; + + private final Handler mHandler = new Handler(); + private final Runnable mTickExecutor = + new Runnable() { + @Override + public void run() { + updateCallDuration(); + mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL); + } + }; + + private static Set actionToMedia(final String action) { + if (ACTION_MAKE_VIDEO_CALL.equals(action)) { + return ImmutableSet.of(Media.AUDIO, Media.VIDEO); + } else { + return ImmutableSet.of(Media.AUDIO); + } + } + + private static void addSink( + final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) { + try { + videoTrack.addSink(surfaceViewRenderer); + } catch (final IllegalStateException e) { + Log.e( + Config.LOGTAG, + "possible race condition on trying to display video track. ignoring", + e); + } + } + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + getWindow() + .addFlags( + WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON + | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD + | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED + | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON); + this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session); + Activities.setStatusAndNavigationBarColors(this, binding.getRoot(), true); + final ViewModelProvider viewModelProvider = + new ViewModelProvider(this, getDefaultViewModelProviderFactory()); + this.viewModel = viewModelProvider.get(RtpSessionViewModel.class); + setSupportActionBar(binding.toolbar); + } + + @Override + public boolean onCreateOptionsMenu(final Menu menu) { + getMenuInflater().inflate(R.menu.activity_rtp_session, menu); + final MenuItem help = menu.findItem(R.id.action_help); + final MenuItem gotoChat = menu.findItem(R.id.action_goto_chat); + final MenuItem switchToVideo = menu.findItem(R.id.action_switch_to_video); + help.setVisible(Config.HELP != null && isHelpButtonVisible()); + gotoChat.setVisible(isSwitchToConversationVisible()); + switchToVideo.setVisible(isSwitchToVideoVisible()); + return super.onCreateOptionsMenu(menu); + } + + @Override + public boolean onKeyDown(final int keyCode, final KeyEvent event) { + if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) { + if (jingleConnectionManager != null) { + if (jingleConnectionManager.getNotificationService().stopSoundAndVibration()) { + return true; + } + } + } + return super.onKeyDown(keyCode, event); + } + + private boolean isHelpButtonVisible() { + try { + return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState()); + } catch (IllegalStateException e) { + final Intent intent = getIntent(); + final String state = + intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null; + if (state != null) { + return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state)); + } else { + return false; + } + } + } + + private boolean isSwitchToConversationVisible() { + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + return connection != null + && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState()); + } + + private boolean isSwitchToVideoVisible() { + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + if (connection == null) { + return false; + } + return connection.isSwitchToVideoAvailable(); + } + + private void switchToConversation() { + // TODO implement me + } + + public boolean onOptionsItemSelected(final MenuItem item) { + switch (item.getItemId()) { + case R.id.action_help: + launchHelpInBrowser(); + return true; + case R.id.action_goto_chat: + switchToConversation(); + return true; + case R.id.action_switch_to_video: + requestPermissionAndSwitchToVideo(); + return true; + } + return super.onOptionsItemSelected(item); + } + + private void launchHelpInBrowser() { + final Intent intent = new Intent(Intent.ACTION_VIEW, Config.HELP); + try { + startActivity(intent); + } catch (final ActivityNotFoundException e) { + Toast.makeText(this, R.string.no_application_found_to_open_link, Toast.LENGTH_LONG) + .show(); + } + } + + private void endCall(View view) { + endCall(); + } + + private void endCall() { + if (this.rtpConnectionReference == null) { + retractSessionProposal(); + finish(); + } else { + requireRtpConnection().endCall(); + } + } + + private void retractSessionProposal() { + final Intent intent = getIntent(); + final String action = intent.getAction(); + final Account account = requireAccount(); + final Jid with = JidCreate.fromOrThrowUnchecked(intent.getStringExtra(EXTRA_WITH)); + final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE); + if (!Intent.ACTION_VIEW.equals(action) + || state == null + || !END_CARD.contains(RtpEndUserState.valueOf(state))) { + resetIntent( + account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction())); + } + requireJingleConnectionManager().retractSessionProposal(with.asBareJid()); + } + + private void rejectCall(View view) { + requireRtpConnection().rejectCall(); + finish(); + } + + private void acceptCall(View view) { + requestPermissionsAndAcceptCall(); + } + + private void acceptContentAdd() { + try { + requireRtpConnection() + .acceptContentAdd(requireRtpConnection().getPendingContentAddition().summary); + } catch (final IllegalStateException e) { + Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show(); + } + } + + private void requestPermissionAndSwitchToVideo() { + final List permissions = permissions(ImmutableSet.of(Media.VIDEO, Media.AUDIO)); + if (PermissionUtils.hasPermission(this, permissions, REQUEST_ADD_CONTENT)) { + switchToVideo(); + } + } + + private void switchToVideo() { + try { + requireRtpConnection().addMedia(Media.VIDEO); + } catch (final IllegalStateException e) { + Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show(); + } + } + + private void acceptContentAdd(final ContentAddition contentAddition) { + if (contentAddition == null + || contentAddition.direction != ContentAddition.Direction.INCOMING) { + Log.d(Config.LOGTAG, "ignore press on content-accept button"); + return; + } + requestPermissionAndAcceptContentAdd(contentAddition); + } + + private void requestPermissionAndAcceptContentAdd(final ContentAddition contentAddition) { + final List permissions = permissions(contentAddition.media()); + if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CONTENT)) { + requireRtpConnection().acceptContentAdd(contentAddition.summary); + } + } + + private void rejectContentAdd(final View view) { + requireRtpConnection().rejectContentAdd(); + } + + private void requestPermissionsAndAcceptCall() { + final List permissions = permissions(getMedia()); + if (PermissionUtils.hasPermission(this, permissions, REQUEST_ACCEPT_CALL)) { + putScreenInCallMode(); + checkRecorderAndAcceptCall(); + } + } + + private List permissions(final Set media) { + final ImmutableList.Builder permissions = ImmutableList.builder(); + if (media.contains(Media.VIDEO)) { + permissions.add(Manifest.permission.CAMERA).add(Manifest.permission.RECORD_AUDIO); + } else { + permissions.add(Manifest.permission.RECORD_AUDIO); + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + permissions.add(Manifest.permission.BLUETOOTH_CONNECT); + } + return permissions.build(); + } + + private void checkRecorderAndAcceptCall() { + checkMicrophoneAvailabilityAsync(); + try { + requireRtpConnection().acceptCall(); + } catch (final IllegalStateException e) { + Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show(); + } + } + + private void checkMicrophoneAvailabilityAsync() { + new Thread(new MicrophoneAvailabilityCheck(this)).start(); + } + + private static class MicrophoneAvailabilityCheck implements Runnable { + + private final WeakReference activityReference; + + private MicrophoneAvailabilityCheck(final Activity activity) { + this.activityReference = new WeakReference<>(activity); + } + + @Override + public void run() { + final long start = SystemClock.elapsedRealtime(); + final boolean isMicrophoneAvailable = AppRTCAudioManager.isMicrophoneAvailable(); + final long stop = SystemClock.elapsedRealtime(); + Log.d(Config.LOGTAG, "checking microphone availability took " + (stop - start) + "ms"); + if (isMicrophoneAvailable) { + return; + } + final Activity activity = activityReference.get(); + if (activity == null) { + return; + } + activity.runOnUiThread( + () -> + Toast.makeText( + activity, + R.string.microphone_unavailable, + Toast.LENGTH_LONG) + .show()); + } + } + + private void putScreenInCallMode() { + putScreenInCallMode(requireRtpConnection().getMedia()); + } + + private void putScreenInCallMode(final Set media) { + getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + if (!media.contains(Media.VIDEO)) { + final JingleRtpConnection rtpConnection = + rtpConnectionReference != null ? rtpConnectionReference.get() : null; + final AppRTCAudioManager audioManager = + rtpConnection == null ? null : rtpConnection.getAudioManager(); + if (audioManager == null + || audioManager.getSelectedAudioDevice() + == AppRTCAudioManager.AudioDevice.EARPIECE) { + acquireProximityWakeLock(); + } + } + } + + @SuppressLint("WakelockTimeout") + private void acquireProximityWakeLock() { + final PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE); + if (powerManager == null) { + Log.e(Config.LOGTAG, "power manager not available"); + return; + } + if (isFinishing()) { + Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing"); + return; + } + if (this.mProximityWakeLock == null) { + this.mProximityWakeLock = + powerManager.newWakeLock( + PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG); + } + if (!this.mProximityWakeLock.isHeld()) { + Log.d(Config.LOGTAG, "acquiring proximity wake lock"); + this.mProximityWakeLock.acquire(); + } + } + + private void releaseProximityWakeLock() { + if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) { + Log.d(Config.LOGTAG, "releasing proximity wake lock"); + this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY); + this.mProximityWakeLock = null; + } + } + + private void putProximityWakeLockInProperState( + final AppRTCAudioManager.AudioDevice audioDevice) { + if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) { + acquireProximityWakeLock(); + } else { + releaseProximityWakeLock(); + } + } + + @Override + public void onNewIntent(final Intent intent) { + Log.d(Config.LOGTAG, this.getClass().getName() + ".onNewIntent()"); + super.onNewIntent(intent); + setIntent(intent); + if (jingleConnectionManager == null) { + Log.d( + Config.LOGTAG, + "RtpSessionActivity: background service wasn't bound in onNewIntent()"); + return; + } + final Account account = requireAccount(); + final String action = intent.getAction(); + final Jid with = JidCreate.fromOrThrowUnchecked(intent.getStringExtra(EXTRA_WITH)); + final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID); + if (sessionId != null) { + Log.d(Config.LOGTAG, "reinitializing from onNewIntent()"); + if (initializeActivityWithRunningRtpSession(account, with, sessionId)) { + return; + } + if (ACTION_ACCEPT_CALL.equals(intent.getAction())) { + Log.d(Config.LOGTAG, "accepting call from onNewIntent()"); + requestPermissionsAndAcceptCall(); + resetIntent(intent.getExtras()); + } + } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) { + proposeJingleRtpSession(account, with, actionToMedia(action)); + setWith(with, null); + } else { + throw new IllegalStateException("received onNewIntent without sessionId"); + } + } + + private void onBackendConnected(final JingleConnectionManager connectionManager) { + this.jingleConnectionManager = connectionManager; + connectionManager.setOnJingleRtpConnectionUpdate(this); + final Intent intent = getIntent(); + final String action = intent.getAction(); + final Account account = requireAccount(); + final Jid with = JidCreate.fromOrThrowUnchecked(intent.getStringExtra(EXTRA_WITH)); + final String sessionId = intent.getStringExtra(EXTRA_SESSION_ID); + if (sessionId != null) { + if (initializeActivityWithRunningRtpSession(account, with, sessionId)) { + return; + } + if (ACTION_ACCEPT_CALL.equals(intent.getAction())) { + Log.d(Config.LOGTAG, "intent action was accept"); + requestPermissionsAndAcceptCall(); + resetIntent(intent.getExtras()); + } + } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(action)) { + proposeJingleRtpSession(account, with, actionToMedia(action)); + setWith(with, null); + } else if (Intent.ACTION_VIEW.equals(action)) { + final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE); + final RtpEndUserState state = + extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState); + if (state != null) { + Log.d(Config.LOGTAG, "restored last state from intent extra"); + updateButtonConfiguration(state); + updateVerifiedShield(false); + updateStateDisplay(state); + updateIncomingCallScreen(state); + invalidateOptionsMenu(); + } + setWith(with, state); + if (connectionManager.fireJingleRtpConnectionStateUpdates()) { + return; + } + if (END_CARD.contains(state) || connectionManager.hasMatchingProposal(with)) { + return; + } + Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing"); + finish(); + } + } + + private void setWidth(final RtpEndUserState state) { + setWith(getWith(), state); + } + + private void setWith(final Jid contact, final RtpEndUserState state) { + // binding.with.setText(contact.getDisplayName()); + if (Arrays.asList(RtpEndUserState.INCOMING_CALL, RtpEndUserState.ACCEPTING_CALL) + .contains(state)) { + binding.withJid.setText(contact.asBareJid().toString()); + binding.withJid.setVisibility(View.VISIBLE); + } else { + binding.withJid.setVisibility(View.GONE); + } + } + + private void proposeJingleRtpSession( + final Account account, final Jid with, final Set media) { + checkMicrophoneAvailabilityAsync(); + if (with.hasNoResource()) { + requireJingleConnectionManager().proposeJingleRtpSession(with, media); + } else { + final String sessionId = + requireJingleConnectionManager().initializeRtpSession(with, media); + initializeActivityWithRunningRtpSession(account, with, sessionId); + resetIntent(account, with, sessionId); + } + putScreenInCallMode(media); + } + + @Override + public void onRequestPermissionsResult( + int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + final PermissionUtils.PermissionResult permissionResult = + PermissionUtils.removeBluetoothConnect(permissions, grantResults); + if (PermissionUtils.allGranted(permissionResult.grantResults)) { + if (requestCode == REQUEST_ACCEPT_CALL) { + checkRecorderAndAcceptCall(); + } else if (requestCode == REQUEST_ACCEPT_CONTENT) { + acceptContentAdd(); + } else if (requestCode == REQUEST_ADD_CONTENT) { + switchToVideo(); + } + } else { + @StringRes int res; + final String firstDenied = + PermissionUtils.getFirstDenied( + permissionResult.grantResults, permissionResult.permissions); + if (firstDenied == null) { + return; + } + if (Manifest.permission.RECORD_AUDIO.equals(firstDenied)) { + res = R.string.no_microphone_permission; + } else if (Manifest.permission.CAMERA.equals(firstDenied)) { + res = R.string.no_camera_permission; + } else { + throw new IllegalStateException("Invalid permission result request"); + } + Toast.makeText(this, getString(res, getString(R.string.app_name)), Toast.LENGTH_SHORT) + .show(); + } + } + + @Override + public void onStart() { + super.onStart(); + final Intent intent = getIntent(); + final long accountId = intent == null ? -1 : intent.getLongExtra(EXTRA_ACCOUNT, -1); + this.viewModel.setAccountId(accountId, this::onBackendConnected); + this.mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL); + this.binding.remoteVideo.setOnAspectRatioChanged(this); + } + + @Override + public void onStop() { + mHandler.removeCallbacks(mTickExecutor); + binding.remoteVideo.release(); + binding.remoteVideo.setOnAspectRatioChanged(null); + binding.localVideo.release(); + final WeakReference weakReference = this.rtpConnectionReference; + final JingleRtpConnection jingleRtpConnection = + weakReference == null ? null : weakReference.get(); + if (jingleRtpConnection != null) { + releaseVideoTracks(jingleRtpConnection); + } + releaseProximityWakeLock(); + super.onStop(); + } + + private void releaseVideoTracks(final JingleRtpConnection jingleRtpConnection) { + final Optional remoteVideo = jingleRtpConnection.getRemoteVideoTrack(); + if (remoteVideo.isPresent()) { + remoteVideo.get().removeSink(binding.remoteVideo); + } + final Optional localVideo = jingleRtpConnection.getLocalVideoTrack(); + if (localVideo.isPresent()) { + localVideo.get().removeSink(binding.localVideo); + } + } + + @Override + public void onBackPressed() { + if (isConnected()) { + if (switchToPictureInPicture()) { + return; + } + } else { + endCall(); + } + super.onBackPressed(); + } + + @Override + public void onUserLeaveHint() { + super.onUserLeaveHint(); + if (switchToPictureInPicture()) { + return; + } + // TODO apparently this method is not getting called on Android 10 when using the task + // switcher + if (emptyReference(rtpConnectionReference) && jingleConnectionManager != null) { + retractSessionProposal(); + } + } + + private boolean isConnected() { + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + final RtpEndUserState endUserState = + connection == null ? null : connection.getEndUserState(); + return STATES_CONSIDERED_CONNECTED.contains(endUserState) + || endUserState == RtpEndUserState.INCOMING_CONTENT_ADD; + } + + private boolean switchToPictureInPicture() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && deviceSupportsPictureInPicture()) { + if (shouldBePictureInPicture()) { + startPictureInPicture(); + return true; + } + } + return false; + } + + @RequiresApi(api = Build.VERSION_CODES.O) + private void startPictureInPicture() { + try { + final Rational rational = this.binding.remoteVideo.getAspectRatio(); + final Rational clippedRational = Rationals.clip(rational); + Log.d( + Config.LOGTAG, + "suggested rational " + rational + ". clipped to " + clippedRational); + enterPictureInPictureMode( + new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build()); + } catch (final IllegalStateException e) { + // this sometimes happens on Samsung phones (possibly when Knox is enabled) + Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e); + } + } + + @Override + public void onAspectRatioChanged(final Rational rational) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) { + final Rational clippedRational = Rationals.clip(rational); + Log.d( + Config.LOGTAG, + "suggested rational after aspect ratio change " + + rational + + ". clipped to " + + clippedRational); + setPictureInPictureParams( + new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build()); + } + } + + private boolean deviceSupportsPictureInPicture() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE); + } else { + return false; + } + } + + private boolean shouldBePictureInPicture() { + try { + final JingleRtpConnection rtpConnection = requireRtpConnection(); + return rtpConnection.getMedia().contains(Media.VIDEO) + && Arrays.asList( + RtpEndUserState.ACCEPTING_CALL, + RtpEndUserState.CONNECTING, + RtpEndUserState.CONNECTED) + .contains(rtpConnection.getEndUserState()); + } catch (final IllegalStateException e) { + return false; + } + } + + private boolean initializeActivityWithRunningRtpSession( + final Account account, Jid with, String sessionId) { + final WeakReference reference = + requireJingleConnectionManager().findJingleRtpConnection(with, sessionId); + if (reference == null || reference.get() == null) { + final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession = + requireJingleConnectionManager().getTerminalSessionState(with, sessionId); + if (terminatedRtpSession == null) { + throw new IllegalStateException( + "failed to initialize activity with running rtp session. session not" + + " found"); + } + initializeWithTerminatedSessionState(account, with, terminatedRtpSession); + return true; + } + this.rtpConnectionReference = reference; + final RtpEndUserState currentState = requireRtpConnection().getEndUserState(); + final boolean verified = requireRtpConnection().isVerified(); + if (currentState == RtpEndUserState.ENDED) { + finish(); + return true; + } + final Set media = getMedia(); + final ContentAddition contentAddition = getPendingContentAddition(); + if (currentState == RtpEndUserState.INCOMING_CALL) { + getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + } + if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains( + requireRtpConnection().getState())) { + putScreenInCallMode(); + } + setWidth(currentState); + updateVideoViews(currentState); + updateStateDisplay(currentState, media, contentAddition); + updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(currentState)); + updateButtonConfiguration(currentState, media, contentAddition); + updateIncomingCallScreen(currentState); + invalidateOptionsMenu(); + return false; + } + + private void initializeWithTerminatedSessionState( + final Account account, + final Jid with, + final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) { + Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()"); + if (terminatedRtpSession.state == RtpEndUserState.ENDED) { + finish(); + return; + } + final RtpEndUserState state = terminatedRtpSession.state; + resetIntent(account, with, terminatedRtpSession.state, terminatedRtpSession.media); + updateButtonConfiguration(state); + updateStateDisplay(state); + updateIncomingCallScreen(state); + updateCallDuration(); + updateVerifiedShield(false); + invalidateOptionsMenu(); + setWith(with, state); + } + + private void reInitializeActivityWithRunningRtpSession( + final Account account, Jid with, String sessionId) { + runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId)); + resetIntent(account, with, sessionId); + } + + private void resetIntent(final Account account, final Jid with, final String sessionId) { + final Intent intent = new Intent(Intent.ACTION_VIEW); + intent.putExtra(EXTRA_ACCOUNT, account.id); + intent.putExtra(EXTRA_WITH, with.toString()); + intent.putExtra(EXTRA_SESSION_ID, sessionId); + setIntent(intent); + } + + private void ensureSurfaceViewRendererIsSetup(final SurfaceViewRenderer surfaceViewRenderer) { + surfaceViewRenderer.setVisibility(View.VISIBLE); + try { + surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null); + } catch (final IllegalStateException e) { + // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized"); + } + surfaceViewRenderer.setEnableHardwareScaler(true); + } + + private void updateStateDisplay(final RtpEndUserState state) { + updateStateDisplay(state, Collections.emptySet(), null); + } + + private void updateStateDisplay( + final RtpEndUserState state, + final Set media, + final ContentAddition contentAddition) { + switch (state) { + case INCOMING_CALL: + Preconditions.checkArgument(media.size() > 0, "Media must not be empty"); + if (media.contains(Media.VIDEO)) { + setTitle(R.string.rtp_state_incoming_video_call); + } else { + setTitle(R.string.rtp_state_incoming_call); + } + break; + case INCOMING_CONTENT_ADD: + if (contentAddition != null && contentAddition.media().contains(Media.VIDEO)) { + setTitle(R.string.rtp_state_content_add_video); + } else { + setTitle(R.string.rtp_state_content_add); + } + break; + case CONNECTING: + setTitle(R.string.rtp_state_connecting); + break; + case CONNECTED: + setTitle(R.string.rtp_state_connected); + break; + case RECONNECTING: + setTitle(R.string.rtp_state_reconnecting); + break; + case ACCEPTING_CALL: + setTitle(R.string.rtp_state_accepting_call); + break; + case ENDING_CALL: + setTitle(R.string.rtp_state_ending_call); + break; + case FINDING_DEVICE: + setTitle(R.string.rtp_state_finding_device); + break; + case RINGING: + setTitle(R.string.rtp_state_ringing); + break; + case DECLINED_OR_BUSY: + setTitle(R.string.rtp_state_declined_or_busy); + break; + case CONNECTIVITY_ERROR: + setTitle(R.string.rtp_state_connectivity_error); + break; + case CONNECTIVITY_LOST_ERROR: + setTitle(R.string.rtp_state_connectivity_lost_error); + break; + case RETRACTED: + setTitle(R.string.rtp_state_retracted); + break; + case APPLICATION_ERROR: + setTitle(R.string.rtp_state_application_failure); + break; + case SECURITY_ERROR: + setTitle(R.string.rtp_state_security_error); + break; + case ENDED: + throw new IllegalStateException( + "Activity should have called finishAndReleaseWakeLock();"); + default: + throw new IllegalStateException( + String.format("State %s has not been handled in UI", state)); + } + } + + private void updateVerifiedShield(final boolean verified) { + if (isPictureInPicture()) { + this.binding.verified.setVisibility(View.GONE); + return; + } + this.binding.verified.setVisibility(verified ? View.VISIBLE : View.GONE); + } + + private void updateIncomingCallScreen(final RtpEndUserState state) { + updateIncomingCallScreen(state, null); + } + + private void updateIncomingCallScreen(final RtpEndUserState state, final Jid contact) { + if (state == RtpEndUserState.INCOMING_CALL || state == RtpEndUserState.ACCEPTING_CALL) { + final boolean show = getResources().getBoolean(R.bool.show_avatar_incoming_call); + /*if (show) { + binding.contactPhoto.setVisibility(View.VISIBLE); + if (contact == null) { + AvatarWorkerTask.loadAvatar( + getWith(), binding.contactPhoto, R.dimen.publish_avatar_size); + } else { + AvatarWorkerTask.loadAvatar( + contact, binding.contactPhoto, R.dimen.publish_avatar_size); + } + } else { + binding.contactPhoto.setVisibility(View.GONE); + }*/ + binding.usingAccount.setVisibility(View.VISIBLE); + binding.usingAccount.setText( + getString(R.string.using_account, requireAccount().address.toString())); + } else { + binding.usingAccount.setVisibility(View.GONE); + binding.contactPhoto.setVisibility(View.GONE); + } + } + + private Set getMedia() { + return requireRtpConnection().getMedia(); + } + + public ContentAddition getPendingContentAddition() { + return requireRtpConnection().getPendingContentAddition(); + } + + private void updateButtonConfiguration(final RtpEndUserState state) { + updateButtonConfiguration(state, Collections.emptySet(), null); + } + + @SuppressLint("RestrictedApi") + private void updateButtonConfiguration( + final RtpEndUserState state, + final Set media, + final ContentAddition contentAddition) { + if (state == RtpEndUserState.ENDING_CALL || isPictureInPicture()) { + this.binding.rejectCall.setVisibility(View.INVISIBLE); + this.binding.endCall.setVisibility(View.INVISIBLE); + this.binding.acceptCall.setVisibility(View.INVISIBLE); + } else if (state == RtpEndUserState.INCOMING_CALL) { + this.binding.rejectCall.setContentDescription(getString(R.string.dismiss_call)); + this.binding.rejectCall.setOnClickListener(this::rejectCall); + this.binding.rejectCall.setImageResource(R.drawable.ic_call_end_24dp); + this.binding.rejectCall.setVisibility(View.VISIBLE); + this.binding.endCall.setVisibility(View.INVISIBLE); + this.binding.acceptCall.setContentDescription(getString(R.string.answer_call)); + this.binding.acceptCall.setOnClickListener(this::acceptCall); + this.binding.acceptCall.setImageResource(R.drawable.ic_call_24dp); + this.binding.acceptCall.setVisibility(View.VISIBLE); + } else if (state == RtpEndUserState.INCOMING_CONTENT_ADD) { + this.binding.rejectCall.setContentDescription( + getString(R.string.reject_switch_to_video)); + this.binding.rejectCall.setOnClickListener(this::rejectContentAdd); + this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp); + this.binding.rejectCall.setVisibility(View.VISIBLE); + this.binding.endCall.setVisibility(View.INVISIBLE); + this.binding.acceptCall.setContentDescription(getString(R.string.accept)); + this.binding.acceptCall.setOnClickListener((v -> acceptContentAdd(contentAddition))); + this.binding.acceptCall.setImageResource(R.drawable.ic_check_24dp); + this.binding.acceptCall.setVisibility(View.VISIBLE); + } else if (state == RtpEndUserState.DECLINED_OR_BUSY) { + this.binding.rejectCall.setContentDescription(getString(R.string.exit)); + this.binding.rejectCall.setOnClickListener(this::exit); + this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp); + this.binding.rejectCall.setVisibility(View.VISIBLE); + this.binding.endCall.setVisibility(View.INVISIBLE); + this.binding.acceptCall.setContentDescription(getString(R.string.record_voice_mail)); + this.binding.acceptCall.setOnClickListener(this::recordVoiceMail); + this.binding.acceptCall.setImageResource(R.drawable.ic_voicemail_24dp); + this.binding.acceptCall.setVisibility(View.VISIBLE); + } else if (asList( + RtpEndUserState.CONNECTIVITY_ERROR, + RtpEndUserState.CONNECTIVITY_LOST_ERROR, + RtpEndUserState.APPLICATION_ERROR, + RtpEndUserState.RETRACTED, + RtpEndUserState.SECURITY_ERROR) + .contains(state)) { + this.binding.rejectCall.setContentDescription(getString(R.string.exit)); + this.binding.rejectCall.setOnClickListener(this::exit); + this.binding.rejectCall.setImageResource(R.drawable.ic_clear_24dp); + this.binding.rejectCall.setVisibility(View.VISIBLE); + this.binding.endCall.setVisibility(View.INVISIBLE); + this.binding.acceptCall.setContentDescription(getString(R.string.try_again)); + this.binding.acceptCall.setOnClickListener(this::retry); + this.binding.acceptCall.setImageResource(R.drawable.ic_replay_24dp); + this.binding.acceptCall.setVisibility(View.VISIBLE); + } else { + this.binding.rejectCall.setVisibility(View.INVISIBLE); + this.binding.endCall.setContentDescription(getString(R.string.hang_up)); + this.binding.endCall.setOnClickListener(this::endCall); + this.binding.endCall.setImageResource(R.drawable.ic_call_end_24dp); + this.binding.endCall.setVisibility(View.VISIBLE); + this.binding.acceptCall.setVisibility(View.INVISIBLE); + } + updateInCallButtonConfiguration(state, media); + } + + private boolean isPictureInPicture() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + return isInPictureInPictureMode(); + } else { + return false; + } + } + + private void updateInCallButtonConfiguration() { + updateInCallButtonConfiguration( + requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia()); + } + + @SuppressLint("RestrictedApi") + private void updateInCallButtonConfiguration( + final RtpEndUserState state, final Set media) { + if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) { + Preconditions.checkArgument(media.size() > 0, "Media must not be empty"); + if (media.contains(Media.VIDEO)) { + final JingleRtpConnection rtpConnection = requireRtpConnection(); + updateInCallButtonConfigurationVideo( + rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable()); + } else { + final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager(); + updateInCallButtonConfigurationSpeaker( + audioManager.getSelectedAudioDevice(), + audioManager.getAudioDevices().size()); + this.binding.inCallActionFarRight.setVisibility(View.GONE); + } + if (media.contains(Media.AUDIO)) { + updateInCallButtonConfigurationMicrophone( + requireRtpConnection().isMicrophoneEnabled()); + } else { + this.binding.inCallActionLeft.setVisibility(View.GONE); + } + } else { + this.binding.inCallActionLeft.setVisibility(View.GONE); + this.binding.inCallActionRight.setVisibility(View.GONE); + this.binding.inCallActionFarRight.setVisibility(View.GONE); + } + } + + @SuppressLint("RestrictedApi") + private void updateInCallButtonConfigurationSpeaker( + final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) { + switch (selectedAudioDevice) { + case EARPIECE: + this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_24dp); + if (numberOfChoices >= 2) { + this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker); + } else { + this.binding.inCallActionRight.setOnClickListener(null); + this.binding.inCallActionRight.setClickable(false); + } + break; + case WIRED_HEADSET: + this.binding.inCallActionRight.setImageResource(R.drawable.ic_headset_24dp); + this.binding.inCallActionRight.setOnClickListener(null); + this.binding.inCallActionRight.setClickable(false); + break; + case SPEAKER_PHONE: + this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_up_24dp); + if (numberOfChoices >= 2) { + this.binding.inCallActionRight.setOnClickListener(this::switchToEarpiece); + } else { + this.binding.inCallActionRight.setOnClickListener(null); + this.binding.inCallActionRight.setClickable(false); + } + break; + case BLUETOOTH: + this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_24dp); + this.binding.inCallActionRight.setOnClickListener(null); + this.binding.inCallActionRight.setClickable(false); + break; + } + this.binding.inCallActionRight.setVisibility(View.VISIBLE); + } + + @SuppressLint("RestrictedApi") + private void updateInCallButtonConfigurationVideo( + final boolean videoEnabled, final boolean isCameraSwitchable) { + this.binding.inCallActionRight.setVisibility(View.VISIBLE); + if (isCameraSwitchable) { + this.binding.inCallActionFarRight.setImageResource( + R.drawable.ic_flip_camera_android_24dp); + this.binding.inCallActionFarRight.setVisibility(View.VISIBLE); + this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera); + } else { + this.binding.inCallActionFarRight.setVisibility(View.GONE); + } + if (videoEnabled) { + this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_24dp); + this.binding.inCallActionRight.setOnClickListener(this::disableVideo); + } else { + this.binding.inCallActionRight.setImageResource(R.drawable.ic_videocam_off_24dp); + this.binding.inCallActionRight.setOnClickListener(this::enableVideo); + } + } + + private void switchCamera(final View view) { + Futures.addCallback( + requireRtpConnection().switchCamera(), + new FutureCallback() { + @Override + public void onSuccess(@Nullable Boolean isFrontCamera) { + binding.localVideo.setMirror(isFrontCamera); + } + + @Override + public void onFailure(@NonNull final Throwable throwable) { + Log.d( + Config.LOGTAG, + "could not switch camera", + Throwables.getRootCause(throwable)); + Toast.makeText( + RtpSessionActivity.this, + R.string.could_not_switch_camera, + Toast.LENGTH_LONG) + .show(); + } + }, + MainThreadExecutor.getInstance()); + } + + private void enableVideo(View view) { + try { + requireRtpConnection().setVideoEnabled(true); + } catch (final IllegalStateException e) { + Toast.makeText(this, R.string.unable_to_enable_video, Toast.LENGTH_SHORT).show(); + return; + } + updateInCallButtonConfigurationVideo(true, requireRtpConnection().isCameraSwitchable()); + } + + private void disableVideo(View view) { + final JingleRtpConnection rtpConnection = requireRtpConnection(); + final ContentAddition pending = rtpConnection.getPendingContentAddition(); + if (pending != null && pending.direction == ContentAddition.Direction.OUTGOING) { + rtpConnection.retractContentAdd(); + return; + } + requireRtpConnection().setVideoEnabled(false); + updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable()); + } + + @SuppressLint("RestrictedApi") + private void updateInCallButtonConfigurationMicrophone(final boolean microphoneEnabled) { + if (microphoneEnabled) { + this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_24dp); + this.binding.inCallActionLeft.setOnClickListener(this::disableMicrophone); + } else { + this.binding.inCallActionLeft.setImageResource(R.drawable.ic_mic_off_24dp); + this.binding.inCallActionLeft.setOnClickListener(this::enableMicrophone); + } + this.binding.inCallActionLeft.setVisibility(View.VISIBLE); + } + + private void updateCallDuration() { + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + if (connection == null || connection.getMedia().contains(Media.VIDEO)) { + this.binding.duration.setVisibility(View.GONE); + return; + } + if (connection.zeroDuration()) { + this.binding.duration.setVisibility(View.GONE); + } else { + this.binding.duration.setText( + TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false)); + this.binding.duration.setVisibility(View.VISIBLE); + } + } + + private void updateVideoViews(final RtpEndUserState state) { + if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) { + binding.localVideo.setVisibility(View.GONE); + binding.localVideo.release(); + binding.remoteVideoWrapper.setVisibility(View.GONE); + binding.remoteVideo.release(); + binding.pipLocalMicOffIndicator.setVisibility(View.GONE); + if (isPictureInPicture()) { + binding.appBarLayout.setVisibility(View.GONE); + binding.pipPlaceholder.setVisibility(View.VISIBLE); + if (Arrays.asList( + RtpEndUserState.APPLICATION_ERROR, + RtpEndUserState.CONNECTIVITY_ERROR, + RtpEndUserState.SECURITY_ERROR) + .contains(state)) { + binding.pipWarning.setVisibility(View.VISIBLE); + binding.pipWaiting.setVisibility(View.GONE); + } else { + binding.pipWarning.setVisibility(View.GONE); + binding.pipWaiting.setVisibility(View.GONE); + } + } else { + binding.appBarLayout.setVisibility(View.VISIBLE); + binding.pipPlaceholder.setVisibility(View.GONE); + } + getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); + return; + } + if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) { + binding.localVideo.setVisibility(View.GONE); + binding.remoteVideoWrapper.setVisibility(View.GONE); + binding.appBarLayout.setVisibility(View.GONE); + binding.pipPlaceholder.setVisibility(View.VISIBLE); + binding.pipWarning.setVisibility(View.GONE); + binding.pipWaiting.setVisibility(View.VISIBLE); + binding.pipLocalMicOffIndicator.setVisibility(View.GONE); + return; + } + final Optional localVideoTrack = getLocalVideoTrack(); + if (localVideoTrack.isPresent() && !isPictureInPicture()) { + ensureSurfaceViewRendererIsSetup(binding.localVideo); + // paint local view over remote view + binding.localVideo.setZOrderMediaOverlay(true); + binding.localVideo.setMirror(requireRtpConnection().isFrontCamera()); + addSink(localVideoTrack.get(), binding.localVideo); + } else { + binding.localVideo.setVisibility(View.GONE); + } + final Optional remoteVideoTrack = getRemoteVideoTrack(); + if (remoteVideoTrack.isPresent()) { + ensureSurfaceViewRendererIsSetup(binding.remoteVideo); + addSink(remoteVideoTrack.get(), binding.remoteVideo); + binding.remoteVideo.setScalingType( + RendererCommon.ScalingType.SCALE_ASPECT_FILL, + RendererCommon.ScalingType.SCALE_ASPECT_FIT); + if (state == RtpEndUserState.CONNECTED) { + binding.appBarLayout.setVisibility(View.GONE); + getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); + binding.remoteVideoWrapper.setVisibility(View.VISIBLE); + } else { + binding.appBarLayout.setVisibility(View.VISIBLE); + getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); + binding.remoteVideoWrapper.setVisibility(View.GONE); + } + if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) { + binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE); + } else { + binding.pipLocalMicOffIndicator.setVisibility(View.GONE); + } + } else { + getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); + binding.remoteVideoWrapper.setVisibility(View.GONE); + binding.pipLocalMicOffIndicator.setVisibility(View.GONE); + } + } + + private Optional getLocalVideoTrack() { + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + if (connection == null) { + return Optional.absent(); + } + return connection.getLocalVideoTrack(); + } + + private Optional getRemoteVideoTrack() { + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + if (connection == null) { + return Optional.absent(); + } + return connection.getRemoteVideoTrack(); + } + + private void disableMicrophone(View view) { + final JingleRtpConnection rtpConnection = requireRtpConnection(); + if (rtpConnection.setMicrophoneEnabled(false)) { + updateInCallButtonConfiguration(); + } + } + + private void enableMicrophone(View view) { + final JingleRtpConnection rtpConnection = requireRtpConnection(); + if (rtpConnection.setMicrophoneEnabled(true)) { + updateInCallButtonConfiguration(); + } + } + + private void switchToEarpiece(View view) { + requireRtpConnection() + .getAudioManager() + .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE); + acquireProximityWakeLock(); + } + + private void switchToSpeaker(View view) { + requireRtpConnection() + .getAudioManager() + .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE); + releaseProximityWakeLock(); + } + + private void retry(View view) { + final Intent intent = getIntent(); + final Account account = requireAccount(); + final Jid with = JidCreate.fromOrThrowUnchecked(intent.getStringExtra(EXTRA_WITH)); + final String lastAction = intent.getStringExtra(EXTRA_LAST_ACTION); + final String action = intent.getAction(); + final Set media = actionToMedia(lastAction == null ? action : lastAction); + this.rtpConnectionReference = null; + Log.d(Config.LOGTAG, "attempting retry with " + with.toString()); + proposeJingleRtpSession(account, with, media); + } + + private void exit(final View view) { + finish(); + } + + private void recordVoiceMail(final View view) { + final Intent intent = getIntent(); + final Account account = requireAccount(); + final Jid with = JidCreate.fromOrThrowUnchecked(intent.getStringExtra(EXTRA_WITH)); + // TODO fix voice mail + /*final Conversation conversation = + xmppConnectionService.findOrCreateConversation(account, with, false, true); + final Intent launchIntent = new Intent(this, ConversationsActivity.class); + launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION); + launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid()); + launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP); + launchIntent.putExtra( + ConversationsActivity.EXTRA_POST_INIT_ACTION, + ConversationsActivity.POST_ACTION_RECORD_VOICE); + startActivity(launchIntent); + finish();*/ + } + + private Jid getWith() { + final AbstractJingleConnection.Id id = requireRtpConnection().getId(); + return id.getWith(); + } + + private JingleRtpConnection requireRtpConnection() { + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + if (connection == null) { + throw new IllegalStateException("No RTP connection found"); + } + return connection; + } + + private JingleConnectionManager requireJingleConnectionManager() { + final var manager = this.jingleConnectionManager; + if (manager == null) { + throw new IllegalStateException("JingleConnectionManager has not been connected yet"); + } + return manager; + } + + private Account requireAccount() { + return requireJingleConnectionManager().getAccount(); + } + + @Override + public void onJingleRtpConnectionUpdate( + Jid with, final String sessionId, RtpEndUserState state) { + Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")"); + if (END_CARD.contains(state)) { + Log.d(Config.LOGTAG, "end card reached"); + releaseProximityWakeLock(); + runOnUiThread( + () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)); + } + if (with.hasNoResource()) { + updateRtpSessionProposalState(requireAccount(), with, state); + return; + } + if (emptyReference(this.rtpConnectionReference)) { + if (END_CARD.contains(state)) { + Log.d(Config.LOGTAG, "not reinitializing session"); + return; + } + // this happens when going from proposed session to actual session + reInitializeActivityWithRunningRtpSession(requireAccount(), with, sessionId); + return; + } + final AbstractJingleConnection.Id id = requireRtpConnection().getId(); + final boolean verified = requireRtpConnection().isVerified(); + final Set media = getMedia(); + final ContentAddition contentAddition = getPendingContentAddition(); + final Jid contact = getWith(); + if (id.with.equals(with) && id.sessionId.equals(sessionId)) { + if (state == RtpEndUserState.ENDED) { + finish(); + return; + } + runOnUiThread( + () -> { + updateStateDisplay(state, media, contentAddition); + updateVerifiedShield( + verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state)); + updateButtonConfiguration(state, media, contentAddition); + updateVideoViews(state); + updateIncomingCallScreen(state, contact); + invalidateOptionsMenu(); + }); + if (END_CARD.contains(state)) { + final JingleRtpConnection rtpConnection = requireRtpConnection(); + resetIntent(requireAccount(), with, state, rtpConnection.getMedia()); + releaseVideoTracks(rtpConnection); + this.rtpConnectionReference = null; + } + } else { + Log.d(Config.LOGTAG, "received update for other rtp session"); + } + } + + @Override + public void onAudioDeviceChanged( + final AppRTCAudioManager.AudioDevice selectedAudioDevice, + final Set availableAudioDevices) { + Log.d( + Config.LOGTAG, + "onAudioDeviceChanged in activity: selected:" + + selectedAudioDevice + + ", available:" + + availableAudioDevices); + try { + final RtpEndUserState endUserState = requireRtpConnection().getEndUserState(); + final Set media = getMedia(); + if (END_CARD.contains(endUserState)) { + Log.d( + Config.LOGTAG, + "onAudioDeviceChanged() nothing to do because end card has been reached"); + } else { + if (Media.audioOnly(media) && endUserState == RtpEndUserState.CONNECTED) { + final AppRTCAudioManager audioManager = + requireRtpConnection().getAudioManager(); + updateInCallButtonConfigurationSpeaker( + audioManager.getSelectedAudioDevice(), + audioManager.getAudioDevices().size()); + } + Log.d( + Config.LOGTAG, + "put proximity wake lock into proper state after device update"); + putProximityWakeLockInProperState(selectedAudioDevice); + } + } catch (final IllegalStateException e) { + Log.d(Config.LOGTAG, "RTP connection was not available when audio device changed"); + } + } + + private void updateRtpSessionProposalState( + final Account account, final Jid with, final RtpEndUserState state) { + final Intent currentIntent = getIntent(); + final String withExtra = + currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH); + if (withExtra == null) { + return; + } + if (JidCreate.fromOrThrowUnchecked(withExtra).asBareJid().equals(with)) { + runOnUiThread( + () -> { + updateVerifiedShield(false); + updateStateDisplay(state); + updateButtonConfiguration(state); + updateIncomingCallScreen(state); + invalidateOptionsMenu(); + }); + resetIntent(account, with, state, actionToMedia(currentIntent.getAction())); + } + } + + private void resetIntent(final Bundle extras) { + final Intent intent = new Intent(Intent.ACTION_VIEW); + intent.putExtras(extras); + setIntent(intent); + } + + private void resetIntent( + final Account account, Jid with, final RtpEndUserState state, final Set media) { + final Intent intent = new Intent(Intent.ACTION_VIEW); + intent.putExtra(EXTRA_ACCOUNT, account.id); + /*if (account.getRoster() + .getContact(with) + .getPresences() + .anySupport(Namespace.JINGLE_MESSAGE)) { + intent.putExtra(EXTRA_WITH, with.asBareJid().toString()); + } else { + intent.putExtra(EXTRA_WITH, with.toString()); + }*/ + intent.putExtra(EXTRA_WITH, with.toString()); + intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString()); + intent.putExtra( + EXTRA_LAST_ACTION, + media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL); + setIntent(intent); + } + + private static boolean emptyReference(final WeakReference weakReference) { + return weakReference == null || weakReference.get() == null; + } +} diff --git a/app/src/main/java/im/conversations/android/ui/model/RtpSessionViewModel.java b/app/src/main/java/im/conversations/android/ui/model/RtpSessionViewModel.java new file mode 100644 index 000000000..d5b3a89d2 --- /dev/null +++ b/app/src/main/java/im/conversations/android/ui/model/RtpSessionViewModel.java @@ -0,0 +1,50 @@ +package im.conversations.android.ui.model; + +import android.app.Application; +import androidx.annotation.NonNull; +import androidx.lifecycle.AndroidViewModel; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; +import im.conversations.android.xmpp.ConnectionPool; +import im.conversations.android.xmpp.manager.JingleConnectionManager; +import java.util.function.Consumer; + +public class RtpSessionViewModel extends AndroidViewModel { + + private long accountId; + + public RtpSessionViewModel(@NonNull Application application) { + super(application); + } + + public void setAccountId( + final long accountId, final Consumer jmcConsumer) { + this.accountId = accountId; + this.connectJingleConnectionManager(accountId, jmcConsumer); + } + + private void connectJingleConnectionManager( + long accountId, final Consumer jmcConsumer) { + final var connectionFuture = ConnectionPool.getInstance(getApplication()).get(accountId); + final var jcmFuture = + Futures.transform( + connectionFuture, + connection -> connection.getManager(JingleConnectionManager.class), + MoreExecutors.directExecutor()); + Futures.addCallback( + jcmFuture, + new FutureCallback<>() { + @Override + public void onSuccess(JingleConnectionManager manager) { + jmcConsumer.accept(manager); + } + + @Override + public void onFailure(Throwable t) { + // TODO show warning in activity + } + }, + MoreExecutors.directExecutor()); + } +} diff --git a/app/src/main/java/im/conversations/android/ui/widget/SurfaceViewRenderer.java b/app/src/main/java/im/conversations/android/ui/widget/SurfaceViewRenderer.java new file mode 100644 index 000000000..6f400a79e --- /dev/null +++ b/app/src/main/java/im/conversations/android/ui/widget/SurfaceViewRenderer.java @@ -0,0 +1,55 @@ +package im.conversations.android.ui.widget; + +import android.content.Context; +import android.util.AttributeSet; +import android.util.Log; +import android.util.Rational; +import eu.siacs.conversations.Config; + +public class SurfaceViewRenderer extends org.webrtc.SurfaceViewRenderer { + + private Rational aspectRatio = new Rational(1, 1); + + private OnAspectRatioChanged onAspectRatioChanged; + + public SurfaceViewRenderer(Context context) { + super(context); + } + + public SurfaceViewRenderer(Context context, AttributeSet attrs) { + super(context, attrs); + } + + public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation) { + super.onFrameResolutionChanged(videoWidth, videoHeight, rotation); + final int rotatedWidth = rotation != 0 && rotation != 180 ? videoHeight : videoWidth; + final int rotatedHeight = rotation != 0 && rotation != 180 ? videoWidth : videoHeight; + final Rational currentRational = this.aspectRatio; + this.aspectRatio = new Rational(rotatedWidth, rotatedHeight); + Log.d( + Config.LOGTAG, + "onFrameResolutionChanged(" + + rotatedWidth + + "," + + rotatedHeight + + "," + + aspectRatio + + ")"); + if (currentRational.equals(this.aspectRatio) || onAspectRatioChanged == null) { + return; + } + onAspectRatioChanged.onAspectRatioChanged(this.aspectRatio); + } + + public void setOnAspectRatioChanged(final OnAspectRatioChanged onAspectRatioChanged) { + this.onAspectRatioChanged = onAspectRatioChanged; + } + + public Rational getAspectRatio() { + return this.aspectRatio; + } + + public interface OnAspectRatioChanged { + void onAspectRatioChanged(final Rational rational); + } +} diff --git a/app/src/main/java/im/conversations/android/util/MainThreadExecutor.java b/app/src/main/java/im/conversations/android/util/MainThreadExecutor.java new file mode 100644 index 000000000..6612311d1 --- /dev/null +++ b/app/src/main/java/im/conversations/android/util/MainThreadExecutor.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 Daniel Gultsch + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.conversations.android.util; + +import android.os.Handler; +import android.os.Looper; +import java.util.concurrent.Executor; + +public class MainThreadExecutor implements Executor { + + private static final MainThreadExecutor INSTANCE = new MainThreadExecutor(); + + private final Handler handler = new Handler(Looper.myLooper()); + + @Override + public void execute(final Runnable command) { + handler.post(command); + } + + public static MainThreadExecutor getInstance() { + return INSTANCE; + } +} diff --git a/app/src/main/java/im/conversations/android/util/PermissionUtils.java b/app/src/main/java/im/conversations/android/util/PermissionUtils.java new file mode 100644 index 000000000..09d9587ba --- /dev/null +++ b/app/src/main/java/im/conversations/android/util/PermissionUtils.java @@ -0,0 +1,91 @@ +package im.conversations.android.util; + +import android.Manifest; +import android.app.Activity; +import android.content.pm.PackageManager; +import android.os.Build; +import androidx.core.app.ActivityCompat; +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Ints; +import java.util.ArrayList; +import java.util.List; + +public class PermissionUtils { + + public static boolean allGranted(int[] grantResults) { + for (int grantResult : grantResults) { + if (grantResult != PackageManager.PERMISSION_GRANTED) { + return false; + } + } + return true; + } + + public static boolean writeGranted(int[] grantResults, String[] permission) { + for (int i = 0; i < grantResults.length; ++i) { + if (Manifest.permission.WRITE_EXTERNAL_STORAGE.equals(permission[i])) { + return grantResults[i] == PackageManager.PERMISSION_GRANTED; + } + } + return false; + } + + public static String getFirstDenied(int[] grantResults, String[] permissions) { + for (int i = 0; i < grantResults.length; ++i) { + if (grantResults[i] == PackageManager.PERMISSION_DENIED) { + return permissions[i]; + } + } + return null; + } + + public static class PermissionResult { + public final String[] permissions; + public final int[] grantResults; + + public PermissionResult(String[] permissions, int[] grantResults) { + this.permissions = permissions; + this.grantResults = grantResults; + } + } + + public static PermissionResult removeBluetoothConnect( + final String[] inPermissions, final int[] inGrantResults) { + final List outPermissions = new ArrayList<>(); + final List outGrantResults = new ArrayList<>(); + for (int i = 0; i < Math.min(inPermissions.length, inGrantResults.length); ++i) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + if (inPermissions[i].equals(Manifest.permission.BLUETOOTH_CONNECT)) { + continue; + } + } + outPermissions.add(inPermissions[i]); + outGrantResults.add(inGrantResults[i]); + } + + return new PermissionResult( + outPermissions.toArray(new String[0]), Ints.toArray(outGrantResults)); + } + + public static boolean hasPermission( + final Activity activity, final List permissions, final int requestCode) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + final ImmutableList.Builder missingPermissions = new ImmutableList.Builder<>(); + for (final String permission : permissions) { + if (ActivityCompat.checkSelfPermission(activity, permission) + != PackageManager.PERMISSION_GRANTED) { + missingPermissions.add(permission); + } + } + final ImmutableList missing = missingPermissions.build(); + if (missing.size() == 0) { + return true; + } + ActivityCompat.requestPermissions( + activity, missing.toArray(new String[0]), requestCode); + return false; + } else { + return true; + } + } +} diff --git a/app/src/main/java/im/conversations/android/util/Rationals.java b/app/src/main/java/im/conversations/android/util/Rationals.java new file mode 100644 index 000000000..d42e4f10d --- /dev/null +++ b/app/src/main/java/im/conversations/android/util/Rationals.java @@ -0,0 +1,22 @@ +package im.conversations.android.util; + +import android.util.Rational; + +public final class Rationals { + + // between 2.39:1 and 1:2.39 (inclusive). + private static final Rational MIN = new Rational(100, 239); + private static final Rational MAX = new Rational(239, 100); + + private Rationals() {} + + public static Rational clip(final Rational input) { + if (input.compareTo(MIN) < 0) { + return MIN; + } + if (input.compareTo(MAX) > 0) { + return MAX; + } + return input; + } +} diff --git a/app/src/main/java/im/conversations/android/util/TimeFrameUtils.java b/app/src/main/java/im/conversations/android/util/TimeFrameUtils.java new file mode 100644 index 000000000..f9026cece --- /dev/null +++ b/app/src/main/java/im/conversations/android/util/TimeFrameUtils.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2018, Daniel Gultsch All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package im.conversations.android.util; + +import android.content.Context; +import android.os.SystemClock; +import androidx.annotation.PluralsRes; +import im.conversations.android.R; +import java.util.Locale; + +public class TimeFrameUtils { + + private static final TimeFrame[] TIME_FRAMES; + + static { + TIME_FRAMES = + new TimeFrame[] { + new TimeFrame(1000L, R.plurals.seconds), + new TimeFrame(60L * 1000, R.plurals.minutes), + new TimeFrame(60L * 60 * 1000, R.plurals.hours), + new TimeFrame(24L * 60 * 60 * 1000, R.plurals.days), + new TimeFrame(7L * 24 * 60 * 60 * 1000, R.plurals.weeks), + new TimeFrame(30L * 24 * 60 * 60 * 1000, R.plurals.months), + }; + } + + public static String resolve(Context context, long timeFrame) { + for (int i = TIME_FRAMES.length - 1; i >= 0; --i) { + long duration = TIME_FRAMES[i].duration; + long threshold = i > 0 ? (TIME_FRAMES[i - 1].duration / 2) : 0; + if (timeFrame >= duration - threshold) { + int count = + (int) + (timeFrame / duration + + ((timeFrame % duration) > (duration / 2) ? 1 : 0)); + return context.getResources().getQuantityString(TIME_FRAMES[i].name, count, count); + } + } + return context.getResources().getQuantityString(TIME_FRAMES[0].name, 0, 0); + } + + public static String formatTimePassed(final long since, final boolean withMilliseconds) { + return formatTimePassed(since, SystemClock.elapsedRealtime(), withMilliseconds); + } + + public static String formatTimePassed( + final long since, final long to, final boolean withMilliseconds) { + final long passed = (since < 0) ? 0 : (to - since); + return formatElapsedTime(passed, withMilliseconds); + } + + public static String formatElapsedTime(final long elapsed, final boolean withMilliseconds) { + final int hours = (int) (elapsed / 3600000); + final int minutes = (int) (elapsed / 60000) % 60; + final int seconds = (int) (elapsed / 1000) % 60; + final int milliseconds = (int) (elapsed / 100) % 10; + if (hours > 0) { + return String.format(Locale.ENGLISH, "%d:%02d:%02d", hours, minutes, seconds); + } else if (withMilliseconds) { + return String.format(Locale.ENGLISH, "%d:%02d.%d", minutes, seconds, milliseconds); + } else { + return String.format(Locale.ENGLISH, "%d:%02d", minutes, seconds); + } + } + + private static class TimeFrame { + final long duration; + public final int name; + + private TimeFrame(long duration, @PluralsRes int name) { + this.duration = duration; + this.name = name; + } + } +} diff --git a/app/src/main/java/im/conversations/android/xml/Element.java b/app/src/main/java/im/conversations/android/xml/Element.java index 9dd608f8d..cb3a6d216 100644 --- a/app/src/main/java/im/conversations/android/xml/Element.java +++ b/app/src/main/java/im/conversations/android/xml/Element.java @@ -155,6 +155,10 @@ public class Element { return this.setAttribute(name, value == null ? null : value.toString()); } + public void setAttribute(final String name, final boolean value) { + this.setAttribute(name, value ? "1" : "0"); + } + public void removeAttribute(final String name) { this.attributes.remove(name); } diff --git a/app/src/main/java/im/conversations/android/xmpp/Managers.java b/app/src/main/java/im/conversations/android/xmpp/Managers.java index 882335eac..6eaa90e7d 100644 --- a/app/src/main/java/im/conversations/android/xmpp/Managers.java +++ b/app/src/main/java/im/conversations/android/xmpp/Managers.java @@ -12,6 +12,7 @@ import im.conversations.android.xmpp.manager.BookmarkManager; import im.conversations.android.xmpp.manager.CarbonsManager; import im.conversations.android.xmpp.manager.ChatStateManager; import im.conversations.android.xmpp.manager.DiscoManager; +import im.conversations.android.xmpp.manager.ExternalDiscoManager; import im.conversations.android.xmpp.manager.HttpUploadManager; import im.conversations.android.xmpp.manager.JingleConnectionManager; import im.conversations.android.xmpp.manager.NickManager; @@ -38,6 +39,7 @@ public final class Managers { .put(CarbonsManager.class, new CarbonsManager(context, connection)) .put(ChatStateManager.class, new ChatStateManager(context, connection)) .put(DiscoManager.class, new DiscoManager(context, connection)) + .put(ExternalDiscoManager.class, new ExternalDiscoManager(context, connection)) .put(HttpUploadManager.class, new HttpUploadManager(context, connection)) .put( JingleConnectionManager.class, diff --git a/app/src/main/java/im/conversations/android/xmpp/XmppConnection.java b/app/src/main/java/im/conversations/android/xmpp/XmppConnection.java index 35c4e7113..8704e3942 100644 --- a/app/src/main/java/im/conversations/android/xmpp/XmppConnection.java +++ b/app/src/main/java/im/conversations/android/xmpp/XmppConnection.java @@ -17,6 +17,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; +import im.conversations.android.AbstractAccountService; import im.conversations.android.BuildConfig; import im.conversations.android.Conversations; import im.conversations.android.IDs; @@ -105,17 +106,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xmlpull.v1.XmlPullParserException; -public class XmppConnection implements Runnable { +public class XmppConnection extends AbstractAccountService implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(XmppConnection.class); private static final boolean EXTENDED_SM_LOGGING = false; private static final int CONNECT_DISCO_TIMEOUT = 20; - protected final Account account; private final SparseArray mStanzaQueue = new SparseArray<>(); private final Hashtable>> packetCallbacks = new Hashtable<>(); - private final Context context; private Socket socket; private XmlReader tagReader; private TagWriter tagWriter = new TagWriter(); @@ -156,8 +155,7 @@ public class XmppConnection implements Runnable { private CountDownLatch mStreamCountDownLatch; public XmppConnection(final Context context, final Account account) { - this.context = context; - this.account = account; + super(context, account); this.connectionAddress = account.address; // these consumers are pure listeners; they don’t have public method except for accept|apply diff --git a/app/src/main/java/im/conversations/android/xmpp/manager/AxolotlManager.java b/app/src/main/java/im/conversations/android/xmpp/manager/AxolotlManager.java index 38f90869d..5bfbcbc25 100644 --- a/app/src/main/java/im/conversations/android/xmpp/manager/AxolotlManager.java +++ b/app/src/main/java/im/conversations/android/xmpp/manager/AxolotlManager.java @@ -2,23 +2,39 @@ package im.conversations.android.xmpp.manager; import android.content.Context; import androidx.annotation.NonNull; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import im.conversations.android.database.AxolotlDatabaseStore; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.xmpp.jingle.OmemoVerification; +import eu.siacs.conversations.xmpp.jingle.OmemoVerifiedRtpContentMap; +import eu.siacs.conversations.xmpp.jingle.RtpContentMap; +import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; +import eu.siacs.conversations.xmpp.jingle.stanzas.OmemoVerifiedIceUdpTransportInfo; +import im.conversations.android.axolotl.AxolotlAddress; +import im.conversations.android.axolotl.AxolotlDecryptionException; +import im.conversations.android.axolotl.AxolotlEncryptionException; +import im.conversations.android.axolotl.AxolotlPayload; +import im.conversations.android.axolotl.AxolotlService; +import im.conversations.android.axolotl.AxolotlSession; +import im.conversations.android.axolotl.EncryptionBuilder; +import im.conversations.android.xml.Element; import im.conversations.android.xml.Namespace; import im.conversations.android.xmpp.IqErrorException; import im.conversations.android.xmpp.NodeConfiguration; import im.conversations.android.xmpp.XmppConnection; -import im.conversations.android.xmpp.axolotl.AxolotlAddress; import im.conversations.android.xmpp.model.axolotl.Bundle; import im.conversations.android.xmpp.model.axolotl.DeviceList; +import im.conversations.android.xmpp.model.axolotl.Encrypted; import im.conversations.android.xmpp.model.pubsub.Items; import java.util.Collection; import java.util.Collections; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; import org.jxmpp.jid.BareJid; @@ -28,7 +44,6 @@ import org.slf4j.LoggerFactory; import org.whispersystems.libsignal.IdentityKey; import org.whispersystems.libsignal.InvalidKeyException; import org.whispersystems.libsignal.SessionBuilder; -import org.whispersystems.libsignal.SessionCipher; import org.whispersystems.libsignal.UntrustedIdentityException; import org.whispersystems.libsignal.state.PreKeyBundle; import org.whispersystems.libsignal.state.SignalProtocolStore; @@ -41,11 +56,11 @@ public class AxolotlManager extends AbstractManager { private static final int NUM_PRE_KEYS_IN_BUNDLE = 30; - private final SignalProtocolStore signalProtocolStore; + private final AxolotlService axolotlService; public AxolotlManager(Context context, XmppConnection connection) { super(context, connection); - this.signalProtocolStore = new AxolotlDatabaseStore(context, connection.getAccount()); + this.axolotlService = new AxolotlService(context, connection.getAccount()); } public void handleItems(final BareJid from, final Items items) { @@ -103,25 +118,27 @@ public class AxolotlManager extends AbstractManager { return getManager(PubSubManager.class).fetchMostRecentItem(address, node, Bundle.class); } - public ListenableFuture getOrCreateSessionCipher( + public ListenableFuture getOrCreateSessionCipher( final AxolotlAddress axolotlAddress) { - if (signalProtocolStore.containsSession(axolotlAddress)) { - return Futures.immediateFuture(new SessionCipher(signalProtocolStore, axolotlAddress)); + final AxolotlSession session = axolotlService.getExistingSession(axolotlAddress); + if (session != null) { + return Futures.immediateFuture(session); } else { final var bundleFuture = fetchBundle(axolotlAddress.getJid(), axolotlAddress.getDeviceId()); return Futures.transform( bundleFuture, bundle -> { - buildSession(axolotlAddress, bundle); - return new SessionCipher(signalProtocolStore, axolotlAddress); + final var identityKey = buildSession(axolotlAddress, bundle); + return AxolotlSession.of( + signalProtocolStore(), identityKey, axolotlAddress); }, MoreExecutors.directExecutor()); } } - private void buildSession(final AxolotlAddress address, final Bundle bundle) { - final var sessionBuilder = new SessionBuilder(signalProtocolStore, address); + private IdentityKey buildSession(final AxolotlAddress address, final Bundle bundle) { + final var sessionBuilder = new SessionBuilder(signalProtocolStore(), address); final var deviceId = address.getDeviceId(); final var preKey = bundle.getRandomPreKey(); final var signedPreKey = bundle.getSignedPreKey(); @@ -139,6 +156,7 @@ public class AxolotlManager extends AbstractManager { if (identityKey == null) { throw new IllegalArgumentException("No IdentityKey found in bundle"); } + final var signalIdentityKey = new IdentityKey(identityKey.asECPublicKey()); final var preKeyBundle = new PreKeyBundle( 0, @@ -148,9 +166,10 @@ public class AxolotlManager extends AbstractManager { signedPreKey.getId(), signedPreKey.asECPublicKey(), signedPreKeySignature.asBytes(), - new IdentityKey(identityKey.asECPublicKey())); + signalIdentityKey); try { sessionBuilder.process(preKeyBundle); + return signalIdentityKey; } catch (final InvalidKeyException | UntrustedIdentityException e) { throw new RuntimeException(e); } @@ -249,7 +268,7 @@ public class AxolotlManager extends AbstractManager { Locale.ROOT, "%s:%d", Namespace.AXOLOTL_BUNDLES, - signalProtocolStore.getLocalRegistrationId()); + signalProtocolStore().getLocalRegistrationId()); return getManager(PepManager.class) .publishSingleton(bundle, node, NodeConfiguration.OPEN); }, @@ -260,7 +279,7 @@ public class AxolotlManager extends AbstractManager { refillPreKeys(); final var bundle = new Bundle(); bundle.setIdentityKey( - signalProtocolStore.getIdentityKeyPair().getPublicKey().getPublicKey()); + signalProtocolStore().getIdentityKeyPair().getPublicKey().getPublicKey()); final var signedPreKeyRecord = getDatabase().axolotlDao().getLatestSignedPreKey(getAccount().id); if (signedPreKeyRecord == null) { @@ -286,11 +305,11 @@ public class AxolotlManager extends AbstractManager { try { signedPreKeyRecord = KeyHelper.generateSignedPreKey( - signalProtocolStore.getIdentityKeyPair(), signedPreKeyId); + signalProtocolStore().getIdentityKeyPair(), signedPreKeyId); } catch (final InvalidKeyException e) { throw new IllegalStateException("Could not generate SignedPreKey", e); } - signalProtocolStore.storeSignedPreKey(signedPreKeyRecord.getId(), signedPreKeyRecord); + signalProtocolStore().storeSignedPreKey(signedPreKeyRecord.getId(), signedPreKeyRecord); LOGGER.info("Generated SignedPreKey #{}", signedPreKeyRecord.getId()); } axolotlDao.setPreKeys(getAccount(), preKeys); @@ -298,4 +317,165 @@ public class AxolotlManager extends AbstractManager { LOGGER.info("Generated {} PreKeys starting with {}", preKeys.size(), start); } } + + private OmemoVerifiedIceUdpTransportInfo encrypt( + final IceUdpTransportInfo element, final AxolotlSession session) + throws AxolotlEncryptionException { + final OmemoVerifiedIceUdpTransportInfo transportInfo = + new OmemoVerifiedIceUdpTransportInfo(); + transportInfo.setAttributes(element.getAttributes()); + for (final Element child : element.getChildren()) { + if ("fingerprint".equals(child.getName()) + && Namespace.JINGLE_APPS_DTLS.equals(child.getNamespace())) { + final Element fingerprint = + new Element("fingerprint", Namespace.OMEMO_DTLS_SRTP_VERIFICATION); + fingerprint.setAttribute("setup", child.getAttribute("setup")); + fingerprint.setAttribute("hash", child.getAttribute("hash")); + final String content = child.getContent(); + final var encrypted = + new EncryptionBuilder() + .sourceDeviceId(signalProtocolStore().getLocalRegistrationId()) + .payload(content) + .session(session) + .build(); + fingerprint.addExtension(encrypted); + transportInfo.addChild(fingerprint); + } else { + transportInfo.addChild(child); + } + } + return transportInfo; + } + + public ListenableFuture> + encrypt(final RtpContentMap rtpContentMap, final Jid jid, final int deviceId) { + final var axolotlAddress = new AxolotlAddress(jid.asBareJid(), deviceId); + final var sessionFuture = getOrCreateSessionCipher(axolotlAddress); + return Futures.transformAsync( + sessionFuture, + session -> encrypt(rtpContentMap, session), + MoreExecutors.directExecutor()); + } + + private ListenableFuture> + encrypt(final RtpContentMap rtpContentMap, final AxolotlSession session) { + if (Config.REQUIRE_RTP_VERIFICATION) { + requireVerification(session); + } + final ImmutableMap.Builder + descriptionTransportBuilder = new ImmutableMap.Builder<>(); + final OmemoVerification omemoVerification = new OmemoVerification(); + omemoVerification.setDeviceId(session.axolotlAddress.getDeviceId()); + omemoVerification.setSessionFingerprint(session.identityKey); + for (final Map.Entry content : + rtpContentMap.contents.entrySet()) { + final RtpContentMap.DescriptionTransport descriptionTransport = content.getValue(); + final OmemoVerifiedIceUdpTransportInfo encryptedTransportInfo; + try { + encryptedTransportInfo = encrypt(descriptionTransport.transport, session); + } catch (final AxolotlEncryptionException e) { + return Futures.immediateFailedFuture(e); + } + descriptionTransportBuilder.put( + content.getKey(), + new RtpContentMap.DescriptionTransport( + descriptionTransport.senders, + descriptionTransport.description, + encryptedTransportInfo)); + } + return Futures.immediateFuture( + new AxolotlService.OmemoVerifiedPayload<>( + omemoVerification, + new OmemoVerifiedRtpContentMap( + rtpContentMap.group, descriptionTransportBuilder.build()))); + } + + public ListenableFuture> decrypt( + OmemoVerifiedRtpContentMap omemoVerifiedRtpContentMap, final Jid from) { + final ImmutableMap.Builder + descriptionTransportBuilder = new ImmutableMap.Builder<>(); + final OmemoVerification omemoVerification = new OmemoVerification(); + final ImmutableList.Builder> pepVerificationFutures = + new ImmutableList.Builder<>(); + for (final Map.Entry content : + omemoVerifiedRtpContentMap.contents.entrySet()) { + final RtpContentMap.DescriptionTransport descriptionTransport = content.getValue(); + final AxolotlService.OmemoVerifiedPayload decryptedTransport; + try { + decryptedTransport = + decrypt( + (OmemoVerifiedIceUdpTransportInfo) descriptionTransport.transport, + from, + pepVerificationFutures); + } catch (final AxolotlDecryptionException e) { + return Futures.immediateFailedFuture(e); + } + omemoVerification.setOrEnsureEqual(decryptedTransport); + descriptionTransportBuilder.put( + content.getKey(), + new RtpContentMap.DescriptionTransport( + descriptionTransport.senders, + descriptionTransport.description, + decryptedTransport.getPayload())); + } + final ImmutableList> sessionFutures = + pepVerificationFutures.build(); + return Futures.transform( + Futures.allAsList(sessionFutures), + sessions -> { + if (Config.REQUIRE_RTP_VERIFICATION) { + for (final AxolotlSession session : sessions) { + requireVerification(session); + } + } + return new AxolotlService.OmemoVerifiedPayload<>( + omemoVerification, + new RtpContentMap( + omemoVerifiedRtpContentMap.group, + descriptionTransportBuilder.build())); + }, + MoreExecutors.directExecutor()); + } + + private AxolotlService.OmemoVerifiedPayload decrypt( + final OmemoVerifiedIceUdpTransportInfo verifiedIceUdpTransportInfo, + final Jid from, + ImmutableList.Builder> pepVerificationFutures) + throws AxolotlDecryptionException { + final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); + transportInfo.setAttributes(verifiedIceUdpTransportInfo.getAttributes()); + final OmemoVerification omemoVerification = new OmemoVerification(); + for (final Element child : verifiedIceUdpTransportInfo.getChildren()) { + if ("fingerprint".equals(child.getName()) + && Namespace.OMEMO_DTLS_SRTP_VERIFICATION.equals(child.getNamespace())) { + final Element fingerprint = new Element("fingerprint", Namespace.JINGLE_APPS_DTLS); + fingerprint.setAttribute("setup", child.getAttribute("setup")); + fingerprint.setAttribute("hash", child.getAttribute("hash")); + final Encrypted encrypted = child.getExtension(Encrypted.class); + final AxolotlPayload axolotlPayload = axolotlService.decrypt(from, encrypted); + fingerprint.setContent(axolotlPayload.payloadAsString()); + omemoVerification.setDeviceId(axolotlPayload.axolotlAddress.getDeviceId()); + omemoVerification.setSessionFingerprint(axolotlPayload.identityKey); + transportInfo.addChild(fingerprint); + } else { + transportInfo.addChild(child); + } + } + return new AxolotlService.OmemoVerifiedPayload<>(omemoVerification, transportInfo); + } + + private static void requireVerification(final AxolotlSession session) { + // TODO fix me; check if identity key is trusted + + /*if (session.getTrust().isVerified()) { + return; + }*/ + throw new AxolotlService.NotVerifiedException( + String.format( + "session with %s was not verified", session.identityKey.getFingerprint())); + } + + private SignalProtocolStore signalProtocolStore() { + return this.axolotlService.getSignalProtocolStore(); + } } diff --git a/app/src/main/java/im/conversations/android/xmpp/manager/DiscoManager.java b/app/src/main/java/im/conversations/android/xmpp/manager/DiscoManager.java index c0bf212ed..c24e3a73d 100644 --- a/app/src/main/java/im/conversations/android/xmpp/manager/DiscoManager.java +++ b/app/src/main/java/im/conversations/android/xmpp/manager/DiscoManager.java @@ -32,7 +32,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; -import org.jxmpp.jid.Jid; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,8 +66,8 @@ public class DiscoManager extends AbstractManager { Namespace.JINGLE_FEATURE_AUDIO, Namespace.JINGLE_FEATURE_VIDEO, Namespace.JINGLE_APPS_RTP, - Namespace.JINGLE_APPS_DTLS, - Namespace.JINGLE_MESSAGE); + Namespace.JINGLE_APPS_DTLS /*, + Namespace.JINGLE_MESSAGE*/); private static final Collection FEATURES_IMPACTING_PRIVACY = Collections.singleton(Namespace.VERSION); @@ -241,16 +240,28 @@ public class DiscoManager extends AbstractManager { MoreExecutors.directExecutor()); } - public boolean hasFeature(final Jid entity, final String feature) { + public boolean hasFeature(final Entity entity, final String feature) { return getDatabase().discoDao().hasFeature(getAccount().id, entity, feature); } + public ListenableFuture hasFeatureAsync(final Entity entity, final String feature) { + return Futures.submit(() -> hasFeature(entity, feature), getDatabase().getQueryExecutor()); + } + public boolean hasAccountFeature(final String feature) { - return hasFeature(getAccount().address, feature); + return hasFeature(Entity.discoItem(getAccount().address), feature); + } + + public ListenableFuture hasAccountFeatureAsync(final String feature) { + return Futures.submit(() -> hasAccountFeature(feature), getDatabase().getQueryExecutor()); } public boolean hasServerFeature(final String feature) { - return hasFeature(getAccount().address.asDomainBareJid(), feature); + return hasFeature(Entity.discoItem(getAccount().address.asDomainBareJid()), feature); + } + + public ListenableFuture hasServerFeatureAsync(final String feature) { + return Futures.submit(() -> hasServerFeature(feature), getDatabase().getQueryExecutor()); } public ServiceDescription getServiceDescription() { diff --git a/app/src/main/java/im/conversations/android/xmpp/manager/ExternalDiscoManager.java b/app/src/main/java/im/conversations/android/xmpp/manager/ExternalDiscoManager.java new file mode 100644 index 000000000..ffccad68f --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/manager/ExternalDiscoManager.java @@ -0,0 +1,55 @@ +package im.conversations.android.xmpp.manager; + +import android.content.Context; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import im.conversations.android.xml.Namespace; +import im.conversations.android.xmpp.XmppConnection; +import im.conversations.android.xmpp.model.disco.external.Service; +import im.conversations.android.xmpp.model.disco.external.Services; +import im.conversations.android.xmpp.model.stanza.Iq; +import java.util.Collection; + +public class ExternalDiscoManager extends AbstractManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExternalDiscoManager.class); + + public ExternalDiscoManager(Context context, XmppConnection connection) { + super(context, connection); + } + + public ListenableFuture> getServices() { + final var hasFeatureFuture = + getManager(DiscoManager.class) + .hasServerFeatureAsync(Namespace.EXTERNAL_SERVICE_DISCOVERY); + final var iqResultFuture = + Futures.transformAsync( + hasFeatureFuture, + hasFeature -> { + if (Boolean.TRUE.equals(hasFeature)) { + final Iq request = new Iq(Iq.Type.GET); + request.setTo(getAccount().address.asDomainBareJid()); + request.addExtension(new Services()); + return connection.sendIqPacket(request); + } + throw new IllegalStateException( + "Server does not support External Service Discovery"); + }, + MoreExecutors.directExecutor()); + return Futures.transform( + iqResultFuture, + result -> { + final var services = result.getExtension(Services.class); + if (services == null) { + throw new IllegalStateException("Server result did not contain services"); + } + return services.getExtensions(Service.class); + }, + MoreExecutors.directExecutor()); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/manager/JingleConnectionManager.java b/app/src/main/java/im/conversations/android/xmpp/manager/JingleConnectionManager.java index 9dfa11a9f..08b3593e3 100644 --- a/app/src/main/java/im/conversations/android/xmpp/manager/JingleConnectionManager.java +++ b/app/src/main/java/im/conversations/android/xmpp/manager/JingleConnectionManager.java @@ -1,16 +1,880 @@ package im.conversations.android.xmpp.manager; import android.content.Context; - +import android.util.Log; +import com.google.common.base.Objects; +import com.google.common.base.Optional; +import com.google.common.base.Strings; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Collections2; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.ImmutableSet; +import eu.siacs.conversations.Config; +import eu.siacs.conversations.generator.MessageGenerator; +import eu.siacs.conversations.services.AppRTCAudioManager; +import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection; +import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection; +import eu.siacs.conversations.xmpp.jingle.Media; +import eu.siacs.conversations.xmpp.jingle.OngoingRtpSession; +import eu.siacs.conversations.xmpp.jingle.RtpEndUserState; +import eu.siacs.conversations.xmpp.jingle.ToneManager; +import eu.siacs.conversations.xmpp.jingle.stanzas.Content; +import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription; +import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; +import eu.siacs.conversations.xmpp.jingle.stanzas.Reason; +import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; +import im.conversations.android.IDs; +import im.conversations.android.database.model.Account; +import im.conversations.android.notification.RtpSessionNotification; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; import im.conversations.android.xmpp.XmppConnection; +import im.conversations.android.xmpp.model.error.Condition; +import im.conversations.android.xmpp.model.error.Error; +import im.conversations.android.xmpp.model.jmi.Accept; +import im.conversations.android.xmpp.model.jmi.JingleMessage; +import im.conversations.android.xmpp.model.jmi.Proceed; +import im.conversations.android.xmpp.model.jmi.Propose; import im.conversations.android.xmpp.model.stanza.Iq; +import im.conversations.android.xmpp.model.stanza.Message; +import java.lang.ref.WeakReference; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import org.jxmpp.jid.Jid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class JingleConnectionManager extends AbstractManager { - public JingleConnectionManager(Context context, XmppConnection connection) { + + private static final Logger LOGGER = LoggerFactory.getLogger(JingleConnectionManager.class); + + public static final ScheduledExecutorService SCHEDULED_EXECUTOR_SERVICE = + Executors.newSingleThreadScheduledExecutor(); + private final HashMap rtpSessionProposals = + new HashMap<>(); + private final ConcurrentHashMap + connections = new ConcurrentHashMap<>(); + + private final Cache terminatedSessions = + CacheBuilder.newBuilder().expireAfterWrite(24, TimeUnit.HOURS).build(); + + private final RtpSessionNotification rtpSessionNotification; + + private OnJingleRtpConnectionUpdate onJingleRtpConnectionUpdate; + + public JingleConnectionManager(final Context context, final XmppConnection connection) { super(context, connection); + this.rtpSessionNotification = new RtpSessionNotification(context); } - public void handleJingle(Iq packet) { + @Override + public Account getAccount() { + return super.getAccount(); + } + public void handleJingle(final Iq iq) { + final JinglePacket packet = JinglePacket.upgrade(iq); + final String sessionId = packet.getSessionId(); + if (sessionId == null) { + respondWithJingleError( + iq, "unknown-session", Error.Type.CANCEL, new Condition.ItemNotFound()); + return; + } + final AbstractJingleConnection.Id id = AbstractJingleConnection.Id.of(packet); + final AbstractJingleConnection existingJingleConnection = connections.get(id); + if (existingJingleConnection != null) { + existingJingleConnection.deliverPacket(packet); + } else if (packet.getAction() == JinglePacket.Action.SESSION_INITIATE) { + final Jid from = packet.getFrom(); + final Content content = packet.getJingleContent(); + final String descriptionNamespace = + content == null ? null : content.getDescriptionNamespace(); + final AbstractJingleConnection connection; + if (Namespace.JINGLE_APPS_RTP.equals(descriptionNamespace) && isUsingClearNet()) { + final boolean sessionEnded = + this.terminatedSessions.asMap().containsKey(PersistableSessionId.of(id)); + final boolean stranger = isWithStrangerAndStrangerNotificationsAreOff(id.with); + if (isBusy() || sessionEnded || stranger) { + LOGGER.debug( + this.connection.getAccount().address + + ": rejected session with " + + id.with + + " because busy. sessionEnded=" + + sessionEnded + + ", stranger=" + + stranger); + this.connection.sendResultFor(packet); + final JinglePacket sessionTermination = + new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId); + sessionTermination.setTo(id.with); + sessionTermination.setReason(Reason.BUSY, null); + this.connection.sendIqPacket(sessionTermination, null); + return; + } + connection = new JingleRtpConnection(context, this.connection, id, from); + } else { + respondWithJingleError( + packet, + "unsupported-info", + Error.Type.CANCEL, + new Condition.FeatureNotImplemented()); + return; + } + connections.put(id, connection); + connection.deliverPacket(packet); + } else { + Log.d(Config.LOGTAG, "unable to route jingle packet: " + packet); + respondWithJingleError( + packet, "unknown-session", Error.Type.CANCEL, new Condition.ItemNotFound()); + } + } + + private boolean isUsingClearNet() { + // todo bring back proper Tor check + return !connection.getAccount().isOnion(); + } + + public boolean isBusy() { + // TODO check if in actual phone call + for (AbstractJingleConnection connection : this.connections.values()) { + if (connection instanceof JingleRtpConnection) { + if (((JingleRtpConnection) connection).isTerminated()) { + continue; + } + return true; + } + } + synchronized (this.rtpSessionProposals) { + return this.rtpSessionProposals.containsValue(DeviceDiscoveryState.DISCOVERED) + || this.rtpSessionProposals.containsValue(DeviceDiscoveryState.SEARCHING) + || this.rtpSessionProposals.containsValue( + JingleConnectionManager.DeviceDiscoveryState.SEARCHING_ACKNOWLEDGED); + } + } + + public void notifyPhoneCallStarted() { + for (AbstractJingleConnection connection : connections.values()) { + if (connection instanceof JingleRtpConnection) { + final JingleRtpConnection rtpConnection = (JingleRtpConnection) connection; + if (rtpConnection.isTerminated()) { + continue; + } + rtpConnection.notifyPhoneCall(); + } + } + } + + private Optional findMatchingSessionProposal( + final Jid with, final Set media) { + synchronized (this.rtpSessionProposals) { + for (Map.Entry entry : + this.rtpSessionProposals.entrySet()) { + final RtpSessionProposal proposal = entry.getKey(); + final DeviceDiscoveryState state = entry.getValue(); + final boolean openProposal = + state == DeviceDiscoveryState.DISCOVERED + || state == DeviceDiscoveryState.SEARCHING + || state == DeviceDiscoveryState.SEARCHING_ACKNOWLEDGED; + if (openProposal + && proposal.with.equals(with.asBareJid()) + && proposal.media.equals(media)) { + return Optional.of(proposal); + } + } + } + return Optional.absent(); + } + + private boolean hasMatchingRtpSession(final Jid with, final Set media) { + for (AbstractJingleConnection connection : this.connections.values()) { + if (connection instanceof JingleRtpConnection) { + final JingleRtpConnection rtpConnection = (JingleRtpConnection) connection; + if (rtpConnection.isTerminated()) { + continue; + } + if (rtpConnection.getId().with.asBareJid().equals(with.asBareJid()) + && rtpConnection.getMedia().equals(media)) { + return true; + } + } + } + return false; + } + + private boolean isWithStrangerAndStrangerNotificationsAreOff(Jid with) { + final boolean notifyForStrangers = rtpSessionNotification.notificationsFromStrangers(); + if (notifyForStrangers) { + return false; + } + return getDatabase().rosterDao().isInRoster(getAccount().id, with.asBareJid()); + } + + public ScheduledFuture schedule( + final Runnable runnable, final long delay, final TimeUnit timeUnit) { + return SCHEDULED_EXECUTOR_SERVICE.schedule(runnable, delay, timeUnit); + } + + private void respondWithJingleError( + final Iq original, String jingleCondition, final Error.Type type, Condition condition) { + // TODO add jingle condation + connection.sendErrorFor(original, type, condition); + } + + public void deliverMessage( + final Jid to, + final Jid from, + final JingleMessage message, + final String remoteMsgId, + final String serverMsgId) { + final String sessionId = message.getSessionId(); + if (Strings.isNullOrEmpty(sessionId)) { + return; + } + if (message instanceof Accept) { + for (AbstractJingleConnection connection : connections.values()) { + if (connection instanceof JingleRtpConnection) { + final JingleRtpConnection rtpConnection = (JingleRtpConnection) connection; + final AbstractJingleConnection.Id id = connection.getId(); + if (id.sessionId.equals(sessionId)) { + rtpConnection.deliveryMessage(from, message, serverMsgId); + return; + } + } + } + return; + } + final boolean fromSelf = from.asBareJid().equals(connection.getBoundAddress().asBareJid()); + final boolean addressedDirectly = to != null && to.equals(connection.getBoundAddress()); + final AbstractJingleConnection.Id id; + if (fromSelf) { + if (to != null && to.hasResource()) { + id = AbstractJingleConnection.Id.of(to, sessionId); + } else { + return; + } + } else { + id = AbstractJingleConnection.Id.of(from, sessionId); + } + final AbstractJingleConnection existingJingleConnection = connections.get(id); + if (existingJingleConnection != null) { + if (existingJingleConnection instanceof JingleRtpConnection) { + ((JingleRtpConnection) existingJingleConnection) + .deliveryMessage(from, message, serverMsgId); + } else { + LOGGER.debug( + connection.getAccount().address + + ": " + + existingJingleConnection.getClass().getName() + + " does not support jingle messages"); + } + return; + } + + if (fromSelf) { + if (message instanceof Proceed) { + + // if we've previously rejected a call because we were busy (which would have + // created a CallLogEntry) but that call was picked up on another one of our devices + // we want to update that CallLogEntry to say picked up (not missed) + + /*final Conversation c = + mXmppConnectionService.findOrCreateConversation( + account, id.with, false, false); + final Message previousBusy = c.findRtpSession(sessionId, Message.STATUS_RECEIVED); + if (previousBusy != null) { + previousBusy.setBody(new RtpSessionStatus(true, 0).toString()); + if (serverMsgId != null) { + previousBusy.setServerMsgId(serverMsgId); + } + previousBusy.setTime(timestamp); + mXmppConnectionService.updateMessage(previousBusy, true); + LOGGER.debug( + connection.getAccount().address + + ": updated previous busy because call got picked up by another device"); + return; + }*/ + } + // TODO handle reject for cases where we don’t have carbon copies (normally reject is to + // be sent to own bare jid as well) + LOGGER.debug(connection.getAccount().address + ": ignore jingle message from self"); + return; + } + + if (message instanceof Propose) { + final Propose propose = (Propose) message; + final List descriptions = propose.getDescriptions(); + final Collection rtpDescriptions = + Collections2.transform( + Collections2.filter(descriptions, d -> d instanceof RtpDescription), + input -> (RtpDescription) input); + if (rtpDescriptions.size() > 0 + && rtpDescriptions.size() == descriptions.size() + && isUsingClearNet()) { + final Collection media = + Collections2.transform(rtpDescriptions, RtpDescription::getMedia); + if (media.contains(Media.UNKNOWN)) { + LOGGER.debug( + connection.getAccount().address + + ": encountered unknown media in session proposal. " + + propose); + return; + } + final Optional matchingSessionProposal = + findMatchingSessionProposal(id.with, ImmutableSet.copyOf(media)); + if (matchingSessionProposal.isPresent()) { + final String ourSessionId = matchingSessionProposal.get().sessionId; + final String theirSessionId = id.sessionId; + if (ComparisonChain.start() + .compare(ourSessionId, theirSessionId) + .compare( + connection.getBoundAddress().toString(), + id.with.toString()) + .result() + > 0) { + LOGGER.debug( + connection.getAccount().address + + ": our session lost tie break. automatically accepting" + + " their session. winning Session=" + + theirSessionId); + // TODO a retract for this reason should probably include some indication of + // tie break + retractSessionProposal(matchingSessionProposal.get()); + final JingleRtpConnection rtpConnection = + new JingleRtpConnection(context, this.connection, id, from); + this.connections.put(id, rtpConnection); + rtpConnection.setProposedMedia(ImmutableSet.copyOf(media)); + rtpConnection.deliveryMessage(from, message, serverMsgId); + } else { + LOGGER.debug( + connection.getAccount().address + + ": our session won tie break. waiting for other party to" + + " accept. winningSession=" + + ourSessionId); + } + return; + } + final boolean stranger = isWithStrangerAndStrangerNotificationsAreOff(id.with); + if (isBusy() || stranger) { + writeLogMissedIncoming(id.with.asBareJid(), id.sessionId, serverMsgId); + if (stranger) { + LOGGER.debug( + connection.getAccount().address + + ": ignoring call proposal from stranger " + + id.with); + return; + } + final int activeDevices = + getDatabase() + .discoDao() + .countPresencesWithFeature( + getAccount(), Namespace.JINGLE_APPS_RTP); + Log.d(Config.LOGTAG, "active devices with rtp capability: " + activeDevices); + if (activeDevices == 0) { + final Message reject = MessageGenerator.sessionReject(from, sessionId); + connection.sendMessagePacket(reject); + } else { + LOGGER.debug( + connection.getAccount().address + + ": ignoring proposal because busy on this device but" + + " there are other devices"); + } + } else { + final JingleRtpConnection rtpConnection = + new JingleRtpConnection(context, this.connection, id, from); + this.connections.put(id, rtpConnection); + rtpConnection.setProposedMedia(ImmutableSet.copyOf(media)); + rtpConnection.deliveryMessage(from, message, serverMsgId); + } + } else { + LOGGER.debug( + connection.getAccount().address + + ": unable to react to proposed session with " + + rtpDescriptions.size() + + " rtp descriptions of " + + descriptions.size() + + " total descriptions"); + } + } else if (addressedDirectly && "proceed".equals(message.getName())) { + synchronized (rtpSessionProposals) { + final RtpSessionProposal proposal = + getRtpSessionProposal(from.asBareJid(), sessionId); + if (proposal != null) { + rtpSessionProposals.remove(proposal); + final JingleRtpConnection rtpConnection = + new JingleRtpConnection( + context, + this.connection, + id, + this.connection.getBoundAddress()); + rtpConnection.setProposedMedia(proposal.media); + this.connections.put(id, rtpConnection); + rtpConnection.transitionOrThrow(AbstractJingleConnection.State.PROPOSED); + rtpConnection.deliveryMessage(from, message, serverMsgId); + } else { + LOGGER.debug( + connection.getAccount().address + + ": no rtp session proposal found for " + + from + + " to deliver proceed"); + if (remoteMsgId == null) { + return; + } + final Message errorMessage = new Message(); + errorMessage.setTo(from); + errorMessage.setId(remoteMsgId); + errorMessage.setType(Message.Type.ERROR); + final Element error = errorMessage.addChild("error"); + error.setAttribute("code", "404"); + error.setAttribute("type", "cancel"); + error.addChild("item-not-found", "urn:ietf:params:xml:ns:xmpp-stanzas"); + connection.sendMessagePacket(errorMessage); + } + } + } else if (addressedDirectly && "reject".equals(message.getName())) { + final RtpSessionProposal proposal = getRtpSessionProposal(from.asBareJid(), sessionId); + synchronized (rtpSessionProposals) { + if (proposal != null && rtpSessionProposals.remove(proposal) != null) { + writeLogMissedOutgoing(proposal.with, proposal.sessionId, serverMsgId); + ToneManager.getInstance(context) + .transition(RtpEndUserState.DECLINED_OR_BUSY, proposal.media); + notifyJingleRtpConnectionUpdate( + proposal.with, proposal.sessionId, RtpEndUserState.DECLINED_OR_BUSY); + } else { + LOGGER.debug( + connection.getAccount().address + + ": no rtp session proposal found for " + + from + + " to deliver reject"); + } + } + } else { + LOGGER.debug( + connection.getAccount().address + + ": retrieved out of order jingle message" + + message); + } + } + + private RtpSessionProposal getRtpSessionProposal(Jid from, String sessionId) { + for (RtpSessionProposal rtpSessionProposal : rtpSessionProposals.keySet()) { + if (rtpSessionProposal.sessionId.equals(sessionId) + && rtpSessionProposal.with.equals(from)) { + return rtpSessionProposal; + } + } + return null; + } + + private void writeLogMissedOutgoing(Jid with, final String sessionId, String serverMsgId) {} + + private void writeLogMissedIncoming(Jid with, final String sessionId, String serverMsgId) {} + + public Optional getOngoingRtpConnection(final Jid contact) { + for (final Map.Entry entry : + this.connections.entrySet()) { + if (entry.getValue() instanceof JingleRtpConnection) { + final AbstractJingleConnection.Id id = entry.getKey(); + if (id.with.asBareJid().equals(contact.asBareJid())) { + return Optional.of(id); + } + } + } + synchronized (this.rtpSessionProposals) { + for (final Map.Entry entry : + this.rtpSessionProposals.entrySet()) { + final RtpSessionProposal proposal = entry.getKey(); + if (contact.asBareJid().equals(proposal.with)) { + final DeviceDiscoveryState preexistingState = entry.getValue(); + if (preexistingState != null + && preexistingState != DeviceDiscoveryState.FAILED) { + return Optional.of(proposal); + } + } + } + } + return Optional.absent(); + } + + void finishConnection(final AbstractJingleConnection connection) { + this.connections.remove(connection.getId()); + } + + public void finishConnectionOrThrow(final AbstractJingleConnection connection) { + final AbstractJingleConnection.Id id = connection.getId(); + if (this.connections.remove(id) == null) { + throw new IllegalStateException( + String.format("Unable to finish connection with id=%s", id.toString())); + } + } + + public boolean fireJingleRtpConnectionStateUpdates() { + boolean firedUpdates = false; + for (final AbstractJingleConnection connection : this.connections.values()) { + if (connection instanceof JingleRtpConnection) { + final JingleRtpConnection jingleRtpConnection = (JingleRtpConnection) connection; + if (jingleRtpConnection.isTerminated()) { + continue; + } + jingleRtpConnection.fireStateUpdate(); + firedUpdates = true; + } + } + return firedUpdates; + } + + public void retractSessionProposal(final Jid with) { + synchronized (this.rtpSessionProposals) { + RtpSessionProposal matchingProposal = null; + for (RtpSessionProposal proposal : this.rtpSessionProposals.keySet()) { + if (with.asBareJid().equals(proposal.with)) { + matchingProposal = proposal; + break; + } + } + if (matchingProposal != null) { + retractSessionProposal(matchingProposal); + } + } + } + + private void retractSessionProposal(RtpSessionProposal rtpSessionProposal) { + ToneManager.getInstance(context) + .transition(RtpEndUserState.ENDED, rtpSessionProposal.media); + LOGGER.debug( + connection.getAccount().address + + ": retracting rtp session proposal with " + + rtpSessionProposal.with); + this.rtpSessionProposals.remove(rtpSessionProposal); + final Message messagePacket = MessageGenerator.sessionRetract(rtpSessionProposal); + writeLogMissedOutgoing(rtpSessionProposal.with, rtpSessionProposal.sessionId, null); + connection.sendMessagePacket(messagePacket); + } + + public String initializeRtpSession(final Jid with, final Set media) { + final AbstractJingleConnection.Id id = AbstractJingleConnection.Id.of(with); + final JingleRtpConnection rtpConnection = + new JingleRtpConnection( + context, this.connection, id, this.connection.getBoundAddress()); + rtpConnection.setProposedMedia(media); + this.connections.put(id, rtpConnection); + rtpConnection.sendSessionInitiate(); + return id.sessionId; + } + + public void proposeJingleRtpSession(final Jid with, final Set media) { + synchronized (this.rtpSessionProposals) { + for (Map.Entry entry : + this.rtpSessionProposals.entrySet()) { + RtpSessionProposal proposal = entry.getKey(); + if (with.asBareJid().equals(proposal.with)) { + final DeviceDiscoveryState preexistingState = entry.getValue(); + if (preexistingState != null + && preexistingState != DeviceDiscoveryState.FAILED) { + final RtpEndUserState endUserState = preexistingState.toEndUserState(); + ToneManager.getInstance(context).transition(endUserState, media); + this.notifyJingleRtpConnectionUpdate( + with, proposal.sessionId, endUserState); + return; + } + } + } + if (isBusy()) { + if (hasMatchingRtpSession(with, media)) { + LOGGER.debug( + "ignoring request to propose jingle session because the other party" + + " already created one for us"); + return; + } + throw new IllegalStateException( + "There is already a running RTP session. This should have been caught by" + + " the UI"); + } + final RtpSessionProposal proposal = RtpSessionProposal.of(with.asBareJid(), media); + this.rtpSessionProposals.put(proposal, DeviceDiscoveryState.SEARCHING); + this.notifyJingleRtpConnectionUpdate( + proposal.with, proposal.sessionId, RtpEndUserState.FINDING_DEVICE); + final Message messagePacket = MessageGenerator.sessionProposal(proposal); + connection.sendMessagePacket(messagePacket); + } + } + + public boolean hasMatchingProposal(final Jid with) { + synchronized (this.rtpSessionProposals) { + for (Map.Entry entry : + this.rtpSessionProposals.entrySet()) { + final RtpSessionProposal proposal = entry.getKey(); + if (with.asBareJid().equals(proposal.with)) { + return true; + } + } + } + return false; + } + + public void notifyRebound() { + for (final AbstractJingleConnection connection : this.connections.values()) { + connection.notifyRebound(); + } + // TODO the old version did this only when SM was enabled?! + resendSessionProposals(); + } + + public WeakReference findJingleRtpConnection(Jid with, String sessionId) { + final AbstractJingleConnection.Id id = AbstractJingleConnection.Id.of(with, sessionId); + final AbstractJingleConnection connection = connections.get(id); + if (connection instanceof JingleRtpConnection) { + return new WeakReference<>((JingleRtpConnection) connection); + } + return null; + } + + private void resendSessionProposals() { + synchronized (this.rtpSessionProposals) { + for (final Map.Entry entry : + this.rtpSessionProposals.entrySet()) { + final RtpSessionProposal proposal = entry.getKey(); + if (entry.getValue() == DeviceDiscoveryState.SEARCHING) { + LOGGER.debug( + connection.getAccount().address + + ": resending session proposal to " + + proposal.with); + final Message messagePacket = MessageGenerator.sessionProposal(proposal); + connection.sendMessagePacket(messagePacket); + } + } + } + } + + public void updateProposedSessionDiscovered( + Jid from, String sessionId, final DeviceDiscoveryState target) { + synchronized (this.rtpSessionProposals) { + final RtpSessionProposal sessionProposal = + getRtpSessionProposal(from.asBareJid(), sessionId); + final DeviceDiscoveryState currentState = + sessionProposal == null ? null : rtpSessionProposals.get(sessionProposal); + if (currentState == null) { + Log.d(Config.LOGTAG, "unable to find session proposal for session id " + sessionId); + return; + } + if (currentState == DeviceDiscoveryState.DISCOVERED) { + LOGGER.debug("session proposal already at discovered. not going to fall back"); + return; + } + this.rtpSessionProposals.put(sessionProposal, target); + final RtpEndUserState endUserState = target.toEndUserState(); + ToneManager.getInstance(context).transition(endUserState, sessionProposal.media); + this.notifyJingleRtpConnectionUpdate( + sessionProposal.with, sessionProposal.sessionId, endUserState); + LOGGER.debug( + connection.getAccount().address + + ": flagging session " + + sessionId + + " as " + + target); + } + } + + public void rejectRtpSession(final String sessionId) { + for (final AbstractJingleConnection connection : this.connections.values()) { + if (connection.getId().sessionId.equals(sessionId)) { + if (connection instanceof JingleRtpConnection) { + try { + ((JingleRtpConnection) connection).rejectCall(); + return; + } catch (final IllegalStateException e) { + Log.w( + Config.LOGTAG, + "race condition on rejecting call from notification", + e); + } + } + } + } + } + + public void endRtpSession(final String sessionId) { + for (final AbstractJingleConnection connection : this.connections.values()) { + if (connection.getId().sessionId.equals(sessionId)) { + if (connection instanceof JingleRtpConnection) { + ((JingleRtpConnection) connection).endCall(); + } + } + } + } + + public void failProceed(final Jid with, final String sessionId, final String message) { + final AbstractJingleConnection.Id id = AbstractJingleConnection.Id.of(with, sessionId); + final AbstractJingleConnection existingJingleConnection = connections.get(id); + if (existingJingleConnection instanceof JingleRtpConnection) { + ((JingleRtpConnection) existingJingleConnection).deliverFailedProceed(message); + } + } + + public void ensureConnectionIsRegistered(final AbstractJingleConnection connection) { + if (connections.containsValue(connection)) { + return; + } + final IllegalStateException e = + new IllegalStateException( + "JingleConnection has not been registered with connection manager"); + Log.e(Config.LOGTAG, "ensureConnectionIsRegistered() failed. Going to throw", e); + throw e; + } + + public void setTerminalSessionState( + AbstractJingleConnection.Id id, final RtpEndUserState state, final Set media) { + this.terminatedSessions.put( + PersistableSessionId.of(id), new TerminatedRtpSession(state, media)); + } + + public TerminatedRtpSession getTerminalSessionState(final Jid with, final String sessionId) { + return this.terminatedSessions.getIfPresent(new PersistableSessionId(with, sessionId)); + } + + public void notifyJingleRtpConnectionUpdate( + final Jid with, final String sessionId, final RtpEndUserState state) { + final var listener = this.onJingleRtpConnectionUpdate; + if (listener == null) { + return; + } + listener.onJingleRtpConnectionUpdate(with, sessionId, state); + } + + public void notifyJingleRtpConnectionUpdate( + AppRTCAudioManager.AudioDevice selectedAudioDevice, + Set availableAudioDevices) { + final var listener = this.onJingleRtpConnectionUpdate; + if (listener == null) { + return; + } + listener.onAudioDeviceChanged(selectedAudioDevice, availableAudioDevices); + } + + public void setOnJingleRtpConnectionUpdate(final OnJingleRtpConnectionUpdate listener) { + this.onJingleRtpConnectionUpdate = listener; + } + + public RtpSessionNotification getNotificationService() { + return this.rtpSessionNotification; + } + + public interface OnJingleRtpConnectionUpdate { + void onJingleRtpConnectionUpdate( + final Jid with, final String sessionId, final RtpEndUserState state); + + void onAudioDeviceChanged( + AppRTCAudioManager.AudioDevice selectedAudioDevice, + Set availableAudioDevices); + } + + private static class PersistableSessionId { + private final Jid with; + private final String sessionId; + + private PersistableSessionId(Jid with, String sessionId) { + this.with = with; + this.sessionId = sessionId; + } + + public static PersistableSessionId of(AbstractJingleConnection.Id id) { + return new PersistableSessionId(id.with, id.sessionId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistableSessionId that = (PersistableSessionId) o; + return Objects.equal(with, that.with) && Objects.equal(sessionId, that.sessionId); + } + + @Override + public int hashCode() { + return Objects.hashCode(with, sessionId); + } + } + + public static class TerminatedRtpSession { + public final RtpEndUserState state; + public final Set media; + + TerminatedRtpSession(RtpEndUserState state, Set media) { + this.state = state; + this.media = media; + } + } + + public enum DeviceDiscoveryState { + SEARCHING, + SEARCHING_ACKNOWLEDGED, + DISCOVERED, + FAILED; + + public RtpEndUserState toEndUserState() { + switch (this) { + case SEARCHING: + case SEARCHING_ACKNOWLEDGED: + return RtpEndUserState.FINDING_DEVICE; + case DISCOVERED: + return RtpEndUserState.RINGING; + default: + return RtpEndUserState.CONNECTIVITY_ERROR; + } + } + } + + public static class RtpSessionProposal implements OngoingRtpSession { + public final Jid with; + public final String sessionId; + public final Set media; + + private RtpSessionProposal(Jid with, String sessionId) { + this(with, sessionId, Collections.emptySet()); + } + + private RtpSessionProposal(Jid with, String sessionId, Set media) { + this.with = with; + this.sessionId = sessionId; + this.media = media; + } + + public static RtpSessionProposal of(Jid with, Set media) { + return new RtpSessionProposal(with, IDs.medium(), media); + } + + @Override + public Jid getWith() { + return with; + } + + @Override + public String getSessionId() { + return sessionId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RtpSessionProposal that = (RtpSessionProposal) o; + return Objects.equal(with, that.with) + && Objects.equal(sessionId, that.sessionId) + && Objects.equal(media, that.media); + } + + @Override + public int hashCode() { + return Objects.hashCode(with, sessionId, media); + } } } diff --git a/app/src/main/java/im/conversations/android/xmpp/manager/StanzaIdManager.java b/app/src/main/java/im/conversations/android/xmpp/manager/StanzaIdManager.java index 586ffb721..014629879 100644 --- a/app/src/main/java/im/conversations/android/xmpp/manager/StanzaIdManager.java +++ b/app/src/main/java/im/conversations/android/xmpp/manager/StanzaIdManager.java @@ -2,6 +2,7 @@ package im.conversations.android.xmpp.manager; import android.content.Context; import im.conversations.android.xml.Namespace; +import im.conversations.android.xmpp.Entity; import im.conversations.android.xmpp.XmppConnection; import im.conversations.android.xmpp.model.stanza.Message; import im.conversations.android.xmpp.model.unique.StanzaId; @@ -25,7 +26,8 @@ public class StanzaIdManager extends AbstractManager { by = connection.getBoundAddress().asBareJid(); } if (message.hasExtension(StanzaId.class) - && getManager(DiscoManager.class).hasFeature(by, Namespace.STANZA_IDS)) { + && getManager(DiscoManager.class) + .hasFeature(Entity.discoItem(by), Namespace.STANZA_IDS)) { return getStanzaIdBy(message, by); } else { return null; diff --git a/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Encrypted.java b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Encrypted.java index d1ac4d73b..1a98068ab 100644 --- a/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Encrypted.java +++ b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Encrypted.java @@ -13,4 +13,12 @@ public class Encrypted extends Extension { public boolean hasPayload() { return hasExtension(Payload.class); } + + public Header getHeader() { + return getExtension(Header.class); + } + + public Payload getPayload() { + return getExtension(Payload.class); + } } diff --git a/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Header.java b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Header.java index 85ea0a363..91e2bd87b 100644 --- a/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Header.java +++ b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Header.java @@ -1,7 +1,11 @@ package im.conversations.android.xmpp.model.axolotl; +import com.google.common.base.Optional; +import com.google.common.collect.Iterables; import im.conversations.android.annotation.XmlElement; import im.conversations.android.xmpp.model.Extension; +import java.util.Collection; +import java.util.Objects; @XmlElement public class Header extends Extension { @@ -9,4 +13,33 @@ public class Header extends Extension { public Header() { super(Header.class); } + + public void addIv(byte[] iv) { + this.addExtension(new IV()).setContent(iv); + } + + public void setSourceDevice(long sourceDeviceId) { + this.setAttribute("sid", sourceDeviceId); + } + + public Optional getSourceDevice() { + return getOptionalIntAttribute("sid"); + } + + public Collection getKeys() { + return this.getExtensions(Key.class); + } + + public Key getKey(final int deviceId) { + return Iterables.find( + getKeys(), key -> Objects.equals(key.getRemoteDeviceId(), deviceId), null); + } + + public byte[] getIv() { + final IV iv = this.getExtension(IV.class); + if (iv == null) { + throw new IllegalStateException("No IV in header"); + } + return iv.asBytes(); + } } diff --git a/app/src/main/java/im/conversations/android/xmpp/model/axolotl/IV.java b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/IV.java new file mode 100644 index 000000000..93e368a00 --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/IV.java @@ -0,0 +1,13 @@ +package im.conversations.android.xmpp.model.axolotl; + +import im.conversations.android.annotation.XmlElement; +import im.conversations.android.xmpp.model.ByteContent; +import im.conversations.android.xmpp.model.Extension; + +@XmlElement +public class IV extends Extension implements ByteContent { + + public IV() { + super(IV.class); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Key.java b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Key.java new file mode 100644 index 000000000..3ad7357b8 --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Key.java @@ -0,0 +1,29 @@ +package im.conversations.android.xmpp.model.axolotl; + +import im.conversations.android.annotation.XmlElement; +import im.conversations.android.xmpp.model.ByteContent; +import im.conversations.android.xmpp.model.Extension; + +@XmlElement +public class Key extends Extension implements ByteContent { + + public Key() { + super(Key.class); + } + + public void setIsPreKey(boolean isPreKey) { + this.setAttribute("prekey", isPreKey); + } + + public boolean isPreKey() { + return this.getAttributeAsBoolean("prekey"); + } + + public void setRemoteDeviceId(final int remoteDeviceId) { + this.setAttribute("rid", remoteDeviceId); + } + + public Integer getRemoteDeviceId() { + return getOptionalIntAttribute("rid").orNull(); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Payload.java b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Payload.java index f8c0324c1..9c5870110 100644 --- a/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Payload.java +++ b/app/src/main/java/im/conversations/android/xmpp/model/axolotl/Payload.java @@ -1,10 +1,11 @@ package im.conversations.android.xmpp.model.axolotl; import im.conversations.android.annotation.XmlElement; +import im.conversations.android.xmpp.model.ByteContent; import im.conversations.android.xmpp.model.Extension; @XmlElement -public class Payload extends Extension { +public class Payload extends Extension implements ByteContent { public Payload() { super(Payload.class); diff --git a/app/src/main/java/im/conversations/android/xmpp/model/disco/external/Service.java b/app/src/main/java/im/conversations/android/xmpp/model/disco/external/Service.java new file mode 100644 index 000000000..86a93af0d --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/disco/external/Service.java @@ -0,0 +1,12 @@ +package im.conversations.android.xmpp.model.disco.external; + +import im.conversations.android.annotation.XmlElement; +import im.conversations.android.xmpp.model.Extension; + +@XmlElement +public class Service extends Extension { + + public Service() { + super(Service.class); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/disco/external/Services.java b/app/src/main/java/im/conversations/android/xmpp/model/disco/external/Services.java new file mode 100644 index 000000000..36338083d --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/disco/external/Services.java @@ -0,0 +1,12 @@ +package im.conversations.android.xmpp.model.disco.external; + +import im.conversations.android.annotation.XmlElement; +import im.conversations.android.xmpp.model.Extension; + +@XmlElement +public class Services extends Extension { + + public Services() { + super(Services.class); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/disco/external/package-info.java b/app/src/main/java/im/conversations/android/xmpp/model/disco/external/package-info.java new file mode 100644 index 000000000..3a2a85a48 --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/disco/external/package-info.java @@ -0,0 +1,5 @@ +@XmlPackage(namespace = Namespace.EXTERNAL_SERVICE_DISCOVERY) +package im.conversations.android.xmpp.model.disco.external; + +import im.conversations.android.annotation.XmlPackage; +import im.conversations.android.xml.Namespace; diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jingle/Jingle.java b/app/src/main/java/im/conversations/android/xmpp/model/jingle/Jingle.java index e181ecdce..8967d0fda 100644 --- a/app/src/main/java/im/conversations/android/xmpp/model/jingle/Jingle.java +++ b/app/src/main/java/im/conversations/android/xmpp/model/jingle/Jingle.java @@ -6,7 +6,6 @@ import im.conversations.android.xmpp.model.Extension; @XmlElement public class Jingle extends Extension { - public Jingle() { super(Jingle.class); } diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jingle/package-info.java b/app/src/main/java/im/conversations/android/xmpp/model/jingle/package-info.java new file mode 100644 index 000000000..ab1bbc86f --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/jingle/package-info.java @@ -0,0 +1,5 @@ +@XmlPackage(namespace = Namespace.JINGLE) +package im.conversations.android.xmpp.model.jingle; + +import im.conversations.android.annotation.XmlPackage; +import im.conversations.android.xml.Namespace; diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jmi/Accept.java b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Accept.java new file mode 100644 index 000000000..562c3d6fe --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Accept.java @@ -0,0 +1,8 @@ +package im.conversations.android.xmpp.model.jmi; + +public class Accept extends JingleMessage { + + public Accept() { + super(Accept.class); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jmi/JingleMessage.java b/app/src/main/java/im/conversations/android/xmpp/model/jmi/JingleMessage.java new file mode 100644 index 000000000..3b69bb440 --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/jmi/JingleMessage.java @@ -0,0 +1,14 @@ +package im.conversations.android.xmpp.model.jmi; + +import im.conversations.android.xmpp.model.Extension; + +public abstract class JingleMessage extends Extension { + + public JingleMessage(Class clazz) { + super(clazz); + } + + public String getSessionId() { + return this.getAttribute("id"); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jmi/Proceed.java b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Proceed.java new file mode 100644 index 000000000..6324d5fc0 --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Proceed.java @@ -0,0 +1,20 @@ +package im.conversations.android.xmpp.model.jmi; + +import com.google.common.primitives.Ints; +import im.conversations.android.xml.Element; + +public class Proceed extends JingleMessage { + + public Proceed() { + super(Propose.class); + } + + public Integer getDeviceId() { + final Element device = this.findChild("device"); + final String id = device == null ? null : device.getAttribute("id"); + if (id == null) { + return null; + } + return Ints.tryParse(id); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jmi/Propose.java b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Propose.java new file mode 100644 index 000000000..9ac28638b --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Propose.java @@ -0,0 +1,33 @@ +package im.conversations.android.xmpp.model.jmi; + +import com.google.common.collect.ImmutableList; +import eu.siacs.conversations.xmpp.jingle.stanzas.FileTransferDescription; +import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription; +import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; +import im.conversations.android.xml.Element; +import im.conversations.android.xml.Namespace; +import java.util.List; + +public class Propose extends JingleMessage { + + public Propose() { + super(Propose.class); + } + + public List getDescriptions() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : this.children) { + if ("description".equals(child.getName())) { + final String namespace = child.getNamespace(); + if (FileTransferDescription.NAMESPACES.contains(namespace)) { + builder.add(FileTransferDescription.upgrade(child)); + } else if (Namespace.JINGLE_APPS_RTP.equals(namespace)) { + builder.add(RtpDescription.upgrade(child)); + } else { + builder.add(GenericDescription.upgrade(child)); + } + } + } + return builder.build(); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jmi/Reject.java b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Reject.java new file mode 100644 index 000000000..3ed513885 --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Reject.java @@ -0,0 +1,8 @@ +package im.conversations.android.xmpp.model.jmi; + +public class Reject extends JingleMessage { + + public Reject() { + super(Reject.class); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jmi/Retract.java b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Retract.java new file mode 100644 index 000000000..6e9b29603 --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/jmi/Retract.java @@ -0,0 +1,8 @@ +package im.conversations.android.xmpp.model.jmi; + +public class Retract extends JingleMessage { + + public Retract() { + super(Retract.class); + } +} diff --git a/app/src/main/java/im/conversations/android/xmpp/model/jmi/package-info.java b/app/src/main/java/im/conversations/android/xmpp/model/jmi/package-info.java new file mode 100644 index 000000000..73b81e29d --- /dev/null +++ b/app/src/main/java/im/conversations/android/xmpp/model/jmi/package-info.java @@ -0,0 +1,5 @@ +@XmlPackage(namespace = Namespace.JINGLE_MESSAGE) +package im.conversations.android.xmpp.model.jmi; + +import im.conversations.android.annotation.XmlPackage; +import im.conversations.android.xml.Namespace; diff --git a/app/src/main/java/im/conversations/android/xmpp/processor/IqProcessor.java b/app/src/main/java/im/conversations/android/xmpp/processor/IqProcessor.java index 893c03dc5..1bbe6b1da 100644 --- a/app/src/main/java/im/conversations/android/xmpp/processor/IqProcessor.java +++ b/app/src/main/java/im/conversations/android/xmpp/processor/IqProcessor.java @@ -73,6 +73,7 @@ public class IqProcessor extends XmppConnection.Delegate implements Consumer if (type == Iq.Type.SET && packet.hasExtension(Jingle.class)) { getManager(JingleConnectionManager.class).handleJingle(packet); + return; } final var extensionIds = packet.getExtensionIds(); diff --git a/app/src/main/res/drawable/ic_bluetooth_audio_24dp.xml b/app/src/main/res/drawable/ic_bluetooth_audio_24dp.xml new file mode 100644 index 000000000..6f3bf7c40 --- /dev/null +++ b/app/src/main/res/drawable/ic_bluetooth_audio_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_call_24dp.xml b/app/src/main/res/drawable/ic_call_24dp.xml new file mode 100644 index 000000000..f56f10fd4 --- /dev/null +++ b/app/src/main/res/drawable/ic_call_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_call_end_24dp.xml b/app/src/main/res/drawable/ic_call_end_24dp.xml new file mode 100644 index 000000000..6b4b22ae7 --- /dev/null +++ b/app/src/main/res/drawable/ic_call_end_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_chat_24dp.xml b/app/src/main/res/drawable/ic_chat_24dp.xml index 649809399..956385a9b 100644 --- a/app/src/main/res/drawable/ic_chat_24dp.xml +++ b/app/src/main/res/drawable/ic_chat_24dp.xml @@ -1,11 +1,5 @@ - - + + diff --git a/app/src/main/res/drawable/ic_check_24dp.xml b/app/src/main/res/drawable/ic_check_24dp.xml new file mode 100644 index 000000000..cf143d4d5 --- /dev/null +++ b/app/src/main/res/drawable/ic_check_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_clear_24dp.xml b/app/src/main/res/drawable/ic_clear_24dp.xml new file mode 100644 index 000000000..844b6b62e --- /dev/null +++ b/app/src/main/res/drawable/ic_clear_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_flip_camera_android_24dp.xml b/app/src/main/res/drawable/ic_flip_camera_android_24dp.xml new file mode 100644 index 000000000..9e8ac18ea --- /dev/null +++ b/app/src/main/res/drawable/ic_flip_camera_android_24dp.xml @@ -0,0 +1,7 @@ + + + + + diff --git a/app/src/main/res/drawable/ic_headset_24dp.xml b/app/src/main/res/drawable/ic_headset_24dp.xml new file mode 100644 index 000000000..7395711a1 --- /dev/null +++ b/app/src/main/res/drawable/ic_headset_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_help_24dp.xml b/app/src/main/res/drawable/ic_help_24dp.xml new file mode 100644 index 000000000..0028ebdbf --- /dev/null +++ b/app/src/main/res/drawable/ic_help_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_mic_24dp.xml b/app/src/main/res/drawable/ic_mic_24dp.xml new file mode 100644 index 000000000..5eb92eb31 --- /dev/null +++ b/app/src/main/res/drawable/ic_mic_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_mic_off_24dp.xml b/app/src/main/res/drawable/ic_mic_off_24dp.xml new file mode 100644 index 000000000..e97571284 --- /dev/null +++ b/app/src/main/res/drawable/ic_mic_off_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_replay_24dp.xml b/app/src/main/res/drawable/ic_replay_24dp.xml new file mode 100644 index 000000000..df3231b41 --- /dev/null +++ b/app/src/main/res/drawable/ic_replay_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_videocam_24dp.xml b/app/src/main/res/drawable/ic_videocam_24dp.xml new file mode 100644 index 000000000..34b26df6f --- /dev/null +++ b/app/src/main/res/drawable/ic_videocam_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_videocam_off_24dp.xml b/app/src/main/res/drawable/ic_videocam_off_24dp.xml new file mode 100644 index 000000000..b573bbba1 --- /dev/null +++ b/app/src/main/res/drawable/ic_videocam_off_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_voicemail_24dp.xml b/app/src/main/res/drawable/ic_voicemail_24dp.xml new file mode 100644 index 000000000..7110cc416 --- /dev/null +++ b/app/src/main/res/drawable/ic_voicemail_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_volume_off_24dp.xml b/app/src/main/res/drawable/ic_volume_off_24dp.xml new file mode 100644 index 000000000..d7b0465cd --- /dev/null +++ b/app/src/main/res/drawable/ic_volume_off_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_volume_up_24dp.xml b/app/src/main/res/drawable/ic_volume_up_24dp.xml new file mode 100644 index 000000000..2551246f0 --- /dev/null +++ b/app/src/main/res/drawable/ic_volume_up_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/drawable/ic_warning_24dp.xml b/app/src/main/res/drawable/ic_warning_24dp.xml new file mode 100644 index 000000000..b1726a361 --- /dev/null +++ b/app/src/main/res/drawable/ic_warning_24dp.xml @@ -0,0 +1,5 @@ + + + diff --git a/app/src/main/res/layout/activity_rtp_session.xml b/app/src/main/res/layout/activity_rtp_session.xml new file mode 100644 index 000000000..0f38f13bd --- /dev/null +++ b/app/src/main/res/layout/activity_rtp_session.xml @@ -0,0 +1,273 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/menu/activity_rtp_session.xml b/app/src/main/res/menu/activity_rtp_session.xml new file mode 100644 index 000000000..b013d7f75 --- /dev/null +++ b/app/src/main/res/menu/activity_rtp_session.xml @@ -0,0 +1,19 @@ + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/values-h360dp/dimens.xml b/app/src/main/res/values-h360dp/dimens.xml new file mode 100644 index 000000000..a72c6a493 --- /dev/null +++ b/app/src/main/res/values-h360dp/dimens.xml @@ -0,0 +1,3 @@ + + 128dp + diff --git a/app/src/main/res/values-h500dp/dimens.xml b/app/src/main/res/values-h500dp/dimens.xml new file mode 100644 index 000000000..607ceb0d5 --- /dev/null +++ b/app/src/main/res/values-h500dp/dimens.xml @@ -0,0 +1,3 @@ + + 192dp + diff --git a/app/src/main/res/values-land/dimens.xml b/app/src/main/res/values-land/dimens.xml index 66ed64d3d..bddb1e27b 100644 --- a/app/src/main/res/values-land/dimens.xml +++ b/app/src/main/res/values-land/dimens.xml @@ -2,4 +2,7 @@ 8dp 12dp 272dp + 96dp + 128dp + 16dp diff --git a/app/src/main/res/values-w384dp/dimens.xml b/app/src/main/res/values-w384dp/dimens.xml new file mode 100644 index 000000000..9eecae4a8 --- /dev/null +++ b/app/src/main/res/values-w384dp/dimens.xml @@ -0,0 +1,4 @@ + + 12dp + 16dp + \ No newline at end of file diff --git a/app/src/main/res/values/bools.xml b/app/src/main/res/values/bools.xml new file mode 100644 index 000000000..0799afb3f --- /dev/null +++ b/app/src/main/res/values/bools.xml @@ -0,0 +1,4 @@ + + + true + \ No newline at end of file diff --git a/app/src/main/res/values/defaults.xml b/app/src/main/res/values/defaults.xml index 2248916c8..a2b3bce49 100644 --- a/app/src/main/res/values/defaults.xml +++ b/app/src/main/res/values/defaults.xml @@ -1,8 +1,10 @@ true + true false true true default_on + content://settings/system/ringtone diff --git a/app/src/main/res/values/dimens.xml b/app/src/main/res/values/dimens.xml index 7f85bd709..b41ec9b26 100644 --- a/app/src/main/res/values/dimens.xml +++ b/app/src/main/res/values/dimens.xml @@ -2,4 +2,10 @@ 48dp 336dp 24dp + 128dp + 96dp + 24dp + 8dp + 12dp + 96dp