narayana/conversations-classic
3
2
Fork 0
Code Issues 3 Pull requests Actions Packages Projects Releases 4 Wiki Activity

add SDP Offer / Answer support

Browse source
This commit is contained in:
Daniel Gultsch 2023-11-13 12:36:20 +01:00
parent 38ca53fcac
commit 96dcc75ac3
No known key found for this signature in database
GPG key ID: F43D18AD2A0982C2
5 changed files with 128 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/main/java/eu/siacs/conversations/xml/Namespace.java
View file

@ -69,4 +69,5 @@ public final class Namespace {
public static final String UNIFIED_PUSH = "http://gultsch.de/xmpp/drafts/unified-push";
public static final String REPORTING = "urn:xmpp:reporting:1";
public static final String REPORTING_REASON_SPAM = "urn:xmpp:reporting:spam";
public static final String SDP_OFFER_ANSWER = "urn:ietf:rfc:3264";
}

76
src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java
View file

@ -635,7 +635,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
}
private void resendCandidatesFromSdp(final SessionDescription answer) {
private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(final SessionDescription answer) {
final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
for(final SessionDescription.Media media : answer.media) {
final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
@ -649,8 +649,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
}
}
final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
sendTransportInfo(candidates);
return candidateBuilder.build();
}
private void receiveContentReject(final JinglePacket jinglePacket) {
@ -1406,8 +1405,9 @@ public class JingleRtpConnection extends AbstractJingleConnection
+ ": ICE servers got discovered when session was already terminated. nothing to do.");
return;
}
final boolean includeCandidates = remoteHasSdpOfferAnswer();
try {
setupWebRTC(media, iceServers);
setupWebRTC(media, iceServers, !includeCandidates);
} catch (final WebRTCWrapper.InitializationException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
webRTCWrapper.close();
@ -1421,8 +1421,8 @@ public class JingleRtpConnection extends AbstractJingleConnection
this.webRTCWrapper.setRemoteDescription(sdp).get();
addIceCandidatesFromBlackLog();
org.webrtc.SessionDescription webRTCSessionDescription =
this.webRTCWrapper.setLocalDescription().get();
prepareSessionAccept(webRTCSessionDescription);
this.webRTCWrapper.setLocalDescription(includeCandidates).get();
prepareSessionAccept(webRTCSessionDescription, includeCandidates);
} catch (final Exception e) {
failureToAcceptSession(e);
}
@ -1459,10 +1459,16 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
private void prepareSessionAccept(
final org.webrtc.SessionDescription webRTCSessionDescription) {
final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates) {
final SessionDescription sessionDescription =
SessionDescription.parse(webRTCSessionDescription.description);
final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
if (includeCandidates) {
candidates = parseCandidates(sessionDescription);
} else {
candidates = ImmutableMultimap.of();
}
this.responderRtpContentMap = respondingRtpContentMap;
storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
@ -1472,9 +1478,19 @@ public class JingleRtpConnection extends AbstractJingleConnection
new FutureCallback<RtpContentMap>() {
@Override
public void onSuccess(final RtpContentMap outgoingContentMap) {
if (includeCandidates) {
Log.d(
Config.LOGTAG,
"including "
+ candidates.size()
+ " candidates in session accept");
sendSessionAccept(outgoingContentMap.withCandidates(candidates));
webRTCWrapper.resetPendingCandidates();
} else {
sendSessionAccept(outgoingContentMap);
webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
}
}
@Override
public void onFailure(@NonNull Throwable throwable) {
@ -1871,8 +1887,9 @@ public class JingleRtpConnection extends AbstractJingleConnection
+ ": ICE servers got discovered when session was already terminated. nothing to do.");
return;
}
final boolean includeCandidates = remoteHasSdpOfferAnswer();
try {
setupWebRTC(media, iceServers);
setupWebRTC(media, iceServers, !includeCandidates);
} catch (final WebRTCWrapper.InitializationException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
webRTCWrapper.close();
@ -1881,8 +1898,8 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
try {
org.webrtc.SessionDescription webRTCSessionDescription =
this.webRTCWrapper.setLocalDescription().get();
prepareSessionInitiate(webRTCSessionDescription, targetState);
this.webRTCWrapper.setLocalDescription(includeCandidates).get();
prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
} catch (final Exception e) {
// TODO sending the error text is worthwhile as well. Especially for FailureToSet
// exceptions
@ -1915,10 +1932,16 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
private void prepareSessionInitiate(
final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates, final State targetState) {
final SessionDescription sessionDescription =
SessionDescription.parse(webRTCSessionDescription.description);
final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
if (includeCandidates) {
candidates = parseCandidates(sessionDescription);
} else {
candidates = ImmutableMultimap.of();
}
this.initiatorRtpContentMap = rtpContentMap;
final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
encryptSessionInitiate(rtpContentMap);
@ -1927,9 +1950,19 @@ public class JingleRtpConnection extends AbstractJingleConnection
new FutureCallback<RtpContentMap>() {
@Override
public void onSuccess(final RtpContentMap outgoingContentMap) {
if (includeCandidates) {
Log.d(
Config.LOGTAG,
"including "
+ candidates.size()
+ " candidates in session initiate");
sendSessionInitiate(outgoingContentMap.withCandidates(candidates), targetState);
webRTCWrapper.resetPendingCandidates();
} else {
sendSessionInitiate(outgoingContentMap, targetState);
webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
}
}
@Override
public void onFailure(@NonNull final Throwable throwable) {
@ -2031,11 +2064,6 @@ public class JingleRtpConnection extends AbstractJingleConnection
send(jinglePacket);
}
private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
// TODO send all candidates in one transport-info
}
private void send(final JinglePacket jinglePacket) {
jinglePacket.setTo(id.with);
xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
@ -2400,10 +2428,10 @@ public class JingleRtpConnection extends AbstractJingleConnection
finish();
}
private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers, final boolean trickle) throws WebRTCWrapper.InitializationException {
this.jingleConnectionManager.ensureConnectionIsRegistered(this);
this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
this.webRTCWrapper.initializePeerConnection(media, iceServers);
this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
}
private void acceptCallFromProposed() {
@ -2736,7 +2764,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
private SessionDescription setLocalSessionDescription()
throws ExecutionException, InterruptedException {
final org.webrtc.SessionDescription sessionDescription =
this.webRTCWrapper.setLocalDescription().get();
this.webRTCWrapper.setLocalDescription(false).get();
return SessionDescription.parse(sessionDescription.description);
}
@ -3024,6 +3052,14 @@ public class JingleRtpConnection extends AbstractJingleConnection
}
private boolean remoteHasVideoFeature() {
return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
}
private boolean remoteHasSdpOfferAnswer() {
return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
}
private boolean remoteHasFeature(final String feature) {
final Contact contact = id.getContact();
final Presence presence =
contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
@ -3031,7 +3067,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
presence == null ? null : presence.getServiceDiscoveryResult();
final List<String> features =
serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
return features != null && features.contains(feature);
}
private interface OnIceServersDiscovered {

19
src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java
View file

@ -8,6 +8,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
@ -196,6 +197,24 @@ public class RtpContentMap {
dt.senders, null, dt.transport.cloneWrapper())));
}
RtpContentMap withCandidates(
ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates) {
final ImmutableMap.Builder<String, DescriptionTransport> contentBuilder =
new ImmutableMap.Builder<>();
for (final Map.Entry<String, DescriptionTransport> entry : this.contents.entrySet()) {
final String name = entry.getKey();
final DescriptionTransport descriptionTransport = entry.getValue();
final var transport = descriptionTransport.transport;
contentBuilder.put(
name,
new DescriptionTransport(
descriptionTransport.senders,
descriptionTransport.description,
transport.withCandidates(candidates.get(name))));
}
return new RtpContentMap(group, contentBuilder.build());
}
public IceUdpTransportInfo.Credentials getDistinctCredentials() {
final Set<IceUdpTransportInfo.Credentials> allCredentials = getCredentials();
final IceUdpTransportInfo.Credentials credentials =

43
src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java
View file

@ -94,6 +94,8 @@ public class WebRTCWrapper {
private TrackWrapper<AudioTrack> localAudioTrack = null;
private TrackWrapper<VideoTrack> localVideoTrack = null;
private VideoTrack remoteVideoTrack = null;
private final SettableFuture<Void> iceGatheringComplete = SettableFuture.create();
private final PeerConnection.Observer peerConnectionObserver =
new PeerConnection.Observer() {
@Override
@ -128,8 +130,11 @@ public class WebRTCWrapper {
@Override
public void onIceGatheringChange(
PeerConnection.IceGatheringState iceGatheringState) {
final PeerConnection.IceGatheringState iceGatheringState) {
Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
if (iceGatheringState == PeerConnection.IceGatheringState.COMPLETE) {
iceGatheringComplete.set(null);
}
}
@Override
@ -256,7 +261,9 @@ public class WebRTCWrapper {
}
synchronized void initializePeerConnection(
final Set<Media> media, final List<PeerConnection.IceServer> iceServers)
final Set<Media> media,
final List<PeerConnection.IceServer> iceServers,
final boolean trickle)
throws InitializationException {
Preconditions.checkState(this.eglBase != null);
Preconditions.checkNotNull(media);
@ -283,7 +290,7 @@ public class WebRTCWrapper {
.createAudioDeviceModule())
.createPeerConnectionFactory();
final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers);
final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers, trickle);
final PeerConnection peerConnection =
requirePeerConnectionFactory()
.createPeerConnection(rtcConfig, peerConnectionObserver);
@ -398,21 +405,27 @@ public class WebRTCWrapper {
}
private static PeerConnection.RTCConfiguration buildConfiguration(
final List<PeerConnection.IceServer> iceServers) {
final List<PeerConnection.IceServer> iceServers, final boolean trickle) {
final PeerConnection.RTCConfiguration rtcConfig =
new PeerConnection.RTCConfiguration(iceServers);
rtcConfig.tcpCandidatePolicy =
PeerConnection.TcpCandidatePolicy.DISABLED; // XEP-0176 doesn't support tcp
if (trickle) {
rtcConfig.continualGatheringPolicy =
PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
} else {
rtcConfig.continualGatheringPolicy =
PeerConnection.ContinualGatheringPolicy.GATHER_ONCE;
}
rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN;
rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE;
rtcConfig.enableImplicitRollback = true;
return rtcConfig;
}
void reconfigurePeerConnection(final List<PeerConnection.IceServer> iceServers) {
requirePeerConnection().setConfiguration(buildConfiguration(iceServers));
void reconfigurePeerConnection(
final List<PeerConnection.IceServer> iceServers, final boolean trickle) {
requirePeerConnection().setConfiguration(buildConfiguration(iceServers, trickle));
}
void restartIceAsync() {
@ -443,6 +456,11 @@ public class WebRTCWrapper {
"setIsReadyToReceiveCandidates(" + ready + ") was=" + was + " is=" + is);
}
public void resetPendingCandidates() {
this.readyToReceivedIceCandidates.set(true);
this.iceCandidates.clear();
}
synchronized void close() {
final PeerConnection peerConnection = this.peerConnection;
final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory;
@ -561,7 +579,7 @@ public class WebRTCWrapper {
throw new IllegalStateException("Local video track does not exist");
}
synchronized ListenableFuture<SessionDescription> setLocalDescription() {
synchronized ListenableFuture<SessionDescription> setLocalDescription(final boolean waitForCandidates) {
this.setIsReadyToReceiveIceCandidates(false);
return Futures.transformAsync(
getPeerConnectionFuture(),
@ -575,7 +593,16 @@ public class WebRTCWrapper {
new SetSdpObserver() {
@Override
public void onSetSuccess() {
future.setFuture(getLocalDescriptionFuture());
final var delay =
waitForCandidates
? iceGatheringComplete
: Futures.immediateVoidFuture();
final var delayedSessionDescription =
Futures.transformAsync(
delay,
v -> getLocalDescriptionFuture(),
MoreExecutors.directExecutor());
future.setFuture(delayedSessionDescription);
}
@Override

11
src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java
View file

@ -12,6 +12,7 @@ import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
@ -152,6 +153,16 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
return transportInfo;
}
public IceUdpTransportInfo withCandidates(ImmutableCollection<Candidate> candidates) {
final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo();
transportInfo.setAttributes(new Hashtable<>(getAttributes()));
transportInfo.setChildren(this.getChildren());
for(final Candidate candidate : candidates) {
transportInfo.addChild(candidate);
}
return transportInfo;
}
public static class Credentials {
public final String ufrag;
public final String password;