From 22c755c5ce9082797d80b5f1fcb812eede9af092 Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Mon, 6 Apr 2020 15:45:06 +0200 Subject: [PATCH] implement session accept --- .../xmpp/jingle/JingleRtpConnection.java | 53 +++++- .../xmpp/jingle/WebRTCWrapper.java | 26 ++- .../jingle/stanzas/IceUdpTransportInfo.java | 151 +++++++++--------- 3 files changed, 147 insertions(+), 83 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index ca28caedb..3e9759c22 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -57,6 +57,9 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web case TRANSPORT_INFO: receiveTransportInfo(jinglePacket); break; + case SESSION_ACCEPT: + receiveSessionAccept(jinglePacket); + break; default: Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction())); break; @@ -72,8 +75,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e); return; } - //TODO pick proper rtpContentMap - final Group originalGroup = this.initiatorRtpContentMap != null ? this.initiatorRtpContentMap.group : null; + final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; + final Group originalGroup = rtpContentMap != null ? rtpContentMap.group : null; final List identificationTags = originalGroup == null ? Collections.emptyList() : originalGroup.getIdentificationTags(); if (identificationTags.size() == 0) { Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices"); @@ -128,10 +131,46 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } } + private void receiveSessionAccept(final JinglePacket jinglePacket) { + if (!isInitiator()) { + Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid())); + //TODO respond with out-of-order + return; + } + final RtpContentMap contentMap; + try { + contentMap = RtpContentMap.of(jinglePacket); + contentMap.requireContentDescriptions(); + } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e); + return; + } + Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents"); + if (transition(State.SESSION_ACCEPTED)) { + receiveSessionAccept(contentMap); + } else { + Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state)); + //TODO out-of-order + } + } + + private void receiveSessionAccept(final RtpContentMap contentMap) { + this.responderRtpContentMap = contentMap; + org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.ANSWER, + SessionDescription.of(contentMap).toString() + ); + try { + this.webRTCWrapper.setRemoteDescription(answer).get(); + } catch (Exception e) { + Log.d(Config.LOGTAG, "unable to receive session accept", e); + } + } + private void sendSessionAccept() { final RtpContentMap rtpContentMap = this.initiatorRtpContentMap; if (rtpContentMap == null) { - throw new IllegalStateException("intital RTP Content Map has not been set"); + throw new IllegalStateException("initiator RTP Content Map has not been set"); } setupWebRTC(); final org.webrtc.SessionDescription offer = new org.webrtc.SessionDescription( @@ -141,10 +180,10 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web try { this.webRTCWrapper.setRemoteDescription(offer).get(); org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get(); - this.webRTCWrapper.setLocalDescription(webRTCSessionDescription); final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description); final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription); sendSessionAccept(respondingRtpContentMap); + this.webRTCWrapper.setLocalDescription(webRTCSessionDescription); } catch (Exception e) { Log.d(Config.LOGTAG, "unable to send session accept", e); @@ -227,8 +266,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) { final RtpContentMap transportInfo; try { - //TODO when responding use responderRtpContentMap - transportInfo = this.initiatorRtpContentMap.transportInfo(contentName, candidate); + final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; + transportInfo = rtpContentMap.transportInfo(contentName, candidate); } catch (Exception e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName); return; @@ -301,7 +340,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web @Override public void onIceCandidate(final IceCandidate iceCandidate) { final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp); - Log.d(Config.LOGTAG, "onIceCandidate: " + iceCandidate.sdp + " mLineIndex=" + iceCandidate.sdpMLineIndex); + Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString()); sendTransportInfo(iceCandidate.sdpMid, candidate); } } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index 67d89bcdb..53210e78c 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -1,6 +1,7 @@ package eu.siacs.conversations.xmpp.jingle; import android.content.Context; +import android.util.Log; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; @@ -25,6 +26,8 @@ import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import eu.siacs.conversations.Config; + public class WebRTCWrapper { private final EventCallback eventCallback; @@ -32,9 +35,15 @@ public class WebRTCWrapper { private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() { @Override public void onSignalingChange(PeerConnection.SignalingState signalingState) { + Log.d(Config.LOGTAG, "onSignalingChange(" + signalingState + ")"); } + @Override + public void onConnectionChange(PeerConnection.PeerConnectionState newState) { + Log.d(Config.LOGTAG, "onConnectionChange(" + newState + ")"); + } + @Override public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { @@ -62,7 +71,10 @@ public class WebRTCWrapper { @Override public void onAddStream(MediaStream mediaStream) { - + Log.d(Config.LOGTAG, "onAddStream"); + for(AudioTrack audioTrack : mediaStream.audioTracks) { + Log.d(Config.LOGTAG,"remote? - audioTrack enabled:"+audioTrack.enabled()+" state="+audioTrack.state()); + } } @Override @@ -82,6 +94,7 @@ public class WebRTCWrapper { @Override public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) { + Log.d(Config.LOGTAG, "onAddTrack()"); } }; @@ -105,6 +118,7 @@ public class WebRTCWrapper { final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints()); final AudioTrack audioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource); + Log.d(Config.LOGTAG,"audioTrack enabled:"+audioTrack.enabled()+" state="+audioTrack.state()); final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream"); stream.addTrack(audioTrack); @@ -117,6 +131,8 @@ public class WebRTCWrapper { throw new IllegalStateException("Unable to create PeerConnection"); } peerConnection.addStream(stream); + peerConnection.setAudioPlayout(true); + peerConnection.setAudioRecording(true); this.peerConnection = peerConnection; } @@ -167,7 +183,7 @@ public class WebRTCWrapper { @Override public void onSetFailure(String s) { - future.setException(new IllegalArgumentException("unable to set local session description: "+s)); + future.setException(new IllegalArgumentException("unable to set local session description: " + s)); } }, sessionDescription); @@ -186,7 +202,7 @@ public class WebRTCWrapper { @Override public void onSetFailure(String s) { - future.setException(new IllegalArgumentException("unable to set remote session description: "+s)); + future.setException(new IllegalArgumentException("unable to set remote session description: " + s)); } }, sessionDescription); @@ -212,6 +228,10 @@ public class WebRTCWrapper { peerConnection.addIceCandidate(iceCandidate); } + public PeerConnection.PeerConnectionState getState() { + return this.peerConnection.connectionState(); + } + private static abstract class SetSdpObserver implements SdpObserver { @Override diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java index 1b3fe18f2..467a25490 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java @@ -16,6 +16,7 @@ import org.checkerframework.checker.nullness.compatqual.NullableDecl; import java.util.Collection; import java.util.HashMap; import java.util.Hashtable; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -30,21 +31,6 @@ public class IceUdpTransportInfo extends GenericTransportInfo { super("transport", Namespace.JINGLE_TRANSPORT_ICE_UDP); } - public Fingerprint getFingerprint() { - final Element fingerprint = this.findChild("fingerprint", Namespace.JINGLE_APPS_DTLS); - return fingerprint == null ? null : Fingerprint.upgrade(fingerprint); - } - - public List getCandidates() { - final ImmutableList.Builder builder = new ImmutableList.Builder<>(); - for (final Element child : getChildren()) { - if ("candidate".equals(child.getName())) { - builder.add(Candidate.upgrade(child)); - } - } - return builder.build(); - } - public static IceUdpTransportInfo upgrade(final Element element) { Preconditions.checkArgument("transport".equals(element.getName()), "Name of provided element is not transport"); Preconditions.checkArgument(Namespace.JINGLE_TRANSPORT_ICE_UDP.equals(element.getNamespace()), "Element does not match ice-udp transport namespace"); @@ -54,12 +40,6 @@ public class IceUdpTransportInfo extends GenericTransportInfo { return transportInfo; } - public IceUdpTransportInfo cloneWrapper() { - final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); - transportInfo.setAttributes(new Hashtable<>(getAttributes())); - return transportInfo; - } - public static IceUdpTransportInfo of(SessionDescription sessionDescription, SessionDescription.Media media) { final String ufrag = Iterables.getFirst(media.attributes.get("ice-ufrag"), null); final String pwd = Iterables.getFirst(media.attributes.get("ice-pwd"), null); @@ -78,12 +58,74 @@ public class IceUdpTransportInfo extends GenericTransportInfo { } + public Fingerprint getFingerprint() { + final Element fingerprint = this.findChild("fingerprint", Namespace.JINGLE_APPS_DTLS); + return fingerprint == null ? null : Fingerprint.upgrade(fingerprint); + } + + public List getCandidates() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : getChildren()) { + if ("candidate".equals(child.getName())) { + builder.add(Candidate.upgrade(child)); + } + } + return builder.build(); + } + + public IceUdpTransportInfo cloneWrapper() { + final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); + transportInfo.setAttributes(new Hashtable<>(getAttributes())); + return transportInfo; + } + public static class Candidate extends Element { private Candidate() { super("candidate"); } + public static Candidate upgrade(final Element element) { + Preconditions.checkArgument("candidate".equals(element.getName())); + final Candidate candidate = new Candidate(); + candidate.setAttributes(element.getAttributes()); + candidate.setChildren(element.getChildren()); + return candidate; + } + + // https://tools.ietf.org/html/draft-ietf-mmusic-ice-sip-sdp-39#section-5.1 + public static Candidate fromSdpAttribute(final String attribute) { + final String[] pair = attribute.split(":", 2); + if (pair.length == 2 && "candidate".equals(pair[0])) { + final String[] segments = pair[1].split(" "); + if (segments.length >= 6) { + final String foundation = segments[0]; + final String component = segments[1]; + final String transport = segments[2]; + final String priority = segments[3]; + final String connectionAddress = segments[4]; + final String port = segments[5]; + final HashMap additional = new HashMap<>(); + for (int i = 6; i < segments.length - 1; i = i + 2) { + additional.put(segments[i], segments[i + 1]); + } + final Candidate candidate = new Candidate(); + candidate.setAttribute("component", component); + candidate.setAttribute("foundation", foundation); + candidate.setAttribute("generation", additional.get("generation")); + candidate.setAttribute("rel-addr", additional.get("raddr")); + candidate.setAttribute("rel-port", additional.get("rport")); + candidate.setAttribute("ip", connectionAddress); + candidate.setAttribute("port", port); + candidate.setAttribute("priority", priority); + candidate.setAttribute("protocol", transport); + candidate.setAttribute("type", additional.get("typ")); + return candidate; + } + } + return null; + } + public int getComponent() { return getAttributeAsInt("component"); } @@ -144,14 +186,6 @@ public class IceUdpTransportInfo extends GenericTransportInfo { } } - public static Candidate upgrade(final Element element) { - Preconditions.checkArgument("candidate".equals(element.getName())); - final Candidate candidate = new Candidate(); - candidate.setAttributes(element.getAttributes()); - candidate.setChildren(element.getChildren()); - return candidate; - } - public String toSdpAttribute(final String ufrag) { final String foundation = this.getAttribute("foundation"); final String component = this.getAttribute("component"); @@ -159,10 +193,14 @@ public class IceUdpTransportInfo extends GenericTransportInfo { final String priority = this.getAttribute("priority"); final String connectionAddress = this.getAttribute("ip"); final String port = this.getAttribute("port"); - final Map additionalParameter = new HashMap<>(); + final Map additionalParameter = new LinkedHashMap<>(); final String relAddr = this.getAttribute("rel-addr"); + final String type = this.getAttribute("type"); + if (type != null) { + additionalParameter.put("typ", type); + } if (relAddr != null) { - additionalParameter.put("raddr",relAddr); + additionalParameter.put("raddr", relAddr); } final String relPort = this.getAttribute("rel-port"); if (relPort != null) { @@ -175,7 +213,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo { if (ufrag != null) { additionalParameter.put("ufrag", ufrag); } - final String parametersString = Joiner.on(' ').join(Collections2.transform(additionalParameter.entrySet(), input -> String.format("%s %s",input.getKey(),input.getValue()))); + final String parametersString = Joiner.on(' ').join(Collections2.transform(additionalParameter.entrySet(), input -> String.format("%s %s", input.getKey(), input.getValue()))); return String.format( "candidate:%s %s %s %s %s %s %s", foundation, @@ -188,52 +226,11 @@ public class IceUdpTransportInfo extends GenericTransportInfo { ); } - - // https://tools.ietf.org/html/draft-ietf-mmusic-ice-sip-sdp-39#section-5.1 - public static Candidate fromSdpAttribute(final String attribute) { - final String[] pair = attribute.split(":", 2); - if (pair.length == 2 && "candidate".equals(pair[0])) { - final String[] segments = pair[1].split(" "); - if (segments.length >= 6) { - final String foundation = segments[0]; - final String component = segments[1]; - final String transport = segments[2]; - final String priority = segments[3]; - final String connectionAddress = segments[4]; - final String port = segments[5]; - final HashMap additional = new HashMap<>(); - for (int i = 6; i < segments.length - 1; i = i + 2) { - additional.put(segments[i], segments[i + 1]); - } - final Candidate candidate = new Candidate(); - candidate.setAttribute("component", component); - candidate.setAttribute("foundation", foundation); - candidate.setAttribute("generation", additional.get("generation")); - candidate.setAttribute("rel-addr", additional.get("raddr")); - candidate.setAttribute("rel-port", additional.get("rport")); - candidate.setAttribute("ip", connectionAddress); - candidate.setAttribute("port", port); - candidate.setAttribute("priority", priority); - candidate.setAttribute("protocol", transport); - candidate.setAttribute("type", additional.get("typ")); - return candidate; - } - } - return null; - } } public static class Fingerprint extends Element { - public String getHash() { - return this.getAttribute("hash"); - } - - public String getSetup() { - return this.getAttribute("setup"); - } - private Fingerprint() { super("fingerprint", Namespace.JINGLE_APPS_DTLS); } @@ -269,5 +266,13 @@ public class IceUdpTransportInfo extends GenericTransportInfo { final Fingerprint fingerprint = of(media.attributes); return fingerprint == null ? of(sessionDescription.attributes) : fingerprint; } + + public String getHash() { + return this.getAttribute("hash"); + } + + public String getSetup() { + return this.getAttribute("setup"); + } } }