package eu.siacs.conversations.xmpp.jingle; import android.util.Log; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; import org.webrtc.DataChannel; import org.webrtc.IceCandidate; import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; import org.webrtc.RtpReceiver; import org.webrtc.SdpObserver; import java.util.Collection; import java.util.Collections; import java.util.Map; import eu.siacs.conversations.Config; import eu.siacs.conversations.xml.Element; import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; import eu.siacs.conversations.xmpp.stanzas.MessagePacket; import rocks.xmpp.addr.Jid; public class JingleRtpConnection extends AbstractJingleConnection { private static final Map> VALID_TRANSITIONS; static { final ImmutableMap.Builder> transitionBuilder = new ImmutableMap.Builder<>(); transitionBuilder.put(State.NULL, ImmutableList.of(State.PROPOSED, State.SESSION_INITIALIZED)); transitionBuilder.put(State.PROPOSED, ImmutableList.of(State.ACCEPTED, State.PROCEED)); transitionBuilder.put(State.PROCEED, ImmutableList.of(State.SESSION_INITIALIZED)); VALID_TRANSITIONS = transitionBuilder.build(); } private State state = State.NULL; private RtpContentMap initialRtpContentMap; public JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) { super(jingleConnectionManager, id, initiator); } @Override void deliverPacket(final JinglePacket jinglePacket) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection"); switch (jinglePacket.getAction()) { case SESSION_INITIATE: receiveSessionInitiate(jinglePacket); break; default: Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction())); break; } } private void receiveSessionInitiate(final JinglePacket jinglePacket) { if (isInitiator()) { Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid())); //TODO respond with out-of-order return; } final RtpContentMap contentMap; try { contentMap = RtpContentMap.of(jinglePacket); } catch (IllegalArgumentException | NullPointerException e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e); return; } Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents"); final State oldState = this.state; if (transition(State.SESSION_INITIALIZED)) { if (oldState == State.PROCEED) { processContents(contentMap); sendSessionAccept(); } else { //TODO start ringing } } else { Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state)); } } private void processContents(final RtpContentMap contentMap) { for (Map.Entry content : contentMap.contents.entrySet()) { final RtpContentMap.DescriptionTransport descriptionTransport = content.getValue(); final RtpDescription rtpDescription = descriptionTransport.description; Log.d(Config.LOGTAG, "receive content with name " + content.getKey() + " and media=" + rtpDescription.getMedia()); for (RtpDescription.PayloadType payloadType : rtpDescription.getPayloadTypes()) { Log.d(Config.LOGTAG, "payload type: " + payloadType.toString()); } for (RtpDescription.RtpHeaderExtension extension : rtpDescription.getHeaderExtensions()) { Log.d(Config.LOGTAG, "extension: " + extension.toString()); } final IceUdpTransportInfo iceUdpTransportInfo = descriptionTransport.transport; Log.d(Config.LOGTAG, "transport: " + descriptionTransport.transport); Log.d(Config.LOGTAG, "fingerprint " + iceUdpTransportInfo.getFingerprint()); } } void deliveryMessage(final Jid from, final Element message) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message); switch (message.getName()) { case "propose": receivePropose(from, message); break; case "proceed": receiveProceed(from, message); default: break; } } private void receivePropose(final Jid from, final Element propose) { final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid()); if (originatedFromMyself) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from mysql. ignoring"); } else if (transition(State.PROPOSED)) { //TODO start ringing or something pickUpCall(); } else { Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state); } } private void receiveProceed(final Jid from, final Element proceed) { if (from.equals(id.with)) { if (isInitiator()) { if (transition(State.PROCEED)) { this.sendSessionInitiate(); } else { Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state)); } } else { Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid())); } } else { Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with)); } } private void sendSessionInitiate() { setupWebRTC(); Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate"); } private void sendSessionInitiate(RtpContentMap rtpContentMap) { this.initialRtpContentMap = rtpContentMap; final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId); Log.d(Config.LOGTAG, sessionInitiate.toString()); send(sessionInitiate); } private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) { final RtpContentMap transportInfo; try { transportInfo = this.initialRtpContentMap.transportInfo(contentName, candidate); } catch (Exception e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName); return; } final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId); Log.d(Config.LOGTAG, jinglePacket.toString()); send(jinglePacket); } private void send(final JinglePacket jinglePacket) { jinglePacket.setTo(id.with); //TODO track errors xmppConnectionService.sendIqPacket(id.account, jinglePacket, null); } private void sendSessionAccept() { Log.d(Config.LOGTAG, "sending session-accept"); } public void pickUpCall() { switch (this.state) { case PROPOSED: pickupCallFromProposed(); break; case SESSION_INITIALIZED: pickupCallFromSessionInitialized(); break; default: throw new IllegalStateException("Can not pick up call from " + this.state); } } private void setupWebRTC() { PeerConnectionFactory.initialize( PeerConnectionFactory.InitializationOptions.builder(xmppConnectionService).createInitializationOptions() ); final PeerConnectionFactory.Options options = new PeerConnectionFactory.Options(); PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder().createPeerConnectionFactory(); final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints()); final AudioTrack audioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource); final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream"); stream.addTrack(audioTrack); PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(Collections.emptyList(), new PeerConnection.Observer() { @Override public void onSignalingChange(PeerConnection.SignalingState signalingState) { } @Override public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { } @Override public void onIceConnectionReceivingChange(boolean b) { } @Override public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) { Log.d(Config.LOGTAG, "onIceGatheringChange() " + iceGatheringState); } @Override public void onIceCandidate(IceCandidate iceCandidate) { IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp); Log.d(Config.LOGTAG, "onIceCandidate: " + iceCandidate.sdp); Log.d(Config.LOGTAG, "xml: " + candidate.toString()); Log.d(Config.LOGTAG, "mid: " + iceCandidate.sdpMid); sendTransportInfo(iceCandidate.sdpMid, candidate); } @Override public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) { } @Override public void onAddStream(MediaStream mediaStream) { } @Override public void onRemoveStream(MediaStream mediaStream) { } @Override public void onDataChannel(DataChannel dataChannel) { } @Override public void onRenegotiationNeeded() { } @Override public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) { } }); peerConnection.addStream(stream); peerConnection.createOffer(new SdpObserver() { @Override public void onCreateSuccess(org.webrtc.SessionDescription description) { final SessionDescription sessionDescription = SessionDescription.parse(description.description); Log.d(Config.LOGTAG, "description: " + description.description); final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription); sendSessionInitiate(rtpContentMap); peerConnection.setLocalDescription(new SdpObserver() { @Override public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) { } @Override public void onSetSuccess() { Log.d(Config.LOGTAG, "onSetSuccess()"); } @Override public void onCreateFailure(String s) { } @Override public void onSetFailure(String s) { } }, description); } @Override public void onSetSuccess() { } @Override public void onCreateFailure(String s) { } @Override public void onSetFailure(String s) { } }, new MediaConstraints()); } private void pickupCallFromProposed() { transitionOrThrow(State.PROCEED); final MessagePacket messagePacket = new MessagePacket(); messagePacket.setTo(id.with); //Note that Movim needs 'accept', correct is 'proceed' https://github.com/movim/movim/issues/916 messagePacket.addChild("proceed", Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId); Log.d(Config.LOGTAG, messagePacket.toString()); xmppConnectionService.sendMessagePacket(id.account, messagePacket); } private void pickupCallFromSessionInitialized() { } private synchronized boolean transition(final State target) { final Collection validTransitions = VALID_TRANSITIONS.get(this.state); if (validTransitions != null && validTransitions.contains(target)) { this.state = target; Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target); return true; } else { return false; } } public void transitionOrThrow(final State target) { if (!transition(target)) { throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target)); } } }